From 3a55e2994dd28c747c36ff7ee96ffbf7670abe2c Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:29:50 +1000 Subject: [PATCH 01/26] feat: resumable SSE streams via pluggable delivery durability Add a transport-level StreamDurability seam to toServerSentEventsResponse: chunks are appended to an ordered log before delivery and each SSE event is tagged with an opaque adapter-owned id: offset. Reconnects (Last-Event-ID) and joins (?offset=-1&runId) replay from the log without re-running the provider. Ships memoryStream (in-core, dev/test) and the new @tanstack/ai-durable-stream package (Durable Streams protocol adapter). Client: fetchServerSentEvents now auto-resumes id-tagged streams, de-dupes replayed prefixes, exposes joinRun(runId), and throws DurableStreamIncompleteError when a durable run ends with no terminal event and no forward progress. Split out of #785 so state persistence and delivery durability land independently. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01A6Arc9bWdLq1aRRnDRCLMt --- .changeset/resumable-streams.md | 27 + docs/chat/connection-adapters.md | 13 + docs/config.json | 13 +- docs/resumable-streams/overview.md | 193 +++ packages/ai-client/src/connection-adapters.ts | 240 +++- packages/ai-client/src/index.ts | 2 + .../connection-adapters-resumable.test.ts | 262 +++++ packages/ai-durable-stream/package.json | 50 + .../ai-durable-stream/src/durable-stream.ts | 636 ++++++++++ packages/ai-durable-stream/src/index.ts | 10 + .../tests/durable-stream-types.test-d.ts | 16 + .../tests/durable-stream.test.ts | 1048 +++++++++++++++++ packages/ai-durable-stream/tsconfig.json | 8 + packages/ai-durable-stream/vite.config.ts | 37 + .../skills/ai-core/chat-experience/SKILL.md | 10 + packages/ai/src/index.ts | 4 + packages/ai/src/stream-durability.ts | 212 ++++ packages/ai/src/stream-to-response.ts | 490 ++++++-- .../ai/tests/stream-delivery-contract.test.ts | 273 +++++ .../tests/stream-durability-types.test-d.ts | 30 + packages/ai/tests/stream-durability.test.ts | 193 +++ .../stream-to-response-durability.test.ts | 276 +++++ pnpm-lock.yaml | 9 + testing/e2e/src/routeTree.gen.ts | 21 + .../e2e/src/routes/api.durable-delivery.ts | 84 ++ testing/e2e/tests/delivery-durability.spec.ts | 104 ++ 26 files changed, 4150 insertions(+), 111 deletions(-) create mode 100644 .changeset/resumable-streams.md create mode 100644 docs/resumable-streams/overview.md create mode 100644 packages/ai-client/tests/connection-adapters-resumable.test.ts create mode 100644 packages/ai-durable-stream/package.json create mode 100644 packages/ai-durable-stream/src/durable-stream.ts create mode 100644 packages/ai-durable-stream/src/index.ts create mode 100644 packages/ai-durable-stream/tests/durable-stream-types.test-d.ts create mode 100644 packages/ai-durable-stream/tests/durable-stream.test.ts create mode 100644 packages/ai-durable-stream/tsconfig.json create mode 100644 packages/ai-durable-stream/vite.config.ts create mode 100644 packages/ai/src/stream-durability.ts create mode 100644 packages/ai/tests/stream-delivery-contract.test.ts create mode 100644 packages/ai/tests/stream-durability-types.test-d.ts create mode 100644 packages/ai/tests/stream-durability.test.ts create mode 100644 packages/ai/tests/stream-to-response-durability.test.ts create mode 100644 testing/e2e/src/routes/api.durable-delivery.ts create mode 100644 testing/e2e/tests/delivery-durability.spec.ts diff --git a/.changeset/resumable-streams.md b/.changeset/resumable-streams.md new file mode 100644 index 000000000..50309f3e9 --- /dev/null +++ b/.changeset/resumable-streams.md @@ -0,0 +1,27 @@ +--- +'@tanstack/ai': minor +'@tanstack/ai-client': minor +'@tanstack/ai-durable-stream': minor +--- + +Resumable streams: reconnect to an in-flight SSE response without re-running +the provider. + +`toServerSentEventsResponse` accepts a `durability: { adapter, batch }` option. +The adapter (`StreamDurability`) records every chunk to an ordered log before +delivery and tags each SSE event with an opaque, adapter-owned `id:` offset. A +reconnect (`Last-Event-ID`) or an explicit `?offset` read replays strictly +after that offset from the log — the lazy provider stream is never iterated on +resume. Producers terminalize the log on completion, cancellation, and failure +(`RUN_ERROR` append + `close()`), so readers are never parked on a dead run. + +Two adapters ship: `memoryStream(request)` in `@tanstack/ai` (process-local, +for development and tests) and the new `@tanstack/ai-durable-stream` package, +a Durable Streams protocol adapter for production backends. + +On the client, `fetchServerSentEvents` is now resumable: it tracks SSE `id:` +values, auto-reconnects with `Last-Event-ID`, de-duplicates the replayed +prefix, and exposes `joinRun(runId)` to attach to an in-flight or finished run +from the start (read-only GET with `offset=-1`). Untagged streams behave +exactly as before. A durable run that ends with no terminal event and no +forward progress now throws `DurableStreamIncompleteError` instead of hanging. diff --git a/docs/chat/connection-adapters.md b/docs/chat/connection-adapters.md index a67e17e6f..367c14d65 100644 --- a/docs/chat/connection-adapters.md +++ b/docs/chat/connection-adapters.md @@ -80,6 +80,19 @@ const { messages } = useChat({ > **Tip:** `body` and `forwardedProps` populate the same wire field. Use `body` for static defaults, the `forwardedProps` constructor option (or per-`sendMessage` `data`) for dynamic values. Runtime values always win. +### Resumable SSE + +`fetchServerSentEvents` watches SSE `id:` values. If a connection drops after +receiving an id, it reconnects with `Last-Event-ID` and de-duplicates the +replayed prefix. `joinRun(runId)` performs a read-only GET with `offset=-1` and +the run id, replaying an in-flight or finished run from the start. + +The ids only appear when the server passes a durability adapter to +`toServerSentEventsResponse`. They are opaque tokens owned by that adapter; the +chat client does not create, parse, or persist them. Without ids, behavior is +identical to a plain single fetch. See +[Resumable Streams](../resumable-streams/overview). + ## HTTP Streaming (NDJSON) For environments that don't speak SSE — some edge runtimes, certain mobile WebViews, or anywhere a proxy strips `text/event-stream` — use raw newline-delimited JSON. The wire format is one JSON `StreamChunk` per line: diff --git a/docs/config.json b/docs/config.json index 7debc1ba4..a003be8f3 100644 --- a/docs/config.json +++ b/docs/config.json @@ -166,7 +166,8 @@ { "label": "Connection Adapters", "to": "chat/connection-adapters", - "addedAt": "2026-04-15" + "addedAt": "2026-04-15", + "updatedAt": "2026-07-17" }, { "label": "Thinking & Reasoning", @@ -180,6 +181,16 @@ } ] }, + { + "label": "Resumable Streams", + "children": [ + { + "label": "Overview", + "to": "resumable-streams/overview", + "addedAt": "2026-07-17" + } + ] + }, { "label": "Protocol", "children": [ diff --git a/docs/resumable-streams/overview.md b/docs/resumable-streams/overview.md new file mode 100644 index 000000000..026772df8 --- /dev/null +++ b/docs/resumable-streams/overview.md @@ -0,0 +1,193 @@ +--- +title: Resumable Streams +id: overview +description: "Reconnect to an in-flight AI response without re-running the model — durable, offset-addressed SSE delivery with replay, multi-tab join, and producer-death recovery." +keywords: + - resumable streams + - resume stream + - reconnect sse + - delivery durability + - durable streams + - last-event-id + - replay + - joinRun +--- + +# Resumable Streams + +A resumable stream lets a client reconnect to an in-flight response — after a +page refresh, a dropped connection, or a suspended tab — without invoking the +provider again. TanStack AI implements this with a **delivery durability** +adapter: an ordered log that records every stream chunk before it is delivered, +so any client can replay from any point. + +Because the log outlives the request, this goes further than in-flight +reconnection: a finished run can be replayed, a second tab can join a live run, +and a run whose producer died can still be terminalized by the backend. + +Two adapters ship: + +- `memoryStream(request)` stores a process-local log for development and tests. +- `durableStream(request, options)` writes to an external + [Durable Streams](https://durablestreams.com) protocol URL. + +Resumable delivery is supported by `toServerSentEventsResponse`. NDJSON +helpers do not accept durability and do not resume. + +## Server setup + +```ts +import { + chat, + chatParamsFromRequest, + toServerSentEventsResponse, +} from '@tanstack/ai' +import { durableStream } from '@tanstack/ai-durable-stream' +import { openaiText } from '@tanstack/ai-openai' + +declare function getDurableStreamsToken(): Promise + +const durableOptions = { + server: 'https://streams.example.com', + streamPrefix: 'chat-runs', + headers: async () => ({ + Authorization: `Bearer ${await getDurableStreamsToken()}`, + }), +} + +type ChatInput = Pick< + Awaited>, + 'messages' | 'threadId' | 'runId' +> + +async function respond(request: Request, input: ChatInput) { + const stream = chat({ + adapter: openaiText('gpt-5.5'), + messages: input.messages, + threadId: input.threadId, + runId: input.runId, + }) + + return toServerSentEventsResponse(stream, { + durability: { + adapter: durableStream(request, durableOptions), + batch: 32, + }, + }) +} + +export async function POST(request: Request) { + return respond(request, await chatParamsFromRequest(request)) +} + +export async function GET(request: Request) { + const runId = new URL(request.url).searchParams.get('runId') + if (!runId) return new Response('runId is required', { status: 400 }) + + // A resume request replays the durability log and never iterates this lazy + // provider stream. The placeholder input is therefore not sent upstream. + return respond(request, { + messages: [], + threadId: `replay:${runId}`, + runId, + }) +} +``` + +Use a static `headers` object for fixed credentials or an async resolver for +rotating tokens. The resolver runs for every create, append, read, and close +request. This option belongs to the external URL adapter; a future direct +Cloudflare binding would use binding authorization instead. + +The external service must return a non-empty `Stream-Next-Offset` header for +create and append operations. Missing headers fail loudly. The adapter never +guesses an offset. + +For development and tests, swap `durableStream(request, durableOptions)` for +`memoryStream(request)` from `@tanstack/ai` — no external service required, +at the cost of the log living only in that process's memory. + +## Client setup + +```tsx +import { fetchServerSentEvents, useChat } from '@tanstack/ai-react' + +export function Chat() { + const chat = useChat({ + connection: fetchServerSentEvents('/api/chat'), + }) + + return +} +``` + +When the server emits SSE `id:` lines, `fetchServerSentEvents` remembers the +last id, reconnects with `Last-Event-ID`, and de-duplicates the replayed prefix. +The id is opaque to the client. + +To attach to a known run from the beginning, use the adapter's `joinRun`: + +```ts +import { fetchServerSentEvents } from '@tanstack/ai-client' + +declare const runId: string +const connection = fetchServerSentEvents('/api/chat') + +for await (const chunk of connection.joinRun(runId)) { + console.log(chunk) +} +``` + +`joinRun` performs a GET with `offset=-1` and `runId`. The endpoint must accept +that GET, as shown above. + +## Offset ownership + +`StreamDurability` owns its offset format. Core only passes returned +values back to that adapter and writes them to SSE `id:` fields. + +For every appended batch: + +1. core calls `append(chunks)` before forwarding the chunks; +2. the adapter returns exactly one offset per chunk in the same order; +3. core rejects missing, extra, empty, or CR/LF-containing offsets; +4. a resume reads strictly after the supplied offset. + +Core never derives an offset from an array index and never stamps an offset onto +the `StreamChunk` object. + +## Completion, stop, and errors + +The producer awaits `close()` on all in-process exits: + +- normal completion; +- `stop()` / response cancellation; +- provider iteration errors; +- caught server-side durability failures. + +Cancellation and provider failure also attempt to append a terminal +`RUN_ERROR` with an aborted or failed error payload before closing. If terminal +append or close fails, the failure is surfaced. This prevents a known error +from leaving readers parked on an apparently live log. + +## Process death + +A process that has already terminated cannot execute cleanup code. Therefore +literal process death cannot be guaranteed by `finally` or `close()` alone. + +Production backends should add a lease/reaper mechanism: + +1. the producer acquires or renews a lease while writing; +2. a timer, alarm, or background worker detects expired leases; +3. the reaper appends or records an aborted terminal state and closes the log; +4. readers observe the terminal state instead of waiting forever. + +This mechanism belongs to the durability service or deployment. It is not +implemented by the in-process response helper. + +## Delivery is not state + +The durability log replays chunks. It is not a queryable source of truth for +thread messages or conversation history — it answers "what did this run +stream?", not "what has this user said?". Keep authoritative state in your own +storage; see [Persistence](../chat/persistence) for the client-side options. diff --git a/packages/ai-client/src/connection-adapters.ts b/packages/ai-client/src/connection-adapters.ts index 3c4010047..d198ec22d 100644 --- a/packages/ai-client/src/connection-adapters.ts +++ b/packages/ai-client/src/connection-adapters.ts @@ -47,6 +47,27 @@ export class StreamTruncatedError extends Error { } } +class StreamReadError extends Error { + constructor(cause: unknown) { + super('SSE response body read failed', { cause }) + this.name = 'StreamReadError' + } +} + +/** + * Thrown when a durable (id-tagged) run's stream ends with no terminal event + * and a reconnect makes no forward progress — the run cannot complete, so the + * consumer must not be left silently hanging on a stream that just stops. + */ +export class DurableStreamIncompleteError extends Error { + constructor() { + super( + 'Durable run ended without a terminal event and could not resume — the run did not complete.', + ) + this.name = 'DurableStreamIncompleteError' + } +} + function generateRunId(prefix: string): string { return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` } @@ -88,6 +109,21 @@ function mergeHeaders( return customHeaders } +function withSearchParams(url: string, values: Record): string { + const hashIndex = url.indexOf('#') + const hash = hashIndex === -1 ? '' : url.slice(hashIndex) + const withoutHash = hashIndex === -1 ? url : url.slice(0, hashIndex) + const queryIndex = withoutHash.indexOf('?') + const base = + queryIndex === -1 ? withoutHash : withoutHash.slice(0, queryIndex) + const search = new URLSearchParams( + queryIndex === -1 ? '' : withoutHash.slice(queryIndex + 1), + ) + for (const [key, value] of Object.entries(values)) search.set(key, value) + const query = search.toString() + return `${base}${query.length === 0 ? '' : `?${query}`}${hash}` +} + /** * Read lines from a stream (newline-delimited) */ @@ -100,7 +136,14 @@ async function* readStreamLines( let buffer = '' while (!abortSignal?.aborted) { - const { done, value } = await reader.read() + let result: ReadableStreamReadResult + try { + result = await reader.read() + } catch (error) { + if (abortSignal?.aborted) return + throw new StreamReadError(error) + } + const { done, value } = result if (done) break buffer += decoder.decode(value, { stream: true }) @@ -141,10 +184,16 @@ async function* readStreamLines( * * A JSON parse failure throws — the consumer surfaces it as an error. */ -async function* responseToSSEChunks( +/** + * Yield StreamChunks parsed from an SSE Response body, each paired with the + * `id:` offset of the SSE event it arrived on (if any). Delivery durability + * tags every event with an adapter-owned opaque `id:`; carrying that id up lets the + * resumable adapter track the last offset (for reconnect) and de-dupe replays. + */ +async function* responseToSSEEvents( response: Response, abortSignal?: AbortSignal, -): AsyncGenerator { +): AsyncGenerator<{ chunk: StreamChunk; id?: string }> { if (!response.ok) { throw new Error( `HTTP error! status: ${response.status} ${response.statusText}`, @@ -154,11 +203,15 @@ async function* responseToSSEChunks( let lastThreadId: string | undefined let lastRunId: string | undefined let lastModel: string | undefined + let pendingId: string | undefined for await (const line of readStreamLines(reader, abortSignal)) { + if (line.startsWith('id:')) { + pendingId = line.slice(3).trim() + continue + } if ( line.startsWith(':') || line.startsWith('event:') || - line.startsWith('id:') || line.startsWith('retry:') ) { continue @@ -173,7 +226,7 @@ async function* responseToSSEChunks( timestamp: Date.now(), finishReason: 'stop', } - yield synthetic + yield { chunk: synthetic } return } const chunk = JSON.parse(data) as StreamChunk @@ -186,10 +239,111 @@ async function* responseToSSEChunks( if ('model' in chunk && typeof chunk.model === 'string') { lastModel = chunk.model } + const id = pendingId + pendingId = undefined + yield { chunk, ...(id !== undefined ? { id } : {}) } + } +} + +async function* responseToSSEChunks( + response: Response, + abortSignal?: AbortSignal, +): AsyncGenerator { + for await (const { chunk } of responseToSSEEvents(response, abortSignal)) { yield chunk } } +/** + * Iterate an SSE endpoint with native-style resumability. Each SSE event's + * adapter-owned `id:` delivery-durability offset is remembered; if the + * connection drops or ends before a terminal event, the request is re-issued + * with a `Last-Event-ID` header so the server replays strictly after the last + * offset. Already-seen offsets are de-duped, so an overlapping replay is safe. + * + * When the server does NOT tag events (no durability), no offset is ever seen, + * so no reconnect happens — behaviour is identical to a plain single fetch. + */ +async function* resumableServerSentEvents( + fetchClient: typeof globalThis.fetch, + url: string, + requestInit: RequestInit, + abortSignal?: AbortSignal, +): AsyncGenerator { + const seen = new Set() + let lastEventId: string | undefined + + for (;;) { + if (abortSignal?.aborted) return + const headers: Record = { + ...(requestInit.headers as Record | undefined), + } + if (lastEventId !== undefined) { + headers['Last-Event-ID'] = lastEventId + } + const response = await fetchClient(url, { + ...requestInit, + headers, + ...(abortSignal ? { signal: abortSignal } : {}), + }) + + let sawTerminal = false + let progressed = false + try { + for await (const { chunk, id } of responseToSSEEvents( + response, + abortSignal, + )) { + if (id !== undefined) { + if (seen.has(id)) continue + seen.add(id) + lastEventId = id + } + progressed = true + if (chunk.type === 'RUN_FINISHED' || chunk.type === 'RUN_ERROR') { + sawTerminal = true + } + yield chunk + if (sawTerminal) return + } + } catch (error) { + if (abortSignal?.aborted) return + // A truncated connection is resumable only if we have an offset and made + // forward progress; otherwise surface the failure. + if ( + (error instanceof StreamTruncatedError || + error instanceof StreamReadError) && + lastEventId !== undefined && + progressed + ) { + continue + } + throw error + } + + if (abortSignal?.aborted) return + + if (lastEventId !== undefined) { + // A durable (id-tagged) run. + if (progressed) { + // Clean end WITHOUT a terminal event but we advanced — the producer is + // still going (or the socket rolled over). Reconnect from the last + // offset. + continue + } + // Ended without a terminal event AND made no forward progress on this + // pass: the run cannot complete. Surface an error rather than returning + // silently, which would leave the consumer with neither a terminal event + // nor a failure. + throw new DurableStreamIncompleteError() + } + + // A non-durable (untagged) stream that ended cleanly. Legitimate — the + // upper layer synthesizes a terminal event. Stop. + return + } +} + /** * Per-send context provided by the chat client to the connection adapter. * The adapter combines this with serialized messages to build a full @@ -221,6 +375,22 @@ export interface ConnectConnectionAdapter { ) => AsyncIterable } +/** + * A {@link ConnectConnectionAdapter} that also supports joining an existing run + * (a second tab, or re-attaching after a full reload) via `joinRun`, replaying + * the ordered stream from the start off the server's delivery-durability sink. + */ +export interface ResumableConnectConnectionAdapter extends ConnectConnectionAdapter { + /** + * Join an in-flight or finished run by id, replaying from the start + * (`?offset=-1`). Read-only — sends no messages. + */ + joinRun: ( + runId: string, + abortSignal?: AbortSignal, + ) => AsyncIterable +} + export interface SubscribeConnectionAdapter { /** * Subscribe to stream chunks. @@ -482,7 +652,7 @@ function buildRunAgentInputBody( * const connection = fetchServerSentEvents('/api/chat', async () => ({ * body: { * provider: 'openai', - * model: 'gpt-4o', + * model: 'gpt-5.5', * } * })); * ``` @@ -492,7 +662,7 @@ export function fetchServerSentEvents( options: | FetchConnectionOptions | (() => FetchConnectionOptions | Promise) = {}, -): ConnectConnectionAdapter { +): ResumableConnectConnectionAdapter { return { async *connect(messages, data, abortSignal, runContext) { // Resolve URL and options if they are functions @@ -524,15 +694,55 @@ export function fetchServerSentEvents( // under `exactOptionalPropertyTypes`), so spread it conditionally // rather than passing `undefined` explicitly. const signal = abortSignal || resolvedOptions.signal - const response = await fetchClient(resolvedUrl, { - method: 'POST', - headers: requestHeaders, - body: JSON.stringify(requestBody), - credentials: resolvedOptions.credentials || 'same-origin', - ...(signal ? { signal } : {}), + const requestUrl = runContext?.runId + ? withSearchParams(resolvedUrl, { runId: runContext.runId }) + : resolvedUrl + + // Resumable SSE: if the server tags events with `id:` offsets (delivery + // durability), a dropped/rolled-over connection auto-reconnects with a + // `Last-Event-ID` header and de-dupes the replayed prefix. With no tags, + // this is a single plain fetch. + yield* resumableServerSentEvents( + fetchClient, + requestUrl, + { + method: 'POST', + headers: requestHeaders, + body: JSON.stringify(requestBody), + credentials: resolvedOptions.credentials || 'same-origin', + }, + signal, + ) + }, + async *joinRun(runId, abortSignal) { + // Read an in-flight or finished run from the start. `?offset=-1` tells the + // server's delivery-durability sink to replay from the beginning; `runId` + // identifies which run. This is a read-only GET — no messages are sent. + const resolvedUrl = typeof url === 'function' ? url() : url + const resolvedOptions = + typeof options === 'function' ? await options() : options + + const joinUrl = withSearchParams(resolvedUrl, { + offset: '-1', + runId, }) - yield* responseToSSEChunks(response, abortSignal) + const requestHeaders: Record = { + ...mergeHeaders(resolvedOptions.headers), + } + const fetchClient = resolvedOptions.fetchClient ?? fetch + const signal = abortSignal || resolvedOptions.signal + + yield* resumableServerSentEvents( + fetchClient, + joinUrl, + { + method: 'GET', + headers: requestHeaders, + credentials: resolvedOptions.credentials || 'same-origin', + }, + signal, + ) }, } } @@ -566,7 +776,7 @@ export function fetchServerSentEvents( * const connection = fetchHttpStream('/api/chat', async () => ({ * body: { * provider: 'openai', - * model: 'gpt-4o', + * model: 'gpt-5.5', * } * })); * ``` diff --git a/packages/ai-client/src/index.ts b/packages/ai-client/src/index.ts index bd9270ff6..f5eece614 100644 --- a/packages/ai-client/src/index.ts +++ b/packages/ai-client/src/index.ts @@ -88,9 +88,11 @@ export { stream, rpcStream, StreamTruncatedError, + DurableStreamIncompleteError, type ConnectConnectionAdapter, type ConnectionAdapter, type FetchConnectionOptions, + type ResumableConnectConnectionAdapter, type RunAgentInputContext, type SubscribeConnectionAdapter, type XhrConnectionOptions, diff --git a/packages/ai-client/tests/connection-adapters-resumable.test.ts b/packages/ai-client/tests/connection-adapters-resumable.test.ts new file mode 100644 index 000000000..31f9d730d --- /dev/null +++ b/packages/ai-client/tests/connection-adapters-resumable.test.ts @@ -0,0 +1,262 @@ +import { describe, expect, it, vi } from 'vitest' +import { EventType } from '@tanstack/ai/client' +import { + DurableStreamIncompleteError, + fetchServerSentEvents, +} from '../src/connection-adapters' +import type { StreamChunk } from '@tanstack/ai/client' + +function sseResponse(body: string): Response { + return new Response(body, { + headers: { 'Content-Type': 'text/event-stream' }, + }) +} + +function failingSseResponse(body: string, error: Error): Response { + const bytes = new TextEncoder().encode(body) + let sent = false + return new Response( + new ReadableStream( + { + pull(controller) { + if (!sent) { + sent = true + controller.enqueue(bytes) + return + } + controller.error(error) + }, + }, + { highWaterMark: 0 }, + ), + { headers: { 'Content-Type': 'text/event-stream' } }, + ) +} + +function contentEvent(id: string, delta: string): string { + const chunk = { + type: EventType.TEXT_MESSAGE_CONTENT, + messageId: 'm', + model: 'test', + timestamp: 0, + delta, + content: delta, + } + return `id: ${id}\ndata: ${JSON.stringify(chunk)}\n\n` +} + +function finishedEvent(id: string): string { + const chunk = { + type: EventType.RUN_FINISHED, + threadId: 't', + runId: 'r', + model: 'test', + timestamp: 0, + finishReason: 'stop', + } + return `id: ${id}\ndata: ${JSON.stringify(chunk)}\n\n` +} + +describe('resumable SSE connection adapter', () => { + it('reconnects with Last-Event-ID and de-dupes already-seen chunks', async () => { + const fetchClient = vi.fn(async (url, init) => { + expect(String(url)).toBe('/api/chat?runId=r') + if (fetchClient.mock.calls.length === 1) { + // First response: 3 tagged chunks, then the connection closes with no + // terminal event (a mid-stream drop). + return sseResponse( + contentEvent('run@1', '1') + + contentEvent('run@2', '2') + + contentEvent('run@3', '3'), + ) + } + // Second response (reconnect): server replays from the offset — it + // re-sends seq 3 (must be de-duped), then the tail + terminal. + expect(new Headers(init?.headers).get('Last-Event-ID')).toBe('run@3') + return sseResponse( + contentEvent('run@3', '3') + + contentEvent('run@4', '4') + + finishedEvent('run@5'), + ) + }) + + const adapter = fetchServerSentEvents('/api/chat', { fetchClient }) + + const chunks: Array = [] + for await (const chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + chunks.push(chunk) + } + + const deltas = chunks + .filter((c) => c.type === EventType.TEXT_MESSAGE_CONTENT) + .map((c) => c.delta) + expect(deltas).toEqual(['1', '2', '3', '4']) + expect(chunks[chunks.length - 1]?.type).toBe(EventType.RUN_FINISHED) + expect(fetchClient).toHaveBeenCalledTimes(2) + }) + + it('preserves query parameters while replacing a stale runId', async () => { + const fetchClient = vi.fn(async () => + sseResponse(finishedEvent('run@1')), + ) + const adapter = fetchServerSentEvents( + '/api/chat?provider=openai&runId=stale#response', + { fetchClient }, + ) + + for await (const _chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'current' }, + )) { + // drain + } + + expect(String(fetchClient.mock.calls[0]![0])).toBe( + '/api/chat?provider=openai&runId=current#response', + ) + }) + + it('joinRun opens the stream from the start with ?offset=-1', async () => { + const fetchClient = vi.fn(async () => + sseResponse(finishedEvent('run@1')), + ) + const adapter = fetchServerSentEvents('/api/chat', { fetchClient }) + + const chunks: Array = [] + for await (const chunk of adapter.joinRun('run-x')) { + chunks.push(chunk) + } + + const calledUrl = String(fetchClient.mock.calls[0]![0]) + expect(calledUrl).toContain('offset=-1') + expect(calledUrl).toContain('runId=run-x') + expect(chunks.map((c) => c.type)).toContain(EventType.RUN_FINISHED) + }) + + // Finding 6: a durable (id-tagged) run that ends with no terminal event and + // makes no forward progress on reconnect must surface an error, not silently + // return leaving the consumer with neither a terminal nor a failure. + it('surfaces an error when a durable run ends without a terminal and cannot progress', async () => { + const fetchClient = vi.fn(async () => { + if (fetchClient.mock.calls.length === 1) { + // First pass: two tagged content events, then a clean end with NO + // terminal — the adapter reconnects from run@2. + return sseResponse( + contentEvent('run@1', '1') + contentEvent('run@2', '2'), + ) + } + // Reconnect: server replays nothing new (no progress) and still no + // terminal — the run cannot complete. + return sseResponse('') + }) + + const adapter = fetchServerSentEvents('/api/chat', { fetchClient }) + + const deltas: Array = [] + await expect(async () => { + for await (const chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + if (chunk.type === EventType.TEXT_MESSAGE_CONTENT) { + deltas.push(chunk.delta) + } + } + }).rejects.toBeInstanceOf(DurableStreamIncompleteError) + + // The consumer still received everything delivered before the failure. + expect(deltas).toEqual(['1', '2']) + expect(fetchClient).toHaveBeenCalledTimes(2) + }) + + it('reconnects after a body reader failure and resumes exactly once', async () => { + const fetchClient = vi.fn(async (_url, init) => { + if (fetchClient.mock.calls.length === 1) { + return failingSseResponse( + contentEvent('run@1', '1'), + new TypeError('socket disconnected'), + ) + } + expect(new Headers(init?.headers).get('Last-Event-ID')).toBe('run@1') + return sseResponse( + contentEvent('run@1', '1') + + contentEvent('run@2', '2') + + finishedEvent('run@3'), + ) + }) + const adapter = fetchServerSentEvents('/api/chat', { fetchClient }) + + const chunks: Array = [] + for await (const chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + chunks.push(chunk) + } + + expect( + chunks + .filter((chunk) => chunk.type === EventType.TEXT_MESSAGE_CONTENT) + .map((chunk) => chunk.delta), + ).toEqual(['1', '2']) + expect(fetchClient).toHaveBeenCalledTimes(2) + }) + + it('does not reconnect a body reader after the caller aborts', async () => { + const fetchClient = vi.fn(async () => + failingSseResponse( + contentEvent('run@1', '1'), + new TypeError('socket disconnected'), + ), + ) + const adapter = fetchServerSentEvents('/api/chat', { fetchClient }) + const controller = new AbortController() + const chunks: Array = [] + + for await (const chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + controller.signal, + { threadId: 't', runId: 'r' }, + )) { + chunks.push(chunk) + controller.abort() + } + + expect(chunks).toHaveLength(1) + expect(fetchClient).toHaveBeenCalledOnce() + }) + + it('does not retry HTTP setup failures after earlier progress', async () => { + const fetchClient = vi.fn(async () => { + if (fetchClient.mock.calls.length === 1) { + return sseResponse(contentEvent('run@1', '1')) + } + return new Response(null, { status: 503, statusText: 'Unavailable' }) + }) + const adapter = fetchServerSentEvents('/api/chat', { fetchClient }) + + await expect(async () => { + for await (const _chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + // drain + } + }).rejects.toThrow(/503 Unavailable/) + expect(fetchClient).toHaveBeenCalledTimes(2) + }) +}) diff --git a/packages/ai-durable-stream/package.json b/packages/ai-durable-stream/package.json new file mode 100644 index 000000000..72cfeffcd --- /dev/null +++ b/packages/ai-durable-stream/package.json @@ -0,0 +1,50 @@ +{ + "name": "@tanstack/ai-durable-stream", + "version": "0.0.0", + "description": "Delivery durability for TanStack AI over the durable-streams HTTP protocol — a resumable StreamDurability transport sink (append/read/resume) that stores zero delivery events itself.", + "author": "", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/TanStack/ai.git", + "directory": "packages/ai-durable-stream" + }, + "keywords": [ + "ai", + "tanstack", + "durable-streams", + "delivery-durability", + "resumable", + "sse" + ], + "type": "module", + "module": "./dist/esm/index.js", + "types": "./dist/esm/index.d.ts", + "exports": { + ".": { + "types": "./dist/esm/index.d.ts", + "import": "./dist/esm/index.js" + } + }, + "files": [ + "dist", + "src" + ], + "scripts": { + "build": "vite build", + "clean": "premove ./build ./dist", + "lint:fix": "eslint ./src --fix", + "test:build": "publint --strict", + "test:eslint": "eslint ./src", + "test:lib": "vitest", + "test:lib:dev": "pnpm test:lib --watch", + "test:types": "tsc" + }, + "peerDependencies": { + "@tanstack/ai": "workspace:^" + }, + "devDependencies": { + "@tanstack/ai": "workspace:*", + "@vitest/coverage-v8": "4.0.14" + } +} diff --git a/packages/ai-durable-stream/src/durable-stream.ts b/packages/ai-durable-stream/src/durable-stream.ts new file mode 100644 index 000000000..638f7baca --- /dev/null +++ b/packages/ai-durable-stream/src/durable-stream.ts @@ -0,0 +1,636 @@ +import type { StreamChunk, StreamDurability } from '@tanstack/ai' + +declare const durableStreamCursorBrand: unique symbol + +/** A validated, versioned offset produced by this adapter. */ +type DurableStreamCursor = string & { + readonly [durableStreamCursorBrand]: true +} + +/** Adapter offsets also include the Durable Streams protocol sentinels. */ +export type DurableStreamOffset = DurableStreamCursor | '-1' | 'now' + +export interface DurableStreamOptions { + /** Base URL of the Durable Streams server (no trailing slash needed). */ + server: string + /** Stream-name prefix. Defaults to `runs`. */ + streamPrefix?: string + /** Fetch implementation. Defaults to the global fetch. */ + fetch?: typeof globalThis.fetch + /** + * Headers applied to every create, append, read, and close request. A + * resolver is called for every request so credentials can rotate. + */ + headers?: HeadersInit | (() => HeadersInit | Promise) +} + +export class DurableStreamError extends Error { + override name = 'DurableStreamError' + + constructor(message: string) { + super(`durableStream: ${message}`) + } +} + +interface SseEvent { + event?: string + data?: string +} + +interface WireRecord { + v: 1 + seq: number + chunk: StreamChunk +} + +interface CursorPayload { + v: 1 + backendOffset: string + seq: number +} + +interface ControlFrame { + streamNextOffset: string + streamCursor?: string + upToDate?: boolean + streamClosed?: boolean +} + +const CURSOR_PREFIX = 'tanstack-ai-ds:v1:' +const READ_ABORTED = Symbol('read aborted') + +class ResponseBodyReadFailure extends Error { + override name = 'ResponseBodyReadFailure' + + constructor(readonly readError: unknown) { + super('response body read failed') + } +} + +function assertTransportField(value: string, name: string): string { + if (value.trim().length === 0 || /[\r\n]/.test(value)) { + throw new DurableStreamError( + `${name} must be non-empty and contain no CR/LF`, + ) + } + return value +} + +function assertRunId(value: string): string { + return assertTransportField(value, 'runId') +} + +function isDurableStreamCursor(value: string): value is DurableStreamCursor { + return value.startsWith(CURSOR_PREFIX) +} + +function isCursorPayload(value: unknown): value is CursorPayload { + return ( + typeof value === 'object' && + value !== null && + 'v' in value && + value.v === 1 && + 'backendOffset' in value && + typeof value.backendOffset === 'string' && + 'seq' in value && + typeof value.seq === 'number' && + Number.isSafeInteger(value.seq) && + value.seq > 0 + ) +} + +function encodeCursor(payload: CursorPayload): DurableStreamCursor { + assertTransportField(payload.backendOffset, 'backend offset') + if (!Number.isSafeInteger(payload.seq) || payload.seq < 1) { + throw new DurableStreamError(`invalid record sequence: ${payload.seq}`) + } + const cursor = `${CURSOR_PREFIX}${encodeURIComponent(JSON.stringify(payload))}` + if (!isDurableStreamCursor(cursor)) { + throw new DurableStreamError('failed to encode cursor') + } + return cursor +} + +function decodeCursor(cursor: string): CursorPayload { + if (!isDurableStreamCursor(cursor)) { + throw new DurableStreamError('invalid or unsupported resume offset') + } + let parsed: unknown + try { + parsed = JSON.parse(decodeURIComponent(cursor.slice(CURSOR_PREFIX.length))) + } catch { + throw new DurableStreamError('invalid or unsupported resume offset') + } + if (!isCursorPayload(parsed)) { + throw new DurableStreamError('invalid or unsupported resume offset') + } + assertTransportField(parsed.backendOffset, 'backend offset') + return parsed +} + +function safeSearchParam(request: Request, key: string): string | null { + try { + return new URL(request.url).searchParams.get(key) + } catch { + return null + } +} + +function parseResumeOffset(raw: string | null): DurableStreamOffset | null { + if (raw === null || raw === '-1' || raw === 'now') return raw + decodeCursor(raw) + if (!isDurableStreamCursor(raw)) { + throw new DurableStreamError('invalid or unsupported resume offset') + } + return raw +} + +async function* readLines( + body: ReadableStream, + signal?: AbortSignal, +): AsyncGenerator { + const reader = body.getReader() + const decoder = new TextDecoder() + let buffer = '' + let completed = false + let cancelled = false + let readFailed = false + try { + for (;;) { + let result: ReadableStreamReadResult | typeof READ_ABORTED + try { + result = await readWithAbort(reader, signal) + } catch (error) { + readFailed = true + throw new ResponseBodyReadFailure(error) + } + if (result === READ_ABORTED) { + cancelled = true + await reader.cancel(signal?.reason) + return + } + if (result.done) { + completed = true + break + } + buffer += decoder.decode(result.value, { stream: true }) + const parts = buffer.split('\n') + buffer = parts.pop() ?? '' + for (const raw of parts) { + yield raw.endsWith('\r') ? raw.slice(0, -1) : raw + } + } + buffer += decoder.decode() + if (buffer.length > 0) { + yield buffer.endsWith('\r') ? buffer.slice(0, -1) : buffer + } + } finally { + try { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- readFailed is set in the catch before its throw; CFA can't see that from finally + if (!completed && !cancelled && !readFailed) await reader.cancel() + } finally { + reader.releaseLock() + } + } +} + +function readWithAbort( + reader: ReadableStreamDefaultReader, + signal: AbortSignal | undefined, +): Promise | typeof READ_ABORTED> { + if (!signal) return reader.read() + if (signal.aborted) return Promise.resolve(READ_ABORTED) + + return new Promise((resolve, reject) => { + const onAbort = () => { + signal.removeEventListener('abort', onAbort) + resolve(READ_ABORTED) + } + signal.addEventListener('abort', onAbort, { once: true }) + reader.read().then( + (result) => { + signal.removeEventListener('abort', onAbort) + resolve(result) + }, + (error: unknown) => { + signal.removeEventListener('abort', onAbort) + reject(error) + }, + ) + }) +} + +async function* parseSseEvents( + body: ReadableStream, + signal?: AbortSignal, +): AsyncGenerator { + let current: SseEvent = {} + let hasField = false + + for await (const line of readLines(body, signal)) { + if (line === '') { + if (hasField) yield current + current = {} + hasField = false + continue + } + if (line.startsWith(':')) continue + + const colon = line.indexOf(':') + const field = colon === -1 ? line : line.slice(0, colon) + let value = colon === -1 ? '' : line.slice(colon + 1) + if (value.startsWith(' ')) value = value.slice(1) + if (field === 'event') { + current.event = value + hasField = true + } else if (field === 'data') { + current.data = + current.data === undefined ? value : `${current.data}\n${value}` + hasField = true + } + } + if (hasField) yield current +} + +function isStreamChunk(value: unknown): value is StreamChunk { + return ( + typeof value === 'object' && + value !== null && + 'type' in value && + typeof value.type === 'string' + ) +} + +function isWireRecord(value: unknown): value is WireRecord { + return ( + typeof value === 'object' && + value !== null && + 'v' in value && + value.v === 1 && + 'seq' in value && + typeof value.seq === 'number' && + Number.isSafeInteger(value.seq) && + value.seq > 0 && + 'chunk' in value && + isStreamChunk(value.chunk) + ) +} + +function parseDataRecords(data: string | undefined): Array { + if (data === undefined) { + throw new DurableStreamError('data event had no payload') + } + let parsed: unknown + try { + parsed = JSON.parse(data) + } catch { + throw new DurableStreamError('data event contained invalid JSON') + } + if (!Array.isArray(parsed)) { + throw new DurableStreamError('data event payload must be a JSON array') + } + const records: Array = [] + for (const value of parsed) { + if (!isWireRecord(value)) { + throw new DurableStreamError('data event contained an invalid record') + } + records.push(value) + } + return records +} + +function optionalBoolean( + value: object, + name: 'upToDate' | 'streamClosed', +): boolean | undefined { + const field = + name === 'upToDate' + ? 'upToDate' in value + ? value.upToDate + : undefined + : 'streamClosed' in value + ? value.streamClosed + : undefined + if (field === undefined) return undefined + if (typeof field !== 'boolean') { + throw new DurableStreamError(`control field ${name} must be boolean`) + } + return field +} + +function parseControlFrame(data: string | undefined): ControlFrame { + if (data === undefined) { + throw new DurableStreamError('control event had no payload') + } + let parsed: unknown + try { + parsed = JSON.parse(data) + } catch { + throw new DurableStreamError('control event contained invalid JSON') + } + if (typeof parsed !== 'object' || parsed === null) { + throw new DurableStreamError('control event payload must be an object') + } + if ( + !('streamNextOffset' in parsed) || + typeof parsed.streamNextOffset !== 'string' + ) { + throw new DurableStreamError( + 'control event requires string streamNextOffset', + ) + } + const streamNextOffset = assertTransportField( + parsed.streamNextOffset, + 'control streamNextOffset', + ) + let streamCursor: string | undefined + if ('streamCursor' in parsed) { + if (typeof parsed.streamCursor !== 'string') { + throw new DurableStreamError('control streamCursor must be a string') + } + streamCursor = assertTransportField( + parsed.streamCursor, + 'control streamCursor', + ) + } + const upToDate = optionalBoolean(parsed, 'upToDate') + const streamClosed = optionalBoolean(parsed, 'streamClosed') + if (streamClosed !== true && streamCursor === undefined) { + throw new DurableStreamError( + 'open control event requires string streamCursor', + ) + } + return { + streamNextOffset, + ...(streamCursor === undefined ? {} : { streamCursor }), + ...(upToDate === undefined ? {} : { upToDate }), + ...(streamClosed === undefined ? {} : { streamClosed }), + } +} + +function requireNextOffset(response: Response, operation: string): string { + const offset = response.headers.get('Stream-Next-Offset') + if (offset === null || offset.trim().length === 0) { + throw new DurableStreamError( + `${operation} response missing non-empty Stream-Next-Offset`, + ) + } + return assertTransportField(offset, `${operation} Stream-Next-Offset`) +} + +function httpFailure( + operation: string, + response: Response, +): DurableStreamError { + return new DurableStreamError( + `failed to ${operation} (${response.status} ${response.statusText})`, + ) +} + +/** External-URL Durable Streams protocol adapter. */ +export function durableStream( + request: Request, + options: DurableStreamOptions, +): StreamDurability { + const fetchFn = options.fetch ?? globalThis.fetch + assertTransportField(options.server, 'server URL') + try { + void new URL(options.server) + } catch { + throw new DurableStreamError( + `invalid server URL: ${JSON.stringify(options.server)}`, + ) + } + const server = options.server.replace(/\/+$/, '') + const prefix = assertTransportField( + options.streamPrefix ?? 'runs', + 'streamPrefix', + ) + const rawResumeOffset = + request.headers.get('Last-Event-ID') ?? safeSearchParam(request, 'offset') + const resumeOffset = parseResumeOffset(rawResumeOffset) + const requestedRunId = safeSearchParam(request, 'runId') + if (resumeOffset !== null && requestedRunId === null) { + throw new DurableStreamError('resume offset requires a runId') + } + const runId = assertRunId(requestedRunId ?? crypto.randomUUID()) + + const streamUrl = `${server}/streams/${encodeURIComponent(`${prefix}/${runId}`)}` + let createPromise: Promise | undefined + let appendTailOffset: string | undefined + let nextSeq = 1 + const producerId = crypto.randomUUID() + const producerEpoch = 0 + let producerSeq = 0 + let closePromise: Promise | undefined + + const resolveHeaders = async (required?: HeadersInit): Promise => { + const configured = + typeof options.headers === 'function' + ? await options.headers() + : options.headers + const headers = new Headers(configured) + if (required) { + new Headers(required).forEach((value, key) => headers.set(key, value)) + } + return headers + } + + const ensureCreated = (): Promise => { + if (createPromise) return createPromise + createPromise = (async () => { + const response = await fetchFn(streamUrl, { + method: 'PUT', + headers: await resolveHeaders({ 'Content-Type': 'application/json' }), + }) + if (!response.ok) throw httpFailure('create stream', response) + const offset = requireNextOffset(response, 'create') + appendTailOffset = offset + return offset + })().catch((error: unknown) => { + createPromise = undefined + throw error + }) + return createPromise + } + + return { + resumeFrom: () => resumeOffset, + append: async (chunks) => { + if (chunks.length === 0) return [] + const batchStartOffset = appendTailOffset ?? (await ensureCreated()) + const firstSeq = nextSeq + const records = chunks.map( + (chunk, index): WireRecord => ({ + v: 1, + seq: firstSeq + index, + chunk, + }), + ) + nextSeq += records.length + const requestProducerSeq = producerSeq + producerSeq += 1 + const requestInit: RequestInit = { + method: 'POST', + headers: await resolveHeaders({ + 'Content-Type': 'application/json', + 'Producer-Id': producerId, + 'Producer-Epoch': String(producerEpoch), + 'Producer-Seq': String(requestProducerSeq), + }), + body: JSON.stringify(records), + } + let response: Response + try { + response = await fetchFn(streamUrl, requestInit) + } catch (firstError) { + try { + response = await fetchFn(streamUrl, requestInit) + } catch (retryError) { + throw new AggregateError( + [firstError, retryError], + 'durableStream: append failed before its outcome could be confirmed', + ) + } + } + if (!response.ok) throw httpFailure('append', response) + const nextOffset = requireNextOffset(response, 'append') + appendTailOffset = nextOffset + return records.map((record) => + encodeCursor({ + v: 1, + backendOffset: batchStartOffset, + seq: record.seq, + }), + ) + }, + close: () => { + if (closePromise) return closePromise + closePromise = (async () => { + await ensureCreated() + const response = await fetchFn(streamUrl, { + method: 'POST', + headers: await resolveHeaders({ 'Stream-Closed': 'true' }), + }) + if (!response.ok) throw httpFailure('close', response) + const nextOffset = requireNextOffset(response, 'close') + if (response.headers.get('Stream-Closed')?.toLowerCase() !== 'true') { + throw new DurableStreamError( + 'close response missing Stream-Closed: true', + ) + } + appendTailOffset = nextOffset + })().catch((error: unknown) => { + closePromise = undefined + throw error + }) + return closePromise + }, + read: async function* (offset, signal) { + let backendOffset: string + let deliveredThroughSeq = 0 + if (offset === '-1' || offset === 'now') { + backendOffset = offset + } else { + const cursor = decodeCursor(offset) + backendOffset = cursor.backendOffset + deliveredThroughSeq = cursor.seq + } + let streamCursor: string | undefined + + for (;;) { + if (signal?.aborted) return + const requestOffset = backendOffset + const requestCursor = streamCursor + const url = new URL(streamUrl) + url.searchParams.set('offset', backendOffset) + url.searchParams.set('live', 'sse') + if (streamCursor !== undefined) { + url.searchParams.set('cursor', streamCursor) + } + + let response: Response + try { + response = await fetchFn(url, { + method: 'GET', + headers: await resolveHeaders(), + signal, + }) + } catch (error) { + if (signal?.aborted) return + throw error + } + if (!response.ok) throw httpFailure('read', response) + if (!response.body) { + throw new DurableStreamError('read response had no body') + } + + let dataStartOffset = backendOffset + let sawControl = false + let dataAwaitingControl = false + let yieldedData = false + try { + for await (const event of parseSseEvents(response.body, signal)) { + if (signal?.aborted) return + if (event.event === 'data') { + dataAwaitingControl = true + for (const record of parseDataRecords(event.data)) { + if (record.seq <= deliveredThroughSeq) continue + deliveredThroughSeq = record.seq + yieldedData = true + yield { + offset: encodeCursor({ + v: 1, + backendOffset: dataStartOffset, + seq: record.seq, + }), + chunk: record.chunk, + } + } + continue + } + if (event.event === 'control') { + const control = parseControlFrame(event.data) + backendOffset = control.streamNextOffset + streamCursor = control.streamCursor + dataStartOffset = backendOffset + sawControl = true + dataAwaitingControl = false + if (control.streamClosed === true) return + continue + } + throw new DurableStreamError( + `unexpected SSE event type: ${JSON.stringify(event.event)}`, + ) + } + } catch (error) { + if (signal?.aborted) return + if (error instanceof ResponseBodyReadFailure) { + if ( + yieldedData || + (sawControl && + (backendOffset !== requestOffset || + streamCursor !== requestCursor)) + ) { + continue + } + throw error.readError + } + throw error + } + + if (signal?.aborted) return + if (dataAwaitingControl || !sawControl) { + throw new DurableStreamError( + 'read SSE window ended without a matching control event', + ) + } + if (backendOffset === requestOffset && streamCursor === requestCursor) { + throw new DurableStreamError( + 'read SSE window ended without advancing offset or cursor', + ) + } + } + }, + } +} diff --git a/packages/ai-durable-stream/src/index.ts b/packages/ai-durable-stream/src/index.ts new file mode 100644 index 000000000..857485aae --- /dev/null +++ b/packages/ai-durable-stream/src/index.ts @@ -0,0 +1,10 @@ +/** + * `@tanstack/ai-durable-stream` — a {@link StreamDurability} backend over the + * durable-streams HTTP protocol, for production-grade delivery durability. + */ +export { durableStream } from './durable-stream' +export { DurableStreamError } from './durable-stream' +export type { + DurableStreamOffset, + DurableStreamOptions, +} from './durable-stream' diff --git a/packages/ai-durable-stream/tests/durable-stream-types.test-d.ts b/packages/ai-durable-stream/tests/durable-stream-types.test-d.ts new file mode 100644 index 000000000..576151967 --- /dev/null +++ b/packages/ai-durable-stream/tests/durable-stream-types.test-d.ts @@ -0,0 +1,16 @@ +import { expectTypeOf } from 'vitest' +import { durableStream } from '../src' +import type { DurableStreamOffset } from '../src' + +declare const durability: ReturnType +declare const offset: DurableStreamOffset + +expectTypeOf( + durability.resumeFrom(), +).toEqualTypeOf() +durability.read(offset) +durability.read('-1') +durability.read('now') + +// @ts-expect-error arbitrary strings are not validated adapter cursors +durability.read('unvalidated-offset') diff --git a/packages/ai-durable-stream/tests/durable-stream.test.ts b/packages/ai-durable-stream/tests/durable-stream.test.ts new file mode 100644 index 000000000..bb69620f2 --- /dev/null +++ b/packages/ai-durable-stream/tests/durable-stream.test.ts @@ -0,0 +1,1048 @@ +import { describe, expect, it, vi } from 'vitest' +import { EventType, toServerSentEventsResponse } from '@tanstack/ai' +import { durableStream } from '../src' +import type { StreamChunk } from '@tanstack/ai' +import type { DurableStreamOffset } from '../src' + +interface WireRecord { + v: 1 + seq: number + chunk: StreamChunk +} + +interface CapturedRequest { + url: URL + method: string + headers: Headers + body?: string + signal?: AbortSignal | null +} + +interface StoredBatch { + startOffset: string + nextOffset: string + records: Array +} + +interface ProtocolServerOptions { + createOffset?: string + appendOffsets?: Array + closeResponse?: () => Promise + readStatus?: number + loseAppendResponses?: number +} + +function textChunk(delta: string) { + return { + type: EventType.TEXT_MESSAGE_CONTENT, + messageId: 'message-1', + delta, + timestamp: 0, + } as const +} + +function textStream(deltas: Array): AsyncIterable { + return { + async *[Symbol.asyncIterator]() { + for (const delta of deltas) yield textChunk(delta) + }, + } +} + +function isStreamChunk(value: unknown): value is StreamChunk { + return ( + typeof value === 'object' && + value !== null && + 'type' in value && + typeof value.type === 'string' + ) +} + +function isWireRecord(value: unknown): value is WireRecord { + return ( + typeof value === 'object' && + value !== null && + 'v' in value && + value.v === 1 && + 'seq' in value && + typeof value.seq === 'number' && + 'chunk' in value && + isStreamChunk(value.chunk) + ) +} + +function parseRecords(body: string | undefined) { + if (body === undefined) throw new Error('Expected an append body') + const parsed: unknown = JSON.parse(body) + if (!Array.isArray(parsed)) throw new Error('Expected a JSON record array') + const records: Array = [] + for (const value of parsed) { + if (!isWireRecord(value)) throw new Error('Invalid versioned wire record') + records.push(value) + } + return records +} + +function requestUrl(input: Parameters[0]) { + if (input instanceof Request) return new URL(input.url) + return new URL(input.toString()) +} + +function createHeaders(offset: string | null, closed = false) { + const headers = new Headers() + if (offset !== null) headers.set('Stream-Next-Offset', offset) + if (closed) headers.set('Stream-Closed', 'true') + return headers +} + +function dataEvent(records: Array) { + return `event: data\ndata: ${JSON.stringify(records)}\n\n` +} + +function controlEvent(control: { + streamNextOffset: string + streamCursor?: string + upToDate?: boolean + streamClosed?: boolean +}) { + return `event: control\ndata: ${JSON.stringify(control)}\n\n` +} + +function makeProtocolServer(options: ProtocolServerOptions = {}) { + const requests: Array = [] + const batches: Array = [] + const createOffset = + options.createOffset ?? 'origin::partition/A?cursor=%2F+==' + let tailOffset = createOffset + let closed = false + let appendIndex = 0 + let lostAppendResponses = options.loseAppendResponses ?? 0 + const producerResponses = new Map() + + const fetchStub = vi.fn(async (input, init) => { + const url = requestUrl(input) + const method = (init?.method ?? 'GET').toUpperCase() + const headers = new Headers(init?.headers) + const body = init?.body === undefined ? undefined : String(init.body) + requests.push({ + url, + method, + headers, + ...(body === undefined ? {} : { body }), + signal: init?.signal, + }) + + if (method === 'PUT') { + return new Response(null, { + status: 201, + headers: createHeaders(createOffset), + }) + } + + if (method === 'POST' && headers.get('Stream-Closed') === 'true') { + if (options.closeResponse) { + const response = await options.closeResponse() + if (response.ok) closed = true + return response + } + closed = true + return new Response(null, { + status: 204, + headers: createHeaders(tailOffset, true), + }) + } + + if (method === 'POST') { + const records = parseRecords(body) + const producerId = headers.get('Producer-Id') + const producerEpoch = headers.get('Producer-Epoch') + const producerSeq = headers.get('Producer-Seq') + const producerHeaders = [producerId, producerEpoch, producerSeq] + if ( + producerHeaders.some((value) => value !== null) && + producerHeaders.some((value) => value === null) + ) { + return new Response(null, { status: 400 }) + } + const producerKey = + producerId === null || producerEpoch === null || producerSeq === null + ? undefined + : `${producerId}:${producerEpoch}:${producerSeq}` + const deduplicatedOffset = + producerKey === undefined + ? undefined + : producerResponses.get(producerKey) + if (deduplicatedOffset !== undefined) { + return new Response(null, { + status: 204, + headers: createHeaders(deduplicatedOffset), + }) + } + const configuredOffset = options.appendOffsets?.[appendIndex] + const nextOffset = + configuredOffset === undefined + ? `opaque::next/${appendIndex}?token=%2B==` + : configuredOffset + appendIndex += 1 + if (nextOffset !== null && nextOffset.length > 0) { + batches.push({ startOffset: tailOffset, nextOffset, records }) + tailOffset = nextOffset + if (producerKey !== undefined) { + producerResponses.set(producerKey, nextOffset) + } + } + if (lostAppendResponses > 0) { + lostAppendResponses -= 1 + throw new TypeError('append response was lost') + } + return new Response(null, { + status: 204, + headers: createHeaders(nextOffset), + }) + } + + if (options.readStatus !== undefined) { + return new Response(null, { status: options.readStatus }) + } + + const requestedOffset = url.searchParams.get('offset') ?? '-1' + let firstBatch = 0 + if (requestedOffset === 'now' || requestedOffset === tailOffset) { + firstBatch = batches.length + } else if (requestedOffset !== '-1') { + firstBatch = batches.findIndex( + (batch) => batch.startOffset === requestedOffset, + ) + if (firstBatch === -1) return new Response(null, { status: 400 }) + } + + let sse = '' + for (let index = firstBatch; index < batches.length; index += 1) { + const batch = batches[index] + if (!batch) continue + const isFinal = index === batches.length - 1 + sse += dataEvent(batch.records) + sse += controlEvent({ + streamNextOffset: batch.nextOffset, + ...(!closed || !isFinal + ? { streamCursor: `collapse::${index}?edge=%2F` } + : {}), + ...(isFinal ? { upToDate: true } : {}), + ...(closed && isFinal ? { streamClosed: true } : {}), + }) + } + if (firstBatch === batches.length) { + sse += controlEvent({ + streamNextOffset: tailOffset, + ...(closed + ? { streamClosed: true } + : { streamCursor: 'collapse::tail', upToDate: true }), + }) + } + return new Response(sse, { + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + }) + }) + + return { + fetchStub, + requests, + batches, + createOffset, + closeCount: () => + requests.filter( + (request) => + request.method === 'POST' && + request.headers.get('Stream-Closed') === 'true', + ).length, + } +} + +function requestWithMethod(requests: Array, method: string) { + const request = requests.find((candidate) => candidate.method === method) + if (!request) throw new Error(`Expected a ${method} request`) + return request +} + +async function readBody(response: Response) { + if (!response.body) throw new Error('Expected a response body') + const reader = response.body.getReader() + const decoder = new TextDecoder() + let body = '' + for (;;) { + const result = await reader.read() + if (result.done) return body + body += decoder.decode(result.value) + } +} + +function parseTransportEvents(body: string) { + return body + .split('\n\n') + .filter((block) => block.length > 0) + .map((block) => { + const lines = block.split('\n') + const id = lines.find((line) => line.startsWith('id: '))?.slice(4) + const data = lines.find((line) => line.startsWith('data: '))?.slice(6) + if (!data) throw new Error(`Missing transport data in ${block}`) + const parsed: unknown = JSON.parse(data) + return { id, data: parsed } + }) +} + +function deltaFrom(value: unknown) { + if ( + typeof value !== 'object' || + value === null || + !('delta' in value) || + typeof value.delta !== 'string' + ) { + throw new Error('Expected a text chunk') + } + return value.delta +} + +function deferred() { + let resolve = (_value: T): void => undefined + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +describe('durableStream official HTTP protocol', () => { + it('appends one versioned batch and returns one adapter offset per record', async () => { + const server = makeProtocolServer() + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-batch'), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + + const offsets = await durability.append([ + textChunk('a'), + textChunk('b'), + textChunk('c'), + ]) + + expect(offsets).toHaveLength(3) + expect(new Set(offsets).size).toBe(3) + expect(server.requests.map((request) => request.method)).toEqual([ + 'PUT', + 'POST', + ]) + expect( + requestWithMethod(server.requests, 'PUT').headers.get('Content-Type'), + ).toBe('application/json') + const appendRequest = requestWithMethod(server.requests, 'POST') + expect(appendRequest.headers.get('Content-Type')).toBe('application/json') + expect(parseRecords(appendRequest.body)).toEqual([ + { v: 1, seq: 1, chunk: textChunk('a') }, + { v: 1, seq: 2, chunk: textChunk('b') }, + { v: 1, seq: 3, chunk: textChunk('c') }, + ]) + }) + + it('retries an ambiguously committed append with one producer tuple', async () => { + const server = makeProtocolServer({ loseAppendResponses: 1 }) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-idempotent-producer'), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + const source: AsyncIterable = { + async *[Symbol.asyncIterator]() { + yield textChunk('persisted-before-failure') + throw new Error('provider failed') + }, + } + + await readBody( + toServerSentEventsResponse(source, { + durability: { adapter: durability }, + }), + ) + + const appendRequests = server.requests.filter( + (request) => + request.method === 'POST' && + request.headers.get('Stream-Closed') !== 'true', + ) + expect( + appendRequests.map((request) => request.headers.get('Producer-Seq')), + ).toEqual(['0', '0', '1']) + expect( + new Set( + appendRequests.map((request) => request.headers.get('Producer-Id')), + ).size, + ).toBe(1) + expect(server.batches).toHaveLength(2) + + const replayed: Array = [] + for await (const { chunk } of durability.read('-1')) replayed.push(chunk) + expect(replayed.map((chunk) => chunk.type)).toEqual([ + EventType.TEXT_MESSAGE_CONTENT, + EventType.RUN_ERROR, + ]) + }) + + it('applies static auth headers to create, append, close, and read', async () => { + const server = makeProtocolServer() + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-static-auth'), + { + server: 'https://ds.test', + fetch: server.fetchStub, + headers: { + Authorization: 'Bearer static-token', + 'Content-Type': 'text/plain', + 'Stream-Closed': 'false', + }, + }, + ) + + await durability.append([textChunk('secured')]) + await durability.close() + for await (const _entry of durability.read('-1')) { + // drain + } + + expect(server.requests).toHaveLength(4) + expect( + server.requests.map((request) => request.headers.get('Authorization')), + ).toEqual(Array.from({ length: 4 }, () => 'Bearer static-token')) + expect(server.requests[0]?.headers.get('Content-Type')).toBe( + 'application/json', + ) + expect(server.requests[1]?.headers.get('Content-Type')).toBe( + 'application/json', + ) + expect(server.requests[2]?.headers.get('Stream-Closed')).toBe('true') + }) + + it('resolves rotating async auth headers for every protocol request', async () => { + const server = makeProtocolServer() + let token = 0 + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-rotating-auth'), + { + server: 'https://ds.test', + fetch: server.fetchStub, + headers: async () => ({ Authorization: `Bearer token-${++token}` }), + }, + ) + + await durability.append([textChunk('secured')]) + await durability.close() + for await (const _entry of durability.read('-1')) { + // drain + } + + expect( + server.requests.map((request) => request.headers.get('Authorization')), + ).toEqual([ + 'Bearer token-1', + 'Bearer token-2', + 'Bearer token-3', + 'Bearer token-4', + ]) + }) + + it.each([ + ['missing', null], + ['empty', ''], + ])('fails when an append returns a %s next offset', async (_name, offset) => { + const server = makeProtocolServer({ appendOffsets: [offset] }) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-bad-offset'), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + + await expect(durability.append([textChunk('x')])).rejects.toThrow( + /Stream-Next-Offset/, + ) + }) + + it('parses conforming id-less data and control events', async () => { + const server = makeProtocolServer() + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-idless'), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + await durability.append([textChunk('a'), textChunk('b')]) + await durability.close() + + const received: Array<{ offset: DurableStreamOffset; delta: string }> = [] + for await (const entry of durability.read('-1')) { + received.push({ + offset: entry.offset, + delta: + entry.chunk.type === EventType.TEXT_MESSAGE_CONTENT + ? entry.chunk.delta + : entry.chunk.type, + }) + } + + expect(received.map((entry) => entry.delta)).toEqual(['a', 'b']) + expect(new Set(received.map((entry) => entry.offset)).size).toBe(2) + }) + + it('reconnects an open SSE window with control offset and cursor', async () => { + const requests: Array = [] + let readNumber = 0 + const fetchStub = vi.fn(async (input, init) => { + const url = requestUrl(input) + const headers = new Headers(init?.headers) + requests.push({ + url, + method: (init?.method ?? 'GET').toUpperCase(), + headers, + signal: init?.signal, + }) + readNumber += 1 + const record = { + v: 1, + seq: readNumber, + chunk: textChunk(String(readNumber)), + } satisfies WireRecord + const nextOffset = `opaque::window/${readNumber}?token=%2F` + return new Response( + dataEvent([record]) + + controlEvent({ + streamNextOffset: nextOffset, + ...(readNumber === 1 + ? { streamCursor: 'collapse::window-1', upToDate: true } + : { streamClosed: true }), + }), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ) + }) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-rollover'), + { server: 'https://ds.test', fetch: fetchStub }, + ) + + const deltas: Array = [] + for await (const { chunk } of durability.read('-1')) { + if (chunk.type === EventType.TEXT_MESSAGE_CONTENT) + deltas.push(chunk.delta) + } + + expect(deltas).toEqual(['1', '2']) + expect(requests[1]?.url.searchParams.get('offset')).toBe( + 'opaque::window/1?token=%2F', + ) + expect(requests[1]?.url.searchParams.get('cursor')).toBe( + 'collapse::window-1', + ) + }) + + it('reconnects after a body read failure from the last valid control', async () => { + const requests: Array = [] + let readNumber = 0 + const fetchStub = vi.fn(async (input, init) => { + const url = requestUrl(input) + requests.push({ + url, + method: (init?.method ?? 'GET').toUpperCase(), + headers: new Headers(init?.headers), + signal: init?.signal, + }) + readNumber += 1 + + if (readNumber === 1) { + let pullNumber = 0 + const body = new ReadableStream({ + pull(controller) { + pullNumber += 1 + if (pullNumber === 1) { + controller.enqueue( + new TextEncoder().encode( + dataEvent([ + { v: 1, seq: 1, chunk: textChunk('before-failure') }, + ]) + + controlEvent({ + streamNextOffset: 'opaque::after/1?token=%2F', + streamCursor: 'collapse::after-1', + upToDate: true, + }), + ), + ) + return + } + controller.error(new TypeError('socket read failed')) + }, + }) + return new Response(body, { + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + }) + } + + return new Response( + dataEvent([{ v: 1, seq: 2, chunk: textChunk('after-reconnect') }]) + + controlEvent({ + streamNextOffset: 'opaque::after/2?token=%2F', + streamClosed: true, + }), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ) + }) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-body-reconnect'), + { server: 'https://ds.test', fetch: fetchStub }, + ) + + const deltas: Array = [] + for await (const { chunk } of durability.read('-1')) { + if (chunk.type === EventType.TEXT_MESSAGE_CONTENT) { + deltas.push(chunk.delta) + } + } + + expect(deltas).toEqual(['before-failure', 'after-reconnect']) + expect(requests).toHaveLength(2) + expect(requests[1]?.url.searchParams.get('offset')).toBe( + 'opaque::after/1?token=%2F', + ) + expect(requests[1]?.url.searchParams.get('cursor')).toBe( + 'collapse::after-1', + ) + }) + + it('reconnects from the same control after data is read before a body failure', async () => { + const requests: Array = [] + let readNumber = 0 + const fetchStub = vi.fn(async (input, init) => { + const url = requestUrl(input) + requests.push({ + url, + method: (init?.method ?? 'GET').toUpperCase(), + headers: new Headers(init?.headers), + signal: init?.signal, + }) + readNumber += 1 + + if (readNumber === 1) { + return new Response( + dataEvent([{ v: 1, seq: 1, chunk: textChunk('before-drop') }]) + + controlEvent({ + streamNextOffset: 'opaque::stable/window?token=%2F', + streamCursor: 'collapse::stable-window', + }), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ) + } + + if (readNumber === 2) { + let pullNumber = 0 + return new Response( + new ReadableStream({ + pull(controller) { + pullNumber += 1 + if (pullNumber === 1) { + controller.enqueue( + new TextEncoder().encode( + dataEvent([ + { v: 1, seq: 2, chunk: textChunk('during-drop') }, + ]), + ), + ) + return + } + controller.error(new TypeError('socket read failed')) + }, + }), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ) + } + + return new Response( + dataEvent([ + { v: 1, seq: 2, chunk: textChunk('during-drop') }, + { v: 1, seq: 3, chunk: textChunk('after-reconnect') }, + ]) + + controlEvent({ + streamNextOffset: 'opaque::tail/window?token=%3D', + streamClosed: true, + }), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ) + }) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-data-body-reconnect'), + { server: 'https://ds.test', fetch: fetchStub }, + ) + + const deltas: Array = [] + for await (const { chunk } of durability.read('-1')) { + if (chunk.type === EventType.TEXT_MESSAGE_CONTENT) { + deltas.push(chunk.delta) + } + } + + expect(deltas).toEqual(['before-drop', 'during-drop', 'after-reconnect']) + expect(requests).toHaveLength(3) + expect(requests[2]?.url.searchParams.get('offset')).toBe( + requests[1]?.url.searchParams.get('offset'), + ) + expect(requests[2]?.url.searchParams.get('cursor')).toBe( + requests[1]?.url.searchParams.get('cursor'), + ) + expect(requests[2]?.url.searchParams.get('offset')).toBe( + 'opaque::stable/window?token=%2F', + ) + expect(requests[2]?.url.searchParams.get('cursor')).toBe( + 'collapse::stable-window', + ) + }) + + it('fails loudly when a control event omits streamNextOffset', async () => { + const fetchStub = vi.fn( + async () => + new Response( + 'event: control\ndata: {"upToDate":true,"streamCursor":"c"}\n\n', + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ), + ) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-bad-control'), + { server: 'https://ds.test', fetch: fetchStub }, + ) + + await expect(async () => { + for await (const _entry of durability.read('-1')) { + // drain + } + }).rejects.toThrow(/streamNextOffset/) + }) + + it('requires a separate runId when resuming from an adapter offset', async () => { + const server = makeProtocolServer() + const producer = durableStream( + new Request('https://app.test/api/chat?runId=run-resume'), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + const [resumeOffset] = await producer.append([textChunk('x')]) + if (!resumeOffset) throw new Error('Expected a resume offset') + + expect(() => + durableStream( + new Request('https://app.test/api/chat', { + headers: { 'Last-Event-ID': resumeOffset }, + }), + { server: 'https://ds.test', fetch: server.fetchStub }, + ), + ).toThrow(/resume offset requires a runId/) + + expect( + durableStream( + new Request('https://app.test/api/chat?runId=run-resume', { + headers: { 'Last-Event-ID': resumeOffset }, + }), + { server: 'https://ds.test', fetch: server.fetchStub }, + ).resumeFrom(), + ).toBe(resumeOffset) + }) + + it('rejects CR/LF injection in run ids, prefixes, cursors, and controls', async () => { + expect(() => + durableStream( + new Request( + `https://app.test/api/chat?runId=${encodeURIComponent('bad\nrun')}`, + ), + { server: 'https://ds.test' }, + ), + ).toThrow(/CR\/LF/) + expect(() => + durableStream(new Request('https://app.test/api/chat'), { + server: 'https://ds.test', + streamPrefix: 'bad\rprefix', + }), + ).toThrow(/CR\/LF/) + + const forgedCursor = `tanstack-ai-ds:v1:${encodeURIComponent( + JSON.stringify({ + v: 1, + backendOffset: 'bad\noffset', + seq: 1, + }), + )}` + expect(() => + durableStream( + new Request('https://app.test/api/chat', { + headers: { 'Last-Event-ID': forgedCursor }, + }), + { server: 'https://ds.test' }, + ), + ).toThrow(/CR\/LF/) + + const fetchStub = vi.fn( + async () => + new Response( + controlEvent({ + streamNextOffset: 'bad\ncontrol-offset', + streamClosed: true, + }), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ), + ) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-control-injection'), + { server: 'https://ds.test', fetch: fetchStub }, + ) + await expect(async () => { + for await (const _entry of durability.read('-1')) { + // drain + } + }).rejects.toThrow(/CR\/LF/) + }) + + it('awaits external close and sends the protocol close header', async () => { + const closing = deferred() + const server = makeProtocolServer({ + closeResponse: () => closing.promise, + }) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-close'), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + await durability.append([textChunk('x')]) + + const closePromise = durability.close() + let settled = false + void closePromise.then(() => { + settled = true + }) + await vi.waitFor(() => expect(server.closeCount()).toBe(1)) + expect(settled).toBe(false) + const closeRequest = server.requests.at(-1) + expect(closeRequest?.headers.get('Stream-Closed')).toBe('true') + + closing.resolve( + new Response(null, { + status: 204, + headers: createHeaders('opaque::closed', true), + }), + ) + await expect(closePromise).resolves.toBeUndefined() + }) + + it('surfaces close and read HTTP failures', async () => { + const closeServer = makeProtocolServer({ + closeResponse: async () => new Response(null, { status: 503 }), + }) + const closeDurability = durableStream( + new Request('https://app.test/api/chat?runId=run-close-error'), + { server: 'https://ds.test', fetch: closeServer.fetchStub }, + ) + await closeDurability.append([textChunk('x')]) + await expect(closeDurability.close()).rejects.toThrow(/failed to close/) + + const readServer = makeProtocolServer({ readStatus: 502 }) + const readDurability = durableStream( + new Request('https://app.test/api/chat?runId=run-read-error'), + { server: 'https://ds.test', fetch: readServer.fetchStub }, + ) + await expect(async () => { + for await (const _entry of readDurability.read('-1')) { + // drain + } + }).rejects.toThrow(/failed to read/) + }) + + it.each([ + ['next offset', createHeaders(null, true)], + ['closed state', createHeaders('opaque::closed', false)], + ])( + 'rejects a successful close response missing its %s', + async (_name, headers) => { + const server = makeProtocolServer({ + closeResponse: async () => new Response(null, { status: 204, headers }), + }) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-invalid-close'), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + await durability.append([textChunk('x')]) + + await expect(durability.close()).rejects.toThrow(/close response/i) + }, + ) + + it('retries close after a transient failure', async () => { + let closeAttempt = 0 + const server = makeProtocolServer({ + closeResponse: async () => { + closeAttempt += 1 + return closeAttempt === 1 + ? new Response(null, { status: 503 }) + : new Response(null, { + status: 204, + headers: createHeaders('opaque::closed', true), + }) + }, + }) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-close-retry'), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + await durability.append([textChunk('x')]) + + await expect(durability.close()).rejects.toThrow(/failed to close/i) + await expect(durability.close()).resolves.toBeUndefined() + expect(server.closeCount()).toBe(2) + }) + + it.each([ + [ + 'closed control', + controlEvent({ streamNextOffset: 'opaque::closed', streamClosed: true }), + ], + ['parse failure', 'event: unexpected\ndata: {}\n\n'], + ])('cancels the replay body after %s', async (_name, body) => { + const cancel = vi.fn(() => undefined) + const response = new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(body)) + }, + cancel, + }), + { headers: { 'Content-Type': 'text/event-stream' } }, + ) + const fetchStub = vi.fn(async () => response) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-cancel-body'), + { server: 'https://ds.test', fetch: fetchStub }, + ) + + const consume = async () => { + for await (const _entry of durability.read('-1')) { + // drain + } + } + if (_name === 'parse failure') { + await expect(consume()).rejects.toThrow(/unexpected SSE event type/) + } else { + await expect(consume()).resolves.toBeUndefined() + } + expect(cancel).toHaveBeenCalledOnce() + }) + + it( + 'cancels a pending replay body when aborted', + { timeout: 500 }, + async () => { + const cancel = vi.fn(() => undefined) + const response = new Response( + new ReadableStream({ cancel }), + { headers: { 'Content-Type': 'text/event-stream' } }, + ) + const fetchStub = vi.fn(async () => response) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-abort-body'), + { server: 'https://ds.test', fetch: fetchStub }, + ) + const controller = new AbortController() + const iterator = durability + .read('-1', controller.signal) + [Symbol.asyncIterator]() + const next = iterator.next() + + await vi.waitFor(() => expect(fetchStub).toHaveBeenCalledOnce()) + controller.abort() + + await expect(next).resolves.toEqual({ done: true, value: undefined }) + expect(cancel).toHaveBeenCalledOnce() + }, + ) + + it('propagates read abort and ends the iterator cleanly', async () => { + const fetchStub = vi.fn( + async (_input, init) => + await new Promise((_resolve, reject) => { + const signal = init?.signal + if (!signal) throw new Error('Expected a read abort signal') + signal.addEventListener( + 'abort', + () => reject(new DOMException('Aborted', 'AbortError')), + { once: true }, + ) + }), + ) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-abort'), + { server: 'https://ds.test', fetch: fetchStub }, + ) + const controller = new AbortController() + const iterator = durability + .read('-1', controller.signal) + [Symbol.asyncIterator]() + const next = iterator.next() + + await vi.waitFor(() => expect(fetchStub).toHaveBeenCalledOnce()) + controller.abort() + await expect(next).resolves.toEqual({ done: true, value: undefined }) + }) +}) + +describe('durableStream exact-once resume', () => { + it('resumes mid-way through a coalesced data batch without gaps or duplicates', async () => { + const server = makeProtocolServer({ + createOffset: 'opaque::batch-start/A?token=%2F+==', + appendOffsets: ['opaque::batch-end/Z?token=%3D'], + }) + const full = ['a', 'b', 'c', 'd', 'e', 'f'] + const producer = durableStream( + new Request('https://app.test/api/chat?runId=run-exact-once'), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + const produced = parseTransportEvents( + await readBody( + toServerSentEventsResponse(textStream(full), { + durability: { adapter: producer }, + }), + ), + ) + + expect(server.batches).toHaveLength(1) + expect(produced.every((event) => event.id !== undefined)).toBe(true) + const beforeDrop = produced.slice(0, 2) + const resumeOffset = beforeDrop.at(-1)?.id + if (!resumeOffset) throw new Error('Expected a resume offset') + expect(decodeURIComponent(resumeOffset)).not.toContain('runId') + const exploding: AsyncIterable = { + [Symbol.asyncIterator]() { + return { + next() { + throw new Error('input stream must not be iterated on resume') + }, + } + }, + } + const reconnect = durableStream( + new Request('https://app.test/api/chat?runId=run-exact-once', { + headers: { 'Last-Event-ID': resumeOffset }, + }), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + const afterDrop = parseTransportEvents( + await readBody( + toServerSentEventsResponse(exploding, { + durability: { adapter: reconnect }, + }), + ), + ) + + expect( + [...beforeDrop, ...afterDrop].map((event) => deltaFrom(event.data)), + ).toEqual(full) + const replayRequest = server.requests.find( + (request) => request.method === 'GET', + ) + expect(replayRequest?.url.searchParams.get('offset')).toBe( + server.createOffset, + ) + expect(server.closeCount()).toBe(1) + }) +}) diff --git a/packages/ai-durable-stream/tsconfig.json b/packages/ai-durable-stream/tsconfig.json new file mode 100644 index 000000000..c38689f4e --- /dev/null +++ b/packages/ai-durable-stream/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": ["src", "tests"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/ai-durable-stream/vite.config.ts b/packages/ai-durable-stream/vite.config.ts new file mode 100644 index 000000000..11f5b20b7 --- /dev/null +++ b/packages/ai-durable-stream/vite.config.ts @@ -0,0 +1,37 @@ +import { defineConfig, mergeConfig } from 'vitest/config' +import { tanstackViteConfig } from '@tanstack/vite-config' +import packageJson from './package.json' + +const config = defineConfig({ + test: { + name: packageJson.name, + dir: './', + watch: false, + + globals: true, + environment: 'node', + include: ['tests/**/*.test.ts'], + coverage: { + provider: 'v8', + reporter: ['text', 'json', 'html', 'lcov'], + exclude: [ + 'node_modules/', + 'dist/', + 'tests/', + '**/*.test.ts', + '**/*.config.ts', + '**/types.ts', + ], + include: ['src/**/*.ts'], + }, + }, +}) + +export default mergeConfig( + config, + tanstackViteConfig({ + entry: ['./src/index.ts'], + srcDir: './src', + cjs: false, + }), +) diff --git a/packages/ai/skills/ai-core/chat-experience/SKILL.md b/packages/ai/skills/ai-core/chat-experience/SKILL.md index 3ae4408dc..89758b6c5 100644 --- a/packages/ai/skills/ai-core/chat-experience/SKILL.md +++ b/packages/ai/skills/ai-core/chat-experience/SKILL.md @@ -148,6 +148,16 @@ const stream = chat({ return toServerSentEventsResponse(stream, { abortController }) ``` +To make the SSE response resumable (reconnect after a drop/refresh without +re-running the provider), pass a delivery-durability adapter: +`toServerSentEventsResponse(stream, { durability: { adapter: memoryStream(request) } })` +(`memoryStream` from `@tanstack/ai` is process-local, for dev/tests) or +`durableStream(request, { server })` from `@tanstack/ai-durable-stream` +(Durable Streams protocol, production). Each SSE event gets an opaque +adapter-owned `id:`; `fetchServerSentEvents` auto-reconnects with +`Last-Event-ID` and exposes `joinRun(runId)` to replay a run from the start. +See `docs/resumable-streams/overview.md`. + **Client:** ```typescript diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 102987c02..f005588ba 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -79,6 +79,10 @@ export { toHttpResponse, } from './stream-to-response' +// Delivery durability (transport layer) +export { memoryStream } from './stream-durability' +export type { StreamDurability } from './stream-durability' + // Tool call management export { ToolCallManager } from './activities/chat/tools/tool-calls' diff --git a/packages/ai/src/stream-durability.ts b/packages/ai/src/stream-durability.ts new file mode 100644 index 000000000..542369fd7 --- /dev/null +++ b/packages/ai/src/stream-durability.ts @@ -0,0 +1,212 @@ +import type { StreamChunk } from './types' + +/** + * A pluggable delivery-durability backend. + * + * Offsets are owned by the adapter and opaque to the transport. The generic + * parameter lets an adapter retain a branded string type across append, read, + * and resume without requiring core to understand its cursor format. + */ +export interface StreamDurability { + /** Return the adapter offset captured from the request, or null for a producer. */ + resumeFrom: () => TOffset | null + /** + * Persist a batch before it is delivered and return exactly one resumable + * offset for each chunk, in the same order. + */ + append: (chunks: Array) => Promise> + /** Replay chunks strictly after the supplied adapter-owned offset. */ + read: ( + offset: TOffset, + signal?: AbortSignal, + ) => AsyncIterable<{ offset: TOffset; chunk: StreamChunk }> + /** + * Terminalize the producer log and unblock live readers. Core awaits this + * for every producer exit, including completion, cancellation, and failure. + */ + close: () => Promise +} + +const MEMORY_OFFSET_PREFIX = 'memory:v1:' + +interface MemoryOffset { + runId: string + seq: number +} + +function encodeMemoryOffset(runId: string, seq: number): string { + return `${MEMORY_OFFSET_PREFIX}${encodeURIComponent(runId)}:${seq}` +} + +function decodeMemoryOffset(offset: string): MemoryOffset { + if (!offset.startsWith(MEMORY_OFFSET_PREFIX)) { + throw new Error(`Invalid memory stream offset: ${offset}`) + } + const encoded = offset.slice(MEMORY_OFFSET_PREFIX.length) + const separator = encoded.lastIndexOf(':') + if (separator === -1) { + throw new Error(`Invalid memory stream offset: ${offset}`) + } + const runId = decodeURIComponent(encoded.slice(0, separator)) + const seq = Number(encoded.slice(separator + 1)) + if (!Number.isSafeInteger(seq) || seq < 1) { + throw new Error(`Invalid memory stream offset: ${offset}`) + } + return { runId, seq } +} + +function readResumeOffset(request: Request): string | null { + const header = request.headers.get('Last-Event-ID') + if (header) return header + try { + return new URL(request.url).searchParams.get('offset') + } catch { + return null + } +} + +function readRunId(request: Request): string | null { + try { + return new URL(request.url).searchParams.get('runId') + } catch { + return null + } +} + +function assertValidRunId(runId: string): string { + if (runId.length === 0 || /[\r\n]/.test(runId)) { + throw new Error( + `Invalid runId (must be non-empty and contain no CR/LF): ${JSON.stringify(runId)}`, + ) + } + return runId +} + +function resolveMemoryRunId( + request: Request, + resumeOffset: string | null, +): string { + if ( + resumeOffset !== null && + resumeOffset !== '-1' && + resumeOffset !== 'now' + ) { + return assertValidRunId(decodeMemoryOffset(resumeOffset).runId) + } + const requestedRunId = readRunId(request) + return requestedRunId === null + ? crypto.randomUUID() + : assertValidRunId(requestedRunId) +} + +function memoryThreshold(offset: string, runId: string, tail: number): number { + if (offset === '-1') return -1 + if (offset === 'now') return tail + const decoded = decodeMemoryOffset(offset) + if (decoded.runId !== runId) { + throw new Error( + `Memory stream offset belongs to run ${JSON.stringify(decoded.runId)}, not ${JSON.stringify(runId)}`, + ) + } + return decoded.seq +} + +function isTerminalChunk(chunk: StreamChunk): boolean { + return chunk.type === 'RUN_FINISHED' || chunk.type === 'RUN_ERROR' +} + +interface MemoryEntry { + seq: number + offset: string + chunk: StreamChunk +} + +interface MemoryLog { + entries: Array + complete: boolean + waiters: Array<() => void> +} + +const memoryLogs = new Map() + +function getOrCreateLog(id: string): MemoryLog { + let log = memoryLogs.get(id) + if (!log) { + log = { entries: [], complete: false, waiters: [] } + memoryLogs.set(id, log) + } + return log +} + +function wakeWaiters(log: MemoryLog): void { + const waiters = log.waiters + log.waiters = [] + for (const wake of waiters) wake() +} + +/** + * The zero-infrastructure delivery-durability backend. Its versioned cursor is + * deliberately private: callers and core only pass the returned string back. + */ +export function memoryStream(request: Request): StreamDurability { + const resumeOffset = readResumeOffset(request) + const runId = resolveMemoryRunId(request, resumeOffset) + + return { + resumeFrom: () => resumeOffset, + append: (chunks) => { + const log = getOrCreateLog(runId) + const firstSeq = (log.entries.at(-1)?.seq ?? 0) + 1 + const offsets = chunks.map((chunk, index) => { + const seq = firstSeq + index + const offset = encodeMemoryOffset(runId, seq) + log.entries.push({ seq, offset, chunk }) + if (isTerminalChunk(chunk)) log.complete = true + return offset + }) + wakeWaiters(log) + return Promise.resolve(offsets) + }, + close: () => { + const log = getOrCreateLog(runId) + log.complete = true + wakeWaiters(log) + return Promise.resolve() + }, + read: async function* (offset, signal) { + const log = getOrCreateLog(runId) + const threshold = memoryThreshold( + offset, + runId, + log.entries.at(-1)?.seq ?? 0, + ) + let index = 0 + + for (;;) { + while (index < log.entries.length) { + const entry = log.entries[index] + index += 1 + if (entry && entry.seq > threshold) { + yield { offset: entry.offset, chunk: entry.chunk } + if (isTerminalChunk(entry.chunk)) return + } + } + if (log.complete || signal?.aborted) return + + await new Promise((resolve) => { + const onAbort = () => { + const waiterIndex = log.waiters.indexOf(wake) + if (waiterIndex !== -1) log.waiters.splice(waiterIndex, 1) + resolve() + } + const wake = () => { + signal?.removeEventListener('abort', onAbort) + resolve() + } + log.waiters.push(wake) + signal?.addEventListener('abort', onAbort, { once: true }) + }) + } + }, + } +} diff --git a/packages/ai/src/stream-to-response.ts b/packages/ai/src/stream-to-response.ts index 9850f4d60..a5a97b734 100644 --- a/packages/ai/src/stream-to-response.ts +++ b/packages/ai/src/stream-to-response.ts @@ -1,4 +1,6 @@ import { toRunErrorPayload } from './activities/error-payload' +import { EventType } from './types' +import type { StreamDurability } from './stream-durability' import type { StreamChunk } from './types' /** @@ -14,7 +16,7 @@ import type { StreamChunk } from './types' * ```typescript * const stream = chat({ * adapter: openaiText(), - * model: 'gpt-4o', + * model: 'gpt-5.5', * messages: [{ role: 'user', content: 'Hello!' }] * }); * const text = await streamToText(stream); @@ -35,6 +37,151 @@ export async function streamToText( return accumulatedContent } +interface RecordedFailure { + error: unknown +} + +function errorMessage(error: unknown): string { + return toRunErrorPayload(error).message +} + +function combineFailures( + primary: unknown, + secondary: unknown, + phase: string, +): unknown { + if (primary === secondary) return primary + const errors = + primary instanceof AggregateError + ? [...primary.errors, secondary] + : [primary, secondary] + return new AggregateError( + errors, + `${errorMessage(primary)}; ${phase}: ${errorMessage(secondary)}`, + ) +} + +function runErrorChunk( + error: unknown, +): Extract { + const payload = toRunErrorPayload(error) + return { + type: EventType.RUN_ERROR, + timestamp: Date.now(), + message: payload.message, + ...(payload.code === undefined ? {} : { code: payload.code }), + error: payload, + } +} + +function isAborted(signal: AbortSignal): boolean { + return signal.aborted +} + +function needsTerminalPersistence( + terminalPersisted: boolean, + cancelled: boolean, + failed: boolean, +): boolean { + return !terminalPersisted && (cancelled || failed) +} + +function toEncodedStream( + stream: AsyncIterable, + abortController: AbortController | undefined, + encodeChunk: (chunk: StreamChunk, index: number) => Uint8Array, + encodeError: (error: unknown) => Uint8Array, +): ReadableStream { + const cancellation = abortController ?? new AbortController() + let iterator: AsyncIterator | undefined + let iteratorCleanup: Promise | undefined + let pumpPromise: Promise = Promise.resolve() + let pumpFailure: RecordedFailure | undefined + let cancelled = false + + const recordPumpFailure = (error: unknown, phase: string): void => { + pumpFailure = { + error: + pumpFailure === undefined + ? error + : combineFailures(pumpFailure.error, error, phase), + } + } + + const closeIterator = (): Promise => { + iteratorCleanup ??= (async () => { + if (iterator?.return) await iterator.return() + })() + return iteratorCleanup + } + + return new ReadableStream({ + start(controller) { + iterator = stream[Symbol.asyncIterator]() + pumpPromise = (async () => { + let index = 0 + let iteratorDone = false + + try { + while (!isAborted(cancellation.signal)) { + const result = await iterator.next() + if (result.done) { + iteratorDone = true + break + } + if (isAborted(cancellation.signal)) break + controller.enqueue(encodeChunk(result.value, index)) + index += 1 + } + } catch (error) { + recordPumpFailure(error, 'stream iteration failed') + } finally { + if (!iteratorDone) { + try { + await closeIterator() + } catch (error) { + recordPumpFailure(error, 'iterator cleanup failed') + } + } + + if ( + !cancelled && + !isAborted(cancellation.signal) && + pumpFailure !== undefined + ) { + controller.enqueue(encodeError(pumpFailure.error)) + } + if (!cancelled) controller.close() + } + })().catch((error: unknown) => { + recordPumpFailure(error, 'stream pump failed') + }) + }, + async cancel(reason) { + cancelled = true + if (!isAborted(cancellation.signal)) cancellation.abort(reason) + + let cancellationFailure: RecordedFailure | undefined + try { + await closeIterator() + } catch (error) { + cancellationFailure = { error } + } + await pumpPromise + + if (pumpFailure !== undefined && cancellationFailure !== undefined) { + throw combineFailures( + pumpFailure.error, + cancellationFailure.error, + 'iterator cancellation failed', + ) + } + if (pumpFailure !== undefined) throw pumpFailure.error + if (cancellationFailure !== undefined) throw cancellationFailure.error + }, + }) +} + /** * Convert a StreamChunk async iterable to a ReadableStream in Server-Sent Events format * @@ -50,53 +197,217 @@ export async function streamToText( export function toServerSentEventsStream( stream: AsyncIterable, abortController?: AbortController, + getId?: (chunk: StreamChunk, index: number) => string | undefined, ): ReadableStream { const encoder = new TextEncoder() + return toEncodedStream( + stream, + abortController, + (chunk, index) => { + const id = getId?.(chunk, index) + const idLine = id === undefined ? '' : `id: ${id}\n` + return encoder.encode(`${idLine}data: ${JSON.stringify(chunk)}\n\n`) + }, + (error) => + encoder.encode(`data: ${JSON.stringify(runErrorChunk(error))}\n\n`), + ) +} - return new ReadableStream({ - async start(controller) { - try { - for await (const chunk of stream) { - // Check if stream was cancelled/aborted - if (abortController?.signal.aborted) { - break - } +/** Default number of chunks buffered before a durability `append`. */ +const DEFAULT_DURABILITY_BATCH = 32 - // Send each chunk as Server-Sent Events format - controller.enqueue( - encoder.encode(`data: ${JSON.stringify(chunk)}\n\n`), - ) - } +/** + * Resolve and validate the durability batch size. A non-positive-integer (0, + * negative, fractional, or `NaN`) is rejected rather than clamped: silently + * `Math.max(1, …)`-ing a `NaN` used to disable size-based flushing entirely + * (`length >= NaN` is always false), which is a subtle footgun. + */ +function resolveBatchSize(batch: number | undefined): number { + if (batch === undefined) return DEFAULT_DURABILITY_BATCH + if (!Number.isInteger(batch) || batch <= 0) { + throw new Error( + `Invalid durability batch size: ${batch}. Must be a positive integer.`, + ) + } + return batch +} - controller.close() - } catch (error: unknown) { - // Don't send error if aborted - if (abortController?.signal.aborted) { - controller.close() - return - } +/** + * Boundaries at which the batching producer flushes early, regardless of the + * batch size — terminal events and tool-call ends. Flushing here keeps the + * durability log promptly consistent at semantically meaningful points. + */ +function isDurabilityFlushBoundary(chunk: StreamChunk): boolean { + return ( + chunk.type === 'RUN_FINISHED' || + chunk.type === 'RUN_ERROR' || + chunk.type === 'TOOL_CALL_END' + ) +} - // Send error event (AG-UI RUN_ERROR) - controller.enqueue( - encoder.encode( - `data: ${JSON.stringify({ - type: 'RUN_ERROR', - timestamp: Date.now(), - error: toRunErrorPayload(error), - })}\n\n`, - ), +/** + * Build the delivery-durable source iterable for a transport helper. + * + * - **Resume** (`resumeFrom()` non-null): replay strictly after the offset, + * reading only from the durability log. The input `stream` is NEVER iterated, + * so `chat()`'s lazy iterator never fires the provider — the untouched + * generator is simply GC'd. This is what makes resume free of re-invocation. + * - **Fresh** (`resumeFrom()` null): iterate `stream`, buffering up to `batch` + * chunks (flushing early at terminal / tool-call boundaries), `append` each + * batch to the log, then forward. Appending BEFORE forwarding guarantees a + * reconnecting client can always replay exactly what it already saw. + * + * The returned `getId` maps each forwarded chunk to the exact opaque offset + * returned by the durability adapter for the SSE `id:` line. + */ +function durableStreamSource( + stream: AsyncIterable, + durability: StreamDurability, + options: { abortController: AbortController; batch?: number }, +): { + source: AsyncIterable + getId: (chunk: StreamChunk) => string | undefined +} { + const resumeOffset = durability.resumeFrom() + const batchSize = resolveBatchSize(options.batch) + const abortController = options.abortController + const idByChunk = new WeakMap() + const seenOffsets = new Set() + const getId = (chunk: StreamChunk): string | undefined => idByChunk.get(chunk) + + const validateOffset = (offset: TOffset): void => { + if (offset.length === 0 || /[\0\r\n]/.test(offset)) { + throw new Error( + `Invalid durability offset for SSE id: ${JSON.stringify(offset)}`, + ) + } + if (seenOffsets.has(offset)) { + throw new Error( + `Durability adapter must return a unique offset per chunk: ${JSON.stringify(offset)}`, + ) + } + seenOffsets.add(offset) + } + + async function* produce(): AsyncIterable { + let batch: Array = [] + let terminalPersisted = false + let failure: RecordedFailure | undefined + let terminalCause: unknown + let hasTerminalCause = false + + const recordFailure = (error: unknown, phase: string): void => { + failure = { + error: + failure === undefined + ? error + : combineFailures(failure.error, error, phase), + } + } + + async function* flush(): AsyncIterable { + if (batch.length === 0) return + const toForward = batch + batch = [] + // Tag each chunk with the exact backend offset. Requiring one opaque + // token per chunk preserves exact-once resume at any batch size. + const offsets = await durability.append(toForward) + if (offsets.length !== toForward.length) { + throw new Error( + `Durability append returned ${offsets.length} offsets for ${toForward.length} chunks`, ) - controller.close() } - }, - cancel() { - // When the ReadableStream is cancelled (e.g., client disconnects), - // abort the underlying stream - if (abortController) { - abortController.abort() + toForward.forEach((chunk, i) => { + const offset = offsets[i] + if (offset === undefined) { + throw new Error(`Durability append omitted offset at index ${i}`) + } + validateOffset(offset) + idByChunk.set(chunk, offset) + }) + if ( + toForward.some( + (chunk) => + chunk.type === 'RUN_FINISHED' || chunk.type === 'RUN_ERROR', + ) + ) { + terminalPersisted = true } - }, - }) + for (const chunk of toForward) yield chunk + } + + try { + if (isAborted(abortController.signal)) return + for await (const chunk of stream) { + if (isAborted(abortController.signal)) break + batch.push(chunk) + if (batch.length >= batchSize || isDurabilityFlushBoundary(chunk)) { + yield* flush() + } + } + if (!isAborted(abortController.signal)) yield* flush() + } catch (error) { + terminalCause = error + hasTerminalCause = true + recordFailure(error, 'producer failed') + // The provider stream threw. Persist a terminal RUN_ERROR to the + // durability log so a resumer / joiner learns the run failed (otherwise + // the log ends with no terminal and they wait forever). Flush any + // buffered chunks first, then append the terminal WITHOUT forwarding it + // live — the transport layer synthesizes the live RUN_ERROR on rethrow, + // so forwarding here too would double-emit. + if (!isAborted(abortController.signal)) { + try { + yield* flush() + } catch (flushError) { + recordFailure(flushError, 'flushing buffered chunks failed') + } + } + } finally { + const cancelled = isAborted(abortController.signal) + if ( + needsTerminalPersistence(terminalPersisted, cancelled, hasTerminalCause) + ) { + const cause = cancelled ? { name: 'AbortError' } : terminalCause + try { + await durability.append([runErrorChunk(cause)]) + terminalPersisted = true + } catch (terminalError) { + recordFailure(terminalError, 'persisting terminal RUN_ERROR failed') + } + } + + try { + await durability.close() + } catch (closeError) { + recordFailure(closeError, 'closing durability stream failed') + } + + // Iterator cancellation must reject when awaited terminalization fails. + // eslint-disable-next-line no-unsafe-finally + if (failure !== undefined) throw failure.error + } + } + + async function* replay(offset: TOffset): AsyncIterable { + // Thread the consumer's abort signal into the read so a live-tailing join + // (a mid-stream reconnect) that is aborted — or that hit a runId with no + // in-process producer — stops parking and ends instead of hanging forever. + for await (const { offset: eventOffset, chunk } of durability.read( + offset, + abortController.signal, + )) { + if (isAborted(abortController.signal)) break + validateOffset(eventOffset) + idByChunk.set(chunk, eventOffset) + yield chunk + } + } + + return { + source: resumeOffset !== null ? replay(resumeOffset) : produce(), + getId, + } } /** @@ -107,21 +418,30 @@ export function toServerSentEventsStream( * - Each chunk is followed by "\n\n" * - Stream ends when the underlying iterable is exhausted (RUN_FINISHED is the terminal event) * + * Pass a `durability` sink (`memoryStream(request)` / `durableStream(request)`) + * to make the stream resumable: fresh runs are appended to the log and each SSE + * event is tagged with an `id:` offset; a reconnect (native `Last-Event-ID`) or + * a `?offset` join replays from the log without re-running the producer. `batch` + * controls how many chunks are buffered per `append` (default 32). + * * @param stream - AsyncIterable of StreamChunks from chat() - * @param init - Optional Response initialization options (including `abortController`) + * @param init - Optional Response initialization options (including `abortController`, `durability`, `batch`) * @returns Response in Server-Sent Events format * * @example * ```typescript - * const stream = chat({ adapter: openaiText(), model: "gpt-4o", messages: [...] }); - * return toServerSentEventsResponse(stream, { abortController }); + * const stream = chat({ adapter: openaiText(), model: "gpt-5.5", messages: [...] }); + * return toServerSentEventsResponse(stream, { durability: { adapter: memoryStream(request) } }); * ``` */ -export function toServerSentEventsResponse( +export function toServerSentEventsResponse( stream: AsyncIterable, - init?: ResponseInit & { abortController?: AbortController }, + init?: ResponseInit & { + abortController?: AbortController + durability?: { adapter: StreamDurability; batch?: number } + }, ): Response { - const { headers, abortController, ...responseInit } = init ?? {} + const { headers, abortController, durability, ...responseInit } = init ?? {} // Start with default SSE headers const mergedHeaders = new Headers({ @@ -139,7 +459,19 @@ export function toServerSentEventsResponse( }) } - return new Response(toServerSentEventsStream(stream, abortController), { + let body: ReadableStream + if (durability) { + const deliveryAbortController = abortController ?? new AbortController() + const { source, getId } = durableStreamSource(stream, durability.adapter, { + abortController: deliveryAbortController, + batch: durability.batch, + }) + body = toServerSentEventsStream(source, deliveryAbortController, getId) + } else { + body = toServerSentEventsStream(stream, abortController) + } + + return new Response(body, { ...responseInit, headers: mergedHeaders, }) @@ -160,7 +492,7 @@ export function toServerSentEventsResponse( * * @example * ```typescript - * const stream = chat({ adapter: openaiText(), model: "gpt-4o", messages: [...] }); + * const stream = chat({ adapter: openaiText(), model: "gpt-5.5", messages: [...] }); * const readableStream = toHttpStream(stream); * // Use with Response for HTTP streaming (not SSE) * return new Response(readableStream, { @@ -173,49 +505,12 @@ export function toHttpStream( abortController?: AbortController, ): ReadableStream { const encoder = new TextEncoder() - - return new ReadableStream({ - async start(controller) { - try { - for await (const chunk of stream) { - // Check if stream was cancelled/aborted - if (abortController?.signal.aborted) { - break - } - - // Send each chunk as newline-delimited JSON - controller.enqueue(encoder.encode(`${JSON.stringify(chunk)}\n`)) - } - - controller.close() - } catch (error: unknown) { - // Don't send error if aborted - if (abortController?.signal.aborted) { - controller.close() - return - } - - // Send error event (AG-UI RUN_ERROR) - controller.enqueue( - encoder.encode( - `${JSON.stringify({ - type: 'RUN_ERROR', - timestamp: Date.now(), - error: toRunErrorPayload(error), - })}\n`, - ), - ) - controller.close() - } - }, - cancel() { - // When the ReadableStream is cancelled (e.g., client disconnects), - // abort the underlying stream - if (abortController) { - abortController.abort() - } - }, - }) + return toEncodedStream( + stream, + abortController, + (chunk) => encoder.encode(`${JSON.stringify(chunk)}\n`), + (error) => encoder.encode(`${JSON.stringify(runErrorChunk(error))}\n`), + ) } /** @@ -233,15 +528,20 @@ export function toHttpStream( * * @example * ```typescript - * const stream = chat({ adapter: openaiText(), model: "gpt-4o", messages: [...] }); - * return toHttpResponse(stream, { abortController }); + * const stream = chat({ adapter: openaiText(), model: "gpt-5.5", messages: [...] }); + * return toHttpResponse(stream); * ``` */ export function toHttpResponse( stream: AsyncIterable, - init?: ResponseInit & { abortController?: AbortController }, + init?: ResponseInit & { + abortController?: AbortController + }, ): Response { - return new Response(toHttpStream(stream, init?.abortController), { - ...init, + const { abortController, ...responseInit } = init ?? {} + const body = toHttpStream(stream, abortController) + + return new Response(body, { + ...responseInit, }) } diff --git a/packages/ai/tests/stream-delivery-contract.test.ts b/packages/ai/tests/stream-delivery-contract.test.ts new file mode 100644 index 000000000..f4e9620d7 --- /dev/null +++ b/packages/ai/tests/stream-delivery-contract.test.ts @@ -0,0 +1,273 @@ +import { describe, expect, it, vi } from 'vitest' +import { toServerSentEventsResponse } from '../src/stream-to-response' +import { ev } from './test-utils' +import type { StreamDurability } from '../src/stream-durability' +import type { StreamChunk } from '../src/types' + +function deferred(): { + promise: Promise + resolve: () => void +} { + let resolve = (): void => undefined + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +async function readBody(response: Response): Promise { + if (!response.body) throw new Error('Expected a response body') + const reader = response.body.getReader() + const decoder = new TextDecoder() + let body = '' + for (;;) { + const result = await reader.read() + if (result.done) return body + body += decoder.decode(result.value) + } +} + +function parseEvents(body: string): Array<{ id?: string; chunk: StreamChunk }> { + return body + .split('\n\n') + .filter((block) => block.length > 0) + .map((block) => { + const lines = block.split('\n') + const id = lines.find((line) => line.startsWith('id: '))?.slice(4) + const data = lines.find((line) => line.startsWith('data: '))?.slice(6) + if (!data) throw new Error(`Missing SSE data line in ${block}`) + return { ...(id === undefined ? {} : { id }), chunk: JSON.parse(data) } + }) +} + +function oneChunkStream(): AsyncIterable { + return { + async *[Symbol.asyncIterator]() { + yield ev.textContent('hello') + }, + } +} + +describe('delivery durability contract', () => { + it('forwards an adapter-owned replay offset unchanged and never closes the producer', async () => { + const resumeOffset = 'backend:v3:resume/token?partition=west' + const replayOffset = 'backend:v3:event/token#17' + const close = vi.fn(async () => undefined) + const durability = { + resumeFrom: () => resumeOffset, + append: async () => [], + read: async function* (offset: string) { + expect(offset).toBe(resumeOffset) + yield { + offset: replayOffset, + chunk: ev.textContent('replayed'), + } + }, + close, + } satisfies StreamDurability + + const response = toServerSentEventsResponse(oneChunkStream(), { + durability: { adapter: durability }, + }) + const events = parseEvents(await readBody(response)) + + expect(events.map((event) => event.id)).toEqual([replayOffset]) + expect(close).not.toHaveBeenCalled() + }) + + it('awaits producer close before completing a normal response', async () => { + const closing = deferred() + const close = vi.fn(() => closing.promise) + const durability = { + resumeFrom: () => null, + append: async (chunks: Array) => + chunks.map((_, index) => `backend:normal:${index}`), + read: async function* () {}, + close, + } satisfies StreamDurability + const bodyPromise = readBody( + toServerSentEventsResponse(oneChunkStream(), { + durability: { adapter: durability, batch: 1 }, + }), + ) + let bodySettled = false + void bodyPromise.then(() => { + bodySettled = true + }) + + await vi.waitFor(() => expect(close).toHaveBeenCalledOnce()) + expect(bodySettled).toBe(false) + + closing.resolve() + await expect(bodyPromise).resolves.toContain('backend:normal:0') + }) + + it('persists an aborted RUN_ERROR and awaits close when the reader cancels', async () => { + const abortController = new AbortController() + const closing = deferred() + const sourceClosed = deferred() + const appended: Array = [] + const close = vi.fn(() => closing.promise) + const durability = { + resumeFrom: () => null, + append: async (chunks: Array) => { + appended.push(...chunks) + return chunks.map((_, index) => `backend:cancel:${index}`) + }, + read: async function* () {}, + close, + } satisfies StreamDurability + const source: AsyncIterable = { + async *[Symbol.asyncIterator]() { + try { + yield ev.textContent('before cancel') + if (!abortController.signal.aborted) { + await new Promise((resolve) => { + abortController.signal.addEventListener( + 'abort', + () => resolve(), + { + once: true, + }, + ) + }) + } + } finally { + sourceClosed.resolve() + } + }, + } + const response = toServerSentEventsResponse(source, { + abortController, + durability: { adapter: durability, batch: 1 }, + }) + if (!response.body) throw new Error('Expected a response body') + const reader = response.body.getReader() + + await reader.read() + const cancelPromise = reader.cancel() + let cancelSettled = false + void cancelPromise.then(() => { + cancelSettled = true + }) + + await vi.waitFor(() => expect(abortController.signal.aborted).toBe(true)) + await vi.waitFor(() => { + expect(appended.at(-1)?.type).toBe('RUN_ERROR') + }) + const terminal = appended.at(-1) + expect(terminal).toMatchObject({ + type: 'RUN_ERROR', + message: 'Request aborted', + code: 'aborted', + error: { message: 'Request aborted', code: 'aborted' }, + }) + await vi.waitFor(() => expect(close).toHaveBeenCalledOnce()) + expect(cancelSettled).toBe(false) + + closing.resolve() + await expect(cancelPromise).resolves.toBeUndefined() + await sourceClosed.promise + }) + + it('preserves a provider error when producer close also fails', async () => { + const providerError = new Error('provider exploded') + const closeError = new Error('close exploded') + const appended: Array = [] + const close = vi.fn(async () => { + throw closeError + }) + const durability = { + resumeFrom: () => null, + append: async (chunks: Array) => { + appended.push(...chunks) + return chunks.map((_, index) => `backend:error:${index}`) + }, + read: async function* () {}, + close, + } satisfies StreamDurability + const source: AsyncIterable = { + async *[Symbol.asyncIterator]() { + yield ev.textContent('before error') + throw providerError + }, + } + + const events = parseEvents( + await readBody( + toServerSentEventsResponse(source, { + durability: { adapter: durability, batch: 1 }, + }), + ), + ) + const persistedTerminal = appended.find( + (chunk) => chunk.type === 'RUN_ERROR', + ) + const liveTerminal = events.find( + (event) => event.chunk.type === 'RUN_ERROR', + )?.chunk + + expect(persistedTerminal).toMatchObject({ + type: 'RUN_ERROR', + message: 'provider exploded', + }) + expect(liveTerminal).toMatchObject({ + type: 'RUN_ERROR', + message: expect.stringContaining('provider exploded'), + }) + expect(liveTerminal).toMatchObject({ + error: { + message: expect.stringContaining('close exploded'), + }, + }) + }) + + it('aggregates provider, terminal persistence, and close failures', async () => { + const appended: Array = [] + const durability = { + resumeFrom: () => null, + append: async (chunks: Array) => { + appended.push(...chunks) + if (chunks.some((chunk) => chunk.type === 'RUN_ERROR')) { + throw new Error('terminal persistence exploded') + } + return chunks.map((_, index) => `backend:aggregate:${index}`) + }, + read: async function* () {}, + close: async () => { + throw new Error('aggregate close exploded') + }, + } satisfies StreamDurability + const source: AsyncIterable = { + async *[Symbol.asyncIterator]() { + yield ev.textContent('before aggregate') + throw new Error('aggregate provider exploded') + }, + } + + const events = parseEvents( + await readBody( + toServerSentEventsResponse(source, { + durability: { adapter: durability, batch: 1 }, + }), + ), + ) + const liveTerminal = events.find( + (event) => event.chunk.type === 'RUN_ERROR', + )?.chunk + + expect(appended.at(-1)?.type).toBe('RUN_ERROR') + expect(liveTerminal).toMatchObject({ + type: 'RUN_ERROR', + message: expect.stringContaining('aggregate provider exploded'), + error: { + message: expect.stringContaining('terminal persistence exploded'), + }, + }) + expect(liveTerminal).toMatchObject({ + error: { + message: expect.stringContaining('aggregate close exploded'), + }, + }) + }) +}) diff --git a/packages/ai/tests/stream-durability-types.test-d.ts b/packages/ai/tests/stream-durability-types.test-d.ts new file mode 100644 index 000000000..502a58e74 --- /dev/null +++ b/packages/ai/tests/stream-durability-types.test-d.ts @@ -0,0 +1,30 @@ +import { expectTypeOf } from 'vitest' +import { + toHttpResponse, + toServerSentEventsResponse, +} from '../src/stream-to-response' +import type { StreamDurability } from '../src/stream-durability' +import type { StreamChunk } from '../src/types' + +declare const backendOffsetBrand: unique symbol +type BackendOffset = string & { + readonly [backendOffsetBrand]: true +} + +declare const durability: StreamDurability +declare const stream: AsyncIterable + +expectTypeOf(durability.resumeFrom()).toEqualTypeOf() +expectTypeOf(durability.append).returns.toEqualTypeOf< + Promise> +>() + +toServerSentEventsResponse(stream, { + durability: { adapter: durability }, +}) + +// @ts-expect-error raw strings cannot be passed to a branded-offset adapter +durability.read('raw-offset') + +type HttpResponseOptions = NonNullable[1]> +expectTypeOf().not.toHaveProperty('durability') diff --git a/packages/ai/tests/stream-durability.test.ts b/packages/ai/tests/stream-durability.test.ts new file mode 100644 index 000000000..58ee65440 --- /dev/null +++ b/packages/ai/tests/stream-durability.test.ts @@ -0,0 +1,193 @@ +import { describe, expect, it } from 'vitest' +import { memoryStream } from '../src/stream-durability' +import { EventType } from '../src/types' +import { ev } from './test-utils' +import type { StreamChunk } from '../src/types' + +function label(chunk: StreamChunk): string { + return chunk.type === EventType.TEXT_MESSAGE_CONTENT + ? chunk.delta + : `[${chunk.type}]` +} + +async function readLabels( + stream: AsyncIterable<{ offset: string; chunk: StreamChunk }>, +): Promise> { + const labels: Array = [] + for await (const { chunk } of stream) labels.push(label(chunk)) + return labels +} + +describe('memoryStream', () => { + it('returns opaque per-chunk offsets and replays them unchanged', async () => { + const durability = memoryStream( + new Request('https://example.test/api/chat', { method: 'POST' }), + ) + + expect(durability.resumeFrom()).toBeNull() + const offsets = await durability.append([ + ev.textContent('a'), + ev.textContent('b'), + ev.textContent('c'), + ]) + expect(offsets).toHaveLength(3) + expect(new Set(offsets).size).toBe(3) + await durability.close() + + const replayedOffsets: Array = [] + const replayedLabels: Array = [] + for await (const entry of durability.read('-1')) { + replayedOffsets.push(entry.offset) + replayedLabels.push(label(entry.chunk)) + } + expect(replayedOffsets).toEqual(offsets) + expect(replayedLabels).toEqual(['a', 'b', 'c']) + }) + + it('resumes strictly after an adapter-owned Last-Event-ID', async () => { + const producer = memoryStream( + new Request('https://example.test/api/chat?runId=run-resume', { + method: 'POST', + }), + ) + const offsets = await producer.append([ + ev.textContent('a'), + ev.textContent('b'), + ev.textContent('c'), + ]) + await producer.close() + + const reconnect = memoryStream( + new Request('https://example.test/api/chat', { + method: 'POST', + headers: { 'Last-Event-ID': offsets[1] ?? '' }, + }), + ) + expect(reconnect.resumeFrom()).toBe(offsets[1]) + + const entries = [] + const resumeOffset = reconnect.resumeFrom() + if (resumeOffset === null) throw new Error('Expected a resume offset') + for await (const entry of reconnect.read(resumeOffset)) entries.push(entry) + expect(entries.map((entry) => entry.offset)).toEqual([offsets[2]]) + expect(entries.map((entry) => label(entry.chunk))).toEqual(['c']) + }) + + it('reads an opaque offset from the query string', async () => { + const producer = memoryStream( + new Request('https://example.test/api/chat?runId=run-query', { + method: 'POST', + }), + ) + const offsets = await producer.append([ + ev.textContent('x'), + ev.textContent('y'), + ]) + await producer.close() + + const joiner = memoryStream( + new Request( + `https://example.test/api/chat?offset=${encodeURIComponent(offsets[0] ?? '')}`, + { method: 'POST' }, + ), + ) + const resumeOffset = joiner.resumeFrom() + if (resumeOffset === null) throw new Error('Expected a resume offset') + expect(await readLabels(joiner.read(resumeOffset))).toEqual(['y']) + }) + + it('live-tails a from-start join through the producer terminal', async () => { + const producer = memoryStream( + new Request('https://example.test/api/chat?runId=run-live', { + method: 'POST', + }), + ) + await producer.append([ev.textContent('a'), ev.textContent('b')]) + + const joiner = memoryStream( + new Request('https://example.test/api/chat?runId=run-live&offset=-1', { + method: 'POST', + }), + ) + const resumeOffset = joiner.resumeFrom() + if (resumeOffset === null) throw new Error('Expected a resume offset') + const received: Array = [] + const done = (async () => { + for await (const { chunk } of joiner.read(resumeOffset)) { + received.push(label(chunk)) + } + })() + + await new Promise((resolve) => setTimeout(resolve, 10)) + expect(received).toEqual(['a', 'b']) + + await producer.append([ev.textContent('c'), ev.textContent('d')]) + await producer.append([ev.runFinished()]) + await done + expect(received).toEqual(['a', 'b', 'c', 'd', '[RUN_FINISHED]']) + }) + + it('supports an adapter-owned tail sentinel for future writes', async () => { + const producer = memoryStream( + new Request('https://example.test/api/chat?runId=run-tail', { + method: 'POST', + }), + ) + await producer.append([ev.textContent('old')]) + const joiner = memoryStream( + new Request('https://example.test/api/chat?runId=run-tail&offset=now', { + method: 'POST', + }), + ) + const resumeOffset = joiner.resumeFrom() + if (resumeOffset === null) throw new Error('Expected a resume offset') + const received: Array = [] + const done = (async () => { + for await (const { chunk } of joiner.read(resumeOffset)) { + received.push(label(chunk)) + } + })() + + await new Promise((resolve) => setTimeout(resolve, 10)) + await producer.append([ev.textContent('new'), ev.runFinished()]) + await done + expect(received).toEqual(['new', '[RUN_FINISHED]']) + }) + + it('ends a parked reader when its signal aborts', async () => { + const controller = new AbortController() + const joiner = memoryStream( + new Request( + 'https://example.test/api/chat?runId=never-produced&offset=-1', + { method: 'POST' }, + ), + ) + const resumeOffset = joiner.resumeFrom() + if (resumeOffset === null) throw new Error('Expected a resume offset') + const iterated = readLabels(joiner.read(resumeOffset, controller.signal)) + + await new Promise((resolve) => setTimeout(resolve, 10)) + controller.abort() + await expect(iterated).resolves.toEqual([]) + }) + + it('rejects invalid run ids and offsets loudly', () => { + expect(() => + memoryStream( + new Request( + `https://example.test/api/chat?runId=${encodeURIComponent('evil\ninjected')}`, + { method: 'POST' }, + ), + ), + ).toThrow(/Invalid runId/) + + expect(() => + memoryStream( + new Request('https://example.test/api/chat', { + method: 'POST', + headers: { 'Last-Event-ID': 'another-backend:cursor' }, + }), + ), + ).toThrow(/Invalid memory stream offset/) + }) +}) diff --git a/packages/ai/tests/stream-to-response-durability.test.ts b/packages/ai/tests/stream-to-response-durability.test.ts new file mode 100644 index 000000000..ecf51cacd --- /dev/null +++ b/packages/ai/tests/stream-to-response-durability.test.ts @@ -0,0 +1,276 @@ +import { describe, expect, it, vi } from 'vitest' +import { memoryStream } from '../src/stream-durability' +import { toServerSentEventsResponse } from '../src/stream-to-response' +import { EventType } from '../src/types' +import { ev } from './test-utils' +import type { StreamDurability } from '../src/stream-durability' +import type { StreamChunk } from '../src/types' + +function fiveChunkStream(): { + stream: AsyncIterable + iterated: () => boolean +} { + let started = false + const stream: AsyncIterable = { + async *[Symbol.asyncIterator]() { + started = true + for (const delta of ['1', '2', '3', '4', '5']) { + yield ev.textContent(delta) + } + }, + } + return { stream, iterated: () => started } +} + +async function readBody(response: Response): Promise { + if (!response.body) throw new Error('Expected a response body') + const reader = response.body.getReader() + const decoder = new TextDecoder() + let body = '' + for (;;) { + const result = await reader.read() + if (result.done) return body + body += decoder.decode(result.value) + } +} + +interface ParsedSseEvent { + id?: string + data: unknown +} + +function parseSseEvents(body: string): Array { + return body + .split('\n\n') + .filter((block) => block.trim().length > 0) + .map((block) => { + const lines = block.split('\n') + const id = lines.find((line) => line.startsWith('id: '))?.slice(4) + const data = lines.find((line) => line.startsWith('data: '))?.slice(6) + if (!data) throw new Error(`Missing SSE data line in ${block}`) + return { ...(id === undefined ? {} : { id }), data: JSON.parse(data) } + }) +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null +} + +function field(event: ParsedSseEvent, name: string): unknown { + if (!isRecord(event.data)) throw new Error('Expected an object SSE payload') + return event.data[name] +} + +function label(chunk: StreamChunk): string { + return chunk.type === EventType.TEXT_MESSAGE_CONTENT + ? chunk.delta + : `[${chunk.type}]` +} + +function fixedOffsetDurability( + offsets: Array, +): StreamDurability { + return { + resumeFrom: () => null, + append: async () => offsets, + close: async () => undefined, + async *read() { + // No replay is needed by these validation tests. + }, + } +} + +describe('toServerSentEventsResponse with durability', () => { + it('appends a fresh run and tags every event with its adapter offset', async () => { + const durability = memoryStream( + new Request('https://example.test/api/chat?runId=response-fresh', { + method: 'POST', + }), + ) + const { stream, iterated } = fiveChunkStream() + + const events = parseSseEvents( + await readBody( + toServerSentEventsResponse(stream, { + durability: { adapter: durability }, + }), + ), + ) + const eventOffsets = events.map((event) => event.id) + + expect(iterated()).toBe(true) + expect(events).toHaveLength(5) + expect(eventOffsets.every((offset) => offset !== undefined)).toBe(true) + expect(new Set(eventOffsets).size).toBe(5) + + const loggedOffsets: Array = [] + const loggedLabels: Array = [] + for await (const entry of durability.read('-1')) { + loggedOffsets.push(entry.offset) + loggedLabels.push(label(entry.chunk)) + } + expect(loggedOffsets).toEqual(eventOffsets) + expect(loggedLabels).toEqual(['1', '2', '3', '4', '5']) + }) + + it('replays opaque IDs from the log without iterating the input stream', async () => { + const { stream } = fiveChunkStream() + const produced = parseSseEvents( + await readBody( + toServerSentEventsResponse(stream, { + durability: { + adapter: memoryStream( + new Request( + 'https://example.test/api/chat?runId=response-replay', + { + method: 'POST', + }, + ), + ), + }, + }), + ), + ) + const resumeOffset = produced[1]?.id + if (!resumeOffset) throw new Error('Expected a replay offset') + const exploding: AsyncIterable = { + [Symbol.asyncIterator]() { + return { + next() { + throw new Error('input stream must not be iterated on resume') + }, + } + }, + } + + const replayed = parseSseEvents( + await readBody( + toServerSentEventsResponse(exploding, { + durability: { + adapter: memoryStream( + new Request('https://example.test/api/chat', { + method: 'POST', + headers: { 'Last-Event-ID': resumeOffset }, + }), + ), + }, + }), + ), + ) + + expect(replayed.map((event) => event.id)).toEqual( + produced.slice(2).map((event) => event.id), + ) + expect(replayed.map((event) => field(event, 'delta'))).toEqual([ + '3', + '4', + '5', + ]) + }) + + it('batches appends to at most the configured batch size', async () => { + const durability = memoryStream( + new Request('https://example.test/api/chat?runId=response-batch', { + method: 'POST', + }), + ) + const appendSpy = vi.spyOn(durability, 'append') + const { stream } = fiveChunkStream() + + await readBody( + toServerSentEventsResponse(stream, { + durability: { adapter: durability, batch: 2 }, + }), + ) + + const batchSizes = appendSpy.mock.calls.map(([chunks]) => chunks.length) + expect(batchSizes.every((size) => size <= 2)).toBe(true) + expect(batchSizes.reduce((sum, size) => sum + size, 0)).toBe(5) + }) + + it('rejects a non-positive-integer batch size', () => { + const durability = memoryStream( + new Request('https://example.test/api/chat?runId=response-bad-batch', { + method: 'POST', + }), + ) + const { stream } = fiveChunkStream() + for (const batch of [0, -1, 1.5, NaN]) { + expect(() => + toServerSentEventsResponse(stream, { + durability: { adapter: durability, batch }, + }), + ).toThrow(/Invalid durability batch size/) + } + }) + + it('rejects duplicate offsets before emitting distinct chunks', async () => { + const { stream } = fiveChunkStream() + const response = toServerSentEventsResponse(stream, { + durability: { + adapter: fixedOffsetDurability(Array.from({ length: 5 }, () => 'same')), + }, + }) + + const events = parseSseEvents(await readBody(response)) + expect(events).toHaveLength(1) + expect(events[0]?.id).toBeUndefined() + expect(field(events[0]!, 'type')).toBe(EventType.RUN_ERROR) + expect(field(events[0]!, 'message')).toMatch(/unique.*offset/i) + }) + + it('rejects SSE offsets containing U+0000', async () => { + const response = toServerSentEventsResponse(textStreamWithOneChunk(), { + durability: { adapter: fixedOffsetDurability(['bad\0offset']) }, + }) + + const events = parseSseEvents(await readBody(response)) + expect(events).toHaveLength(1) + expect(events[0]?.id).toBeUndefined() + expect(field(events[0]!, 'type')).toBe(EventType.RUN_ERROR) + expect(field(events[0]!, 'message')).toMatch(/Invalid durability offset/) + }) + + it('persists a terminal RUN_ERROR before closing when the source throws', async () => { + const durability = memoryStream( + new Request('https://example.test/api/chat?runId=response-error', { + method: 'POST', + }), + ) + const throwing: AsyncIterable = { + async *[Symbol.asyncIterator]() { + yield ev.textContent('1') + throw new Error('provider exploded') + }, + } + + const liveEvents = parseSseEvents( + await readBody( + toServerSentEventsResponse(throwing, { + durability: { adapter: durability }, + }), + ), + ) + expect(liveEvents.map((event) => field(event, 'type'))).toContain( + 'RUN_ERROR', + ) + + const logged: Array = [] + for await (const { chunk } of durability.read('-1')) logged.push(chunk) + expect(logged.map((chunk) => chunk.type)).toEqual([ + 'TEXT_MESSAGE_CONTENT', + 'RUN_ERROR', + ]) + expect(logged.at(-1)).toMatchObject({ + message: 'provider exploded', + }) + }) +}) + +function textStreamWithOneChunk(): AsyncIterable { + return { + async *[Symbol.asyncIterator]() { + yield ev.textContent('one') + }, + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5d92af110..adbeee227 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1658,6 +1658,15 @@ importers: specifier: ^2.11.10 version: 2.11.10(solid-js@1.9.10)(vite@7.3.3(@types/node@24.10.3)(jiti@2.6.1)(less@4.6.6)(lightningcss@1.30.2)(sass@1.101.0)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)) + packages/ai-durable-stream: + devDependencies: + '@tanstack/ai': + specifier: workspace:* + version: link:../ai + '@vitest/coverage-v8': + specifier: 4.0.14 + version: 4.0.14(vitest@4.1.4) + packages/ai-elevenlabs: dependencies: '@elevenlabs/client': diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index 14ffbf330..8abc6f917 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -45,6 +45,7 @@ import { Route as ApiMcpAppsChatRouteImport } from './routes/api.mcp-apps-chat' import { Route as ApiMcpAppsCallRouteImport } from './routes/api.mcp-apps-call' import { Route as ApiLazyToolsWireRouteImport } from './routes/api.lazy-tools-wire' import { Route as ApiImageRouteImport } from './routes/api.image' +import { Route as ApiDurableDeliveryRouteImport } from './routes/api.durable-delivery' import { Route as ApiChatRouteImport } from './routes/api.chat' import { Route as ApiAudioRouteImport } from './routes/api.audio' import { Route as ApiArktypeToolWireRouteImport } from './routes/api.arktype-tool-wire' @@ -242,6 +243,11 @@ const ApiImageRoute = ApiImageRouteImport.update({ path: '/api/image', getParentRoute: () => rootRouteImport, } as any) +const ApiDurableDeliveryRoute = ApiDurableDeliveryRouteImport.update({ + id: '/api/durable-delivery', + path: '/api/durable-delivery', + getParentRoute: () => rootRouteImport, +} as any) const ApiChatRoute = ApiChatRouteImport.update({ id: '/api/chat', path: '/api/chat', @@ -323,6 +329,7 @@ export interface FileRoutesByFullPath { '/api/arktype-tool-wire': typeof ApiArktypeToolWireRoute '/api/audio': typeof ApiAudioRouteWithChildren '/api/chat': typeof ApiChatRoute + '/api/durable-delivery': typeof ApiDurableDeliveryRoute '/api/image': typeof ApiImageRouteWithChildren '/api/lazy-tools-wire': typeof ApiLazyToolsWireRoute '/api/mcp-apps-call': typeof ApiMcpAppsCallRoute @@ -373,6 +380,7 @@ export interface FileRoutesByTo { '/api/arktype-tool-wire': typeof ApiArktypeToolWireRoute '/api/audio': typeof ApiAudioRouteWithChildren '/api/chat': typeof ApiChatRoute + '/api/durable-delivery': typeof ApiDurableDeliveryRoute '/api/image': typeof ApiImageRouteWithChildren '/api/lazy-tools-wire': typeof ApiLazyToolsWireRoute '/api/mcp-apps-call': typeof ApiMcpAppsCallRoute @@ -424,6 +432,7 @@ export interface FileRoutesById { '/api/arktype-tool-wire': typeof ApiArktypeToolWireRoute '/api/audio': typeof ApiAudioRouteWithChildren '/api/chat': typeof ApiChatRoute + '/api/durable-delivery': typeof ApiDurableDeliveryRoute '/api/image': typeof ApiImageRouteWithChildren '/api/lazy-tools-wire': typeof ApiLazyToolsWireRoute '/api/mcp-apps-call': typeof ApiMcpAppsCallRoute @@ -476,6 +485,7 @@ export interface FileRouteTypes { | '/api/arktype-tool-wire' | '/api/audio' | '/api/chat' + | '/api/durable-delivery' | '/api/image' | '/api/lazy-tools-wire' | '/api/mcp-apps-call' @@ -526,6 +536,7 @@ export interface FileRouteTypes { | '/api/arktype-tool-wire' | '/api/audio' | '/api/chat' + | '/api/durable-delivery' | '/api/image' | '/api/lazy-tools-wire' | '/api/mcp-apps-call' @@ -576,6 +587,7 @@ export interface FileRouteTypes { | '/api/arktype-tool-wire' | '/api/audio' | '/api/chat' + | '/api/durable-delivery' | '/api/image' | '/api/lazy-tools-wire' | '/api/mcp-apps-call' @@ -627,6 +639,7 @@ export interface RootRouteChildren { ApiArktypeToolWireRoute: typeof ApiArktypeToolWireRoute ApiAudioRoute: typeof ApiAudioRouteWithChildren ApiChatRoute: typeof ApiChatRoute + ApiDurableDeliveryRoute: typeof ApiDurableDeliveryRoute ApiImageRoute: typeof ApiImageRouteWithChildren ApiLazyToolsWireRoute: typeof ApiLazyToolsWireRoute ApiMcpAppsCallRoute: typeof ApiMcpAppsCallRoute @@ -908,6 +921,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiImageRouteImport parentRoute: typeof rootRouteImport } + '/api/durable-delivery': { + id: '/api/durable-delivery' + path: '/api/durable-delivery' + fullPath: '/api/durable-delivery' + preLoaderRoute: typeof ApiDurableDeliveryRouteImport + parentRoute: typeof rootRouteImport + } '/api/chat': { id: '/api/chat' path: '/api/chat' @@ -1072,6 +1092,7 @@ const rootRouteChildren: RootRouteChildren = { ApiArktypeToolWireRoute: ApiArktypeToolWireRoute, ApiAudioRoute: ApiAudioRouteWithChildren, ApiChatRoute: ApiChatRoute, + ApiDurableDeliveryRoute: ApiDurableDeliveryRoute, ApiImageRoute: ApiImageRouteWithChildren, ApiLazyToolsWireRoute: ApiLazyToolsWireRoute, ApiMcpAppsCallRoute: ApiMcpAppsCallRoute, diff --git a/testing/e2e/src/routes/api.durable-delivery.ts b/testing/e2e/src/routes/api.durable-delivery.ts new file mode 100644 index 000000000..26a600dac --- /dev/null +++ b/testing/e2e/src/routes/api.durable-delivery.ts @@ -0,0 +1,84 @@ +import { createFileRoute } from '@tanstack/react-router' +import { memoryStream, toServerSentEventsResponse } from '@tanstack/ai' +import type { StreamChunk } from '@tanstack/ai' + +/** + * A provider-free delivery-durability harness route. It streams a FIXED + * sequence of AG-UI events through the transport helper's `durability` sink + * (`memoryStream`), so the delivery e2e can assert disconnect→reconnect→ordered + * resume and second-tab join deterministically, with no LLM in the loop. + * + * - `POST` with no offset → fresh run: produce + append the fixed sequence, + * tagging each SSE event with an opaque adapter-owned offset. + * - `POST` with `Last-Event-ID` → reconnect: replay strictly after the offset + * from the log (the fixed sequence is never re-produced). + * - `GET ?offset=-1&runId=…` → second-tab join: replay from the start. + */ +function fixedRun(threadId: string, runId: string): AsyncIterable { + return (async function* () { + yield { + type: 'RUN_STARTED', + threadId, + runId, + timestamp: Date.now(), + } as StreamChunk + for (let i = 1; i <= 5; i++) { + yield { + type: 'TEXT_MESSAGE_CONTENT', + messageId: 'm', + model: 'fixed', + delta: String(i), + content: String(i), + timestamp: Date.now(), + } as StreamChunk + } + yield { + type: 'RUN_FINISHED', + threadId, + runId, + model: 'fixed', + finishReason: 'stop', + timestamp: Date.now(), + } as StreamChunk + })() +} + +function durableRun(request: Request) { + const url = new URL(request.url) + const runId = url.searchParams.get('runId') ?? crypto.randomUUID() + url.searchParams.set('runId', runId) + return { + durability: memoryStream(new Request(url, request)), + runId, + } +} + +function withRunId(response: Response, runId: string): Response { + response.headers.set('X-Run-Id', runId) + return response +} + +export const Route = createFileRoute('/api/durable-delivery')({ + server: { + handlers: { + POST: async ({ request }) => { + const { durability, runId } = durableRun(request) + return withRunId( + toServerSentEventsResponse(fixedRun('thread-durable', runId), { + durability: { adapter: durability, batch: 2 }, + }), + runId, + ) + }, + GET: async ({ request }) => { + const { durability, runId } = durableRun(request) + return withRunId( + toServerSentEventsResponse(fixedRun('thread-durable', runId), { + durability: { adapter: durability }, + }), + runId, + ) + }, + }, + }, +}) diff --git a/testing/e2e/tests/delivery-durability.spec.ts b/testing/e2e/tests/delivery-durability.spec.ts new file mode 100644 index 000000000..db10f9650 --- /dev/null +++ b/testing/e2e/tests/delivery-durability.spec.ts @@ -0,0 +1,104 @@ +import { expect, test } from '@playwright/test' + +/** + * Delivery durability (transport layer) over real HTTP. + * + * These exercise the server-side `durability` sink end-to-end: a fresh run is + * appended to the log and each SSE event is tagged with an opaque adapter-owned + * offset; a reconnect (native `Last-Event-ID`) or a `?offset` join replays from + * the log without re-producing. + * + * We test only the **producer-alive** path deterministically (the log is fully + * written by the first request). The **producer-dead** case — where the client + * disconnects mid-run and the producer process is torn down with it — cannot be + * resumed by the log alone; it needs the producer to outlive the socket + * (`waitUntil` / durable object / queue), which is a per-platform deployment + * concern, not something this transport can guarantee. The client-side + * auto-reconnect (Last-Event-ID resend + de-dupe) is covered by unit tests in + * `@tanstack/ai-client`. + */ + +interface SseEvent { + id?: string + data: unknown +} + +function parseSse(body: string): Array { + return body + .split('\n\n') + .filter((block) => block.trim().length > 0) + .map((block) => { + const lines = block.split('\n') + const idLine = lines.find((l) => l.startsWith('id:')) + const dataLine = lines.find((l) => l.startsWith('data:')) + const rawData = dataLine + ? dataLine.slice(dataLine.indexOf(':') + 1).trim() + : '' + return { + ...(idLine ? { id: idLine.slice(idLine.indexOf(':') + 1).trim() } : {}), + data: JSON.parse(rawData), + } + }) +} + +function eventType(e: SseEvent): string { + return (e.data as { type: string }).type +} + +function contentDeltas(events: Array): Array { + return events + .filter((e) => eventType(e) === 'TEXT_MESSAGE_CONTENT') + .map((e) => (e.data as { delta: string }).delta) +} + +test.describe('delivery durability', () => { + test('disconnect → reconnect resumes the ordered stream exactly once', async ({ + request, + }) => { + // Fresh run: produce + persist the full ordered sequence. + const produce = await request.post('/api/durable-delivery', { data: {} }) + expect(produce.ok()).toBeTruthy() + const produced = parseSse(await produce.text()) + + // Sequence: RUN_STARTED(1), content 1..5 (seq 2..6), RUN_FINISHED(7). + expect(contentDeltas(produced)).toEqual(['1', '2', '3', '4', '5']) + expect(produced.every((e) => e.id !== undefined)).toBeTruthy() + + // Client received through seq 3 (RUN_STARTED, content "1", content "2") + // then dropped. Reconnect from that offset via native Last-Event-ID. + const resumeFrom = produced[2]!.id! + const reconnect = await request.post('/api/durable-delivery', { + headers: { 'Last-Event-ID': resumeFrom }, + data: {}, + }) + expect(reconnect.ok()).toBeTruthy() + const resumed = parseSse(await reconnect.text()) + + // Exactly the tail, in order, exactly once — content "3","4","5" then RUN_FINISHED. + expect(contentDeltas(resumed)).toEqual(['3', '4', '5']) + expect(eventType(resumed[resumed.length - 1]!)).toBe('RUN_FINISHED') + // Adapter offsets are opaque; no replayed event may reuse the acknowledged + // resume offset. + expect(resumed.map((event) => event.id)).not.toContain(resumeFrom) + }) + + test('a second tab joins an existing run from the start', async ({ + request, + }) => { + const produce = await request.post('/api/durable-delivery', { data: {} }) + const produced = parseSse(await produce.text()) + const runId = produce.headers()['x-run-id'] + if (!runId) throw new Error('Missing X-Run-Id response header') + + // Join from the start with ?offset=-1 (a fresh tab attaching to the run). + const join = await request.get( + `/api/durable-delivery?offset=-1&runId=${encodeURIComponent(runId)}`, + ) + expect(join.ok()).toBeTruthy() + const joined = parseSse(await join.text()) + + expect(contentDeltas(joined)).toEqual(['1', '2', '3', '4', '5']) + expect(eventType(joined[0]!)).toBe('RUN_STARTED') + expect(eventType(joined[joined.length - 1]!)).toBe('RUN_FINISHED') + }) +}) From 21483a2187008f3af573b4b96405902b2b319dbf Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Fri, 17 Jul 2026 17:06:02 +1000 Subject: [PATCH 02/26] docs(resumable-streams): Cloudflare Durable Streams backend via service binding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document running against the Durable Streams Cloudflare Workers + DO backend: same protocol, no new adapter — inject the service binding's fetch via the adapter's injectable fetch option, or point server at the deployed Worker URL. Note the DO alarm satisfies the lease/reaper needed for producer-death terminalization. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01A6Arc9bWdLq1aRRnDRCLMt --- docs/config.json | 3 +- docs/resumable-streams/overview.md | 56 ++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/docs/config.json b/docs/config.json index a003be8f3..78b816d65 100644 --- a/docs/config.json +++ b/docs/config.json @@ -187,7 +187,8 @@ { "label": "Overview", "to": "resumable-streams/overview", - "addedAt": "2026-07-17" + "addedAt": "2026-07-17", + "updatedAt": "2026-07-17" } ] }, diff --git a/docs/resumable-streams/overview.md b/docs/resumable-streams/overview.md index 026772df8..50f4d158c 100644 --- a/docs/resumable-streams/overview.md +++ b/docs/resumable-streams/overview.md @@ -100,13 +100,65 @@ request. This option belongs to the external URL adapter; a future direct Cloudflare binding would use binding authorization instead. The external service must return a non-empty `Stream-Next-Offset` header for -create and append operations. Missing headers fail loudly. The adapter never -guesses an offset. +create, append, and close operations. Missing headers fail loudly. The adapter +never guesses an offset. For development and tests, swap `durableStream(request, durableOptions)` for `memoryStream(request)` from `@tanstack/ai` — no external service required, at the cost of the log living only in that process's memory. +### Cloudflare Durable Streams + +[Durable Streams](https://durablestreams.com) ships a Cloudflare Workers + +Durable Objects backend that speaks this same protocol, so `durableStream` +talks to it directly — no new adapter. + +When your TanStack AI endpoint also runs on Workers, reach the backend over a +[service binding](https://developers.cloudflare.com/workers/runtime-apis/bindings/service-bindings/) +rather than a public URL. The adapter's injectable `fetch` routes every create, +append, read, and close through the binding, so traffic never leaves +Cloudflare's network and the binding — not a bearer token — authorizes the call: + +```ts +import { durableStream } from '@tanstack/ai-durable-stream' + +interface Env { + // A service binding to the deployed Durable Streams Worker. + DURABLE_STREAMS: { fetch: typeof fetch } +} + +function cloudflareAdapter(request: Request, env: Env) { + return durableStream(request, { + streamPrefix: 'chat-runs', + // Any absolute URL parses; a service binding dispatches to the bound + // Worker regardless of host, and only the `/streams/...` path is used. + server: 'https://durable-streams', + fetch: env.DURABLE_STREAMS.fetch.bind(env.DURABLE_STREAMS), + }) +} +``` + +If the backend runs elsewhere, or the caller is off-platform, point `server` at +the deployed Worker's public URL instead: + +```ts +import { durableStream } from '@tanstack/ai-durable-stream' + +const cloudflareOptions = { + server: 'https://durable-streams.example.workers.dev', + streamPrefix: 'chat-runs', +} + +function urlAdapter(request: Request) { + return durableStream(request, cloudflareOptions) +} +``` + +Running on a Durable Object also satisfies the lease/reaper described under +[Process death](#process-death): a DO alarm can terminalize a run whose producer +died, so a reconnecting client observes a terminal state instead of waiting +forever. + ## Client setup ```tsx From 70315413cf71ccf73651d72297f5833122b702bd Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Fri, 17 Jul 2026 17:13:57 +1000 Subject: [PATCH 03/26] docs(durable-stream): Cloudflare service-binding example; make server optional when fetch is provided MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The durableStream adapter is a protocol client, so a Cloudflare Workers + Durable Objects backend that speaks the same protocol needs no new adapter — just the injected `fetch` seam. Over a service binding the host is irrelevant (dispatch routes to the bound Worker by path), so `server` is now optional whenever `fetch` is supplied and defaults to a reserved `.internal` base. Passing neither `server` nor `fetch` throws loudly. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/resumable-streams/overview.md | 5 ++-- .../ai-durable-stream/src/durable-stream.ts | 26 ++++++++++++++----- .../tests/durable-stream.test.ts | 26 +++++++++++++++++++ 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/docs/resumable-streams/overview.md b/docs/resumable-streams/overview.md index 50f4d158c..e6509dd2a 100644 --- a/docs/resumable-streams/overview.md +++ b/docs/resumable-streams/overview.md @@ -128,11 +128,10 @@ interface Env { } function cloudflareAdapter(request: Request, env: Env) { + // No `server` needed — the binding routes by path, so the adapter uses an + // internal placeholder base and only the `/streams/...` path matters. return durableStream(request, { streamPrefix: 'chat-runs', - // Any absolute URL parses; a service binding dispatches to the bound - // Worker regardless of host, and only the `/streams/...` path is used. - server: 'https://durable-streams', fetch: env.DURABLE_STREAMS.fetch.bind(env.DURABLE_STREAMS), }) } diff --git a/packages/ai-durable-stream/src/durable-stream.ts b/packages/ai-durable-stream/src/durable-stream.ts index 638f7baca..b8af0386f 100644 --- a/packages/ai-durable-stream/src/durable-stream.ts +++ b/packages/ai-durable-stream/src/durable-stream.ts @@ -11,8 +11,14 @@ type DurableStreamCursor = string & { export type DurableStreamOffset = DurableStreamCursor | '-1' | 'now' export interface DurableStreamOptions { - /** Base URL of the Durable Streams server (no trailing slash needed). */ - server: string + /** + * Base URL of the Durable Streams server (no trailing slash needed). + * Optional when `fetch` is supplied — e.g. a Cloudflare service binding that + * ignores the host and dispatches to the bound Worker by path — in which case + * an internal placeholder base is used and only the `/streams/...` path + * matters. + */ + server?: string /** Stream-name prefix. Defaults to `runs`. */ streamPrefix?: string /** Fetch implementation. Defaults to the global fetch. */ @@ -393,15 +399,23 @@ export function durableStream( options: DurableStreamOptions, ): StreamDurability { const fetchFn = options.fetch ?? globalThis.fetch - assertTransportField(options.server, 'server URL') + if (options.server === undefined && options.fetch === undefined) { + throw new DurableStreamError( + 'server is required unless a fetch implementation is provided', + ) + } + // When a custom fetch routes by path (e.g. a service binding), the host is + // irrelevant; a reserved `.internal` base parses without ever resolving. + const rawServer = options.server ?? 'https://durable-streams.internal' + assertTransportField(rawServer, 'server URL') try { - void new URL(options.server) + void new URL(rawServer) } catch { throw new DurableStreamError( - `invalid server URL: ${JSON.stringify(options.server)}`, + `invalid server URL: ${JSON.stringify(rawServer)}`, ) } - const server = options.server.replace(/\/+$/, '') + const server = rawServer.replace(/\/+$/, '') const prefix = assertTransportField( options.streamPrefix ?? 'runs', 'streamPrefix', diff --git a/packages/ai-durable-stream/tests/durable-stream.test.ts b/packages/ai-durable-stream/tests/durable-stream.test.ts index bb69620f2..097b21e28 100644 --- a/packages/ai-durable-stream/tests/durable-stream.test.ts +++ b/packages/ai-durable-stream/tests/durable-stream.test.ts @@ -797,6 +797,32 @@ describe('durableStream official HTTP protocol', () => { }).rejects.toThrow(/CR\/LF/) }) + it('omits server when a routing fetch (e.g. a service binding) is provided', async () => { + const server = makeProtocolServer() + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-binding'), + { fetch: server.fetchStub }, + ) + + const [offset] = await durability.append([textChunk('x')]) + expect(offset).toBeTruthy() + + // The adapter routed through the provided fetch using an internal base URL, + // so only the `/streams/...` path is meaningful. + const firstCall = server.fetchStub.mock.calls[0] + if (!firstCall) throw new Error('Expected the adapter to call fetch') + expect(String(firstCall[0])).toContain('/streams/') + }) + + it('requires server when no fetch is provided', () => { + expect(() => + durableStream( + new Request('https://app.test/api/chat?runId=run-no-fetch'), + {}, + ), + ).toThrow(/server is required unless a fetch/) + }) + it('awaits external close and sends the protocol close header', async () => { const closing = deferred() const server = makeProtocolServer({ From 5b19a9909590e76905a04ea0b90c79cc820f395c Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Fri, 17 Jul 2026 17:43:22 +1000 Subject: [PATCH 04/26] fix(resumable-streams): bound eviction, reconnection, and surface silent failures Addresses review findings on the resumable-streams PR: #1 memoryStream never evicted its process-global log Map (unbounded growth). Completed logs are now swept after a grace window with a hard LRU cap; active runs are never evicted. Adds MemoryStreamOptions. #3 memoryStream join/resume of an unknown or evicted run parked forever. A concrete resume of a missing log now throws; a from-start join bounds the wait for the first chunk (firstChunkDeadlineMs) instead of hanging. #2 The client resumable-SSE reconnect loop and the durableStream read loop were unbounded and backoff-free. The client now throttles between attempts and caps the total (StreamReconnectLimitError); durableStream caps consecutive body-read-failure retries. Normal long-poll advancement is never throttled. Adds reconnect options to both. #4 Durability terminal-append / close failures are rethrown to the live consumer but invisible to a replaying joiner. toServerSentEventsResponse now accepts `debug` to record the real cause server-side via the library's logger. Also: durableStream `server` is optional when `fetch` is provided (service bindings). Docs + changeset updated; unit tests added for each path (timing- and eviction-based behavior is covered by unit tests rather than the aimock e2e harness, which can't exercise it deterministically). Co-Authored-By: Claude Opus 4.8 (1M context) --- .changeset/resumable-streams.md | 15 +++ docs/resumable-streams/overview.md | 45 ++++++- packages/ai-client/src/connection-adapters.ts | 85 ++++++++++++- packages/ai-client/src/index.ts | 2 + .../connection-adapters-resumable.test.ts | 65 ++++++++++ .../ai-durable-stream/src/durable-stream.ts | 45 +++++++ .../tests/durable-stream.test.ts | 51 ++++++++ packages/ai/src/index.ts | 2 +- packages/ai/src/stream-durability.ts | 116 ++++++++++++++++-- packages/ai/src/stream-to-response.ts | 29 ++++- packages/ai/tests/stream-durability.test.ts | 100 ++++++++++++++- .../stream-to-response-durability.test.ts | 38 ++++++ 12 files changed, 578 insertions(+), 15 deletions(-) diff --git a/.changeset/resumable-streams.md b/.changeset/resumable-streams.md index 50309f3e9..3c2cfbb8a 100644 --- a/.changeset/resumable-streams.md +++ b/.changeset/resumable-streams.md @@ -25,3 +25,18 @@ prefix, and exposes `joinRun(runId)` to attach to an in-flight or finished run from the start (read-only GET with `offset=-1`). Untagged streams behave exactly as before. A durable run that ends with no terminal event and no forward progress now throws `DurableStreamIncompleteError` instead of hanging. + +Reconnection and durability are bounded so failures surface rather than hang or +loop: + +- `memoryStream` evicts completed logs after a grace window (unbounded growth + is gone); resuming an expired/unknown run throws, and a from-start join to a + run that never produces fails after `MemoryStreamOptions.firstChunkDeadlineMs`. +- `fetchServerSentEvents` accepts `reconnect: { maxAttempts, delayMs }` — a + throttle plus a total ceiling that fails with the new + `StreamReconnectLimitError` instead of reconnecting endlessly. +- `durableStream` accepts `reconnect: { maxReadFailures, delayMs }` to bound its + read-retry loop, and `server` is now optional when `fetch` is provided (e.g. a + Cloudflare service binding). +- `toServerSentEventsResponse` accepts `debug` to record durability terminal / + close failures server-side, where a replaying joiner cannot observe them. diff --git a/docs/resumable-streams/overview.md b/docs/resumable-streams/overview.md index e6509dd2a..39c3c0c58 100644 --- a/docs/resumable-streams/overview.md +++ b/docs/resumable-streams/overview.md @@ -105,7 +105,11 @@ never guesses an offset. For development and tests, swap `durableStream(request, durableOptions)` for `memoryStream(request)` from `@tanstack/ai` — no external service required, -at the cost of the log living only in that process's memory. +at the cost of the log living only in that process's memory. Because that store +is process-global, `memoryStream` is for development and single-process +deployments only: completed runs are evicted after a grace window (so resuming +an expired or unknown run fails loudly instead of hanging), and a from-start +join to a run that never produces fails after `firstChunkDeadlineMs`. ### Cloudflare Durable Streams @@ -218,8 +222,43 @@ The producer awaits `close()` on all in-process exits: Cancellation and provider failure also attempt to append a terminal `RUN_ERROR` with an aborted or failed error payload before closing. If terminal -append or close fails, the failure is surfaced. This prevents a known error -from leaving readers parked on an apparently live log. +append or close fails, the failure is surfaced to the live consumer. Because a +_joiner_ replaying the log only sees a generic incomplete error, pass +`debug` to `toServerSentEventsResponse` to record the real cause server-side: + +```ts +import { memoryStream, toServerSentEventsResponse } from '@tanstack/ai' +import type { StreamChunk } from '@tanstack/ai' + +function respond(request: Request, stream: AsyncIterable) { + return toServerSentEventsResponse(stream, { + durability: { adapter: memoryStream(request) }, + debug: true, // or { logger } for a custom Logger + }) +} +``` + +## Reconnection bounding + +Reconnection only follows forward progress, so a dropped connection resumes from +the last offset. To keep a flapping producer (or a proxy that rolls the socket +after every event) from reconnecting without end, the client throttles between +attempts and caps the total, failing with `StreamReconnectLimitError`: + +```ts +import { fetchServerSentEvents } from '@tanstack/ai-client' + +function makeConnection() { + return fetchServerSentEvents('/api/chat', { + reconnect: { maxAttempts: 1000, delayMs: 250 }, // defaults shown + }) +} +``` + +`durableStream` bounds its own read loop the same way — after a mid-window body +read failure it retries from the last valid position, capping consecutive +failures (`reconnect: { maxReadFailures: 10, delayMs: 250 }`). Normal +long-poll window advancement is never throttled. ## Process death diff --git a/packages/ai-client/src/connection-adapters.ts b/packages/ai-client/src/connection-adapters.ts index d198ec22d..0593719b1 100644 --- a/packages/ai-client/src/connection-adapters.ts +++ b/packages/ai-client/src/connection-adapters.ts @@ -68,6 +68,69 @@ export class DurableStreamIncompleteError extends Error { } } +/** + * Thrown when a durable run exceeds its reconnect ceiling. Bounds the + * otherwise-unbounded reconnect loop so a flapping producer (or a proxy that + * rolls the socket after every event) surfaces a failure instead of + * reconnecting without end. + */ +export class StreamReconnectLimitError extends Error { + constructor(attempts: number) { + super( + `Durable run exceeded its reconnect ceiling of ${attempts} attempts — giving up.`, + ) + this.name = 'StreamReconnectLimitError' + } +} + +/** + * Reconnect bounding for resumable SSE. Reconnection only follows forward + * progress (a no-progress end already fails), so a constant throttle delay + * prevents a hot loop against the origin without penalizing a legitimately + * long-lived run, and a total-attempts ceiling caps a pathological + * progress-then-drop flapper. + */ +export interface ReconnectOptions { + /** + * Total reconnect attempts before failing with + * {@link StreamReconnectLimitError}. A normal long run reconnects only on + * rare drops and never approaches this. Default 1000. + */ + maxAttempts?: number + /** Delay between reconnect attempts, in ms, to avoid hammering. Default 250. */ + delayMs?: number +} + +interface ResolvedReconnectOptions { + maxAttempts: number + delayMs: number +} + +function resolveReconnectOptions( + options: ReconnectOptions | undefined, +): ResolvedReconnectOptions { + return { + maxAttempts: options?.maxAttempts ?? 1000, + delayMs: options?.delayMs ?? 250, + } +} + +/** Resolve after `ms`, or immediately once `signal` aborts. Never rejects. */ +function abortableDelay(ms: number, signal?: AbortSignal): Promise { + if (ms <= 0 || signal?.aborted) return Promise.resolve() + return new Promise((resolve) => { + const onAbort = () => { + clearTimeout(timer) + resolve() + } + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort) + resolve() + }, ms) + signal?.addEventListener('abort', onAbort, { once: true }) + }) +} + function generateRunId(prefix: string): string { return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` } @@ -269,9 +332,23 @@ async function* resumableServerSentEvents( url: string, requestInit: RequestInit, abortSignal?: AbortSignal, + reconnectOptions?: ReconnectOptions, ): AsyncGenerator { const seen = new Set() let lastEventId: string | undefined + const reconnect = resolveReconnectOptions(reconnectOptions) + let reconnectAttempts = 0 + + // Throttle before re-issuing the request, and enforce the total ceiling so a + // producer that keeps dropping after each event is bounded rather than + // reconnecting forever. + async function waitBeforeReconnect(): Promise { + reconnectAttempts += 1 + if (reconnectAttempts > reconnect.maxAttempts) { + throw new StreamReconnectLimitError(reconnect.maxAttempts) + } + await abortableDelay(reconnect.delayMs, abortSignal) + } for (;;) { if (abortSignal?.aborted) return @@ -316,6 +393,7 @@ async function* resumableServerSentEvents( lastEventId !== undefined && progressed ) { + await waitBeforeReconnect() continue } throw error @@ -328,7 +406,8 @@ async function* resumableServerSentEvents( if (progressed) { // Clean end WITHOUT a terminal event but we advanced — the producer is // still going (or the socket rolled over). Reconnect from the last - // offset. + // offset (backing off to avoid a hot loop against the origin). + await waitBeforeReconnect() continue } // Ended without a terminal event AND made no forward progress on this @@ -574,6 +653,8 @@ export interface FetchConnectionOptions { signal?: AbortSignal body?: Record fetchClient?: typeof globalThis.fetch + /** Bounding for resumable-SSE reconnection (throttle delay + attempt ceiling). */ + reconnect?: ReconnectOptions } /** @@ -712,6 +793,7 @@ export function fetchServerSentEvents( credentials: resolvedOptions.credentials || 'same-origin', }, signal, + resolvedOptions.reconnect, ) }, async *joinRun(runId, abortSignal) { @@ -742,6 +824,7 @@ export function fetchServerSentEvents( credentials: resolvedOptions.credentials || 'same-origin', }, signal, + resolvedOptions.reconnect, ) }, } diff --git a/packages/ai-client/src/index.ts b/packages/ai-client/src/index.ts index f5eece614..e0967625b 100644 --- a/packages/ai-client/src/index.ts +++ b/packages/ai-client/src/index.ts @@ -89,9 +89,11 @@ export { rpcStream, StreamTruncatedError, DurableStreamIncompleteError, + StreamReconnectLimitError, type ConnectConnectionAdapter, type ConnectionAdapter, type FetchConnectionOptions, + type ReconnectOptions, type ResumableConnectConnectionAdapter, type RunAgentInputContext, type SubscribeConnectionAdapter, diff --git a/packages/ai-client/tests/connection-adapters-resumable.test.ts b/packages/ai-client/tests/connection-adapters-resumable.test.ts index 31f9d730d..b8cea68d0 100644 --- a/packages/ai-client/tests/connection-adapters-resumable.test.ts +++ b/packages/ai-client/tests/connection-adapters-resumable.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest' import { EventType } from '@tanstack/ai/client' import { DurableStreamIncompleteError, + StreamReconnectLimitError, fetchServerSentEvents, } from '../src/connection-adapters' import type { StreamChunk } from '@tanstack/ai/client' @@ -213,6 +214,70 @@ describe('resumable SSE connection adapter', () => { expect(fetchClient).toHaveBeenCalledTimes(2) }) + it('bounds reconnection with a total-attempts ceiling', async () => { + // Every pass delivers one NEW tagged event then ends cleanly with no + // terminal — an endless progress-then-drop flapper the ceiling must stop. + let pass = 0 + const fetchClient = vi.fn(async () => { + pass += 1 + return sseResponse(contentEvent(`run@${pass}`, 'x')) + }) + const adapter = fetchServerSentEvents('/api/chat', { + fetchClient, + reconnect: { maxAttempts: 3, delayMs: 0 }, + }) + + await expect(async () => { + for await (const _chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + // drain + } + }).rejects.toBeInstanceOf(StreamReconnectLimitError) + + // Initial fetch + 3 permitted reconnects; the 4th trips the ceiling before + // re-fetching. + expect(fetchClient).toHaveBeenCalledTimes(4) + }) + + it('stops reconnecting promptly when aborted during the throttle delay', async () => { + const controller = new AbortController() + let pass = 0 + const fetchClient = vi.fn(async () => { + pass += 1 + return sseResponse(contentEvent(`run@${pass}`, 'x')) + }) + const adapter = fetchServerSentEvents('/api/chat', { + fetchClient, + signal: controller.signal, + reconnect: { delayMs: 10_000 }, + }) + + const chunks: Array = [] + const done = (async () => { + for await (const chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + chunks.push(chunk) + } + })() + + // Let the first pass finish and enter the 10s throttle, then abort — the + // delay must resolve immediately rather than stalling for 10s. + await new Promise((resolve) => setTimeout(resolve, 20)) + controller.abort() + await done + + expect(chunks).toHaveLength(1) + expect(fetchClient).toHaveBeenCalledTimes(1) + }) + it('does not reconnect a body reader after the caller aborts', async () => { const fetchClient = vi.fn(async () => failingSseResponse( diff --git a/packages/ai-durable-stream/src/durable-stream.ts b/packages/ai-durable-stream/src/durable-stream.ts index b8af0386f..a8d3fdbb2 100644 --- a/packages/ai-durable-stream/src/durable-stream.ts +++ b/packages/ai-durable-stream/src/durable-stream.ts @@ -28,6 +28,38 @@ export interface DurableStreamOptions { * resolver is called for every request so credentials can rotate. */ headers?: HeadersInit | (() => HeadersInit | Promise) + /** + * Bounding for the read reconnect loop. After a response-body read failure + * mid-window, `read` retries from the last valid position; these cap + * consecutive retries and throttle them so a persistently failing backend + * surfaces the error instead of looping without end. Normal window + * advancement (long-poll) is never throttled. + */ + reconnect?: { + /** + * Consecutive body-read-failure retries before surfacing the underlying + * read error. Default 10. + */ + maxReadFailures?: number + /** Delay between read retries, in ms. Default 250. */ + delayMs?: number + } +} + +/** Resolve after `ms`, or immediately once `signal` aborts. Never rejects. */ +function abortableDelay(ms: number, signal?: AbortSignal): Promise { + if (ms <= 0 || signal?.aborted) return Promise.resolve() + return new Promise((resolve) => { + const onAbort = () => { + clearTimeout(timer) + resolve() + } + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort) + resolve() + }, ms) + signal?.addEventListener('abort', onAbort, { once: true }) + }) } export class DurableStreamError extends Error { @@ -416,6 +448,8 @@ export function durableStream( ) } const server = rawServer.replace(/\/+$/, '') + const maxReadFailures = options.reconnect?.maxReadFailures ?? 10 + const readRetryDelayMs = options.reconnect?.delayMs ?? 250 const prefix = assertTransportField( options.streamPrefix ?? 'runs', 'streamPrefix', @@ -551,6 +585,7 @@ export function durableStream( deliveredThroughSeq = cursor.seq } let streamCursor: string | undefined + let consecutiveReadFailures = 0 for (;;) { if (signal?.aborted) return @@ -626,6 +661,12 @@ export function durableStream( (backendOffset !== requestOffset || streamCursor !== requestCursor)) ) { + // Made progress before the body failed — retry from the last valid + // position, but cap consecutive failures and throttle so a + // persistently failing backend surfaces the error, not a hot loop. + consecutiveReadFailures += 1 + if (consecutiveReadFailures > maxReadFailures) throw error.readError + await abortableDelay(readRetryDelayMs, signal) continue } throw error.readError @@ -633,6 +674,10 @@ export function durableStream( throw error } + // A window read to completion (no body failure) clears the streak; only + // consecutive failures accumulate toward the ceiling. + consecutiveReadFailures = 0 + if (signal?.aborted) return if (dataAwaitingControl || !sawControl) { throw new DurableStreamError( diff --git a/packages/ai-durable-stream/tests/durable-stream.test.ts b/packages/ai-durable-stream/tests/durable-stream.test.ts index 097b21e28..dab3f1fbc 100644 --- a/packages/ai-durable-stream/tests/durable-stream.test.ts +++ b/packages/ai-durable-stream/tests/durable-stream.test.ts @@ -609,6 +609,57 @@ describe('durableStream official HTTP protocol', () => { ) }) + it('caps consecutive body-read failures and surfaces the read error', async () => { + let readNumber = 0 + const fetchStub = vi.fn(async () => { + readNumber += 1 + const seq = readNumber + let pullNumber = 0 + const body = new ReadableStream({ + pull(controller) { + pullNumber += 1 + if (pullNumber === 1) { + controller.enqueue( + new TextEncoder().encode( + dataEvent([{ v: 1, seq, chunk: textChunk(`d${seq}`) }]) + + controlEvent({ + // Advance the offset every window so each pass makes real + // progress before the body fails. + streamNextOffset: `opaque::after/${readNumber}?token=%2F`, + streamCursor: `collapse::${readNumber}`, + upToDate: true, + }), + ), + ) + return + } + controller.error(new TypeError('socket read failed')) + }, + }) + return new Response(body, { + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + }) + }) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-read-cap'), + { + server: 'https://ds.test', + fetch: fetchStub, + reconnect: { maxReadFailures: 3, delayMs: 0 }, + }, + ) + + await expect(async () => { + for await (const _entry of durability.read('-1')) { + // drain + } + }).rejects.toThrow(/socket read failed/) + + // Initial read + 3 permitted retries; the 4th failure trips the ceiling. + expect(fetchStub).toHaveBeenCalledTimes(4) + }) + it('reconnects from the same control after data is read before a body failure', async () => { const requests: Array = [] let readNumber = 0 diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index f005588ba..f8bde1f52 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -81,7 +81,7 @@ export { // Delivery durability (transport layer) export { memoryStream } from './stream-durability' -export type { StreamDurability } from './stream-durability' +export type { MemoryStreamOptions, StreamDurability } from './stream-durability' // Tool call management export { ToolCallManager } from './activities/chat/tools/tool-calls' diff --git a/packages/ai/src/stream-durability.ts b/packages/ai/src/stream-durability.ts index 542369fd7..8742177af 100644 --- a/packages/ai/src/stream-durability.ts +++ b/packages/ai/src/stream-durability.ts @@ -124,20 +124,79 @@ interface MemoryEntry { interface MemoryLog { entries: Array complete: boolean + /** Epoch ms when the log was terminalized; undefined while still producing. */ + completedAt: number | undefined waiters: Array<() => void> } +/** + * Bounds for the in-process log store. `memoryStream` is the dev/single-process + * backend; without eviction its module-global Map would grow without bound on a + * long-lived server (one retained chunk buffer per run, forever). Completed logs + * are swept after a grace window — late resumers/joiners still work briefly — + * and a hard cap drops the oldest completed logs under pressure. Active + * (incomplete) logs are never evicted, so an in-flight run is never dropped. + */ +const MAX_MEMORY_RUNS = 1024 +const COMPLETED_LOG_TTL_MS = 5 * 60_000 + +/** + * How long a from-start join (`-1` / `now`) waits for a run's first chunk before + * failing. Bounds the "joined a run that never produces" case so a consumer + * gets a surfaced error instead of an indefinitely-open, event-less connection. + */ +const DEFAULT_FIRST_CHUNK_DEADLINE_MS = 30_000 + +/** Options for the in-process delivery-durability backend. */ +export interface MemoryStreamOptions { + /** + * Milliseconds a from-start join waits for the run's first chunk before + * throwing. Defaults to {@link DEFAULT_FIRST_CHUNK_DEADLINE_MS}. + */ + firstChunkDeadlineMs?: number +} + const memoryLogs = new Map() +/** + * Evict completed logs past their grace window, then, if still over the cap, + * drop the oldest completed logs (the Map preserves insertion order) until back + * under the cap. Never touches an incomplete (in-flight) log. + */ +function sweepMemoryLogs(now: number): void { + for (const [id, log] of memoryLogs) { + if ( + log.complete && + log.completedAt !== undefined && + now - log.completedAt > COMPLETED_LOG_TTL_MS + ) { + memoryLogs.delete(id) + } + } + if (memoryLogs.size <= MAX_MEMORY_RUNS) return + for (const [id, log] of memoryLogs) { + if (memoryLogs.size <= MAX_MEMORY_RUNS) break + if (log.complete) memoryLogs.delete(id) + } +} + function getOrCreateLog(id: string): MemoryLog { let log = memoryLogs.get(id) if (!log) { - log = { entries: [], complete: false, waiters: [] } + sweepMemoryLogs(Date.now()) + log = { entries: [], complete: false, completedAt: undefined, waiters: [] } memoryLogs.set(id, log) } return log } +function markComplete(log: MemoryLog): void { + if (!log.complete) { + log.complete = true + log.completedAt = Date.now() + } +} + function wakeWaiters(log: MemoryLog): void { const waiters = log.waiters log.waiters = [] @@ -147,10 +206,20 @@ function wakeWaiters(log: MemoryLog): void { /** * The zero-infrastructure delivery-durability backend. Its versioned cursor is * deliberately private: callers and core only pass the returned string back. + * + * Logs live in a process-global map, so this backend is for development, tests, + * and single-process deployments only. Completed runs are evicted after a grace + * window (see {@link COMPLETED_LOG_TTL_MS}); a resume of an evicted or unknown + * run fails loudly rather than hanging. */ -export function memoryStream(request: Request): StreamDurability { +export function memoryStream( + request: Request, + options: MemoryStreamOptions = {}, +): StreamDurability { const resumeOffset = readResumeOffset(request) const runId = resolveMemoryRunId(request, resumeOffset) + const firstChunkDeadlineMs = + options.firstChunkDeadlineMs ?? DEFAULT_FIRST_CHUNK_DEADLINE_MS return { resumeFrom: () => resumeOffset, @@ -161,7 +230,7 @@ export function memoryStream(request: Request): StreamDurability { const seq = firstSeq + index const offset = encodeMemoryOffset(runId, seq) log.entries.push({ seq, offset, chunk }) - if (isTerminalChunk(chunk)) log.complete = true + if (isTerminalChunk(chunk)) markComplete(log) return offset }) wakeWaiters(log) @@ -169,12 +238,23 @@ export function memoryStream(request: Request): StreamDurability { }, close: () => { const log = getOrCreateLog(runId) - log.complete = true + markComplete(log) wakeWaiters(log) return Promise.resolve() }, read: async function* (offset, signal) { const log = getOrCreateLog(runId) + + // A concrete resume offset means data provably existed for this run. If + // the log holds nothing, the run was evicted (or never lived in this + // process) and will not reappear — fail rather than park forever. + const isFromStartJoin = offset === '-1' || offset === 'now' + if (!isFromStartJoin && log.entries.length === 0 && !log.complete) { + throw new Error( + `Unknown or expired memory stream run: ${JSON.stringify(runId)}`, + ) + } + const threshold = memoryThreshold( offset, runId, @@ -193,18 +273,40 @@ export function memoryStream(request: Request): StreamDurability { } if (log.complete || signal?.aborted) return - await new Promise((resolve) => { - const onAbort = () => { + // Bound only the wait for the very first chunk: once a run has produced + // anything, its producer owns termination and a caught-up reader may + // legitimately park indefinitely between chunks. + const deadlineForFirstChunk = + log.entries.length === 0 ? firstChunkDeadlineMs : undefined + + await new Promise((resolve, reject) => { + let timer: ReturnType | undefined + const cleanup = () => { + if (timer !== undefined) clearTimeout(timer) + signal?.removeEventListener('abort', onAbort) const waiterIndex = log.waiters.indexOf(wake) if (waiterIndex !== -1) log.waiters.splice(waiterIndex, 1) + } + const onAbort = () => { + cleanup() resolve() } const wake = () => { - signal?.removeEventListener('abort', onAbort) + cleanup() resolve() } log.waiters.push(wake) signal?.addEventListener('abort', onAbort, { once: true }) + if (deadlineForFirstChunk !== undefined) { + timer = setTimeout(() => { + cleanup() + reject( + new Error( + `Memory stream run produced no data within ${deadlineForFirstChunk}ms: ${JSON.stringify(runId)}`, + ), + ) + }, deadlineForFirstChunk) + } }) } }, diff --git a/packages/ai/src/stream-to-response.ts b/packages/ai/src/stream-to-response.ts index a5a97b734..1eddc0e36 100644 --- a/packages/ai/src/stream-to-response.ts +++ b/packages/ai/src/stream-to-response.ts @@ -1,5 +1,8 @@ import { toRunErrorPayload } from './activities/error-payload' import { EventType } from './types' +import { resolveDebugOption } from './logger/resolve' +import type { InternalLogger } from './logger/internal-logger' +import type { DebugOption } from './logger/types' import type { StreamDurability } from './stream-durability' import type { StreamChunk } from './types' @@ -263,7 +266,11 @@ function isDurabilityFlushBoundary(chunk: StreamChunk): boolean { function durableStreamSource( stream: AsyncIterable, durability: StreamDurability, - options: { abortController: AbortController; batch?: number }, + options: { + abortController: AbortController + batch?: number + logger?: InternalLogger + }, ): { source: AsyncIterable getId: (chunk: StreamChunk) => string | undefined @@ -271,6 +278,7 @@ function durableStreamSource( const resumeOffset = durability.resumeFrom() const batchSize = resolveBatchSize(options.batch) const abortController = options.abortController + const logger = options.logger const idByChunk = new WeakMap() const seenOffsets = new Set() const getId = (chunk: StreamChunk): string | undefined => idByChunk.get(chunk) @@ -373,6 +381,12 @@ function durableStreamSource( await durability.append([runErrorChunk(cause)]) terminalPersisted = true } catch (terminalError) { + // Rethrown to the live consumer below, but a joiner replaying the log + // only ever sees a generic incomplete error — so record the real + // cause server-side where an operator can act on it. + logger?.errors('persisting terminal RUN_ERROR failed', { + error: terminalError, + }) recordFailure(terminalError, 'persisting terminal RUN_ERROR failed') } } @@ -380,6 +394,9 @@ function durableStreamSource( try { await durability.close() } catch (closeError) { + // A failed close leaves the durable log unterminated for joiners; the + // live consumer gets the rethrow, but log it for the joiner's sake. + logger?.errors('closing durability stream failed', { error: closeError }) recordFailure(closeError, 'closing durability stream failed') } @@ -439,9 +456,16 @@ export function toServerSentEventsResponse( init?: ResponseInit & { abortController?: AbortController durability?: { adapter: StreamDurability; batch?: number } + /** + * Debug logging for durability failure paths (terminal-append and close). + * These are rethrown to the live consumer, but a joiner only sees a generic + * incomplete error, so enabling this surfaces the real cause server-side. + */ + debug?: DebugOption }, ): Response { - const { headers, abortController, durability, ...responseInit } = init ?? {} + const { headers, abortController, durability, debug, ...responseInit } = + init ?? {} // Start with default SSE headers const mergedHeaders = new Headers({ @@ -465,6 +489,7 @@ export function toServerSentEventsResponse( const { source, getId } = durableStreamSource(stream, durability.adapter, { abortController: deliveryAbortController, batch: durability.batch, + ...(debug !== undefined ? { logger: resolveDebugOption(debug) } : {}), }) body = toServerSentEventsStream(source, deliveryAbortController, getId) } else { diff --git a/packages/ai/tests/stream-durability.test.ts b/packages/ai/tests/stream-durability.test.ts index 58ee65440..21164cf74 100644 --- a/packages/ai/tests/stream-durability.test.ts +++ b/packages/ai/tests/stream-durability.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { memoryStream } from '../src/stream-durability' import { EventType } from '../src/types' import { ev } from './test-utils' @@ -171,6 +171,104 @@ describe('memoryStream', () => { await expect(iterated).resolves.toEqual([]) }) + it('fails a from-start join that never receives data before the deadline', async () => { + const joiner = memoryStream( + new Request( + 'https://example.test/api/chat?runId=run-no-producer&offset=-1', + { method: 'POST' }, + ), + { firstChunkDeadlineMs: 20 }, + ) + const resumeOffset = joiner.resumeFrom() + if (resumeOffset === null) throw new Error('Expected a resume offset') + + await expect(readLabels(joiner.read(resumeOffset))).rejects.toThrow( + /produced no data within 20ms/, + ) + }) + + it('does not apply the first-chunk deadline once a run has produced data', async () => { + const producer = memoryStream( + new Request('https://example.test/api/chat?runId=run-slow-tail', { + method: 'POST', + }), + ) + await producer.append([ev.textContent('a')]) + + const joiner = memoryStream( + new Request('https://example.test/api/chat?runId=run-slow-tail&offset=-1', { + method: 'POST', + }), + { firstChunkDeadlineMs: 20 }, + ) + const resumeOffset = joiner.resumeFrom() + if (resumeOffset === null) throw new Error('Expected a resume offset') + + const received: Array = [] + const done = (async () => { + for await (const { chunk } of joiner.read(resumeOffset)) { + received.push(label(chunk)) + } + })() + + // Well past the 20ms first-chunk deadline: a caught-up reader keeps parking + // because the run already produced data. + await new Promise((resolve) => setTimeout(resolve, 60)) + expect(received).toEqual(['a']) + + await producer.append([ev.textContent('b'), ev.runFinished()]) + await done + expect(received).toEqual(['a', 'b', '[RUN_FINISHED]']) + }) + + it('fails a resume of an evicted run rather than hanging', async () => { + vi.useFakeTimers() + vi.setSystemTime(0) + try { + const producer = memoryStream( + new Request('https://example.test/api/chat?runId=run-evictable', { + method: 'POST', + }), + ) + const offsets = await producer.append([ev.textContent('a')]) + await producer.append([ev.runFinished()]) + await producer.close() + const resumeFrom = offsets[0] + if (resumeFrom === undefined) throw new Error('Expected an offset') + + // Within the grace window the completed run still resumes. + const early = memoryStream( + new Request('https://example.test/api/chat?runId=run-evictable', { + method: 'POST', + headers: { 'Last-Event-ID': resumeFrom }, + }), + ) + expect(await readLabels(early.read(resumeFrom))).toEqual([ + '[RUN_FINISHED]', + ]) + + // Past the grace window, creating a new log sweeps the completed one, and + // resuming the evicted run surfaces an error instead of parking. + vi.setSystemTime(6 * 60_000) + await memoryStream( + new Request('https://example.test/api/chat?runId=run-sweep-trigger', { + method: 'POST', + }), + ).append([ev.textContent('x')]) + const late = memoryStream( + new Request('https://example.test/api/chat?runId=run-evictable', { + method: 'POST', + headers: { 'Last-Event-ID': resumeFrom }, + }), + ) + await expect(readLabels(late.read(resumeFrom))).rejects.toThrow( + /Unknown or expired memory stream run/, + ) + } finally { + vi.useRealTimers() + } + }) + it('rejects invalid run ids and offsets loudly', () => { expect(() => memoryStream( diff --git a/packages/ai/tests/stream-to-response-durability.test.ts b/packages/ai/tests/stream-to-response-durability.test.ts index ecf51cacd..61c47ce72 100644 --- a/packages/ai/tests/stream-to-response-durability.test.ts +++ b/packages/ai/tests/stream-to-response-durability.test.ts @@ -113,6 +113,44 @@ describe('toServerSentEventsResponse with durability', () => { expect(loggedLabels).toEqual(['1', '2', '3', '4', '5']) }) + it('logs a durability close failure server-side when debug is enabled', async () => { + const closeError = new Error('close boom') + let seq = 0 + const durability: StreamDurability = { + resumeFrom: () => null, + append: async (chunks) => chunks.map(() => `off-${seq++}`), + close: async () => { + throw closeError + }, + async *read() { + // Not exercised by this test. + }, + } + const errorLog = vi.fn() + const logger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: errorLog, + } + const { stream } = fiveChunkStream() + + // The rethrown close failure lands in-band as an error event; the consumer + // drains cleanly. The point is that the cause is recorded server-side, where + // a joiner (who only sees a generic incomplete error) cannot observe it. + await readBody( + toServerSentEventsResponse(stream, { + durability: { adapter: durability }, + debug: { logger }, + }), + ) + + expect(errorLog).toHaveBeenCalledWith( + expect.stringContaining('closing durability stream failed'), + expect.objectContaining({ error: closeError }), + ) + }) + it('replays opaque IDs from the log without iterating the input stream', async () => { const { stream } = fiveChunkStream() const produced = parseSseEvents( From cc295c0cce4bbfb7ffe2abded53172490263a2ca Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 07:49:19 +0000 Subject: [PATCH 05/26] ci: apply automated fixes --- packages/ai-durable-stream/src/durable-stream.ts | 3 ++- packages/ai/src/stream-to-response.ts | 4 +++- packages/ai/tests/stream-durability.test.ts | 9 ++++++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/ai-durable-stream/src/durable-stream.ts b/packages/ai-durable-stream/src/durable-stream.ts index a8d3fdbb2..e2043b077 100644 --- a/packages/ai-durable-stream/src/durable-stream.ts +++ b/packages/ai-durable-stream/src/durable-stream.ts @@ -665,7 +665,8 @@ export function durableStream( // position, but cap consecutive failures and throttle so a // persistently failing backend surfaces the error, not a hot loop. consecutiveReadFailures += 1 - if (consecutiveReadFailures > maxReadFailures) throw error.readError + if (consecutiveReadFailures > maxReadFailures) + throw error.readError await abortableDelay(readRetryDelayMs, signal) continue } diff --git a/packages/ai/src/stream-to-response.ts b/packages/ai/src/stream-to-response.ts index 1eddc0e36..5a4797fe4 100644 --- a/packages/ai/src/stream-to-response.ts +++ b/packages/ai/src/stream-to-response.ts @@ -396,7 +396,9 @@ function durableStreamSource( } catch (closeError) { // A failed close leaves the durable log unterminated for joiners; the // live consumer gets the rethrow, but log it for the joiner's sake. - logger?.errors('closing durability stream failed', { error: closeError }) + logger?.errors('closing durability stream failed', { + error: closeError, + }) recordFailure(closeError, 'closing durability stream failed') } diff --git a/packages/ai/tests/stream-durability.test.ts b/packages/ai/tests/stream-durability.test.ts index 21164cf74..657d96870 100644 --- a/packages/ai/tests/stream-durability.test.ts +++ b/packages/ai/tests/stream-durability.test.ts @@ -196,9 +196,12 @@ describe('memoryStream', () => { await producer.append([ev.textContent('a')]) const joiner = memoryStream( - new Request('https://example.test/api/chat?runId=run-slow-tail&offset=-1', { - method: 'POST', - }), + new Request( + 'https://example.test/api/chat?runId=run-slow-tail&offset=-1', + { + method: 'POST', + }, + ), { firstChunkDeadlineMs: 20 }, ) const resumeOffset = joiner.resumeFrom() From 7bd1727c51bc940eb65263c93707d70f31097a18 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Fri, 17 Jul 2026 18:01:44 +1000 Subject: [PATCH 06/26] docs(example): add resumable-streams demo to ts-react-chat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New /resumable route pair: api.resumable.ts (memoryStream-backed POST that appends+tags each SSE event, plus a GET joinRun replay endpoint) and resumable.tsx (start a run, then join it by run ID — in a second tab or after a reload — replaying from the durability log without re-running the model). Nav link added to Header. Kept the shared api.tanchat route untouched. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ts-react-chat/src/components/Header.tsx | 18 +- examples/ts-react-chat/src/routeTree.gen.ts | 42 +++++ .../ts-react-chat/src/routes/api.resumable.ts | 67 ++++++++ .../ts-react-chat/src/routes/resumable.tsx | 159 ++++++++++++++++++ 4 files changed, 284 insertions(+), 2 deletions(-) create mode 100644 examples/ts-react-chat/src/routes/api.resumable.ts create mode 100644 examples/ts-react-chat/src/routes/resumable.tsx diff --git a/examples/ts-react-chat/src/components/Header.tsx b/examples/ts-react-chat/src/components/Header.tsx index 061f74d78..d6a3716fb 100644 --- a/examples/ts-react-chat/src/components/Header.tsx +++ b/examples/ts-react-chat/src/components/Header.tsx @@ -11,12 +11,13 @@ import { Guitar, Home, Image, - Menu, LayoutGrid, - Mic, + Menu, MessageSquare, + Mic, Music, Plug, + RefreshCw, Server, Sparkles, Video, @@ -284,6 +285,19 @@ export default function Header() { Server Function Chat + setIsOpen(false)} + className="flex items-center gap-3 p-3 rounded-lg hover:bg-gray-800 transition-colors mb-2" + activeProps={{ + className: + 'flex items-center gap-3 p-3 rounded-lg bg-cyan-600 hover:bg-cyan-700 transition-colors mb-2', + }} + > + + Resumable Streams + + setIsOpen(false)} diff --git a/examples/ts-react-chat/src/routeTree.gen.ts b/examples/ts-react-chat/src/routeTree.gen.ts index b223bce09..f6e0f8ff1 100644 --- a/examples/ts-react-chat/src/routeTree.gen.ts +++ b/examples/ts-react-chat/src/routeTree.gen.ts @@ -13,6 +13,7 @@ import { Route as TypesafeToolsRouteImport } from './routes/typesafe-tools' import { Route as ThreadsRouteImport } from './routes/threads' import { Route as ServerFnChatRouteImport } from './routes/server-fn-chat' import { Route as SandboxesRouteImport } from './routes/sandboxes' +import { Route as ResumableRouteImport } from './routes/resumable' import { Route as RealtimeRouteImport } from './routes/realtime' import { Route as McpDemoRouteImport } from './routes/mcp-demo' import { Route as McpAppsRouteImport } from './routes/mcp-apps' @@ -37,6 +38,7 @@ import { Route as ApiSummarizeRouteImport } from './routes/api.summarize' import { Route as ApiStructuredOutputRouteImport } from './routes/api.structured-output' import { Route as ApiStructuredChatRouteImport } from './routes/api.structured-chat' import { Route as ApiSandboxTriageRouteImport } from './routes/api.sandbox-triage' +import { Route as ApiResumableRouteImport } from './routes/api.resumable' import { Route as ApiMcpStatusRouteImport } from './routes/api.mcp-status' import { Route as ApiMcpPoolRouteImport } from './routes/api.mcp-pool' import { Route as ApiMcpManualRouteImport } from './routes/api.mcp-manual' @@ -75,6 +77,11 @@ const SandboxesRoute = SandboxesRouteImport.update({ path: '/sandboxes', getParentRoute: () => rootRouteImport, } as any) +const ResumableRoute = ResumableRouteImport.update({ + id: '/resumable', + path: '/resumable', + getParentRoute: () => rootRouteImport, +} as any) const RealtimeRoute = RealtimeRouteImport.update({ id: '/realtime', path: '/realtime', @@ -198,6 +205,11 @@ const ApiSandboxTriageRoute = ApiSandboxTriageRouteImport.update({ path: '/api/sandbox-triage', getParentRoute: () => rootRouteImport, } as any) +const ApiResumableRoute = ApiResumableRouteImport.update({ + id: '/api/resumable', + path: '/api/resumable', + getParentRoute: () => rootRouteImport, +} as any) const ApiMcpStatusRoute = ApiMcpStatusRouteImport.update({ id: '/api/mcp-status', path: '/api/mcp-status', @@ -294,6 +306,7 @@ export interface FileRoutesByFullPath { '/mcp-apps': typeof McpAppsRoute '/mcp-demo': typeof McpDemoRoute '/realtime': typeof RealtimeRoute + '/resumable': typeof ResumableRoute '/sandboxes': typeof SandboxesRoute '/server-fn-chat': typeof ServerFnChatRoute '/threads': typeof ThreadsRoute @@ -309,6 +322,7 @@ export interface FileRoutesByFullPath { '/api/mcp-manual': typeof ApiMcpManualRoute '/api/mcp-pool': typeof ApiMcpPoolRoute '/api/mcp-status': typeof ApiMcpStatusRoute + '/api/resumable': typeof ApiResumableRoute '/api/sandbox-triage': typeof ApiSandboxTriageRoute '/api/structured-chat': typeof ApiStructuredChatRoute '/api/structured-output': typeof ApiStructuredOutputRoute @@ -341,6 +355,7 @@ export interface FileRoutesByTo { '/mcp-apps': typeof McpAppsRoute '/mcp-demo': typeof McpDemoRoute '/realtime': typeof RealtimeRoute + '/resumable': typeof ResumableRoute '/sandboxes': typeof SandboxesRoute '/server-fn-chat': typeof ServerFnChatRoute '/threads': typeof ThreadsRoute @@ -356,6 +371,7 @@ export interface FileRoutesByTo { '/api/mcp-manual': typeof ApiMcpManualRoute '/api/mcp-pool': typeof ApiMcpPoolRoute '/api/mcp-status': typeof ApiMcpStatusRoute + '/api/resumable': typeof ApiResumableRoute '/api/sandbox-triage': typeof ApiSandboxTriageRoute '/api/structured-chat': typeof ApiStructuredChatRoute '/api/structured-output': typeof ApiStructuredOutputRoute @@ -389,6 +405,7 @@ export interface FileRoutesById { '/mcp-apps': typeof McpAppsRoute '/mcp-demo': typeof McpDemoRoute '/realtime': typeof RealtimeRoute + '/resumable': typeof ResumableRoute '/sandboxes': typeof SandboxesRoute '/server-fn-chat': typeof ServerFnChatRoute '/threads': typeof ThreadsRoute @@ -404,6 +421,7 @@ export interface FileRoutesById { '/api/mcp-manual': typeof ApiMcpManualRoute '/api/mcp-pool': typeof ApiMcpPoolRoute '/api/mcp-status': typeof ApiMcpStatusRoute + '/api/resumable': typeof ApiResumableRoute '/api/sandbox-triage': typeof ApiSandboxTriageRoute '/api/structured-chat': typeof ApiStructuredChatRoute '/api/structured-output': typeof ApiStructuredOutputRoute @@ -438,6 +456,7 @@ export interface FileRouteTypes { | '/mcp-apps' | '/mcp-demo' | '/realtime' + | '/resumable' | '/sandboxes' | '/server-fn-chat' | '/threads' @@ -453,6 +472,7 @@ export interface FileRouteTypes { | '/api/mcp-manual' | '/api/mcp-pool' | '/api/mcp-status' + | '/api/resumable' | '/api/sandbox-triage' | '/api/structured-chat' | '/api/structured-output' @@ -485,6 +505,7 @@ export interface FileRouteTypes { | '/mcp-apps' | '/mcp-demo' | '/realtime' + | '/resumable' | '/sandboxes' | '/server-fn-chat' | '/threads' @@ -500,6 +521,7 @@ export interface FileRouteTypes { | '/api/mcp-manual' | '/api/mcp-pool' | '/api/mcp-status' + | '/api/resumable' | '/api/sandbox-triage' | '/api/structured-chat' | '/api/structured-output' @@ -532,6 +554,7 @@ export interface FileRouteTypes { | '/mcp-apps' | '/mcp-demo' | '/realtime' + | '/resumable' | '/sandboxes' | '/server-fn-chat' | '/threads' @@ -547,6 +570,7 @@ export interface FileRouteTypes { | '/api/mcp-manual' | '/api/mcp-pool' | '/api/mcp-status' + | '/api/resumable' | '/api/sandbox-triage' | '/api/structured-chat' | '/api/structured-output' @@ -580,6 +604,7 @@ export interface RootRouteChildren { McpAppsRoute: typeof McpAppsRoute McpDemoRoute: typeof McpDemoRoute RealtimeRoute: typeof RealtimeRoute + ResumableRoute: typeof ResumableRoute SandboxesRoute: typeof SandboxesRoute ServerFnChatRoute: typeof ServerFnChatRoute ThreadsRoute: typeof ThreadsRoute @@ -595,6 +620,7 @@ export interface RootRouteChildren { ApiMcpManualRoute: typeof ApiMcpManualRoute ApiMcpPoolRoute: typeof ApiMcpPoolRoute ApiMcpStatusRoute: typeof ApiMcpStatusRoute + ApiResumableRoute: typeof ApiResumableRoute ApiSandboxTriageRoute: typeof ApiSandboxTriageRoute ApiStructuredChatRoute: typeof ApiStructuredChatRoute ApiStructuredOutputRoute: typeof ApiStructuredOutputRoute @@ -648,6 +674,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof SandboxesRouteImport parentRoute: typeof rootRouteImport } + '/resumable': { + id: '/resumable' + path: '/resumable' + fullPath: '/resumable' + preLoaderRoute: typeof ResumableRouteImport + parentRoute: typeof rootRouteImport + } '/realtime': { id: '/realtime' path: '/realtime' @@ -816,6 +849,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiSandboxTriageRouteImport parentRoute: typeof rootRouteImport } + '/api/resumable': { + id: '/api/resumable' + path: '/api/resumable' + fullPath: '/api/resumable' + preLoaderRoute: typeof ApiResumableRouteImport + parentRoute: typeof rootRouteImport + } '/api/mcp-status': { id: '/api/mcp-status' path: '/api/mcp-status' @@ -948,6 +988,7 @@ const rootRouteChildren: RootRouteChildren = { McpAppsRoute: McpAppsRoute, McpDemoRoute: McpDemoRoute, RealtimeRoute: RealtimeRoute, + ResumableRoute: ResumableRoute, SandboxesRoute: SandboxesRoute, ServerFnChatRoute: ServerFnChatRoute, ThreadsRoute: ThreadsRoute, @@ -963,6 +1004,7 @@ const rootRouteChildren: RootRouteChildren = { ApiMcpManualRoute: ApiMcpManualRoute, ApiMcpPoolRoute: ApiMcpPoolRoute, ApiMcpStatusRoute: ApiMcpStatusRoute, + ApiResumableRoute: ApiResumableRoute, ApiSandboxTriageRoute: ApiSandboxTriageRoute, ApiStructuredChatRoute: ApiStructuredChatRoute, ApiStructuredOutputRoute: ApiStructuredOutputRoute, diff --git a/examples/ts-react-chat/src/routes/api.resumable.ts b/examples/ts-react-chat/src/routes/api.resumable.ts new file mode 100644 index 000000000..0a4788420 --- /dev/null +++ b/examples/ts-react-chat/src/routes/api.resumable.ts @@ -0,0 +1,67 @@ +import { createFileRoute } from '@tanstack/react-router' +import { + chat, + chatParamsFromRequestBody, + memoryStream, + toServerSentEventsResponse, +} from '@tanstack/ai' +import { openaiText } from '@tanstack/ai-openai' +import type { StreamChunk } from '@tanstack/ai' + +/** + * Resumable-streams demo endpoint. + * + * The `memoryStream(request)` durability sink records every chunk to an ordered + * log before delivery and tags each SSE event with an opaque `id:` offset, so a + * client can reconnect (`Last-Event-ID`) or a second tab can join (`?offset=-1`) + * without re-running the model. + * + * `memoryStream` is process-local — fine for a single-process demo. In + * production, swap it for `durableStream(request, { server })` from + * `@tanstack/ai-durable-stream` backed by Cloudflare Durable Streams, Electric, + * or any Durable Streams protocol server. + */ + +// A join/replay never iterates the provider stream — the durability log is the +// source of truth — so the GET handler passes this placeholder. +const REPLAY_ONLY_STREAM: AsyncIterable = { + async *[Symbol.asyncIterator]() { + // intentionally empty + }, +} + +export const Route = createFileRoute('/api/resumable')({ + server: { + handlers: { + // Fresh run (no `Last-Event-ID`) → produce with the model and append to + // the log. Reconnect (client re-sends with `Last-Event-ID`) → the sink + // replays strictly after that offset and the lazy `chat()` stream below + // is never iterated. + POST: async ({ request }) => { + const abortController = new AbortController() + const params = await chatParamsFromRequestBody(await request.json()) + + const stream = chat({ + adapter: openaiText('gpt-5.5'), + messages: params.messages, + threadId: params.threadId, + runId: params.runId, + abortController, + }) + + return toServerSentEventsResponse(stream, { + durability: { adapter: memoryStream(request) }, + abortController, + }) + }, + + // Join an in-flight or finished run from the start (`?offset=-1&runId=…`), + // replaying the ordered log. Read-only — no messages are sent. + GET: ({ request }) => { + return toServerSentEventsResponse(REPLAY_ONLY_STREAM, { + durability: { adapter: memoryStream(request) }, + }) + }, + }, + }, +}) diff --git a/examples/ts-react-chat/src/routes/resumable.tsx b/examples/ts-react-chat/src/routes/resumable.tsx new file mode 100644 index 000000000..650d83fd8 --- /dev/null +++ b/examples/ts-react-chat/src/routes/resumable.tsx @@ -0,0 +1,159 @@ +import { createFileRoute } from '@tanstack/react-router' +import { useRef, useState } from 'react' +import { fetchServerSentEvents } from '@tanstack/ai-client' +import type { ModelMessage, StreamChunk } from '@tanstack/ai' + +export const Route = createFileRoute('/resumable')({ + component: ResumablePage, +}) + +// One resumable connection to the durability-backed endpoint. `connect` starts +// a run; `joinRun` replays an existing run from the start off the server's log. +const connection = fetchServerSentEvents('/api/resumable') + +/** Append every TEXT_MESSAGE_CONTENT delta from a chunk stream to React state. */ +async function drainInto( + chunks: AsyncIterable, + onText: (delta: string) => void, +): Promise { + for await (const chunk of chunks) { + if (chunk.type === 'TEXT_MESSAGE_CONTENT') onText(chunk.delta) + } +} + +function randomId(prefix: string): string { + return `${prefix}-${crypto.randomUUID()}` +} + +function ResumablePage() { + const [prompt, setPrompt] = useState('Write a short haiku about durable streams.') + const [runId, setRunId] = useState('') + const [producerText, setProducerText] = useState('') + const [producerStatus, setProducerStatus] = useState<'idle' | 'streaming' | 'done' | 'error'>('idle') + const producerAbort = useRef(null) + + const [joinId, setJoinId] = useState('') + const [joinText, setJoinText] = useState('') + const [joinStatus, setJoinStatus] = useState<'idle' | 'streaming' | 'done' | 'error'>('idle') + const joinAbort = useRef(null) + + async function startRun() { + producerAbort.current?.abort() + const controller = new AbortController() + producerAbort.current = controller + + const nextRunId = randomId('run') + const threadId = randomId('thread') + setRunId(nextRunId) + setJoinId(nextRunId) + setProducerText('') + setProducerStatus('streaming') + + const messages: Array = [{ role: 'user', content: prompt }] + try { + await drainInto( + connection.connect(messages, undefined, controller.signal, { + threadId, + runId: nextRunId, + }), + (delta) => setProducerText((prev) => prev + delta), + ) + setProducerStatus('done') + } catch { + if (!controller.signal.aborted) setProducerStatus('error') + } + } + + async function joinRun() { + if (!joinId) return + joinAbort.current?.abort() + const controller = new AbortController() + joinAbort.current = controller + + setJoinText('') + setJoinStatus('streaming') + try { + await drainInto(connection.joinRun(joinId, controller.signal), (delta) => + setJoinText((prev) => prev + delta), + ) + setJoinStatus('done') + } catch { + if (!controller.signal.aborted) setJoinStatus('error') + } + } + + return ( +
+

Resumable streams

+

+ Start a run, then join it by run ID — here, in a second + tab, or after the original tab reloads. The join replays the same + response from the server’s durability log without re-running the + model. Because the connection is durable, a dropped socket also + auto-reconnects with Last-Event-ID. +

+ +
+

1. Start a run

+