From 41a6e195a8f61a95f3fdaee6033700b4ce034b97 Mon Sep 17 00:00:00 2001 From: Brandon Julio Thenaro Date: Sat, 1 Aug 2026 15:17:25 +0700 Subject: [PATCH] Add OpenAI prompt cache breakpoints --- .../add-openai-prompt-cache-breakpoints.md | 6 + packages/ai/openai/src/OpenAiLanguageModel.ts | 108 +++++++++++- packages/ai/openai/src/OpenAiSchema.ts | 33 +++- .../openai/test/OpenAiLanguageModel.test.ts | 156 ++++++++++++++++++ packages/effect/src/unstable/ai/Prompt.ts | 2 +- 5 files changed, 293 insertions(+), 12 deletions(-) create mode 100644 .changeset/add-openai-prompt-cache-breakpoints.md diff --git a/.changeset/add-openai-prompt-cache-breakpoints.md b/.changeset/add-openai-prompt-cache-breakpoints.md new file mode 100644 index 00000000000..dfef155aa7a --- /dev/null +++ b/.changeset/add-openai-prompt-cache-breakpoints.md @@ -0,0 +1,6 @@ +--- +"@effect/ai-openai": patch +"effect": patch +--- + +Support explicit OpenAI prompt-cache breakpoints on Responses API input content and correct message constructor provider option types. diff --git a/packages/ai/openai/src/OpenAiLanguageModel.ts b/packages/ai/openai/src/OpenAiLanguageModel.ts index e8559fea08e..bcf2d11a192 100644 --- a/packages/ai/openai/src/OpenAiLanguageModel.ts +++ b/packages/ai/openai/src/OpenAiLanguageModel.ts @@ -124,6 +124,45 @@ export class Config extends Context.Service< // ============================================================================= declare module "effect/unstable/ai/Prompt" { + /** + * OpenAI-specific options for system messages. + * + * **Details** + * + * A prompt-cache breakpoint is placed on the system message's input text block. + * + * @category request + * @since 4.0.0 + */ + export interface SystemMessageOptions extends ProviderOptions { + readonly openai?: { + /** + * Marks the end of reusable prompt content eligible for caching. + */ + readonly promptCacheBreakpoint?: typeof OpenAiSchema.PromptCacheBreakpoint.Encoded | null + } | null + } + + /** + * OpenAI-specific options for user messages. + * + * **Details** + * + * A message-level prompt-cache breakpoint is used as a fallback for the last + * content part when that part has no breakpoint of its own. + * + * @category request + * @since 4.0.0 + */ + export interface UserMessageOptions extends ProviderOptions { + readonly openai?: { + /** + * Marks the end of reusable prompt content eligible for caching. + */ + readonly promptCacheBreakpoint?: typeof OpenAiSchema.PromptCacheBreakpoint.Encoded | null + } | null + } + /** * OpenAI-specific options for file prompt parts. * @@ -139,6 +178,10 @@ declare module "effect/unstable/ai/Prompt" { * The detail level of the image to be sent to the model. One of `high`, `low`, or `auto`. Defaults to `auto`. */ readonly imageDetail?: ImageDetail | null + /** + * Marks the end of reusable prompt content eligible for caching. + */ + readonly promptCacheBreakpoint?: typeof OpenAiSchema.PromptCacheBreakpoint.Encoded | null } | null } @@ -241,6 +284,10 @@ declare module "effect/unstable/ai/Prompt" { * A list of annotations that apply to the output text. */ readonly annotations?: ReadonlyArray | null + /** + * Marks the end of reusable prompt content eligible for caching. + */ + readonly promptCacheBreakpoint?: typeof OpenAiSchema.PromptCacheBreakpoint.Encoded | null } | null } } @@ -814,9 +861,15 @@ const prepareMessages = Effect.fnUntraced( for (const message of prompt.content) { switch (message.role) { case "system": { + const prompt_cache_breakpoint = getPromptCacheBreakpoint(message) + messages.push({ role: getSystemMessageMode(config.model as string), - content: [{ type: "input_text", text: message.content }] + content: [{ + type: "input_text", + text: message.content, + ...(Predicate.isNotNull(prompt_cache_breakpoint) ? { prompt_cache_breakpoint } : undefined) + }] }) break } @@ -826,10 +879,17 @@ const prepareMessages = Effect.fnUntraced( for (let index = 0; index < message.content.length; index++) { const part = message.content[index] + const prompt_cache_breakpoint = getPromptCacheBreakpoint(part) ?? ( + index === message.content.length - 1 ? getPromptCacheBreakpoint(message) : null + ) switch (part.type) { case "text": { - content.push({ type: "input_text", text: part.text }) + content.push({ + type: "input_text", + text: part.text, + ...(Predicate.isNotNull(prompt_cache_breakpoint) ? { prompt_cache_breakpoint } : undefined) + }) break } @@ -839,32 +899,60 @@ const prepareMessages = Effect.fnUntraced( const mediaType = part.mediaType === "image/*" ? "image/jpeg" : part.mediaType if (typeof part.data === "string" && isFileId(part.data, config)) { - content.push({ type: "input_image", file_id: part.data, detail }) + content.push({ + type: "input_image", + file_id: part.data, + detail, + ...(Predicate.isNotNull(prompt_cache_breakpoint) ? { prompt_cache_breakpoint } : undefined) + }) } if (part.data instanceof URL) { - content.push({ type: "input_image", image_url: part.data.toString(), detail }) + content.push({ + type: "input_image", + image_url: part.data.toString(), + detail, + ...(Predicate.isNotNull(prompt_cache_breakpoint) ? { prompt_cache_breakpoint } : undefined) + }) } if (part.data instanceof Uint8Array) { const base64 = Encoding.encodeBase64(part.data) const imageUrl = `data:${mediaType};base64,${base64}` - content.push({ type: "input_image", image_url: imageUrl, detail }) + content.push({ + type: "input_image", + image_url: imageUrl, + detail, + ...(Predicate.isNotNull(prompt_cache_breakpoint) ? { prompt_cache_breakpoint } : undefined) + }) } } else if (part.mediaType === "application/pdf") { if (typeof part.data === "string" && isFileId(part.data, config)) { - content.push({ type: "input_file", file_id: part.data }) + content.push({ + type: "input_file", + file_id: part.data, + ...(Predicate.isNotNull(prompt_cache_breakpoint) ? { prompt_cache_breakpoint } : undefined) + }) } if (part.data instanceof URL) { - content.push({ type: "input_file", file_url: part.data.toString() }) + content.push({ + type: "input_file", + file_url: part.data.toString(), + ...(Predicate.isNotNull(prompt_cache_breakpoint) ? { prompt_cache_breakpoint } : undefined) + }) } if (part.data instanceof Uint8Array) { const base64 = Encoding.encodeBase64(part.data) const fileName = part.fileName ?? `part-${index}.pdf` const fileData = `data:application/pdf;base64,${base64}` - content.push({ type: "input_file", filename: fileName, file_data: fileData }) + content.push({ + type: "input_file", + filename: fileName, + file_data: fileData, + ...(Predicate.isNotNull(prompt_cache_breakpoint) ? { prompt_cache_breakpoint } : undefined) + }) } } else { return yield* AiError.make({ @@ -2877,6 +2965,10 @@ const getEncryptedContent = ( const getImageDetail = (part: Prompt.FilePart): ImageDetail => part.options.openai?.imageDetail ?? "auto" +const getPromptCacheBreakpoint = ( + input: Prompt.SystemMessage | Prompt.UserMessage | Prompt.TextPart | Prompt.FilePart +): typeof OpenAiSchema.PromptCacheBreakpoint.Encoded | null => input.options.openai?.promptCacheBreakpoint ?? null + const makeItemIdMetadata = (itemId: string | undefined) => Predicate.isNotUndefined(itemId) ? { itemId } : {} const makeEncryptedContentMetadata = (encryptedContent: string | null | undefined) => diff --git a/packages/ai/openai/src/OpenAiSchema.ts b/packages/ai/openai/src/OpenAiSchema.ts index 4afd616d228..dfd3fa458e7 100644 --- a/packages/ai/openai/src/OpenAiSchema.ts +++ b/packages/ai/openai/src/OpenAiSchema.ts @@ -18,6 +18,29 @@ const MessageRole = Schema.Literals(["system", "developer", "user", "assistant"] const ImageDetail = Schema.Literals(["low", "high", "auto"]) +/** + * Schema for an explicit prompt-cache breakpoint on an OpenAI input content block. + * + * **Details** + * + * The breakpoint includes the marked block and all preceding prompt content in + * the candidate cached prefix. OpenAI currently accepts only `"explicit"`. + * + * @category schemas + * @since 4.0.0 + */ +export const PromptCacheBreakpoint = Schema.Struct({ + mode: Schema.Literal("explicit") +}) + +/** + * Explicit prompt-cache breakpoint attached to an OpenAI input content block. + * + * @category models + * @since 4.0.0 + */ +export type PromptCacheBreakpoint = typeof PromptCacheBreakpoint.Type + /** * Schema for optional `include` values supported by the local handwritten * Responses client schema. @@ -75,14 +98,16 @@ export type MessageStatus = typeof MessageStatus.Type const InputTextContent = Schema.Struct({ type: Schema.Literal("input_text"), - text: Schema.String + text: Schema.String, + prompt_cache_breakpoint: Schema.optionalKey(PromptCacheBreakpoint) }) const InputImageContent = Schema.Struct({ type: Schema.Literal("input_image"), image_url: Schema.optionalKey(Schema.NullOr(Schema.String)), file_id: Schema.optionalKey(Schema.NullOr(Schema.String)), - detail: Schema.optionalKey(Schema.NullOr(ImageDetail)) + detail: Schema.optionalKey(Schema.NullOr(ImageDetail)), + prompt_cache_breakpoint: Schema.optionalKey(PromptCacheBreakpoint) }) const InputFileContent = Schema.Struct({ @@ -90,7 +115,8 @@ const InputFileContent = Schema.Struct({ file_id: Schema.optionalKey(Schema.NullOr(Schema.String)), filename: Schema.optionalKey(Schema.String), file_url: Schema.optionalKey(Schema.String), - file_data: Schema.optionalKey(Schema.String) + file_data: Schema.optionalKey(Schema.String), + prompt_cache_breakpoint: Schema.optionalKey(PromptCacheBreakpoint) }) /** @@ -99,6 +125,7 @@ const InputFileContent = Schema.Struct({ * **Details** * * Accepted block variants are `input_text`, `input_image`, and `input_file`. + * Each variant can carry an explicit prompt-cache breakpoint. * * @see {@link InputItem} for request input item shapes that can contain these content blocks * diff --git a/packages/ai/openai/test/OpenAiLanguageModel.test.ts b/packages/ai/openai/test/OpenAiLanguageModel.test.ts index 1250b88b922..8a4605097f6 100644 --- a/packages/ai/openai/test/OpenAiLanguageModel.test.ts +++ b/packages/ai/openai/test/OpenAiLanguageModel.test.ts @@ -31,6 +31,162 @@ describe("OpenAiLanguageModel", () => { describe("generateText", () => { describe("message preparation", () => { + describe("prompt cache breakpoints", () => { + const breakpoint = { mode: "explicit" } as const + + it.effect("places a breakpoint on system input_text", () => + Effect.gen(function*() { + yield* LanguageModel.generateText({ + prompt: Prompt.make([ + Prompt.systemMessage({ + content: "Stable instructions", + options: { openai: { promptCacheBreakpoint: breakpoint } } + }), + { role: "user", content: "Hello" } + ]) + }).pipe(Effect.provide(OpenAiLanguageModel.model("gpt-5.6"))) + + const requests = yield* MockHttpClient.requests + const body = yield* getRequestBody(requests[0]) + const systemMessage = body.input.find((message: any) => message.role === "developer") + + deepStrictEqual(systemMessage.content, [{ + type: "input_text", + text: "Stable instructions", + prompt_cache_breakpoint: breakpoint + }]) + }).pipe(Effect.provide(makeTestLayer({ body: { model: "gpt-5.6" } })))) + + it.effect("places a breakpoint on a user input_text part", () => + Effect.gen(function*() { + yield* LanguageModel.generateText({ + prompt: Prompt.make([{ + role: "user", + content: [Prompt.textPart({ + text: "Stable context", + options: { openai: { promptCacheBreakpoint: breakpoint } } + })] + }]) + }).pipe(Effect.provide(OpenAiLanguageModel.model("gpt-5.6"))) + + const requests = yield* MockHttpClient.requests + const body = yield* getRequestBody(requests[0]) + + deepStrictEqual(body.input[0].content, [{ + type: "input_text", + text: "Stable context", + prompt_cache_breakpoint: breakpoint + }]) + }).pipe(Effect.provide(makeTestLayer({ body: { model: "gpt-5.6" } })))) + + it.effect("places a breakpoint on an input_image part", () => + Effect.gen(function*() { + yield* LanguageModel.generateText({ + prompt: Prompt.make([{ + role: "user", + content: [Prompt.filePart({ + mediaType: "image/png", + data: new URL("https://example.com/stable.png"), + options: { openai: { promptCacheBreakpoint: breakpoint } } + })] + }]) + }).pipe(Effect.provide(OpenAiLanguageModel.model("gpt-5.6"))) + + const requests = yield* MockHttpClient.requests + const body = yield* getRequestBody(requests[0]) + + deepStrictEqual(body.input[0].content, [{ + type: "input_image", + image_url: "https://example.com/stable.png", + detail: "auto", + prompt_cache_breakpoint: breakpoint + }]) + }).pipe(Effect.provide(makeTestLayer({ body: { model: "gpt-5.6" } })))) + + it.effect("uses a user message breakpoint on the last input_file part", () => + Effect.gen(function*() { + yield* LanguageModel.generateText({ + prompt: Prompt.make([{ + role: "user", + content: [ + Prompt.textPart({ text: "Read this file" }), + Prompt.filePart({ + mediaType: "application/pdf", + data: new URL("https://example.com/stable.pdf") + }) + ], + options: { openai: { promptCacheBreakpoint: breakpoint } } + }]) + }).pipe(Effect.provide(OpenAiLanguageModel.model("gpt-5.6"))) + + const requests = yield* MockHttpClient.requests + const body = yield* getRequestBody(requests[0]) + + deepStrictEqual(body.input[0].content, [ + { type: "input_text", text: "Read this file" }, + { + type: "input_file", + file_url: "https://example.com/stable.pdf", + prompt_cache_breakpoint: breakpoint + } + ]) + }).pipe(Effect.provide(makeTestLayer({ body: { model: "gpt-5.6" } })))) + + it.effect("prefers a part breakpoint over the user message fallback", () => + Effect.gen(function*() { + const messageOptions = { + openai: { + get promptCacheBreakpoint(): never { + throw new Error("message fallback should not be read") + } + } + } + + yield* LanguageModel.generateText({ + prompt: Prompt.fromMessages([Prompt.userMessage({ + content: [Prompt.textPart({ + text: "Stable context", + options: { openai: { promptCacheBreakpoint: breakpoint } } + })], + options: messageOptions + })]) + }).pipe(Effect.provide(OpenAiLanguageModel.model("gpt-5.6"))) + + const requests = yield* MockHttpClient.requests + const body = yield* getRequestBody(requests[0]) + + deepStrictEqual(body.input[0].content[0].prompt_cache_breakpoint, breakpoint) + }).pipe(Effect.provide(makeTestLayer({ body: { model: "gpt-5.6" } })))) + + it.effect("omits breakpoints when no OpenAI provider option is set", () => + Effect.gen(function*() { + yield* LanguageModel.generateText({ + prompt: Prompt.make([{ + role: "system", + content: "Instructions" + }, { + role: "user", + content: [ + Prompt.textPart({ text: "Question" }), + Prompt.filePart({ + mediaType: "image/png", + data: new URL("https://example.com/image.png") + }) + ] + }]) + }).pipe(Effect.provide(OpenAiLanguageModel.model("gpt-5.6"))) + + const requests = yield* MockHttpClient.requests + const body = yield* getRequestBody(requests[0]) + + for (const message of body.input) { + if (Array.isArray(message.content)) { + assert.isTrue(message.content.every((part: any) => !("prompt_cache_breakpoint" in part))) + } + } + }).pipe(Effect.provide(makeTestLayer({ body: { model: "gpt-5.6" } })))) + }) + describe("system messages", () => { it.effect("uses system role for standard models", () => Effect.gen(function*() { diff --git a/packages/effect/src/unstable/ai/Prompt.ts b/packages/effect/src/unstable/ai/Prompt.ts index 59896694fab..309ef719f4a 100644 --- a/packages/effect/src/unstable/ai/Prompt.ts +++ b/packages/effect/src/unstable/ai/Prompt.ts @@ -1088,7 +1088,7 @@ export type MessageConstructorParams = Omit