From 57c9bb0d602f31e1d9b7ec1688eb93e30b20e872 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Wed, 26 Aug 2026 19:52:09 -0400 Subject: [PATCH] feat(studio): give an agent eyes with studio_frame Renders the composition to a PNG at a given time and returns the URL. This is what turns the tool set from a remote control into a loop: author a change, capture the instant it affects, look, adjust. No agent can judge motion from source, because "what does this look like at 2.4 seconds" is not a question a file answers. Reuses Studio's existing capture endpoint via `buildFrameCaptureUrl` rather than inventing a second one. Two things this does not fake: It reports the time the playhead LANDED on, not the time requested. The player clamps, so those differ at the ends, and attaching the wrong time to a frame is how an agent draws a confident wrong conclusion about motion. It waits before capturing, by default 150ms. The frame is rendered from the file on disk, and the render cache is cleared by a file watcher with a 40ms write-stability threshold, so a capture that beats the watcher renders the PRE-edit composition. That exact staleness was a real bug here once. An agent reading a stale frame as "my edit failed" would thrash, so the wait is on by default, `settleMs` makes it tunable, and the tool description names the failure rather than leaving it to be rediscovered. It probes with HEAD before returning, so a URL that 404s comes back as a failure with a hint instead of as a link the agent cannot render. --- .../studio/src/webmcp/StudioAgentTools.tsx | 23 ++- .../src/webmcp/tools/frameTools.test.ts | 146 ++++++++++++++++++ .../studio/src/webmcp/tools/frameTools.ts | 133 ++++++++++++++++ .../src/webmcp/useStudioAgentTools.test.tsx | 11 +- .../studio/src/webmcp/useStudioAgentTools.ts | 19 ++- 5 files changed, 327 insertions(+), 5 deletions(-) create mode 100644 packages/studio/src/webmcp/tools/frameTools.test.ts create mode 100644 packages/studio/src/webmcp/tools/frameTools.ts diff --git a/packages/studio/src/webmcp/StudioAgentTools.tsx b/packages/studio/src/webmcp/StudioAgentTools.tsx index 0130ce1a9a..e26612c175 100644 --- a/packages/studio/src/webmcp/StudioAgentTools.tsx +++ b/packages/studio/src/webmcp/StudioAgentTools.tsx @@ -57,8 +57,29 @@ export function StudioAgentTools() { isPlaying: player.isPlaying, }; }, + getProjectId: () => projectId, + getCompositionPath: () => activeCompPath, + // HEAD, not GET: the tool only needs to know the frame renders. Pulling + // the PNG here would download it once for nothing, since the agent + // fetches the URL itself. + probeFrame: async (url) => { + try { + const response = await fetch(url, { method: "HEAD" }); + return { ok: response.ok, status: response.status }; + } catch { + return { ok: false, status: 0 }; + } + }, + wait: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), }), - [getSnapshot, previewIframeRef, buildDomSelectionFromTarget, applyDomSelection], + [ + getSnapshot, + previewIframeRef, + buildDomSelectionFromTarget, + applyDomSelection, + projectId, + activeCompPath, + ], ); useStudioAgentTools(deps); diff --git a/packages/studio/src/webmcp/tools/frameTools.test.ts b/packages/studio/src/webmcp/tools/frameTools.test.ts new file mode 100644 index 0000000000..e98d5c2e46 --- /dev/null +++ b/packages/studio/src/webmcp/tools/frameTools.test.ts @@ -0,0 +1,146 @@ +// @vitest-environment jsdom +import { describe, expect, it, vi } from "vitest"; +import { studioFrame, type FrameToolDeps, type StudioFrameResult } from "./frameTools"; +import type { ToolFailure, ToolResult } from "../toolResult"; + +function frameDeps(overrides: Partial = {}): FrameToolDeps { + return { + getProjectId: () => "demo", + getCompositionPath: () => "index.html", + readPlayhead: () => ({ currentTime: 2.4, duration: 10, isPlaying: false }), + requestSeek: () => undefined, + probeFrame: async () => ({ ok: true, status: 200 }), + wait: async () => undefined, + ...overrides, + }; +} + +function expectOk(result: ToolResult): { ok: true } & T { + if (!result.ok) throw new Error(`expected ok, got ${JSON.stringify(result)}`); + return result; +} + +function expectFailure(result: ToolResult): ToolFailure { + if (result.ok) throw new Error(`expected failure, got ${JSON.stringify(result)}`); + return result; +} + +describe("studioFrame", () => { + it("returns a URL for the composition at the playhead", async () => { + const result = await studioFrame(frameDeps()); + + const ok = expectOk(result); + expect(ok.time).toBe(2.4); + expect(ok.compositionPath).toBe("index.html"); + expect(ok.url).toContain("/thumbnail/"); + expect(ok.url).toContain("t=2.400"); + expect(ok.url).toContain("format=png"); + }); + + it("seeks first when given a time", async () => { + const requestSeek = vi.fn(); + + await studioFrame(frameDeps({ requestSeek }), { time: 5 }); + + expect(requestSeek).toHaveBeenCalledWith(5); + }); + + it("captures where the playhead LANDED, not what was asked for", async () => { + // The player clamps. Reporting the request would attach the wrong time to + // the frame, and an agent judging motion would draw the wrong conclusion. + const result = await studioFrame( + frameDeps({ readPlayhead: () => ({ currentTime: 10, duration: 10, isPlaying: false }) }), + { time: 999 }, + ); + + const ok = expectOk(result); + expect(ok.time).toBe(10); + expect(ok.url).toContain("t=10.000"); + }); + + it("waits before capturing, so a just-made edit is in the frame", async () => { + // The render cache is cleared by a file watcher with a write-stability + // threshold. Capturing faster than that renders the PRE-edit composition. + const wait = vi.fn(async () => undefined); + const order: string[] = []; + + await studioFrame( + frameDeps({ + wait: async (ms) => { + order.push(`wait:${ms}`); + await wait(); + }, + probeFrame: async () => { + order.push("probe"); + return { ok: true, status: 200 }; + }, + }), + ); + + expect(order).toEqual(["wait:150", "probe"]); + }); + + it("honours a caller-supplied settle time and reports it", async () => { + const result = await studioFrame(frameDeps(), { settleMs: 800 }); + + expect(expectOk(result).settledMs).toBe(800); + }); + + it("clamps an absurd settle time rather than hanging", async () => { + const result = await studioFrame(frameDeps(), { settleMs: 10 * 60 * 1000 }); + + expect(expectOk(result).settledMs).toBe(5000); + }); + + it("falls back to the default for a nonsense settle time", async () => { + for (const settleMs of [-1, Number.NaN]) { + const result = await studioFrame(frameDeps(), { settleMs }); + expect(expectOk(result).settledMs).toBe(150); + } + }); + + it("skips the wait entirely when asked for zero", async () => { + const wait = vi.fn(async () => undefined); + + await studioFrame(frameDeps({ wait }), { settleMs: 0 }); + + expect(wait).not.toHaveBeenCalled(); + }); + + it("reports a renderer failure instead of handing back a dead URL", async () => { + const result = expectFailure( + await studioFrame(frameDeps({ probeFrame: async () => ({ ok: false, status: 500 }) })), + ); + + expect(result.kind).toBe("failed"); + expect(result.reason).toContain("500"); + expect(result.hint).toBeDefined(); + }); + + it("fails when no project is open, before touching the renderer", async () => { + const probeFrame = vi.fn(); + + const result = expectFailure( + await studioFrame(frameDeps({ getProjectId: () => null, probeFrame })), + ); + + expect(result.kind).toBe("blocked"); + expect(probeFrame).not.toHaveBeenCalled(); + }); + + it("rejects a negative or non-finite time without seeking", async () => { + const requestSeek = vi.fn(); + + for (const time of [-1, Number.NaN, Number.POSITIVE_INFINITY]) { + const result = expectFailure(await studioFrame(frameDeps({ requestSeek }), { time })); + expect(result.kind).toBe("invalid"); + } + expect(requestSeek).not.toHaveBeenCalled(); + }); + + it("captures the master composition when no path is active", async () => { + const result = await studioFrame(frameDeps({ getCompositionPath: () => null })); + + expect(expectOk(result).compositionPath).toBe("index.html"); + }); +}); diff --git a/packages/studio/src/webmcp/tools/frameTools.ts b/packages/studio/src/webmcp/tools/frameTools.ts new file mode 100644 index 0000000000..839e4c231d --- /dev/null +++ b/packages/studio/src/webmcp/tools/frameTools.ts @@ -0,0 +1,133 @@ +/** + * `studio_frame`: the eyes. + * + * Without this the tool set is a remote control. With it an agent can author a + * change, look at the instant it affects, judge it, and adjust. That loop is the + * one thing source alone cannot support, because "what does this look like at + * 2.4 seconds" is not a question a file can answer. + * + * Reuses Studio's existing capture endpoint (`utils/frameCapture`) rather than + * inventing a second one. The server renders the composition with Puppeteer, so + * the frame reflects the file on disk, not the live preview DOM. + */ + +import { buildFrameCaptureUrl } from "../../utils/frameCapture"; +import { toolFailure, toolOk, type ToolResult } from "../toolResult"; + +export interface FrameToolDeps { + getProjectId: () => string | null; + getCompositionPath: () => string | null; + readPlayhead: () => { currentTime: number; duration: number; isPlaying: boolean }; + requestSeek: (time: number) => void; + /** Confirms the URL renders. Injected so tests need no network. */ + probeFrame: (url: string) => Promise<{ ok: boolean; status: number }>; + wait: (ms: number) => Promise; +} + +export interface StudioFrameResult { + /** Fetch this to see the frame. A PNG of the composition at `time`. */ + url: string; + time: number; + compositionPath: string; + /** How long the tool waited for a pending write to settle before capturing. */ + settledMs: number; +} + +export interface StudioFrameInput { + /** Seconds. Omit to capture wherever the playhead already is. */ + time?: number; + /** + * Milliseconds to wait before capturing, so a just-written edit is visible. + * See the staleness note in the description. + */ + settleMs?: number; +} + +/** + * Long enough to cover the project watcher's 40ms write-stability threshold + * plus filesystem latency, short enough not to be felt. This is the mitigation + * for a real, previously-fixed bug: the preview signature is invalidated by a + * file watcher, and a capture that beats the watcher renders the PRE-edit + * composition. An agent reading that as "my edit failed" would thrash. + */ +const DEFAULT_SETTLE_MS = 150; +const MAX_SETTLE_MS = 5_000; + +export async function studioFrame( + deps: FrameToolDeps, + input: StudioFrameInput = {}, +): Promise> { + const projectId = deps.getProjectId(); + if (!projectId) { + return toolFailure("blocked", "no project is open"); + } + + if (input.time !== undefined) { + if (typeof input.time !== "number" || !Number.isFinite(input.time) || input.time < 0) { + return toolFailure("invalid", "time must be a non-negative, finite number of seconds"); + } + deps.requestSeek(input.time); + } + + const settledMs = clampSettle(input.settleMs); + if (settledMs > 0) await deps.wait(settledMs); + + // Capture whatever the playhead now reads, rather than what was requested: + // the player clamps, so those can differ and the frame belongs to the former. + const { currentTime } = deps.readPlayhead(); + const compositionPath = deps.getCompositionPath(); + const url = buildFrameCaptureUrl({ projectId, compositionPath, currentTime }); + + const probe = await deps.probeFrame(url); + if (!probe.ok) { + return toolFailure( + "failed", + `the renderer returned ${probe.status} for this frame`, + "The composition may not build. Try `hyperframes check`.", + ); + } + + return toolOk({ + url, + time: currentTime, + compositionPath: compositionPath ?? "index.html", + settledMs, + }); +} + +function clampSettle(requested: number | undefined): number { + if (requested === undefined) return DEFAULT_SETTLE_MS; + if (typeof requested !== "number" || !Number.isFinite(requested) || requested < 0) { + return DEFAULT_SETTLE_MS; + } + return Math.min(requested, MAX_SETTLE_MS); +} + +export const STUDIO_FRAME_INPUT_SCHEMA = { + type: "object", + properties: { + time: { + type: "number", + minimum: 0, + description: "Seconds. Omit to capture wherever the playhead already is.", + }, + settleMs: { + type: "integer", + minimum: 0, + maximum: MAX_SETTLE_MS, + description: `Wait this long before capturing so a just-made edit is included. Default ${DEFAULT_SETTLE_MS}.`, + }, + }, + additionalProperties: false, +} as const; + +export const STUDIO_FRAME_DESCRIPTION = [ + "Render the composition to a PNG at a given time and return its URL, so you can", + "SEE the result instead of inferring it from source. Use this to judge a change:", + "edit, capture the instant it affects, look, adjust.", + "The frame is rendered from the file on disk, not the live preview.", + "A capture taken immediately after an edit can therefore predate that edit, because", + "the render cache is cleared by a file watcher. The tool waits briefly to cover that;", + "raise `settleMs` if a frame still looks stale, rather than concluding the edit failed.", + "Returns `ok: true` with `url` and the `time` actually captured, or `ok: false`.", +].join(" "); diff --git a/packages/studio/src/webmcp/useStudioAgentTools.test.tsx b/packages/studio/src/webmcp/useStudioAgentTools.test.tsx index d27665c4a7..d9c7dd4081 100644 --- a/packages/studio/src/webmcp/useStudioAgentTools.test.tsx +++ b/packages/studio/src/webmcp/useStudioAgentTools.test.tsx @@ -38,6 +38,10 @@ function deps(overrides: Partial = {}): StudioAgentToolsDe applySelection: () => undefined, requestSeek: () => undefined, readPlayhead: () => ({ currentTime: 0, duration: 10, isPlaying: false }), + getProjectId: () => "demo", + getCompositionPath: () => "index.html", + probeFrame: async () => ({ ok: true, status: 200 }), + wait: async () => undefined, ...overrides, }; } @@ -102,6 +106,7 @@ describe("useStudioAgentTools", () => { "studio_look", "studio_select", "studio_seek", + "studio_frame", ]); expect(trackEvent).toHaveBeenCalledWith("webmcp.native_present"); }); @@ -116,14 +121,14 @@ describe("useStudioAgentTools", () => { await act(async () => { harness = mountTools(deps({ getSnapshot: () => snapshot() })); }); - expect(registerTool).toHaveBeenCalledTimes(3); + expect(registerTool).toHaveBeenCalledTimes(4); await act(async () => { harness?.rerenderWith(deps({ getSnapshot: () => snapshot({ currentTime: 5 }) })); harness?.rerenderWith(deps({ getSnapshot: () => snapshot({ currentTime: 6 }) })); }); - expect(registerTool).toHaveBeenCalledTimes(3); + expect(registerTool).toHaveBeenCalledTimes(4); }); it("executes against the LATEST deps, not the ones present at registration", async () => { @@ -196,7 +201,7 @@ describe("useStudioAgentTools", () => { mountTools(deps({ getSnapshot: () => snapshot() })); }); - expect(registerTool).toHaveBeenCalledTimes(3); + expect(registerTool).toHaveBeenCalledTimes(4); }); it("reports a non-abort registration failure through production telemetry", async () => { diff --git a/packages/studio/src/webmcp/useStudioAgentTools.ts b/packages/studio/src/webmcp/useStudioAgentTools.ts index 2c67d23328..9af5b8cc61 100644 --- a/packages/studio/src/webmcp/useStudioAgentTools.ts +++ b/packages/studio/src/webmcp/useStudioAgentTools.ts @@ -25,6 +25,14 @@ import { type StudioSeekResult, type StudioSelectResult, } from "./tools/selectionTools"; +import { + studioFrame, + STUDIO_FRAME_DESCRIPTION, + STUDIO_FRAME_INPUT_SCHEMA, + type FrameToolDeps, + type StudioFrameInput, + type StudioFrameResult, +} from "./tools/frameTools"; const log = makeStudioDebugLogger("webmcp"); @@ -38,7 +46,7 @@ function reportRegistration(report: ToolRegistrationReport, native: boolean): vo } } -export interface StudioAgentToolsDeps extends SelectionToolDeps { +export interface StudioAgentToolsDeps extends SelectionToolDeps, FrameToolDeps { /** Read Studio's current state. Called per tool invocation, never cached. */ getSnapshot: () => StudioLookSnapshot; } @@ -90,6 +98,15 @@ function buildStudioTools(depsRef: { readonly current: StudioAgentToolsDeps }): studioSeek(depsRef.current, readNumberInput(input, "time")), ), }, + { + name: "studio_frame", + title: "See the composition", + description: STUDIO_FRAME_DESCRIPTION, + inputSchema: STUDIO_FRAME_INPUT_SCHEMA, + annotations: { readOnlyHint: true, untrustedContentHint: true }, + execute: (input): Promise> => + runToolBody("studio_frame", () => studioFrame(depsRef.current, input as StudioFrameInput)), + }, ]; }