From e2621d74870112ef25628e2e9a137ed82f3a1c65 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Wed, 26 Aug 2026 18:58:33 -0400 Subject: [PATCH 1/5] feat(studio): expose Studio's live state to an agentic browser Registers a `studio_look` tool on `document.modelContext`, so an agent in a browser that supports it can read what Studio knows: the open project and composition, the playhead, the human's current selection with its capabilities, and the timeline's elements with a handle for each. The API is `document.modelContext`, not `navigator.modelContext`. The latter is a polyfill compatibility shim rather than a spec member, so feature detecting it is wrong even where a published sample appears to work. Three decisions worth knowing: Registration happens ONCE per mount, with the dependencies held in a ref that every render refreshes. Depending on the handlers instead re-runs on nearly every interaction, because the DomEdit actions object changes identity with the selection and the element list. Each re-run aborts the registration signal and unregisters everything, and the spec warns that a quick unregister-then- reregister can apply an old call's arguments against the new schema. The test for this is the important one in the unit; breaking the empty dependency array fails it and nothing else. Tools resolve with a tagged result, they never reject. That is forced by the spec: a rejected `execute` has its reason discarded and the caller sees a bare UnknownError, so rejecting would guarantee the agent cannot learn why an edit failed. Elements are addressed by a minted handle, not by `TimelineElement.id`. That id is a synthesised identity, so `getElementById` misses most elements; the handle carries `data-hf-id`, else the DOM id, else a selector plus occurrence. Mounted from `EditorShell` rather than `App`, because the DomEdit contexts are only readable below `DomEditProvider` and `App.tsx` is three lines under the 600-line cap. The undo signal is reported as the shell actually exposes it, `canUndo` and a label, rather than as a revision counter. The depth lives in component-local state and is not reachable without plumbing it through the shell context, so the field says what it is instead of implying precision it does not have. Writes are not in this change. `canWrite` is optimistic and the comment says so; the write tools need a real guard against the paused-save and external- conflict states, which are not on any context this component can reach yet. --- .../studio/src/components/EditorShell.tsx | 4 + .../studio/src/utils/studioUiPreferences.ts | 9 + .../studio/src/webmcp/StudioAgentTools.tsx | 52 +++++ packages/studio/src/webmcp/handles.test.ts | 130 +++++++++++ packages/studio/src/webmcp/handles.ts | 129 +++++++++++ packages/studio/src/webmcp/registrar.test.ts | 150 +++++++++++++ packages/studio/src/webmcp/registrar.ts | 112 ++++++++++ packages/studio/src/webmcp/toolResult.ts | 67 ++++++ .../studio/src/webmcp/tools/lookTools.test.ts | 207 ++++++++++++++++++ packages/studio/src/webmcp/tools/lookTools.ts | 199 +++++++++++++++++ packages/studio/src/webmcp/types.ts | 76 +++++++ .../src/webmcp/useStudioAgentTools.test.tsx | 203 +++++++++++++++++ .../studio/src/webmcp/useStudioAgentTools.ts | 92 ++++++++ 13 files changed, 1430 insertions(+) create mode 100644 packages/studio/src/webmcp/StudioAgentTools.tsx create mode 100644 packages/studio/src/webmcp/handles.test.ts create mode 100644 packages/studio/src/webmcp/handles.ts create mode 100644 packages/studio/src/webmcp/registrar.test.ts create mode 100644 packages/studio/src/webmcp/registrar.ts create mode 100644 packages/studio/src/webmcp/toolResult.ts create mode 100644 packages/studio/src/webmcp/tools/lookTools.test.ts create mode 100644 packages/studio/src/webmcp/tools/lookTools.ts create mode 100644 packages/studio/src/webmcp/types.ts create mode 100644 packages/studio/src/webmcp/useStudioAgentTools.test.tsx create mode 100644 packages/studio/src/webmcp/useStudioAgentTools.ts diff --git a/packages/studio/src/components/EditorShell.tsx b/packages/studio/src/components/EditorShell.tsx index 472a91df64..f2ab2f3755 100644 --- a/packages/studio/src/components/EditorShell.tsx +++ b/packages/studio/src/components/EditorShell.tsx @@ -15,6 +15,7 @@ import { usePlayerStore, type TimelineElement } from "../player"; import type { BlockPreviewInfo } from "./sidebar/BlocksTab"; import type { GestureRecordingState } from "./editor/GestureRecordControl"; import { useTimelineSelectionPreviewSync } from "../hooks/useTimelineSelectionPreviewSync"; +import { StudioAgentTools } from "../webmcp/StudioAgentTools"; type RenderClipContent = ( element: TimelineElement, @@ -262,6 +263,9 @@ function EditorShellBody({ onKeyDown={handleKeyDown} tabIndex={-1} > + {/* Renders nothing; exposes Studio's state to an agentic browser. Mounted + here rather than in App because it needs the DomEdit contexts. */} + {/* Top row: [left | preview | right] — outer padding + the 8px resize seams give the panels CapCut-style separation on the dark canvas. */}
diff --git a/packages/studio/src/utils/studioUiPreferences.ts b/packages/studio/src/utils/studioUiPreferences.ts index 801a584858..e21c303bd2 100644 --- a/packages/studio/src/utils/studioUiPreferences.ts +++ b/packages/studio/src/utils/studioUiPreferences.ts @@ -34,6 +34,12 @@ export interface StudioUiPreferences { timelineZoomMode?: "fit" | "manual"; /** Manual timeline zoom percent, paired with `timelineZoomMode: "manual"`. */ timelineManualZoomPercent?: number; + /** + * Expose Studio's editing capabilities to an agentic browser as WebMCP tools. + * Absent means on: the browser still gates every actual call behind its own + * permission prompt, so "registered" is not "reachable without consent". + */ + agentToolsEnabled?: boolean; } const STUDIO_UI_PREFERENCES_KEY = "hf-studio-ui-preferences"; @@ -140,6 +146,9 @@ function readStorage(storage: Storage | null): StudioUiPreferences { ) { preferences.timelineManualZoomPercent = parsed.timelineManualZoomPercent; } + if (typeof parsed.agentToolsEnabled === "boolean") { + preferences.agentToolsEnabled = parsed.agentToolsEnabled; + } return preferences; } catch { return {}; diff --git a/packages/studio/src/webmcp/StudioAgentTools.tsx b/packages/studio/src/webmcp/StudioAgentTools.tsx new file mode 100644 index 0000000000..6d4de4d8b6 --- /dev/null +++ b/packages/studio/src/webmcp/StudioAgentTools.tsx @@ -0,0 +1,52 @@ +import { useCallback } from "react"; +import { useDomEditSelectionContext } from "../contexts/DomEditContext"; +import { useStudioShellContext } from "../contexts/StudioContext"; +import { usePlayerStore } from "../player"; +import { useStudioAgentTools } from "./useStudioAgentTools"; +import type { StudioLookSnapshot } from "./tools/lookTools"; + +/** + * Mounts Studio's WebMCP tool surface. Renders nothing. + * + * Lives inside `EditorShell` rather than `App` for two reasons: the DomEdit + * contexts are only readable below `DomEditProvider`, which `App` renders, and + * `App.tsx` sits three lines under the 600-line cap. + * + * The player store is read IMPERATIVELY through `getState()` inside the + * snapshot callback rather than subscribed to. Subscribing to `currentTime` + * would re-render this component on every animation frame during playback for + * a value nothing here displays. + */ +export function StudioAgentTools() { + const { projectId, activeCompPath, editHistory } = useStudioShellContext(); + const { domEditSelection } = useDomEditSelectionContext(); + + const getSnapshot = useCallback((): StudioLookSnapshot => { + const player = usePlayerStore.getState(); + return { + projectId, + compositionPath: activeCompPath, + currentTime: player.currentTime, + duration: player.duration, + isPlaying: player.isPlaying, + elements: player.elements, + selection: domEditSelection, + selectedElementIds: [...player.selectedElementIds], + history: { + canUndo: editHistory.canUndo, + canRedo: editHistory.canRedo, + undoLabel: editHistory.undoLabel ?? null, + redoLabel: editHistory.redoLabel ?? null, + }, + // TODO(webmcp): the save-queue and external-conflict states live on + // App's previewPersistence and externalFileChanges, which are not on any + // context this component can read. Until they are, `canWrite` is + // optimistic. The write tools land in a later unit and MUST NOT ship + // trusting this field; they need the real guard. + writeBlockedReason: null, + }; + }, [projectId, activeCompPath, domEditSelection, editHistory]); + + useStudioAgentTools({ getSnapshot }); + return null; +} diff --git a/packages/studio/src/webmcp/handles.test.ts b/packages/studio/src/webmcp/handles.test.ts new file mode 100644 index 0000000000..d164dbadff --- /dev/null +++ b/packages/studio/src/webmcp/handles.test.ts @@ -0,0 +1,130 @@ +// @vitest-environment jsdom +import { describe, expect, it } from "vitest"; +import type { TimelineElement } from "../player/store/timelineElement"; +import { + mintElementHandle, + parseElementHandle, + resolveElementHandle, + timelineElementAddress, +} from "./handles"; + +function timelineElement(overrides: Partial): TimelineElement { + return { id: "synthetic-id", tag: "div", start: 0, duration: 1, track: 0, ...overrides }; +} + +/** A separate document, standing in for the preview iframe's realm. */ +function previewDoc(html: string): Document { + const iframe = document.createElement("iframe"); + document.body.append(iframe); + const doc = iframe.contentDocument; + if (!doc) throw new Error("expected iframe document"); + doc.body.innerHTML = html; + return doc; +} + +describe("mintElementHandle", () => { + it("prefers data-hf-id, the stable patch target", () => { + const handle = mintElementHandle( + timelineElementAddress( + timelineElement({ hfId: "abc123", domId: "headline", selector: ".title" }), + ), + ); + expect(handle).toBe("hf:abc123"); + }); + + it("falls back to the DOM id when there is no hf id", () => { + expect( + mintElementHandle( + timelineElementAddress(timelineElement({ domId: "headline", selector: ".title" })), + ), + ).toBe("dom:headline"); + }); + + it("falls back to a selector with its occurrence index", () => { + expect( + mintElementHandle( + timelineElementAddress(timelineElement({ selector: ".card", selectorIndex: 2 })), + ), + ).toBe("sel:.card#2"); + }); + + it("defaults a missing occurrence index to the first match", () => { + expect(mintElementHandle(timelineElementAddress(timelineElement({ selector: ".card" })))).toBe( + "sel:.card#0", + ); + }); + + it("returns null when the element carries no way to address it", () => { + // The synthesised `id` is deliberately NOT used: it cannot resolve. + expect(mintElementHandle(timelineElementAddress(timelineElement({})))).toBeNull(); + }); +}); + +describe("parseElementHandle", () => { + it("splits the index off the LAST hash, so id selectors survive", () => { + expect(parseElementHandle("sel:#card > .title#3")).toEqual({ + scheme: "sel", + value: "#card > .title", + index: 3, + }); + }); + + it("treats a selector with no index as the first match", () => { + expect(parseElementHandle("sel:.card")).toEqual({ scheme: "sel", value: ".card", index: 0 }); + }); + + it("rejects an unknown scheme", () => { + expect(parseElementHandle("xpath://div")).toBeNull(); + }); + + it("rejects a handle with no value", () => { + expect(parseElementHandle("dom:")).toBeNull(); + expect(parseElementHandle("")).toBeNull(); + expect(parseElementHandle(":headline")).toBeNull(); + }); +}); + +describe("resolveElementHandle", () => { + it("round-trips every handle scheme a read can mint", () => { + const doc = previewDoc( + `
A
+
first
+
second
`, + ); + + expect(resolveElementHandle(doc, "hf:abc123")?.id).toBe("headline"); + expect(resolveElementHandle(doc, "dom:headline")?.id).toBe("headline"); + expect(resolveElementHandle(doc, "sel:.card#1")?.textContent).toBe("second"); + }); + + it("resolves across realms, where a naive instanceof check fails", () => { + const doc = previewDoc('
A
'); + const resolved = resolveElementHandle(doc, "dom:headline"); + + expect(resolved).not.toBeNull(); + // The preview element is NOT an instance of Studio's own HTMLElement. + expect(resolved instanceof HTMLElement).toBe(false); + }); + + it("returns null for a handle that no longer matches", () => { + const doc = previewDoc('
A
'); + expect(resolveElementHandle(doc, "dom:deleted")).toBeNull(); + expect(resolveElementHandle(doc, "hf:missing")).toBeNull(); + expect(resolveElementHandle(doc, "sel:.card#0")).toBeNull(); + }); + + it("returns null for an out-of-range occurrence rather than the wrong element", () => { + const doc = previewDoc('
only
'); + expect(resolveElementHandle(doc, "sel:.card#4")).toBeNull(); + }); + + it("returns null for a selector that is invalid in this document", () => { + const doc = previewDoc('
only
'); + expect(resolveElementHandle(doc, "sel:>>>broken#0")).toBeNull(); + }); + + it("returns null for a malformed handle", () => { + const doc = previewDoc('
A
'); + expect(resolveElementHandle(doc, "nonsense")).toBeNull(); + }); +}); diff --git a/packages/studio/src/webmcp/handles.ts b/packages/studio/src/webmcp/handles.ts new file mode 100644 index 0000000000..9f8420bcd1 --- /dev/null +++ b/packages/studio/src/webmcp/handles.ts @@ -0,0 +1,129 @@ +/** + * Opaque element handles: the one thing reads mint and writes consume. + * + * `TimelineElement.id` cannot do this job. It is a SYNTHESISED identity built + * from label, index, selector and source file when the clip has no authored id + * (`timelineElementHelpers.buildTimelineElementIdentity`), so + * `getElementById(element.id)` misses most elements. The real addressing fields + * are separate: `hfId` (the `data-hf-id` the codebase calls the stable primary + * patch target), `domId`, and a `selector` plus occurrence index. + * + * Handles are strings so they survive a JSON round trip through the agent + * untouched. The agent never builds one; it passes back what a read gave it. + */ + +import type { TimelineElement } from "../player/store/timelineElement"; +import type { PatchTarget } from "../utils/sourcePatcher"; + +const SEPARATOR = ":"; +const INDEX_SEPARATOR = "#"; + +/** + * How to find one element. `TimelineElement` calls the DOM id `domId` and + * `PatchTarget` calls it `id`, so both adapt into this rather than the minter + * knowing about either. + */ +export interface ElementAddress { + hfId?: string; + domId?: string | null; + selector?: string; + selectorIndex?: number; +} + +/** + * Address an element the same way Studio's own patcher does, most stable first. + * `data-hf-id` survives edits that renumber or reorder; a bare selector does not. + */ +export function mintElementHandle(address: ElementAddress): string | null { + if (address.hfId) return `hf${SEPARATOR}${address.hfId}`; + if (address.domId) return `dom${SEPARATOR}${address.domId}`; + if (address.selector) { + const index = address.selectorIndex ?? 0; + return `sel${SEPARATOR}${address.selector}${INDEX_SEPARATOR}${index}`; + } + return null; +} + +export function timelineElementAddress(element: TimelineElement): ElementAddress { + return { + hfId: element.hfId, + domId: element.domId, + selector: element.selector, + selectorIndex: element.selectorIndex, + }; +} + +export function patchTargetAddress(target: PatchTarget): ElementAddress { + return { + hfId: target.hfId, + domId: target.id, + selector: target.selector, + selectorIndex: target.selectorIndex, + }; +} + +interface ParsedHandle { + scheme: "hf" | "dom" | "sel"; + value: string; + index: number; +} + +export function parseElementHandle(handle: string): ParsedHandle | null { + const separatorAt = handle.indexOf(SEPARATOR); + if (separatorAt <= 0) return null; + const scheme = handle.slice(0, separatorAt); + const rest = handle.slice(separatorAt + 1); + if (!rest) return null; + if (scheme === "hf" || scheme === "dom") return { scheme, value: rest, index: 0 }; + if (scheme !== "sel") return null; + + // Only the LAST `#` splits the index off: CSS selectors contain `#` themselves. + const indexAt = rest.lastIndexOf(INDEX_SEPARATOR); + if (indexAt <= 0) return { scheme, value: rest, index: 0 }; + const index = Number(rest.slice(indexAt + 1)); + if (!Number.isInteger(index) || index < 0) return { scheme, value: rest, index: 0 }; + return { scheme, value: rest.slice(0, indexAt), index }; +} + +/** + * Resolve a handle against the preview document. + * + * Always re-resolve per call rather than holding an element across calls: a + * preview reload replaces the document, and a node from the destroyed one is + * detached but still looks like an element. + */ +export function resolveElementHandle(doc: Document, handle: string): HTMLElement | null { + const parsed = parseElementHandle(handle); + if (!parsed) return null; + + if (parsed.scheme === "dom") return asHtmlElement(doc, doc.getElementById(parsed.value)); + if (parsed.scheme === "hf") { + return asHtmlElement(doc, doc.querySelector(`[data-hf-id="${cssEscape(parsed.value)}"]`)); + } + + let matches: NodeListOf; + try { + matches = doc.querySelectorAll(parsed.value); + } catch { + // A selector minted from a previous document can be invalid in this one. + return null; + } + return asHtmlElement(doc, matches.item(parsed.index)); +} + +function cssEscape(value: string): string { + // ponytail: happy-dom and jsdom don't always ship CSS.escape; quoting the two + // characters that can break out of an attribute selector covers this use. + return typeof CSS?.escape === "function" ? CSS.escape(value) : value.replace(/["\\]/g, "\\$&"); +} + +/** + * `instanceof HTMLElement` is checked against the OWNING document's realm. + * The preview lives in an iframe, so Studio's own `HTMLElement` is a different + * constructor and the naive check fails on every real preview element. + */ +function asHtmlElement(doc: Document, node: Element | null): HTMLElement | null { + if (!node) return null; + const ctor = doc.defaultView?.HTMLElement; + return ctor && node instanceof ctor ? node : null; +} diff --git a/packages/studio/src/webmcp/registrar.test.ts b/packages/studio/src/webmcp/registrar.test.ts new file mode 100644 index 0000000000..212c62b9fc --- /dev/null +++ b/packages/studio/src/webmcp/registrar.test.ts @@ -0,0 +1,150 @@ +// @vitest-environment jsdom +import { describe, expect, it, vi } from "vitest"; +import { findToolDefinitionError, registerStudioTools } from "./registrar"; +import type { ModelContext, ModelContextTool } from "./types"; + +function tool(overrides: Partial = {}): ModelContextTool { + return { + name: "studio_look", + description: "Read Studio's live state.", + execute: async () => ({ ok: true }), + ...overrides, + }; +} + +function fakeModelContext( + registerTool: ModelContext["registerTool"] = vi.fn().mockResolvedValue(undefined), +): ModelContext { + return { registerTool }; +} + +function domException(name: string, message = name): DOMException { + return new DOMException(message, name); +} + +describe("findToolDefinitionError", () => { + it("accepts the names Studio actually uses", () => { + expect(findToolDefinitionError(tool({ name: "studio_look" }))).toBeNull(); + expect(findToolDefinitionError(tool({ name: "studio.look-2" }))).toBeNull(); + }); + + it("rejects a name the browser would reject, naming the tool", () => { + expect(findToolDefinitionError(tool({ name: "studio look" }))).toMatch(/name must be/); + expect(findToolDefinitionError(tool({ name: "a".repeat(129) }))).toMatch(/name must be/); + expect(findToolDefinitionError(tool({ name: "" }))).toMatch(/name must be/); + }); + + it("rejects an empty description", () => { + expect(findToolDefinitionError(tool({ description: " " }))).toBe( + "description must not be empty", + ); + }); +}); + +describe("registerStudioTools", () => { + it("registers every tool with the shared abort signal", async () => { + const registerTool = vi.fn().mockResolvedValue(undefined); + const controller = new AbortController(); + + const report = await registerStudioTools( + fakeModelContext(registerTool), + [tool({ name: "studio_look" }), tool({ name: "studio_frame" })], + controller.signal, + ); + + expect(report.registered).toEqual(["studio_look", "studio_frame"]); + expect(report.failed).toEqual([]); + expect(registerTool).toHaveBeenCalledTimes(2); + expect(registerTool.mock.calls[0]?.[1]).toEqual({ signal: controller.signal }); + }); + + it("stops silently when the signal aborts mid-registration", async () => { + // A StrictMode mount-cleanup-mount rejects the in-flight registrations with + // AbortError. That is teardown working; it must not surface as a failure or + // escape as an unhandled rejection. + const registerTool = vi + .fn() + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(domException("AbortError")); + + const report = await registerStudioTools( + fakeModelContext(registerTool), + [tool({ name: "studio_look" }), tool({ name: "studio_frame" })], + new AbortController().signal, + ); + + expect(report.registered).toEqual(["studio_look"]); + expect(report.failed).toEqual([]); + }); + + it("keeps the DOMException name, which is the only thing that tells the gates apart", async () => { + const registerTool = vi + .fn() + .mockRejectedValue(domException("SecurityError", "not origin-keyed")); + + const report = await registerStudioTools( + fakeModelContext(registerTool), + [tool()], + new AbortController().signal, + ); + + expect(report.failed).toHaveLength(1); + expect(report.failed[0]?.tool).toBe("studio_look"); + expect(report.failed[0]?.name).toBe("SecurityError"); + // Substring, not equality: jsdom prefixes DOMException.message with the + // name and real browsers do not. + expect(report.failed[0]?.message).toContain("not origin-keyed"); + expect(report.registered).toEqual([]); + }); + + it("keeps going after one tool fails", async () => { + const registerTool = vi + .fn() + .mockRejectedValueOnce(domException("NotAllowedError", "tools policy")) + .mockResolvedValueOnce(undefined); + + const report = await registerStudioTools( + fakeModelContext(registerTool), + [tool({ name: "studio_look" }), tool({ name: "studio_frame" })], + new AbortController().signal, + ); + + expect(report.registered).toEqual(["studio_frame"]); + expect(report.failed.map((f) => f.tool)).toEqual(["studio_look"]); + }); + + it("catches a duplicate name before the browser does, so the report names it", async () => { + const registerTool = vi.fn().mockResolvedValue(undefined); + + const report = await registerStudioTools( + fakeModelContext(registerTool), + [tool({ name: "studio_look" }), tool({ name: "studio_look" })], + new AbortController().signal, + ); + + expect(report.registered).toEqual(["studio_look"]); + expect(report.failed).toEqual([ + { + tool: "studio_look", + name: "InvalidStateError", + message: "duplicate tool name in this registration set", + }, + ]); + // Registering the same name twice REJECTS rather than replacing, so the + // second one must never reach the browser. + expect(registerTool).toHaveBeenCalledTimes(1); + }); + + it("does not send a tool the browser would reject", async () => { + const registerTool = vi.fn().mockResolvedValue(undefined); + + const report = await registerStudioTools( + fakeModelContext(registerTool), + [tool({ name: "studio look" })], + new AbortController().signal, + ); + + expect(registerTool).not.toHaveBeenCalled(); + expect(report.failed[0]?.name).toBe("InvalidStateError"); + }); +}); diff --git a/packages/studio/src/webmcp/registrar.ts b/packages/studio/src/webmcp/registrar.ts new file mode 100644 index 0000000000..628cd79769 --- /dev/null +++ b/packages/studio/src/webmcp/registrar.ts @@ -0,0 +1,112 @@ +/** + * Registers Studio's tools with the browser, once. + * + * The only file besides `types.ts` that touches the WebMCP API, so a spec + * change lands here. + */ + +import type { ModelContext, ModelContextTool } from "./types"; + +export interface ToolRegistrationFailure { + tool: string; + /** The DOMException name where there is one. It is the only thing that tells + * a duplicate name (InvalidStateError) apart from a document that is not + * origin-keyed (SecurityError) or not permitted to use `tools` + * (NotAllowedError), and all three look identical without it. */ + name: string; + message: string; +} + +export interface ToolRegistrationReport { + registered: string[]; + failed: ToolRegistrationFailure[]; +} + +/** Max 128 chars, ASCII alphanumeric plus `_`, `-`, `.` (`index.bs`). */ +const VALID_TOOL_NAME = /^[A-Za-z0-9_.-]{1,128}$/; + +/** + * A tool whose name or description the browser would reject anyway. Caught here + * so the failure names the offending tool instead of arriving as one of N + * identical InvalidStateErrors. + */ +export function findToolDefinitionError(tool: ModelContextTool): string | null { + if (!VALID_TOOL_NAME.test(tool.name)) { + return `name must be 1-128 chars of A-Z a-z 0-9 _ - . (got ${JSON.stringify(tool.name)})`; + } + if (!tool.description.trim()) return "description must not be empty"; + return null; +} + +function isAbortError(error: unknown): boolean { + return error instanceof DOMException && error.name === "AbortError"; +} + +function invalidState(tool: string, message: string): ToolRegistrationFailure { + return { tool, name: "InvalidStateError", message }; +} + +/** + * `"aborted"` is a third outcome, not a failure: teardown got there first and + * the caller should stop rather than record anything. + */ +type RegisterOneOutcome = + | { status: "registered" } + | { status: "failed"; failure: ToolRegistrationFailure } + | { status: "aborted" }; + +async function registerOne( + modelContext: ModelContext, + tool: ModelContextTool, + signal: AbortSignal, +): Promise { + const definitionError = findToolDefinitionError(tool); + if (definitionError) { + return { status: "failed", failure: invalidState(tool.name, definitionError) }; + } + + try { + await modelContext.registerTool(tool, { signal }); + return { status: "registered" }; + } catch (error) { + // A mount-cleanup-mount cycle (React StrictMode in dev) aborts the signal in + // the same task the registration promise is queued in, which rejects every + // registerTool with AbortError. That is teardown working, not a failure, and + // letting it escape fills the dev console with unhandled rejections. + if (isAbortError(error)) return { status: "aborted" }; + return { + status: "failed", + failure: { + tool: tool.name, + name: error instanceof DOMException ? error.name : "Error", + message: error instanceof Error ? error.message : String(error), + }, + }; + } +} + +export async function registerStudioTools( + modelContext: ModelContext, + tools: readonly ModelContextTool[], + signal: AbortSignal, +): Promise { + const report: ToolRegistrationReport = { registered: [], failed: [] }; + const seen = new Set(); + + for (const tool of tools) { + // Registering a name twice REJECTS rather than replacing, so a duplicate + // must never reach the browser. + if (seen.has(tool.name)) { + report.failed.push(invalidState(tool.name, "duplicate tool name in this registration set")); + continue; + } + seen.add(tool.name); + + const outcome = await registerOne(modelContext, tool, signal); + if (outcome.status === "aborted") return report; + if (outcome.status === "failed") report.failed.push(outcome.failure); + else report.registered.push(tool.name); + } + + return report; +} diff --git a/packages/studio/src/webmcp/toolResult.ts b/packages/studio/src/webmcp/toolResult.ts new file mode 100644 index 0000000000..fd803ec10c --- /dev/null +++ b/packages/studio/src/webmcp/toolResult.ts @@ -0,0 +1,67 @@ +/** + * The shape every Studio tool resolves with. + * + * Tools resolve, they never reject. That is forced by the spec, not a style + * choice: a rejected `execute` has its reason DISCARDED and the caller is + * rejected with a bare `UnknownError` (`index.bs`, the execute-tool completion + * steps). Rejecting would therefore guarantee the agent cannot see why the edit + * failed, which is the one thing it needs most. + * + * There is no `outputSchema` in the platform yet, so this discriminant is + * invisible to the agent's schema layer. Every tool's `description` has to say + * that it returns `ok`. + */ + +export type ToolFailureKind = + /** A real state the agent can route around: save queue paused, capability off. */ + | "blocked" + /** The agent's fault: unknown handle, bad enum, out of range. */ + | "invalid" + /** Exogenous: the server said no, the patch target could not be resolved. */ + | "failed" + /** Our bug. Reported AND re-thrown, so it is findable instead of plausible. */ + | "internal"; + +export interface ToolFailure { + ok: false; + kind: ToolFailureKind; + reason: string; + /** What to try instead. This is what turns a failure into a next action. */ + hint?: string; +} + +export type ToolResult = ({ ok: true } & T) | ToolFailure; + +export function toolOk(value: T): { ok: true } & T { + return { ok: true, ...value }; +} + +function toolFailure(kind: ToolFailureKind, reason: string, hint?: string): ToolFailure { + return hint ? { ok: false, kind, reason, hint } : { ok: false, kind, reason }; +} + +/** + * Wrap a tool body so a thrown bug becomes a legible result instead of a + * rejection the agent cannot read. + * + * The split matters. A `TypeError` means a handler signature moved under us and + * the tool is permanently broken; reporting that as an ordinary failure would + * let it ship looking like a bad request forever. So it is tagged `internal` + * AND re-thrown to the console, where it is findable. Everything else is a + * failure the agent should route around. + */ +export async function runToolBody( + toolName: string, + body: () => Promise>, +): Promise> { + try { + return await body(); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + if (error instanceof TypeError || error instanceof ReferenceError) { + console.error(`[hf-webmcp] ${toolName} threw`, error); + return toolFailure("internal", reason); + } + return toolFailure("failed", reason); + } +} diff --git a/packages/studio/src/webmcp/tools/lookTools.test.ts b/packages/studio/src/webmcp/tools/lookTools.test.ts new file mode 100644 index 0000000000..ec7fa6069a --- /dev/null +++ b/packages/studio/src/webmcp/tools/lookTools.test.ts @@ -0,0 +1,207 @@ +// @vitest-environment jsdom +import { describe, expect, it } from "vitest"; +import type { DomEditSelection } from "../../components/editor/domEditingTypes"; +import type { TimelineElement } from "../../player/store/timelineElement"; +import { buildStudioLook, type StudioLookSnapshot } from "./lookTools"; + +function element(overrides: Partial): TimelineElement { + return { id: "synthetic", tag: "div", start: 0, duration: 1, track: 0, ...overrides }; +} + +function snapshot(overrides: Partial = {}): StudioLookSnapshot { + return { + projectId: "demo", + compositionPath: "index.html", + currentTime: 1.5, + duration: 10, + isPlaying: false, + elements: [], + selection: null, + selectedElementIds: [], + history: { canUndo: true, canRedo: false, undoLabel: "Move layer", redoLabel: null }, + writeBlockedReason: null, + ...overrides, + }; +} + +function selection(overrides: Partial = {}): DomEditSelection { + return { + id: "headline", + hfId: "abc123", + element: document.createElement("div"), + label: "Headline", + tagName: "h1", + sourceFile: "index.html", + compositionPath: "index.html", + isCompositionHost: false, + isInsideLockedComposition: false, + boundingBox: { x: 40, y: 12, width: 880, height: 96 }, + textContent: "Ship it", + dataAttributes: {}, + inlineStyles: {}, + computedStyles: {}, + textFields: [], + capabilities: { + canSelect: true, + canEditStyles: true, + canCrop: true, + canMove: true, + canResize: true, + canApplyManualOffset: true, + canApplyManualSize: true, + canApplyManualRotation: true, + }, + ...overrides, + }; +} + +function expectOk(result: { ok: boolean } & Record): T { + if (!result.ok) throw new Error(`expected ok, got ${JSON.stringify(result)}`); + return result as unknown as T; +} + +describe("buildStudioLook", () => { + it("reports the playhead, duration and undo label an agent needs to checkpoint", () => { + const look = expectOk<{ + playhead: number; + duration: number; + history: { undoLabel: string | null }; + }>( + buildStudioLook( + snapshot({ + currentTime: 2.4, + duration: 30, + history: { canUndo: true, canRedo: false, undoLabel: "Edit text", redoLabel: null }, + }), + ), + ); + + expect(look.playhead).toBe(2.4); + expect(look.duration).toBe(30); + expect(look.history.undoLabel).toBe("Edit text"); + }); + + it("gives every addressable element a handle a write tool can consume", () => { + const look = expectOk<{ elements: { handle: string | null; label: string | null }[] }>( + buildStudioLook( + snapshot({ + elements: [ + element({ hfId: "abc", label: "Headline" }), + element({ domId: "cta", label: "Button" }), + element({ selector: ".card", selectorIndex: 2, label: "Card" }), + ], + }), + ), + ); + + expect(look.elements.map((e) => e.handle)).toEqual(["hf:abc", "dom:cta", "sel:.card#2"]); + }); + + it("reports an unaddressable element with a null handle rather than hiding it", () => { + const look = expectOk<{ elements: { handle: string | null }[]; elementCount: number }>( + buildStudioLook(snapshot({ elements: [element({ label: "Anonymous" })] })), + ); + + expect(look.elementCount).toBe(1); + expect(look.elements[0]?.handle).toBeNull(); + }); + + it("returns an empty list for an empty timeline, not a failure", () => { + const look = expectOk<{ elements: unknown[]; elementCount: number }>( + buildStudioLook(snapshot()), + ); + + expect(look.elements).toEqual([]); + expect(look.elementCount).toBe(0); + }); + + it("filters on label, tag and handle, case-insensitively", () => { + const elements = [ + element({ hfId: "abc", label: "Headline", tag: "h1" }), + element({ domId: "cta", label: "Button", tag: "button" }), + ]; + + const byLabel = expectOk<{ elements: { handle: string | null }[] }>( + buildStudioLook(snapshot({ elements }), { filter: "HEADLINE" }), + ); + const byTag = expectOk<{ elements: { handle: string | null }[] }>( + buildStudioLook(snapshot({ elements }), { filter: "button" }), + ); + const byHandle = expectOk<{ elements: { handle: string | null }[] }>( + buildStudioLook(snapshot({ elements }), { filter: "hf:abc" }), + ); + + expect(byLabel.elements.map((e) => e.handle)).toEqual(["hf:abc"]); + expect(byTag.elements.map((e) => e.handle)).toEqual(["dom:cta"]); + expect(byHandle.elements.map((e) => e.handle)).toEqual(["hf:abc"]); + }); + + it("keeps the true match count when the list is truncated", () => { + const elements = Array.from({ length: 5 }, (_, index) => + element({ domId: `el-${index}`, label: "Card" }), + ); + + const look = expectOk<{ elements: unknown[]; elementCount: number }>( + buildStudioLook(snapshot({ elements }), { limit: 2 }), + ); + + // A truncated list must not read as "that is all there is". + expect(look.elements).toHaveLength(2); + expect(look.elementCount).toBe(5); + }); + + it("clamps a nonsense limit instead of failing the call", () => { + const elements = [element({ domId: "a" }), element({ domId: "b" })]; + + for (const limit of [0, -3, 1.5, Number.NaN]) { + const look = expectOk<{ elements: unknown[] }>( + buildStudioLook(snapshot({ elements }), { limit }), + ); + expect(look.elements).toHaveLength(2); + } + }); + + it("surfaces the selection with its capabilities and a usable handle", () => { + const look = expectOk<{ + selection: { handle: string | null; box: { width: number }; can: { editStyles: boolean } }; + }>(buildStudioLook(snapshot({ selection: selection() }))); + + expect(look.selection?.handle).toBe("hf:abc123"); + expect(look.selection?.box.width).toBe(880); + expect(look.selection?.can.editStyles).toBe(true); + }); + + it("passes the disabled reason through so the agent learns it from a read", () => { + const locked = selection({ + capabilities: { + ...selection().capabilities, + canEditStyles: false, + canMove: false, + canApplyManualOffset: false, + reasonIfDisabled: "Element is inside a locked composition", + }, + }); + + const look = expectOk<{ + selection: { can: { editStyles: boolean; move: boolean; reasonIfDisabled: string | null } }; + }>(buildStudioLook(snapshot({ selection: locked }))); + + expect(look.selection?.can.editStyles).toBe(false); + expect(look.selection?.can.move).toBe(false); + expect(look.selection?.can.reasonIfDisabled).toBe("Element is inside a locked composition"); + }); + + it("reports null selection rather than an empty one when nothing is selected", () => { + const look = expectOk<{ selection: unknown }>(buildStudioLook(snapshot({ selection: null }))); + expect(look.selection).toBeNull(); + }); + + it("tells the agent writes are blocked, and why, before it tries one", () => { + const look = expectOk<{ canWrite: boolean; writeBlockedReason: string | null }>( + buildStudioLook(snapshot({ writeBlockedReason: "Auto-save is paused" })), + ); + + expect(look.canWrite).toBe(false); + expect(look.writeBlockedReason).toBe("Auto-save is paused"); + }); +}); diff --git a/packages/studio/src/webmcp/tools/lookTools.ts b/packages/studio/src/webmcp/tools/lookTools.ts new file mode 100644 index 0000000000..ff618677bc --- /dev/null +++ b/packages/studio/src/webmcp/tools/lookTools.ts @@ -0,0 +1,199 @@ +/** + * `studio_look`: the one call that orients an agent. + * + * Deliberately fat. Every field here is one the agent would otherwise have to + * spend a round trip discovering, and several of them (the capabilities, the + * write-blocked reason) exist to stop it attempting a write that cannot land. + * + * The building is a pure function over a snapshot so it can be tested with + * values. Gathering the snapshot is the React layer's job. + */ + +import type { DomEditSelection } from "../../components/editor/domEditingTypes"; +import type { TimelineElement } from "../../player/store/timelineElement"; +import { mintElementHandle, patchTargetAddress, timelineElementAddress } from "../handles"; +import { toolOk, type ToolResult } from "../toolResult"; + +export interface StudioLookSnapshot { + projectId: string | null; + compositionPath: string | null; + currentTime: number; + duration: number; + isPlaying: boolean; + elements: readonly TimelineElement[]; + selection: DomEditSelection | null; + selectedElementIds: readonly string[]; + /** + * The undo stack as Studio's shell actually exposes it. + * + * This is a weaker signal than a revision counter, and deliberately not + * dressed up as one: the depth lives in component-local state and is not + * reachable here without plumbing it through the shell context. What an agent + * CAN do is checkpoint `undoLabel` before a batch and notice it change to + * something it did not do, which means a human pressed undo and its earlier + * edits are gone. + */ + history: { + canUndo: boolean; + canRedo: boolean; + undoLabel: string | null; + redoLabel: string | null; + }; + /** Why a write would be refused right now, or null when writes are possible. */ + writeBlockedReason: string | null; +} + +interface LookElement { + /** Pass back to any tool that takes a handle. Null means unaddressable. */ + handle: string | null; + label: string | null; + tag: string; + kind: string | null; + start: number; + duration: number; + track: number; + zIndex: number | null; +} + +interface LookSelection { + handle: string | null; + label: string; + tagName: string; + sourceFile: string; + box: { x: number; y: number; width: number; height: number }; + text: string | null; + /** What this element will and will not accept, straight from Studio. */ + can: { + editStyles: boolean; + move: boolean; + resize: boolean; + editText: boolean; + reasonIfDisabled: string | null; + }; + animationCount: number; +} + +export interface StudioLook { + projectId: string | null; + compositionPath: string | null; + playhead: number; + duration: number; + isPlaying: boolean; + canWrite: boolean; + writeBlockedReason: string | null; + history: StudioLookSnapshot["history"]; + selection: LookSelection | null; + elementCount: number; + elements: LookElement[]; +} + +export interface StudioLookInput { + /** Case-insensitive substring match against label, tag, and handle. */ + filter?: string; + /** Cap the returned list. The full count is always reported separately. */ + limit?: number; +} + +const DEFAULT_LIMIT = 200; + +function describeElement(element: TimelineElement): LookElement { + return { + handle: mintElementHandle(timelineElementAddress(element)), + label: element.label ?? null, + tag: element.tag, + kind: element.kind ?? null, + start: element.start, + duration: element.duration, + track: element.track, + zIndex: element.zIndex ?? null, + }; +} + +function describeSelection(selection: DomEditSelection): LookSelection { + const { capabilities } = selection; + return { + handle: mintElementHandle(patchTargetAddress(selection)), + label: selection.label, + tagName: selection.tagName, + sourceFile: selection.sourceFile, + box: selection.boundingBox, + text: selection.textContent, + can: { + editStyles: capabilities.canEditStyles, + move: capabilities.canMove || capabilities.canApplyManualOffset, + resize: capabilities.canResize || capabilities.canApplyManualSize, + editText: selection.textFields.length > 0, + reasonIfDisabled: capabilities.reasonIfDisabled ?? null, + }, + animationCount: selection.gsapAnimations?.length ?? 0, + }; +} + +function matchesFilter(element: LookElement, needle: string): boolean { + return ( + (element.label?.toLowerCase().includes(needle) ?? false) || + element.tag.toLowerCase().includes(needle) || + (element.handle?.toLowerCase().includes(needle) ?? false) + ); +} + +export function buildStudioLook( + snapshot: StudioLookSnapshot, + input: StudioLookInput = {}, +): ToolResult { + const described = snapshot.elements.map(describeElement); + const needle = input.filter?.trim().toLowerCase(); + const matched = needle + ? described.filter((element) => matchesFilter(element, needle)) + : described; + + // Clamp rather than reject: a bad limit should not cost the agent a round trip + // when the answer it wants is right here. + const requested = + Number.isInteger(input.limit) && input.limit! > 0 ? input.limit! : DEFAULT_LIMIT; + const limit = Math.min(requested, DEFAULT_LIMIT); + + return toolOk({ + projectId: snapshot.projectId, + compositionPath: snapshot.compositionPath, + playhead: snapshot.currentTime, + duration: snapshot.duration, + isPlaying: snapshot.isPlaying, + canWrite: snapshot.writeBlockedReason === null, + writeBlockedReason: snapshot.writeBlockedReason, + history: snapshot.history, + selection: snapshot.selection ? describeSelection(snapshot.selection) : null, + // The count is of everything that MATCHED, so a truncated list is visible + // as a truncated list rather than reading as "that is all there is". + elementCount: matched.length, + elements: matched.slice(0, limit), + }); +} + +export const STUDIO_LOOK_INPUT_SCHEMA = { + type: "object", + properties: { + filter: { + type: "string", + description: "Case-insensitive substring matched against element label, tag, and handle.", + }, + limit: { + type: "integer", + minimum: 1, + maximum: DEFAULT_LIMIT, + description: `Cap the returned elements (default and max ${DEFAULT_LIMIT}). elementCount always reports the full match count.`, + }, + }, + additionalProperties: false, +} as const; + +export const STUDIO_LOOK_DESCRIPTION = [ + "Read HyperFrames Studio's live state in one call: the open project and composition,", + "the playhead and duration, what the human currently has selected (including what that", + "element will and will not accept), and the timeline's elements with a handle for each.", + "Pass a handle back to any tool that edits an element.", + "Returns an object with `ok: true`, or `ok: false` with `kind`, `reason` and often a `hint`.", + "Check `canWrite` before attempting an edit. `history.undoLabel` is worth checkpointing", + "before a batch: if it later names something you did not do, a human pressed undo and your", + "earlier edits are gone.", +].join(" "); diff --git a/packages/studio/src/webmcp/types.ts b/packages/studio/src/webmcp/types.ts new file mode 100644 index 0000000000..f7be58d254 --- /dev/null +++ b/packages/studio/src/webmcp/types.ts @@ -0,0 +1,76 @@ +/** + * Local typings for the WebMCP browser API, which is not in lib.dom yet. + * + * Mirrors the WebIDL in the W3C spec (`webmachinelearning/webmcp`, `index.bs`) + * as of 2026-08-26. Two things worth knowing before editing this file: + * + * - The API hangs off `document`, NOT `navigator`. `navigator.modelContext` is + * a polyfill compatibility shim, not a spec member, so feature-detecting it + * is wrong even where an article's sample "works". + * - The spec is pre-stable (Origin Trial). This file and `registrar.ts` are the + * only places that touch the API, so a spec change is a two-file edit. Re-read + * `index.bs` rather than trusting this transcription. + * + * Only the surface Studio actually uses is declared. `getTools` and + * `executeTool` are the consumer side; Studio registers, it does not call. + */ + +export interface ModelContextToolAnnotations { + /** The tool does not change state. Lets an agent decide when calling is free. */ + readOnlyHint?: boolean; + /** The tool's output contains data the page's author does not vouch for. */ + untrustedContentHint?: boolean; +} + +export interface ToolExecuteCallbackOptions { + /** + * Aborted when the caller cancels. Note that Studio's commit path is not + * cancellable once dispatched, so tools check this BEFORE dispatching and + * document that a late abort does not unwind a write. + */ + signal: AbortSignal; +} + +export interface ModelContextTool { + /** + * Max 128 characters, ASCII alphanumeric plus `_`, `-`, `.`. Registering a + * name that already exists REJECTS with InvalidStateError; it does not + * replace. + */ + name: string; + title?: string; + /** Required and non-empty; an empty string rejects with InvalidStateError. */ + description: string; + /** JSON Schema. Nothing in the platform validates input against it. */ + inputSchema?: object; + /** + * The user agent JSON-serializes whatever this resolves with, so it must + * return an object. Returning `undefined` fails the serialization. + * + * A rejection is NOT a usable error channel: the spec discards the reason and + * rejects the caller with a bare UnknownError. Resolve with a tagged failure + * instead. See `toolResult.ts`. + */ + execute: (input: object, options: ToolExecuteCallbackOptions) => Promise; + annotations?: ModelContextToolAnnotations; +} + +export interface ModelContextRegisterToolOptions { + exposedTo?: string[]; + /** Aborting unregisters the tool. It does not cancel a running `execute`. */ + signal?: AbortSignal; +} + +export interface ModelContext { + registerTool(tool: ModelContextTool, options?: ModelContextRegisterToolOptions): Promise; +} + +interface DocumentWithModelContext extends Document { + modelContext?: ModelContext; +} + +/** The live WebMCP entry point, or null when this browser has not shipped it. */ +export function getModelContext(doc: Document = document): ModelContext | null { + const candidate = (doc as DocumentWithModelContext).modelContext; + return typeof candidate?.registerTool === "function" ? candidate : null; +} diff --git a/packages/studio/src/webmcp/useStudioAgentTools.test.tsx b/packages/studio/src/webmcp/useStudioAgentTools.test.tsx new file mode 100644 index 0000000000..7341a34eeb --- /dev/null +++ b/packages/studio/src/webmcp/useStudioAgentTools.test.tsx @@ -0,0 +1,203 @@ +// @vitest-environment jsdom +import { act } from "react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mountReactHarness } from "../hooks/domSelectionTestHarness"; +import { writeStudioUiPreferences } from "../utils/studioUiPreferences"; +import { useStudioAgentTools, type StudioAgentToolsDeps } from "./useStudioAgentTools"; +import type { ModelContext, ModelContextRegisterToolOptions, ModelContextTool } from "./types"; +import type { StudioLookSnapshot } from "./tools/lookTools"; + +Reflect.set(globalThis, "IS_REACT_ACT_ENVIRONMENT", true); + +let cleanup: (() => void) | null = null; + +function snapshot(overrides: Partial = {}): StudioLookSnapshot { + return { + projectId: "demo", + compositionPath: "index.html", + currentTime: 0, + duration: 10, + isPlaying: false, + elements: [], + selection: null, + selectedElementIds: [], + history: { canUndo: false, canRedo: false, undoLabel: null, redoLabel: null }, + writeBlockedReason: null, + ...overrides, + }; +} + +/** Install a fake `document.modelContext` and report what got registered. */ +function installModelContext() { + const registered: ModelContextTool[] = []; + const registerTool = vi.fn( + async (tool: ModelContextTool, _options?: ModelContextRegisterToolOptions) => { + registered.push(tool); + }, + ); + const modelContext: ModelContext = { registerTool }; + Object.defineProperty(document, "modelContext", { + value: modelContext, + configurable: true, + writable: true, + }); + return { registered, registerTool }; +} + +function removeModelContext() { + Reflect.deleteProperty(document, "modelContext"); +} + +function mountTools(deps: StudioAgentToolsDeps) { + function Probe({ current }: { current: StudioAgentToolsDeps }) { + useStudioAgentTools(current); + return null; + } + const root = mountReactHarness(); + cleanup = () => act(() => root.unmount()); + return { + rerenderWith(next: StudioAgentToolsDeps) { + act(() => root.render()); + }, + }; +} + +beforeEach(() => { + window.localStorage.clear(); +}); + +afterEach(() => { + cleanup?.(); + cleanup = null; + removeModelContext(); + window.localStorage.clear(); + vi.restoreAllMocks(); +}); + +describe("useStudioAgentTools", () => { + it("registers the tool set once on mount", async () => { + const { registered } = installModelContext(); + + await act(async () => { + mountTools({ getSnapshot: () => snapshot() }); + }); + + expect(registered.map((tool) => tool.name)).toEqual(["studio_look"]); + }); + + it("does not re-register when the deps object changes identity", async () => { + // The regression test for the whole design. The DomEdit actions object + // changes identity on nearly every interaction; if registration depended on + // it, the signal would abort and unregister the tools each time. + const { registerTool } = installModelContext(); + + let harness: ReturnType | null = null; + await act(async () => { + harness = mountTools({ getSnapshot: () => snapshot() }); + }); + expect(registerTool).toHaveBeenCalledTimes(1); + + await act(async () => { + harness?.rerenderWith({ getSnapshot: () => snapshot({ currentTime: 5 }) }); + harness?.rerenderWith({ getSnapshot: () => snapshot({ currentTime: 6 }) }); + }); + + expect(registerTool).toHaveBeenCalledTimes(1); + }); + + it("executes against the LATEST deps, not the ones present at registration", async () => { + // The other half of the ref: registering once must not freeze the state the + // tools read, or every answer after the first render would be stale. + const { registered } = installModelContext(); + + let harness: ReturnType | null = null; + await act(async () => { + harness = mountTools({ getSnapshot: () => snapshot({ currentTime: 1 }) }); + }); + + await act(async () => { + harness?.rerenderWith({ getSnapshot: () => snapshot({ currentTime: 42 }) }); + }); + + const look = registered[0]; + if (!look) throw new Error("expected studio_look to be registered"); + const result = (await look.execute({}, { signal: new AbortController().signal })) as { + ok: boolean; + playhead: number; + }; + + expect(result.ok).toBe(true); + expect(result.playhead).toBe(42); + }); + + it("unregisters on unmount by aborting the registration signal", async () => { + const { registerTool } = installModelContext(); + + await act(async () => { + mountTools({ getSnapshot: () => snapshot() }); + }); + const signal = registerTool.mock.calls[0]?.[1]?.signal; + expect(signal?.aborted).toBe(false); + + cleanup?.(); + cleanup = null; + + expect(signal?.aborted).toBe(true); + }); + + it("registers nothing when the browser has no WebMCP", async () => { + removeModelContext(); + + await act(async () => { + mountTools({ getSnapshot: () => snapshot() }); + }); + + // The assertion is that mounting did not throw; a browser without the API + // must still boot Studio. + expect(document).not.toHaveProperty("modelContext"); + }); + + it("registers nothing when the preference is turned off", async () => { + writeStudioUiPreferences({ agentToolsEnabled: false }); + const { registerTool } = installModelContext(); + + await act(async () => { + mountTools({ getSnapshot: () => snapshot() }); + }); + + expect(registerTool).not.toHaveBeenCalled(); + }); + + it("registers when the preference is absent, because on is the default", async () => { + const { registerTool } = installModelContext(); + + await act(async () => { + mountTools({ getSnapshot: () => snapshot() }); + }); + + expect(registerTool).toHaveBeenCalledTimes(1); + }); + + it("reports a tool that throws as an internal failure instead of rejecting", async () => { + const { registered } = installModelContext(); + + await act(async () => { + mountTools({ + getSnapshot: () => { + throw new TypeError("handler signature moved"); + }, + }); + }); + vi.spyOn(console, "error").mockImplementation(() => {}); + + const look = registered[0]; + if (!look) throw new Error("expected studio_look to be registered"); + const result = (await look.execute({}, { signal: new AbortController().signal })) as { + ok: boolean; + kind: string; + }; + + expect(result.ok).toBe(false); + expect(result.kind).toBe("internal"); + }); +}); diff --git a/packages/studio/src/webmcp/useStudioAgentTools.ts b/packages/studio/src/webmcp/useStudioAgentTools.ts new file mode 100644 index 0000000000..21c4bed985 --- /dev/null +++ b/packages/studio/src/webmcp/useStudioAgentTools.ts @@ -0,0 +1,92 @@ +import { useEffect, useRef } from "react"; +import { readStudioUiPreferences } from "../utils/studioUiPreferences"; +import { makeStudioDebugLogger } from "../utils/studioDebug"; +import { registerStudioTools } from "./registrar"; +import { runToolBody, type ToolResult } from "./toolResult"; +import { getModelContext, type ModelContext, type ModelContextTool } from "./types"; +import { + buildStudioLook, + STUDIO_LOOK_DESCRIPTION, + STUDIO_LOOK_INPUT_SCHEMA, + type StudioLook, + type StudioLookInput, + type StudioLookSnapshot, +} from "./tools/lookTools"; + +const log = makeStudioDebugLogger("webmcp"); + +export interface StudioAgentToolsDeps { + /** Read Studio's current state. Called per tool invocation, never cached. */ + getSnapshot: () => StudioLookSnapshot; +} + +/** + * Build the tool list once. + * + * Every `execute` reads `depsRef.current` at CALL time rather than closing over + * a snapshot, which is what lets the list be built once and still see live + * state. That is the whole point of the ref: see the registration note below. + */ +function buildStudioTools(depsRef: { readonly current: StudioAgentToolsDeps }): ModelContextTool[] { + return [ + { + name: "studio_look", + title: "Look at the composition", + description: STUDIO_LOOK_DESCRIPTION, + inputSchema: STUDIO_LOOK_INPUT_SCHEMA, + annotations: { + readOnlyHint: true, + // The labels, text and ids come from the user's composition, which can + // contain anything. This is a hint to the agent, not a sanitiser. + untrustedContentHint: true, + }, + execute: (input): Promise> => + runToolBody("studio_look", async () => + buildStudioLook(depsRef.current.getSnapshot(), input as StudioLookInput), + ), + }, + ]; +} + +/** + * Register Studio's tools with the browser, exactly once per mount. + * + * The effect has an EMPTY dependency array on purpose, and the deps live in a + * ref that every render refreshes. The obvious alternative — depend on the + * handlers — re-runs on nearly every interaction, because the DomEdit actions + * object changes identity whenever the selection or the element list does. + * Re-running means the registration signal aborts and unregisters everything, + * `toolchange` fires constantly so a connected agent watches the tool list + * churn, and the spec warns that a quick unregister-then-reregister can apply + * an old call's arguments against the new schema. + * + * `useStudioTestHooks` carries a comment about the same class of bug already hit + * in this codebase, where effect teardown revoked a lease moments after it was + * taken because writing state changed the effect's dependency identities. + */ +export function useStudioAgentTools(deps: StudioAgentToolsDeps): void { + const depsRef = useRef(deps); + depsRef.current = deps; + + // eslint-disable-next-line no-restricted-syntax + useEffect(() => { + if (readStudioUiPreferences().agentToolsEnabled === false) { + log("skipped", { why: "disabled by preference" }); + return; + } + + const modelContext: ModelContext | null = getModelContext(); + if (!modelContext) { + // Expected on any browser that has not shipped WebMCP. Not an error. + log("skipped", { why: "document.modelContext absent" }); + return; + } + + const controller = new AbortController(); + void registerStudioTools(modelContext, buildStudioTools(depsRef), controller.signal).then( + (report) => log("registered", { ...report }), + ); + + return () => controller.abort(); + }, []); +} From 20a8e3f1ea6f6a1f6246050a5a2aafe1297e573b Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Wed, 26 Aug 2026 22:18:08 -0400 Subject: [PATCH 2/5] fix(studio): bound WebMCP look filters --- .../studio/src/webmcp/tools/lookTools.test.ts | 15 ++++++++++++++- packages/studio/src/webmcp/tools/lookTools.ts | 4 +++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/studio/src/webmcp/tools/lookTools.test.ts b/packages/studio/src/webmcp/tools/lookTools.test.ts index ec7fa6069a..e6dddc53e2 100644 --- a/packages/studio/src/webmcp/tools/lookTools.test.ts +++ b/packages/studio/src/webmcp/tools/lookTools.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import type { DomEditSelection } from "../../components/editor/domEditingTypes"; import type { TimelineElement } from "../../player/store/timelineElement"; -import { buildStudioLook, type StudioLookSnapshot } from "./lookTools"; +import { buildStudioLook, STUDIO_LOOK_INPUT_SCHEMA, type StudioLookSnapshot } from "./lookTools"; function element(overrides: Partial): TimelineElement { return { id: "synthetic", tag: "div", start: 0, duration: 1, track: 0, ...overrides }; @@ -136,6 +136,19 @@ describe("buildStudioLook", () => { expect(byHandle.elements.map((e) => e.handle)).toEqual(["hf:abc"]); }); + it("bounds a filter before normalizing it", () => { + const boundedFilter = "x".repeat(128); + const look = expectOk<{ elements: { handle: string | null }[] }>( + buildStudioLook( + snapshot({ elements: [element({ domId: "bounded", label: boundedFilter })] }), + { filter: `${boundedFilter}${"y".repeat(10_000)}` }, + ), + ); + + expect(look.elements.map((entry) => entry.handle)).toEqual(["dom:bounded"]); + expect(STUDIO_LOOK_INPUT_SCHEMA.properties.filter.maxLength).toBe(128); + }); + it("keeps the true match count when the list is truncated", () => { const elements = Array.from({ length: 5 }, (_, index) => element({ domId: `el-${index}`, label: "Card" }), diff --git a/packages/studio/src/webmcp/tools/lookTools.ts b/packages/studio/src/webmcp/tools/lookTools.ts index ff618677bc..1610fb76ec 100644 --- a/packages/studio/src/webmcp/tools/lookTools.ts +++ b/packages/studio/src/webmcp/tools/lookTools.ts @@ -95,6 +95,7 @@ export interface StudioLookInput { } const DEFAULT_LIMIT = 200; +const MAX_FILTER_LENGTH = 128; function describeElement(element: TimelineElement): LookElement { return { @@ -142,7 +143,7 @@ export function buildStudioLook( input: StudioLookInput = {}, ): ToolResult { const described = snapshot.elements.map(describeElement); - const needle = input.filter?.trim().toLowerCase(); + const needle = input.filter?.slice(0, MAX_FILTER_LENGTH).trim().toLowerCase(); const matched = needle ? described.filter((element) => matchesFilter(element, needle)) : described; @@ -175,6 +176,7 @@ export const STUDIO_LOOK_INPUT_SCHEMA = { properties: { filter: { type: "string", + maxLength: MAX_FILTER_LENGTH, description: "Case-insensitive substring matched against element label, tag, and handle.", }, limit: { From 719a0bff551f74c698e531eb9465b7592d07b6fe Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Wed, 26 Aug 2026 22:53:34 -0400 Subject: [PATCH 3/5] fix(studio): remove premature WebMCP write state --- packages/studio/src/webmcp/StudioAgentTools.tsx | 7 ------- .../studio/src/webmcp/tools/lookTools.test.ts | 12 ++++-------- packages/studio/src/webmcp/tools/lookTools.ts | 15 +++------------ .../src/webmcp/useStudioAgentTools.test.tsx | 2 -- 4 files changed, 7 insertions(+), 29 deletions(-) diff --git a/packages/studio/src/webmcp/StudioAgentTools.tsx b/packages/studio/src/webmcp/StudioAgentTools.tsx index 6d4de4d8b6..2d26cca56a 100644 --- a/packages/studio/src/webmcp/StudioAgentTools.tsx +++ b/packages/studio/src/webmcp/StudioAgentTools.tsx @@ -31,19 +31,12 @@ export function StudioAgentTools() { isPlaying: player.isPlaying, elements: player.elements, selection: domEditSelection, - selectedElementIds: [...player.selectedElementIds], history: { canUndo: editHistory.canUndo, canRedo: editHistory.canRedo, undoLabel: editHistory.undoLabel ?? null, redoLabel: editHistory.redoLabel ?? null, }, - // TODO(webmcp): the save-queue and external-conflict states live on - // App's previewPersistence and externalFileChanges, which are not on any - // context this component can read. Until they are, `canWrite` is - // optimistic. The write tools land in a later unit and MUST NOT ship - // trusting this field; they need the real guard. - writeBlockedReason: null, }; }, [projectId, activeCompPath, domEditSelection, editHistory]); diff --git a/packages/studio/src/webmcp/tools/lookTools.test.ts b/packages/studio/src/webmcp/tools/lookTools.test.ts index e6dddc53e2..6718d0fec4 100644 --- a/packages/studio/src/webmcp/tools/lookTools.test.ts +++ b/packages/studio/src/webmcp/tools/lookTools.test.ts @@ -17,9 +17,7 @@ function snapshot(overrides: Partial = {}): StudioLookSnapsh isPlaying: false, elements: [], selection: null, - selectedElementIds: [], history: { canUndo: true, canRedo: false, undoLabel: "Move layer", redoLabel: null }, - writeBlockedReason: null, ...overrides, }; } @@ -209,12 +207,10 @@ describe("buildStudioLook", () => { expect(look.selection).toBeNull(); }); - it("tells the agent writes are blocked, and why, before it tries one", () => { - const look = expectOk<{ canWrite: boolean; writeBlockedReason: string | null }>( - buildStudioLook(snapshot({ writeBlockedReason: "Auto-save is paused" })), - ); + it("does not advertise write readiness before the real write gate exists", () => { + const look = expectOk>(buildStudioLook(snapshot())); - expect(look.canWrite).toBe(false); - expect(look.writeBlockedReason).toBe("Auto-save is paused"); + expect(look).not.toHaveProperty("canWrite"); + expect(look).not.toHaveProperty("writeBlockedReason"); }); }); diff --git a/packages/studio/src/webmcp/tools/lookTools.ts b/packages/studio/src/webmcp/tools/lookTools.ts index 1610fb76ec..04173bf819 100644 --- a/packages/studio/src/webmcp/tools/lookTools.ts +++ b/packages/studio/src/webmcp/tools/lookTools.ts @@ -2,8 +2,7 @@ * `studio_look`: the one call that orients an agent. * * Deliberately fat. Every field here is one the agent would otherwise have to - * spend a round trip discovering, and several of them (the capabilities, the - * write-blocked reason) exist to stop it attempting a write that cannot land. + * spend a round trip discovering. * * The building is a pure function over a snapshot so it can be tested with * values. Gathering the snapshot is the React layer's job. @@ -22,7 +21,6 @@ export interface StudioLookSnapshot { isPlaying: boolean; elements: readonly TimelineElement[]; selection: DomEditSelection | null; - selectedElementIds: readonly string[]; /** * The undo stack as Studio's shell actually exposes it. * @@ -39,8 +37,6 @@ export interface StudioLookSnapshot { undoLabel: string | null; redoLabel: string | null; }; - /** Why a write would be refused right now, or null when writes are possible. */ - writeBlockedReason: string | null; } interface LookElement { @@ -79,8 +75,6 @@ export interface StudioLook { playhead: number; duration: number; isPlaying: boolean; - canWrite: boolean; - writeBlockedReason: string | null; history: StudioLookSnapshot["history"]; selection: LookSelection | null; elementCount: number; @@ -160,8 +154,6 @@ export function buildStudioLook( playhead: snapshot.currentTime, duration: snapshot.duration, isPlaying: snapshot.isPlaying, - canWrite: snapshot.writeBlockedReason === null, - writeBlockedReason: snapshot.writeBlockedReason, history: snapshot.history, selection: snapshot.selection ? describeSelection(snapshot.selection) : null, // The count is of everything that MATCHED, so a truncated list is visible @@ -195,7 +187,6 @@ export const STUDIO_LOOK_DESCRIPTION = [ "element will and will not accept), and the timeline's elements with a handle for each.", "Pass a handle back to any tool that edits an element.", "Returns an object with `ok: true`, or `ok: false` with `kind`, `reason` and often a `hint`.", - "Check `canWrite` before attempting an edit. `history.undoLabel` is worth checkpointing", - "before a batch: if it later names something you did not do, a human pressed undo and your", - "earlier edits are gone.", + "`history.undoLabel` is worth checkpointing before a batch: if it later names something", + "you did not do, a human pressed undo and your earlier edits are gone.", ].join(" "); diff --git a/packages/studio/src/webmcp/useStudioAgentTools.test.tsx b/packages/studio/src/webmcp/useStudioAgentTools.test.tsx index 7341a34eeb..5685eea688 100644 --- a/packages/studio/src/webmcp/useStudioAgentTools.test.tsx +++ b/packages/studio/src/webmcp/useStudioAgentTools.test.tsx @@ -20,9 +20,7 @@ function snapshot(overrides: Partial = {}): StudioLookSnapsh isPlaying: false, elements: [], selection: null, - selectedElementIds: [], history: { canUndo: false, canRedo: false, undoLabel: null, redoLabel: null }, - writeBlockedReason: null, ...overrides, }; } From 36e718286fd368a4bccc3d0ed08abbaf241d7a12 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Wed, 26 Aug 2026 23:00:17 -0400 Subject: [PATCH 4/5] docs(studio): name WebMCP singleton assumption --- packages/studio/src/webmcp/registrar.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/studio/src/webmcp/registrar.ts b/packages/studio/src/webmcp/registrar.ts index 628cd79769..7480caac0f 100644 --- a/packages/studio/src/webmcp/registrar.ts +++ b/packages/studio/src/webmcp/registrar.ts @@ -2,7 +2,10 @@ * Registers Studio's tools with the browser, once. * * The only file besides `types.ts` that touches the WebMCP API, so a spec - * change lands here. + * change lands here. Tool names are document-scoped, so Studio relies on its + * single live `EditorShell` mounting one `StudioAgentTools`. A second live + * shell would register the same names and receive `InvalidStateError`; the + * duplicate check below only owns duplicates within one registration set. */ import type { ModelContext, ModelContextTool } from "./types"; From 3d3db4759754a2072ab93c5a86bac0bca671be67 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Wed, 26 Aug 2026 23:16:37 -0400 Subject: [PATCH 5/5] fix(studio): surface WebMCP registration failures --- .../studio/src/utils/studioUiPreferences.ts | 2 ++ .../studio/src/webmcp/StudioAgentTools.tsx | 5 +++-- .../studio/src/webmcp/tools/lookTools.test.ts | 9 +++++++++ packages/studio/src/webmcp/tools/lookTools.ts | 15 +++++++++++--- .../src/webmcp/useStudioAgentTools.test.tsx | 19 ++++++++++++++++++ .../studio/src/webmcp/useStudioAgentTools.ts | 20 +++++++++++++++++-- 6 files changed, 63 insertions(+), 7 deletions(-) diff --git a/packages/studio/src/utils/studioUiPreferences.ts b/packages/studio/src/utils/studioUiPreferences.ts index e21c303bd2..1a14318d7e 100644 --- a/packages/studio/src/utils/studioUiPreferences.ts +++ b/packages/studio/src/utils/studioUiPreferences.ts @@ -38,6 +38,8 @@ export interface StudioUiPreferences { * Expose Studio's editing capabilities to an agentic browser as WebMCP tools. * Absent means on: the browser still gates every actual call behind its own * permission prompt, so "registered" is not "reachable without consent". + * Changes take effect on the next Studio reload because registration is + * intentionally scoped to one mount. */ agentToolsEnabled?: boolean; } diff --git a/packages/studio/src/webmcp/StudioAgentTools.tsx b/packages/studio/src/webmcp/StudioAgentTools.tsx index 2d26cca56a..8060d7382b 100644 --- a/packages/studio/src/webmcp/StudioAgentTools.tsx +++ b/packages/studio/src/webmcp/StudioAgentTools.tsx @@ -19,7 +19,7 @@ import type { StudioLookSnapshot } from "./tools/lookTools"; */ export function StudioAgentTools() { const { projectId, activeCompPath, editHistory } = useStudioShellContext(); - const { domEditSelection } = useDomEditSelectionContext(); + const { domEditSelection, selectedGsapAnimations } = useDomEditSelectionContext(); const getSnapshot = useCallback((): StudioLookSnapshot => { const player = usePlayerStore.getState(); @@ -31,6 +31,7 @@ export function StudioAgentTools() { isPlaying: player.isPlaying, elements: player.elements, selection: domEditSelection, + selectionAnimationCount: selectedGsapAnimations.length, history: { canUndo: editHistory.canUndo, canRedo: editHistory.canRedo, @@ -38,7 +39,7 @@ export function StudioAgentTools() { redoLabel: editHistory.redoLabel ?? null, }, }; - }, [projectId, activeCompPath, domEditSelection, editHistory]); + }, [projectId, activeCompPath, domEditSelection, selectedGsapAnimations, editHistory]); useStudioAgentTools({ getSnapshot }); return null; diff --git a/packages/studio/src/webmcp/tools/lookTools.test.ts b/packages/studio/src/webmcp/tools/lookTools.test.ts index 6718d0fec4..33c37ce9c6 100644 --- a/packages/studio/src/webmcp/tools/lookTools.test.ts +++ b/packages/studio/src/webmcp/tools/lookTools.test.ts @@ -17,6 +17,7 @@ function snapshot(overrides: Partial = {}): StudioLookSnapsh isPlaying: false, elements: [], selection: null, + selectionAnimationCount: 0, history: { canUndo: true, canRedo: false, undoLabel: "Move layer", redoLabel: null }, ...overrides, }; @@ -182,6 +183,14 @@ describe("buildStudioLook", () => { expect(look.selection?.can.editStyles).toBe(true); }); + it("reports the live animation count supplied outside the DOM selection", () => { + const look = expectOk<{ selection: { animationCount: number } | null }>( + buildStudioLook(snapshot({ selection: selection(), selectionAnimationCount: 3 })), + ); + + expect(look.selection?.animationCount).toBe(3); + }); + it("passes the disabled reason through so the agent learns it from a read", () => { const locked = selection({ capabilities: { diff --git a/packages/studio/src/webmcp/tools/lookTools.ts b/packages/studio/src/webmcp/tools/lookTools.ts index 04173bf819..1cadcffb2b 100644 --- a/packages/studio/src/webmcp/tools/lookTools.ts +++ b/packages/studio/src/webmcp/tools/lookTools.ts @@ -21,6 +21,8 @@ export interface StudioLookSnapshot { isPlaying: boolean; elements: readonly TimelineElement[]; selection: DomEditSelection | null; + /** Live animations for the current selection arrive outside DomEditSelection. */ + selectionAnimationCount: number; /** * The undo stack as Studio's shell actually exposes it. * @@ -69,6 +71,11 @@ interface LookSelection { animationCount: number; } +/** + * Session-scoped response shape. There is intentionally no schema version: + * WebMCP consumers discover the current tool and schema when they connect + * rather than pinning a cached REST response contract. + */ export interface StudioLook { projectId: string | null; compositionPath: string | null; @@ -104,7 +111,7 @@ function describeElement(element: TimelineElement): LookElement { }; } -function describeSelection(selection: DomEditSelection): LookSelection { +function describeSelection(selection: DomEditSelection, animationCount: number): LookSelection { const { capabilities } = selection; return { handle: mintElementHandle(patchTargetAddress(selection)), @@ -120,7 +127,7 @@ function describeSelection(selection: DomEditSelection): LookSelection { editText: selection.textFields.length > 0, reasonIfDisabled: capabilities.reasonIfDisabled ?? null, }, - animationCount: selection.gsapAnimations?.length ?? 0, + animationCount, }; } @@ -155,7 +162,9 @@ export function buildStudioLook( duration: snapshot.duration, isPlaying: snapshot.isPlaying, history: snapshot.history, - selection: snapshot.selection ? describeSelection(snapshot.selection) : null, + selection: snapshot.selection + ? describeSelection(snapshot.selection, snapshot.selectionAnimationCount) + : null, // The count is of everything that MATCHED, so a truncated list is visible // as a truncated list rather than reading as "that is all there is". elementCount: matched.length, diff --git a/packages/studio/src/webmcp/useStudioAgentTools.test.tsx b/packages/studio/src/webmcp/useStudioAgentTools.test.tsx index 5685eea688..544192ba43 100644 --- a/packages/studio/src/webmcp/useStudioAgentTools.test.tsx +++ b/packages/studio/src/webmcp/useStudioAgentTools.test.tsx @@ -7,6 +7,9 @@ import { useStudioAgentTools, type StudioAgentToolsDeps } from "./useStudioAgent import type { ModelContext, ModelContextRegisterToolOptions, ModelContextTool } from "./types"; import type { StudioLookSnapshot } from "./tools/lookTools"; +const trackEvent = vi.hoisted(() => vi.fn()); +vi.mock("../telemetry/client", () => ({ trackEvent })); + Reflect.set(globalThis, "IS_REACT_ACT_ENVIRONMENT", true); let cleanup: (() => void) | null = null; @@ -20,6 +23,7 @@ function snapshot(overrides: Partial = {}): StudioLookSnapsh isPlaying: false, elements: [], selection: null, + selectionAnimationCount: 0, history: { canUndo: false, canRedo: false, undoLabel: null, redoLabel: null }, ...overrides, }; @@ -62,6 +66,7 @@ function mountTools(deps: StudioAgentToolsDeps) { beforeEach(() => { window.localStorage.clear(); + trackEvent.mockReset(); }); afterEach(() => { @@ -176,6 +181,20 @@ describe("useStudioAgentTools", () => { expect(registerTool).toHaveBeenCalledTimes(1); }); + it("reports a non-abort registration failure through production telemetry", async () => { + const { registerTool } = installModelContext(); + registerTool.mockRejectedValue(new DOMException("blocked", "NotAllowedError")); + + await act(async () => { + mountTools({ getSnapshot: () => snapshot() }); + }); + + expect(trackEvent).toHaveBeenCalledWith("webmcp_registration_failed", { + error_name: "NotAllowedError", + tool_name: "studio_look", + }); + }); + it("reports a tool that throws as an internal failure instead of rejecting", async () => { const { registered } = installModelContext(); diff --git a/packages/studio/src/webmcp/useStudioAgentTools.ts b/packages/studio/src/webmcp/useStudioAgentTools.ts index 21c4bed985..b691df2a74 100644 --- a/packages/studio/src/webmcp/useStudioAgentTools.ts +++ b/packages/studio/src/webmcp/useStudioAgentTools.ts @@ -1,7 +1,8 @@ import { useEffect, useRef } from "react"; +import { trackEvent } from "../telemetry/client"; import { readStudioUiPreferences } from "../utils/studioUiPreferences"; import { makeStudioDebugLogger } from "../utils/studioDebug"; -import { registerStudioTools } from "./registrar"; +import { registerStudioTools, type ToolRegistrationReport } from "./registrar"; import { runToolBody, type ToolResult } from "./toolResult"; import { getModelContext, type ModelContext, type ModelContextTool } from "./types"; import { @@ -15,6 +16,16 @@ import { const log = makeStudioDebugLogger("webmcp"); +function reportRegistration(report: ToolRegistrationReport): void { + log("registered", { ...report }); + for (const failure of report.failed) { + trackEvent("webmcp_registration_failed", { + error_name: failure.name, + tool_name: failure.tool, + }); + } +} + export interface StudioAgentToolsDeps { /** Read Studio's current state. Called per tool invocation, never cached. */ getSnapshot: () => StudioLookSnapshot; @@ -63,6 +74,11 @@ function buildStudioTools(depsRef: { readonly current: StudioAgentToolsDeps }): * `useStudioTestHooks` carries a comment about the same class of bug already hit * in this codebase, where effect teardown revoked a lease moments after it was * taken because writing state changed the effect's dependency identities. + * + * Any fallback must be awaited inside this effect before registration and then + * re-read here. Installing one from a sibling effect would race this mount-only + * lookup. Hot-module replacement can still create a brief unregister/register + * window in development; production has one document-scoped registration. */ export function useStudioAgentTools(deps: StudioAgentToolsDeps): void { const depsRef = useRef(deps); @@ -84,7 +100,7 @@ export function useStudioAgentTools(deps: StudioAgentToolsDeps): void { const controller = new AbortController(); void registerStudioTools(modelContext, buildStudioTools(depsRef), controller.signal).then( - (report) => log("registered", { ...report }), + reportRegistration, ); return () => controller.abort();