diff --git a/docs/docs.json b/docs/docs.json index 91a3048fd1..632c0a2a3a 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -872,7 +872,8 @@ "group": "Agent setup", "pages": [ "guides/authentication", - "guides/skills" + "guides/skills", + "guides/webmcp" ] }, { diff --git a/docs/guides/webmcp.mdx b/docs/guides/webmcp.mdx new file mode 100644 index 0000000000..78848a3844 --- /dev/null +++ b/docs/guides/webmcp.mdx @@ -0,0 +1,149 @@ +--- +title: "Let an agent drive Studio" +sidebarTitle: "Agent tools (WebMCP)" +description: "Studio exposes its editing capabilities as WebMCP tools, so an agent in your browser can see the composition and change it alongside you." +--- + +Studio registers its own capabilities as WebMCP tools, so an AI agent running in your browser can read what Studio knows and make the same edits you can. + + + This is not the same as [creating through an AI chat](/guides/mcp). That page covers the hosted + HyperFrames MCP connector, which builds and renders a video from a conversation. This page is + about an agent working *inside Studio*, on a composition already open in front of you. + + +## What it looks like + +With the tools available, an agent can do this without touching your files: + +```text +studio_look -> the project, playhead, selection, and every element +studio_select hf:abc123 -> selects the headline, same as clicking it +studio_inspect -> its resolved styles, text, and animations +studio_set_style {"color":"red"} -> writes it, through Studio's own commit path +studio_frame 2.4 -> a PNG of the composition at 2.4 seconds +``` + +The last one matters most. It is what lets an agent judge a change instead of guessing at it. + +## Turning it on + +The tools register automatically when Studio loads. Whether an agent can *reach* them depends on the browser. + +| Browser | Status | +| --- | --- | +| Chrome 149 | Origin Trial | +| Edge 150 | Origin Trial | +| ChatGPT Desktop | Shipped | +| Brave (Leo) | Experimental | +| Firefox, Safari | Not yet | + +For local development in Chrome, enable the flag and restart: + +```text chrome://flags +chrome://flags/#enable-webmcp-testing +``` + +Then confirm the tools are there from Studio's console: + +```javascript +const tools = await document.modelContext.getTools(); +console.log(tools.map((tool) => tool.name)); +// ["studio_look", "studio_select", "studio_seek", ...] +``` + + + Registration is asynchronous, so a caller that reads `getTools()` the instant Studio loads can + see a partial list. Wait for the `toolchange` event, or poll until the count settles at twelve. + + + + The API is `document.modelContext`, not `navigator.modelContext`. Many published examples use + the second one. It is a compatibility shim some polyfills add, not part of the specification, so + feature-detecting it will mislead you. + + +On browsers without native support, Studio loads a polyfill so a WebMCP bridge extension can still +connect. Nothing is downloaded on a browser that has the API already. + +## What an agent can do + +### Read + +| Tool | Answers | +| --- | --- | +| `studio_look` | The open project and composition, the playhead, what you have selected, and every element with a handle | +| `studio_inspect` | One element in full: resolved styles, text fields, box, animations, and what it will accept | +| `studio_frame` | A PNG of the composition at any time | + +`studio_look` gives every element a **handle**. Pass it back to any tool that edits an element. + +### Change + +| Tool | Does | +| --- | --- | +| `studio_select` | Selects an element, exactly as clicking it does | +| `studio_seek` | Moves the playhead | +| `studio_set_text` | Rewrites text | +| `studio_set_style` | Sets inline styles | +| `studio_transform` | Moves, resizes or rotates | +| `studio_add_animation` | Adds a GSAP animation at the playhead | +| `studio_update_animation` | Changes a duration, ease or position | +| `studio_add_keyframe` | Adds a keyframe to an animation | +| `studio_delete_animation` | Removes an animation | + +Every edit runs through the same commit path a mouse gesture uses, so it lands in your file with the +same undo entry and the same save behaviour. There is no separate agent write path. + +## Two rules worth knowing + +**Select first, then edit.** Most editing tools act on the current selection rather than taking an +element. That is how Studio itself works: click, then type. An agent that edits without selecting +gets an error telling it to select. + +**Check what came back.** Tools report what actually happened, not what was asked for. +`studio_transform` reads the element's box back after writing and tells you which operations took +effect. `studio_frame` reports the time it actually captured. When something could not be verified, +the tool says so rather than claiming success. + +## Working alongside an agent + +This is built for you and an agent looking at the same composition. Studio shows you every change as +it happens: an agent selecting an element draws the same selection box, and an edit appears in your +undo history under its own name. + +That shared view is doing real work. Some of Studio's write paths report a failure through a toast +rather than a return value, so **you** are the one who sees it. Leave Studio visible while an agent +is working. + + + Studio refuses agent writes while auto-save is paused or an external change to the file is waiting + for your decision, and tells the agent why. Resolve the banner and it can continue. + + +## Turning it off + +There is no settings toggle yet. The switch is a Studio preference, so set it from the console and +reload: + +```javascript +const KEY = "hf-studio-ui-preferences"; +const prefs = JSON.parse(localStorage.getItem(KEY) ?? "{}"); +localStorage.setItem(KEY, JSON.stringify({ ...prefs, agentToolsEnabled: false })); +location.reload(); +``` + +Read the existing object and spread it, as above. Writing `{agentToolsEnabled: false}` on its own +replaces the whole preferences blob and loses your panel sizes, zoom and timeline settings. + +Set it back to `true`, or delete the key, to re-enable. + +The browser gates tool access behind its own permission prompt, so registering a tool is not the same +as granting access to it. How often you are asked, once per site or every call, is up to the browser +and is still changing while the API is in origin trial. + +## Related topics + +- [Create through an AI chat](/guides/mcp) +- [Install and update agent skills](/guides/skills) +- [Work on the project in Studio](/studio) diff --git a/packages/studio/src/webmcp/tools/contentTools.test.ts b/packages/studio/src/webmcp/tools/contentTools.test.ts index 7b5d825ade..63f41b79e6 100644 --- a/packages/studio/src/webmcp/tools/contentTools.test.ts +++ b/packages/studio/src/webmcp/tools/contentTools.test.ts @@ -29,7 +29,8 @@ describe("studioSetText", () => { const ok = expectOk(result); expect(ok.text).toBe("Ship it faster"); expect(ok.changed).toBe(true); - expect(setText).toHaveBeenCalledWith("Ship it faster", undefined); + // The single field is resolved and named, rather than left undefined. + expect(setText).toHaveBeenCalledWith("Ship it faster", "self"); }); it("reports changed:false when the text already said that", async () => { @@ -105,6 +106,79 @@ describe("studioSetText", () => { expect(result.hint).toMatch(/studio_select/); expect(setText).not.toHaveBeenCalled(); }); + + it("targets the element's ACTUAL text field, not a field called self", async () => { + // Found end to end, not by these tests. An element's text usually lives in a + // child field keyed like `child:0:h1`. Passing no key planned zero + // operations, and the server rejected the empty patch with + // "target and operations required" -- a persist failure that looked like a + // server problem and was not. + const element = previewElement('

Ship it

', "headline"); + const selection = selectionFor(element); + selection.textFields = [{ ...selection.textFields[0]!, key: "child:0:h1" }]; + const setText = vi.fn(async () => ({ ok: true }) as const); + + await studioSetText(contentDeps({ getCurrentSelection: () => selection, setText }), { + text: "Shipped it", + }); + + expect(setText).toHaveBeenCalledWith("Shipped it", "child:0:h1"); + }); + + it("rejects a field the element does not have, rather than writing nowhere", async () => { + const element = previewElement('

Ship it

', "headline"); + const selection = selectionFor(element); + selection.textFields = [{ ...selection.textFields[0]!, key: "child:0:h1" }]; + const setText = vi.fn(); + + const result = expectFailure( + await studioSetText(contentDeps({ getCurrentSelection: () => selection, setText }), { + text: "x", + field: "self", + }), + ); + + expect(result.kind).toBe("invalid"); + expect(result.hint).toContain("child:0:h1"); + expect(setText).not.toHaveBeenCalled(); + }); + + it("asks which field when the element has several", async () => { + const element = previewElement('
a
', "card"); + const selection = selectionFor(element); + const base = selection.textFields[0]!; + selection.textFields = [ + { ...base, key: "child:0:h2" }, + { ...base, key: "child:1:p" }, + ]; + const setText = vi.fn(); + + const result = expectFailure( + await studioSetText(contentDeps({ getCurrentSelection: () => selection, setText }), { + text: "x", + }), + ); + + expect(result.kind).toBe("invalid"); + expect(result.reason).toMatch(/2 text fields/); + expect(setText).not.toHaveBeenCalled(); + }); + + it("reports an element with no text field as blocked", async () => { + const element = previewElement('
', "box"); + const selection = selectionFor(element); + selection.textFields = []; + const setText = vi.fn(); + + const result = expectFailure( + await studioSetText(contentDeps({ getCurrentSelection: () => selection, setText }), { + text: "x", + }), + ); + + expect(result.kind).toBe("blocked"); + expect(setText).not.toHaveBeenCalled(); + }); }); describe("studioSetStyle", () => { diff --git a/packages/studio/src/webmcp/tools/contentTools.ts b/packages/studio/src/webmcp/tools/contentTools.ts index 6e5a85b943..19d26dda39 100644 --- a/packages/studio/src/webmcp/tools/contentTools.ts +++ b/packages/studio/src/webmcp/tools/contentTools.ts @@ -79,12 +79,45 @@ export async function studioSetText( if (typeof input.text !== "string") { return toolFailure("invalid", "text must be a string"); } - const field = typeof input.field === "string" && input.field ? input.field : undefined; const blocked = guardWrite(deps); if (blocked) return blocked; - const before = deps.getCurrentSelection()?.textContent ?? null; + const selection = deps.getCurrentSelection(); + if (!selection) return toolFailure("invalid", "nothing is selected"); + + const fields = selection.textFields; + const requested = typeof input.field === "string" && input.field ? input.field : undefined; + if (requested && !fields.some((candidate) => candidate.key === requested)) { + return toolFailure( + "invalid", + `this element has no text field "${requested}"`, + `Its fields are: ${fields.map((candidate) => candidate.key).join(", ") || "none"}.`, + ); + } + + // Resolving the field is NOT optional. An element's text usually lives in a + // child field keyed like `child:0:h1`, not in one called `self`, and passing + // no key plans zero operations. The server then rejects the empty patch with + // "target and operations required", which surfaces as a persist failure that + // looks like a server problem and is not. + const field = requested ?? (fields.length === 1 ? fields[0]?.key : undefined); + if (!field) { + if (fields.length === 0) { + return toolFailure( + "blocked", + "this element has no editable text field", + "studio_inspect lists an element's textFields.", + ); + } + return toolFailure( + "invalid", + `this element has ${fields.length} text fields, so one must be named`, + `Pass field as one of: ${fields.map((candidate) => candidate.key).join(", ")}.`, + ); + } + + const before = selection.textContent ?? null; const outcome = await deps.setText(input.text, field); const failure = fromOutcome(outcome, "the text"); if (failure) return failure;