feat(studio): expose Studio's live state to an agentic browser (WebMCP)

packages/studio/src/components/EditorShell.tsx

@@ -15,6 +15,7 @@ import { usePlayerStore, type TimelineElement } from "../player";
 import type { BlockPreviewInfo } from "./sidebar/BlocksTab";
 import type { GestureRecordingState } from "./editor/GestureRecordControl";
 import { useTimelineSelectionPreviewSync } from "../hooks/useTimelineSelectionPreviewSync";
+import { StudioAgentTools } from "../webmcp/StudioAgentTools";
 
 type RenderClipContent = (
   element: TimelineElement,
@@ -262,6 +263,9 @@ function EditorShellBody({
       onKeyDown={handleKeyDown}
       tabIndex={-1}
     >
+      {/* Renders nothing; exposes Studio's state to an agentic browser. Mounted
+          here rather than in App because it needs the DomEdit contexts. */}
+      <StudioAgentTools />
       {/* Top row: [left | preview | right] — outer padding + the 8px resize
           seams give the panels CapCut-style separation on the dark canvas. */}
       <div className="flex flex-row flex-1 min-h-0 px-px pt-px">

packages/studio/src/utils/studioUiPreferences.ts

@@ -34,6 +34,14 @@ export interface StudioUiPreferences {
   timelineZoomMode?: "fit" | "manual";
   /** Manual timeline zoom percent, paired with `timelineZoomMode: "manual"`. */
   timelineManualZoomPercent?: number;
+  /**
+   * Expose Studio's editing capabilities to an agentic browser as WebMCP tools.
+   * Absent means on: the browser still gates every actual call behind its own
+   * permission prompt, so "registered" is not "reachable without consent".
+   * Changes take effect on the next Studio reload because registration is
+   * intentionally scoped to one mount.
+   */
+  agentToolsEnabled?: boolean;
 }
 
 const STUDIO_UI_PREFERENCES_KEY = "hf-studio-ui-preferences";
@@ -140,6 +148,9 @@ function readStorage(storage: Storage | null): StudioUiPreferences {
     ) {
       preferences.timelineManualZoomPercent = parsed.timelineManualZoomPercent;
     }
+    if (typeof parsed.agentToolsEnabled === "boolean") {
+      preferences.agentToolsEnabled = parsed.agentToolsEnabled;
+    }
     return preferences;
   } catch {
     return {};

packages/studio/src/webmcp/StudioAgentTools.tsx

@@ -0,0 +1,46 @@
+import { useCallback } from "react";
+import { useDomEditSelectionContext } from "../contexts/DomEditContext";
+import { useStudioShellContext } from "../contexts/StudioContext";
+import { usePlayerStore } from "../player";
+import { useStudioAgentTools } from "./useStudioAgentTools";
+import type { StudioLookSnapshot } from "./tools/lookTools";
+
+/**
+ * Mounts Studio's WebMCP tool surface. Renders nothing.
+ *
+ * Lives inside `EditorShell` rather than `App` for two reasons: the DomEdit
+ * contexts are only readable below `DomEditProvider`, which `App` renders, and
+ * `App.tsx` sits three lines under the 600-line cap.
+ *
+ * The player store is read IMPERATIVELY through `getState()` inside the
+ * snapshot callback rather than subscribed to. Subscribing to `currentTime`
+ * would re-render this component on every animation frame during playback for
+ * a value nothing here displays.
+ */
+export function StudioAgentTools() {
+  const { projectId, activeCompPath, editHistory } = useStudioShellContext();
+  const { domEditSelection, selectedGsapAnimations } = useDomEditSelectionContext();
+
+  const getSnapshot = useCallback((): StudioLookSnapshot => {
+    const player = usePlayerStore.getState();
+    return {
+      projectId,
+      compositionPath: activeCompPath,
+      currentTime: player.currentTime,
+      duration: player.duration,
+      isPlaying: player.isPlaying,
+      elements: player.elements,
+      selection: domEditSelection,
+      selectionAnimationCount: selectedGsapAnimations.length,
+      history: {
+        canUndo: editHistory.canUndo,
+        canRedo: editHistory.canRedo,
+        undoLabel: editHistory.undoLabel ?? null,
+        redoLabel: editHistory.redoLabel ?? null,
+      },
+    };
+  }, [projectId, activeCompPath, domEditSelection, selectedGsapAnimations, editHistory]);
+
+  useStudioAgentTools({ getSnapshot });
+  return null;
+}

packages/studio/src/webmcp/handles.test.ts

@@ -0,0 +1,130 @@
+// @vitest-environment jsdom
+import { describe, expect, it } from "vitest";
+import type { TimelineElement } from "../player/store/timelineElement";
+import {
+  mintElementHandle,
+  parseElementHandle,
+  resolveElementHandle,
+  timelineElementAddress,
+} from "./handles";
+
+function timelineElement(overrides: Partial<TimelineElement>): TimelineElement {
+  return { id: "synthetic-id", tag: "div", start: 0, duration: 1, track: 0, ...overrides };
+}
+
+/** A separate document, standing in for the preview iframe's realm. */
+function previewDoc(html: string): Document {
+  const iframe = document.createElement("iframe");
+  document.body.append(iframe);
+  const doc = iframe.contentDocument;
+  if (!doc) throw new Error("expected iframe document");
+  doc.body.innerHTML = html;
+  return doc;
+}
+
+describe("mintElementHandle", () => {
+  it("prefers data-hf-id, the stable patch target", () => {
+    const handle = mintElementHandle(
+      timelineElementAddress(
+        timelineElement({ hfId: "abc123", domId: "headline", selector: ".title" }),
+      ),
+    );
+    expect(handle).toBe("hf:abc123");
+  });
+
+  it("falls back to the DOM id when there is no hf id", () => {
+    expect(
+      mintElementHandle(
+        timelineElementAddress(timelineElement({ domId: "headline", selector: ".title" })),
+      ),
+    ).toBe("dom:headline");
+  });
+
+  it("falls back to a selector with its occurrence index", () => {
+    expect(
+      mintElementHandle(
+        timelineElementAddress(timelineElement({ selector: ".card", selectorIndex: 2 })),
+      ),
+    ).toBe("sel:.card#2");
+  });
+
+  it("defaults a missing occurrence index to the first match", () => {
+    expect(mintElementHandle(timelineElementAddress(timelineElement({ selector: ".card" })))).toBe(
+      "sel:.card#0",
+    );
+  });
+
+  it("returns null when the element carries no way to address it", () => {
+    // The synthesised `id` is deliberately NOT used: it cannot resolve.
+    expect(mintElementHandle(timelineElementAddress(timelineElement({})))).toBeNull();
+  });
+});
+
+describe("parseElementHandle", () => {
+  it("splits the index off the LAST hash, so id selectors survive", () => {
+    expect(parseElementHandle("sel:#card > .title#3")).toEqual({
+      scheme: "sel",
+      value: "#card > .title",
+      index: 3,
+    });
+  });
+
+  it("treats a selector with no index as the first match", () => {
+    expect(parseElementHandle("sel:.card")).toEqual({ scheme: "sel", value: ".card", index: 0 });
+  });
+
+  it("rejects an unknown scheme", () => {
+    expect(parseElementHandle("xpath://div")).toBeNull();
+  });
+
+  it("rejects a handle with no value", () => {
+    expect(parseElementHandle("dom:")).toBeNull();
+    expect(parseElementHandle("")).toBeNull();
+    expect(parseElementHandle(":headline")).toBeNull();
+  });
+});
+
+describe("resolveElementHandle", () => {
+  it("round-trips every handle scheme a read can mint", () => {
+    const doc = previewDoc(
+      `<div id="headline" data-hf-id="abc123">A</div>
+       <div class="card">first</div>
+       <div class="card">second</div>`,
+    );
+
+    expect(resolveElementHandle(doc, "hf:abc123")?.id).toBe("headline");
+    expect(resolveElementHandle(doc, "dom:headline")?.id).toBe("headline");
+    expect(resolveElementHandle(doc, "sel:.card#1")?.textContent).toBe("second");
+  });
+
+  it("resolves across realms, where a naive instanceof check fails", () => {
+    const doc = previewDoc('<div id="headline">A</div>');
+    const resolved = resolveElementHandle(doc, "dom:headline");
+
+    expect(resolved).not.toBeNull();
+    // The preview element is NOT an instance of Studio's own HTMLElement.
+    expect(resolved instanceof HTMLElement).toBe(false);
+  });
+
+  it("returns null for a handle that no longer matches", () => {
+    const doc = previewDoc('<div id="headline">A</div>');
+    expect(resolveElementHandle(doc, "dom:deleted")).toBeNull();
+    expect(resolveElementHandle(doc, "hf:missing")).toBeNull();
+    expect(resolveElementHandle(doc, "sel:.card#0")).toBeNull();
+  });
+
+  it("returns null for an out-of-range occurrence rather than the wrong element", () => {
+    const doc = previewDoc('<div class="card">only</div>');
+    expect(resolveElementHandle(doc, "sel:.card#4")).toBeNull();
+  });
+
+  it("returns null for a selector that is invalid in this document", () => {
+    const doc = previewDoc('<div class="card">only</div>');
+    expect(resolveElementHandle(doc, "sel:>>>broken#0")).toBeNull();
+  });
+
+  it("returns null for a malformed handle", () => {
+    const doc = previewDoc('<div id="headline">A</div>');
+    expect(resolveElementHandle(doc, "nonsense")).toBeNull();
+  });
+});

packages/studio/src/webmcp/handles.ts

@@ -0,0 +1,129 @@
+/**
+ * Opaque element handles: the one thing reads mint and writes consume.
+ *
+ * `TimelineElement.id` cannot do this job. It is a SYNTHESISED identity built
+ * from label, index, selector and source file when the clip has no authored id
+ * (`timelineElementHelpers.buildTimelineElementIdentity`), so
+ * `getElementById(element.id)` misses most elements. The real addressing fields
+ * are separate: `hfId` (the `data-hf-id` the codebase calls the stable primary
+ * patch target), `domId`, and a `selector` plus occurrence index.
+ *
+ * Handles are strings so they survive a JSON round trip through the agent
+ * untouched. The agent never builds one; it passes back what a read gave it.
+ */
+
+import type { TimelineElement } from "../player/store/timelineElement";
+import type { PatchTarget } from "../utils/sourcePatcher";
+
+const SEPARATOR = ":";
+const INDEX_SEPARATOR = "#";
+
+/**
+ * How to find one element. `TimelineElement` calls the DOM id `domId` and
+ * `PatchTarget` calls it `id`, so both adapt into this rather than the minter
+ * knowing about either.
+ */
+export interface ElementAddress {
+  hfId?: string;
+  domId?: string | null;
+  selector?: string;
+  selectorIndex?: number;
+}
+
+/**
+ * Address an element the same way Studio's own patcher does, most stable first.
+ * `data-hf-id` survives edits that renumber or reorder; a bare selector does not.
+ */
+export function mintElementHandle(address: ElementAddress): string | null {
+  if (address.hfId) return `hf${SEPARATOR}${address.hfId}`;
+  if (address.domId) return `dom${SEPARATOR}${address.domId}`;
+  if (address.selector) {
+    const index = address.selectorIndex ?? 0;
+    return `sel${SEPARATOR}${address.selector}${INDEX_SEPARATOR}${index}`;
+  }
+  return null;
+}
+
+export function timelineElementAddress(element: TimelineElement): ElementAddress {
+  return {
+    hfId: element.hfId,
+    domId: element.domId,
+    selector: element.selector,
+    selectorIndex: element.selectorIndex,
+  };
+}
+
+export function patchTargetAddress(target: PatchTarget): ElementAddress {
+  return {
+    hfId: target.hfId,
+    domId: target.id,
+    selector: target.selector,
+    selectorIndex: target.selectorIndex,
+  };
+}
+
+interface ParsedHandle {
+  scheme: "hf" | "dom" | "sel";
+  value: string;
+  index: number;
+}
+
+export function parseElementHandle(handle: string): ParsedHandle | null {
+  const separatorAt = handle.indexOf(SEPARATOR);
+  if (separatorAt <= 0) return null;
+  const scheme = handle.slice(0, separatorAt);
+  const rest = handle.slice(separatorAt + 1);
+  if (!rest) return null;
+  if (scheme === "hf" || scheme === "dom") return { scheme, value: rest, index: 0 };
+  if (scheme !== "sel") return null;
+
+  // Only the LAST `#` splits the index off: CSS selectors contain `#` themselves.
+  const indexAt = rest.lastIndexOf(INDEX_SEPARATOR);
+  if (indexAt <= 0) return { scheme, value: rest, index: 0 };
+  const index = Number(rest.slice(indexAt + 1));
+  if (!Number.isInteger(index) || index < 0) return { scheme, value: rest, index: 0 };
+  return { scheme, value: rest.slice(0, indexAt), index };
+}
+
+/**
+ * Resolve a handle against the preview document.
+ *
+ * Always re-resolve per call rather than holding an element across calls: a
+ * preview reload replaces the document, and a node from the destroyed one is
+ * detached but still looks like an element.
+ */
+export function resolveElementHandle(doc: Document, handle: string): HTMLElement | null {
+  const parsed = parseElementHandle(handle);
+  if (!parsed) return null;
+
+  if (parsed.scheme === "dom") return asHtmlElement(doc, doc.getElementById(parsed.value));
+  if (parsed.scheme === "hf") {
+    return asHtmlElement(doc, doc.querySelector(`[data-hf-id="${cssEscape(parsed.value)}"]`));
+  }
+
+  let matches: NodeListOf<Element>;
+  try {
+    matches = doc.querySelectorAll(parsed.value);
+  } catch {
+    // A selector minted from a previous document can be invalid in this one.
+    return null;
+  }
+  return asHtmlElement(doc, matches.item(parsed.index));
+}
+
+function cssEscape(value: string): string {
+  // ponytail: happy-dom and jsdom don't always ship CSS.escape; quoting the two
+  // characters that can break out of an attribute selector covers this use.
+  return typeof CSS?.escape === "function" ? CSS.escape(value) : value.replace(/["\\]/g, "\\$&");
+}
+
+/**
+ * `instanceof HTMLElement` is checked against the OWNING document's realm.
+ * The preview lives in an iframe, so Studio's own `HTMLElement` is a different
+ * constructor and the naive check fails on every real preview element.
+ */
+function asHtmlElement(doc: Document, node: Element | null): HTMLElement | null {
+  if (!node) return null;
+  const ctor = doc.defaultView?.HTMLElement;
+  return ctor && node instanceof ctor ? node : null;
+}