feat(studio): let an agent drive Studio's selection and playhead

packages/studio/src/webmcp/StudioAgentTools.tsx

@@ -1,8 +1,8 @@
-import { useCallback } from "react";
-import { useDomEditSelectionContext } from "../contexts/DomEditContext";
+import { useCallback, useMemo } from "react";
+import { useDomEditActionsContext, useDomEditSelectionContext } from "../contexts/DomEditContext";
 import { useStudioShellContext } from "../contexts/StudioContext";
 import { usePlayerStore } from "../player";
-import { useStudioAgentTools } from "./useStudioAgentTools";
+import { useStudioAgentTools, type StudioAgentToolsDeps } from "./useStudioAgentTools";
 import type { StudioLookSnapshot } from "./tools/lookTools";
 
 /**
@@ -12,14 +12,15 @@ import type { StudioLookSnapshot } from "./tools/lookTools";
  * contexts are only readable below `DomEditProvider`, which `App` renders, and
  * `App.tsx` sits three lines under the 600-line cap.
  *
- * The player store is read IMPERATIVELY through `getState()` inside the
- * snapshot callback rather than subscribed to. Subscribing to `currentTime`
- * would re-render this component on every animation frame during playback for
- * a value nothing here displays.
+ * The player store is read IMPERATIVELY through `getState()` rather than
+ * subscribed to. Subscribing to `currentTime` would re-render this component on
+ * every animation frame during playback for a value nothing here displays.
  */
 export function StudioAgentTools() {
   const { projectId, activeCompPath, editHistory } = useStudioShellContext();
   const { domEditSelection, selectedGsapAnimations } = useDomEditSelectionContext();
+  const { previewIframeRef, buildDomSelectionFromTarget, applyDomSelection } =
+    useDomEditActionsContext();
 
   const getSnapshot = useCallback((): StudioLookSnapshot => {
     const player = usePlayerStore.getState();
@@ -41,6 +42,25 @@ export function StudioAgentTools() {
     };
   }, [projectId, activeCompPath, domEditSelection, selectedGsapAnimations, editHistory]);
 
-  useStudioAgentTools({ getSnapshot });
+  const deps = useMemo<StudioAgentToolsDeps>(
+    () => ({
+      getSnapshot,
+      getPreviewDocument: () => previewIframeRef.current?.contentDocument ?? null,
+      buildSelection: (element) => buildDomSelectionFromTarget(element),
+      applySelection: (selection) => applyDomSelection(selection, { revealPanel: true }),
+      requestSeek: (time) => usePlayerStore.getState().requestSeek(time),
+      readPlayhead: () => {
+        const player = usePlayerStore.getState();
+        return {
+          currentTime: player.currentTime,
+          duration: player.duration,
+          isPlaying: player.isPlaying,
+        };
+      },
+    }),
+    [getSnapshot, previewIframeRef, buildDomSelectionFromTarget, applyDomSelection],
+  );
+
+  useStudioAgentTools(deps);
   return null;
 }

packages/studio/src/webmcp/toolResult.ts

@@ -36,7 +36,7 @@ export function toolOk<T extends object>(value: T): { ok: true } & T {
   return { ok: true, ...value };
 }
 
-function toolFailure(kind: ToolFailureKind, reason: string, hint?: string): ToolFailure {
+export function toolFailure(kind: ToolFailureKind, reason: string, hint?: string): ToolFailure {
   return hint ? { ok: false, kind, reason, hint } : { ok: false, kind, reason };
 }
 

packages/studio/src/webmcp/tools/selectionTools.test.ts

@@ -0,0 +1,214 @@
+// @vitest-environment jsdom
+import { describe, expect, it, vi } from "vitest";
+import type { DomEditSelection } from "../../components/editor/domEditingTypes";
+import {
+  studioSeek,
+  studioSelect,
+  type SelectionToolDeps,
+  type StudioSeekResult,
+  type StudioSelectResult,
+} from "./selectionTools";
+import type { ToolFailure, ToolResult } from "../toolResult";
+
+function previewDoc(html: string): Document {
+  const iframe = document.createElement("iframe");
+  document.body.append(iframe);
+  const doc = iframe.contentDocument;
+  if (!doc) throw new Error("expected iframe document");
+  doc.body.innerHTML = html;
+  return doc;
+}
+
+function selectionFor(element: HTMLElement): DomEditSelection {
+  return {
+    id: element.id || undefined,
+    hfId: element.getAttribute("data-hf-id") ?? undefined,
+    element,
+    label: "Headline",
+    tagName: element.tagName.toLowerCase(),
+    sourceFile: "index.html",
+    compositionPath: "index.html",
+    isCompositionHost: false,
+    isInsideLockedComposition: false,
+    boundingBox: { x: 40, y: 12, width: 880, height: 96 },
+    textContent: element.textContent,
+    dataAttributes: {},
+    inlineStyles: {},
+    computedStyles: {},
+    textFields: [],
+    capabilities: {
+      canSelect: true,
+      canEditStyles: true,
+      canCrop: true,
+      canMove: true,
+      canResize: true,
+      canApplyManualOffset: true,
+      canApplyManualSize: true,
+      canApplyManualRotation: true,
+    },
+  };
+}
+
+function selectionDeps(overrides: Partial<SelectionToolDeps> = {}): SelectionToolDeps {
+  return {
+    getPreviewDocument: () => null,
+    buildSelection: async (element) => selectionFor(element),
+    applySelection: () => undefined,
+    requestSeek: () => undefined,
+    readPlayhead: () => ({ currentTime: 0, duration: 10, isPlaying: false }),
+    ...overrides,
+  };
+}
+
+function expectFailure(result: ToolResult<unknown>): ToolFailure {
+  if (result.ok) throw new Error(`expected failure, got ${JSON.stringify(result)}`);
+  return result;
+}
+
+function expectOk<T>(result: ToolResult<T>): { ok: true } & T {
+  if (!result.ok) throw new Error(`expected ok, got ${JSON.stringify(result)}`);
+  return result;
+}
+
+describe("studioSelect", () => {
+  it("applies the selection a click would produce and reports it back", async () => {
+    const doc = previewDoc('<h1 id="headline" data-hf-id="abc">Ship it</h1>');
+    const applySelection = vi.fn();
+
+    const result = await studioSelect(
+      selectionDeps({ getPreviewDocument: () => doc, applySelection }),
+      "hf:abc",
+    );
+
+    const ok = expectOk<StudioSelectResult>(result);
+    expect(ok.handle).toBe("hf:abc");
+    expect(ok.label).toBe("Headline");
+    expect(ok.box.width).toBe(880);
+    // Reveals the inspector, which is what makes the human see what the agent did.
+    expect(applySelection).toHaveBeenCalledTimes(1);
+  });
+
+  it("distinguishes a preview that is not mounted from a handle that does not match", async () => {
+    const notMounted = expectFailure(await studioSelect(selectionDeps(), "dom:headline"));
+    expect(notMounted.kind).toBe("blocked");
+    expect(notMounted.reason).toMatch(/not mounted/);
+
+    const doc = previewDoc('<h1 id="headline">Ship it</h1>');
+    const noMatch = expectFailure(
+      await studioSelect(selectionDeps({ getPreviewDocument: () => doc }), "dom:missing"),
+    );
+    expect(noMatch.kind).toBe("invalid");
+    expect(noMatch.reason).toMatch(/no element matches/);
+    // The two must not be the same message: waiting and re-reading are different fixes.
+    expect(noMatch.reason).not.toBe(notMounted.reason);
+  });
+
+  it("reports an element Studio cannot build a selection for, as a third case", async () => {
+    const doc = previewDoc('<h1 id="headline">Ship it</h1>');
+
+    const result = expectFailure(
+      await studioSelect(
+        selectionDeps({ getPreviewDocument: () => doc, buildSelection: async () => null }),
+        "dom:headline",
+      ),
+    );
+
+    expect(result.kind).toBe("blocked");
+    expect(result.reason).toMatch(/cannot select/);
+  });
+
+  it("rejects a missing handle without touching the preview", async () => {
+    const getPreviewDocument = vi.fn(() => null);
+
+    const result = expectFailure(await studioSelect(selectionDeps({ getPreviewDocument }), "  "));
+
+    expect(result.kind).toBe("invalid");
+    expect(getPreviewDocument).not.toHaveBeenCalled();
+  });
+
+  it("leaves the existing selection alone when it fails", async () => {
+    const doc = previewDoc('<h1 id="headline">Ship it</h1>');
+    const applySelection = vi.fn();
+
+    await studioSelect(
+      selectionDeps({ getPreviewDocument: () => doc, applySelection }),
+      "dom:missing",
+    );
+
+    expect(applySelection).not.toHaveBeenCalled();
+  });
+});
+
+describe("studioSeek", () => {
+  it("reports where the playhead landed, not what was requested", () => {
+    // The player clamps against the ADAPTER's duration, which the wrapper
+    // deliberately does not second-guess.
+    let currentTime = 0;
+    const result = studioSeek(
+      selectionDeps({
+        requestSeek: () => {
+          currentTime = 10;
+        },
+        readPlayhead: () => ({ currentTime, duration: 10, isPlaying: false }),
+      }),
+      999,
+    );
+
+    const ok = expectOk<StudioSeekResult>(result);
+    expect(ok.playhead).toBe(10);
+    expect(ok.moved).toBe(true);
+  });
+
+  it("reports that playback stopped", () => {
+    let isPlaying = true;
+    let currentTime = 0;
+    const result = studioSeek(
+      selectionDeps({
+        requestSeek: () => {
+          currentTime = 2;
+          isPlaying = false;
+        },
+        readPlayhead: () => ({ currentTime, duration: 10, isPlaying }),
+      }),
+      2,
+    );
+
+    expect(expectOk<StudioSeekResult>(result).isPlaying).toBe(false);
+  });
+
+  it("fails rather than claiming a seek the player never received", () => {
+    // `requestSeek` is fire-and-forget: with no adapter mounted it silently does
+    // nothing, and reporting ok would be a lie the agent builds on.
+    const result = expectFailure(
+      studioSeek(
+        selectionDeps({ readPlayhead: () => ({ currentTime: 0, duration: 10, isPlaying: false }) }),
+        5,
+      ),
+    );
+
+    expect(result.kind).toBe("blocked");
+    expect(result.reason).toMatch(/did not move/);
+  });
+
+  it("succeeds when asked to seek to where the playhead already is", () => {
+    const result = studioSeek(
+      selectionDeps({ readPlayhead: () => ({ currentTime: 3, duration: 10, isPlaying: false }) }),
+      3,
+    );
+
+    // Nothing moved, but nothing failed either, and `moved` says which.
+    const ok = expectOk<StudioSeekResult>(result);
+    expect(ok.moved).toBe(false);
+    expect(ok.playhead).toBe(3);
+  });
+
+  it("rejects a non-finite time without calling the player", () => {
+    const requestSeek = vi.fn();
+
+    for (const time of [Number.NaN, Number.POSITIVE_INFINITY]) {
+      const result = expectFailure(studioSeek(selectionDeps({ requestSeek }), time));
+      expect(result.kind).toBe("invalid");
+    }
+    expect(requestSeek).not.toHaveBeenCalled();
+  });
+});