From c8a0591ddddd18c66b37b3277286fbda483a3aa9 Mon Sep 17 00:00:00 2001 From: ukimsanov Date: Sat, 11 Jul 2026 13:55:23 -0700 Subject: [PATCH] feat(studio): clip thumbnail modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit What: ImageThumbnail (+tests) and thumbnailUtils (+tests) — frame decode with SVG/AVIF format fallbacks and rounded-corner clipping — plus VideoThumbnail updates. Why: the decode layer for timeline clip thumbnails, ahead of the visual refresh that renders them. How: new modules + one modified file; purely presentational. Test plan: bunx vitest run on both test files; tsc --noEmit; fallow audit clean. --- .../components/nle/AssetPreviewOverlay.tsx | 4 +- .../src/components/nle/useCompositionStack.ts | 5 +- .../src/components/sidebar/AssetCard.tsx | 26 ++- .../src/components/sidebar/AudioRow.tsx | 3 +- .../player/components/ImageThumbnail.test.tsx | 173 ++++++++++++++++++ .../src/player/components/ImageThumbnail.tsx | 160 ++++++++++++++++ .../player/components/VideoThumbnail.test.tsx | 152 +++++++++++++++ .../src/player/components/VideoThumbnail.tsx | 35 +++- .../player/components/thumbnailUtils.test.ts | 127 +++++++++++++ .../src/player/components/thumbnailUtils.ts | 54 ++++++ 10 files changed, 719 insertions(+), 20 deletions(-) create mode 100644 packages/studio/src/player/components/ImageThumbnail.test.tsx create mode 100644 packages/studio/src/player/components/ImageThumbnail.tsx create mode 100644 packages/studio/src/player/components/VideoThumbnail.test.tsx create mode 100644 packages/studio/src/player/components/thumbnailUtils.test.ts create mode 100644 packages/studio/src/player/components/thumbnailUtils.ts diff --git a/packages/studio/src/components/nle/AssetPreviewOverlay.tsx b/packages/studio/src/components/nle/AssetPreviewOverlay.tsx index 5364289f15..6245c4985e 100644 --- a/packages/studio/src/components/nle/AssetPreviewOverlay.tsx +++ b/packages/studio/src/components/nle/AssetPreviewOverlay.tsx @@ -11,6 +11,7 @@ import { useEffect, useCallback } from "react"; import { VIDEO_EXT, IMAGE_EXT } from "../../utils/mediaTypes"; import { useAssetPreviewStore } from "../../utils/assetPreviewStore"; +import { resolveMediaPreviewUrl } from "../../player/components/thumbnailUtils"; function basename(path: string): string { return path.split("/").pop() ?? path; @@ -95,8 +96,7 @@ export function AssetPreviewOverlay() { if (!previewAsset || !previewProjectId) return null; - const encodedAsset = previewAsset.split("/").map(encodeURIComponent).join("/"); - const serveUrl = `/api/projects/${previewProjectId}/preview/${encodedAsset}`; + const serveUrl = resolveMediaPreviewUrl(previewAsset, previewProjectId); const name = basename(previewAsset); return ( diff --git a/packages/studio/src/components/nle/useCompositionStack.ts b/packages/studio/src/components/nle/useCompositionStack.ts index e7e0e2dc7f..77e65ebe28 100644 --- a/packages/studio/src/components/nle/useCompositionStack.ts +++ b/packages/studio/src/components/nle/useCompositionStack.ts @@ -2,6 +2,7 @@ import { useState, useCallback, useRef, useEffect } from "react"; import { usePlayerStore } from "../../player"; import type { CompositionLevel } from "./CompositionBreadcrumb"; +import { encodePreviewPath } from "../../player/components/thumbnailUtils"; interface UseCompositionStackOptions { projectId: string; @@ -84,7 +85,7 @@ export function useCompositionStack({ .split("/") .pop() ?.replace(/\.html$/, "") || resolvedPath; - const previewUrl = `/api/projects/${projectId}/preview/comp/${resolvedPath}`; + const previewUrl = `/api/projects/${projectId}/preview/comp/${encodePreviewPath(resolvedPath)}`; return [...prev, { id: resolvedPath, label, previewUrl }]; }); }, @@ -100,7 +101,7 @@ export function useCompositionStack({ updateCompositionStack((prev) => (prev.length > 1 ? [prev[0]] : prev)); } else if (activeCompositionPath && activeCompositionPath.startsWith("compositions/")) { const label = activeCompositionPath.replace(/^compositions\//, "").replace(/\.html$/, ""); - const previewUrl = `/api/projects/${projectId}/preview/comp/${activeCompositionPath}`; + const previewUrl = `/api/projects/${projectId}/preview/comp/${encodePreviewPath(activeCompositionPath)}`; usePlayerStore.getState().setElements([]); updateCompositionStack((prev) => { if (prev[prev.length - 1]?.id === activeCompositionPath) return prev; diff --git a/packages/studio/src/components/sidebar/AssetCard.tsx b/packages/studio/src/components/sidebar/AssetCard.tsx index e2ef1bf4f6..a083d34d25 100644 --- a/packages/studio/src/components/sidebar/AssetCard.tsx +++ b/packages/studio/src/components/sidebar/AssetCard.tsx @@ -11,6 +11,7 @@ import { usePlayerStore } from "../../player/store/playerStore"; import { useAssetPreviewStore } from "../../utils/assetPreviewStore"; import { findClipForAsset, isPointerClick } from "../../utils/assetClickBehavior"; import { basename, ext, truncateMiddle, formatDuration } from "./assetHelpers"; +import { resolveMediaPreviewUrl } from "../../player/components/thumbnailUtils"; /** Drag payload writer shared by the asset tile and the font row: copy effect * plus the timeline-asset MIME and a plain-text path fallback. */ @@ -40,25 +41,32 @@ function useProbedDuration(src: string, skip: boolean): number | null | undefine useEffect(() => { if (skip) return; let cancelled = false; + let retryTimer: ReturnType | undefined; + // The in-flight probe element, so unmount cleanup can abort its network + // fetch (clearing `src`) instead of leaving it to finish in the background. + let liveVid: HTMLVideoElement | null = null; + + function teardown(vid: HTMLVideoElement) { + vid.onloadedmetadata = null; + vid.onerror = null; + vid.src = ""; + } function probe(attempt: number) { if (cancelled) return; const vid = document.createElement("video"); + liveVid = vid; vid.preload = "metadata"; vid.muted = true; vid.onloadedmetadata = () => { const d = Number.isFinite(vid.duration) && vid.duration > 0 ? vid.duration : null; + teardown(vid); if (!cancelled) setDuration(d); - vid.onloadedmetadata = null; - vid.onerror = null; - vid.src = ""; }; vid.onerror = () => { - vid.onloadedmetadata = null; - vid.onerror = null; - vid.src = ""; + teardown(vid); if (!cancelled) { - if (attempt < 1) setTimeout(() => probe(attempt + 1), 50); + if (attempt < 1) retryTimer = setTimeout(() => probe(attempt + 1), 50); else setDuration(null); } }; @@ -68,6 +76,8 @@ function useProbedDuration(src: string, skip: boolean): number | null | undefine probe(0); return () => { cancelled = true; + if (retryTimer) clearTimeout(retryTimer); + if (liveVid) teardown(liveVid); }; }, [src, skip]); return duration; @@ -111,7 +121,7 @@ export function AssetCard({ const fullName = asset.split("/").pop() ?? asset; const name = basename(asset); const extension = ext(asset); - const serveUrl = `/api/projects/${projectId}/preview/${asset}`; + const serveUrl = resolveMediaPreviewUrl(asset, projectId); const isVideo = VIDEO_EXT.test(asset); const isImage = IMAGE_EXT.test(asset); const probedDuration = useProbedDuration(serveUrl, !isVideo || duration != null); diff --git a/packages/studio/src/components/sidebar/AudioRow.tsx b/packages/studio/src/components/sidebar/AudioRow.tsx index 46532420a4..445ea6732d 100644 --- a/packages/studio/src/components/sidebar/AudioRow.tsx +++ b/packages/studio/src/components/sidebar/AudioRow.tsx @@ -5,6 +5,7 @@ import { TIMELINE_ASSET_MIME } from "../../utils/timelineAssetDrop"; import { usePlayerStore } from "../../player/store/playerStore"; import { useAssetPreviewStore } from "../../utils/assetPreviewStore"; import { findClipForAsset, isPointerClick } from "../../utils/assetClickBehavior"; +import { resolveMediaPreviewUrl } from "../../player/components/thumbnailUtils"; export function AudioRow({ projectId, @@ -37,7 +38,7 @@ export function AudioRow({ const animRef = useRef(0); const name = basename(asset); const subtype = getAudioSubtype(asset); - const serveUrl = `/api/projects/${projectId}/preview/${asset}`; + const serveUrl = resolveMediaPreviewUrl(asset, projectId); // CapCut-style click behavior: drag-threshold gate. const pointerDownRef = useRef<{ x: number; y: number } | null>(null); diff --git a/packages/studio/src/player/components/ImageThumbnail.test.tsx b/packages/studio/src/player/components/ImageThumbnail.test.tsx new file mode 100644 index 0000000000..f1463a9b40 --- /dev/null +++ b/packages/studio/src/player/components/ImageThumbnail.test.tsx @@ -0,0 +1,173 @@ +// @vitest-environment happy-dom + +import React, { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { ImageThumbnail } from "./ImageThumbnail"; + +Object.defineProperty(globalThis, "IS_REACT_ACT_ENVIRONMENT", { + configurable: true, + value: true, +}); + +// --- Observer stubs: fire "intersecting" immediately on observe --- +class MockIntersectionObserver { + private cb: IntersectionObserverCallback; + constructor(cb: IntersectionObserverCallback) { + this.cb = cb; + } + observe() { + this.cb( + [{ isIntersecting: true } as IntersectionObserverEntry], + this as unknown as IntersectionObserver, + ); + } + disconnect() {} + unobserve() {} + takeRecords(): IntersectionObserverEntry[] { + return []; + } +} + +class MockResizeObserver { + observe() {} + disconnect() {} + unobserve() {} +} + +// --- Image stub: captures instances so tests fire load/error deterministically --- +class MockImage { + static instances: MockImage[] = []; + onload: (() => void) | null = null; + onerror: (() => void) | null = null; + naturalWidth = 0; + naturalHeight = 0; + src = ""; + constructor() { + MockImage.instances.push(this); + } +} + +const originalIO = globalThis.IntersectionObserver; +const originalRO = globalThis.ResizeObserver; +const originalImage = globalThis.Image; + +let host: HTMLDivElement; +let root: Root | null = null; + +beforeEach(() => { + globalThis.IntersectionObserver = + MockIntersectionObserver as unknown as typeof IntersectionObserver; + globalThis.ResizeObserver = MockResizeObserver as unknown as typeof ResizeObserver; + globalThis.Image = MockImage as unknown as typeof Image; + MockImage.instances = []; + host = document.createElement("div"); + document.body.append(host); +}); + +afterEach(() => { + act(() => root?.unmount()); + root = null; + globalThis.IntersectionObserver = originalIO; + globalThis.ResizeObserver = originalRO; + globalThis.Image = originalImage; + document.body.innerHTML = ""; +}); + +function render(props: { imageSrc: string; label?: string; labelColor?: string }) { + root = createRoot(host); + act(() => { + root!.render( + , + ); + }); +} + +function lastProbe(): MockImage { + const probe = MockImage.instances.at(-1); + expect(probe).toBeDefined(); + return probe!; +} + +/** Assert at least one tile rendered and the first tile serves `expectedSrc`. */ +function expectFirstTileSrc(expectedSrc: string): void { + const imgs = [...host.querySelectorAll("img")]; + expect(imgs.length).toBeGreaterThanOrEqual(1); + expect(imgs[0].getAttribute("src")).toBe(expectedSrc); +} + +describe("ImageThumbnail", () => { + it("shows the loading shimmer before the image resolves", () => { + render({ imageSrc: "/api/projects/p/preview/assets/pic.png" }); + expect(host.querySelector(".animate-pulse")).not.toBeNull(); + expect(host.querySelectorAll("img").length).toBe(0); + }); + + it("probes the resolved src and renders repeated object-cover tiles on load", () => { + render({ imageSrc: "/api/projects/p/preview/assets/pic.png" }); + const probe = lastProbe(); + expect(probe.src).toBe("/api/projects/p/preview/assets/pic.png"); + + act(() => { + probe.naturalWidth = 1920; + probe.naturalHeight = 1080; + probe.onload?.(); + }); + + const imgs = [...host.querySelectorAll("img")]; + expect(imgs.length).toBeGreaterThanOrEqual(1); + for (const img of imgs) { + expect(img.getAttribute("src")).toBe("/api/projects/p/preview/assets/pic.png"); + expect(img.className).toContain("object-cover"); + } + expect(host.querySelector(".animate-pulse")).toBeNull(); + }); + + it("drops the shimmer and renders no tiles when a raster image fails to load", () => { + render({ imageSrc: "/api/projects/p/preview/assets/missing.png" }); + act(() => lastProbe().onerror?.()); + expect(host.querySelectorAll("img").length).toBe(0); + expect(host.querySelector(".animate-pulse")).toBeNull(); + }); + + it("renders tiles at 16:9 when an SVG has no intrinsic dimensions (naturalWidth=0)", () => { + render({ imageSrc: "/api/projects/p/preview/assets/logo.svg" }); + const probe = lastProbe(); + + act(() => { + // naturalWidth stays 0 — SVG with no width/height attribute + probe.onload?.(); + }); + + expectFirstTileSrc("/api/projects/p/preview/assets/logo.svg"); + expect(host.querySelector(".animate-pulse")).toBeNull(); + }); + + it("renders SVG tiles at 16:9 fallback even when the probe fires onerror", () => { + // Some browser/sandbox environments fire onerror for SVGs even though the + // element itself can render the file — we must not blank the strip. + render({ imageSrc: "/api/projects/p/preview/assets/icon.svg" }); + + act(() => lastProbe().onerror?.()); + + expectFirstTileSrc("/api/projects/p/preview/assets/icon.svg"); + expect(host.querySelector(".animate-pulse")).toBeNull(); + }); + + it("renders the label above the strip when provided", () => { + render({ imageSrc: "/x.png", label: "hero", labelColor: "#abc" }); + act(() => { + const probe = lastProbe(); + probe.naturalWidth = 100; + probe.naturalHeight = 100; + probe.onload?.(); + }); + const label = [...host.querySelectorAll("span")].find((s) => s.textContent === "hero"); + expect(label).toBeDefined(); + expect(label!.closest(".z-10")).not.toBeNull(); + }); +}); diff --git a/packages/studio/src/player/components/ImageThumbnail.tsx b/packages/studio/src/player/components/ImageThumbnail.tsx new file mode 100644 index 0000000000..33914ae2c1 --- /dev/null +++ b/packages/studio/src/player/components/ImageThumbnail.tsx @@ -0,0 +1,160 @@ +import { memo, useRef, useState, useCallback, useEffect } from "react"; +import { useMountEffect } from "../../hooks/useMountEffect"; +import { computeThumbnailStrip } from "./thumbnailUtils"; + +interface ImageThumbnailProps { + imageSrc: string; + label: string; + labelColor: string; +} + +/** + * Renders a film-strip of a still image for a timeline clip. The image is a + * fixed-width tile (sized by its natural aspect ratio) repeated to fill the + * clip width — matching VideoThumbnail's visual pattern. Loading is lazy + * (IntersectionObserver) with the same shimmer fallback while decoding. + */ +export const ImageThumbnail = memo(function ImageThumbnail({ + imageSrc, + label, + labelColor, +}: ImageThumbnailProps) { + const [containerWidth, setContainerWidth] = useState(0); + const [visible, setVisible] = useState(false); + const [status, setStatus] = useState<"loading" | "loaded" | "error">("loading"); + const [aspect, setAspect] = useState(16 / 9); + const ioRef = useRef(null); + const roRef = useRef(null); + + const setContainerRef = useCallback((el: HTMLDivElement | null) => { + ioRef.current?.disconnect(); + roRef.current?.disconnect(); + if (!el) return; + + const measured = el.parentElement?.clientWidth || el.clientWidth; + setContainerWidth(measured); + + ioRef.current = new IntersectionObserver( + ([entry]) => { + if (entry.isIntersecting) { + setVisible(true); + ioRef.current?.disconnect(); + } + }, + { rootMargin: "200px" }, + ); + // fallow-ignore-next-line code-duplication + ioRef.current.observe(el); + + const target = el.parentElement || el; + roRef.current = new ResizeObserver(([entry]) => { + setContainerWidth(entry.contentRect.width); + }); + roRef.current.observe(target); + }, []); + + useMountEffect(() => () => { + ioRef.current?.disconnect(); + roRef.current?.disconnect(); + }); + + // Probe the image once visible — measures the natural aspect ratio so the + // tile width matches, and flips to the error state (plain clip background) + // if the src can't load. The browser cache makes the tile s free. + // + // SVG handling: SVGs without intrinsic width/height report naturalWidth=0 on + // load (treat as success with the 16:9 default aspect) and may fire onerror + // in some environments even though the file is valid and can be displayed — + // fall back to loaded-at-16:9 rather than hiding the strip entirely. + // eslint-disable-next-line no-restricted-syntax + useEffect(() => { + if (!visible) return; + let cancelled = false; + setStatus("loading"); + + const isSvg = /\.svg($|\?)/i.test(imageSrc); + + const probe = new Image(); + probe.onload = () => { + if (cancelled) return; + if (probe.naturalWidth > 0 && probe.naturalHeight > 0) { + setAspect(probe.naturalWidth / probe.naturalHeight); + } + // naturalWidth===0 (e.g. SVG with no intrinsic dimensions) falls through + // to "loaded" with the default 16:9 aspect already set in state. + setStatus("loaded"); + }; + probe.onerror = () => { + if (cancelled) return; + // SVGs can fail the probe in certain browser/sandbox environments even + // though the tiles themselves render fine (different security + // context). Show the strip at the 16:9 fallback rather than blanking. + if (isSvg) { + setStatus("loaded"); + } else { + setStatus("error"); + } + }; + probe.src = imageSrc; + + return () => { + cancelled = true; + probe.onload = null; + probe.onerror = null; + probe.src = ""; + }; + }, [visible, imageSrc]); + + const { frameW, frameCount } = computeThumbnailStrip(containerWidth, aspect); + + return ( +
+ {visible && status === "loaded" && ( +
+ {Array.from({ length: frameCount }).map((_, i) => ( +
+ +
+ ))} +
+ )} + + {visible && status === "loading" && ( +
+ )} + + {label && ( +
+ + {label} + +
+ )} +
+ ); +}); diff --git a/packages/studio/src/player/components/VideoThumbnail.test.tsx b/packages/studio/src/player/components/VideoThumbnail.test.tsx new file mode 100644 index 0000000000..82f49d8966 --- /dev/null +++ b/packages/studio/src/player/components/VideoThumbnail.test.tsx @@ -0,0 +1,152 @@ +// @vitest-environment happy-dom +import React, { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { VideoThumbnail } from "./VideoThumbnail"; + +Object.defineProperty(globalThis, "IS_REACT_ACT_ENVIRONMENT", { + configurable: true, + value: true, +}); + +// Fire "intersecting" immediately on observe so the extraction effect runs. +class MockIntersectionObserver { + private cb: IntersectionObserverCallback; + constructor(cb: IntersectionObserverCallback) { + this.cb = cb; + } + observe() { + this.cb( + [{ isIntersecting: true } as IntersectionObserverEntry], + this as unknown as IntersectionObserver, + ); + } + disconnect() {} + unobserve() {} + takeRecords(): IntersectionObserverEntry[] { + return []; + } +} + +class MockResizeObserver { + observe() {} + disconnect() {} + unobserve() {} +} + +const originalIO = globalThis.IntersectionObserver; +const originalRO = globalThis.ResizeObserver; + +let host: HTMLDivElement; +let root: Root | null = null; +let createdVideos: HTMLVideoElement[]; + +beforeEach(() => { + globalThis.IntersectionObserver = + MockIntersectionObserver as unknown as typeof IntersectionObserver; + globalThis.ResizeObserver = MockResizeObserver as unknown as typeof ResizeObserver; + + createdVideos = []; + const origCreate = document.createElement.bind(document); + vi.spyOn(document, "createElement").mockImplementation((tag: string) => { + const el = origCreate(tag); + if (tag === "video") createdVideos.push(el as HTMLVideoElement); + return el; + }); + + // happy-dom's