diff --git a/docs/reference/html-schema.mdx b/docs/reference/html-schema.mdx index c71613c98c..f9717430f3 100644 --- a/docs/reference/html-schema.mdx +++ b/docs/reference/html-schema.mdx @@ -126,6 +126,7 @@ Audio has no visual lifecycle. | `data-playback-rate` | Video, audio, nested composition | Playback multiplier from `0.1` to `5` | | `data-volume` | Video and audio | Static volume from `0` to `1` | | `data-has-audio="true"` | Video | Declares that the video contributes audio | +| `data-native-audio` | Audio | Keeps the element on the browser's own output instead of routing it through Web Audio. The runtime already does this automatically for cross-origin media with no `crossorigin` opt-in — the Web Audio spec makes such a source silent — so this is the escape hatch for the cases a URL cannot settle, such as a same-origin path that redirects to a CDN. Native output cannot carry `data-fx-chain`, `data-automation`, `data-audio-group`, or a `data-volume` above `1`; the runtime reports what it dropped, and `hyperframes check` surfaces it as a `web_audio_bypass` finding. | Video and audio may omit `data-duration` when their intrinsic duration is known and the whole remaining source should play. diff --git a/packages/cli/src/utils/checkBrowser.test.ts b/packages/cli/src/utils/checkBrowser.test.ts index 8a6a4920ab..ce83152e32 100644 --- a/packages/cli/src/utils/checkBrowser.test.ts +++ b/packages/cli/src/utils/checkBrowser.test.ts @@ -436,6 +436,53 @@ it("surfaces the runtime's media-proxy-unavailable console.info line as its own ); }); +it("surfaces the runtime's web-audio-bypass console.info line as its own info finding", async () => { + // The whole complaint in #3458 is that nothing was reported. The runtime + // emits this from media DISCOVERY, not from playback scheduling, precisely + // because `check` seeks and never calls play() — a diagnostic raised from + // the transport would never reach this scraper. + vi.spyOn(Date, "now").mockReturnValue(100); + mountCanvasFixture(); + const page = fakePage(); + const bypassMessage = fakeConsoleMessage( + "info", + '[hyperframes] runtime_web_audio_bypass: "https://cdn.example.com/track.mp3" ' + + "(cross_origin_no_cors): Web Audio capture withheld; the track plays through native " + + "HTMLMediaElement output. Native playback cannot reproduce: fx-chain — proxy or download " + + "the asset to a same-origin URL to keep it.", + ); + const authorInfo = fakeConsoleMessage("info", "debug runtime_web_audio_bypass lookalike"); + page.on = vi.fn( + (event: string, handler: (message: ReturnType) => void) => { + if (event === "console") { + handler(bypassMessage); + handler(authorInfo); + } + }, + ); + installSessionMock(page); + + const result = await runBrowserCheck( + PROJECT, + { ...DEFAULT_CHECK_OPTIONS, samples: 1, contrast: false }, + { kind: "none" }, + runAuditGrid, + ); + + expect(result.runtimeFindings).toContainEqual( + expect.objectContaining({ + code: "web_audio_bypass", + severity: "info", + message: bypassMessage.text(), + }), + ); + // Prefix-anchored, so a composition author's own console.info that merely + // mentions the code is not promoted into a finding. + expect(result.runtimeFindings.some((finding) => finding.message === authorInfo.text())).toBe( + false, + ); +}); + it("elevates and deduplicates WebGPU validation warnings while preserving ordinary warnings", async () => { vi.spyOn(Date, "now").mockReturnValue(100); mountCanvasFixture(); diff --git a/packages/cli/src/utils/checkBrowser.ts b/packages/cli/src/utils/checkBrowser.ts index 012708b217..b84f4e92ca 100644 --- a/packages/cli/src/utils/checkBrowser.ts +++ b/packages/cli/src/utils/checkBrowser.ts @@ -264,6 +264,11 @@ export async function captureFindingCrops( // `console.info` from a composition author's own script must not. const MEDIA_PROXY_MARKER_PREFIX = "[hyperframes] runtime_media_proxy_"; const MEDIA_PROXY_UNAVAILABLE_MARKER = "[hyperframes] runtime_media_proxy_unavailable"; +// `reportWebAudioMediaRoute` (packages/core/src/runtime/webAudioRoute.ts) uses +// the same code-in-the-console-line contract. It is emitted from the media +// DISCOVERY phase rather than from playback scheduling, precisely so this +// scraper can see it — `check` seeks, it never plays. +const WEB_AUDIO_BYPASS_MARKER = "[hyperframes] runtime_web_audio_bypass"; const WEBGPU_RUNTIME_FAILURE = /\b(?:GPUValidationError|GPUOutOfMemoryError|GPUInternalError)\b|WebGPU uncaptured error|(?:destroyed\b.*\b(?:GPU )?(?:resource|buffer|texture)\b.*\bsubmit)|(?:(?:GPU )?(?:resource|buffer|texture)\b.*\bdestroyed\b.*\bsubmit)/i; @@ -290,6 +295,20 @@ function pushRuntimeDraft(drafts: RuntimeDraft[], draft: RuntimeDraft): void { drafts.push({ ...draft, count: 1 }); } +/** + * The finding code for a runtime-emitted `console.info` line, or null for the + * ordinary info logging a composition author's own script produces. Matching is + * prefix-anchored on the stable diagnostic codes the runtime deliberately embeds + * in the text, so a line that merely mentions one is not promoted. + */ +function runtimeInfoFindingCode(text: string): string | null { + if (text.startsWith(WEB_AUDIO_BYPASS_MARKER)) return "web_audio_bypass"; + if (!text.startsWith(MEDIA_PROXY_MARKER_PREFIX)) return null; + return text.includes(MEDIA_PROXY_UNAVAILABLE_MARKER) + ? "media_proxy_unavailable" + : "media_proxy_fallback"; +} + function wireRuntimeListeners(page: Page, drafts: RuntimeDraft[], currentTime: () => number): void { page.on("console", (message) => { const type = message.type(); @@ -315,12 +334,12 @@ function wireRuntimeListeners(page: Page, drafts: RuntimeDraft[], currentTime: ( url: location.url, line: location.lineNumber, }); - } else if (type === "info" && text.startsWith(MEDIA_PROXY_MARKER_PREFIX)) { + } else if (type === "info") { + const code = runtimeInfoFindingCode(text); + if (!code) return; const location = message.location(); pushRuntimeDraft(drafts, { - code: text.includes(MEDIA_PROXY_UNAVAILABLE_MARKER) - ? "media_proxy_unavailable" - : "media_proxy_fallback", + code, severity: "info", message: text, time: currentTime(), diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.ts index 262a08321d..dc8321fae6 100644 --- a/packages/core/src/runtime/init.test.ts +++ b/packages/core/src/runtime/init.test.ts @@ -3053,4 +3053,236 @@ describe("initSandboxRuntimeModular", () => { }).not.toThrow(); }); }); + + // #3458: cross-origin media with no CORS opt-in. `createMediaElementSource` + // returns a node that outputs silence per the Web Audio spec rather than + // throwing, so the composition played through with visuals animating and no + // sound, and nothing was logged. + describe("cross-origin audio without a CORS opt-in", () => { + // `WebAudioTransport.init()` does `new AudioContext()`, which jsdom does not + // provide — without a stub it returns false, `webAudioReady` stays false, + // and `scheduleWebAudioForActiveClips` is never reached at all, so every + // assertion below would pass for the wrong reason. + class MockAudioContext { + currentTime = 0; + state = "running"; + destination = {}; + resume() { + return Promise.resolve(); + } + createGain() { + return { gain: { value: 1 }, connect() {}, disconnect() {} }; + } + } + const originalAudioContext = (globalThis as Record).AudioContext; + + beforeEach(() => { + (globalThis as Record).AudioContext = MockAudioContext; + }); + + afterEach(() => { + (globalThis as Record).AudioContext = originalAudioContext; + }); + + /** `webAudio.init()` resolves on a microtask, so `webAudioReady` is still + * false on the tick `initSandboxRuntimeModular()` returns. */ + async function startPlayback() { + initSandboxRuntimeModular(); + await Promise.resolve(); + window.__player?.play(); + await Promise.resolve(); + await Promise.resolve(); + } + + function mountAudio(src: string, attrs: Record = {}) { + const root = document.createElement("div"); + root.setAttribute("data-composition-id", "main"); + root.setAttribute("data-root", "true"); + root.setAttribute("data-start", "0"); + root.setAttribute("data-duration", "10"); + root.setAttribute("data-width", "1920"); + root.setAttribute("data-height", "1080"); + document.body.appendChild(root); + + const audio = document.createElement("audio"); + audio.setAttribute("data-start", "0"); + audio.setAttribute("data-duration", "10"); + audio.setAttribute("src", src); + for (const [name, value] of Object.entries(attrs)) audio.setAttribute(name, value); + audio.load = () => {}; + audio.play = vi.fn(() => Promise.resolve()); + root.appendChild(audio); + + window.__timelines = { main: createMockTimeline(10) }; + return audio; + } + + it("withholds Web Audio capture but still tries decode, which keeps the FX graph", async () => { + // Decode is the BEST outcome here, not a consolation: a CDN that sends + // `Access-Control-Allow-Origin` while the author simply never wrote the + // `crossorigin` attribute decodes fine, and that route keeps every + // effect and automation lane the media-element route would have had. + const audio = mountAudio("https://cdn.example.com/track.mp3"); + vi.spyOn(console, "info").mockImplementation(() => {}); + const captureSpy = vi.spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback"); + const decodeSpy = vi + .spyOn(WebAudioTransport.prototype, "decodeAudioElement") + .mockResolvedValue(null); + + await startPlayback(); + + expect(captureSpy).not.toHaveBeenCalled(); + expect(decodeSpy).toHaveBeenCalledWith(audio); + }); + + it("leaves the element audible on native output when decode also fails", async () => { + const audio = mountAudio("https://cdn.example.com/track.mp3"); + vi.spyOn(console, "info").mockImplementation(() => {}); + vi.spyOn(WebAudioTransport.prototype, "decodeAudioElement").mockResolvedValue(null); + + await startPlayback(); + + // The three things that add up to "the user hears it". + expect(audio.muted).toBe(false); + expect(audio.volume).toBeGreaterThan(0); + expect(audio.play).toHaveBeenCalled(); + expect(window.__player?.isPlaying()).toBe(true); + }); + + it("does not fail closed into silence for an FX track it deliberately withheld", async () => { + // The pre-existing non-unit-rate rule mutes a processed track rather than + // let it lose its graph. On this route capture was withheld ON PURPOSE + // and native output IS the fix, so muting would hand back the exact + // silence being fixed — now with the runtime's blessing. + const audio = mountAudio("https://cdn.example.com/track.mp3", { + "data-fx-chain": "[]", + "data-playback-rate": "2", + }); + vi.spyOn(console, "info").mockImplementation(() => {}); + vi.spyOn(WebAudioTransport.prototype, "decodeAudioElement").mockResolvedValue(null); + + await startPlayback(); + + expect(audio.muted).toBe(false); + }); + + it("reports the bypass at media discovery, without anyone calling play()", () => { + // `hyperframes check` seeks, it never plays. A diagnostic raised only + // from the schedule path would be invisible to the one gate whose job is + // to surface this. + mountAudio("https://cdn.example.com/track.mp3", { "data-fx-chain": "[]" }); + const info = vi.spyOn(console, "info").mockImplementation(() => {}); + + initSandboxRuntimeModular(); + + const line = info.mock.calls.find(([first]) => + String(first).includes("runtime_web_audio_bypass"), + ); + expect(line).toBeDefined(); + // Names what native playback cannot carry, so the author knows the track + // is audible but no longer processed. + expect(String(line?.[0])).toContain("fx-chain"); + }); + + it("says nothing about a cross-origin