From 88f4603f7e55ee29651656a83ea01472eaa43d94 Mon Sep 17 00:00:00 2001 From: desenmeng Date: Mon, 24 Aug 2026 19:24:51 +0800 Subject: [PATCH] fix(core): stop createMediaElementSource silencing cross-origin audio Cross-origin media without a crossorigin opt-in produces silence when captured through createMediaElementSource. The call succeeds instead of throwing and permanently reroutes the element, so the existing fallback cannot recover after capture. Classify audio before capture and keep three explicit routes: eligible media stays on Web Audio, unsafe cross-origin media skips capture while retaining the decode/native fallback chain, and data-native-audio opts an element out of both Web Audio paths. Recheck the route at the transport's irreversible capture boundary and account for currentSrc, src, and source children. Emit a stable preview diagnostic that hyperframes check can surface, while suppressing it during export rendering where audio is mixed offline. Keep the existing non-unit-rate fail-closed rule scoped to fx-chain and automation so this fix does not newly mute grouped or above-unity tracks. Document the escape hatch and cover routing, fallback, diagnostics, and CLI reporting with regression tests. Fixes #3458 --- docs/reference/html-schema.mdx | 1 + packages/cli/src/utils/checkBrowser.test.ts | 47 ++++ packages/cli/src/utils/checkBrowser.ts | 27 +- packages/core/src/runtime/init.test.ts | 232 ++++++++++++++++ packages/core/src/runtime/init.ts | 112 ++++++-- .../core/src/runtime/webAudioRoute.test.ts | 260 ++++++++++++++++++ packages/core/src/runtime/webAudioRoute.ts | 239 ++++++++++++++++ .../src/runtime/webAudioTransport.test.ts | 58 ++++ .../core/src/runtime/webAudioTransport.ts | 38 ++- 9 files changed, 976 insertions(+), 38 deletions(-) create mode 100644 packages/core/src/runtime/webAudioRoute.test.ts create mode 100644 packages/core/src/runtime/webAudioRoute.ts diff --git a/docs/reference/html-schema.mdx b/docs/reference/html-schema.mdx index c71613c98c..f9717430f3 100644 --- a/docs/reference/html-schema.mdx +++ b/docs/reference/html-schema.mdx @@ -126,6 +126,7 @@ Audio has no visual lifecycle. | `data-playback-rate` | Video, audio, nested composition | Playback multiplier from `0.1` to `5` | | `data-volume` | Video and audio | Static volume from `0` to `1` | | `data-has-audio="true"` | Video | Declares that the video contributes audio | +| `data-native-audio` | Audio | Keeps the element on the browser's own output instead of routing it through Web Audio. The runtime already does this automatically for cross-origin media with no `crossorigin` opt-in — the Web Audio spec makes such a source silent — so this is the escape hatch for the cases a URL cannot settle, such as a same-origin path that redirects to a CDN. Native output cannot carry `data-fx-chain`, `data-automation`, `data-audio-group`, or a `data-volume` above `1`; the runtime reports what it dropped, and `hyperframes check` surfaces it as a `web_audio_bypass` finding. | Video and audio may omit `data-duration` when their intrinsic duration is known and the whole remaining source should play. diff --git a/packages/cli/src/utils/checkBrowser.test.ts b/packages/cli/src/utils/checkBrowser.test.ts index 8a6a4920ab..ce83152e32 100644 --- a/packages/cli/src/utils/checkBrowser.test.ts +++ b/packages/cli/src/utils/checkBrowser.test.ts @@ -436,6 +436,53 @@ it("surfaces the runtime's media-proxy-unavailable console.info line as its own ); }); +it("surfaces the runtime's web-audio-bypass console.info line as its own info finding", async () => { + // The whole complaint in #3458 is that nothing was reported. The runtime + // emits this from media DISCOVERY, not from playback scheduling, precisely + // because `check` seeks and never calls play() — a diagnostic raised from + // the transport would never reach this scraper. + vi.spyOn(Date, "now").mockReturnValue(100); + mountCanvasFixture(); + const page = fakePage(); + const bypassMessage = fakeConsoleMessage( + "info", + '[hyperframes] runtime_web_audio_bypass: "https://cdn.example.com/track.mp3" ' + + "(cross_origin_no_cors): Web Audio capture withheld; the track plays through native " + + "HTMLMediaElement output. Native playback cannot reproduce: fx-chain — proxy or download " + + "the asset to a same-origin URL to keep it.", + ); + const authorInfo = fakeConsoleMessage("info", "debug runtime_web_audio_bypass lookalike"); + page.on = vi.fn( + (event: string, handler: (message: ReturnType) => void) => { + if (event === "console") { + handler(bypassMessage); + handler(authorInfo); + } + }, + ); + installSessionMock(page); + + const result = await runBrowserCheck( + PROJECT, + { ...DEFAULT_CHECK_OPTIONS, samples: 1, contrast: false }, + { kind: "none" }, + runAuditGrid, + ); + + expect(result.runtimeFindings).toContainEqual( + expect.objectContaining({ + code: "web_audio_bypass", + severity: "info", + message: bypassMessage.text(), + }), + ); + // Prefix-anchored, so a composition author's own console.info that merely + // mentions the code is not promoted into a finding. + expect(result.runtimeFindings.some((finding) => finding.message === authorInfo.text())).toBe( + false, + ); +}); + it("elevates and deduplicates WebGPU validation warnings while preserving ordinary warnings", async () => { vi.spyOn(Date, "now").mockReturnValue(100); mountCanvasFixture(); diff --git a/packages/cli/src/utils/checkBrowser.ts b/packages/cli/src/utils/checkBrowser.ts index 012708b217..b84f4e92ca 100644 --- a/packages/cli/src/utils/checkBrowser.ts +++ b/packages/cli/src/utils/checkBrowser.ts @@ -264,6 +264,11 @@ export async function captureFindingCrops( // `console.info` from a composition author's own script must not. const MEDIA_PROXY_MARKER_PREFIX = "[hyperframes] runtime_media_proxy_"; const MEDIA_PROXY_UNAVAILABLE_MARKER = "[hyperframes] runtime_media_proxy_unavailable"; +// `reportWebAudioMediaRoute` (packages/core/src/runtime/webAudioRoute.ts) uses +// the same code-in-the-console-line contract. It is emitted from the media +// DISCOVERY phase rather than from playback scheduling, precisely so this +// scraper can see it — `check` seeks, it never plays. +const WEB_AUDIO_BYPASS_MARKER = "[hyperframes] runtime_web_audio_bypass"; const WEBGPU_RUNTIME_FAILURE = /\b(?:GPUValidationError|GPUOutOfMemoryError|GPUInternalError)\b|WebGPU uncaptured error|(?:destroyed\b.*\b(?:GPU )?(?:resource|buffer|texture)\b.*\bsubmit)|(?:(?:GPU )?(?:resource|buffer|texture)\b.*\bdestroyed\b.*\bsubmit)/i; @@ -290,6 +295,20 @@ function pushRuntimeDraft(drafts: RuntimeDraft[], draft: RuntimeDraft): void { drafts.push({ ...draft, count: 1 }); } +/** + * The finding code for a runtime-emitted `console.info` line, or null for the + * ordinary info logging a composition author's own script produces. Matching is + * prefix-anchored on the stable diagnostic codes the runtime deliberately embeds + * in the text, so a line that merely mentions one is not promoted. + */ +function runtimeInfoFindingCode(text: string): string | null { + if (text.startsWith(WEB_AUDIO_BYPASS_MARKER)) return "web_audio_bypass"; + if (!text.startsWith(MEDIA_PROXY_MARKER_PREFIX)) return null; + return text.includes(MEDIA_PROXY_UNAVAILABLE_MARKER) + ? "media_proxy_unavailable" + : "media_proxy_fallback"; +} + function wireRuntimeListeners(page: Page, drafts: RuntimeDraft[], currentTime: () => number): void { page.on("console", (message) => { const type = message.type(); @@ -315,12 +334,12 @@ function wireRuntimeListeners(page: Page, drafts: RuntimeDraft[], currentTime: ( url: location.url, line: location.lineNumber, }); - } else if (type === "info" && text.startsWith(MEDIA_PROXY_MARKER_PREFIX)) { + } else if (type === "info") { + const code = runtimeInfoFindingCode(text); + if (!code) return; const location = message.location(); pushRuntimeDraft(drafts, { - code: text.includes(MEDIA_PROXY_UNAVAILABLE_MARKER) - ? "media_proxy_unavailable" - : "media_proxy_fallback", + code, severity: "info", message: text, time: currentTime(), diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.ts index 262a08321d..dc8321fae6 100644 --- a/packages/core/src/runtime/init.test.ts +++ b/packages/core/src/runtime/init.test.ts @@ -3053,4 +3053,236 @@ describe("initSandboxRuntimeModular", () => { }).not.toThrow(); }); }); + + // #3458: cross-origin media with no CORS opt-in. `createMediaElementSource` + // returns a node that outputs silence per the Web Audio spec rather than + // throwing, so the composition played through with visuals animating and no + // sound, and nothing was logged. + describe("cross-origin audio without a CORS opt-in", () => { + // `WebAudioTransport.init()` does `new AudioContext()`, which jsdom does not + // provide — without a stub it returns false, `webAudioReady` stays false, + // and `scheduleWebAudioForActiveClips` is never reached at all, so every + // assertion below would pass for the wrong reason. + class MockAudioContext { + currentTime = 0; + state = "running"; + destination = {}; + resume() { + return Promise.resolve(); + } + createGain() { + return { gain: { value: 1 }, connect() {}, disconnect() {} }; + } + } + const originalAudioContext = (globalThis as Record).AudioContext; + + beforeEach(() => { + (globalThis as Record).AudioContext = MockAudioContext; + }); + + afterEach(() => { + (globalThis as Record).AudioContext = originalAudioContext; + }); + + /** `webAudio.init()` resolves on a microtask, so `webAudioReady` is still + * false on the tick `initSandboxRuntimeModular()` returns. */ + async function startPlayback() { + initSandboxRuntimeModular(); + await Promise.resolve(); + window.__player?.play(); + await Promise.resolve(); + await Promise.resolve(); + } + + function mountAudio(src: string, attrs: Record = {}) { + const root = document.createElement("div"); + root.setAttribute("data-composition-id", "main"); + root.setAttribute("data-root", "true"); + root.setAttribute("data-start", "0"); + root.setAttribute("data-duration", "10"); + root.setAttribute("data-width", "1920"); + root.setAttribute("data-height", "1080"); + document.body.appendChild(root); + + const audio = document.createElement("audio"); + audio.setAttribute("data-start", "0"); + audio.setAttribute("data-duration", "10"); + audio.setAttribute("src", src); + for (const [name, value] of Object.entries(attrs)) audio.setAttribute(name, value); + audio.load = () => {}; + audio.play = vi.fn(() => Promise.resolve()); + root.appendChild(audio); + + window.__timelines = { main: createMockTimeline(10) }; + return audio; + } + + it("withholds Web Audio capture but still tries decode, which keeps the FX graph", async () => { + // Decode is the BEST outcome here, not a consolation: a CDN that sends + // `Access-Control-Allow-Origin` while the author simply never wrote the + // `crossorigin` attribute decodes fine, and that route keeps every + // effect and automation lane the media-element route would have had. + const audio = mountAudio("https://cdn.example.com/track.mp3"); + vi.spyOn(console, "info").mockImplementation(() => {}); + const captureSpy = vi.spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback"); + const decodeSpy = vi + .spyOn(WebAudioTransport.prototype, "decodeAudioElement") + .mockResolvedValue(null); + + await startPlayback(); + + expect(captureSpy).not.toHaveBeenCalled(); + expect(decodeSpy).toHaveBeenCalledWith(audio); + }); + + it("leaves the element audible on native output when decode also fails", async () => { + const audio = mountAudio("https://cdn.example.com/track.mp3"); + vi.spyOn(console, "info").mockImplementation(() => {}); + vi.spyOn(WebAudioTransport.prototype, "decodeAudioElement").mockResolvedValue(null); + + await startPlayback(); + + // The three things that add up to "the user hears it". + expect(audio.muted).toBe(false); + expect(audio.volume).toBeGreaterThan(0); + expect(audio.play).toHaveBeenCalled(); + expect(window.__player?.isPlaying()).toBe(true); + }); + + it("does not fail closed into silence for an FX track it deliberately withheld", async () => { + // The pre-existing non-unit-rate rule mutes a processed track rather than + // let it lose its graph. On this route capture was withheld ON PURPOSE + // and native output IS the fix, so muting would hand back the exact + // silence being fixed — now with the runtime's blessing. + const audio = mountAudio("https://cdn.example.com/track.mp3", { + "data-fx-chain": "[]", + "data-playback-rate": "2", + }); + vi.spyOn(console, "info").mockImplementation(() => {}); + vi.spyOn(WebAudioTransport.prototype, "decodeAudioElement").mockResolvedValue(null); + + await startPlayback(); + + expect(audio.muted).toBe(false); + }); + + it("reports the bypass at media discovery, without anyone calling play()", () => { + // `hyperframes check` seeks, it never plays. A diagnostic raised only + // from the schedule path would be invisible to the one gate whose job is + // to surface this. + mountAudio("https://cdn.example.com/track.mp3", { "data-fx-chain": "[]" }); + const info = vi.spyOn(console, "info").mockImplementation(() => {}); + + initSandboxRuntimeModular(); + + const line = info.mock.calls.find(([first]) => + String(first).includes("runtime_web_audio_bypass"), + ); + expect(line).toBeDefined(); + // Names what native playback cannot carry, so the author knows the track + // is audible but no longer processed. + expect(String(line?.[0])).toContain("fx-chain"); + }); + + it("says nothing about a cross-origin