diff --git a/docs/PRD-memory-optimization.md b/docs/PRD-memory-optimization.md new file mode 100644 index 00000000..c0730de9 --- /dev/null +++ b/docs/PRD-memory-optimization.md @@ -0,0 +1,306 @@ +# PRD: Reduce Memory Usage in Recordly + +**Author:** Gurpreet Kait +**Date:** 2026-05-09 +**Branch:** `refactor/reduce-memory-usage` +**Status:** Draft + +--- + +## 1. Why This Matters + +When a user records their screen for 20+ minutes or exports a long video at high resolution, Recordly's memory usage climbs steadily. On machines with 8 GB of RAM, this can cause the system to slow down, show memory warnings, or in worst cases crash the app entirely. + +The root cause is not one big leak — it's several small things adding up: log strings that grow forever, arrays that hold duplicate data, canvases that aren't freed after use, and decode buffers sized too generously for the hardware. + +This PRD documents the specific issues found, what we plan to fix, and what we're leaving for later. + +--- + +## 2. Goals + +- Users should be able to record for 30+ minutes without the app getting sluggish +- Exporting a 10-minute 1080p video should not spike memory to multiple GB +- After an export finishes, memory should drop back to baseline (no lingering waste) +- Zero changes to output quality or features — this is purely internal cleanup + +## 3. Non-Goals + +- Rewriting the rendering engine or export pipeline +- Optimizing CPU usage or disk I/O +- Reducing memory of the video preview/editor (future work) + +--- + +## 4. Current Issues Identified + +We audited the full recording and export pipelines. Below are the real memory problems we found, ordered by how much memory they waste in practice. + +### Issue 1: Audio decode temporarily doubles memory for long recordings + +**When it happens:** Every time a user exports a recording that has audio (which is almost always). + +**What goes wrong:** When preparing audio for export, Recordly decodes the full audio track into small chunks, then copies all those chunks into a single final buffer. During the copy step, both the chunks and the final buffer exist in memory at the same time — temporarily doubling the audio memory. This happens regardless of video resolution since audio size only depends on duration and sample rate. + +**Real-world impact (stereo, 48 kHz):** +- 5-minute recording: ~55 MB wasted (110 MB peak instead of 55 MB) +- 10-minute recording: ~220 MB wasted (440 MB peak instead of 220 MB) +- **30-minute recording: ~660 MB wasted** (1.3 GB peak instead of 660 MB) + +**Where:** `src/lib/exporter/audioEncoder.ts` — the `streamDecodeFromUrl()` method holds `channelChunks[][]` and the final `AudioBuffer` simultaneously. + +--- + +### Issue 2: Recording logs grow without limit + +**When it happens:** During every screen recording, especially long ones. + +**What goes wrong:** Recordly spawns a native helper process (on macOS, Windows, or via FFmpeg) to capture the screen. The stdout and stderr output from these processes is stored in string variables that grow for the entire duration of the recording. These logs are only useful for error diagnostics, but they're never trimmed. + +**Real-world impact:** A 1-hour recording with a verbose capture helper can accumulate 2-5 MB of log strings per process. With both stdout and stderr, and sometimes multiple processes (capture + cursor monitor), this can reach 10+ MB of strings sitting in the main process memory doing nothing. + +**Where:** +- macOS: `electron/ipc/register/recording.ts` — `nativeCaptureOutputBuffer` +- Windows: same file — `windowsCaptureOutputBuffer` via local `captureOutput` +- FFmpeg: same file — `ffmpegCaptureOutputBuffer` + +--- + +### Issue 3: FFmpeg export logs grow without limit + +**When it happens:** During every native (Breeze) video export. + +**What goes wrong:** Same pattern as Issue 2, but during export instead of recording. FFmpeg's stderr output is appended to a string (`session.stderrOutput`) for the entire duration of the export with no cap. + +**Real-world impact:** A 30-minute 4K export can produce 1-3 MB of FFmpeg log output. Not huge on its own, but it adds up alongside all the other export memory. + +**Where:** `electron/ipc/register/export.ts` — the `stderrOutput` field on `NativeVideoExportSession`. + +--- + +### Issue 4: Cursor tracking data is duplicated in memory + +**When it happens:** During any recording longer than a few minutes. + +**What goes wrong:** Recordly samples cursor position 30 times per second during recording. Periodically, these samples are "snapshotted" for persistence — copied from an `activeCursorSamples` array into a `pendingCursorSamples` array. But the active array is never cleared after the copy, so both arrays hold the same data. + +**Real-world impact:** +- 10-minute recording: ~18K samples duplicated = ~3.4 MB wasted +- 30-minute recording: ~54K samples duplicated = ~10 MB wasted + +**Where:** `electron/ipc/cursor/telemetry.ts` — `snapshotCursorTelemetryForPersistence()` copies but doesn't clear `activeCursorSamples`. + +--- + +### Issue 5: Export canvases not fully cleaned up after export + +**When it happens:** After every video export finishes. + +**What goes wrong:** The frame renderer (`FrameRenderer`) allocates several HTML canvases at the output resolution for compositing, staging video frames, and drawing backgrounds. When the export finishes and `destroy()` is called, most of these canvases are released — but four references are missed: +- `backgroundVideoFrameStagingCanvas` and its context +- `compositeCanvas` and its context + +These canvases stay in memory until JavaScript's garbage collector eventually gets to them, which in Electron can take a long time. + +**Real-world impact:** ~16 MB of canvas memory lingers after each 1080p export. **At 4K, that's ~64 MB.** If a user does multiple exports in a row, this stacks up. + +**Where:** `src/lib/exporter/modernFrameRenderer.ts` — the `destroy()` method at line 3336. + +--- + +### Issue 6: Frame capture canvas never released + +**When it happens:** After the first native (Breeze) export in a session. + +**What goes wrong:** The native frame capture module uses a fallback canvas for reading pixel data from the GPU. This canvas is created at the output resolution and stored in a module-level variable — meaning it lives forever once created, even after the export is done and the user goes back to editing. + +**Real-world impact:** ~8 MB at 1080p, **~33 MB at 4K**, persisting for the rest of the app session. + +**Where:** `src/lib/exporter/nativeFrameCapture.ts` — module-level `fallbackCanvas` and `fallbackContext`. + +--- + +### Issue 7: Too many video frames buffered on low-end machines + +**When it happens:** When exporting on a machine with 4 or fewer CPU cores, or at very high resolutions (4K+). + +**What goes wrong:** Recordly uses a "backpressure" system to control how many decoded video frames sit in memory waiting to be encoded. The system has a "conservative" mode for slower machines, but even this mode allows 12-20 pending frames. At high resolutions, each frame is large. + +**Real-world impact:** +- 1080p export on 4-core machine: 12 frames × ~8 MB = ~96 MB in buffers (acceptable) +- 4K export on 4-core machine: 12 frames × ~33 MB = ~396 MB in buffers (too much) + +The machine is already constrained — holding this much data increases the chance of the OS swapping to disk, which makes the export even slower. + +**Where:** `src/lib/exporter/exportTuning.ts` — the `breeze-conservative` and `webcodecs-conservative` profiles. + +--- + +### Known issues NOT addressed here (future work) + +These are real but require bigger changes or UX decisions: + +- **Undo history has no depth limit** — Each undo snapshot in the editor stores zoom/clip region arrays. Heavy editing sessions can accumulate significant undo data. Fixing this requires deciding how many undo levels to support (UX decision). +- **Full video loaded for preview** — The editor loads the entire video for playback preview. Streaming preview would save memory but requires rearchitecting the preview pipeline. +- **Source audio buffers held during offline render** — The offline audio pipeline processes in 30-second chunks, but keeps the full decoded source audio in memory for the entire render. Streaming decode during render would fix this but adds significant complexity. + +--- + +## 5. How Recordly Uses Memory (Simplified) + +### During Recording + +```text +User hits Record + └─> Spawn capture process (macOS / Windows / FFmpeg) + └─> Sample cursor position 30x per second into an array + └─> Capture process logs accumulate in string buffers + └─> On Stop: save cursor data to disk, combine audio tracks +``` + +Memory grows linearly with recording duration via log buffers and cursor arrays. + +### During Export + +```text +User hits Export + └─> Decode video frames one-by-one (WebCodecs) + └─> Render each frame with effects (PixiJS + canvases) + └─> Encode frame (WebCodecs or FFmpeg pipe) + └─> Decode + process audio (full track into memory) + └─> Combine video + audio into final MP4 +``` + +Peak memory is dominated by: decoded frame buffers, PixiJS GPU context, audio decode buffers, and staging canvases. + +--- + +## 6. Proposed Fixes + +### Fix 1: Release audio decode chunks as they're copied (Issue 1) + +**What:** After copying each channel's decoded chunks into the final AudioBuffer, immediately clear the chunks array so the intermediate data can be garbage collected. + +**Files:** `src/lib/exporter/audioEncoder.ts` +**Memory saved:** ~660 MB for a 30-minute recording (resolution-independent) +**Risk:** Low — the chunks are never read again after being copied + +--- + +### Fix 2: Cap recording log buffers at 256 KB (Issues 2) + +**What:** Add a constant `MAX_CAPTURE_OUTPUT_BUFFER_LENGTH = 256 KB`. After each log append, if the buffer exceeds this limit, trim it to keep only the most recent 128 KB. We keep the tail because the most recent output is the most useful for debugging. + +**Files:** `electron/ipc/constants.ts`, `electron/ipc/register/recording.ts` +**Memory saved:** ~10 MB for a 30-minute recording +**Risk:** Very low — old log lines are only used for error diagnostics and are rarely needed + +--- + +### Fix 3: Cap FFmpeg export stderr at 256 KB (Issue 3) + +**What:** Same approach as Fix 2, applied to the FFmpeg stderr string during native export. + +**Files:** `electron/ipc/register/export.ts` +**Memory saved:** ~3 MB for a 30-minute 4K export +**Risk:** Very low + +--- + +### Fix 4: Clear cursor samples after snapshotting (Issue 4) + +**What:** After copying active cursor samples into the pending array, set `activeCursorSamples.length = 0`. New samples will continue accumulating from scratch. The next snapshot merges only the new ones. + +**Files:** `electron/ipc/cursor/telemetry.ts` +**Memory saved:** ~10 MB for a 30-minute recording (resolution-independent) +**Risk:** Low — verified that `activeCursorSamples` is only used for pushing new samples and for snapshotting, and is already cleared on recording stop + +--- + +### Fix 5: Null the four missed canvas references in destroy() (Issue 5) + +**What:** Add `this.backgroundVideoFrameStagingCanvas = null`, `this.backgroundVideoFrameStagingCtx = null`, `this.compositeCanvas = null`, and `this.compositeCtx = null` to the `destroy()` method. + +**Files:** `src/lib/exporter/modernFrameRenderer.ts` +**Memory saved:** ~64 MB for a 4K export +**Risk:** Very low — these are simple null assignments following the same pattern as the 20+ other canvas cleanups already in `destroy()` + +--- + +### Fix 6: Release fallback canvas after export (Issue 6) + +**What:** Add a `releaseNativeFrameCaptureResources()` function that nulls the module-level canvas and context. Call it from the exporter's cleanup method after each export. + +**Files:** `src/lib/exporter/nativeFrameCapture.ts`, `src/lib/exporter/modernVideoExporter.ts` +**Memory saved:** ~33 MB for a 4K export +**Risk:** Very low — the canvas is lazily recreated on next use if needed + +--- + +### Fix 7: Lower decode buffer limits on constrained systems (Issue 7) + +**What:** Reduce the conservative backpressure profile limits: +- `breeze-conservative`: pending frames 12 → 8, decode queue 6 → 4 +- `webcodecs-conservative`: pending frames 20 → 12, decode queue 8 → 6 + +**Files:** `src/lib/exporter/exportTuning.ts` +**Memory saved:** ~132 MB for a 4K export on a 4-core machine (4 fewer pending frames × ~33 MB each) +**Risk:** Low-medium — export may be slightly slower on edge-case hardware due to less pipeline buffering, but avoids the much worse scenario of the OS swapping to disk under memory pressure + +--- + +## 7. Summary + +All figures below are for the same reference scenario: **30-minute recording, 4K export, 4-core machine.** + +| Fix | Addresses | Memory Saved | Risk | +|-----|-----------|-------------|------| +| 1. Release audio chunks early | Audio decode doubling | ~660 MB | Low | +| 2. Cap recording log buffers | Unbounded log strings | ~10 MB | Very low | +| 3. Cap FFmpeg export stderr | Unbounded FFmpeg logs | ~3 MB | Very low | +| 4. Clear cursor samples after snapshot | Duplicate cursor data | ~10 MB | Low | +| 5. Null missed canvas refs | Canvas leak in destroy() | ~64 MB | Very low | +| 6. Release fallback canvas | Persistent singleton | ~33 MB | Very low | +| 7. Lower decode buffer limits | Excessive buffering | ~132 MB | Low-medium | + +**Combined savings for this scenario: ~912 MB.** + +Note: Fixes 2 and 4 reduce memory in the main process (during recording), while fixes 1, 3, 5, 6, and 7 reduce memory in the renderer process (during export). The combined figure is the total across both processes for a record-then-export workflow. + +--- + +## 8. Testing Plan + +### Does everything still work? +- [ ] Record 5+ minutes on macOS — video plays correctly, cursor telemetry is intact +- [ ] Record on Windows — same checks +- [ ] Export 1080p with zoom, webcam, annotations, captions — output matches pre-change quality +- [ ] Export as GIF — output is correct +- [ ] Export with speed changes and audio — audio stays in sync +- [ ] Cancel export mid-way — no errors, cleanup runs +- [ ] Record with pause/resume — cursor telemetry is continuous across pauses + +### Is memory actually lower? +- [ ] Record for 15 minutes — main process memory stabilizes (no steady climb) +- [ ] Export a 10-minute 1080p video — peak renderer memory is lower than before +- [ ] Export twice in a row — memory drops back to baseline between exports + +### Edge cases +- [ ] 30+ minute recording — cursor samples are complete in telemetry file +- [ ] Export with no audio — no crash +- [ ] Export on a machine with 4 GB RAM — completes without OOM + +--- + +## 9. Rollback Plan + +Every fix is independent and can be reverted on its own. No data formats, APIs, or file structures are changed. + +| Fix | How to revert | +|-----|---------------| +| 1 | Remove the `channelChunks[ch].length = 0` line | +| 2, 3 | Remove the `if (length > max)` trim blocks | +| 4 | Remove the `activeCursorSamples.length = 0` line | +| 5 | Remove the four added null assignments | +| 6 | Remove the `releaseNativeFrameCaptureResources()` call | +| 7 | Restore the original numeric values in the profile objects | diff --git a/electron/ipc/constants.ts b/electron/ipc/constants.ts index e3c34590..7dbed775 100644 --- a/electron/ipc/constants.ts +++ b/electron/ipc/constants.ts @@ -28,3 +28,6 @@ export const COMPANION_AUDIO_LAYOUTS = [ export const CURSOR_TELEMETRY_VERSION = 2; export const CURSOR_SAMPLE_INTERVAL_MS = 33; export const MAX_CURSOR_SAMPLES = 60 * 60 * 30; // 1 hour @ 30Hz + +export const MAX_CAPTURE_OUTPUT_BUFFER_LENGTH = 256 * 1024; // 256 KB +export const CAPTURE_OUTPUT_BUFFER_TRIM_TARGET = 128 * 1024; // keep most recent 128 KB diff --git a/electron/ipc/cursor/telemetry.ts b/electron/ipc/cursor/telemetry.ts index ebedfe72..b5abeae7 100644 --- a/electron/ipc/cursor/telemetry.ts +++ b/electron/ipc/cursor/telemetry.ts @@ -282,14 +282,16 @@ export function snapshotCursorTelemetryForPersistence() { if (pendingCursorSamples.length === 0) { setPendingCursorSamples([...activeCursorSamples]); - return; + } else { + const lastPendingTimeMs = + pendingCursorSamples[pendingCursorSamples.length - 1]?.timeMs ?? -1; + setPendingCursorSamples([ + ...pendingCursorSamples, + ...activeCursorSamples.filter((sample) => sample.timeMs > lastPendingTimeMs), + ]); } - const lastPendingTimeMs = pendingCursorSamples[pendingCursorSamples.length - 1]?.timeMs ?? -1; - setPendingCursorSamples([ - ...pendingCursorSamples, - ...activeCursorSamples.filter((sample) => sample.timeMs > lastPendingTimeMs), - ]); + activeCursorSamples.length = 0; } export function startCursorSampling() { diff --git a/electron/ipc/register/export.ts b/electron/ipc/register/export.ts index ede2272f..2a6dd57a 100644 --- a/electron/ipc/register/export.ts +++ b/electron/ipc/register/export.ts @@ -42,6 +42,10 @@ import { type NativeExportEncodingMode, type NativeVideoExportFinishOptions, } from "../nativeVideoExport"; +import { + MAX_CAPTURE_OUTPUT_BUFFER_LENGTH, + CAPTURE_OUTPUT_BUFFER_TRIM_TARGET, +} from "../constants"; import { isAllowedLocalReadPath, resolveApprovedLocalMediaPath } from "../project/manager"; import { approveUserPath } from "../utils"; @@ -360,6 +364,11 @@ export function registerExportHandlers() { ffmpegProcess.stderr.on("data", (chunk: Buffer) => { session.stderrOutput += chunk.toString(); + if (session.stderrOutput.length > MAX_CAPTURE_OUTPUT_BUFFER_LENGTH) { + session.stderrOutput = session.stderrOutput.slice( + session.stderrOutput.length - CAPTURE_OUTPUT_BUFFER_TRIM_TARGET, + ); + } }); nativeVideoExportSessions.set(sessionId, session); diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index 9e9e6e96..9c27c12a 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -714,12 +714,16 @@ export function registerRecordingHandlers( console.error("Failed to start native ScreenCaptureKit recording:", error); const errorStr = String(error); - // Detect TCC (screen recording permission) errors and show a helpful dialog - if ( + // Detect TCC (screen recording permission) errors and show a helpful dialog. + // A timeout waiting for the recorder to start is almost always a + // permission issue — ScreenCaptureKit silently stalls when access + // hasn't been granted. + const isPermissionError = errorStr.includes("declined TCC") || errorStr.includes("declined TCCs") || - errorStr.includes("SCREEN_RECORDING_PERMISSION_DENIED") - ) { + errorStr.includes("SCREEN_RECORDING_PERMISSION_DENIED") || + errorStr.includes("Timed out waiting for ScreenCaptureKit"); + if (isPermissionError) { const { response } = await dialog.showMessageBox({ type: "warning", title: "Screen Recording Permission Required", diff --git a/electron/ipc/state.ts b/electron/ipc/state.ts index a0a41744..cd5209ef 100644 --- a/electron/ipc/state.ts +++ b/electron/ipc/state.ts @@ -9,6 +9,17 @@ import type { SystemCursorAsset, WindowBounds, } from "./types"; +import { + MAX_CAPTURE_OUTPUT_BUFFER_LENGTH, + CAPTURE_OUTPUT_BUFFER_TRIM_TARGET, +} from "./constants"; + +function trimCaptureOutputBuffer(buffer: string): string { + if (buffer.length <= MAX_CAPTURE_OUTPUT_BUFFER_LENGTH) { + return buffer; + } + return buffer.slice(buffer.length - CAPTURE_OUTPUT_BUFFER_TRIM_TARGET); +} // ── Source selection ────────────────────────────────────────────────────────── export let selectedSource: SelectedSource | null = null; @@ -129,7 +140,7 @@ export function setNativeCaptureProcess(v: ChildProcessWithoutNullStreams | null nativeCaptureProcess = v; } export function setNativeCaptureOutputBuffer(v: string) { - nativeCaptureOutputBuffer = v; + nativeCaptureOutputBuffer = trimCaptureOutputBuffer(v); } export function setNativeCaptureTargetPath(v: string | null) { nativeCaptureTargetPath = v; @@ -158,7 +169,7 @@ export function setWindowsCaptureProcess(v: ChildProcessWithoutNullStreams | nul windowsCaptureProcess = v; } export function setWindowsCaptureOutputBuffer(v: string) { - windowsCaptureOutputBuffer = v; + windowsCaptureOutputBuffer = trimCaptureOutputBuffer(v); } export function setWindowsCaptureTargetPath(v: string | null) { windowsCaptureTargetPath = v; @@ -196,7 +207,7 @@ export function setFfmpegCaptureProcess(v: ChildProcessWithoutNullStreams | null ffmpegCaptureProcess = v; } export function setFfmpegCaptureOutputBuffer(v: string) { - ffmpegCaptureOutputBuffer = v; + ffmpegCaptureOutputBuffer = trimCaptureOutputBuffer(v); } export function setFfmpegCaptureTargetPath(v: string | null) { ffmpegCaptureTargetPath = v; diff --git a/electron/native/bin/darwin-arm64/recordly-native-cursor-monitor b/electron/native/bin/darwin-arm64/recordly-native-cursor-monitor index e0e478f1..a3faa6c4 100755 Binary files a/electron/native/bin/darwin-arm64/recordly-native-cursor-monitor and b/electron/native/bin/darwin-arm64/recordly-native-cursor-monitor differ diff --git a/electron/native/bin/darwin-arm64/recordly-screencapturekit-helper b/electron/native/bin/darwin-arm64/recordly-screencapturekit-helper index 91b54570..9394d14b 100755 Binary files a/electron/native/bin/darwin-arm64/recordly-screencapturekit-helper and b/electron/native/bin/darwin-arm64/recordly-screencapturekit-helper differ diff --git a/electron/native/bin/darwin-arm64/recordly-system-cursors b/electron/native/bin/darwin-arm64/recordly-system-cursors index f4b41ab6..641e2b51 100755 Binary files a/electron/native/bin/darwin-arm64/recordly-system-cursors and b/electron/native/bin/darwin-arm64/recordly-system-cursors differ diff --git a/electron/native/bin/darwin-arm64/recordly-window-list b/electron/native/bin/darwin-arm64/recordly-window-list index 76a7dab4..6e89cba7 100755 Binary files a/electron/native/bin/darwin-arm64/recordly-window-list and b/electron/native/bin/darwin-arm64/recordly-window-list differ diff --git a/electron/native/bin/darwin-x64/recordly-native-cursor-monitor b/electron/native/bin/darwin-x64/recordly-native-cursor-monitor index d577b1a6..36c1c0fd 100755 Binary files a/electron/native/bin/darwin-x64/recordly-native-cursor-monitor and b/electron/native/bin/darwin-x64/recordly-native-cursor-monitor differ diff --git a/electron/native/bin/darwin-x64/recordly-screencapturekit-helper b/electron/native/bin/darwin-x64/recordly-screencapturekit-helper index 3696e45d..58e49cea 100755 Binary files a/electron/native/bin/darwin-x64/recordly-screencapturekit-helper and b/electron/native/bin/darwin-x64/recordly-screencapturekit-helper differ diff --git a/electron/native/bin/darwin-x64/recordly-system-cursors b/electron/native/bin/darwin-x64/recordly-system-cursors index 54561362..04579222 100755 Binary files a/electron/native/bin/darwin-x64/recordly-system-cursors and b/electron/native/bin/darwin-x64/recordly-system-cursors differ diff --git a/electron/native/bin/darwin-x64/recordly-window-list b/electron/native/bin/darwin-x64/recordly-window-list index e165257a..ee9b2578 100755 Binary files a/electron/native/bin/darwin-x64/recordly-window-list and b/electron/native/bin/darwin-x64/recordly-window-list differ diff --git a/src/components/video-editor/ExportSettingsMenu.tsx b/src/components/video-editor/ExportSettingsMenu.tsx index 08333bbd..de62a979 100644 --- a/src/components/video-editor/ExportSettingsMenu.tsx +++ b/src/components/video-editor/ExportSettingsMenu.tsx @@ -6,6 +6,7 @@ import { useScopedT } from "@/contexts/I18nContext"; import type { ExportEncodingMode, ExportFormat, + ExportMemoryUsage, ExportMp4FrameRate, ExportPipelineModel, ExportQuality, @@ -22,6 +23,8 @@ interface ExportSettingsMenuProps { onExportQualityChange?: (quality: ExportQuality) => void; exportEncodingMode: ExportEncodingMode; onExportEncodingModeChange?: (encodingMode: ExportEncodingMode) => void; + exportMemoryUsage: ExportMemoryUsage; + onExportMemoryUsageChange?: (memoryUsage: ExportMemoryUsage) => void; mp4FrameRate: ExportMp4FrameRate; onMp4FrameRateChange?: (frameRate: ExportMp4FrameRate) => void; exportPipelineModel?: ExportPipelineModel; @@ -45,6 +48,8 @@ export function ExportSettingsMenu({ onExportQualityChange, exportEncodingMode, onExportEncodingModeChange, + exportMemoryUsage, + onExportMemoryUsageChange, mp4FrameRate, onMp4FrameRateChange, exportPipelineModel = "modern", @@ -226,6 +231,59 @@ export function ExportSettingsMenu({ ); })} +
+ + {tSettings("export.memoryTitle", "Memory")} + +
+
+ {( + [ + { value: "low", label: tSettings("export.memory.low", "Low") }, + { + value: "balanced", + label: tSettings("export.memory.balanced", "Balanced"), + }, + { + value: "high", + label: tSettings("export.memory.high", "High"), + }, + ] as const + ).map((option) => { + const isActive = exportMemoryUsage === option.value; + return ( + + ); + })} +
{tSettings("export.fpsTitle", "FPS")} diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 9a4642fa..2dfa5c54 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -58,6 +58,7 @@ import { type ExportBackendPreference, type ExportEncodingMode, type ExportFormat, + type ExportMemoryUsage, type ExportMp4FrameRate, type ExportPipelineModel, type ExportProgress, @@ -722,6 +723,9 @@ export default function VideoEditor() { const [exportEncodingMode, setExportEncodingMode] = useState( initialEditorPreferences.exportEncodingMode, ); + const [exportMemoryUsage, setExportMemoryUsage] = useState( + initialEditorPreferences.exportMemoryUsage, + ); const [exportBackendPreference, setExportBackendPreference] = useState( initialEditorPreferences.exportBackendPreference, ); @@ -2560,6 +2564,7 @@ export default function VideoEditor() { webcam, aspectRatio, exportEncodingMode, + exportMemoryUsage, exportBackendPreference, exportPipelineModel, exportQuality, @@ -2611,6 +2616,7 @@ export default function VideoEditor() { webcam, aspectRatio, exportEncodingMode, + exportMemoryUsage, exportBackendPreference, exportPipelineModel, exportQuality, @@ -4329,6 +4335,7 @@ export default function VideoEditor() { settings.encodingMode ?? exportEncodingMode) : (settings.encodingMode ?? exportEncodingMode); + const memoryUsage = settings.memoryUsage ?? exportMemoryUsage; const selectedMp4FrameRate = smokeExportConfig.enabled ? (smokeExportConfig.fps ?? settings.mp4FrameRate ?? mp4FrameRate) : (settings.mp4FrameRate ?? mp4FrameRate); @@ -4376,6 +4383,7 @@ export default function VideoEditor() { bitrate, codec: DEFAULT_MP4_CODEC, encodingMode, + memoryUsage, preferredEncoderPath: supportedSourceDimensions.encoderPath, preferredRenderBackend: smokeExportConfig.renderBackend, experimentalNativeExport: useExperimentalNativeExport, @@ -4861,6 +4869,7 @@ export default function VideoEditor() { const settings: ExportSettings = { format: exportFormat, encodingMode: exportFormat === "mp4" ? exportEncodingMode : undefined, + memoryUsage: exportFormat === "mp4" ? exportMemoryUsage : undefined, mp4FrameRate: exportFormat === "mp4" ? mp4FrameRate : undefined, backendPreference: exportFormat === "mp4" ? exportBackendPreference : undefined, pipelineModel: exportFormat === "mp4" ? exportPipelineModel : undefined, @@ -4898,6 +4907,7 @@ export default function VideoEditor() { const handleCancelExport = useCallback(() => { if (exporterRef.current) { exporterRef.current.cancel(); + exporterRef.current = null; toast.info("Export canceled"); clearPendingExportSave(); setShowExportDropdown(false); @@ -5628,6 +5638,8 @@ export default function VideoEditor() { onExportFormatChange={setExportFormat} exportEncodingMode={exportEncodingMode} onExportEncodingModeChange={setExportEncodingMode} + exportMemoryUsage={exportMemoryUsage} + onExportMemoryUsageChange={setExportMemoryUsage} mp4FrameRate={mp4FrameRate} onMp4FrameRateChange={setMp4FrameRate} exportPipelineModel={exportPipelineModel} diff --git a/src/components/video-editor/editorPreferences.ts b/src/components/video-editor/editorPreferences.ts index 751e9f78..0e3b134a 100644 --- a/src/components/video-editor/editorPreferences.ts +++ b/src/components/video-editor/editorPreferences.ts @@ -46,6 +46,7 @@ type PersistedEditorControls = Pick< | "webcam" | "aspectRatio" | "exportEncodingMode" + | "exportMemoryUsage" | "exportBackendPreference" | "exportPipelineModel" | "exportQuality" @@ -127,6 +128,7 @@ export const DEFAULT_EDITOR_PREFERENCES: EditorPreferences = { webcam: DEFAULT_EDITOR_CONTROLS.webcam, aspectRatio: DEFAULT_EDITOR_CONTROLS.aspectRatio, exportEncodingMode: DEFAULT_EDITOR_CONTROLS.exportEncodingMode, + exportMemoryUsage: DEFAULT_EDITOR_CONTROLS.exportMemoryUsage, exportBackendPreference: DEFAULT_EDITOR_CONTROLS.exportBackendPreference, exportPipelineModel: DEFAULT_EDITOR_CONTROLS.exportPipelineModel, exportQuality: DEFAULT_EDITOR_CONTROLS.exportQuality, @@ -315,6 +317,7 @@ function normalizeEditorControls( webcam: raw.webcam ?? fallback.webcam, aspectRatio: raw.aspectRatio ?? fallback.aspectRatio, exportEncodingMode: raw.exportEncodingMode ?? fallback.exportEncodingMode, + exportMemoryUsage: raw.exportMemoryUsage ?? fallback.exportMemoryUsage, exportBackendPreference: raw.exportBackendPreference === undefined ? fallback.exportBackendPreference @@ -375,6 +378,7 @@ function normalizeEditorControls( webcam: normalized.webcam, aspectRatio: normalized.aspectRatio, exportEncodingMode: normalized.exportEncodingMode, + exportMemoryUsage: normalized.exportMemoryUsage, exportBackendPreference: normalized.exportBackendPreference, exportPipelineModel: normalized.exportPipelineModel, exportQuality: normalized.exportQuality, diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index 9adb3608..ea319695 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -2,6 +2,7 @@ import type { ExportBackendPreference, ExportEncodingMode, ExportFormat, + ExportMemoryUsage, ExportMp4FrameRate, ExportPipelineModel, ExportQuality, @@ -131,6 +132,7 @@ export interface ProjectEditorState { sourceAudioTrackSettingsByClip?: Record; defaultSourceAudioTrackSettings?: SourceAudioTrackSettings; exportEncodingMode: ExportEncodingMode; + exportMemoryUsage: ExportMemoryUsage; exportBackendPreference: ExportBackendPreference; exportPipelineModel: ExportPipelineModel; exportQuality: ExportQuality; @@ -164,6 +166,14 @@ export function normalizeExportEncodingMode(value: unknown): ExportEncodingMode return "balanced"; } +export function normalizeExportMemoryUsage(value: unknown): ExportMemoryUsage { + if (value === "low" || value === "balanced" || value === "high") { + return value; + } + + return "low"; +} + export function normalizeExportBackendPreference(value: unknown): ExportBackendPreference { if (value === "auto" || value === "webcodecs" || value === "breeze") { return value; @@ -1008,6 +1018,7 @@ export function normalizeProjectEditor(editor: Partial): Pro ? (editor.aspectRatio as AspectRatio) : "16:9", exportEncodingMode: normalizeExportEncodingMode(editor.exportEncodingMode), + exportMemoryUsage: normalizeExportMemoryUsage(editor.exportMemoryUsage), exportBackendPreference: normalizeExportBackendPreference(editor.exportBackendPreference), exportPipelineModel: normalizeExportPipelineModel(editor.exportPipelineModel), exportQuality: diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index b9a2d00a..98c2f5ed 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -193,6 +193,12 @@ "high": "High", "original": "Original" }, + "memoryTitle": "Memory", + "memory": { + "low": "Low", + "balanced": "Balanced", + "high": "High" + }, "fpsTitle": "FPS", "loop": "Loop", "outputDimensions": "Output: {{dimensions}}px", diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 8e98091e..875aa493 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -1204,6 +1204,7 @@ export class AudioProcessor { channelData.set(chunk, writeOffset); writeOffset += chunk.length; } + channelChunks[ch].length = 0; } return audioBuffer; diff --git a/src/lib/exporter/exportTuning.test.ts b/src/lib/exporter/exportTuning.test.ts index e2733916..2ef734a8 100644 --- a/src/lib/exporter/exportTuning.test.ts +++ b/src/lib/exporter/exportTuning.test.ts @@ -47,12 +47,12 @@ describe("exportTuning", () => { }); expect(webCodecsProfile.name).toBe("webcodecs-balanced-plus"); - expect(webCodecsProfile.maxDecodeQueue).toBe(12); - expect(webCodecsProfile.maxPendingFrames).toBe(32); + expect(webCodecsProfile.maxDecodeQueue).toBe(8); + expect(webCodecsProfile.maxPendingFrames).toBe(18); expect(breezeProfile.name).toBe("breeze-balanced-plus"); - expect(breezeProfile.maxDecodeQueue).toBe(14); - expect(breezeProfile.maxPendingFrames).toBe(40); - expect(breezeProfile.maxInFlightNativeWrites).toBe(8); + expect(breezeProfile.maxDecodeQueue).toBe(8); + expect(breezeProfile.maxPendingFrames).toBe(20); + expect(breezeProfile.maxInFlightNativeWrites).toBe(4); }); it("falls back to conservative native settings on low-core or very heavy workloads", () => { @@ -72,12 +72,55 @@ describe("exportTuning", () => { }); expect(breezeLowCoreProfile.name).toBe("breeze-conservative"); - expect(breezeLowCoreProfile.maxDecodeQueue).toBe(8); - expect(breezeLowCoreProfile.maxPendingFrames).toBe(16); - expect(breezeLowCoreProfile.maxInFlightNativeWrites).toBe(2); + expect(breezeLowCoreProfile.maxDecodeQueue).toBe(4); + expect(breezeLowCoreProfile.maxPendingFrames).toBe(8); + expect(breezeLowCoreProfile.maxInFlightNativeWrites).toBe(1); expect(breezeHeavyProfile.name).toBe("breeze-conservative"); - expect(breezeHeavyProfile.maxDecodeQueue).toBe(8); - expect(breezeHeavyProfile.maxPendingFrames).toBe(16); + expect(breezeHeavyProfile.maxDecodeQueue).toBe(4); + expect(breezeHeavyProfile.maxPendingFrames).toBe(8); + }); + + it("scales conservative profile buffers by memory usage setting", () => { + const base = { encodeBackend: "ffmpeg" as const, width: 1280, height: 720, frameRate: 60, hardwareConcurrency: 4 }; + + const low = getExportBackpressureProfile({ ...base, memoryUsage: "low" }); + const balanced = getExportBackpressureProfile({ ...base, memoryUsage: "balanced" }); + const high = getExportBackpressureProfile({ ...base, memoryUsage: "high" }); + + expect(low.maxDecodeQueue).toBe(4); + expect(low.maxPendingFrames).toBe(8); + expect(low.maxInFlightNativeWrites).toBe(1); + + expect(balanced.maxDecodeQueue).toBe(6); + expect(balanced.maxPendingFrames).toBe(12); + expect(balanced.maxInFlightNativeWrites).toBe(1); + + expect(high.maxDecodeQueue).toBe(8); + expect(high.maxPendingFrames).toBe(16); + expect(high.maxInFlightNativeWrites).toBe(2); + }); + + it("defaults to low memory usage when not specified", () => { + const withoutMemory = getExportBackpressureProfile({ + encodeBackend: "ffmpeg", width: 1280, height: 720, frameRate: 60, hardwareConcurrency: 4, + }); + const withLow = getExportBackpressureProfile({ + encodeBackend: "ffmpeg", width: 1280, height: 720, frameRate: 60, hardwareConcurrency: 4, memoryUsage: "low", + }); + + expect(withoutMemory.maxDecodeQueue).toBe(withLow.maxDecodeQueue); + expect(withoutMemory.maxPendingFrames).toBe(withLow.maxPendingFrames); + }); + + it("applies memory usage to balanced-plus profiles on high-core systems", () => { + const base = { encodeBackend: "ffmpeg" as const, width: 1280, height: 720, frameRate: 60, hardwareConcurrency: 8 }; + + const low = getExportBackpressureProfile({ ...base, memoryUsage: "low" }); + const high = getExportBackpressureProfile({ ...base, memoryUsage: "high" }); + + expect(low.name).toBe("breeze-balanced-plus"); + expect(high.name).toBe("breeze-balanced-plus"); + expect(high.maxPendingFrames).toBeGreaterThan(low.maxPendingFrames); }); }); diff --git a/src/lib/exporter/exportTuning.ts b/src/lib/exporter/exportTuning.ts index 8e54343b..3493ff07 100644 --- a/src/lib/exporter/exportTuning.ts +++ b/src/lib/exporter/exportTuning.ts @@ -1,4 +1,4 @@ -import type { ExportEncodeBackend, ExportEncodingMode } from "./types"; +import type { ExportEncodeBackend, ExportEncodingMode, ExportMemoryUsage } from "./types"; const DEFAULT_ENCODING_MODE: ExportEncodingMode = "balanced"; type WebCodecsLatencyMode = "quality" | "realtime"; @@ -79,6 +79,7 @@ interface ExportBackpressureProfileOptions { height: number; frameRate: number; encodingMode?: ExportEncodingMode; + memoryUsage?: ExportMemoryUsage; hardwareConcurrency?: number; } @@ -110,6 +111,52 @@ export function getWebCodecsKeyFrameInterval( return Math.max(1, Math.round(frameRate * KEYFRAME_INTERVAL_SECONDS[resolvedEncodingMode])); } +interface MemoryTieredProfile { + low: Omit; + balanced: Omit; + high: Omit; +} + +const BREEZE_CONSERVATIVE: MemoryTieredProfile = { + low: { maxDecodeQueue: 4, maxPendingFrames: 8, maxInFlightNativeWrites: 1 }, + balanced: { maxDecodeQueue: 6, maxPendingFrames: 12, maxInFlightNativeWrites: 1 }, + high: { maxDecodeQueue: 8, maxPendingFrames: 16, maxInFlightNativeWrites: 2 }, +}; + +const BREEZE_BALANCED: MemoryTieredProfile = { + low: { maxDecodeQueue: 8, maxPendingFrames: 16, maxInFlightNativeWrites: 2 }, + balanced: { maxDecodeQueue: 12, maxPendingFrames: 28, maxInFlightNativeWrites: 4 }, + high: { maxDecodeQueue: 12, maxPendingFrames: 28, maxInFlightNativeWrites: 4 }, +}; + +const BREEZE_BALANCED_PLUS: MemoryTieredProfile = { + low: { maxDecodeQueue: 8, maxPendingFrames: 20, maxInFlightNativeWrites: 4 }, + balanced: { maxDecodeQueue: 14, maxPendingFrames: 40, maxInFlightNativeWrites: 8 }, + high: { maxDecodeQueue: 14, maxPendingFrames: 40, maxInFlightNativeWrites: 8 }, +}; + +const WEBCODECS_CONSERVATIVE: MemoryTieredProfile = { + low: { maxDecodeQueue: 6, maxPendingFrames: 12, maxInFlightNativeWrites: 1 }, + balanced: { maxDecodeQueue: 7, maxPendingFrames: 16, maxInFlightNativeWrites: 1 }, + high: { maxDecodeQueue: 8, maxPendingFrames: 20, maxInFlightNativeWrites: 1 }, +}; + +const WEBCODECS_BALANCED: MemoryTieredProfile = { + low: { maxDecodeQueue: 6, maxPendingFrames: 14, maxInFlightNativeWrites: 1 }, + balanced: { maxDecodeQueue: 10, maxPendingFrames: 24, maxInFlightNativeWrites: 1 }, + high: { maxDecodeQueue: 10, maxPendingFrames: 24, maxInFlightNativeWrites: 1 }, +}; + +const WEBCODECS_BALANCED_PLUS: MemoryTieredProfile = { + low: { maxDecodeQueue: 8, maxPendingFrames: 18, maxInFlightNativeWrites: 1 }, + balanced: { maxDecodeQueue: 12, maxPendingFrames: 32, maxInFlightNativeWrites: 1 }, + high: { maxDecodeQueue: 12, maxPendingFrames: 32, maxInFlightNativeWrites: 1 }, +}; + +function resolveMemoryUsage(memoryUsage?: ExportMemoryUsage): ExportMemoryUsage { + return memoryUsage ?? "low"; +} + export function getExportBackpressureProfile( options: ExportBackpressureProfileOptions, ): ExportBackpressureProfile { @@ -124,15 +171,14 @@ export function getExportBackpressureProfile( const isHeavyWorkload = relativePixelRate >= 1.5; const isExtremeWorkload = relativePixelRate >= 3; const maxEncodeQueue = getWebCodecsEncodeQueueLimit(options.frameRate, options.encodingMode); + const mem = resolveMemoryUsage(options.memoryUsage); if (options.encodeBackend === "ffmpeg") { if (isLowCoreSystem || isExtremeWorkload) { return { name: "breeze-conservative", maxEncodeQueue, - maxDecodeQueue: 8, - maxPendingFrames: 16, - maxInFlightNativeWrites: 2, + ...BREEZE_CONSERVATIVE[mem], }; } @@ -140,18 +186,14 @@ export function getExportBackpressureProfile( return { name: "breeze-balanced-plus", maxEncodeQueue, - maxDecodeQueue: 14, - maxPendingFrames: 40, - maxInFlightNativeWrites: 8, + ...BREEZE_BALANCED_PLUS[mem], }; } return { name: "breeze-balanced", maxEncodeQueue, - maxDecodeQueue: 12, - maxPendingFrames: 28, - maxInFlightNativeWrites: 4, + ...BREEZE_BALANCED[mem], }; } @@ -159,9 +201,7 @@ export function getExportBackpressureProfile( return { name: "webcodecs-conservative", maxEncodeQueue, - maxDecodeQueue: 8, - maxPendingFrames: 20, - maxInFlightNativeWrites: 1, + ...WEBCODECS_CONSERVATIVE[mem], }; } @@ -169,17 +209,13 @@ export function getExportBackpressureProfile( return { name: "webcodecs-balanced-plus", maxEncodeQueue, - maxDecodeQueue: 12, - maxPendingFrames: 32, - maxInFlightNativeWrites: 1, + ...WEBCODECS_BALANCED_PLUS[mem], }; } return { name: "webcodecs-balanced", maxEncodeQueue, - maxDecodeQueue: 10, - maxPendingFrames: 24, - maxInFlightNativeWrites: 1, + ...WEBCODECS_BALANCED[mem], }; } diff --git a/src/lib/exporter/index.ts b/src/lib/exporter/index.ts index 3561cb3a..cf2b1b97 100644 --- a/src/lib/exporter/index.ts +++ b/src/lib/exporter/index.ts @@ -19,6 +19,7 @@ export type { ExportEncodeBackend, ExportEncodingMode, ExportFormat, + ExportMemoryUsage, ExportMetrics, ExportMp4FrameRate, ExportPipelineModel, diff --git a/src/lib/exporter/modernFrameRenderer.ts b/src/lib/exporter/modernFrameRenderer.ts index cf52faea..bde41c41 100644 --- a/src/lib/exporter/modernFrameRenderer.ts +++ b/src/lib/exporter/modernFrameRenderer.ts @@ -3826,6 +3826,10 @@ export class FrameRenderer { this.sceneVideoFrameStagingCtx = null; this.webcamVideoFrameStagingCanvas = null; this.webcamVideoFrameStagingCtx = null; + this.backgroundVideoFrameStagingCanvas = null; + this.backgroundVideoFrameStagingCtx = null; + this.compositeCanvas = null; + this.compositeCtx = null; this.videoTextureUsesStartupStaging = false; this.webcamTextureUsesStartupStaging = false; this.closeRetainedVideoFrame("scene"); diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index 55f47923..bff7092a 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -67,6 +67,7 @@ import { } from "./finalizationTimeout"; import { getLocalFilePath } from "./localMediaSource"; import { FrameRenderer as ModernFrameRenderer } from "./modernFrameRenderer"; +import { releaseNativeFrameCaptureResources } from "./nativeFrameCapture"; import { getOrderedSupportedMp4EncoderCandidates, type SupportedMp4EncoderPath, @@ -431,6 +432,7 @@ export class ModernVideoExporter { height: this.config.height, frameRate: this.config.frameRate, encodingMode: this.config.encodingMode, + memoryUsage: this.config.memoryUsage, }); this.maxNativeWriteInFlight = useNativeEncoder ? Math.max( @@ -540,6 +542,7 @@ export class ModernVideoExporter { height: this.config.height, frameRate: this.config.frameRate, encodingMode: this.config.encodingMode, + memoryUsage: this.config.memoryUsage, }); this.maxNativeWriteInFlight = 1; await this.initializeEncoder(); @@ -3426,6 +3429,8 @@ export class ModernVideoExporter { if (nativeStaticLayoutSessionId && typeof window !== "undefined") { void window.electronAPI?.nativeStaticLayoutExportCancel?.(nativeStaticLayoutSessionId); } + + this.cleanup(); } private cleanup(): void { @@ -3460,6 +3465,7 @@ export class ModernVideoExporter { this.muxer = null; this.audioProcessor?.cancel(); this.audioProcessor = null; + releaseNativeFrameCaptureResources(); this.disposeNativeH264Encoder(); const nativeExportSessionId = this.nativeExportSessionId; this.nativeExportSessionId = null; diff --git a/src/lib/exporter/nativeFrameCapture.ts b/src/lib/exporter/nativeFrameCapture.ts index 5d69f697..509db38c 100644 --- a/src/lib/exporter/nativeFrameCapture.ts +++ b/src/lib/exporter/nativeFrameCapture.ts @@ -72,6 +72,11 @@ function flipRgbaRowsInPlace(buffer: Uint8Array, width: number, height: number): } } +export function releaseNativeFrameCaptureResources(): void { + fallbackCanvas = null; + fallbackContext = null; +} + export async function captureCanvasFrameForNativeExport( canvas: HTMLCanvasElement, timestamp: number, diff --git a/src/lib/exporter/types.ts b/src/lib/exporter/types.ts index e0eb7934..c5517c7c 100644 --- a/src/lib/exporter/types.ts +++ b/src/lib/exporter/types.ts @@ -12,6 +12,7 @@ export interface ExportConfig { maxDecodeQueue?: number; maxPendingFrames?: number; maxInFlightNativeWrites?: number; + memoryUsage?: ExportMemoryUsage; sourceAudioFallbackStartDelayMsByPath?: Record; } @@ -171,6 +172,8 @@ export interface VideoFrameData { export type ExportEncodingMode = "fast" | "balanced" | "quality"; +export type ExportMemoryUsage = "low" | "balanced" | "high"; + export type ExportQuality = "medium" | "good" | "high" | "source"; export type ExportMp4FrameRate = 24 | 30 | 60; @@ -198,6 +201,7 @@ export interface ExportSettings { mp4FrameRate?: ExportMp4FrameRate; backendPreference?: ExportBackendPreference; pipelineModel?: ExportPipelineModel; + memoryUsage?: ExportMemoryUsage; // GIF settings gifConfig?: GifExportConfig; }