diff --git a/apps/common-app/src/examples/Worklets/Worklets.tsx b/apps/common-app/src/examples/Worklets/Worklets.tsx index 172f38487..fd8c50d26 100644 --- a/apps/common-app/src/examples/Worklets/Worklets.tsx +++ b/apps/common-app/src/examples/Worklets/Worklets.tsx @@ -1,10 +1,11 @@ import { useEffect, useRef, useState } from 'react'; import { StyleSheet, Text, View } from 'react-native'; +import { AudioContext } from 'react-native-audio-api'; import { - AudioContext, - OscillatorNode, -} from 'react-native-audio-api'; -import { WorkletNode } from 'react-native-audio-worklets'; + WorkletNode, + WorkletProcessingNode, + WorkletSourceNode, +} from 'react-native-audio-worklets'; import Animated, { Extrapolation, interpolate, @@ -68,9 +69,9 @@ function VisualizerBar({ function Worklets() { const audioContextRef = useRef(null); + const workletSourceRef = useRef(null); + const workletProcessingRef = useRef(null); const workletNodeRef = useRef(null); - const oscillatorRef = useRef(null); - const lfoRef = useRef(null); const heavyWorkAccRef = useRef(0); const jsWorkloadTimerRef = useRef | null>(null); @@ -120,21 +121,64 @@ function Worklets() { return; } + const sampleRate = ctx.sampleRate; + + workletSourceRef.current = new WorkletSourceNode( + ctx, + (audioData, outputChannelCount, framesToProcess, currentTime, startOffset) => { + 'worklet'; + + const frequency = 440; + + for (let channel = 0; channel < outputChannelCount; channel++) { + for (let i = 0; i < framesToProcess; i++) { + const sampleTime = currentTime + (startOffset + i) / sampleRate; + const phase = 2 * Math.PI * frequency * sampleTime; + audioData[channel]![i] = Math.sin(phase) * 0.25; + } + } + } + ); + + workletProcessingRef.current = new WorkletProcessingNode( + ctx, + ( + inputData, + outputData, + inputChannelCount, + outputChannelCount, + framesToProcess, + currentTime + ) => { + 'worklet'; + + const tremolo = + 0.75 + 0.25 * Math.sin(2 * Math.PI * 2 * currentTime); + + for (let ch = 0; ch < outputChannelCount; ch++) { + const input = inputData[Math.min(ch, inputChannelCount - 1)]!; + const output = outputData[ch]!; + + for (let i = 0; i < framesToProcess; i++) { + output[i] = Math.tanh(input[i]! * 1.2) * tremolo; + } + } + } + ); + workletNodeRef.current = new WorkletNode( ctx, - (audioBuffers, numberOfChannels) => { + (audioData, numberOfChannels) => { 'worklet'; if (numberOfChannels < 1) { return; } - const buffer = audioBuffers[0]; - if (buffer == null) { - return; - } - const channel = new Float32Array(buffer); + + const channel = audioData[0]!; let sum = 0; + for (let i = 0; i < channel.length; i++) { - sum += channel[i] * channel[i]; + sum += channel[i]! * channel[i]!; } const rms = Math.sqrt(sum / channel.length); const scaledAmplitude = Math.min(rms * 4, 1); @@ -147,30 +191,15 @@ function Worklets() { damping: 18, stiffness: 120, }); - } + }, + 1024 ); - const oscillator = ctx.createOscillator(); - oscillator.frequency.value = 440; - - const lfo = ctx.createOscillator(); - lfo.frequency.value = 2; - const lfoGain = ctx.createGain(); - lfoGain.gain.value = 0.18; - const amp = ctx.createGain(); - amp.gain.value = 0.25; - - lfo.connect(lfoGain); - lfoGain.connect(amp.gain); - oscillator.connect(amp); - amp.connect(workletNodeRef.current); + workletSourceRef.current.connect(workletProcessingRef.current); + workletProcessingRef.current.connect(workletNodeRef.current); workletNodeRef.current.connect(ctx.destination); - oscillator.start(); - lfo.start(); - - oscillatorRef.current = oscillator; - lfoRef.current = lfo; + workletSourceRef.current.start(); if (ctx.state === 'suspended') { ctx.resume(); @@ -180,12 +209,13 @@ function Worklets() { }; const stop = () => { - oscillatorRef.current?.stop(); - lfoRef.current?.stop(); + workletSourceRef.current?.stop(); + workletSourceRef.current?.disconnect(); + workletProcessingRef.current?.disconnect(); workletNodeRef.current?.disconnect(); - oscillatorRef.current = null; - lfoRef.current = null; + workletSourceRef.current = null; + workletProcessingRef.current = null; workletNodeRef.current = null; bar0.value = withSpring(0, { damping: 20, stiffness: 100 }); @@ -205,7 +235,7 @@ function Worklets() { Audio Worklets Visualizer - Oscillator → WorkletNode (RMS on UI runtime) → destination + WorkletSource → WorkletProcessing → WorkletNode → destination diff --git a/packages/audiodocs/docs/core/base-audio-context.mdx b/packages/audiodocs/docs/core/base-audio-context.mdx index 65c1277bc..05c88554a 100644 --- a/packages/audiodocs/docs/core/base-audio-context.mdx +++ b/packages/audiodocs/docs/core/base-audio-context.mdx @@ -180,61 +180,6 @@ Creates [`WaveShaperNode`](/docs/effects/wave-shaper-node). #### Returns `WaveShaperNode`. -### `createWorkletNode` - -Creates [`WorkletNode`](/docs/worklets/worklet-node). - -| Parameter | Type | Description | -| :---: | :---: | :---- | -| `worklet` | `(Array, number) => void` | The worklet to be executed. | -| `bufferLength` | `number` | The size of the buffer that will be passed to the worklet on each call. | -| `inputChannelCount` | `number` | The number of channels that the node expects as input (it will get min(expected, provided)). | -| `workletRuntime` | `AudioWorkletRuntime` | The kind of runtime to use for the worklet. See [worklet runtimes](/docs/worklets/worklets-introduction#what-kind-of-worklets-are-used-in-react-native-audio-api) for details. | - -#### Errors - -| Error type | Description | -| :---: | :---- | -| `Error` | `react-native-worklet` is not found as dependency. | -| `NotSupportedError` | `bufferLength` < `1`. | -| `NotSupportedError` | `inputChannelCount` is not in range [`1`, `32`]. | - -#### Returns `WorkletNode`. - -### `createWorkletSourceNode` - -Creates [`WorkletSourceNode`](/docs/worklets/worklet-source-node). - -| Parameter | Type | Description | -| :---: | :---: | :---- | -| `worklet` | `(Array, number, number, number) => void` | The worklet to be executed. | -| `workletRuntime` | `AudioWorkletRuntime` | The kind of runtime to use for the worklet. See [worklet runtimes](/docs/worklets/worklets-introduction#what-kind-of-worklets-are-used-in-react-native-audio-api) for details. | - -#### Errors - -| Error type | Description | -| :---: | :---- | -| `Error` | `react-native-worklet` is not found as dependency. | - -#### Returns `WorkletSourceNode`. - -### `createWorkletProcessingNode` - -Creates [`WorkletProcessingNode`](/docs/worklets/worklet-processing-node). - -| Parameter | Type | Description | -| :---: | :---: | :---- | -| `worklet` | `(Array, Array, number, number) => void` | The worklet to be executed. | -| `workletRuntime` | `AudioWorkletRuntime` | The kind of runtime to use for the worklet. See [worklet runtimes](/docs/worklets/worklets-introduction#what-kind-of-worklets-are-used-in-react-native-audio-api) for details. | - -#### Errors - -| Error type | Description | -| :---: | :---- | -| `Error` | `react-native-worklet` is not found as dependency. | - -#### Returns `WorkletProcessingNode`. - ### `decodeAudioData` diff --git a/packages/audiodocs/docs/fundamentals/getting-started.mdx b/packages/audiodocs/docs/fundamentals/getting-started.mdx index fbaa65dec..a26a740a7 100644 --- a/packages/audiodocs/docs/fundamentals/getting-started.mdx +++ b/packages/audiodocs/docs/fundamentals/getting-started.mdx @@ -113,11 +113,10 @@ bash -c 'echo Hello World!' ### Possible additional dependencies -If you plan to use any of [`WorkletNode`](/docs/worklets/worklet-node), [`WorkletSourceNode`](/docs/worklets/worklet-source-node), [`WorkletProcessingNode`](/docs/worklets/worklet-processing-node), it is required to have -`react-native-worklets` library set up with version 0.6.0 or higher. See [worklets getting-started page](https://docs.swmansion.com/react-native-worklets/docs/) for info how to do it. +To use [`WorkletNode`](/docs/worklets/worklet-node), install [`react-native-audio-worklets`](https://www.npmjs.com/package/react-native-audio-worklets) and [`react-native-worklets`](https://www.npmjs.com/package/react-native-worklets) **>= 0.10.0**. See the [Worklets introduction](/docs/worklets/introduction) for the full setup guide. :::info -If you are not planning to use any of mentioned nodes, `react-native-worklets` dependency is **OPTIONAL** and your app will build successfully without them. +If you are not planning to use worklet nodes, `react-native-worklets` and `react-native-audio-worklets` are **optional** — the main library builds successfully without them. ::: ### Usage with expo diff --git a/packages/audiodocs/docs/other/testing.mdx b/packages/audiodocs/docs/other/testing.mdx index b6010b176..bddac886f 100644 --- a/packages/audiodocs/docs/other/testing.mdx +++ b/packages/audiodocs/docs/other/testing.mdx @@ -142,27 +142,31 @@ describe('Offline Processing Tests', () => { ### Custom worklet testing ```typescript -import { AudioContext, WorkletProcessingNode } from 'react-native-audio-api/mock'; +import { AudioContext } from 'react-native-audio-api'; +import { WorkletProcessingNode } from 'react-native-audio-worklets'; describe('Worklet Tests', () => { it('should create custom audio processing', () => { const context = new AudioContext(); - const processingCallback = jest.fn((inputData, outputData, framesToProcess) => { - // Mock audio processing logic - for (let channel = 0; channel < outputData.length; channel++) { - for (let i = 0; i < framesToProcess; i++) { - outputData[channel][i] = inputData[channel][i] * 0.5; // Simple gain + const processingCallback = jest.fn( + ( + inputData, + outputData, + inputChannelCount, + outputChannelCount, + framesToProcess + ) => { + for (let channel = 0; channel < outputChannelCount; channel++) { + for (let i = 0; i < framesToProcess; i++) { + outputData[channel][i] = inputData[channel][i] * 0.5; + } } } - }); - - const workletNode = new WorkletProcessingNode( - context, - 'AudioRuntime', - processingCallback ); + const workletNode = new WorkletProcessingNode(context, processingCallback); + expect(workletNode.context).toBe(context); }); }); diff --git a/packages/audiodocs/docs/worklets/introduction.mdx b/packages/audiodocs/docs/worklets/introduction.mdx index 4987c100a..18c667bc3 100644 --- a/packages/audiodocs/docs/worklets/introduction.mdx +++ b/packages/audiodocs/docs/worklets/introduction.mdx @@ -1,76 +1,134 @@ --- -id: worklets-introduction sidebar_label: Introduction sidebar_position: 1 --- import { MobileOnly } from '@site/src/components/Badges'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; -# RNWorklets Support +# Worklets -The `RNWorklets` library was originally part of Reanimated until version 4.0.0; since then, it has become a separate library. +Worklet support in React Native Audio API is provided by a separate package — [`react-native-audio-worklets`](https://www.npmjs.com/package/react-native-audio-worklets). It bridges the audio engine with [`react-native-worklets`](https://docs.swmansion.com/react-native-worklets/) so you can react to live audio on the UI thread — for example, to drive Reanimated visualizers from an RMS meter. -To use the worklet features provided by `react-native-audio-api`, you need to install this library: +:::info Requirements +- [`react-native-audio-api`](https://www.npmjs.com/package/react-native-audio-api) **>= 1.0.0** +- [`react-native-worklets`](https://www.npmjs.com/package/react-native-worklets) **>= 0.10.0** +- React Native **New Architecture** (TurboModules / Fabric) +::: + +## Installation + +Install both peer dependencies and the worklets integration package: + + + + +```bash +npm install react-native-audio-api react-native-worklets react-native-audio-worklets +``` + + + + +```bash +yarn add react-native-audio-api react-native-worklets react-native-audio-worklets +``` + + + + +Follow the [`react-native-worklets` setup guide](https://docs.swmansion.com/react-native-worklets/docs/) (Babel plugin, native rebuild). Then rebuild your app so the native modules are linked: + + + ```bash -npm install react-native-worklets +cd ios && pod install && cd .. +npx react-native run-ios ``` -> **Note**: Supported versions of `react-native-worklets` are [0.6.x, 0.7.x]. They are checked and updated manually with each release. Nightly versions are always supported but your build may fail. -If the library is not installed, you will encounter runtime errors when trying to use features that depend on worklets and do not have documented fallback implementations. + + + +```bash +npx react-native run-android +``` + + + + +Import `react-native-audio-worklets` before creating any worklet node. The package installs its native extensions on top of `react-native-audio-api`: + +```tsx +import 'react-native-audio-api'; +import { + WorkletNode, + WorkletProcessingNode, + WorkletSourceNode, +} from 'react-native-audio-worklets'; +``` + +:::tip +If you only use core audio nodes and never import `react-native-audio-worklets`, you do **not** need `react-native-worklets` installed — the main library builds without it. +::: ## What is a worklet? You can read more about worklets in the [RNWorklets documentation](https://docs.swmansion.com/react-native-worklets/). -Simply put, a worklet is a piece of code that can be executed on a runtime different from the main JavaScript runtime (or more formally, the runtime on which the code was created). +Simply put, a worklet is a piece of JavaScript that can run on a runtime other than the one it was created on — for example, the shared UI worklet runtime used by Reanimated. + +## UI runtime + +`WorkletNode` dispatches your callback on the **UI worklet runtime** provided by `react-native-worklets`. That lets you update Reanimated shared values and animated styles directly from audio-driven logic. + +**Use it when you want to:** + +- Build visualizers or meters from live audio +- Drive UI animations from audio analysis (RMS, peak level, etc.) +- Integrate with Reanimated shared values and `withSpring` / `withTiming` -## What kind of worklets are used in react-native-audio-api? +Audio still flows through the node unchanged — the worklet receives a **read-only snapshot** for analysis. -We support two types of worklet runtimes, each optimized for different use cases: +## Audio runtime -### UIRuntime -Worklets executed on the UI runtime provided by the `RNWorklets` library. This allows the use of Reanimated utilities and features inside the worklets. The main goal is to enable seamless integration with the UI - for example, creating animations from audio data. +`WorkletSourceNode` and `WorkletProcessingNode` invoke your callback synchronously on a **dedicated audio worklet runtime** each render quantum (128 frames). -**Use UIRuntime when:** -- You need to update UI elements from audio data -- Creating visualizations or animations based on audio -- Integrating with Reanimated shared values -- Performance is less critical than UI responsiveness +**Use them when you want to:** -### AudioRuntime -Worklets executed on the audio rendering thread for maximum performance and minimal latency. This runtime is optimized for real-time audio processing where timing is critical. +- Build custom synthesizers or procedural generators (`WorkletSourceNode`) +- Implement in-graph JavaScript effects, filters, or dynamics processors (`WorkletProcessingNode`) +- Process audio sample-by-sample with synchronous control over the output buffer -**Use AudioRuntime when:** -- Performance and low latency are crucial -- Processing audio in real-time without dropouts -- Generating audio with precise timing -- Audio processing doesn't need to interact with UI +These callbacks run on the audio path. Keep them short and allocation-free — a slow worklet can cause dropouts. -You can specify the runtime type when creating worklet nodes using the `workletRuntime` parameter. +## How to use audio worklets mindfully -## How to use worklets in react-native-audio-api mindfully? +### `WorkletNode` (UI visualization) -Our API is specifically designed to support high throughput to enable audio playback at `44.1` kHz, which is the default frequency for most modern devices. +`WorkletNode` is designed for **UI visualization**, not sample-accurate audio processing. The audio thread accumulates frames into a snapshot buffer and schedules your callback on the UI thread — it never blocks playback. -However, this introduces several limitations on what can be done inside a worklet. Since a worklet must be executed on the JavaScript runtime, each execution introduces latency. +`bufferLength` is the number of **frames per channel** in each snapshot. It controls how often the callback runs when the UI keeps up: -$$ 44.1\text{Hz} \equiv 44100\text{ samples} \equiv 1\text{ s} $$ +$$ +\text{callback rate (Hz)} \approx \frac{\text{sampleRate}}{\text{bufferLength}} +$$ -This means the sample rate indicates how many frames are processed in one second. Most features that allow using worklets as callbacks should also allow setting `bufferLength` for worklet input. +At 44.1 kHz with `bufferLength = 128`, that is roughly **344 callbacks per second** (~2.9 ms between dispatches). A slow UI callback does **not** cause audio dropouts — while your worklet is still running, new snapshots are skipped and audio continues to pass through unchanged. You may simply miss visualization updates. -If you set `bufferLength` to `128` (which is the default internal buffer size of our API used to process the graph), you must remember that your worklet should not take more than: +- **Smaller `bufferLength`** — lower analysis latency, more frequent UI callbacks, more JS work +- **Larger `bufferLength`** — smoother analysis windows (e.g. RMS over more samples), fewer callbacks -$$ 1\text{ s} = 1000\text{ ms} $$ +Use a larger `bufferLength` such as `256`, `512`, or `1024` if you do not need more than ~40 fps of meter updates. -$$ \frac{44100}{128} \approx 344 $$ +### `WorkletSourceNode` and `WorkletProcessingNode` (audio path) -$$ \frac{1000\text{ ms}}{344} \approx 2.9\text{ ms} $$ +These nodes invoke your worklet **synchronously** on the audio thread each render quantum (128 frames). The entire callback — including your DSP and any other graph processing in that quantum — must finish within one quantum's time budget. -This means that if your worklet, plus the rest of the processing, takes more than `2.9` ms, you may start to experience audio dropouts or other playback issues. +At 44.1 kHz, 128 frames is roughly **2.9 ms**. If your worklet takes longer, you may experience audio dropouts or glitches. -### Recommendations +### General tips -- Use a larger `bufferLength`, like `256`, `512` or even `1024` if you don't need more than `40` fps. -- Avoid blocking operations in the worklet (e.g., calling APIs - use JS callbacks for these instead). -- Do not overuse worklets. Before creating 5 or 6, consider if it can be done with a single one. Creating chained nodes that invoke worklets increases latency linearly. +- Avoid blocking operations inside any worklet (e.g. network calls or async APIs). Offload that work to the JS thread via callbacks. +- Do not overuse worklets. Before creating several chained worklet nodes, consider whether a single node can do the job — each one adds latency on the audio path. - Measure performance and memory usage, and check logs to ensure you are not dropping frames. diff --git a/packages/audiodocs/docs/worklets/worklet-node.mdx b/packages/audiodocs/docs/worklets/worklet-node.mdx index fe143150d..ec5559bc7 100644 --- a/packages/audiodocs/docs/worklets/worklet-node.mdx +++ b/packages/audiodocs/docs/worklets/worklet-node.mdx @@ -9,57 +9,110 @@ import { ReadOnly } from '@site/src/components/Badges'; # WorkletNode :::warning -This node is dependent on `react-native-worklets` and you need to install them in order to use this node. Refer to [getting-started page](/docs/fundamentals/getting-started#possible-additional-dependencies) for more info. +Requires [`react-native-audio-worklets`](https://www.npmjs.com/package/react-native-audio-worklets), [`react-native-worklets`](https://www.npmjs.com/package/react-native-worklets) **>= 0.10.0**, and [`react-native-audio-api`](https://www.npmjs.com/package/react-native-audio-api) **>= 1.0.0**. See the [Worklets introduction](/docs/worklets/introduction) for installation. ::: -The `WorkletNode` interface represents a node in the audio processing graph that can execute a worklet. +`WorkletNode` is a pass-through analysis node. It accumulates incoming audio frames until `bufferLength` is reached, then invokes your worklet on the **UI runtime** with a read-only snapshot. Audio output is unchanged — modifications inside the callback do not affect the signal. -Worklets are a way to run JavaScript code in the audio rendering thread, allowing for low-latency audio processing. For more information, see our introduction [Introduction to worklets](/docs/worklets/worklets-introduction). -This node lets you execute a worklet on the UI thread. bufferLength specifies the size of the buffer that will be passed to the worklet on each call. The inputChannelCount specifies the number of channels that will be passed to the worklet. +For background on runtimes and performance tips, see [How to use audio worklets mindfully](/docs/worklets/introduction#how-to-use-audio-worklets-mindfully). ## Constructor ```tsx -constructor( - context: BaseAudioContext, - runtime: AudioWorkletRuntime, - callback: (audioData: Array, channelCount: number) => void, - bufferLength: number, - inputChannelCount: number) +import { AudioContext } from 'react-native-audio-api'; +import { WorkletNode } from 'react-native-audio-worklets'; + +const node = new WorkletNode(context, callback, bufferLength); ``` -Or by using `BaseAudioContext` factory method: -[`BaseAudioContext.createWorkletNode(worklet, bufferLength, inputChannelCount, workletRuntime)`](/docs/core/base-audio-context#createworkletnode-) + +### Parameters + +| Parameter | Type | Description | +| :---: | :---: | :---- | +| `context` | `BaseAudioContext` | The audio context that owns this node. | +| `callback` | `WorkletNodeCallback` | Worklet invoked on the UI runtime when a snapshot is ready. Must include the `'worklet'` directive. | +| `bufferLength` | `number` | Number of frames per channel in each snapshot. Frames are accumulated across render quanta until this length is reached. | + +### `WorkletNodeCallback` + +```tsx +type WorkletNodeCallback = ( + audioData: Array, + numberOfChannels: number +) => void; +``` + +| Argument | Description | +| :--- | :---- | +| `audioData` | One stable `Float32Array` per channel of 32-bit float PCM (not interleaved), each of length `bufferLength`. The same view objects are reused every callback; only the underlying sample memory is refilled. | +| `numberOfChannels` | Active channel count for this callback (`audioData.length`). | + +:::caution +Do **not** assign `audioData` or a channel view to a Reanimated shared value (or any state read later on the UI thread). The native buffer pool is reused for the next snapshot and may be written from the audio thread after your callback returns. + +- **Safe:** compute a scalar inside the callback and assign that (e.g. `amplitude.value = rms`). +- **Safe:** copy samples if you need waveform data later — `Float32Array.from(audioData[0]!)`. +- **Unsafe:** `waveform.value = audioData[0]` — you keep a live alias, not a frozen snapshot. +::: + +### Errors + +| Error type | Condition | +| :---: | :---- | +| `Error` | `react-native-audio-worklets` native module not installed or New Architecture is disabled. | +| `Error` | `react-native-worklets` is missing or below the supported version. | +| `NotSupportedError` | `bufferLength` is not a finite positive integer. | + +## How buffering works + +1. Each render quantum appends frames into an internal snapshot buffer. +2. When `bufferLength` frames have been collected, the callback is scheduled on the UI thread. +3. While the callback is running, new frames are **skipped** (audio still passes through). +4. After the callback completes, accumulation starts again from the beginning. + +If a single quantum contains more frames than needed to fill the buffer, only the frames required to reach `bufferLength` are copied — the rest are discarded. ## Example + +Oscillator → `WorkletNode` (RMS on UI) → destination, driving Reanimated bars: + ```tsx -import { AudioContext, AudioRecorder, AudioManager } from 'react-native-audio-api'; +import { AudioContext } from 'react-native-audio-api'; +import { WorkletNode } from 'react-native-audio-worklets'; +import { useSharedValue, withSpring } from 'react-native-reanimated'; -AudioManager.setAudioSessionOptions({ - iosCategory: "playAndRecord", - iosMode: "measurement", - iosOptions: ["mixWithOthers"], -}) +function Visualizer() { + const amplitude = useSharedValue(0); -// This example shows how we can use a WorkletNode to process microphone audio data in real-time. -async function App() { - const recorder = new AudioRecorder(); + const start = () => { + const ctx = new AudioContext(); - const audioContext = new AudioContext({ sampleRate: 16000 }); - const worklet = (audioData: Array, inputChannelCount: number) => { + const workletNode = new WorkletNode( + ctx, + (audioData, numberOfChannels) => { 'worklet'; - // here you have access to the number of input channels and the audio data - // audio data is a two dimensional array where first index is the channel number and second is buffer of exactly bufferLength size - // !IMPORTANT: here you can only read audio data any modifications will not be reflected in the audio output of this node - // !VERY IMPORTANT: please read the Known Issue section below - }; - const workletNode = audioContext.createWorkletNode(worklet, 1024, 2, 'UIRuntime'); - - await AudioManager.setAudioSessionActivity(true); - adapterNode.connect(workletNode); - workletNode.connect(audioContext.destination); - recorder.connect(adapterNode); - recorder.start(); - audioContext.resume(); + if (numberOfChannels < 1) return; + + const channel = audioData[0]!; + let sum = 0; + for (let i = 0; i < channel.length; i++) { + sum += channel[i] * channel[i]; + } + const rms = Math.sqrt(sum / channel.length); + amplitude.value = withSpring(Math.min(rms * 4, 1)); + }, + 1024 + ); + + const oscillator = ctx.createOscillator(); + oscillator.frequency.value = 440; + oscillator.connect(workletNode); + workletNode.connect(ctx.destination); + oscillator.start(); + ctx.resume(); + }; + + // ... } ``` @@ -79,13 +132,14 @@ Inherits all methods from [`AudioNode`](/docs/core/audio-node#methods). `WorkletNode` does not define any additional methods. -## Known Issue -It might happen that the worklet side effect is not visible on the UI (when you are using UIRuntime kind). For example you have some animated style which depends on some shared value modified in the worklet. -This is happening because microtask queue is not always being flushed properly, bla bla bla... +## Known issue + +When using Reanimated shared values, UI updates from the worklet may not appear immediately because the microtask queue is not always flushed on the UI runtime. + +Add this at the end of your callback if you see missing updates: -To workaround this issue just add this line at the end of your worklet callback function: ```ts requestAnimationFrame(() => {}); ``` -This will ensure that microtask queue is flushed and your UI will be updated properly. But be aware that this might have some performance implications so it is not included by default. -So use this only after confirming that your worklet side effects are not visible on the UI. + +Use only after confirming the issue — it adds a small scheduling cost. diff --git a/packages/audiodocs/docs/worklets/worklet-processing-node.mdx b/packages/audiodocs/docs/worklets/worklet-processing-node.mdx index 0af22ad58..c10007590 100644 --- a/packages/audiodocs/docs/worklets/worklet-processing-node.mdx +++ b/packages/audiodocs/docs/worklets/worklet-processing-node.mdx @@ -9,125 +9,128 @@ import { ReadOnly } from '@site/src/components/Badges'; # WorkletProcessingNode :::warning -This node is dependent on `react-native-worklets` and you need to install them in order to use this node. Refer to [getting-started page](/docs/fundamentals/getting-started#possible-additional-dependencies) for more info. +Requires [`react-native-audio-worklets`](https://www.npmjs.com/package/react-native-audio-worklets), [`react-native-worklets`](https://www.npmjs.com/package/react-native-worklets) **>= 0.10.0**, and [`react-native-audio-api`](https://www.npmjs.com/package/react-native-audio-api) **>= 1.0.0**. See the [Worklets introduction](/docs/worklets/introduction) for installation. ::: -The `WorkletProcessingNode` interface represents a node in the audio processing graph that can process audio using a worklet function. Unlike [`WorkletNode`](/docs/worklets/worklet-node) which only provides read-only access to audio data, `WorkletProcessingNode` allows you to modify the audio signal by providing both input and output buffers. +`WorkletProcessingNode` is an effect node that processes audio synchronously on the **audio worklet runtime** each render quantum. Unlike [`WorkletNode`](/docs/worklets/worklet-node), which only provides read-only snapshots on the UI runtime, this node receives input audio and writes processed output back into the graph. -This node lets you execute a worklet that receives input audio data and produces output audio data, making it perfect for creating custom audio effects, filters, and processors. The worklet processes the exact number of frames provided by the audio system in each call. +Use it for custom effects, filters, dynamics processors, and other in-graph DSP written in JavaScript. -For more information about worklets, see our [Introduction to worklets](/docs/worklets/worklets-introduction). +For runtime differences and performance tips, see [How to use audio worklets mindfully](/docs/worklets/introduction#how-to-use-audio-worklets-mindfully). ## Constructor ```tsx -constructor( - context: BaseAudioContext, - runtime: AudioWorkletRuntime, - callback: ( - inputData: Array, - outputData: Array, - framesToProcess: number, - currentTime: number - ) => void) +import { AudioContext } from 'react-native-audio-api'; +import { WorkletProcessingNode } from 'react-native-audio-worklets'; + +const node = new WorkletProcessingNode(context, callback); ``` -Or by using `BaseAudioContext` factory method: -[`BaseAudioContext.createWorkletProcessingNode(worklet, workletRuntime)`](/docs/core/base-audio-context#createworkletprocessingnode-) -## Example +### Parameters + +| Parameter | Type | Description | +| :---: | :---: | :---- | +| `context` | `BaseAudioContext` | The audio context that owns this node. | +| `callback` | `WorkletProcessingNodeCallback` | Worklet invoked on the audio runtime each render quantum. Must include the `'worklet'` directive. | + +### `WorkletProcessingNodeCallback` ```tsx -import { AudioContext, AudioRecorder } from 'react-native-audio-api'; - -// This example shows how to create a simple gain effect using WorkletProcessingNode -function App() { - const recorder = new AudioRecorder({ - sampleRate: 16000, - bufferLengthInSamples: 16000, - }); - - const audioContext = new AudioContext({ sampleRate: 16000 }); - - // Create a simple gain worklet that multiplies the input by a gain value - const gainWorklet = ( - inputData: Array, - outputData: Array, - framesToProcess: number, - currentTime: number - ) => { - 'worklet'; - const gain = 0.5; // 50% volume +type WorkletProcessingNodeCallback = ( + inputData: Array, + outputData: Array, + inputChannelCount: number, + outputChannelCount: number, + framesToProcess: number, + currentTime: number +) => void; +``` - for (let ch = 0; ch < inputData.length; ch++) { - const input = inputData[ch]; - const output = outputData[ch]; +| Argument | Description | +| :--- | :---- | +| `inputData` | Stable per-channel `Float32Array` views over the input pool (read-only for your callback). Each view spans the full render quantum (128 frames). | +| `outputData` | Stable per-channel `Float32Array` views over the output pool. Write processed samples into indices `0 .. framesToProcess - 1`. | +| `inputChannelCount` | Active input channel count (`inputData.length`). | +| `outputChannelCount` | Active output channel count (`outputData.length`). | +| `framesToProcess` | Number of frames in this quantum (at most 128). | +| `currentTime` | Audio context time in seconds at the start of this quantum. | + +:::note +Input and output use **separate** buffer pools. You must write every output sample you want to hear — unwritten frames are not automatically copied from input. +::: - for (let i = 0; i < framesToProcess; i++) { - output[i] = input[i] * gain; - } - } - }; +### Errors - const workletProcessingNode = audioContext.createWorkletProcessingNode( - gainWorklet, - 'AudioRuntime' - ); +| Error type | Condition | +| :---: | :---- | +| `Error` | `react-native-audio-worklets` native module not installed or New Architecture is disabled. | +| `Error` | `react-native-worklets` is missing or below the supported version. | - recorder - .connect(audioContext, workletProcessingNode) - .connect(audioContext.destination); - recorder.start(); -} -``` +## Example -## Worklet Parameters Explanation +Simple gain effect: -The worklet function receives four parameters: +```tsx +import { AudioContext } from 'react-native-audio-api'; +import { WorkletProcessingNode } from 'react-native-audio-worklets'; + +function GainEffect() { + const start = () => { + const ctx = new AudioContext(); + + const gainNode = new WorkletProcessingNode( + ctx, + (inputData, outputData, inputChannelCount, outputChannelCount, framesToProcess) => { + 'worklet'; -### `inputData: Array` -A two-dimensional array where: -- First dimension represents the audio channel (`0` = left, `1` = right for stereo) -- Second dimension contains the input audio samples for that channel -- You should **read** from these buffers to get the input audio data -- The length of each `Float32Array` equals the `framesToProcess` parameter + const gain = 0.5; -### `outputData: Array` -A two-dimensional array where: -- First dimension represents the audio channel (`0` = left, `1` = right for stereo) -- Second dimension contains the output audio samples for that channel -- You must **write** to these buffers to produce the processed audio output -- The length of each `Float32Array` equals the `framesToProcess` parameter + for (let ch = 0; ch < Math.min(inputChannelCount, outputChannelCount); ch++) { + const input = inputData[ch]!; + const output = outputData[ch]!; -### `framesToProcess: number` -The number of audio samples to process in this call. This determines how many samples you need to process in each channel's buffer. This value will be at most `128`. + for (let i = 0; i < framesToProcess; i++) { + output[i] = input[i]! * gain; + } + } + } + ); -### `currentTime: number` -The current audio context time in seconds when this worklet call begins. This represents the absolute time since the audio context was created. + const oscillator = ctx.createOscillator(); + oscillator.connect(gainNode); + gainNode.connect(ctx.destination); + oscillator.start(); + ctx.resume(); + }; + + // ... +} +``` ## Audio Processing Pattern A typical WorkletProcessingNode worklet follows this pattern: ```tsx -const audioProcessor = ( - inputData: Array, - outputData: Array, - framesToProcess: number, - currentTime: number +const processor = ( + inputData: Array, + outputData: Array, + inputChannelCount: number, + outputChannelCount: number, + framesToProcess: number, + currentTime: number ) => { - 'worklet'; + 'worklet'; - for (let channel = 0; channel < inputData.length; channel++) { - const input = inputData[channel]; - const output = outputData[channel]; + for (let channel = 0; channel < outputChannelCount; channel++) { + const input = inputData[Math.min(channel, inputChannelCount - 1)]!; + const output = outputData[channel]!; - for (let sample = 0; sample < framesToProcess; sample++) { - // Process each sample - // Read from: input[sample] - // Write to: output[sample] - output[sample] = processAudioSample(input[sample]); - } + for (let sample = 0; sample < framesToProcess; sample++) { + output[sample] = processSample(input[sample]!, currentTime); } + } }; ``` @@ -147,21 +150,12 @@ Inherits all methods from [`AudioNode`](/docs/core/audio-node#methods). `WorkletProcessingNode` does not define any additional methods. -## Performance Considerations - -Since `WorkletProcessingNode` processes audio in real-time, performance is critical: +## Performance considerations -- Keep worklet functions lightweight and efficient -- Avoid complex calculations that could cause audio dropouts -- Process samples in-place when possible -- Consider using lookup tables for expensive operations -- Use `AudioRuntime` for better performance, `UIRuntime` for UI integration -- Test on target devices to ensure smooth audio processing +Callbacks run synchronously on the audio path each render quantum. Keep worklet functions lightweight — heavy math or allocations can cause dropouts. -## Use Cases +## See also -- **Audio Effects**: Reverb, delay, distortion, filters -- **Audio Processing**: Compression, limiting, normalization -- **Real-time Filters**: EQ, high-pass, low-pass, band-pass filters -- **Custom Algorithms**: Noise reduction, pitch shifting, spectral processing -- **Signal Analysis**: Feature extraction while passing audio through +- [`WorkletSourceNode`](/docs/worklets/worklet-source-node) — generate audio with a worklet +- [`WorkletNode`](/docs/worklets/worklet-node) — read-only snapshots on the UI runtime for visualizers +- [Worklets introduction](/docs/worklets/introduction) diff --git a/packages/audiodocs/docs/worklets/worklet-source-node.mdx b/packages/audiodocs/docs/worklets/worklet-source-node.mdx index 6af4b7991..942e9f518 100644 --- a/packages/audiodocs/docs/worklets/worklet-source-node.mdx +++ b/packages/audiodocs/docs/worklets/worklet-source-node.mdx @@ -2,160 +2,137 @@ sidebar_position: 3 --- -import { ReadOnly, MobileOnly } from '@site/src/components/Badges'; +import AudioNodePropsTable from '@site/src/components/AudioNodePropsTable'; +import { ReadOnly } from '@site/src/components/Badges'; # WorkletSourceNode :::warning -This node is dependent on `react-native-worklets` and you need to install them in order to use this node. Refer to [getting-started page](/docs/fundamentals/getting-started#possible-additional-dependencies) for more info. +Requires [`react-native-audio-worklets`](https://www.npmjs.com/package/react-native-audio-worklets), [`react-native-worklets`](https://www.npmjs.com/package/react-native-worklets) **>= 0.10.0**, and [`react-native-audio-api`](https://www.npmjs.com/package/react-native-audio-api) **>= 1.0.0**. See the [Worklets introduction](/docs/worklets/introduction) for installation. ::: -The `WorkletSourceNode` interface represents a scheduled source node in the audio processing graph that generates audio using a worklet function. It extends [`AudioScheduledSourceNode`](/docs/sources/audio-scheduled-source-node), providing the ability to start and stop audio generation at specific times. +`WorkletSourceNode` is a scheduled source that generates audio synchronously on the **audio worklet runtime** each render quantum. It extends [`AudioScheduledSourceNode`](/docs/sources/audio-scheduled-source-node), so you can `start()` and `stop()` it at specific times. -This node allows you to generate audio procedurally using JavaScript worklets, making it perfect for creating custom synthesizers, audio generators, or real-time audio effects that produce sound rather than just process it. +Use it for custom synthesizers, procedural generators, and other nodes that **produce** audio rather than process an existing input. -For more information about worklets, see our [Introduction to worklets](/docs/worklets/worklets-introduction). +For runtime differences and performance tips, see [How to use audio worklets mindfully](/docs/worklets/introduction#how-to-use-audio-worklets-mindfully). ## Constructor -```tsx -constructor( - context: BaseAudioContext, - runtime: AudioWorkletRuntime, - callback: ( - audioData: Array, - framesToProcess: number, - currentTime: number, - startOffset: number - ) => void) -``` -Or by using `BaseAudioContext` factory method: -[`BaseAudioContext.createWorkletSourceNode(worklet, workletRuntime)`](/docs/core/base-audio-context#createworkletsourcenode-) - -## Example - ```tsx import { AudioContext } from 'react-native-audio-api'; +import { WorkletSourceNode } from 'react-native-audio-worklets'; -function App() { - const audioContext = new AudioContext({ sampleRate: 44100 }); - - // Create a simple sine wave generator worklet - const sineWaveWorklet = ( - audioData: Array, - framesToProcess: number, - currentTime: number, - startOffset: number - ) => { - 'worklet'; - - const frequency = 440; // A4 note - const sampleRate = 44100; - - // Generate audio for each channel - for (let channel = 0; channel < audioData.length; channel++) { - for (let i = 0; i < framesToProcess; i++) { - // Calculate the absolute time for this sample - const sampleTime = currentTime + (startOffset + i) / sampleRate; - - // Generate sine wave - const phase = 2 * Math.PI * frequency * sampleTime; - audioData[channel][i] = Math.sin(phase) * 0.5; // 50% volume - } - } - }; +const node = new WorkletSourceNode(context, callback); +``` - const workletSourceNode = audioContext.createWorkletSourceNode( - sineWaveWorklet, - 'AudioRuntime' - ); +### Parameters - // Connect to output and start playback - workletSourceNode.connect(audioContext.destination); - workletSourceNode.start(); // Start immediately +| Parameter | Type | Description | +| :---: | :---: | :---- | +| `context` | `BaseAudioContext` | The audio context that owns this node. | +| `callback` | `WorkletSourceNodeCallback` | Worklet invoked on the audio runtime each render quantum. Must include the `'worklet'` directive. | - // Stop after 2 seconds - setTimeout(() => { - workletSourceNode.stop(); - }, 2000); -} +### `WorkletSourceNodeCallback` + +```tsx +type WorkletSourceNodeCallback = ( + audioData: Array, + outputChannelCount: number, + framesToProcess: number, + currentTime: number, + startOffset: number +) => void; ``` -## Worklet Parameters Explanation +| Argument | Description | +| :--- | :---- | +| `audioData` | Stable per-channel `Float32Array` views over a reused native buffer pool. Each view spans the full render quantum (128 frames); write samples into indices `0 .. framesToProcess - 1`. | +| `outputChannelCount` | Active output channel count (`audioData.length`). | +| `framesToProcess` | Number of frames to generate this quantum. May be less than 128 when playback starts or stops mid-buffer. | +| `currentTime` | Audio context time in seconds at the start of this quantum. | +| `startOffset` | Silent prefix length (in samples) already zeroed in the output buffer when playback starts mid-quantum. Use for time math, not as a write index into `audioData`. | -The worklet function receives four parameters: +### Errors -### `audioData: Array` -A two-dimensional array where: -- First dimension represents the audio channel (`0` = left, `1` = right for stereo) -- Second dimension contains the audio samples for that channel -- You must **write** audio data to these buffers to generate sound -- The length of each `Float32Array` equals `framesToProcess` +| Error type | Condition | +| :---: | :---- | +| `Error` | `react-native-audio-worklets` native module not installed or New Architecture is disabled. | +| `Error` | `react-native-worklets` is missing or below the supported version. | -### `framesToProcess: number` -The number of audio samples to generate in this call. This determines how many samples you need to fill in each channel's buffer. +## Example -### `currentTime: number` -The current audio context time in seconds when this worklet call begins. This represents the absolute time since the audio context was created. +Sine generator → destination: -### `startOffset: number` -The sample offset within the current processing block where your generated audio should begin. This is particularly important for precise timing when the node starts or stops mid-block. +```tsx +import { AudioContext } from 'react-native-audio-api'; +import { WorkletSourceNode } from 'react-native-audio-worklets'; + +function SineGenerator() { + const start = () => { + const ctx = new AudioContext(); + const sampleRate = ctx.sampleRate; + + const source = new WorkletSourceNode( + ctx, + (audioData, outputChannelCount, framesToProcess, currentTime, startOffset) => { + 'worklet'; + + const frequency = 440; + + for (let channel = 0; channel < outputChannelCount; channel++) { + for (let i = 0; i < framesToProcess; i++) { + const sampleTime = currentTime + (startOffset + i) / sampleRate; + const phase = 2 * Math.PI * frequency * sampleTime; + audioData[channel]![i] = Math.sin(phase) * 0.25; + } + } + } + ); -## Understanding `startOffset` and `currentTime` + source.connect(ctx.destination); + source.start(); + ctx.resume(); + }; -The relationship between `currentTime` and `startOffset` is crucial for generating continuous audio: + // ... +} +``` -```tsx -const worklet = (audioData, framesToProcess, currentTime, startOffset) => { - 'worklet'; +## Understanding `startOffset` and `currentTime` - const sampleRate = 44100; +When a source starts mid-quantum, native code zeroes the silent prefix in the output buffer. Your worklet still writes from index `0`, and native code copies those samples to the correct offset. - for (let i = 0; i < framesToProcess; i++) { - // Calculate the exact time for this sample - const sampleTime = currentTime + (startOffset + i) / sampleRate; +The absolute time for sample `i` is: - // Use sampleTime for phase calculations, LFOs, envelopes, etc. - const phase = 2 * Math.PI * frequency * sampleTime; - audioData[0][i] = Math.sin(phase); - } -}; +``` +currentTime + (startOffset + i) / sampleRate ``` -**Key points:** -- `currentTime` represents the audio context time at the start of the processing block -- `startOffset` tells you which sample within the block to start generating audio -- The absolute time for sample `i` is: `currentTime + (startOffset + i) / sampleRate` -- This ensures phase continuity and precise timing across processing blocks +Use this for phase-continuous oscillators, LFOs, and envelopes. ## Properties -It has no own properties but inherits from [`AudioScheduledSourceNode`](/docs/sources/audio-scheduled-source-node). - -## Methods - -It has no own methods but inherits from [`AudioScheduledSourceNode`](/docs/sources/audio-scheduled-source-node): +Inherits all properties from [`AudioScheduledSourceNode`](/docs/sources/audio-scheduled-source-node). -## Performance Considerations + -Since `WorkletSourceNode` generates audio in real-time, performance is critical: +## Methods -- Keep worklet functions lightweight and efficient -- Avoid complex calculations that could cause audio dropouts -- Consider using lookup tables for expensive operations like trigonometric functions -- Test on target devices to ensure smooth audio generation -- Use `AudioRuntime` for better performance, `UIRuntime` for UI integration +Inherits all methods from [`AudioScheduledSourceNode`](/docs/sources/audio-scheduled-source-node), including `start()` and `stop()`. -## Use Cases +## Performance considerations -- **Custom Synthesizers**: Generate waveforms, apply modulation, create complex timbres -- **Audio Generators**: White noise, pink noise, test tones, sweeps -- **Procedural Audio**: Dynamic soundscapes, generative music -- **Real-time Effects**: Audio that responds to user input or external data -- **Educational Tools**: Demonstrate audio synthesis concepts interactively +Callbacks run synchronously on the audio path each render quantum. Keep worklet functions lightweight — heavy math or allocations can cause dropouts. -## See Also +## See also -- [WorkletNode](/docs/worklets/worklet-node) - For processing existing audio with worklets -- [Introduction to worklets](/docs/worklets/worklets-introduction) - Understanding worklet fundamentals -- [AudioScheduledSourceNode](/docs/sources/audio-scheduled-source-node) - Base class for scheduled sources +- [`WorkletProcessingNode`](/docs/worklets/worklet-processing-node) — process an existing signal with a worklet +- [`WorkletNode`](/docs/worklets/worklet-node) — read-only snapshots on the UI runtime for visualizers +- [Worklets introduction](/docs/worklets/introduction) diff --git a/packages/react-native-audio-api/common/cpp/audioapi/compatibility/EXTENSION_API.md b/packages/react-native-audio-api/common/cpp/audioapi/compatibility/EXTENSION_API.md index 657643706..0d06e5fe5 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/compatibility/EXTENSION_API.md +++ b/packages/react-native-audio-api/common/cpp/audioapi/compatibility/EXTENSION_API.md @@ -103,10 +103,14 @@ obvious in the podspec: | `audioapi::AudioNodeHostObject` | Base HostObject for custom nodes exposed to JS | | `audioapi::BaseAudioContext` | Context handle; `getGraph()`, scheduling, sample rate | | `audioapi::AudioNode` | Base class for custom `AudioNode` implementations | +| `audioapi::AudioScheduledSourceNodeHostObject` | HostObject base for scheduled source nodes (`start`/`stop`/`onEnded`) | +| `audioapi::AudioScheduledSourceNode` | Base class for scheduled sources (e.g. worklet generators) | +| `AudioScheduledSourceNodeOptions` (via `types/NodeOptions.h`) | Options for scheduled source HostObjects | | `audioapi::utils::graph::Graph` | Audio graph owned by the context | | `RENDER_QUANTUM_SIZE` etc. (via `core/utils/Constants.h`) | Frame-size and engine constants | | `audioapi::AudioBuffer` | Multi-channel float buffer | -| `audioapi::AudioArrayBuffer` | Single-channel view for JSI handoff | +| `audioapi::AudioArrayBuffer` | Single-channel float buffer; implements `jsi::MutableBuffer` for zero-copy JSI `ArrayBuffer` handoff | +| `DELETE_COPY_AND_MOVE` (via `utils/Macros.h`) | Macro to delete copy and move constructors/assignment for extension-owned classes | --- diff --git a/packages/react-native-audio-api/common/cpp/audioapi/compatibility/StableAPI.h b/packages/react-native-audio-api/common/cpp/audioapi/compatibility/StableAPI.h index 83348d4f5..b4f78cdb8 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/compatibility/StableAPI.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/compatibility/StableAPI.h @@ -10,8 +10,12 @@ #include #include +#include #include #include +#include #include +#include #include #include +#include diff --git a/packages/react-native-audio-worklets/common/cpp/audioworklets/AudioWorkletsInstaller.h b/packages/react-native-audio-worklets/common/cpp/audioworklets/AudioWorkletsInstaller.h index f2d235ae4..9db392feb 100644 --- a/packages/react-native-audio-worklets/common/cpp/audioworklets/AudioWorkletsInstaller.h +++ b/packages/react-native-audio-worklets/common/cpp/audioworklets/AudioWorkletsInstaller.h @@ -2,7 +2,10 @@ #include #include +#include +#include #include +#include #include #include @@ -21,8 +24,26 @@ class AudioWorkletsInstaller { jsi::Function::createFromHostFunction( runtime, jsi::PropNameID::forAscii(runtime, "__createWorkletNode"), - 4, + 5, createWorkletNode)); + + runtime.global().setProperty( + runtime, + "__createWorkletSourceNode", + jsi::Function::createFromHostFunction( + runtime, + jsi::PropNameID::forAscii(runtime, "__createWorkletSourceNode"), + 3, + createWorkletSourceNode)); + + runtime.global().setProperty( + runtime, + "__createWorkletProcessingNode", + jsi::Function::createFromHostFunction( + runtime, + jsi::PropNameID::forAscii(runtime, "__createWorkletProcessingNode"), + 3, + createWorkletProcessingNode)); } private: @@ -50,19 +71,26 @@ class AudioWorkletsInstaller { worklets::Serializable::ValueType::WorkletType); } + static std::weak_ptr getAudioRuntimeOrThrow( + jsi::Runtime &runtime, + const jsi::Value &arg) { + return worklets::extractWorkletRuntime(runtime, arg); + } + static jsi::Value createWorkletNode( jsi::Runtime &runtime, const jsi::Value & /*thisValue*/, const jsi::Value *args, size_t count) { - if (count < 4) { + if (count < 5) { throw jsi::JSError( - runtime, "[react-native-audio-worklets] __createWorkletNode expects 4 arguments"); + runtime, "[react-native-audio-worklets] __createWorkletNode expects 5 arguments"); } const auto &context = getContextOrThrow(runtime, args[0]); auto serializableWorklet = getSerializableWorkletOrThrow(runtime, args[1]); + const auto bufferLength = static_cast(args[2].asNumber()); - auto uiRuntimeHolder = args[2].asObject(runtime); + auto uiRuntimeHolder = args[3].asObject(runtime); auto uiRuntime = worklets::getWorkletRuntimeFromHolder(runtime, uiRuntimeHolder); if (uiRuntime == nullptr) { throw jsi::JSError( @@ -71,7 +99,7 @@ class AudioWorkletsInstaller { "Make sure react-native-worklets is installed."); } - auto uiSchedulerHolder = args[3].asObject(runtime); + auto uiSchedulerHolder = args[4].asObject(runtime); auto uiScheduler = worklets::getUISchedulerFromHolder(runtime, uiSchedulerHolder); if (uiScheduler == nullptr) { throw jsi::JSError( @@ -85,7 +113,51 @@ class AudioWorkletsInstaller { context, std::move(uiRuntime), std::move(uiScheduler), - std::move(serializableWorklet)); + std::move(serializableWorklet), + bufferLength); + + auto object = jsi::Object::createFromHostObject(runtime, hostObject); + object.setExternalMemoryPressure(runtime, hostObject->getMemoryPressure()); + return object; + } + + static jsi::Value createWorkletSourceNode( + jsi::Runtime &runtime, + const jsi::Value & /*thisValue*/, + const jsi::Value *args, + size_t count) { + if (count < 3) { + throw jsi::JSError( + runtime, "[react-native-audio-worklets] __createWorkletSourceNode expects 3 arguments"); + } + + const auto &context = getContextOrThrow(runtime, args[0]); + auto serializableWorklet = getSerializableWorkletOrThrow(runtime, args[1]); + const auto workletRuntime = getAudioRuntimeOrThrow(runtime, args[2]); + + auto hostObject = std::make_shared( + context->getGraph(), context, workletRuntime, serializableWorklet); + + return jsi::Object::createFromHostObject(runtime, hostObject); + } + + static jsi::Value createWorkletProcessingNode( + jsi::Runtime &runtime, + const jsi::Value & /*thisValue*/, + const jsi::Value *args, + size_t count) { + if (count < 3) { + throw jsi::JSError( + runtime, + "[react-native-audio-worklets] __createWorkletProcessingNode expects 3 arguments"); + } + + const auto &context = getContextOrThrow(runtime, args[0]); + auto serializableWorklet = getSerializableWorkletOrThrow(runtime, args[1]); + const auto workletRuntime = getAudioRuntimeOrThrow(runtime, args[2]); + + auto hostObject = std::make_shared( + context->getGraph(), context, workletRuntime, serializableWorklet); auto object = jsi::Object::createFromHostObject(runtime, hostObject); object.setExternalMemoryPressure(runtime, hostObject->getMemoryPressure()); diff --git a/packages/react-native-audio-worklets/common/cpp/audioworklets/AudioWorkletsRunner.cpp b/packages/react-native-audio-worklets/common/cpp/audioworklets/AudioWorkletsRunner.cpp new file mode 100644 index 000000000..a4edf5932 --- /dev/null +++ b/packages/react-native-audio-worklets/common/cpp/audioworklets/AudioWorkletsRunner.cpp @@ -0,0 +1,92 @@ +#include + +#include +#include + +namespace audioworklets { + +AudioWorkletsRunner::AudioWorkletsRunner( + std::weak_ptr weakRuntime, + const std::shared_ptr &serializableWorklet) + : weakRuntime_(std::move(weakRuntime)) { + auto strongRuntime = weakRuntime_.lock(); + if (strongRuntime == nullptr) { + return; + } + + unsafeRuntimePtr_ = &worklets::getJSIRuntimeFromWorkletRuntime(strongRuntime); + strongRuntime->runSync([this, serializableWorklet](jsi::Runtime &rt) -> jsi::Value { + new (reinterpret_cast(unsafeWorklet_.data())) + jsi::Function(serializableWorklet->toJSValue(rt).asObject(rt).asFunction(rt)); + workletInitialized_ = true; + return jsi::Value::undefined(); + }); +} + +AudioWorkletsRunner::AudioWorkletsRunner(AudioWorkletsRunner &&other) noexcept + : weakRuntime_(std::move(other.weakRuntime_)), + unsafeRuntimePtr_(other.unsafeRuntimePtr_), + workletInitialized_(other.workletInitialized_) { + if (workletInitialized_) { + unsafeWorklet_ = other.unsafeWorklet_; + other.workletInitialized_ = false; + other.unsafeRuntimePtr_ = nullptr; + } +} + +AudioWorkletsRunner &AudioWorkletsRunner::operator=(AudioWorkletsRunner &&other) noexcept { + if (this == &other) { + return *this; + } + + destroyCachedWorklet(); + + weakRuntime_ = std::move(other.weakRuntime_); + unsafeRuntimePtr_ = other.unsafeRuntimePtr_; + workletInitialized_ = other.workletInitialized_; + + if (workletInitialized_) { + unsafeWorklet_ = other.unsafeWorklet_; + other.workletInitialized_ = false; + other.unsafeRuntimePtr_ = nullptr; + } + + return *this; +} + +AudioWorkletsRunner::~AudioWorkletsRunner() { + destroyCachedWorklet(); +} + +void AudioWorkletsRunner::destroyCachedWorklet() { + if (!workletInitialized_) { + return; + } + + auto strongRuntime = weakRuntime_.lock(); + if (strongRuntime == nullptr) { + workletInitialized_ = false; + unsafeRuntimePtr_ = nullptr; + return; + } + + strongRuntime->runSync([this](jsi::Runtime & /*rt*/) -> jsi::Value { + reinterpret_cast(unsafeWorklet_.data())->~Function(); + workletInitialized_ = false; + unsafeRuntimePtr_ = nullptr; + return jsi::Value::undefined(); + }); +} + +std::shared_ptr AudioWorkletsRunner::createChannelViews( + size_t frameCount, + size_t channelCount) { + auto strongRuntime = weakRuntime_.lock(); + if (strongRuntime == nullptr) { + return nullptr; + } + + return std::make_shared(strongRuntime, frameCount, channelCount); +} + +} // namespace audioworklets diff --git a/packages/react-native-audio-worklets/common/cpp/audioworklets/AudioWorkletsRunner.h b/packages/react-native-audio-worklets/common/cpp/audioworklets/AudioWorkletsRunner.h new file mode 100644 index 000000000..90172b352 --- /dev/null +++ b/packages/react-native-audio-worklets/common/cpp/audioworklets/AudioWorkletsRunner.h @@ -0,0 +1,75 @@ +#pragma once + +#include +#include + +#include + +#include + +#include +#include +#include +#include + +namespace audioworklets { + +using namespace facebook; + +/** + * Invokes a serializable worklet synchronously on the audio WorkletRuntime from + * the audio thread. + * + * Used by `WorkletSourceNode` and `WorkletProcessingNode`. Channel views are + * pre-built via `createChannelViews` and passed to the worklet on each render + * quantum without per-call JSI allocation. + */ +class AudioWorkletsRunner { + public: + AudioWorkletsRunner( + std::weak_ptr weakRuntime, + const std::shared_ptr &serializableWorklet); + + AudioWorkletsRunner(const AudioWorkletsRunner &) = delete; + AudioWorkletsRunner &operator=(const AudioWorkletsRunner &) = delete; + AudioWorkletsRunner(AudioWorkletsRunner &&other) noexcept; + AudioWorkletsRunner &operator=(AudioWorkletsRunner &&other) noexcept; + ~AudioWorkletsRunner(); + + [[nodiscard]] bool isActive() const { + return workletInitialized_ && unsafeRuntimePtr_ != nullptr; + } + + /// @brief Invokes the cached worklet on the audio worklet runtime. + /// @note Audio Thread only. Caller must check `isActive()` first. + template + jsi::Value callUnsafe(Args &&...args) const { + return getUnsafeWorklet().call(*unsafeRuntimePtr_, std::forward(args)...); + } + + /// @brief Allocates channel buffers and pre-builds stable `Float32Array[]` views on + /// the audio worklet runtime. + /// @param frameCount Number of frames per channel (view length). + /// @param channelCount Number of channel slots in the pool. + /// @note Must be called once before the first `callUnsafe`. + [[nodiscard]] std::shared_ptr createChannelViews( + size_t frameCount, + size_t channelCount); + + private: + std::weak_ptr weakRuntime_; + jsi::Runtime *unsafeRuntimePtr_ = nullptr; + + /// @brief Placement storage for the deserialized worklet `jsi::Function`. + alignas(jsi::Function) std::array unsafeWorklet_{}; + + bool workletInitialized_ = false; + + void destroyCachedWorklet(); + + [[nodiscard]] const jsi::Function &getUnsafeWorklet() const { + return *reinterpret_cast(unsafeWorklet_.data()); + } +}; + +} // namespace audioworklets diff --git a/packages/react-native-audio-worklets/common/cpp/audioworklets/HostObjects/WorkletNodeHostObject.h b/packages/react-native-audio-worklets/common/cpp/audioworklets/HostObjects/WorkletNodeHostObject.h index f8cd5710a..1070b9b86 100644 --- a/packages/react-native-audio-worklets/common/cpp/audioworklets/HostObjects/WorkletNodeHostObject.h +++ b/packages/react-native-audio-worklets/common/cpp/audioworklets/HostObjects/WorkletNodeHostObject.h @@ -18,7 +18,8 @@ class WorkletNodeHostObject : public audioapi::AudioNodeHostObject { const std::shared_ptr &context, std::shared_ptr uiRuntime, std::shared_ptr uiScheduler, - std::shared_ptr serializableWorklet) + std::shared_ptr serializableWorklet, + size_t bufferLength) : audioapi::AudioNodeHostObject( graph, std::make_unique( @@ -26,13 +27,17 @@ class WorkletNodeHostObject : public audioapi::AudioNodeHostObject { UIWorkletsRunner( std::move(uiRuntime), std::move(uiScheduler), - std::move(serializableWorklet)))) {} + std::move(serializableWorklet)), + bufferLength)), + bufferLength_(bufferLength) {} [[nodiscard]] size_t getMemoryPressure() const override { return AudioNodeHostObject::getMemoryPressure() + - static_cast(audioapi::MAX_CHANNEL_COUNT) * audioapi::RENDER_QUANTUM_SIZE * - sizeof(float); + static_cast(audioapi::MAX_CHANNEL_COUNT) * bufferLength_ * sizeof(float); } + + private: + size_t bufferLength_; }; } // namespace audioworklets diff --git a/packages/react-native-audio-worklets/common/cpp/audioworklets/HostObjects/WorkletProcessingNodeHostObject.h b/packages/react-native-audio-worklets/common/cpp/audioworklets/HostObjects/WorkletProcessingNodeHostObject.h new file mode 100644 index 000000000..798952b9c --- /dev/null +++ b/packages/react-native-audio-worklets/common/cpp/audioworklets/HostObjects/WorkletProcessingNodeHostObject.h @@ -0,0 +1,34 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace audioworklets { + +using namespace facebook; + +class WorkletProcessingNodeHostObject : public audioapi::AudioNodeHostObject { + public: + WorkletProcessingNodeHostObject( + const std::shared_ptr &graph, + const std::shared_ptr &context, + std::weak_ptr workletRuntime, + const std::shared_ptr &serializableWorklet) + : audioapi::AudioNodeHostObject( + graph, + std::make_unique( + context, + AudioWorkletsRunner(std::move(workletRuntime), serializableWorklet))) {} + + [[nodiscard]] size_t getMemoryPressure() const override { + return AudioNodeHostObject::getMemoryPressure() + + 2 * static_cast(audioapi::MAX_CHANNEL_COUNT) * audioapi::RENDER_QUANTUM_SIZE * + sizeof(float); + } +}; + +} // namespace audioworklets diff --git a/packages/react-native-audio-worklets/common/cpp/audioworklets/HostObjects/WorkletSourceNodeHostObject.h b/packages/react-native-audio-worklets/common/cpp/audioworklets/HostObjects/WorkletSourceNodeHostObject.h new file mode 100644 index 000000000..4b77d22d7 --- /dev/null +++ b/packages/react-native-audio-worklets/common/cpp/audioworklets/HostObjects/WorkletSourceNodeHostObject.h @@ -0,0 +1,34 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace audioworklets { + +using namespace facebook; + +class WorkletSourceNodeHostObject : public audioapi::AudioScheduledSourceNodeHostObject { + public: + WorkletSourceNodeHostObject( + const std::shared_ptr &graph, + const std::shared_ptr &context, + std::weak_ptr workletRuntime, + const std::shared_ptr &serializableWorklet) + : audioapi::AudioScheduledSourceNodeHostObject( + graph, + std::make_unique( + context, + AudioWorkletsRunner(std::move(workletRuntime), serializableWorklet))) {} + + [[nodiscard]] size_t getMemoryPressure() const override { + return AudioNodeHostObject::getMemoryPressure() + + static_cast(audioapi::MAX_CHANNEL_COUNT) * audioapi::RENDER_QUANTUM_SIZE * + sizeof(float); + } +}; + +} // namespace audioworklets diff --git a/packages/react-native-audio-worklets/common/cpp/audioworklets/UIWorkletsRunner.cpp b/packages/react-native-audio-worklets/common/cpp/audioworklets/UIWorkletsRunner.cpp index 7c1c08023..4d375c2b5 100644 --- a/packages/react-native-audio-worklets/common/cpp/audioworklets/UIWorkletsRunner.cpp +++ b/packages/react-native-audio-worklets/common/cpp/audioworklets/UIWorkletsRunner.cpp @@ -1,49 +1,108 @@ #include +#include -#include #include #include namespace audioworklets { UIWorkletsRunner::UIWorkletsRunner( - std::shared_ptr uiRuntime, - std::shared_ptr uiScheduler, + const std::shared_ptr &uiRuntime, + const std::shared_ptr &uiScheduler, std::shared_ptr serializableWorklet) - : uiRuntime_(std::move(uiRuntime)), - uiScheduler_(std::move(uiScheduler)), - serializableWorklet_(std::move(serializableWorklet)) {} - -void UIWorkletsRunner::invokeOnUI( - std::shared_ptr>> channels, - size_t channelCount, - std::function onComplete) const { - worklets::scheduleOnUI( - uiScheduler_, - [uiRuntime = uiRuntime_, - serializableWorklet = serializableWorklet_, - channels = std::move(channels), - channelCount, - onComplete = std::move(onComplete)]() { - std::atomic_thread_fence(std::memory_order_acquire); - - jsi::Runtime &rt = worklets::getJSIRuntimeFromWorkletRuntime(uiRuntime); - - auto audioData = jsi::Array(rt, channelCount); - for (size_t ch = 0; ch < channelCount; ++ch) { - audioData.setValueAtIndex(rt, ch, jsi::ArrayBuffer(rt, (*channels)[ch])); - } - - worklets::runSyncOnRuntime( - uiRuntime, - serializableWorklet, - jsi::Value(std::move(audioData)), - jsi::Value(static_cast(channelCount))); - - if (onComplete) { - onComplete(); - } - }); + : alive_(std::make_shared>(true)), + uiRuntime_(uiRuntime), + uiScheduler_(uiScheduler), + serializableWorklet_(std::move(serializableWorklet)), + job_(std::make_shared()) { + job_->alive = alive_; + job_->uiRuntime = uiRuntime_; + job_->uiScheduler = uiScheduler_; + job_->serializableWorklet = serializableWorklet_; +} + +void UIWorkletsRunner::deactivate() { + alive_->store(false, std::memory_order_release); + + auto channelViews = channelViews_; + if (channelViews == nullptr) { + return; + } + + auto uiScheduler = uiScheduler_.lock(); + if (uiScheduler == nullptr) { + channelViews->releaseJsValues(); + return; + } + + worklets::scheduleOnUI(uiScheduler, [channelViews]() { channelViews->releaseJsValues(); }); +} + +bool UIWorkletsRunner::isActive() const { + return alive_->load(std::memory_order_acquire); +} + +std::shared_ptr UIWorkletsRunner::createChannelViews( + size_t frameCount, + size_t channelCount) { + auto uiRuntime = uiRuntime_.lock(); + if (!uiRuntime) { + return nullptr; + } + + channelViews_ = std::make_shared(uiRuntime, frameCount, channelCount); + job_->channelViews = channelViews_; + return channelViews_; +} + +void UIWorkletsRunner::call(size_t channelCount, std::function onComplete) const { + auto uiRuntime = uiRuntime_.lock(); + auto uiScheduler = uiScheduler_.lock(); + if (!isActive() || !serializableWorklet_ || !uiRuntime || !uiScheduler) { + if (onComplete) { + onComplete(); + } + return; + } + + job_->channelCount = channelCount; + job_->onComplete = std::move(onComplete); + + worklets::scheduleOnUI(uiScheduler, [job = job_]() { runUIWorkletJob(job); }); +} + +void UIWorkletsRunner::runUIWorkletJob(const std::shared_ptr &job) { + if (job == nullptr) { + return; + } + + if (!job->alive->load(std::memory_order_acquire)) { + if (job->onComplete) { + job->onComplete(); + } + return; + } + + auto runtime = job->uiRuntime.lock(); + auto channelViews = job->channelViews; + const jsi::Value *audioData = + channelViews != nullptr ? channelViews->channelsArray(job->channelCount) : nullptr; + if (!runtime || !job->uiScheduler.lock() || audioData == nullptr) { + if (job->onComplete) { + job->onComplete(); + } + return; + } + + worklets::runSyncOnRuntime( + runtime, + job->serializableWorklet, + *audioData, + jsi::Value(static_cast(job->channelCount))); + + if (job->onComplete) { + job->onComplete(); + } } } // namespace audioworklets diff --git a/packages/react-native-audio-worklets/common/cpp/audioworklets/UIWorkletsRunner.h b/packages/react-native-audio-worklets/common/cpp/audioworklets/UIWorkletsRunner.h index 752ceed6f..424d417c2 100644 --- a/packages/react-native-audio-worklets/common/cpp/audioworklets/UIWorkletsRunner.h +++ b/packages/react-native-audio-worklets/common/cpp/audioworklets/UIWorkletsRunner.h @@ -1,13 +1,13 @@ #pragma once #include +#include #include -#include - +#include +#include #include #include -#include namespace audioworklets { @@ -27,36 +27,57 @@ using namespace facebook; * `scheduleOnUI`. This is fire-and-forget: UI animation does not need a result * and must not block the audio thread. * - * All state is held in shared pointers so a scheduled job stays valid even if - * the owning node is destroyed before the job runs on the UI thread. + * Scheduled jobs capture a pre-allocated `UIWorkletJob` shared_ptr so the + * `scheduleOnUI` closure stays small. Only one job may be in flight at a time. */ class UIWorkletsRunner { public: UIWorkletsRunner( - std::shared_ptr uiRuntime, - std::shared_ptr uiScheduler, + const std::shared_ptr &uiRuntime, + const std::shared_ptr &uiScheduler, std::shared_ptr serializableWorklet); - [[nodiscard]] bool isValid() const { - return uiRuntime_ != nullptr && uiScheduler_ != nullptr && serializableWorklet_ != nullptr; - } + /// Stops scheduling new UI jobs and causes in-flight lambdas to no-op. + /// JSI views are released on the UI scheduler after any in-flight job completes. + /// Safe to call multiple times. + void deactivate(); + + [[nodiscard]] bool isActive() const; + + /// Allocates channel buffers and pre-builds stable `Float32Array[]` views on + /// the UI worklet runtime. Must be called once before the first `call`. + [[nodiscard]] std::shared_ptr createChannelViews( + size_t frameCount, + size_t channelCount); - /// @brief Schedules the worklet to run on the UI thread with the given audio - /// data. The channel buffer pool is captured (kept alive) by the scheduled job; - /// the JSI arrays are built on the UI runtime, inside the UI thread. - /// @param channels Shared, fixed-size pool of per-channel snapshot buffers. + /// @brief Schedules the worklet to run on the UI thread with pre-built channel + /// views from `createChannelViews`. /// @param channelCount Number of channels to expose to the worklet. /// @param onComplete Invoked on the UI thread once the worklet returns. /// @note Audio Thread only. - void invokeOnUI( - std::shared_ptr>> channels, - size_t channelCount, - std::function onComplete) const; + void call(size_t channelCount, std::function onComplete) const; private: - std::shared_ptr uiRuntime_; - std::shared_ptr uiScheduler_; + struct UIWorkletJob { + std::shared_ptr> alive; + std::weak_ptr uiRuntime; + std::weak_ptr uiScheduler; + std::shared_ptr serializableWorklet; + std::shared_ptr channelViews; + size_t channelCount{0}; + std::function onComplete; + }; + + static void runUIWorkletJob(const std::shared_ptr &job); + + /// Shared with scheduled UI lambdas so they can bail out after teardown/reload. + std::shared_ptr> alive_; + std::weak_ptr uiRuntime_; + std::weak_ptr uiScheduler_; std::shared_ptr serializableWorklet_; + std::shared_ptr channelViews_; + /// Reused across dispatches; only one job may be in flight at a time. + std::shared_ptr job_; }; } // namespace audioworklets diff --git a/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletNode.cpp b/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletNode.cpp index 95c677ab2..ad9659f6d 100644 --- a/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletNode.cpp +++ b/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletNode.cpp @@ -2,7 +2,6 @@ #include #include -#include #include #include @@ -10,68 +9,80 @@ namespace audioworklets { WorkletNode::WorkletNode( const std::shared_ptr &context, - UIWorkletsRunner workletRunner) + UIWorkletsRunner workletRunner, + size_t bufferLength) : audioapi::AudioNode(context), workletRunner_(std::move(workletRunner)), - busy_(std::make_shared>(false)), - minFramesBetweenDispatch_( - static_cast( - std::max(1.0f, std::floor(context->getSampleRate() / kMaxUiDispatchRateHz)))) { - snapshotBuffers_ = std::make_shared>>( - static_cast(audioapi::MAX_CHANNEL_COUNT)); - for (size_t ch = 0; ch < static_cast(audioapi::MAX_CHANNEL_COUNT); ++ch) { - (*snapshotBuffers_)[ch] = - std::make_shared(audioapi::RENDER_QUANTUM_SIZE); - } + bufferLength_(bufferLength), + busy_(std::make_shared>(false)) { + channelViews_ = workletRunner_.createChannelViews( + bufferLength_, static_cast(audioapi::MAX_CHANNEL_COUNT)); +} + +WorkletNode::~WorkletNode() { + workletRunner_.deactivate(); } void WorkletNode::processNode(int framesToProcess) { + if (!workletRunner_.isActive() || channelViews_ == nullptr) { + return; + } + if (framesToProcess <= 0) { return; } - const size_t frameCount = static_cast(framesToProcess); - const size_t channelCount = audioBuffer_->getNumberOfChannels(); + if (busy_->load(std::memory_order_acquire)) { + return; + } + + const auto frameCount = static_cast(framesToProcess); + const auto channelCount = audioBuffer_->getNumberOfChannels(); if (channelCount == 0) { return; } - framesSinceLastDispatch_ += frameCount; + const size_t channelsToCopy = + std::min(channelCount, static_cast(audioapi::MAX_CHANNEL_COUNT)); - if (framesSinceLastDispatch_ < minFramesBetweenDispatch_) { + const size_t remaining = bufferLength_ - framesFilled_; + if (remaining == 0) { return; } - framesSinceLastDispatch_ = 0; - dispatchToUI(frameCount, channelCount); + const size_t framesToCopy = std::min(frameCount, remaining); + + for (size_t ch = 0; ch < channelsToCopy; ++ch) { + channelViews_->channelBuffer(ch)->copy( + *audioBuffer_->getChannel(ch), 0, framesFilled_, framesToCopy); + } + + framesFilled_ += framesToCopy; + + if (framesFilled_ >= bufferLength_) { + dispatchToUI(channelsToCopy); + } } -void WorkletNode::dispatchToUI(size_t frameCount, size_t channelCount) { - if (!workletRunner_.isValid()) { +void WorkletNode::dispatchToUI(size_t channelCount) { + if (!workletRunner_.isActive()) { + framesFilled_ = 0; return; } - const size_t channelsToCopy = - std::min(channelCount, static_cast(audioapi::MAX_CHANNEL_COUNT)); - - // Drop this quantum if the previous UI invocation has not completed yet. bool expected = false; if (!busy_->compare_exchange_strong( expected, true, std::memory_order_acq_rel, std::memory_order_acquire)) { return; } - for (size_t ch = 0; ch < channelsToCopy; ++ch) { - (*snapshotBuffers_)[ch]->copy(*audioBuffer_->getChannel(ch), 0, 0, frameCount); - } - std::atomic_thread_fence(std::memory_order_release); auto busy = busy_; - workletRunner_.invokeOnUI(snapshotBuffers_, channelsToCopy, [busy]() { - busy->store(false, std::memory_order_release); - }); + workletRunner_.call(channelCount, [busy]() { busy->store(false, std::memory_order_release); }); + + framesFilled_ = 0; } } // namespace audioworklets diff --git a/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletNode.h b/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletNode.h index b99496fac..820d79894 100644 --- a/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletNode.h +++ b/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletNode.h @@ -2,52 +2,47 @@ #include #include +#include #include #include #include -#include namespace audioworklets { /** - * A pass-through analysis node that hands render-quantum audio snapshots to a + * A pass-through analysis node that hands buffered audio snapshots to a * JavaScript worklet on the UI runtime so the UI can be animated from live * audio (e.g. amplitude/RMS visualizers). * - * UI dispatches are capped at ~120 Hz (sample-rate aware) so the audio thread - * does not flood the UI scheduler. Audio flows through unchanged. + * Render-quantum frames are copied directly into snapshot buffers until + * `bufferLength` is reached, then a snapshot is dispatched to the UI scheduler. + * While the UI callback is running, incoming frames are skipped. Audio flows + * through unchanged. */ class WorkletNode : public audioapi::AudioNode { public: WorkletNode( const std::shared_ptr &context, - UIWorkletsRunner workletRunner); + UIWorkletsRunner workletRunner, + size_t bufferLength); + + ~WorkletNode() override; protected: void processNode(int framesToProcess) override; private: - static constexpr float kMaxUiDispatchRateHz = 120.0f; - - void dispatchToUI(size_t frameCount, size_t channelCount); + void dispatchToUI(size_t channelCount); UIWorkletsRunner workletRunner_; + std::shared_ptr channelViews_; + size_t bufferLength_; + size_t framesFilled_{0}; - /// @brief Fixed pool of per-channel UI snapshot buffers (MAX_CHANNEL_COUNT × render quantum). - /// Allocated once in the constructor; never reallocated. Held in a shared_ptr so - /// scheduled UI jobs keep the pool alive without per-quantum vector allocation. - std::shared_ptr>> snapshotBuffers_; - - /// @brief True while a UI-thread worklet invocation is still pending. Prevents - /// the audio thread from flooding the UI scheduler; quanta are dropped instead. + /// @brief True while a UI-thread worklet invocation is still pending. Snapshot buffers + /// are not filled until the callback completes. std::shared_ptr> busy_; - - /// @brief Audio-thread accumulator for sample-rate-aware UI dispatch throttling. - size_t framesSinceLastDispatch_{0}; - - /// @brief Minimum audio frames between UI worklet dispatches (~120 Hz at context sample rate). - size_t minFramesBetweenDispatch_{0}; }; } // namespace audioworklets diff --git a/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletProcessingNode.cpp b/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletProcessingNode.cpp new file mode 100644 index 000000000..00b27acc8 --- /dev/null +++ b/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletProcessingNode.cpp @@ -0,0 +1,75 @@ +#include + +#include + +#include +#include + +namespace audioworklets { + +WorkletProcessingNode::WorkletProcessingNode( + const std::shared_ptr &context, + AudioWorkletsRunner &&workletRunner) + : audioapi::AudioNode(context), workletRunner_(std::move(workletRunner)) { + const auto frameCount = audioapi::RENDER_QUANTUM_SIZE; + const auto channelCount = static_cast(audioapi::MAX_CHANNEL_COUNT); + + inputChannelViews_ = workletRunner_.createChannelViews(frameCount, channelCount); + outputChannelViews_ = workletRunner_.createChannelViews(frameCount, channelCount); +} + +WorkletProcessingNode::~WorkletProcessingNode() { + if (inputChannelViews_ != nullptr) { + inputChannelViews_->releaseJsValues(); + } + if (outputChannelViews_ != nullptr) { + outputChannelViews_->releaseJsValues(); + } +} + +void WorkletProcessingNode::processNode(int framesToProcess) { + if (inputChannelViews_ == nullptr || outputChannelViews_ == nullptr || framesToProcess <= 0) { + return; + } + + const size_t inputChannelCount = getInputBuffer()->getNumberOfChannels(); + const size_t outputChannelCount = getOutputBuffer()->getNumberOfChannels(); + + if (inputChannelCount == 0 || outputChannelCount == 0) { + return; + } + + for (size_t ch = 0; ch < inputChannelCount; ++ch) { + inputChannelViews_->channelBuffer(ch)->copy( + *getInputBuffer()->getChannel(ch), 0, 0, framesToProcess); + } + + double time = 0.0; + if (std::shared_ptr context = context_.lock()) { + time = context->getCurrentTime(); + } + + const jsi::Value *inputData = inputChannelViews_->channelsArray(inputChannelCount); + const jsi::Value *outputData = outputChannelViews_->channelsArray(outputChannelCount); + if (inputData == nullptr || outputData == nullptr || !workletRunner_.isActive()) { + for (size_t ch = 0; ch < outputChannelCount; ++ch) { + getOutputBuffer()->getChannel(ch)->zero(0, framesToProcess); + } + return; + } + + workletRunner_.callUnsafe( + *inputData, + *outputData, + jsi::Value(static_cast(inputChannelCount)), + jsi::Value(static_cast(outputChannelCount)), + jsi::Value(framesToProcess), + jsi::Value(time)); + + for (size_t ch = 0; ch < outputChannelCount; ++ch) { + getOutputBuffer()->getChannel(ch)->copy( + *outputChannelViews_->channelBuffer(ch), 0, 0, framesToProcess); + } +} + +} // namespace audioworklets diff --git a/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletProcessingNode.h b/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletProcessingNode.h new file mode 100644 index 000000000..37dd7728a --- /dev/null +++ b/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletProcessingNode.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace audioworklets { + +/** + * An effect node that processes audio synchronously on the audio worklet runtime + * each render quantum. + * + * Input samples are copied into an input `AudioChannelViews` pool, the worklet + * callback receives separate stable `Float32Array[]` views for input and output, + * and processed samples are copied from the output pool back to the node's audio + * buffer. + */ +class WorkletProcessingNode : public audioapi::AudioNode { + public: + WorkletProcessingNode( + const std::shared_ptr &context, + AudioWorkletsRunner &&workletRunner); + + ~WorkletProcessingNode() override; + DELETE_COPY_AND_MOVE(WorkletProcessingNode); + + protected: + void processNode(int framesToProcess) override; + + private: + AudioWorkletsRunner workletRunner_; + std::shared_ptr inputChannelViews_; + std::shared_ptr outputChannelViews_; +}; + +} // namespace audioworklets diff --git a/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletSourceNode.cpp b/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletSourceNode.cpp new file mode 100644 index 000000000..bcd47d8d8 --- /dev/null +++ b/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletSourceNode.cpp @@ -0,0 +1,78 @@ +#include + +#include + +#include +#include + +namespace audioworklets { + +WorkletSourceNode::WorkletSourceNode( + const std::shared_ptr &context, + AudioWorkletsRunner &&workletRunner) + : audioapi::AudioScheduledSourceNode(context), workletRunner_(std::move(workletRunner)) { + outputChannelViews_ = workletRunner_.createChannelViews( + audioapi::RENDER_QUANTUM_SIZE, static_cast(audioapi::MAX_CHANNEL_COUNT)); +} + +WorkletSourceNode::~WorkletSourceNode() { + if (outputChannelViews_ != nullptr) { + outputChannelViews_->releaseJsValues(); + } +} + +void WorkletSourceNode::processNode(int framesToProcess) { + if (outputChannelViews_ == nullptr || framesToProcess <= 0) { + return; + } + + if (isUnscheduled() || isFinished()) { + audioBuffer_->zero(); + return; + } + + size_t startOffset = 0; + auto nonSilentFramesToProcess = static_cast(framesToProcess); + + std::shared_ptr context = context_.lock(); + if (context == nullptr) { + audioBuffer_->zero(); + return; + } + + updatePlaybackInfo( + audioBuffer_, + framesToProcess, + startOffset, + nonSilentFramesToProcess, + context->getSampleRate(), + context->getCurrentSampleFrame()); + + if ((!isPlaying() && !isStopScheduled()) || nonSilentFramesToProcess == 0) { + audioBuffer_->zero(); + return; + } + + const size_t outputChannelCount = audioBuffer_->getNumberOfChannels(); + const jsi::Value *outputData = outputChannelViews_->channelsArray(outputChannelCount); + if (outputData == nullptr || !workletRunner_.isActive()) { + audioBuffer_->zero(); + return; + } + + workletRunner_.callUnsafe( + *outputData, + jsi::Value(static_cast(outputChannelCount)), + jsi::Value(static_cast(nonSilentFramesToProcess)), + jsi::Value(context->getCurrentTime()), + jsi::Value(static_cast(startOffset))); + + for (size_t i = 0; i < outputChannelCount; ++i) { + audioBuffer_->getChannel(i)->copy( + *outputChannelViews_->channelBuffer(i), 0, startOffset, nonSilentFramesToProcess); + } + + handleStopScheduled(); +} + +} // namespace audioworklets diff --git a/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletSourceNode.h b/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletSourceNode.h new file mode 100644 index 000000000..c0addf5a8 --- /dev/null +++ b/packages/react-native-audio-worklets/common/cpp/audioworklets/core/WorkletSourceNode.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +#include + +namespace audioworklets { + +/** + * A scheduled source node that generates audio synchronously on the audio worklet + * runtime each render quantum. + * + * The worklet callback receives stable `Float32Array[]` views over an output pool; + * generated samples are copied into the node's audio buffer at the scheduled offset. + */ +class WorkletSourceNode : public audioapi::AudioScheduledSourceNode { + public: + WorkletSourceNode( + const std::shared_ptr &context, + AudioWorkletsRunner &&workletRunner); + + ~WorkletSourceNode() override; + DELETE_COPY_AND_MOVE(WorkletSourceNode); + + protected: + void processNode(int framesToProcess) override; + + private: + AudioWorkletsRunner workletRunner_; + std::shared_ptr outputChannelViews_; +}; + +} // namespace audioworklets diff --git a/packages/react-native-audio-worklets/common/cpp/audioworklets/utils/AudioChannelViews.cpp b/packages/react-native-audio-worklets/common/cpp/audioworklets/utils/AudioChannelViews.cpp new file mode 100644 index 000000000..2f756d084 --- /dev/null +++ b/packages/react-native-audio-worklets/common/cpp/audioworklets/utils/AudioChannelViews.cpp @@ -0,0 +1,102 @@ +#include +#include + +#include +#include +#include + +namespace audioworklets { + +AudioChannelViews::AudioChannelViews( + const std::shared_ptr &runtime, + size_t frameCount, + size_t channelCount) + : runtime_(runtime), frameCount_(frameCount) { + if (runtime == nullptr || frameCount == 0 || channelCount == 0) { + throw std::invalid_argument("AudioChannelViews: invalid construction arguments"); + } + + channelBuffers_.resize(channelCount); + + for (size_t ch = 0; ch < channelCount; ++ch) { + channelBuffers_[ch] = std::make_shared(frameCount); + } + + runtime_->runSync([this](jsi::Runtime &rt) -> jsi::Value { + createFloat32ChannelViews(rt); + createChannelsArraysByCount(rt); + return jsi::Value::undefined(); + }); +} + +AudioChannelViews::~AudioChannelViews() { + releaseJsValues(); +} + +void AudioChannelViews::releaseJsValues() { + if (jsValuesReleased_ || runtime_ == nullptr) { + return; + } + + runtime_->runSync([this](jsi::Runtime &) -> jsi::Value { + arrayBufferValues_.clear(); + float32ChannelValues_.clear(); + channelsArraysByCount_.clear(); + return jsi::Value::undefined(); + }); + jsValuesReleased_ = true; + runtime_.reset(); +} + +void AudioChannelViews::createFloat32ChannelViews(jsi::Runtime &rt) { + const auto channelCount = channelBuffers_.size(); + auto float32ArrayCtor = rt.global().getPropertyAsFunction(rt, "Float32Array"); + + arrayBufferValues_.resize(channelCount); + float32ChannelValues_.resize(channelCount); + + for (size_t ch = 0; ch < channelCount; ++ch) { + auto arrayBuffer = jsi::ArrayBuffer(rt, channelBuffers_[ch]); + arrayBufferValues_[ch] = jsi::Value(std::move(arrayBuffer)); + + auto float32Array = float32ArrayCtor + .callAsConstructor( + rt, + arrayBufferValues_[ch], + jsi::Value(0), + jsi::Value(static_cast(frameCount_))) + .getObject(rt); + + float32Array.setExternalMemoryPressure(rt, channelBuffers_[ch]->size()); + float32ChannelValues_[ch] = jsi::Value(std::move(float32Array)); + } +} + +void AudioChannelViews::createChannelsArraysByCount(jsi::Runtime &rt) { + const auto maxChannelCount = float32ChannelValues_.size(); + channelsArraysByCount_.resize(maxChannelCount + 1); + + for (size_t count = 1; count <= maxChannelCount; ++count) { + jsi::Array channels(rt, count); + for (size_t ch = 0; ch < count; ++ch) { + channels.setValueAtIndex(rt, ch, float32ChannelValues_[ch]); + } + channelsArraysByCount_[count] = jsi::Value(std::move(channels)); + } +} + +const jsi::Value *AudioChannelViews::channelsArray(size_t activeChannelCount) const { + if (jsValuesReleased_ || activeChannelCount == 0 || + activeChannelCount >= channelsArraysByCount_.size()) { + return nullptr; + } + + return &channelsArraysByCount_[activeChannelCount]; +} + +const std::shared_ptr &AudioChannelViews::channelBuffer( + size_t channelIndex) const { + return channelBuffers_.at(channelIndex); +} + +} // namespace audioworklets diff --git a/packages/react-native-audio-worklets/common/cpp/audioworklets/utils/AudioChannelViews.h b/packages/react-native-audio-worklets/common/cpp/audioworklets/utils/AudioChannelViews.h new file mode 100644 index 000000000..75febb8b7 --- /dev/null +++ b/packages/react-native-audio-worklets/common/cpp/audioworklets/utils/AudioChannelViews.h @@ -0,0 +1,70 @@ +#pragma once + +#include +#include + +#include + +#include +#include +#include + +namespace audioworklets { + +using namespace facebook; + +/// @brief Owns a fixed pool of per-channel `AudioArrayBuffer`s and stable `Float32Array[]` +/// views over that memory on a worklet runtime. Native code writes samples into the pool; +/// JavaScript reads them through the pre-built views (zero-copy). +class AudioChannelViews { + public: + /// @brief Allocates `channelCount` native buffers of `frameCount` frames and builds JSI views on `runtime`. + /// @param runtime Worklet runtime on which all JSI values are created. + /// @param frameCount Number of frames per channel (view length). + /// @param channelCount Number of channel slots in the pool. + AudioChannelViews( + const std::shared_ptr &runtime, + size_t frameCount, + size_t channelCount); + + DELETE_COPY_AND_MOVE(AudioChannelViews); + ~AudioChannelViews(); + + /// @brief Returns a stable `Float32Array[]` whose length equals `activeChannelCount`, + /// or `nullptr` if the views were released or `activeChannelCount` is out of range. + /// @param activeChannelCount Number of active channels for this callback. + [[nodiscard]] const jsi::Value *channelsArray(size_t activeChannelCount) const; + + /// @brief Native per-channel buffer for audio-thread reads/writes. + /// @note Audio Thread only. Do not access from JS. + /// @param channelIndex Zero-based channel index. + [[nodiscard]] const std::shared_ptr &channelBuffer( + size_t channelIndex) const; + + [[nodiscard]] size_t channelCount() const { + return channelBuffers_.size(); + } + + [[nodiscard]] size_t frameCount() const { + return frameCount_; + } + + /// @brief Destroys JSI views on the worklet runtime. Safe to call multiple times. + void releaseJsValues(); + + private: + void createFloat32ChannelViews(jsi::Runtime &rt); + void createChannelsArraysByCount(jsi::Runtime &rt); + + /// Keeps the worklet runtime alive until JSI views are released. + std::shared_ptr runtime_; + bool jsValuesReleased_{false}; + size_t frameCount_{0}; + std::vector> channelBuffers_; + std::vector arrayBufferValues_; + std::vector float32ChannelValues_; + /// @brief `channelsArraysByCount_[n]` is a `Float32Array[]` of length `n` (index 0 unused). + std::vector channelsArraysByCount_; +}; + +} // namespace audioworklets diff --git a/packages/react-native-audio-worklets/src/AudioWorkletsModule.ts b/packages/react-native-audio-worklets/src/AudioWorkletsModule.ts index ae68672ce..8742b7e53 100644 --- a/packages/react-native-audio-worklets/src/AudioWorkletsModule.ts +++ b/packages/react-native-audio-worklets/src/AudioWorkletsModule.ts @@ -3,15 +3,18 @@ // we install the worklet extensions on top of them. import 'react-native-audio-api'; import semverGte from 'semver/functions/gte'; +import type { WorkletRuntime } from 'react-native-worklets'; import NativeAudioWorkletsModule from './specs/NativeAudioWorkletsModule'; -import type { IWorkletsModule } from './types'; const MIN_WORKLETS_VERSION = '0.10.0'; +type WorkletsModule = typeof import('react-native-worklets'); + class AudioWorkletsModuleImpl { - #workletsModule: IWorkletsModule | null = null; + #workletsModule: WorkletsModule | null = null; #workletsVersion = 'unknown'; + #audioRuntime?: WorkletRuntime; constructor() { this.#verifyWorklets(); @@ -30,7 +33,7 @@ class AudioWorkletsModuleImpl { } #verifyWorklets(): void { - let workletsPackage: IWorkletsModule; + let workletsPackage: WorkletsModule; try { workletsPackage = require('react-native-worklets'); this.#workletsVersion = @@ -52,12 +55,28 @@ class AudioWorkletsModuleImpl { } #isInstalled(): boolean { - return globalThis.__createWorkletNode != null; + return ( + globalThis.__createWorkletNode != null && + globalThis.__createWorkletSourceNode != null && + globalThis.__createWorkletProcessingNode != null + ); } - get workletsModule(): IWorkletsModule { + get workletsModule(): WorkletsModule { return this.#workletsModule!; } + + getAudioRuntime(): WorkletRuntime { + if (this.#audioRuntime === undefined) { + this.#audioRuntime = this.#workletsModule!.createWorkletRuntime({ + name: 'AudioWorkletRuntime', + enableEventLoop: false, + queue: null, + }); + } + + return this.#audioRuntime; + } } const AudioWorkletsModule = new AudioWorkletsModuleImpl(); diff --git a/packages/react-native-audio-worklets/src/WorkletNode.ts b/packages/react-native-audio-worklets/src/WorkletNode.ts index c5c6733b8..77ebcbe99 100644 --- a/packages/react-native-audio-worklets/src/WorkletNode.ts +++ b/packages/react-native-audio-worklets/src/WorkletNode.ts @@ -2,9 +2,16 @@ import { AudioNode, BaseAudioContext } from 'react-native-audio-api'; import AudioWorkletsModule from './AudioWorkletsModule'; import type { WorkletNodeCallback } from './types'; +import { validateBufferLength } from './utils'; export default class WorkletNode extends AudioNode { - constructor(context: BaseAudioContext, callback: WorkletNodeCallback) { + constructor( + context: BaseAudioContext, + callback: WorkletNodeCallback, + bufferLength: number + ) { + const length = validateBufferLength(bufferLength); + const workletsModule = AudioWorkletsModule.workletsModule; const shareableWorklet = workletsModule.createSerializable(callback); @@ -17,6 +24,7 @@ export default class WorkletNode extends AudioNode { const node = globalThis.__createWorkletNode( context.context, shareableWorklet, + length, workletsModule.getUIRuntimeHolder(), workletsModule.getUISchedulerHolder() ); diff --git a/packages/react-native-audio-worklets/src/WorkletProcessingNode.ts b/packages/react-native-audio-worklets/src/WorkletProcessingNode.ts new file mode 100644 index 000000000..b5a7c606d --- /dev/null +++ b/packages/react-native-audio-worklets/src/WorkletProcessingNode.ts @@ -0,0 +1,28 @@ +import { AudioNode, BaseAudioContext } from 'react-native-audio-api'; + +import AudioWorkletsModule from './AudioWorkletsModule'; +import type { WorkletProcessingNodeCallback } from './types'; + +export default class WorkletProcessingNode extends AudioNode { + constructor( + context: BaseAudioContext, + callback: WorkletProcessingNodeCallback + ) { + if (globalThis.__createWorkletProcessingNode == null) { + throw new Error( + 'react-native-audio-worklets: worklet extensions are not installed.' + ); + } + + const workletsModule = AudioWorkletsModule.workletsModule; + const shareableWorklet = workletsModule.createSerializable(callback); + + const node = globalThis.__createWorkletProcessingNode( + context.context, + shareableWorklet, + AudioWorkletsModule.getAudioRuntime() + ); + + super(context, node); + } +} diff --git a/packages/react-native-audio-worklets/src/WorkletSourceNode.ts b/packages/react-native-audio-worklets/src/WorkletSourceNode.ts new file mode 100644 index 000000000..debd04b09 --- /dev/null +++ b/packages/react-native-audio-worklets/src/WorkletSourceNode.ts @@ -0,0 +1,28 @@ +import { + AudioScheduledSourceNode, + BaseAudioContext, +} from 'react-native-audio-api'; + +import AudioWorkletsModule from './AudioWorkletsModule'; +import type { WorkletSourceNodeCallback } from './types'; + +export default class WorkletSourceNode extends AudioScheduledSourceNode { + constructor(context: BaseAudioContext, callback: WorkletSourceNodeCallback) { + if (globalThis.__createWorkletSourceNode == null) { + throw new Error( + 'react-native-audio-worklets: worklet extensions are not installed.' + ); + } + + const workletsModule = AudioWorkletsModule.workletsModule; + const shareableWorklet = workletsModule.createSerializable(callback); + + const node = globalThis.__createWorkletSourceNode( + context.context, + shareableWorklet, + AudioWorkletsModule.getAudioRuntime() + ); + + super(context, node); + } +} diff --git a/packages/react-native-audio-worklets/src/globals.d.ts b/packages/react-native-audio-worklets/src/globals.d.ts index 8155379f0..517f89eed 100644 --- a/packages/react-native-audio-worklets/src/globals.d.ts +++ b/packages/react-native-audio-worklets/src/globals.d.ts @@ -5,10 +5,29 @@ declare global { | (( audioContext: unknown, shareableWorklet: unknown, + bufferLength: number, uiRuntimeHolder: unknown, uiSchedulerHolder: unknown ) => any) | undefined; + + // eslint-disable-next-line no-var + var __createWorkletSourceNode: + | (( + audioContext: unknown, + shareableWorklet: unknown, + audioRuntime: unknown + ) => any) + | undefined; + + // eslint-disable-next-line no-var + var __createWorkletProcessingNode: + | (( + audioContext: unknown, + shareableWorklet: unknown, + audioRuntime: unknown + ) => any) + | undefined; } /* eslint-enable @typescript-eslint/no-explicit-any */ diff --git a/packages/react-native-audio-worklets/src/index.ts b/packages/react-native-audio-worklets/src/index.ts index ea6a81790..0183045cf 100644 --- a/packages/react-native-audio-worklets/src/index.ts +++ b/packages/react-native-audio-worklets/src/index.ts @@ -1,2 +1,8 @@ export { default as WorkletNode } from './WorkletNode'; -export type { WorkletNodeCallback } from './types'; +export { default as WorkletSourceNode } from './WorkletSourceNode'; +export { default as WorkletProcessingNode } from './WorkletProcessingNode'; +export type { + WorkletNodeCallback, + WorkletProcessingNodeCallback, + WorkletSourceNodeCallback, +} from './types'; diff --git a/packages/react-native-audio-worklets/src/types.ts b/packages/react-native-audio-worklets/src/types.ts index d896582c2..038495d53 100644 --- a/packages/react-native-audio-worklets/src/types.ts +++ b/packages/react-native-audio-worklets/src/types.ts @@ -1,25 +1,64 @@ -export interface IWorkletsModule { - /** Creates a serializable value. */ - createSerializable: (value: T) => T; - /** Returns the holder object wrapping the shared UI worklet runtime. */ - getUIRuntimeHolder: () => object; - /** Returns the holder object wrapping the shared UI scheduler. */ - getUISchedulerHolder: () => object; -} - /** - * Invoked on the UI worklet runtime at most ~120 times per second with the - * latest render-quantum snapshot (when the prior callback has finished). - * - * @param audioBuffers - One `ArrayBuffer` per channel of **32-bit float PCM** - * (not interleaved). Wrap with `new Float32Array(buffer)` for zero-copy - * access. - * @param numberOfChannels - Active channel count for this quantum (same as - * `audioBuffers.length`). Use when iterating channels in a loop. + * Invoked on the UI worklet runtime once `bufferLength` frames have been + * accumulated (when the prior callback has finished). * - * The function must include the `'worklet'` directive. + * @remarks + * Do not store `audioData` (or individual channel views) in Reanimated shared + * values or other state read asynchronously later. The backing memory reused + * for the next snapshot and may be written from the audio thread after your + * callback returns. Derive scalars inside the callback (e.g. RMS, peak) or + * copy samples (`Float32Array.from(channel)`) if you need to keep waveform + * data for later UI reads. + * @param audioData - Stable per-channel `Float32Array` views (zero-copy over a + * reused native buffer pool). Each view spans the full `bufferLength`. The + * same object identities are passed on every callback; only the underlying + * sample memory is refilled. + * @param numberOfChannels - Active channel count (`audioData.length`). */ export type WorkletNodeCallback = ( - audioBuffers: Array, + audioData: Array, numberOfChannels: number ) => void; + +/** + * Invoked synchronously on the audio worklet runtime each render quantum. + * + * @param audioData - Stable per-channel `Float32Array` views over the output + * pool. Write samples into indices `0 .. framesToProcess - 1`. Each view + * spans the full render quantum size; only the first `framesToProcess` + * samples are read by native code. + * @param outputChannelCount - Active output channel count (`audioData.length`). + * @param framesToProcess - Number of frames to generate this quantum (may be + * less than the render quantum when playback starts or stops mid-buffer). + * @param startOffset - Silent prefix length already zeroed in the output + * buffer; use when computing absolute sample time, not as a write offset into + * `audioData`. + */ +export type WorkletSourceNodeCallback = ( + audioData: Array, + outputChannelCount: number, + framesToProcess: number, + currentTime: number, + startOffset: number +) => void; + +/** + * Invoked synchronously on the audio worklet runtime each render quantum. + * + * @param inputData - Stable per-channel `Float32Array` views over the input + * pool. + * @param outputData - Stable per-channel `Float32Array` views over the output + * pool. Write processed samples here. Each view length equals the render + * quantum size. + * @param inputChannelCount - Active input channel count (`inputData.length`). + * @param outputChannelCount - Active output channel count + * (`outputData.length`). + */ +export type WorkletProcessingNodeCallback = ( + inputData: Array, + outputData: Array, + inputChannelCount: number, + outputChannelCount: number, + framesToProcess: number, + currentTime: number +) => void; diff --git a/packages/react-native-audio-worklets/src/utils.ts b/packages/react-native-audio-worklets/src/utils.ts new file mode 100644 index 000000000..72c78c40e --- /dev/null +++ b/packages/react-native-audio-worklets/src/utils.ts @@ -0,0 +1,29 @@ +import { NotSupportedError } from 'react-native-audio-api'; + +export function validateBufferLength(bufferLength: unknown): number { + if (typeof bufferLength !== 'number') { + throw new NotSupportedError( + `bufferLength must be a number, got ${typeof bufferLength}` + ); + } + + if (!Number.isFinite(bufferLength)) { + throw new NotSupportedError( + `bufferLength must be a finite number, got ${bufferLength}` + ); + } + + if (!Number.isInteger(bufferLength)) { + throw new NotSupportedError( + `bufferLength must be an integer, got ${bufferLength}` + ); + } + + if (bufferLength < 1) { + throw new NotSupportedError( + `bufferLength must be greater than or equal to 1, got ${bufferLength}` + ); + } + + return bufferLength; +}