feat: add Fishjam background blur integration

[chmjkb]

Description

Adds react-native-executorch-webrtc package for real-time background blur in Fishjam WebRTC video calls using on-device ExecuTorch segmentation models.

Key features:

• useBackgroundBlur hook providing blurMiddleware for Fishjam's useCamera
• blur compositing (OpenGL ES on Android, Core Image on iOS)
• Morphological mask cleaning + EMA temporal smoothing (C++/OpenCV)

Architecture:

• Reuses BaseSemanticSegmentation from react-native-executorch for inference
• Registers custom VideoFrameProcessor with Fishjam's WebRTC pipeline
• All heavy processing in native (C++/Objective-C++) for performance

Introduces a breaking change?

• Yes
• No

Type of change

• Bug fix (change which fixes an issue)
• New feature (change which adds functionality)
• Documentation update (improves or adds clarity to existing documentation)
• Other (chores, tests, code style improvements etc.)

Tested on

• iOS
• Android

Testing instructions

You'll need to setup your fishjam account, and verify this example works properly:

import { StatusBar } from 'expo-status-bar';
import { StyleSheet, Text, View, TouchableOpacity } from 'react-native';
import {
  FishjamProvider,
  useConnection,
  useCamera,
  useInitializeDevices,
  useSandbox,
  RTCView,
} from '@fishjam-cloud/react-native-client';
import { useBackgroundBlur } from 'react-native-executorch-webrtc';
import { ResourceFetcher, SELFIE_SEGMENTATION, initExecutorch } from 'react-native-executorch';
import { ExpoResourceFetcher } from 'expo-resource-fetcher';

initExecutorch({ resourceFetcher: ExpoResourceFetcher });

const FISHJAM_ID = 'your-id';

function CameraScreen() {
  const { initializeDevices } = useInitializeDevices();
  const { cameraStream, cameraDevices, currentCamera, selectCamera, setCameraTrackMiddleware } = useCamera();
  const { joinRoom, leaveRoom, peerStatus } = useConnection();
  const { getSandboxPeerToken } = useSandbox();
  const [isJoining, setIsJoining] = useState(false);
  const [modelPath, setModelPath] = useState<string | null>(null);
  const [downloadProgress, setDownloadProgress] = useState(0);
  const [blurEnabled, setBlurEnabled] = useState(false);

  const { blurMiddleware } = useBackgroundBlur({
    modelUri: modelPath || '',
    blurRadius: 12,
  });

  // Download the selfie segmentation model
  useEffect(() => {
    const downloadModel = async () => {
      try {
        const paths = await ResourceFetcher.fetch(
          (progress) => setDownloadProgress(progress),
          SELFIE_SEGMENTATION.modelSource
        );
        if (paths?.[0]) {
          setModelPath(paths[0]);
        }
      } catch (error) {
        console.error('Failed to download model:', error);
      }
    };
    downloadModel();
  }, []);

  const handleFlipCamera = async () => {
    if (cameraDevices.length < 2) return;
    const currentIndex = cameraDevices.findIndex(
      (device) => device.deviceId === currentCamera?.deviceId
    );
    const nextIndex = (currentIndex + 1) % cameraDevices.length;
    await selectCamera(cameraDevices[nextIndex].deviceId);
  };

  const handleToggleBlur = async () => {
    if (!modelPath) return;
    if (blurEnabled) {
      await setCameraTrackMiddleware(null);
      setBlurEnabled(false);
    } else {
      await setCameraTrackMiddleware(blurMiddleware);
      setBlurEnabled(true);
    }
  };

  useEffect(() => {
    initializeDevices();
  }, []);

  const handleJoinRoom = async () => {
    setIsJoining(true);
    try {
      const roomName = 'demo-room';
      const peerName = `user_${Date.now()}`;
      const peerToken = await getSandboxPeerToken(roomName, peerName);
      await joinRoom({ peerToken });
    } catch (error) {
      console.error('Failed to join room:', error);
    } finally {
      setIsJoining(false);
    }
  };

  return (
    <View style={styles.container}>
      <StatusBar style="light" />

      <View style={styles.videoContainer}>
        {cameraStream ? (
          <RTCView
            mediaStream={cameraStream}
            style={styles.video}
            objectFit="cover"
            mirror={true}
          />
        ) : (
          <View style={styles.placeholder}>
            <Text style={styles.placeholderText}>Starting camera...</Text>
          </View>
        )}
      </View>

      <View style={styles.controls}>
        <Text style={styles.status}>Status: {peerStatus}</Text>
        {downloadProgress > 0 && downloadProgress < 1 && (
          <Text style={styles.status}>
            Downloading model: {(downloadProgress * 100).toFixed(0)}%
          </Text>
        )}
        <View style={styles.buttons}>
          <TouchableOpacity style={styles.flipButton} onPress={handleFlipCamera}>
            <Text style={styles.buttonText}>Flip</Text>
          </TouchableOpacity>
          <TouchableOpacity
            style={[styles.blurButton, blurEnabled && styles.blurButtonActive, !modelPath && styles.buttonDisabled]}
            onPress={handleToggleBlur}
            disabled={!modelPath}
          >
            <Text style={styles.buttonText}>{blurEnabled ? 'Blur On' : 'Blur'}</Text>
          </TouchableOpacity>
          {peerStatus === 'connected' ? (
            <TouchableOpacity style={styles.leaveButton} onPress={leaveRoom}>
              <Text style={styles.buttonText}>Leave Room</Text>
            </TouchableOpacity>
          ) : (
            <TouchableOpacity
              style={[styles.button, isJoining && styles.buttonDisabled]}
              onPress={handleJoinRoom}
              disabled={isJoining}
            >
              <Text style={styles.buttonText}>
                {isJoining ? 'Joining...' : 'Join Room'}
              </Text>
            </TouchableOpacity>
          )}
        </View>
      </View>
    </View>
  );
}

export default function App() {
  return (
    <FishjamProvider fishjamId={FISHJAM_ID}>
      <CameraScreen />
    </FishjamProvider>
  );
}

const styles = StyleSheet.create({
  container: {
    flex: 1,
    backgroundColor: '#000',
  },
  videoContainer: {
    flex: 1,
  },
  video: {
    flex: 1,
  },
  placeholder: {
    flex: 1,
    alignItems: 'center',
    justifyContent: 'center',
    backgroundColor: '#1a1a1a',
  },
  placeholderText: {
    color: '#666',
    fontSize: 18,
  },
  controls: {
    padding: 20,
    paddingBottom: 40,
    alignItems: 'center',
    gap: 12,
  },
  status: {
    color: '#888',
    fontSize: 14,
  },
  buttons: {
    flexDirection: 'row',
    gap: 12,
  },
  button: {
    backgroundColor: '#007AFF',
    paddingHorizontal: 32,
    paddingVertical: 14,
    borderRadius: 12,
  },
  flipButton: {
    backgroundColor: '#333',
    paddingHorizontal: 24,
    paddingVertical: 14,
    borderRadius: 12,
  },
  leaveButton: {
    backgroundColor: '#FF3B30',
    paddingHorizontal: 32,
    paddingVertical: 14,
    borderRadius: 12,
  },
  blurButton: {
    backgroundColor: '#5856D6',
    paddingHorizontal: 24,
    paddingVertical: 14,
    borderRadius: 12,
  },
  blurButtonActive: {
    backgroundColor: '#34C759',
  },
  buttonDisabled: {
    backgroundColor: '#444',
  },
  buttonText: {
    color: '#fff',
    fontSize: 16,
    fontWeight: '600',
  },
});

Screenshots

Related issues

Checklist

• I have performed a self-review of my code
• I have commented my code, particularly in hard-to-understand areas
• I have updated the documentation accordingly
• My changes generate no new warnings

Additional notes