diff --git a/translation-demo/.env.example b/translation-demo/.env.example new file mode 100644 index 0000000..9932d85 --- /dev/null +++ b/translation-demo/.env.example @@ -0,0 +1,7 @@ +# Required: your Fishjam ID — the relay's root namespace and the connection path segment. +# MoQ tokens are scoped to paths under it (a stream `` token authorizes `/`). +VITE_FISHJAM_ID=ea94930e3ffd47c490f5144c43d57340 + +# Required: Fishjam sandbox API base URL. The app fetches a MoQ JWT from the sandbox and +# appends it to the relay connection URL as `?jwt=`. Get it at https://fishjam.io/app/sandbox. +VITE_SANDBOX_API_URL=https://fishjam.io/api/v1/connect/ea94930e3ffd47c490f5144c43d57340/room-manager diff --git a/translation-demo/.gitignore b/translation-demo/.gitignore new file mode 100644 index 0000000..09a7283 --- /dev/null +++ b/translation-demo/.gitignore @@ -0,0 +1,27 @@ +# Dependencies +node_modules + +# Build output +dist +dist-ssr +*.local + +# Env +.env + +# Yarn (Berry) +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/sdks +!.yarn/versions + +# Logs +*.log + +# Editor / OS +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store diff --git a/translation-demo/.yarnrc.yml b/translation-demo/.yarnrc.yml new file mode 100644 index 0000000..5628a4a --- /dev/null +++ b/translation-demo/.yarnrc.yml @@ -0,0 +1,15 @@ +nodeLinker: node-modules + +packageExtensions: + "@moq/hang@0.2.10": + dependencies: + "@svta/cml-utils": "1.5.0" + "@moq/loc@0.1.1": + dependencies: + zod: "^4.4.3" + "@moq/publish@0.2.15": + dependencies: + zod: "^4.4.3" + "@moq/watch@0.2.17": + dependencies: + zod: "^4.4.3" diff --git a/translation-demo/README.md b/translation-demo/README.md new file mode 100644 index 0000000..036b3eb --- /dev/null +++ b/translation-demo/README.md @@ -0,0 +1,28 @@ +# Translation Demo + +An example React app that streams over the MoQ protocol with Fishjam, with live audio translation and captions for the viewer. + +## Getting Started + +Install dependencies: + +```bash +yarn +``` + +Configure the MoQ relay (required): + +```bash +cp .env.example .env +# then set VITE_MOQ_URL and VITE_MOQ_FISHJAM_ID in .env to your MoQ relay URL +``` + +Start the development server: + +```bash +yarn dev +``` + +## Environment Variables + +- `VITE_MOQ_URL` (required) — URL of the MoQ relay to connect to. The app has no built-in default and will not connect until this is set. diff --git a/translation-demo/index.html b/translation-demo/index.html new file mode 100644 index 0000000..2867acc --- /dev/null +++ b/translation-demo/index.html @@ -0,0 +1,18 @@ + + + + + + + Translation Demo + + + + + +
+ + + diff --git a/translation-demo/package.json b/translation-demo/package.json new file mode 100644 index 0000000..6c5f1ac --- /dev/null +++ b/translation-demo/package.json @@ -0,0 +1,48 @@ +{ + "name": "@moq/translation-demo", + "private": true, + "version": "0.1.0", + "description": "MoQ streaming demo with live audio translation and captions", + "license": "(MIT OR Apache-2.0)", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc && vite build", + "preview": "vite preview" + }, + "dependencies": { + "@fishjam-cloud/react-client": "0.28.1", + "@moq/publish": "0.2.15", + "@moq/signals": "0.1.9", + "@moq/watch": "0.2.17", + "@radix-ui/react-icons": "^1.3.2", + "@radix-ui/react-label": "^2.1.0", + "@radix-ui/react-popover": "^1.1.17", + "@radix-ui/react-select": "^2.1.4", + "@radix-ui/react-slot": "^1.1.1", + "@svta/cml-utils": "1.5.0", + "class-variance-authority": "^0.7.0", + "clsx": "^2.1.1", + "lucide-react": "^0.476.0", + "qrcode.react": "^4.2.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "react-router": "^7.1.5", + "sonner": "^2.0.3", + "tailwind-merge": "^3.0.2", + "tailwindcss-animate": "^1.0.7", + "unique-names-generator": "^4.7.1" + }, + "devDependencies": { + "@types/node": "^22.12.0", + "@types/react": "19.2.17", + "@types/react-dom": "19.2.3", + "@vitejs/plugin-react-swc": "^3.7.2", + "autoprefixer": "^10.4.20", + "postcss": "^8.5.1", + "tailwindcss": "^3.4.17", + "typescript": "^5.7.3", + "vite": "^6.0.11" + }, + "packageManager": "yarn@4.6.0" +} diff --git a/translation-demo/postcss.config.cjs b/translation-demo/postcss.config.cjs new file mode 100644 index 0000000..12a703d --- /dev/null +++ b/translation-demo/postcss.config.cjs @@ -0,0 +1,6 @@ +module.exports = { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +}; diff --git a/translation-demo/public/favicon.svg b/translation-demo/public/favicon.svg new file mode 100644 index 0000000..c5971d1 --- /dev/null +++ b/translation-demo/public/favicon.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/translation-demo/public/fishjam-logo.svg b/translation-demo/public/fishjam-logo.svg new file mode 100644 index 0000000..cbae672 --- /dev/null +++ b/translation-demo/public/fishjam-logo.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/translation-demo/public/gemini-logo.svg b/translation-demo/public/gemini-logo.svg new file mode 100644 index 0000000..721d153 --- /dev/null +++ b/translation-demo/public/gemini-logo.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/translation-demo/src/components/BrandHeader.tsx b/translation-demo/src/components/BrandHeader.tsx new file mode 100644 index 0000000..db076c6 --- /dev/null +++ b/translation-demo/src/components/BrandHeader.tsx @@ -0,0 +1,33 @@ +import { cn } from "@/utils/cn"; + +type Props = { + className?: string; +}; + +// The Gemini wordmark sits a touch low in its viewBox, so a small upward nudge +// (-translate-y-2) keeps it baseline-aligned with the Fishjam mark. +export const BrandHeader = ({ className }: Props) => ( +
+
+ + Fishjam + + × + Gemini +
+

+ Live streaming with real-time AI translation — powered by Fishjam, Gemini, + and Media over QUIC. +

+
+); diff --git a/translation-demo/src/components/DeviceSelect.tsx b/translation-demo/src/components/DeviceSelect.tsx new file mode 100644 index 0000000..33e23d8 --- /dev/null +++ b/translation-demo/src/components/DeviceSelect.tsx @@ -0,0 +1,49 @@ +import type { FC } from "react"; + +import { Label } from "./ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "./ui/select"; + +type SelectableDevice = { + deviceId: string; + label: string; + kind?: string; +}; + +type Props = { + devices: SelectableDevice[]; + onSelectDevice: (deviceId: string) => void; + selectedDeviceId?: string; +}; + +export const DeviceSelect: FC = ({ + devices, + onSelectDevice, + selectedDeviceId, +}) => { + const validDevices = devices.filter((device) => device.deviceId); + + if (!validDevices.length) { + return ; + } + + return ( + + ); +}; diff --git a/translation-demo/src/components/PublisherPanel.tsx b/translation-demo/src/components/PublisherPanel.tsx new file mode 100644 index 0000000..ada2172 --- /dev/null +++ b/translation-demo/src/components/PublisherPanel.tsx @@ -0,0 +1,152 @@ +import { QRCodeSVG } from "qrcode.react"; +import { Copy, Loader2, Radio } from "lucide-react"; +import { toast } from "sonner"; + +import { BrandHeader } from "@/components/BrandHeader"; +import { DeviceSelect } from "@/components/DeviceSelect"; +import VideoPlayer from "@/components/VideoPlayer"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Label } from "@/components/ui/label"; + +import { usePublisher } from "@/hooks/usePublisher"; + +const statusLabel = { + connected: "Live", + connecting: "Connecting", + disconnected: "Offline", +} as const; + +export const PublisherPanel = () => { + const { + shareUrl, + connectionStatus, + cameraDevices, + microphoneDevices, + selectedCameraId, + selectedMicrophoneId, + previewStream, + selectCamera, + selectMicrophone, + start, + stop, + } = usePublisher(); + + const cameraPreview = previewStream && ( + + ); + + return ( +
+ + + {shareUrl ? ( + + + {statusLabel[connectionStatus]} + + + You are live + + Share the QR code or link so others can watch this stream. + + + + + {cameraPreview} + +
+ +
+ +
+ + {shareUrl} + + +
+ + + + + + + ) : ( + + + Start a stream + + Pick your camera and microphone, then go live! + + + + + {cameraPreview} + +
+ + +
+ +
+ + +
+ + + + + + + )} +
+ ); +}; diff --git a/translation-demo/src/components/StreamPlayer.tsx b/translation-demo/src/components/StreamPlayer.tsx new file mode 100644 index 0000000..c70109d --- /dev/null +++ b/translation-demo/src/components/StreamPlayer.tsx @@ -0,0 +1,117 @@ +import { Captions, Loader2 } from "lucide-react"; + +import { TranslationMenu } from "@/components/TranslationMenu"; +import { VideoSurface } from "@/components/VideoSurface"; +import { useSignalValue } from "@/hooks/useSignalValue"; +import type { SyncedStreamPlayer } from "@/hooks/useSyncedStreamPlayer"; +import { cn } from "@/utils/cn"; +import type { MoqStream, TranslationOption } from "@/utils/types"; + +const EMPTY_TRANSLATIONS: TranslationOption[] = []; + +type Props = { + stream: MoqStream; + onTranslationChange?: (translationKey: string | undefined) => void; + selectedTranslationKey?: string; + captionsEnabled?: boolean; + // The selected translation has no transcription track; the CC toggle is disabled. + captionsUnavailable?: boolean; + onCaptionsToggle?: (enabled: boolean) => void; + captionText?: string; + player?: SyncedStreamPlayer | null; + // A newly selected language is warming up; the previous audio still plays until it's ready. + audioLoading?: boolean; + style?: React.CSSProperties; +}; + +export const StreamPlayer = ({ + stream, + style, + onTranslationChange, + selectedTranslationKey, + captionsEnabled, + captionsUnavailable, + onCaptionsToggle, + captionText, + player, + audioLoading, +}: Props) => { + const translations = stream.translations ?? EMPTY_TRANSLATIONS; + const selectedTranslation = selectedTranslationKey + ? translations.find((option) => option.key === selectedTranslationKey) + : undefined; + + // True once the first video frame has decoded. Collapsed to a boolean so the per-frame + // timestamp updates only re-render when it actually flips. `shouldShowVideo` also gates on + // `stream.hasVideo`, so a stream without video never shows the surface. + const hasRenderedFrame = useSignalValue( + player?.videoTimestamp, + (timestamp) => timestamp !== undefined, + ); + + const shouldShowVideo = stream.hasVideo && hasRenderedFrame; + + return ( +
+ {player && ( + + )} + + {stream.hasVideo && !hasRenderedFrame && ( +
+ +
+ )} + + {captionText && ( +
+ {captionText} +
+ )} + + {shouldShowVideo && translations.length > 0 && ( +
+ {})} + audioLoading={audioLoading} + /> + + {selectedTranslation && onCaptionsToggle && ( + + )} +
+ )} +
+ ); +}; diff --git a/translation-demo/src/components/StreamToolbar.tsx b/translation-demo/src/components/StreamToolbar.tsx new file mode 100644 index 0000000..fb968e7 --- /dev/null +++ b/translation-demo/src/components/StreamToolbar.tsx @@ -0,0 +1,19 @@ +import { Button } from "@/components/ui/button"; + +type Props = { + onDisconnect: () => void; +}; + +export const StreamToolbar = ({ onDisconnect }: Props) => { + return ( +
+ +
+ ); +}; diff --git a/translation-demo/src/components/StreamView.tsx b/translation-demo/src/components/StreamView.tsx new file mode 100644 index 0000000..a84fd86 --- /dev/null +++ b/translation-demo/src/components/StreamView.tsx @@ -0,0 +1,182 @@ +import { Loader2 } from "lucide-react"; +import { type ReactNode, useEffect, useState } from "react"; + +import { BrandHeader } from "@/components/BrandHeader"; +import { StreamToolbar } from "@/components/StreamToolbar"; +import { Button } from "@/components/ui/button"; +import { useWakeLock } from "@/hooks/useWakeLock"; +import { cn } from "@/utils/cn"; + +import type { MoqConnectionSignal, MoqStream } from "@/utils/types"; +import { useSignalValue } from "@/hooks/useSignalValue"; +import { + ORIGINAL_AUDIO_KEY, + useSyncedStreamPlayer, +} from "@/hooks/useSyncedStreamPlayer"; +import { useTranslationTranscription } from "@/hooks/useTranslationTranscription"; +import { getTranslationTargetId } from "@/utils/translation"; +import { StreamPlayer } from "@/components/StreamPlayer"; + +type Props = { + connection: MoqConnectionSignal; + stream?: MoqStream; + onDisconnect: () => void; + // Optional overlay rendered on top of the player area only (e.g. a Play gate). + playOverlay?: ReactNode; + // True once playback has been requested: show a loading spinner while connecting and + // buffering, instead of a blank frame. + pending?: boolean; + // True when the stream is not available (it ended, or never existed for this link). + unavailable?: boolean; + // Navigate to the main page to start an own stream (shown when `unavailable`). + onStartOwn?: () => void; +}; + +export const StreamView = ({ + connection, + stream, + onDisconnect, + playOverlay, + pending, + unavailable, + onStartOwn, +}: Props) => { + useWakeLock(); + + const [selectedTranslationKey, setSelectedTranslationKey] = useState< + string | undefined + >(undefined); + const [captionsEnabled, setCaptionsEnabled] = useState(false); + + const selectedTranslation = selectedTranslationKey + ? stream?.translations?.find( + (option) => option.key === selectedTranslationKey, + ) + : undefined; + + // Shared-clock player: original video + selectable audio (original or a translation), all + // delayed and aligned by timestamp. + const player = useSyncedStreamPlayer(stream, connection); + + // Drive the audible audio track from the selection. The player keeps the previous track + // playing until the new one has warmed up (requesting a not-yet-produced language first), so + // this never freezes the picture. + useEffect(() => { + player?.selectAudio(selectedTranslation); + }, [player, selectedTranslation]); + + // The player's audible/pending track, mirrored from its signals for the UI and captions. + const audibleKey = useSignalValue( + player?.audibleKey, + (key) => key ?? ORIGINAL_AUDIO_KEY, + ); + const pendingKey = useSignalValue(player?.pendingKey); + + // Captions follow the track actually being heard (which lags the selection during a warm-up), + // so they stay in sync with the audio. The player times every track to the video play-head, so + // the shared player clock is the right reference for revealing caption segments. + const audibleTranslation = + audibleKey === ORIGINAL_AUDIO_KEY + ? undefined + : stream?.translations?.find((option) => option.key === audibleKey); + const { caption, unavailableTarget } = useTranslationTranscription( + captionsEnabled ? audibleTranslation : undefined, + player?.sync, + ); + + // The heard translation has no transcription track: turn captions off and keep the CC toggle + // disabled while it stays selected. + const audibleTarget = audibleTranslation + ? getTranslationTargetId( + audibleTranslation.path, + audibleTranslation.trackName, + ) + : undefined; + const captionsUnavailable = + !!audibleTranslation && unavailableTarget === audibleTarget; + useEffect(() => { + if (captionsUnavailable) { + setCaptionsEnabled(false); + } + }, [captionsUnavailable]); + + // Drop the selection if its option is no longer announced (or the stream left). + useEffect(() => { + if ( + selectedTranslationKey && + !stream?.translations?.some( + (option) => option.key === selectedTranslationKey, + ) + ) { + setSelectedTranslationKey(undefined); + setCaptionsEnabled(false); + } + }, [stream, selectedTranslationKey]); + + const handleTranslationChange = (translationKey: string | undefined) => { + setSelectedTranslationKey(translationKey); + + // Captions follow the chosen language; turning the translation off turns them off too. + if (!translationKey) { + setCaptionsEnabled(false); + } + }; + + return ( +
+
+ +
+ +
+ {playOverlay} + + {/* Fixed-size player frame. The stream player fills the exact same box as the + pre-connect placeholder, so the player size doesn't change on Play. */} +
+
+ {stream ? ( + + ) : unavailable ? ( +
+
+

+ This stream isn't available +

+

+ It may have ended or hasn't started yet. +

+
+ +
+ ) : pending ? ( +
+ +
+ ) : ( +
+ )} +
+
+
+ + +
+ ); +}; diff --git a/translation-demo/src/components/TranslationMenu.tsx b/translation-demo/src/components/TranslationMenu.tsx new file mode 100644 index 0000000..d831711 --- /dev/null +++ b/translation-demo/src/components/TranslationMenu.tsx @@ -0,0 +1,211 @@ +import { Check, Languages, Loader2, Plus, Search } from "lucide-react"; +import { useMemo, useState } from "react"; + +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/components/ui/popover"; +import { cn } from "@/utils/cn"; +import { formatProviderLabel } from "@/utils/translation"; +import type { TranslationOption } from "@/utils/types"; + +type TranslationMenuItemProps = { + option: TranslationOption; + selected: boolean; + onSelect: () => void; +}; + +const TranslationMenuItem = ({ + option, + selected, + onSelect, +}: TranslationMenuItemProps) => { + const Icon = option.status === "active" ? Languages : Plus; + + return ( + + ); +}; + +type Props = { + translations: TranslationOption[]; + selectedKey?: string; + onSelect: (translationKey: string | undefined) => void; + // A newly selected language is warming up; the previous audio still plays until it's ready. + audioLoading?: boolean; +}; + +// Language picker for the stream: choose the original audio or one of the available/requestable +// translations. Built on the Radix Popover, which handles portalling, collision-aware +// positioning, outside-click/Escape dismissal, and focus management for us. +export const TranslationMenu = ({ + translations, + selectedKey, + onSelect, + audioLoading, +}: Props) => { + const [open, setOpen] = useState(false); + const [languageQuery, setLanguageQuery] = useState(""); + + const selectedTranslation = selectedKey + ? translations.find((option) => option.key === selectedKey) + : undefined; + const selectedTranslationPending = + selectedTranslation?.status === "requestable"; + + const activeTranslations = useMemo( + () => translations.filter((option) => option.status === "active"), + [translations], + ); + const requestableTranslations = useMemo( + () => translations.filter((option) => option.status === "requestable"), + [translations], + ); + const filteredRequestableTranslations = useMemo(() => { + const needle = languageQuery.trim().toLowerCase(); + + if (!needle) { + return requestableTranslations; + } + + return requestableTranslations.filter( + (option) => + option.label.toLowerCase().includes(needle) || + option.language.toLowerCase().includes(needle) || + formatProviderLabel(option.provider).toLowerCase().includes(needle), + ); + }, [languageQuery, requestableTranslations]); + + const select = (translationKey: string | undefined) => { + onSelect(translationKey); + setOpen(false); + }; + + return ( + { + setOpen(next); + // Reset the search each time the menu closes so it reopens clean. + if (!next) { + setLanguageQuery(""); + } + }} + > + + + + + + + + {activeTranslations.length > 0 && ( +
+

+ Active +

+
+ {activeTranslations.map((option) => ( + select(option.key)} + /> + ))} +
+
+ )} + + {requestableTranslations.length > 0 && ( +
+
+ + setLanguageQuery(event.target.value)} + aria-label="Search languages" + className="h-8 w-full rounded-sm border border-stone-200 bg-white pl-7 pr-2 text-sm outline-none focus:border-stone-400" + placeholder="Search languages" + /> +
+ + {selectedTranslationPending && ( +
+ + + {selectedTranslation.label} waiting for catalog + +
+ )} + +
+ {filteredRequestableTranslations.length > 0 ? ( + filteredRequestableTranslations.map((option) => ( + select(option.key)} + /> + )) + ) : ( +

+ No matching languages +

+ )} +
+
+ )} + + + ); +}; diff --git a/translation-demo/src/components/VideoPlayer.tsx b/translation-demo/src/components/VideoPlayer.tsx new file mode 100644 index 0000000..9392aa6 --- /dev/null +++ b/translation-demo/src/components/VideoPlayer.tsx @@ -0,0 +1,21 @@ +import type { FC } from "react"; +import { useEffect, useRef } from "react"; + +interface VideoPlayerProps extends React.HTMLAttributes { + stream?: MediaStream | null; +} + +const VideoPlayer: FC = ({ stream, ...props }) => { + const videoRef = useRef(null); + + useEffect(() => { + if (!videoRef.current) { + return; + } + videoRef.current.srcObject = stream ?? null; + }, [stream]); + + return