diff --git a/rs/libmoq/src/publish.rs b/rs/libmoq/src/publish.rs index b49cfb379..b5a514091 100644 --- a/rs/libmoq/src/publish.rs +++ b/rs/libmoq/src/publish.rs @@ -1,6 +1,5 @@ use std::str::FromStr; -use bytes::Buf; use moq_mux::import; use crate::{Error, Id, NonZeroSlab}; @@ -52,29 +51,20 @@ impl Publish { Ok(()) } - pub fn media_ordered(&mut self, broadcast: Id, format: &str, mut init: &[u8]) -> Result { + pub fn media_ordered(&mut self, broadcast: Id, format: &str, init: &[u8]) -> Result { let (broadcast, catalog) = self.broadcasts.get(broadcast).ok_or(Error::BroadcastNotFound)?; let format = import::FramedFormat::from_str(format).map_err(|_| Error::UnknownFormat(format.to_string()))?; - let decoder = import::Framed::new(broadcast.clone(), catalog.clone(), format, &mut init)?; + let decoder = import::Framed::new(broadcast.clone(), catalog.clone(), format, init)?; let id = self.media.insert(decoder)?; Ok(id) } - pub fn media_frame( - &mut self, - media: Id, - mut data: &[u8], - timestamp: hang::container::Timestamp, - ) -> Result<(), Error> { + pub fn media_frame(&mut self, media: Id, data: &[u8], timestamp: hang::container::Timestamp) -> Result<(), Error> { let media = self.media.get_mut(media).ok_or(Error::MediaNotFound)?; - media.decode_frame(&mut data, Some(timestamp))?; - - if data.has_remaining() { - return Err(Error::BufferNotConsumed); - } + media.decode(data, Some(timestamp))?; Ok(()) } diff --git a/rs/moq-boy/src/main.rs b/rs/moq-boy/src/main.rs index 03a610c07..f7c4e534d 100644 --- a/rs/moq-boy/src/main.rs +++ b/rs/moq-boy/src/main.rs @@ -90,8 +90,8 @@ pub struct Config { /// track monitors, and async tasks. struct Session { video_encoder: video::VideoEncoder, - video_track: moq_net::TrackProducer, - audio_track: moq_net::TrackProducer, + video_track: moq_net::TrackDemand, + audio_track: moq_net::TrackDemand, /// Whether anyone is subscribed to the video/audio tracks. video_active: AtomicBool, @@ -109,7 +109,7 @@ struct Session { impl Session { /// Monitor a single track's subscription state. /// Sets the flag when a viewer subscribes, clears it when all unsubscribe. - async fn run_track_monitor(&self, name: &str, track: &moq_net::TrackProducer, flag: &AtomicBool) { + async fn run_track_monitor(&self, name: &str, track: &moq_net::TrackDemand, flag: &AtomicBool) { loop { if track.used().await.is_err() { break; @@ -246,8 +246,8 @@ async fn run(config: &Config) -> Result<()> { let audio_encoder = audio::AudioEncoder::new(broadcast.clone(), catalog.clone(), 44100)?; - let video_track = video_encoder.track.clone(); - let audio_track = audio_encoder.track().clone(); + let video_track = video_encoder.demand.clone(); + let audio_track = audio_encoder.track().demand(); let status_publisher = status::StatusPublisher::new(&mut broadcast)?; diff --git a/rs/moq-boy/src/video.rs b/rs/moq-boy/src/video.rs index 3eda1ddd2..104274aed 100644 --- a/rs/moq-boy/src/video.rs +++ b/rs/moq-boy/src/video.rs @@ -19,8 +19,8 @@ use crate::emulator::{HEIGHT, WIDTH}; /// Frames are submitted via `try_frame()` (non-blocking, drops if full). pub struct VideoEncoder { tx: tokio::sync::mpsc::Sender, - /// Clone of the video track producer, for monitoring used/unused. - pub track: moq_net::TrackProducer, + /// Watch-only handle to the video track, for monitoring used/unused. + pub demand: moq_net::TrackDemand, force_keyframe: Arc, /// Latest encode duration in microseconds. encode_duration: Arc, @@ -36,7 +36,7 @@ impl VideoEncoder { pub fn spawn(broadcast: moq_net::BroadcastProducer, catalog: moq_mux::catalog::Producer) -> Self { let (tx, rx) = tokio::sync::mpsc::channel(4); let producer = moq_video::encode::Producer::new(broadcast, catalog).expect("failed to create avc3 producer"); - let track = producer.track().expect("avc3 track is eagerly created").clone(); + let demand = producer.demand(); let force_keyframe = Arc::new(AtomicBool::new(false)); let encode_duration = Arc::new(AtomicU64::new(0)); @@ -49,7 +49,7 @@ impl VideoEncoder { Self { tx, - track, + demand, force_keyframe, encode_duration, _thread: thread, diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index bf2b68e4c..6bbed7692 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -80,7 +80,10 @@ pub struct CaptureArgs { } enum PublishDecoder { - Avc3(Box), + Avc3 { + split: moq_mux::codec::h264::Split, + import: Box, + }, Fmp4(Box), Ts(Box), Flv(Box), @@ -91,13 +94,27 @@ impl PublishDecoder { /// Decode a chunk of bytes from stdin (Avc3, Fmp4, Ts, or Flv). fn decode_buf(&mut self, buffer: &mut bytes::BytesMut) -> anyhow::Result<()> { match self { - Self::Avc3(d) => Ok(d.decode_stream(buffer, None)?), + Self::Avc3 { split, import } => { + let frames = split.decode(buffer, None)?; + import.decode(frames)?; + Ok(()) + } Self::Fmp4(d) => Ok(d.decode(buffer)?), Self::Ts(d) => Ok(d.decode(buffer)?), Self::Flv(d) => Ok(d.decode(buffer)?), Self::Hls(_) => unreachable!(), } } + + /// Flush any in-flight access unit at end of stream. The avc3 split holds the + /// final AU until the next start code, so stdin EOF must flush it. + fn finish(&mut self) -> anyhow::Result<()> { + if let Self::Avc3 { split, import } = self { + let tail = split.flush(None)?; + import.decode(tail)?; + } + Ok(()) + } } // Exactly one Source exists per process, so the size gap between the small @@ -130,9 +147,13 @@ impl Publish { let source = match format { PublishFormat::Avc3 => { - let avc3 = moq_mux::codec::h264::Import::new(broadcast.clone(), catalog.clone()) - .with_mode(moq_mux::codec::h264::Mode::Avc3)?; - Source::Stream(PublishDecoder::Avc3(Box::new(avc3))) + let track = moq_mux::import::unique_track(&mut broadcast, ".avc3")?; + let import = moq_mux::codec::h264::Import::new(track, catalog.clone()); + let split = moq_mux::codec::h264::Split::new(); + Source::Stream(PublishDecoder::Avc3 { + split, + import: Box::new(import), + }) } PublishFormat::Fmp4 => { let fmp4 = fmp4::Import::new(broadcast.clone(), catalog.clone()); @@ -179,6 +200,7 @@ impl Publish { loop { let n = tokio::io::AsyncReadExt::read_buf(&mut stdin, &mut buffer).await?; if n == 0 { + decoder.finish()?; return Ok(()); } decoder.decode_buf(&mut buffer)?; diff --git a/rs/moq-ffi/src/producer.rs b/rs/moq-ffi/src/producer.rs index 0dc04040f..839230c3d 100644 --- a/rs/moq-ffi/src/producer.rs +++ b/rs/moq-ffi/src/producer.rs @@ -1,8 +1,6 @@ use std::str::FromStr; use std::sync::Arc; -use bytes::Buf; - use crate::consumer::{MoqBroadcastConsumer, MoqGroupConsumer, MoqTrackConsumer}; use crate::error::MoqError; use crate::ffi::Task; @@ -16,7 +14,7 @@ pub(crate) struct BroadcastProducer { struct MediaProducer { decoder: moq_mux::import::Framed, - track: moq_net::TrackProducer, + demand: moq_net::TrackDemand, } struct MediaStreamProducer { @@ -127,21 +125,15 @@ impl MoqBroadcastProducer { let format = moq_mux::import::FramedFormat::from_str(&format) .map_err(|_| MoqError::Codec(format!("unknown format: {format}")))?; - let mut buf = init.as_slice(); - let decoder = moq_mux::import::Framed::new(state.broadcast.clone(), state.catalog.clone(), format, &mut buf) + let decoder = moq_mux::import::Framed::new(state.broadcast.clone(), state.catalog.clone(), format, &init) .map_err(|err| MoqError::Codec(format!("init failed: {err}")))?; - if buf.has_remaining() { - return Err(MoqError::Codec("init failed: trailing bytes".into())); - } - - let track = decoder - .track() - .map_err(|err| MoqError::Codec(format!("track unavailable: {err}")))? - .clone(); + let demand = decoder + .demand() + .map_err(|err| MoqError::Codec(format!("track unavailable: {err}")))?; Ok(Arc::new(MoqMediaProducer { - inner: std::sync::Mutex::new(Some(MediaProducer { decoder, track })), + inner: std::sync::Mutex::new(Some(MediaProducer { decoder, demand })), })) } @@ -167,22 +159,17 @@ impl MoqBroadcastProducer { guard.as_ref().ok_or_else(|| MoqError::Closed)?.clone() }; - let mut buf = init.as_slice(); let decoder = - moq_mux::import::Framed::new_with_track(track_clone.clone(), state.catalog.clone(), format, &mut buf) + moq_mux::import::Framed::new_with_track(track_clone.clone(), state.catalog.clone(), format, &init) .map_err(|err| MoqError::Codec(format!("init failed: {err}")))?; - if buf.has_remaining() { - return Err(MoqError::Codec("init failed: trailing bytes".into())); - } - let mut guard = track.inner.lock().unwrap(); guard.take().ok_or_else(|| MoqError::Closed)?; Ok(Arc::new(MoqMediaProducer { inner: std::sync::Mutex::new(Some(MediaProducer { decoder, - track: track_clone, + demand: track_clone.demand(), })), })) } @@ -393,20 +380,20 @@ impl MoqMediaProducer { let _guard = crate::ffi::RUNTIME.enter(); let guard = self.inner.lock().unwrap(); let media = guard.as_ref().ok_or_else(|| MoqError::Closed)?; - Ok(media.track.name().to_string()) + Ok(media.demand.name().to_string()) } /// Wait until this media track has at least one active consumer. pub async fn used(&self) -> Result<(), MoqError> { - let track = self + let demand = self .inner .lock() .unwrap() .as_ref() .ok_or(MoqError::Closed)? - .track + .demand .clone(); - match crate::ffi::RUNTIME.spawn(async move { track.used().await }).await { + match crate::ffi::RUNTIME.spawn(async move { demand.used().await }).await { Ok(result) => result.map_err(Into::into), Err(e) if e.is_cancelled() => Err(MoqError::Cancelled), Err(e) => Err(MoqError::Task(e)), @@ -415,15 +402,15 @@ impl MoqMediaProducer { /// Wait until this media track has no active consumers. pub async fn unused(&self) -> Result<(), MoqError> { - let track = self + let demand = self .inner .lock() .unwrap() .as_ref() .ok_or(MoqError::Closed)? - .track + .demand .clone(); - match crate::ffi::RUNTIME.spawn(async move { track.unused().await }).await { + match crate::ffi::RUNTIME.spawn(async move { demand.unused().await }).await { Ok(result) => result.map_err(Into::into), Err(e) if e.is_cancelled() => Err(MoqError::Cancelled), Err(e) => Err(MoqError::Task(e)), @@ -439,16 +426,11 @@ impl MoqMediaProducer { let media = guard.as_mut().ok_or_else(|| MoqError::Closed)?; let timestamp = hang::container::Timestamp::from_micros(timestamp_us)?; - let mut data = payload.as_slice(); media .decoder - .decode_frame(&mut data, Some(timestamp)) + .decode(&payload, Some(timestamp)) .map_err(|err| MoqError::Codec(format!("decode failed: {err}")))?; - if data.has_remaining() { - return Err(MoqError::Codec("buffer was not fully consumed".into())); - } - Ok(()) } diff --git a/rs/moq-gst/src/sink/imp.rs b/rs/moq-gst/src/sink/imp.rs index b5c7d03ac..ed57825ca 100644 --- a/rs/moq-gst/src/sink/imp.rs +++ b/rs/moq-gst/src/sink/imp.rs @@ -448,32 +448,32 @@ fn handle_caps(runtime: &mut RuntimeState, pad_name: String, caps: gst::Caps) -> let structure = caps.structure(0).context("empty caps")?; let decoder: moq_mux::import::Framed = match structure.name().as_str() { "video/x-h264" => { - let mut bytes = Bytes::new(); - new_decoder(runtime, moq_mux::import::FramedFormat::Avc3, &mut bytes)? + let bytes = Bytes::new(); + new_decoder(runtime, moq_mux::import::FramedFormat::Avc3, &bytes)? } "video/x-h265" => { - let mut bytes = Bytes::new(); - new_decoder(runtime, moq_mux::import::FramedFormat::Hev1, &mut bytes)? + let bytes = Bytes::new(); + new_decoder(runtime, moq_mux::import::FramedFormat::Hev1, &bytes)? } "video/x-av1" => { - let mut bytes = Bytes::new(); - new_decoder(runtime, moq_mux::import::FramedFormat::Av01, &mut bytes)? + let bytes = Bytes::new(); + new_decoder(runtime, moq_mux::import::FramedFormat::Av01, &bytes)? } "video/x-vp8" => { - let mut bytes = Bytes::new(); - new_decoder(runtime, moq_mux::import::FramedFormat::Vp8, &mut bytes)? + let bytes = Bytes::new(); + new_decoder(runtime, moq_mux::import::FramedFormat::Vp8, &bytes)? } "video/x-vp9" => { - let mut bytes = Bytes::new(); - new_decoder(runtime, moq_mux::import::FramedFormat::Vp9, &mut bytes)? + let bytes = Bytes::new(); + new_decoder(runtime, moq_mux::import::FramedFormat::Vp9, &bytes)? } "audio/mpeg" => { let codec_data = structure .get::("codec_data") .context("AAC caps missing codec_data")?; let map = codec_data.map_readable().context("failed to map codec_data")?; - let mut data = Bytes::copy_from_slice(map.as_slice()); - new_decoder(runtime, moq_mux::import::FramedFormat::Aac, &mut data)? + let data = Bytes::copy_from_slice(map.as_slice()); + new_decoder(runtime, moq_mux::import::FramedFormat::Aac, &data)? } "audio/x-opus" => { let channels: i32 = structure.get("channels").unwrap_or(2); @@ -486,7 +486,8 @@ fn handle_caps(runtime: &mut RuntimeState, pad_name: String, caps: gst::Caps) -> sample_rate, channel_count, }; - moq_mux::codec::opus::Import::new(runtime.broadcast.clone(), runtime.catalog.clone(), config)?.into() + let track = moq_mux::import::unique_track(&mut runtime.broadcast, ".opus")?; + moq_mux::codec::opus::Import::new(track, runtime.catalog.clone(), config)?.into() } other => anyhow::bail!("unsupported caps: {}", other), }; @@ -504,18 +505,13 @@ fn handle_caps(runtime: &mut RuntimeState, pad_name: String, caps: gst::Caps) -> fn new_decoder( runtime: &mut RuntimeState, format: moq_mux::import::FramedFormat, - buf: &mut Bytes, + buf: &[u8], ) -> Result { let decoder = moq_mux::import::Framed::new(runtime.broadcast.clone(), runtime.catalog.clone(), format, buf)?; Ok(decoder) } -fn handle_buffer( - runtime: &mut RuntimeState, - pad_name: String, - mut data: Bytes, - pts: Option, -) -> Result<()> { +fn handle_buffer(runtime: &mut RuntimeState, pad_name: String, data: Bytes, pts: Option) -> Result<()> { let pad = runtime.pads.get_mut(&pad_name).context("pad not configured")?; let ts = pts.and_then(|pts| { @@ -524,5 +520,5 @@ fn handle_buffer( hang::container::Timestamp::from_micros(relative.nseconds() / 1000).ok() }); - pad.decoder.decode_frame(&mut data, ts).map_err(|e| anyhow::anyhow!(e)) + pad.decoder.decode(&data, ts).map_err(|e| anyhow::anyhow!(e)) } diff --git a/rs/moq-mux/src/catalog/mod.rs b/rs/moq-mux/src/catalog/mod.rs index 6ae373965..e0bb25aef 100644 --- a/rs/moq-mux/src/catalog/mod.rs +++ b/rs/moq-mux/src/catalog/mod.rs @@ -29,6 +29,7 @@ mod format; mod producer; mod stream; mod target; +mod tracks; pub use consumer::Consumer; pub use filter::{Filter, FilterAudio, FilterVideo}; @@ -36,3 +37,4 @@ pub use format::*; pub use producer::{Guard, Producer}; pub use stream::Stream; pub use target::{Target, TargetAudio, TargetVideo}; +pub use tracks::{AudioTrack, VideoTrack}; diff --git a/rs/moq-mux/src/catalog/producer.rs b/rs/moq-mux/src/catalog/producer.rs index 5f14f36d4..1a7c87b76 100644 --- a/rs/moq-mux/src/catalog/producer.rs +++ b/rs/moq-mux/src/catalog/producer.rs @@ -25,6 +25,11 @@ pub struct Producer { msf_track: moq_net::TrackProducer, current: Arc>>, + + /// Shared wall clock for the broadcast's tracks. Every importer on this catalog + /// gets a clone (a `Copy` of the same epoch), so timestamps they synthesize when + /// a caller has none land on one timeline and audio/video stay in sync. + clock: crate::Clock, } // Manual Clone so a producer is cheaply clonable regardless of whether `E` is. @@ -34,6 +39,7 @@ impl Clone for Producer { hang: self.hang.clone(), msf_track: self.msf_track.clone(), current: self.current.clone(), + clock: self.clock, } } } @@ -62,9 +68,22 @@ impl Producer { hang, msf_track, current: Arc::new(Mutex::new(catalog)), + clock: crate::Clock::new(), }) } + /// Resolve a timestamp, synthesizing one from the broadcast's shared + /// [`Clock`](crate::Clock) when the caller has none. + /// + /// Sharing the clock across the catalog's tracks keeps concurrently-produced + /// audio and video on a single timeline. + pub fn timestamp(&self, hint: Option) -> crate::Result { + match hint { + Some(pts) => Ok(pts), + None => Ok(moq_net::Timestamp::from_micros(self.clock.micros())?), + } + } + /// Get mutable access to the catalog, publishing it after any changes. pub fn lock(&mut self) -> Guard<'_, E> { Guard { @@ -80,6 +99,20 @@ impl Producer { self.current.lock().unwrap().clone() } + /// A handle for one importer to publish a video rendition, retired on drop. + /// + /// See [`VideoTrack`](super::VideoTrack). + pub fn video_track(&self, name: impl Into) -> super::VideoTrack { + super::VideoTrack::new(self.clone(), name) + } + + /// A handle for one importer to publish an audio rendition, retired on drop. + /// + /// See [`AudioTrack`](super::AudioTrack). + pub fn audio_track(&self, name: impl Into) -> super::AudioTrack { + super::AudioTrack::new(self.clone(), name) + } + /// Create a consumer for this catalog, receiving updates as they're published. pub fn consume(&self) -> Result, moq_net::Error> { Ok(Consumer::new(self.hang.consume())) diff --git a/rs/moq-mux/src/catalog/tracks.rs b/rs/moq-mux/src/catalog/tracks.rs new file mode 100644 index 000000000..5963a0a8b --- /dev/null +++ b/rs/moq-mux/src/catalog/tracks.rs @@ -0,0 +1,118 @@ +use super::Producer; +use super::hang::CatalogExt; + +/// A single video track's catalog rendition, retired on drop. +/// +/// Made via [`Producer::video_track`]. An importer holds one and publishes its +/// rendition through it ([`set`](Self::set), refined in place with +/// [`update`](Self::update)). When the importer drops, the rendition is removed +/// from the shared catalog, so the broadcast catalog stays out of the importer's +/// type while still being published into. +pub struct VideoTrack { + catalog: Producer, + name: String, + /// Whether a config has been published yet, so a lazily-configured importer + /// (e.g. H.264 before its SPS) can hold the handle without a catalog entry, and + /// drop without a spurious removal. + present: bool, +} + +impl VideoTrack { + pub(super) fn new(catalog: Producer, name: impl Into) -> Self { + Self { + catalog, + name: name.into(), + present: false, + } + } + + /// The track name this rendition is keyed by. + pub fn name(&self) -> &str { + &self.name + } + + /// Resolve a timestamp on the broadcast's shared clock (see [`Producer::timestamp`]). + pub fn timestamp(&self, hint: Option) -> crate::Result { + self.catalog.timestamp(hint) + } + + /// Insert or replace the rendition, publishing the catalog. + pub fn set(&mut self, config: hang::catalog::VideoConfig) { + self.catalog.lock().video.renditions.insert(self.name.clone(), config); + self.present = true; + } + + /// Refine the rendition in place (e.g. observed jitter), publishing if present. + pub fn update(&mut self, f: impl FnOnce(&mut hang::catalog::VideoConfig)) { + if !self.present { + return; + } + let mut guard = self.catalog.lock(); + if let Some(config) = guard.video.renditions.get_mut(&self.name) { + f(config); + } + } +} + +impl Drop for VideoTrack { + fn drop(&mut self) { + if self.present { + self.catalog.lock().video.renditions.remove(&self.name); + } + } +} + +/// A single audio track's catalog rendition, retired on drop. +/// +/// The audio counterpart of [`VideoTrack`]; made via [`Producer::audio_track`]. +pub struct AudioTrack { + catalog: Producer, + name: String, + present: bool, +} + +impl AudioTrack { + pub(super) fn new(catalog: Producer, name: impl Into) -> Self { + Self { + catalog, + name: name.into(), + present: false, + } + } + + /// The track name this rendition is keyed by. + pub fn name(&self) -> &str { + &self.name + } + + /// Resolve a timestamp on the broadcast's shared clock (see [`Producer::timestamp`]). + pub fn timestamp(&self, hint: Option) -> crate::Result { + self.catalog.timestamp(hint) + } + + /// Insert or replace the rendition, publishing the catalog. + pub fn set(&mut self, config: hang::catalog::AudioConfig) { + self.catalog.lock().audio.renditions.insert(self.name.clone(), config); + self.present = true; + } + + /// Refine the rendition in place (e.g. a synthesized description or jitter), + /// publishing if present. + pub fn update(&mut self, f: impl FnOnce(&mut hang::catalog::AudioConfig)) { + if !self.present { + return; + } + let mut guard = self.catalog.lock(); + if let Some(config) = guard.audio.renditions.get_mut(&self.name) { + f(config); + } + } +} + +impl Drop for AudioTrack { + fn drop(&mut self) { + if self.present { + self.catalog.lock().audio.renditions.remove(&self.name); + } + } +} diff --git a/rs/moq-mux/src/codec/aac/import.rs b/rs/moq-mux/src/codec/aac/import.rs index a956482bb..f9e8ec002 100644 --- a/rs/moq-mux/src/codec/aac/import.rs +++ b/rs/moq-mux/src/codec/aac/import.rs @@ -1,48 +1,27 @@ -use bytes::{Buf, BytesMut}; - use super::Config; use crate::catalog::hang::CatalogExt; +use crate::container::Frame; /// AAC importer. /// /// Initialized from an AudioSpecificConfig blob (variable-length, typically extracted from -/// an MP4 ESDS atom). Each input buffer passed to [`decode`](Self::decode) is published as -/// one hang frame in its own group, so the relay can forward each frame without waiting for -/// a group boundary. The codec's packet loss concealment handles drops. +/// an MP4 ESDS atom), so its catalog is known up front. Each packet passed to +/// [`decode`](Self::decode) is published as one hang frame in its own group, so the relay can +/// forward each frame without waiting for a group boundary. The codec's packet loss +/// concealment handles drops. Build it with [`new`](Self::new), passing the track producer +/// and the [`catalog::Producer`](crate::catalog::Producer) it publishes its rendition into. pub struct Import { - catalog: crate::catalog::Producer, track: crate::container::Producer, - zero: Option, + rendition: crate::catalog::AudioTrack, } impl Import { + /// Publish on an existing track producer, registering the rendition in `catalog`. pub fn new( - broadcast: moq_net::BroadcastProducer, - catalog: crate::catalog::Producer, - config: Config, - ) -> crate::Result { - Self::new_with_source( - crate::track_provider::TrackProvider::unique(broadcast, ".aac"), - catalog, - config, - ) - } - - pub fn new_with_track( track: moq_net::TrackProducer, catalog: crate::catalog::Producer, config: Config, ) -> crate::Result { - Self::new_with_source(crate::track_provider::TrackProvider::fixed(track), catalog, config) - } - - fn new_with_source( - mut tracks: crate::track_provider::TrackProvider, - mut catalog: crate::catalog::Producer, - config: Config, - ) -> crate::Result { - let track = tracks.create()?; - let mut audio_config = hang::catalog::AudioConfig::new( hang::catalog::AAC { profile: config.profile, @@ -53,22 +32,27 @@ impl Import { audio_config.container = hang::catalog::Container::Legacy; tracing::debug!(name = ?track.name(), config = ?audio_config, "starting track"); - catalog - .lock() - .audio - .renditions - .insert(track.name().to_string(), audio_config); + + let mut rendition = catalog.audio_track(track.name()); + rendition.set(audio_config); Ok(Self { - catalog, track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), - zero: None, + rendition, }) } - /// Returns a reference to the underlying track producer. - pub fn track(&self) -> &moq_net::TrackProducer { - self.track.track() + /// A watch-only handle to this track's subscriber demand. + pub fn demand(&self) -> moq_net::TrackDemand { + self.track.track().demand() + } + + /// Refine the single audio rendition in place, republishing the catalog. + /// + /// The TS importer uses this to set the synthesized `description` and an + /// audio-burst `jitter` once it knows them. + pub(crate) fn update_rendition(&mut self, f: impl FnOnce(&mut hang::catalog::AudioConfig)) { + self.rendition.update(f); } /// Finish the track, flushing the current group. @@ -83,46 +67,16 @@ impl Import { Ok(()) } - pub fn decode(&mut self, buf: &mut T, pts: Option) -> crate::Result<()> { - let pts = self.pts(pts)?; - - // Collect the input into a contiguous Bytes payload. - let mut payload = BytesMut::with_capacity(buf.remaining()); - while buf.has_remaining() { - let chunk = buf.chunk(); - payload.extend_from_slice(chunk); - let len = chunk.len(); - buf.advance(len); - } - - // Each frame is its own group so the relay can forward it immediately. - // The codec's packet loss concealment handles drops. - let frame = crate::container::Frame { - timestamp: pts, - payload: payload.freeze(), + /// Publish one AAC packet as its own group, stamping `pts` or a wall clock when absent. + pub fn decode(&mut self, frame: &[u8], pts: Option) -> crate::Result<()> { + let timestamp = self.rendition.timestamp(pts)?; + self.track.write(Frame { + timestamp, + payload: bytes::Bytes::copy_from_slice(frame), keyframe: true, duration: None, - }; - - self.track.write(frame)?; + })?; self.track.finish_group()?; - Ok(()) } - - fn pts(&mut self, hint: Option) -> crate::Result { - if let Some(pts) = hint { - return Ok(pts); - } - - let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); - Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) - } -} - -impl Drop for Import { - fn drop(&mut self) { - tracing::debug!(name = ?self.track.name(), "ending track"); - self.catalog.lock().audio.renditions.remove(self.track.name()); - } } diff --git a/rs/moq-mux/src/codec/ac3.rs b/rs/moq-mux/src/codec/ac3.rs index fd4a34686..a2b792acd 100644 --- a/rs/moq-mux/src/codec/ac3.rs +++ b/rs/moq-mux/src/codec/ac3.rs @@ -25,18 +25,26 @@ const CHANNELS: [u32; 8] = [2, 1, 2, 3, 3, 4, 4, 5]; const SAMPLES_PER_FRAME: u64 = 1536; /// Parse an AC-3 sync frame header from the start of `data` (needs >= 7 bytes). -pub(crate) fn parse_header(data: &[u8]) -> anyhow::Result { - anyhow::ensure!(data.len() >= 7, "AC-3 header needs 7 bytes"); - anyhow::ensure!(data[0] == 0x0B && data[1] == 0x77, "missing AC-3 sync word"); +pub(crate) fn parse_header(data: &[u8]) -> legacy::Result { + if data.len() < 7 { + return Err(legacy::Error::Ac3HeaderTooShort); + } + if !(data[0] == 0x0B && data[1] == 0x77) { + return Err(legacy::Error::Ac3MissingSyncWord); + } let fscod = data[4] >> 6; let frmsizecod = (data[4] & 0x3F) as usize; - anyhow::ensure!(frmsizecod <= 37, "invalid AC-3 frame size code"); + if frmsizecod > 37 { + return Err(legacy::Error::Ac3InvalidFrameSizeCode); + } let bitrate_kbps = BITRATE[frmsizecod >> 1] as usize; // bsid > 8 is E-AC-3 or a low-sample-rate variant, neither parsed here. let bsid = data[5] >> 3; - anyhow::ensure!(bsid <= 8, "unsupported AC-3 bsid {bsid}"); + if bsid > 8 { + return Err(legacy::Error::Ac3UnsupportedBsid(bsid)); + } // At 44.1 kHz the frame doesn't divide evenly, so the low frmsizecod bit // selects the padded size (A/52 Table 5.18). @@ -44,7 +52,7 @@ pub(crate) fn parse_header(data: &[u8]) -> anyhow::Result { 0b00 => (48000, 4 * bitrate_kbps), 0b01 => (44100, 2 * (320 * bitrate_kbps / 147 + (frmsizecod & 1))), 0b10 => (32000, 6 * bitrate_kbps), - _ => anyhow::bail!("reserved AC-3 sample-rate code"), + _ => return Err(legacy::Error::Ac3ReservedSampleRate), }; // acmod decides which mix-level fields precede lfeon; skip them bit by bit. diff --git a/rs/moq-mux/src/codec/av1/import.rs b/rs/moq-mux/src/codec/av1/import.rs index 9076913d3..2657d8c4c 100644 --- a/rs/moq-mux/src/codec/av1/import.rs +++ b/rs/moq-mux/src/codec/av1/import.rs @@ -1,66 +1,103 @@ -use crate::container::jitter::MinFrameDuration; - -use bytes::BytesMut; -use bytes::{Buf, Bytes}; +//! AV1 importer. +//! +//! Publishes raw AV1 (OBU-framed, inline sequence headers) on a single moq +//! track and resolves the catalog rendition. The codec config comes from the +//! sequence header the splitter packages into the first keyframe (scanned out of +//! the frame here), or from an av1C record handed to +//! [`initialize`](Import::initialize). A keyframe that can't be configured is an +//! error; non-keyframes before the first config are written through to the +//! producer, which reports [`MissingKeyframe`](crate::container::MissingKeyframe) +//! for a mid-stream join. OBU byte parsing lives in [`Split`](super::Split); this type is a +//! pure frame publisher that whoever owns the split drives via [`decode`](Import::decode). + +use bytes::Bytes; use scuffle_av1::seq::SequenceHeaderObu; +use scuffle_av1::{ObuHeader, ObuType}; use super::Error; +use super::split::ObuIterator; use crate::Result; +use crate::catalog::hang::CatalogExt; +use crate::container::Frame; +use crate::container::jitter::MinFrameDuration; -/// A decoder for AV1 with inline sequence headers. -pub struct Import { - // Where new media tracks come from. - tracks: crate::track_provider::TrackProvider, - - // The catalog being produced. - catalog: crate::catalog::Producer, - - // The track being produced. - track: Option>, - - // Whether the track has been initialized. +/// A pure-publisher importer for AV1 with inline sequence headers. +/// +/// Build it with [`new`](Self::new), passing the track producer and the +/// [`catalog::Producer`](crate::catalog::Producer) it publishes into, and feed it +/// frames a [`Split`](super::Split) produced via [`decode`](Self::decode). The +/// catalog rendition fills in lazily once the config is known. +pub struct Import { + track: crate::container::Producer, + rendition: crate::catalog::VideoTrack, config: Option, - - // The current frame being built. - current: Frame, - - // Used to compute wall clock timestamps if needed. - zero: Option, - - // Tracks the minimum frame duration and updates the catalog `jitter` field. + last_seq: Option, jitter: MinFrameDuration, } -#[derive(Default)] -struct Frame { - chunks: BytesMut, - contains_keyframe: bool, - contains_frame: bool, -} - -impl Import { - pub fn new(broadcast: moq_net::BroadcastProducer, catalog: crate::catalog::Producer) -> Self { +impl Import { + /// Publish on an existing track producer, registering the rendition in `catalog`. + pub fn new(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { + let rendition = catalog.video_track(track.name()); Self { - tracks: crate::track_provider::TrackProvider::unique(broadcast, ".av01"), - catalog, - track: None, + track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), + rendition, config: None, - current: Default::default(), - zero: None, + last_seq: None, jitter: MinFrameDuration::new(), } } - pub fn new_with_track(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { - Self { - tracks: crate::track_provider::TrackProvider::fixed(track), - catalog, - track: None, - config: None, - current: Default::default(), - zero: None, - jitter: MinFrameDuration::new(), + /// Resolve the codec config from a sequence header / av1C and other metadata. + /// + /// - **av1C** (leading `0x81` marker): the buffer is parsed as an + /// AV1CodecConfigurationRecord, which resolves the config. + /// - **raw OBUs**: any sequence header resolves the config. + /// + /// Optional, since the importer also self-initializes from the first keyframe. + /// The buffer is *not* consumed: the dispatcher-owned [`Split`](super::Split) + /// consumes it (seeding the sequence header so it prefixes the first keyframe). + pub fn initialize(&mut self, buf: &[u8]) -> Result<()> { + let data = buf; + + // av1C box starts with 0x81 (marker=1, version=1) per ISO/IEC 14496-15. + if data.len() >= 16 && data[0] == 0x81 { + self.init_from_av1c(data)?; + return Ok(()); + } + + // Raw OBUs: resolve the config from any sequence header. + if let Some(seq) = find_sequence_header(data) { + self.configure_from_seq(&seq)?; } + Ok(()) + } + + fn init_from_av1c(&mut self, data: &[u8]) -> Result<()> { + let seq_profile = (data[1] >> 5) & 0x07; + let seq_level_idx = data[1] & 0x1F; + let tier = ((data[2] >> 7) & 0x01) == 1; + let high_bitdepth = ((data[2] >> 6) & 0x01) == 1; + let twelve_bit = ((data[2] >> 5) & 0x01) == 1; + + // Resolution is unknown from av1C; it's filled when the first sequence header arrives. + let mut config = hang::catalog::VideoConfig::new(hang::catalog::AV1 { + profile: seq_profile, + level: seq_level_idx, + tier: if tier { 'H' } else { 'M' }, + bitdepth: super::bitdepth(twelve_bit, high_bitdepth), + mono_chrome: ((data[2] >> 4) & 0x01) == 1, + chroma_subsampling_x: ((data[2] >> 3) & 0x01) == 1, + chroma_subsampling_y: ((data[2] >> 2) & 0x01) == 1, + chroma_sample_position: data[2] & 0x03, + color_primaries: 1, + transfer_characteristics: 1, + matrix_coefficients: 1, + full_range: false, + }); + config.container = hang::catalog::Container::Legacy; + self.apply_config(config); + Ok(()) } fn init(&mut self, seq_header: &SequenceHeaderObu) -> Result<()> { @@ -94,46 +131,18 @@ impl Import { config.coded_width = Some(seq_header.max_frame_width as u32); config.coded_height = Some(seq_header.max_frame_height as u32); config.container = hang::catalog::Container::Legacy; - - if let Some(old) = &self.config - && old == &config - { - return Ok(()); - } - - if self.track.is_some() && self.tracks.is_fixed() { - return Err(Error::FixedTrackReconfigured.into()); - } - - if let Some(track) = self.track.take() { - tracing::debug!(name = ?track.name(), "reinitializing track"); - self.catalog.lock().video.renditions.remove(track.name()); - } - - let track = self.tracks.create()?; - tracing::debug!(name = ?track.name(), ?config, "starting track"); - self.catalog - .lock() - .video - .renditions - .insert(track.name().to_string(), config.clone()); - - self.config = Some(config); - self.track = Some(crate::container::Producer::new( - track, - crate::catalog::hang::Container::Legacy, - )); - + self.apply_config(config); Ok(()) } - /// Initialize with minimal config if sequence header parsing fails + /// Minimal config when sequence-header parsing fails, so the stream can still + /// flow (the catalog just won't carry full codec info). fn init_minimal(&mut self) -> Result<()> { let mut config = hang::catalog::VideoConfig::new(hang::catalog::AV1 { - profile: 0, // Main profile - level: 0, // Unknown - tier: 'M', // Main tier - bitdepth: 8, // Assume 8-bit + profile: 0, + level: 0, + tier: 'M', + bitdepth: 8, mono_chrome: false, chroma_subsampling_x: true, // 4:2:0 chroma_subsampling_y: true, @@ -144,385 +153,116 @@ impl Import { full_range: false, }); config.container = hang::catalog::Container::Legacy; - - if self.track.is_some() && self.tracks.is_fixed() { - return Err(Error::FixedTrackReconfigured.into()); - } - - let track = self.tracks.create()?; - tracing::debug!(name = ?track.name(), "starting track with minimal config"); - self.catalog - .lock() - .video - .renditions - .insert(track.name().to_string(), config.clone()); - - self.config = Some(config); - self.track = Some(crate::container::Producer::new( - track, - crate::catalog::hang::Container::Legacy, - )); - - Ok(()) - } - - /// Initialize the decoder with sequence header and other metadata OBUs. - pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { - let data = buf.as_ref(); - - // Handle av1C format (MP4/container initialization) - // av1C box starts with 0x81 (marker=1, version=1) per ISO/IEC 14496-15 - if data.len() >= 4 && data[0] == 0x81 && data.len() >= 16 { - self.init_from_av1c(data)?; - buf.advance(data.len()); - return Ok(()); - } - - // Handle raw OBU format - let mut obus = ObuIterator::new(buf); - while let Some(obu) = obus.next().transpose()? { - self.decode_obu(obu, None)?; - } - - if let Some(obu) = obus.flush()? { - self.decode_obu(obu, None)?; - } - + self.apply_config(config); Ok(()) } - fn init_from_av1c(&mut self, data: &[u8]) -> Result<()> { - // Parse av1C box structure - let seq_profile = (data[1] >> 5) & 0x07; - let seq_level_idx = data[1] & 0x1F; - let tier = ((data[2] >> 7) & 0x01) == 1; - let high_bitdepth = ((data[2] >> 6) & 0x01) == 1; - let twelve_bit = ((data[2] >> 5) & 0x01) == 1; - - // Resolution unknown from av1C - will be updated when first sequence header arrives - let mut config = hang::catalog::VideoConfig::new(hang::catalog::AV1 { - profile: seq_profile, - level: seq_level_idx, - tier: if tier { 'H' } else { 'M' }, - bitdepth: super::bitdepth(twelve_bit, high_bitdepth), - mono_chrome: ((data[2] >> 4) & 0x01) == 1, - chroma_subsampling_x: ((data[2] >> 3) & 0x01) == 1, - chroma_subsampling_y: ((data[2] >> 2) & 0x01) == 1, - chroma_sample_position: data[2] & 0x03, - color_primaries: 1, - transfer_characteristics: 1, - matrix_coefficients: 1, - full_range: false, - }); - config.container = hang::catalog::Container::Legacy; - - if let Some(old) = &self.config - && old == &config - { - return Ok(()); - } - - if self.track.is_some() && self.tracks.is_fixed() { - return Err(Error::FixedTrackReconfigured.into()); - } - - if let Some(track) = self.track.take() { - self.catalog.lock().video.renditions.remove(track.name()); + /// Apply a resolved config, updating the catalog rendition in place. + /// + /// A changed config just re-mirrors the rendition; there are no fixed tracks + /// to reject a reconfiguration. + fn apply_config(&mut self, config: hang::catalog::VideoConfig) { + if self.config.as_ref() == Some(&config) { + return; } - - let track = self.tracks.create()?; - self.catalog - .lock() - .video - .renditions - .insert(track.name().to_string(), config.clone()); - + tracing::debug!(name = ?self.track.name(), ?config, "starting track"); + self.rendition.set(config.clone()); self.config = Some(config); - self.track = Some(crate::container::Producer::new( - track, - crate::catalog::hang::Container::Legacy, - )); - - Ok(()) - } - - /// Returns a reference to the underlying track producer. - pub fn track(&self) -> Result<&moq_net::TrackProducer> { - Ok(self.track.as_ref().ok_or(Error::NotInitialized)?.track()) - } - - /// Decode as much data as possible from the given buffer. - pub fn decode_stream>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - let obus = ObuIterator::new(buf); - - for obu in obus { - // Generate PTS for each OBU to avoid reusing same timestamp - let pts = self.pts(pts)?; - self.decode_obu(obu?, Some(pts))?; - } - - Ok(()) - } - - /// Decode all data in the buffer, assuming the buffer contains (the rest of) a frame. - pub fn decode_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - let pts = self.pts(pts)?; - let mut obus = ObuIterator::new(buf); - - while let Some(obu) = obus.next().transpose()? { - self.decode_obu(obu, Some(pts))?; - } - - if let Some(obu) = obus.flush()? { - self.decode_obu(obu, Some(pts))?; - } - - self.maybe_start_frame(Some(pts))?; - - Ok(()) } - fn decode_obu(&mut self, obu_data: Bytes, pts: Option) -> Result<()> { - if obu_data.is_empty() { - return Err(Error::ObuTooShort.into()); + /// Resolve the config from a sequence-header OBU, falling back to a minimal + /// config if it fails to parse. + fn configure_from_seq(&mut self, seq_obu: &Bytes) -> Result<()> { + if self.last_seq.as_ref() == Some(seq_obu) { + return Ok(()); } + self.last_seq = Some(seq_obu.clone()); - // Parse OBU header - this consumes header + extension + LEB128 size - let mut reader = &obu_data[..]; - let header = scuffle_av1::ObuHeader::parse(&mut reader)?; - - // Calculate payload offset by seeing how much the parser consumed - let payload_offset = obu_data.len() - reader.len(); - - // Match on the ObuType enum directly - use scuffle_av1::ObuType; - match header.obu_type { - ObuType::SequenceHeader => { - match SequenceHeaderObu::parse(header, &mut &obu_data[payload_offset..]) { - Ok(seq_header) => { - self.init(&seq_header)?; - } - Err(_) => { - // Use minimal config so stream can work (catalog won't have full info) - if self.track.is_none() { - tracing::debug!("Sequence header parsing failed, initializing with minimal config"); - self.init_minimal()?; - } - } - } - - self.current.contains_keyframe = true; - } - ObuType::TemporalDelimiter => { - self.maybe_start_frame(pts)?; - } - ObuType::FrameHeader | ObuType::Frame => { - let is_keyframe = if obu_data.len() > payload_offset { - let data = &obu_data[payload_offset..]; - if data.is_empty() { - false - } else { - let first_byte = data[0]; - - let show_existing_frame = (first_byte >> 7) & 1; - - if show_existing_frame == 1 { - self.current.contains_keyframe - } else { - let frame_type = (first_byte >> 5) & 0b11; + let mut reader = &seq_obu[..]; + let header = ObuHeader::parse(&mut reader)?; + let payload_offset = seq_obu.len() - reader.len(); - frame_type == 0 - } - } - } else { - tracing::warn!( - "Frame OBU too short: {} bytes (payload_offset={})", - obu_data.len(), - payload_offset - ); - false - }; - - if is_keyframe || self.current.contains_keyframe { - self.current.contains_keyframe = true; - } - - self.current.contains_frame = true; - } - ObuType::Metadata => { - self.maybe_start_frame(pts)?; - } - ObuType::TileGroup | ObuType::TileList => { - self.current.contains_frame = true; - } - _ => { - // Other OBU types - just include them + match SequenceHeaderObu::parse(header, &mut &seq_obu[payload_offset..]) { + Ok(seq_header) => self.init(&seq_header), + Err(_) if self.config.is_none() => { + tracing::debug!("sequence header parse failed, using minimal config"); + self.init_minimal() } + Err(_) => Ok(()), } - - tracing::trace!(?header.obu_type, "parsed OBU"); - - self.current.chunks.extend_from_slice(&obu_data); - - Ok(()) } - fn maybe_start_frame(&mut self, pts: Option) -> Result<()> { - if !self.current.contains_frame { - return Ok(()); - } - - let track = self.track.as_mut().ok_or(Error::MissingSequenceHeader)?; - let pts = pts.ok_or(Error::MissingTimestamp)?; - - let payload = std::mem::take(&mut self.current.chunks).freeze(); - - let frame = crate::container::Frame { - timestamp: pts, - payload, - keyframe: self.current.contains_keyframe, - duration: None, - }; - - track.write(frame)?; - - if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.lock().video.renditions.get_mut(track.name()) - { - c.jitter = Some(jitter); - } - - self.current.contains_keyframe = false; - self.current.contains_frame = false; + /// A watch-only handle to this track's subscriber demand. + pub fn demand(&self) -> moq_net::TrackDemand { + self.track.track().demand() + } - Ok(()) + /// True once the config is known and the catalog has been populated. + pub fn is_initialized(&self) -> bool { + self.config.is_some() } /// Finish the track, flushing the current group. pub fn finish(&mut self) -> Result<()> { - let track = self.track.as_mut().ok_or(Error::NotInitialized)?; - track.finish()?; + self.track.finish()?; Ok(()) } /// Close the current group and open the next one at `sequence`. - /// - /// Any in-flight access unit is dropped. Pre-seek OBUs would otherwise leak - /// into the post-seek group with the wrong timestamp. pub fn seek(&mut self, sequence: u64) -> Result<()> { - self.current = Frame::default(); - let track = self.track.as_mut().ok_or(Error::NotInitialized)?; - track.seek(sequence)?; + self.track.seek(sequence)?; Ok(()) } - pub fn is_initialized(&self) -> bool { - self.track.is_some() - } + /// Write split frames to the track, resolving the config from the first + /// keyframe's inline sequence header and refining the catalog jitter. + fn write_frames(&mut self, frames: impl IntoIterator) -> Result<()> { + for frame in frames { + if frame.keyframe + && let Some(seq) = find_sequence_header(&frame.payload) + { + self.configure_from_seq(&seq)?; + } - fn pts(&mut self, hint: Option) -> Result { - if let Some(pts) = hint { - return Ok(pts); - } + // A keyframe we couldn't configure (no sequence header) is undecodable. + if frame.keyframe && self.config.is_none() { + return Err(Error::MissingSequenceHeader.into()); + } - let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); - Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) - } -} + let pts = frame.timestamp; + // A pre-keyframe delta has no group to anchor it: the producer returns + // MissingKeyframe, which a caller joining mid-stream skips. + self.track.write(frame)?; -impl Drop for Import { - fn drop(&mut self) { - if let Some(track) = self.track.take() { - tracing::debug!(name = ?track.name(), "ending track"); - self.catalog.lock().video.renditions.remove(track.name()); + if let Some(jitter) = self.jitter.observe(pts) { + self.rendition.update(|c| c.jitter = Some(jitter)); + } } + Ok(()) } -} - -/// Iterator over AV1 Open Bitstream Units (OBUs) -struct ObuIterator<'a, T: Buf + AsRef<[u8]> + 'a> { - buf: &'a mut T, -} - -impl<'a, T: Buf + AsRef<[u8]> + 'a> ObuIterator<'a, T> { - pub fn new(buf: &'a mut T) -> Self { - Self { buf } - } - - pub fn flush(self) -> Result> { - let remaining = self.buf.remaining(); - if remaining == 0 { - return Ok(None); - } - let obu = self.buf.copy_to_bytes(remaining); - Ok(Some(obu)) + /// Publish split frames, resolving the config from the first keyframe's inline + /// sequence header and refining the catalog jitter. + pub fn decode(&mut self, frames: impl IntoIterator) -> Result<()> { + self.write_frames(frames) } } -impl<'a, T: Buf + AsRef<[u8]> + 'a> Iterator for ObuIterator<'a, T> { - type Item = Result; - - fn next(&mut self) -> Option { - if self.buf.remaining() == 0 { - return None; - } - - // Parse OBU header to get size - let data = self.buf.as_ref(); - if data.is_empty() { - return None; - } - - // OBU header format: - // - obu_forbidden_bit (1) - // - obu_type (4) - // - obu_extension_flag (1) - // - obu_has_size_field (1) - // - obu_reserved_1bit (1) - - let header = data[0]; - let has_extension = (header >> 2) & 1 == 1; - let has_size = (header >> 1) & 1 == 1; - - if !has_size { - let remaining = self.buf.remaining(); - let obu = self.buf.copy_to_bytes(remaining); - return Some(Ok(obu)); - } - - // LEB128 size field starts after header byte and optional extension byte - let mut size: usize = 0; - let mut offset = if has_extension { 2 } else { 1 }; - let mut shift = 0; - - loop { - if offset >= data.len() { - return None; - } - - let byte = data[offset]; - offset += 1; - - size |= ((byte & 0x7F) as usize) << shift; - shift += 7; - - if byte & 0x80 == 0 { - break; - } - - if shift >= 56 { - return Some(Err(Error::ObuSizeTooLarge.into())); - } - } - - let total_size = offset + size; +fn is_sequence_header(obu: &[u8]) -> bool { + let mut reader = obu; + ObuHeader::parse(&mut reader) + .map(|h| h.obu_type == ObuType::SequenceHeader) + .unwrap_or(false) +} - if total_size > self.buf.remaining() { - return None; +/// Find the first sequence-header OBU in a payload, if any. +fn find_sequence_header(payload: &[u8]) -> Option { + let mut buf = Bytes::copy_from_slice(payload); + let mut obus = ObuIterator::new(&mut buf); + while let Some(Ok(obu)) = obus.next() { + if is_sequence_header(&obu) { + return Some(obu); } - - let obu = self.buf.copy_to_bytes(total_size); - Some(Ok(obu)) } + obus.flush().ok().flatten().filter(|obu| is_sequence_header(obu)) } diff --git a/rs/moq-mux/src/codec/av1/mod.rs b/rs/moq-mux/src/codec/av1/mod.rs index a5f519564..8a602b81a 100644 --- a/rs/moq-mux/src/codec/av1/mod.rs +++ b/rs/moq-mux/src/codec/av1/mod.rs @@ -5,8 +5,10 @@ //! raw AV1 bitstreams (OBU-framed) to a moq broadcast. mod import; +mod split; pub use import::*; +pub use split::*; use hang::catalog::AV1; @@ -23,9 +25,6 @@ pub enum Error { #[error("not initialized")] NotInitialized, - #[error("fixed track cannot be reconfigured")] - FixedTrackReconfigured, - #[error("expected sequence header before any frames")] MissingSequenceHeader, diff --git a/rs/moq-mux/src/codec/av1/split.rs b/rs/moq-mux/src/codec/av1/split.rs new file mode 100644 index 000000000..2ee5186f3 --- /dev/null +++ b/rs/moq-mux/src/codec/av1/split.rs @@ -0,0 +1,321 @@ +//! AV1 OBU stream splitter. +//! +//! The AV1 analogue of [`crate::codec::h264::Split`]: turns a raw OBU byte +//! stream into [`crate::container::Frame`]s. It finds temporal-unit boundaries +//! and flags keyframes (a sequence header or a `KEY_FRAME`), and stamps +//! wall-clock timestamps when the caller has none (stdin). It owns no track, +//! catalog, or codec config. AV1 carries the sequence header inline ahead of +//! keyframes, so unlike H.264/H.265 there is nothing to cache or re-insert; the +//! importer parses the config out of the frames it emits. + +use bytes::{Buf, Bytes, BytesMut}; +use scuffle_av1::{ObuHeader, ObuType}; + +use super::Error; +use crate::Result; + +/// AV1 OBU stream splitter: bytes in, [`Frame`](crate::container::Frame)s out. +/// +/// Feed bytes via [`decode`](Self::decode) (unknown frame boundaries, e.g. +/// stdin); call [`flush`](Self::flush) to emit the final in-flight temporal unit. +pub struct Split { + current: Au, + zero: Option, + pending: Vec, +} + +#[derive(Default)] +struct Au { + chunks: BytesMut, + contains_keyframe: bool, + contains_frame: bool, +} + +impl Default for Split { + fn default() -> Self { + Self::new() + } +} + +impl Split { + /// A fresh splitter. + pub fn new() -> Self { + Self { + current: Au::default(), + zero: None, + pending: Vec::new(), + } + } + + /// Decode a buffer where frame boundaries are unknown, returning the temporal + /// units it can complete. The final temporal unit stays buffered until the + /// next call (or [`flush`](Self::flush)). + pub fn decode>( + &mut self, + buf: &mut T, + pts: impl Into>, + ) -> Result> { + let hint = pts.into(); + let obus = ObuIterator::new(buf); + for obu in obus { + // Resolve a timestamp per OBU so a wall-clock stream doesn't reuse one. + let pts = self.pts(hint)?; + self.decode_obu(obu?, Some(pts))?; + } + Ok(std::mem::take(&mut self.pending)) + } + + /// Emit the in-flight temporal unit, if any. Call after the last + /// [`decode`](Self::decode) when a caller handed over a complete temporal unit + /// (or at end of stream) so the final unit isn't left buffered. + pub fn flush(&mut self, pts: impl Into>) -> Result> { + let pts = self.pts(pts.into())?; + self.maybe_start_frame(Some(pts))?; + Ok(std::mem::take(&mut self.pending)) + } + + fn decode_obu(&mut self, obu_data: Bytes, pts: Option) -> Result<()> { + if obu_data.is_empty() { + return Err(Error::ObuTooShort.into()); + } + + // Parse the OBU header to learn the type; the payload offset is whatever + // the parser consumed (header + optional extension + LEB128 size). + let mut reader = &obu_data[..]; + let header = ObuHeader::parse(&mut reader)?; + let payload_offset = obu_data.len() - reader.len(); + + match header.obu_type { + // A sequence header anchors a keyframe; the importer parses the config. + ObuType::SequenceHeader => { + self.current.contains_keyframe = true; + } + ObuType::TemporalDelimiter => { + self.maybe_start_frame(pts)?; + } + ObuType::FrameHeader | ObuType::Frame => { + let is_keyframe = obu_data.get(payload_offset).is_some_and(|first_byte| { + let show_existing_frame = (first_byte >> 7) & 1; + if show_existing_frame == 1 { + self.current.contains_keyframe + } else { + let frame_type = (first_byte >> 5) & 0b11; + frame_type == 0 // KEY_FRAME + } + }); + + if is_keyframe { + self.current.contains_keyframe = true; + } + self.current.contains_frame = true; + } + ObuType::Metadata => { + self.maybe_start_frame(pts)?; + } + ObuType::TileGroup | ObuType::TileList => { + self.current.contains_frame = true; + } + _ => {} + } + + tracing::trace!(?header.obu_type, "parsed OBU"); + + self.current.chunks.extend_from_slice(&obu_data); + Ok(()) + } + + fn maybe_start_frame(&mut self, pts: Option) -> Result<()> { + if !self.current.contains_frame { + return Ok(()); + } + + let pts = pts.ok_or(Error::MissingTimestamp)?; + let keyframe = self.current.contains_keyframe; + let payload = std::mem::take(&mut self.current.chunks).freeze(); + self.current.contains_keyframe = false; + self.current.contains_frame = false; + + self.pending.push(crate::container::Frame { + timestamp: pts, + payload, + keyframe, + duration: None, + }); + Ok(()) + } + + /// Drop any in-flight temporal unit. + /// + /// Pre-reset OBUs would otherwise leak into a later frame with the wrong + /// timestamp. + pub fn reset(&mut self) { + self.current = Au::default(); + } + + fn pts(&mut self, hint: Option) -> Result { + if let Some(pts) = hint { + return Ok(pts); + } + let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); + Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) + } +} + +/// Iterator over AV1 Open Bitstream Units (OBUs). +pub(super) struct ObuIterator<'a, T: Buf + AsRef<[u8]> + 'a> { + buf: &'a mut T, +} + +impl<'a, T: Buf + AsRef<[u8]> + 'a> ObuIterator<'a, T> { + pub fn new(buf: &'a mut T) -> Self { + Self { buf } + } + + pub fn flush(self) -> Result> { + let remaining = self.buf.remaining(); + if remaining == 0 { + return Ok(None); + } + Ok(Some(self.buf.copy_to_bytes(remaining))) + } +} + +impl<'a, T: Buf + AsRef<[u8]> + 'a> Iterator for ObuIterator<'a, T> { + type Item = Result; + + fn next(&mut self) -> Option { + if self.buf.remaining() == 0 { + return None; + } + + let data = self.buf.as_ref(); + if data.is_empty() { + return None; + } + + // OBU header: forbidden(1) | type(4) | extension_flag(1) | has_size(1) | reserved(1) + let header = data[0]; + let has_extension = (header >> 2) & 1 == 1; + let has_size = (header >> 1) & 1 == 1; + + if !has_size { + let remaining = self.buf.remaining(); + return Some(Ok(self.buf.copy_to_bytes(remaining))); + } + + // LEB128 size field follows the header byte and optional extension byte. + let mut size: usize = 0; + let mut offset = if has_extension { 2 } else { 1 }; + let mut shift = 0; + + loop { + if offset >= data.len() { + return None; + } + + let byte = data[offset]; + offset += 1; + + size |= ((byte & 0x7F) as usize) << shift; + shift += 7; + + if byte & 0x80 == 0 { + break; + } + if shift >= 56 { + return Some(Err(Error::ObuSizeTooLarge.into())); + } + } + + let total_size = offset + size; + if total_size > self.buf.remaining() { + return None; + } + + Some(Ok(self.buf.copy_to_bytes(total_size))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // OBU header byte: forbidden(0) | type(4) | extension_flag(0) | has_size(1) | reserved(0). + fn obu(obu_type: u8, payload: &[u8]) -> Vec { + let mut o = vec![(obu_type << 3) | 0b010, payload.len() as u8]; + o.extend_from_slice(payload); + o + } + + fn td() -> Vec { + obu(2, &[]) // OBU_TEMPORAL_DELIMITER + } + fn seq_header() -> Vec { + obu(1, &[0xaa, 0xbb]) // OBU_SEQUENCE_HEADER (payload not parsed by the splitter) + } + fn key_frame() -> Vec { + obu(6, &[0x00, 0x11]) // OBU_FRAME, first byte: show_existing=0, frame_type=0 (KEY_FRAME) + } + fn inter_frame() -> Vec { + obu(6, &[0x20, 0x11]) // OBU_FRAME, first byte: frame_type=1 (INTER_FRAME) + } + + fn cat(parts: &[Vec]) -> BytesMut { + let mut buf = BytesMut::new(); + for p in parts { + buf.extend_from_slice(p); + } + buf + } + + fn ts() -> moq_net::Timestamp { + moq_net::Timestamp::from_micros(0).unwrap() + } + + /// Decode one complete temporal unit handed over as a single buffer: `decode` + /// buffers it, `flush` emits it. + fn decode_one(split: &mut Split, buf: &mut BytesMut, pts: moq_net::Timestamp) -> Vec { + let mut frames = split.decode(buf, Some(pts)).unwrap(); + frames.extend(split.flush(Some(pts)).unwrap()); + frames + } + + /// A temporal unit with a sequence header + KEY_FRAME emits one keyframe. + #[tokio::test(start_paused = true)] + async fn decode_keyframe() { + let mut split = Split::new(); + let frames = decode_one(&mut split, &mut cat(&[td(), seq_header(), key_frame()]), ts()); + assert_eq!(frames.len(), 1); + assert!(frames[0].keyframe); + } + + /// A frame with no sequence header and INTER frame_type is not a keyframe. + #[tokio::test(start_paused = true)] + async fn decode_delta_is_not_keyframe() { + let mut split = Split::new(); + let frames = decode_one(&mut split, &mut cat(&[td(), inter_frame()]), ts()); + assert_eq!(frames.len(), 1); + assert!(!frames[0].keyframe); + } + + /// In streaming mode the next temporal delimiter closes the previous unit, so + /// the trailing one stays buffered until `flush`. + #[tokio::test(start_paused = true)] + async fn decode_emits_on_next_boundary() { + let mut split = Split::new(); + let frames = split + .decode( + &mut cat(&[td(), seq_header(), key_frame(), td(), inter_frame()]), + Some(ts()), + ) + .unwrap(); + // Only the keyframe is complete; the inter frame waits for the next TD. + assert_eq!(frames.len(), 1); + assert!(frames[0].keyframe); + + // Flushing closes the buffered inter frame. + let tail = split.flush(Some(ts())).unwrap(); + assert_eq!(tail.len(), 1); + assert!(!tail[0].keyframe); + } +} diff --git a/rs/moq-mux/src/codec/eac3.rs b/rs/moq-mux/src/codec/eac3.rs index 920766fcd..621e0bb4c 100644 --- a/rs/moq-mux/src/codec/eac3.rs +++ b/rs/moq-mux/src/codec/eac3.rs @@ -26,41 +26,51 @@ const BLOCKS: [u64; 4] = [1, 2, 3, 6]; const CHANNELS: [u32; 8] = [2, 1, 2, 3, 3, 4, 4, 5]; /// Parse an E-AC-3 sync frame header from the start of `data` (needs >= 6 bytes). -pub(crate) fn parse_header(data: &[u8]) -> anyhow::Result { - anyhow::ensure!(data.len() >= 6, "E-AC-3 header needs 6 bytes"); - anyhow::ensure!(data[0] == 0x0B && data[1] == 0x77, "missing E-AC-3 sync word"); +pub(crate) fn parse_header(data: &[u8]) -> legacy::Result { + if data.len() < 6 { + return Err(legacy::Error::Eac3HeaderTooShort); + } + if !(data[0] == 0x0B && data[1] == 0x77) { + return Err(legacy::Error::Eac3MissingSyncWord); + } // bsid 11..=16 is E-AC-3; plain AC-3 (bsid <= 8) is routed by stream_type and // never reaches this parser. let bsid = data[5] >> 3; - anyhow::ensure!((11..=16).contains(&bsid), "not an E-AC-3 bitstream (bsid {bsid})"); + if !(11..=16).contains(&bsid) { + return Err(legacy::Error::Eac3NotEac3Bsid(bsid)); + } // A dependent substream (strmtyp 1) extends a prior program to 7.1+, and a // nonzero substreamid carries additional programs in the same PID. Either // would make this track's channel count a lie, so they are rejected rather // than mis-described. let strmtyp = data[2] >> 6; - anyhow::ensure!(strmtyp != 3, "reserved E-AC-3 stream type"); - anyhow::ensure!( - strmtyp != 1, - "E-AC-3 dependent substream (7.1+ layout) is not supported; only a single independent substream" - ); + if strmtyp == 3 { + return Err(legacy::Error::Eac3ReservedStreamType); + } + if strmtyp == 1 { + return Err(legacy::Error::Eac3DependentSubstream); + } let substreamid = (data[2] >> 3) & 0x07; - anyhow::ensure!( - substreamid == 0, - "E-AC-3 additional substream {substreamid} is not supported; only a single independent substream" - ); + if substreamid != 0 { + return Err(legacy::Error::Eac3AdditionalSubstream(substreamid)); + } let frmsiz = (((data[2] & 0x07) as usize) << 8) | data[3] as usize; let len = (frmsiz + 1) * 2; // frmsiz is a raw field; corrupt input can declare a "frame" shorter than the // header just parsed, which would surface later as a confusing sync error. - anyhow::ensure!(len >= 6, "E-AC-3 frame length {len} shorter than its header"); + if len < 6 { + return Err(legacy::Error::Eac3FrameShorterThanHeader(len)); + } let fscod = data[4] >> 6; let (sample_rate, blocks) = if fscod == 0b11 { let fscod2 = (data[4] >> 4) & 0x03; - anyhow::ensure!(fscod2 != 3, "reserved E-AC-3 sample-rate code"); + if fscod2 == 3 { + return Err(legacy::Error::Eac3ReservedSampleRate); + } // Reduced rates always run 6 blocks. (SAMPLE_RATE_REDUCED[fscod2 as usize], 6) } else { diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index bede4fdd3..96bde5b2f 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -1,157 +1,79 @@ -//! H.264 importer for both wire shapes. +//! H.264 importer. //! -//! [`Import`] accepts either length-prefixed NALU input with an -//! out-of-band [`AVCDecoderConfigurationRecord`](super::Avcc) (the "avc1" -//! shape) or Annex-B input with inline SPS/PPS (the "avc3" shape). The shape -//! is detected at [`initialize`](Import::initialize) time by looking for a -//! leading start code; callers that already know it can also force the -//! mode via [`with_mode`](Import::with_mode). +//! [`Import`] publishes already-split H.264 frames on a single moq track and +//! resolves the catalog rendition. It is a pure frame publisher: byte parsing +//! and framing live in [`Split`](super::Split), and whoever drives the import owns the split. +//! Frames arrive via [`decode`](Import::decode). +//! +//! The codec config comes from exactly one of two places: an avcC handed to +//! [`initialize`](Import::initialize) (the "avc1" shape), or the SPS the splitter +//! packages into the first keyframe (the "avc3" shape, scanned out of the frame +//! here). A keyframe that can't be configured from either is an error; +//! non-keyframes before the first config are tolerated (mid-stream joins). -use bytes::{Buf, Bytes, BytesMut}; -use tokio::io::{AsyncRead, AsyncReadExt}; +use bytes::Bytes; -use super::{Error, Sps}; +use super::{Error, NAL_TYPE_SPS, Sps}; use crate::Result; use crate::catalog::hang::CatalogExt; -use crate::codec::annexb::{NalIterator, START_CODE}; +use crate::codec::annexb::NalIterator; +use crate::container::Frame; use crate::container::jitter::MinFrameDuration; -/// The wire shape an [`Import`] is processing. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum Mode { - /// Length-prefixed NALU with out-of-band AVCDecoderConfigurationRecord - /// (catalog `H264 { inline: false }`, `description = avcC`). - Avc1, - /// Annex-B (start-code prefixed) with inline SPS/PPS - /// (catalog `H264 { inline: true }`, no description). - Avc3, -} - -/// H.264 importer. Handles both avc1 (length-prefixed) and avc3 (Annex-B) -/// input streams; the shape is detected from the first bytes the caller -/// supplies, or forced explicitly via [`with_mode`](Self::with_mode). +/// H.264 importer: a pure frame publisher that resolves the catalog rendition. +/// +/// Build it with [`new`](Self::new), passing the track producer and the +/// [`catalog::Producer`](crate::catalog::Producer) it publishes its rendition into. +/// Feed it frames a [`Split`](super::Split) produced via [`decode`](Self::decode). +/// The catalog rendition fills in lazily once the codec config is known (avcC via +/// [`initialize`](Self::initialize) for avc1, the first SPS for avc3). pub struct Import { - tracks: crate::track_provider::TrackProvider, - catalog: crate::catalog::Producer, - track: Option>, + /// True for the avc1 shape: the codec config is out-of-band (avcC), so + /// keyframes are not scanned for an inline SPS. + avc1: bool, + track: crate::container::Producer, + rendition: crate::catalog::VideoTrack, config: Option, - state: State, - zero: Option, + last_sps: Option, jitter: MinFrameDuration, } -enum State { - /// No bytes seen yet; mode pinned ahead of time or unknown. - Pending { mode_hint: Option }, - /// avc1 wire shape: length-prefixed NALU, codec config out-of-band. - Avc1 { length_size: usize }, - /// avc3 wire shape: Annex-B NALU, inline SPS/PPS. - Avc3 { - current: Avc3Frame, - sps: Option, - pps: Option, - }, -} - -#[derive(Default)] -struct Avc3Frame { - chunks: BytesMut, - contains_idr: bool, - contains_slice: bool, - contains_sps: bool, - contains_pps: bool, -} - impl Import { - pub fn new(broadcast: moq_net::BroadcastProducer, catalog: crate::catalog::Producer) -> Self { + /// Publish on an existing track producer, registering the rendition in `catalog`. + pub fn new(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { + let rendition = catalog.video_track(track.name()); Self { - tracks: crate::track_provider::TrackProvider::unique(broadcast, ".avc3"), - catalog, - track: None, + avc1: false, + track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), + rendition, config: None, - state: State::Pending { mode_hint: None }, - zero: None, + last_sps: None, jitter: MinFrameDuration::new(), } } - pub fn new_with_track(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { - Self { - tracks: crate::track_provider::TrackProvider::fixed(track), - catalog, - track: None, - config: None, - state: State::Pending { mode_hint: None }, - zero: None, - jitter: MinFrameDuration::new(), - } - } - - /// Pin the wire shape ahead of time; skips the leading-bytes auto-detect - /// inside [`initialize`](Self::initialize). Eagerly creates the broadcast - /// track for avc3 sources so the caller can observe subscriber state - /// (`used()` / `unused()`) before any frames arrive. - pub fn with_mode(mut self, mode: Mode) -> Result { - match mode { - Mode::Avc1 => { - self.tracks.set_suffix(".avc1"); - self.state = State::Pending { - mode_hint: Some(Mode::Avc1), - }; - } - Mode::Avc3 => { - self.tracks.set_suffix(".avc3"); - let track = self.tracks.create()?; - self.track = Some( - crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy) - .with_lenient_start(), - ); - self.state = State::Avc3 { - current: Avc3Frame::default(), - sps: None, - pps: None, - }; - } - } - Ok(self) - } - - /// Returns a reference to the underlying track producer, e.g. for - /// monitoring subscriber state via `used()` / `unused()`. Available only - /// after the track has been created. i.e. after [`with_mode`](Self::with_mode) - /// for avc3 or after [`initialize`](Self::initialize) for avc1. - pub fn track(&self) -> Option<&moq_net::TrackProducer> { - self.track.as_ref().map(|t| t.track()) - } - - /// Initialize from the codec's leading bytes. + /// Resolve the codec config from the codec's leading bytes. /// - /// - **avc1** (no leading start code): the buffer is parsed as an - /// `AVCDecoderConfigurationRecord` and stored as the catalog `description`. - /// - **avc3** (leading `0x00 0x00 0x01` or `0x00 0x00 0x00 0x01`): the buffer - /// is parsed as Annex-B NALs to seed the cached SPS/PPS. + /// - **avc1** (no leading start code): parsed as an `AVCDecoderConfigurationRecord`, + /// which resolves the config and is stored as the catalog `description`. Required + /// for avc1. + /// - **avc3** (leading start code): parsed as Annex-B; any SPS resolves the config. + /// Optional, since avc3 also self-initializes from the first keyframe. /// - /// The buffer is fully consumed. - pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { - let mode = match &self.state { - State::Pending { mode_hint } => mode_hint.unwrap_or_else(|| detect_mode(buf.as_ref())), - State::Avc1 { .. } => Mode::Avc1, - State::Avc3 { .. } => Mode::Avc3, - }; - - match mode { - Mode::Avc1 => self.initialize_avc1(buf), - Mode::Avc3 => self.initialize_avc3(buf), + /// Takes a read-only slice: the dispatcher-owned [`Split`](super::Split) is what + /// consumes the stream (and reads the same avcC for the NALU length size). The + /// shape is detected from the leading bytes. + pub fn initialize(&mut self, buf: &[u8]) -> Result<()> { + if detect_avc1(buf) { + self.initialize_avc1(buf) + } else { + self.initialize_avc3(buf) } } - /// Initialize the avc1 path from an `AVCDecoderConfigurationRecord` buffer. - fn initialize_avc1>(&mut self, buf: &mut T) -> Result<()> { - let avcc_bytes = buf.as_ref(); + fn initialize_avc1(&mut self, avcc_bytes: &[u8]) -> Result<()> { + self.avc1 = true; let avcc = super::Avcc::parse(avcc_bytes)?; - self.state = State::Avc1 { - length_size: avcc.length_size, - }; let mut config = hang::catalog::VideoConfig::new(hang::catalog::H264 { profile: avcc.profile, @@ -164,215 +86,59 @@ impl Import { config.description = Some(Bytes::copy_from_slice(avcc_bytes)); config.container = hang::catalog::Container::Legacy; - self.tracks.set_suffix(".avc1"); - self.swap_config(config)?; - buf.advance(buf.remaining()); - + self.apply_config(config); Ok(()) } - /// Initialize the avc3 path by parsing Annex-B NALs (SPS/PPS seed the - /// catalog rendition; the track is created eagerly on first SPS). - fn initialize_avc3>(&mut self, buf: &mut T) -> Result<()> { - // Eager-create the track + state on first switch into Avc3 mode so - // callers can observe `used()` / `unused()` before any frames arrive. - if !matches!(self.state, State::Avc3 { .. }) { - self.state = State::Avc3 { - current: Avc3Frame::default(), - sps: None, - pps: None, - }; - if self.track.is_none() { - self.tracks.set_suffix(".avc3"); - let track = self.tracks.create()?; - self.track = Some( - crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy) - .with_lenient_start(), - ); - } - } - - let mut nals = NalIterator::new(buf); + fn initialize_avc3(&mut self, data: &[u8]) -> Result<()> { + // Resolve the config from any SPS in the seed buffer. Scan a clone so the + // caller's buffer is left intact for the splitter to consume. + let mut scan = Bytes::copy_from_slice(data); + let mut nals = NalIterator::new(&mut scan); while let Some(nal) = nals.next().transpose()? { - self.decode_nal(nal, None)?; - } - if let Some(nal) = nals.flush()? { - self.decode_nal(nal, None)?; + if is_sps(&nal) { + self.configure_from_sps(&nal)?; + } } - - Ok(()) - } - - pub fn is_initialized(&self) -> bool { - self.track.is_some() - } - - /// Decode from an asynchronous reader. avc3 only — for avc1, the caller - /// already has framed buffers and uses [`decode_frame`](Self::decode_frame). - pub async fn decode_from(&mut self, reader: &mut T) -> Result<()> { - let mut buffer = BytesMut::new(); - while reader.read_buf(&mut buffer).await? > 0 { - self.decode_stream(&mut buffer, None)?; + if let Some(nal) = nals.flush()? + && is_sps(&nal) + { + self.configure_from_sps(&nal)?; } Ok(()) } - /// Decode a buffer where frame boundaries are unknown (avc3 streaming - /// input). The leading start code of the *next* frame is what signals the - /// previous frame is done. - pub fn decode_stream>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - if !matches!(self.state, State::Avc3 { .. }) { - return Err(Error::StreamNotAvc3.into()); - } - let pts = self.pts(pts)?; - let nals = NalIterator::new(buf); - for nal in nals { - self.decode_nal(nal?, Some(pts))?; - } - Ok(()) + /// A watch-only handle to this track's subscriber demand. + pub fn demand(&self) -> moq_net::TrackDemand { + self.track.track().demand() } - /// Decode a buffer assumed to hold (the rest of) a single frame. - /// - /// - avc1: the buffer is written as one length-prefixed-NALU frame. - /// - avc3: NALs are parsed; any trailing NAL without a start code is - /// flushed as the last NAL of this frame. - pub fn decode_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - match &self.state { - State::Avc1 { .. } => self.decode_avc1(buf, pts), - State::Avc3 { .. } => self.decode_avc3_frame(buf, pts), - State::Pending { .. } => Err(Error::NotInitialized.into()), - } + /// True once the codec config is known and the catalog rendition is published. + pub fn is_initialized(&self) -> bool { + self.config.is_some() } - fn decode_avc1>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - let State::Avc1 { length_size } = self.state else { - unreachable!("checked by decode_frame") - }; - let data = buf.as_ref(); - let pts = self.pts(pts)?; - let keyframe = avc1_is_keyframe(data, length_size); - let track = self.track.as_mut().ok_or(Error::NotInitialized)?; - - track.write(crate::container::Frame { - timestamp: pts, - payload: data.to_vec().into(), - keyframe, - duration: None, - })?; - - if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.lock().video.renditions.get_mut(track.name()) - { - c.jitter = Some(jitter); - } - - buf.advance(buf.remaining()); + /// Finish the track, flushing any buffered data. + pub fn finish(&mut self) -> Result<()> { + self.track.finish()?; Ok(()) } - fn decode_avc3_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - let pts = self.pts(pts)?; - let mut nals = NalIterator::new(buf); - while let Some(nal) = nals.next().transpose()? { - self.decode_nal(nal, Some(pts))?; - } - if let Some(nal) = nals.flush()? { - self.decode_nal(nal, Some(pts))?; - } - self.maybe_start_frame(Some(pts))?; + /// Close the current group and open the next one at `sequence`. + pub fn seek(&mut self, sequence: u64) -> Result<()> { + self.track.seek(sequence)?; Ok(()) } - fn decode_nal(&mut self, nal: Bytes, pts: Option) -> Result<()> { - let header = nal.first().ok_or(Error::NalTooShort)?; - let forbidden_zero_bit = (header >> 7) & 1; - if forbidden_zero_bit != 0 { - return Err(Error::ForbiddenZeroBit.into()); - } - - let nal_unit_type = header & 0b11111; - let nal_type = Avc3NalType::try_from(nal_unit_type).ok(); - - match nal_type { - Some(Avc3NalType::Sps) => { - self.maybe_start_frame(pts)?; - let sps = Sps::parse(&nal)?; - self.init_from_sps(&sps)?; - let State::Avc3 { current, sps, pps } = &mut self.state else { - unreachable!("decode_nal is avc3 only") - }; - if sps.as_ref().is_some_and(|cached| cached != &nal) { - // SPS changed mid-AU. The cached PPS is tied to the old SPS - // and may already have been appended to current.chunks - // earlier in this AU; reset the AU so the new SPS+PPS pair - // is the only parameter set we emit. - *pps = None; - current.chunks.clear(); - current.contains_pps = false; - current.contains_sps = false; - } - *sps = Some(nal.clone()); - current.contains_sps = true; - } - Some(Avc3NalType::Pps) => { - self.maybe_start_frame(pts)?; - let State::Avc3 { current, pps, .. } = &mut self.state else { - unreachable!() - }; - *pps = Some(nal.clone()); - current.contains_pps = true; - } - Some(Avc3NalType::Aud) | Some(Avc3NalType::Sei) => { - self.maybe_start_frame(pts)?; - } - Some(Avc3NalType::IdrSlice) => { - let State::Avc3 { current, sps, pps } = &mut self.state else { - unreachable!() - }; - if !current.contains_sps - && let Some(sps) = sps.as_ref() - { - current.chunks.extend_from_slice(&START_CODE); - current.chunks.extend_from_slice(sps); - current.contains_sps = true; - } - if !current.contains_pps - && let Some(pps) = pps.as_ref() - { - current.chunks.extend_from_slice(&START_CODE); - current.chunks.extend_from_slice(pps); - current.contains_pps = true; - } - current.contains_idr = true; - current.contains_slice = true; - } - Some(Avc3NalType::NonIdrSlice) - | Some(Avc3NalType::DataPartitionA) - | Some(Avc3NalType::DataPartitionB) - | Some(Avc3NalType::DataPartitionC) => { - if nal.get(1).ok_or(Error::NalTooShort)? & 0x80 != 0 { - self.maybe_start_frame(pts)?; - } - let State::Avc3 { current, .. } = &mut self.state else { - unreachable!() - }; - current.contains_slice = true; - } - _ => {} + /// Resolve the avc3 config from an inline SPS, updating it in place. + /// + /// avc3 carries SPS inline, so a resolution change just updates the config + /// (no new init segment, unlike avc1). + fn configure_from_sps(&mut self, sps_nal: &Bytes) -> Result<()> { + if self.last_sps.as_ref() == Some(sps_nal) { + return Ok(()); } - - tracing::trace!(kind = ?nal_type, "parsed NAL"); - - let State::Avc3 { current, .. } = &mut self.state else { - unreachable!() - }; - current.chunks.extend_from_slice(&START_CODE); - current.chunks.extend_from_slice(&nal); - Ok(()) - } - - fn init_from_sps(&mut self, sps: &Sps) -> Result<()> { + let sps = Sps::parse(sps_nal)?; let mut config = hang::catalog::VideoConfig::new(hang::catalog::H264 { profile: sps.profile, constraints: sps.constraints, @@ -383,274 +149,218 @@ impl Import { config.coded_height = Some(sps.coded_height); config.container = hang::catalog::Container::Legacy; - if let Some(old) = &self.config - && old == &config - { - return Ok(()); - } - - // The avc3 track was created eagerly in initialize_avc3; just publish - // (or republish) the catalog rendition with the latest config. - let track_name = self - .track - .as_ref() - .ok_or(Error::Avc3TrackNotCreated)? - .name() - .to_string(); - let mut catalog = self.catalog.lock(); - catalog.video.renditions.insert(track_name, config.clone()); - self.config = Some(config); + self.last_sps = Some(sps_nal.clone()); + self.apply_config(config); Ok(()) } - fn maybe_start_frame(&mut self, pts: Option) -> Result<()> { - let State::Avc3 { current, .. } = &mut self.state else { - return Ok(()); - }; - if !current.contains_slice { - return Ok(()); - } - let pts = pts.ok_or(Error::MissingTimestamp)?; - let payload = std::mem::take(&mut current.chunks).freeze(); - let keyframe = current.contains_idr; - current.contains_idr = false; - current.contains_slice = false; - current.contains_sps = false; - current.contains_pps = false; - - let track = self.track.as_mut().ok_or(Error::Avc3TrackNotCreated)?; - track.write(crate::container::Frame { - timestamp: pts, - payload, - keyframe, - duration: None, - })?; - - if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.lock().video.renditions.get_mut(track.name()) - { - c.jitter = Some(jitter); + /// Apply a resolved config, updating the catalog rendition in place. + /// + /// A changed config (new avcC, or a new inline SPS) just re-mirrors the + /// rendition; there are no fixed tracks to reject a reconfiguration. + fn apply_config(&mut self, config: hang::catalog::VideoConfig) { + if self.config.as_ref() == Some(&config) { + return; } - Ok(()) + tracing::debug!(?config, "starting H.264 track"); + self.rendition.set(config.clone()); + self.config = Some(config); } - /// Replace the current track + catalog rendition with `config`. Used by - /// the avc1 path on every (re)initialization. - fn swap_config(&mut self, config: hang::catalog::VideoConfig) -> Result<()> { - if let Some(old) = &self.config - && old == &config - { - return Ok(()); - } - - let mut catalog = self.catalog.lock(); - if let Some(track) = self.track.take() { - if self.tracks.is_fixed() { - self.track = Some(track); - return Err(Error::FixedTrackReconfigured.into()); + /// Write split frames to the track, resolving the avc3 config from the first + /// keyframe's inline SPS and refining the catalog jitter as it goes. + fn write_frames(&mut self, frames: impl IntoIterator) -> Result<()> { + for frame in frames { + // avc1 config arrives out-of-band via initialize(); avc3 carries SPS + // inline on keyframes. + if !self.avc1 + && frame.keyframe + && let Some(sps) = find_sps(&frame.payload) + { + self.configure_from_sps(&sps)?; } - tracing::debug!(name = ?track.name(), "reinitializing H.264 track"); - catalog.video.renditions.remove(track.name()); - } - let track = self.tracks.create()?; - tracing::debug!(name = ?track.name(), ?config, "starting H.264 track"); - catalog - .video - .renditions - .insert(track.name().to_string(), config.clone()); - self.config = Some(config); - self.track = - Some(crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy).with_lenient_start()); - Ok(()) - } + if self.config.is_none() { + // A keyframe we still can't configure is undecodable, so bail + // loudly. A non-keyframe before config is a mid-stream-join + // leftover: write it through, and the producer reports + // MissingKeyframe (which a mid-stream join skips). + if frame.keyframe { + return Err(Error::NotInitialized.into()); + } + } - /// Finish the track, flushing any buffered data. - pub fn finish(&mut self) -> Result<()> { - let track = self.track.as_mut().ok_or(Error::NotInitialized)?; - track.finish()?; - Ok(()) - } + let pts = frame.timestamp; + self.track.write(frame)?; - /// Close the current group and open the next one at `sequence`. - /// - /// Any in-flight avc3 access unit is dropped. Pre-seek NALs would otherwise - /// leak into the post-seek group with the wrong timestamp. - pub fn seek(&mut self, sequence: u64) -> Result<()> { - if let State::Avc3 { current, .. } = &mut self.state { - *current = Avc3Frame::default(); + if let Some(jitter) = self.jitter.observe(pts) { + self.rendition.update(|c| c.jitter = Some(jitter)); + } } - let track = self.track.as_mut().ok_or(Error::NotInitialized)?; - track.seek(sequence)?; Ok(()) } - fn pts(&mut self, hint: Option) -> Result { - if let Some(pts) = hint { - return Ok(pts); - } - let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); - Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) + /// Publish split frames, resolving the avc3 config from the first keyframe's + /// inline SPS and refining the catalog jitter as it goes. + pub fn decode(&mut self, frames: impl IntoIterator) -> Result<()> { + self.write_frames(frames) } } -impl Drop for Import { - fn drop(&mut self) { - if let Some(track) = self.track.take() { - tracing::debug!(name = ?track.name(), "ending H.264 track"); - self.catalog.lock().video.renditions.remove(track.name()); - } - } +/// Detect the avc1 wire shape from leading bytes: a 3- or 4-byte Annex-B start +/// code means avc3, otherwise an AVCDecoderConfigurationRecord (avc1). +fn detect_avc1(bytes: &[u8]) -> bool { + !(matches!(bytes, [0, 0, 1, ..]) || matches!(bytes, [0, 0, 0, 1, ..])) } -/// Detect the wire shape from leading bytes: a 3- or 4-byte Annex-B start -/// code means avc3, otherwise an AVCDecoderConfigurationRecord (avc1). -fn detect_mode(bytes: &[u8]) -> Mode { - let three_byte = matches!(bytes, [0, 0, 1, ..]); - let four_byte = matches!(bytes, [0, 0, 0, 1, ..]); - if three_byte || four_byte { - Mode::Avc3 - } else { - Mode::Avc1 - } +fn is_sps(nal: &[u8]) -> bool { + nal.first().is_some_and(|h| h & 0x1f == NAL_TYPE_SPS) } -/// Detect if an avc1-shaped (length-prefixed) buffer contains an IDR slice. -fn avc1_is_keyframe(data: &[u8], length_size: usize) -> bool { - let mut offset = 0; - while offset + length_size <= data.len() { - let nal_len = match length_size { - 1 => data[offset] as usize, - 2 => u16::from_be_bytes([data[offset], data[offset + 1]]) as usize, - 3 => u32::from_be_bytes([0, data[offset], data[offset + 1], data[offset + 2]]) as usize, - 4 => u32::from_be_bytes([data[offset], data[offset + 1], data[offset + 2], data[offset + 3]]) as usize, - _ => return false, - }; - offset += length_size; - if offset + nal_len > data.len() { - break; +/// Find the first SPS NAL in an Annex-B payload, if any. +fn find_sps(payload: &[u8]) -> Option { + let mut buf = Bytes::copy_from_slice(payload); + let mut nals = NalIterator::new(&mut buf); + while let Some(Ok(nal)) = nals.next() { + if is_sps(&nal) { + return Some(nal); } - if nal_len > 0 && data[offset] & 0x1f == 5 { - return true; // IDR slice - } - offset += nal_len; } - false + nals.flush().ok().flatten().filter(|nal| is_sps(nal)) } #[cfg(test)] mod tests { - use super::*; + use bytes::BytesMut; - #[test] - fn detect_mode_avc1_avcc_buffer() { - // AVCDecoderConfigurationRecord starts with configurationVersion = 1, profile, ... - // First byte is 0x01, definitely not a start code. - let avcc: &[u8] = &[ - 0x01, 0x42, 0xc0, 0x1f, 0xff, 0xe1, 0x00, 0x06, 0x67, 0x42, 0xc0, 0x1f, 0xde, 0xad, - ]; - assert_eq!(detect_mode(avcc), Mode::Avc1); - } - - #[test] - fn detect_mode_avc3_3byte_start_code() { - let nals: &[u8] = &[0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1f]; - assert_eq!(detect_mode(nals), Mode::Avc3); - } - - #[test] - fn detect_mode_avc3_4byte_start_code() { - let nals: &[u8] = &[0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1f]; - assert_eq!(detect_mode(nals), Mode::Avc3); + use super::*; + use crate::codec::h264::Split; + + fn setup(name: &str) -> (moq_net::TrackProducer, crate::catalog::Producer) { + let mut broadcast = moq_net::BroadcastInfo::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); + let track = broadcast + .create_track( + name, + moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), + ) + .unwrap(); + (track, catalog) } - /// Auto-detect routes an avcC initializer into the avc1 path and stores - /// it in the catalog `description`. + /// An avcC initializer resolves a config with the avcC stored as `description`. #[tokio::test(start_paused = true)] - async fn auto_detect_avc1_lands_in_catalog() { - // Minimal AVCDecoderConfigurationRecord: version(1) profile(0x42) compat(0xc0) level(0x1f) - // length_size_minus_one + 0xfc | 3 = 0xff - // reserved | num_sps = 0xe1 - // sps_len = 4, sps bytes (NAL header 0x67 + profile/level for parsing). + async fn initialize_avc1_lands_in_catalog() { let sps_nal = [0x67, 0x42, 0xc0, 0x1f]; let mut avcc = vec![0x01, 0x42, 0xc0, 0x1f, 0xff, 0xe1, 0x00, sps_nal.len() as u8]; avcc.extend_from_slice(&sps_nal); avcc.extend_from_slice(&[0x01, 0x00, 0x04, 0x68, 0xce, 0x3c, 0x80]); // num_pps + pps - let broadcast = moq_net::BroadcastInfo::new(); - let mut producer = broadcast.produce(); - let catalog = crate::catalog::Producer::new(&mut producer).unwrap(); - - let mut importer = Import::new(producer, catalog.clone()); + let (track, catalog) = setup("video"); + let mut import = Import::new(track, catalog.clone()); + // initialize() must not consume the buffer (the split owns the consume). let mut buf = bytes::BytesMut::from(avcc.as_slice()); - importer.initialize(&mut buf).expect("initialize avc1"); + import.initialize(&mut buf).expect("initialize avc1"); + assert_eq!(buf.len(), avcc.len(), "initialize must not consume the buffer"); let snapshot = catalog.snapshot(); - assert_eq!(snapshot.video.renditions.len(), 1); - let cfg = snapshot.video.renditions.values().next().unwrap(); + let cfg = snapshot.video.renditions.get("video").expect("rendition"); let hang::catalog::VideoCodec::H264(h264) = &cfg.codec else { panic!("expected H.264 codec") }; assert!(!h264.inline, "avc1 source should land as inline=false"); assert_eq!(h264.profile, 0x42); assert_eq!(h264.level, 0x1f); - let desc = cfg.description.as_ref().expect("description set"); - assert_eq!(desc.as_ref(), avcc.as_slice()); + assert_eq!(cfg.description.as_ref().expect("description").as_ref(), avcc.as_slice()); } - /// Auto-detect routes an Annex-B initializer into the avc3 path; the - /// catalog rendition reports inline=true and no description. + /// An avc3 stream self-initializes: the config is resolved from the SPS the + /// splitter packages into the first keyframe. #[tokio::test(start_paused = true)] - async fn auto_detect_avc3_lands_in_catalog() { + async fn avc3_self_initializes_from_first_keyframe() { let sps: &[u8] = &[ 0x67, 0x42, 0xc0, 0x1f, 0xda, 0x01, 0x40, 0x16, 0xe9, 0xb8, 0x08, 0x08, 0x0a, 0x00, 0x00, 0x07, 0xd0, 0x00, 0x01, 0xd4, 0xc0, 0x80, ]; let pps: &[u8] = &[0x68, 0xce, 0x3c, 0x80]; - let mut annexb = bytes::BytesMut::new(); - annexb.extend_from_slice(&[0, 0, 0, 1]); - annexb.extend_from_slice(sps); - annexb.extend_from_slice(&[0, 0, 0, 1]); - annexb.extend_from_slice(pps); + let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; + + let mut annexb = BytesMut::new(); + for nal in [sps, pps, idr] { + annexb.extend_from_slice(&[0, 0, 0, 1]); + annexb.extend_from_slice(nal); + } - let broadcast = moq_net::BroadcastInfo::new(); - let mut producer = broadcast.produce(); - let catalog = crate::catalog::Producer::new(&mut producer).unwrap(); + let mut split = Split::new(); + let (track, catalog) = setup("video"); + let mut import = Import::new(track, catalog.clone()); + assert!( + catalog.snapshot().video.renditions.is_empty(), + "no config before any frame" + ); - let mut importer = Import::new(producer, catalog.clone()); - importer.initialize(&mut annexb).expect("initialize avc3"); + let pts = moq_net::Timestamp::from_micros(0).unwrap(); + let mut frames = split.decode(&mut annexb, pts).expect("split keyframe"); + frames.extend(split.flush(pts).expect("flush keyframe")); + import.decode(frames).expect("decode keyframe"); let snapshot = catalog.snapshot(); - assert_eq!(snapshot.video.renditions.len(), 1); - let cfg = snapshot.video.renditions.values().next().unwrap(); - let hang::catalog::VideoCodec::H264(h264) = &cfg.codec else { + let h264_cfg = snapshot.video.renditions.get("video").expect("rendition"); + let hang::catalog::VideoCodec::H264(h264) = &h264_cfg.codec else { panic!("expected H.264 codec") }; assert!(h264.inline, "avc3 source should land as inline=true"); - assert!(cfg.description.is_none(), "avc3 has no out-of-band description"); + assert!(h264_cfg.description.is_none(), "avc3 has no out-of-band description"); assert_eq!(h264.profile, sps[1]); assert_eq!(h264.level, sps[3]); } -} -#[derive(Debug, Clone, Copy, PartialEq, Eq, num_enum::TryFromPrimitive)] -#[repr(u8)] -enum Avc3NalType { - Unspecified = 0, - NonIdrSlice = 1, - DataPartitionA = 2, - DataPartitionB = 3, - DataPartitionC = 4, - IdrSlice = 5, - Sei = 6, - Sps = 7, - Pps = 8, - Aud = 9, - EndOfSeq = 10, - EndOfStream = 11, - Filler = 12, - SpsExt = 13, - Prefix = 14, - SubsetSps = 15, - DepthParameterSet = 16, + /// A keyframe that carries no SPS (and no avcC/seed to fall back on) is + /// undecodable, so it's a hard error rather than an uncatalogued frame. + #[tokio::test(start_paused = true)] + async fn keyframe_without_sps_errors() { + let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; // IDR slice, no inline SPS + let mut annexb = BytesMut::new(); + annexb.extend_from_slice(&[0, 0, 0, 1]); + annexb.extend_from_slice(idr); + + let mut split = Split::new(); + let (track, catalog) = setup("video"); + let mut import = Import::new(track, catalog); + + let pts = moq_net::Timestamp::from_micros(0).unwrap(); + let mut frames = split.decode(&mut annexb, pts).expect("split keyframe"); + frames.extend(split.flush(pts).expect("flush keyframe")); + let err = import + .decode(frames) + .expect_err("an unconfigurable keyframe must error"); + assert!(matches!(err, crate::Error::H264(Error::NotInitialized)), "got {err:?}"); + } + + /// A non-keyframe before the first keyframe has no group to anchor it, so the + /// producer surfaces MissingKeyframe (which a mid-stream join skips). It must + /// not silently abort the import. + #[tokio::test(start_paused = true)] + async fn delta_before_init_reports_missing_keyframe() { + let pslice: &[u8] = &[0x61, 0xe0, 0x12, 0x34]; // non-IDR slice + let mut annexb = BytesMut::new(); + annexb.extend_from_slice(&[0, 0, 0, 1]); + annexb.extend_from_slice(pslice); + + let mut split = Split::new(); + let (track, catalog) = setup("video"); + let mut import = Import::new(track, catalog.clone()); + + let pts = moq_net::Timestamp::from_micros(0).unwrap(); + let mut frames = split.decode(&mut annexb, pts).expect("split delta"); + frames.extend(split.flush(pts).expect("flush delta")); + let err = import + .decode(frames) + .expect_err("a delta before any keyframe must report MissingKeyframe"); + assert!(matches!(err, crate::Error::MissingKeyframe(_)), "got {err:?}"); + assert!( + catalog.snapshot().video.renditions.is_empty(), + "no config yet, so no catalog" + ); + } } diff --git a/rs/moq-mux/src/codec/h264/mod.rs b/rs/moq-mux/src/codec/h264/mod.rs index c85bf64f4..c7e94fc2a 100644 --- a/rs/moq-mux/src/codec/h264/mod.rs +++ b/rs/moq-mux/src/codec/h264/mod.rs @@ -5,14 +5,18 @@ //! (inline SPS/PPS) as length-prefixed NALU + out-of-band avcC, which is //! what every CMAF and MKV consumer expects. [`Export`] subscribes to a //! catalog-narrowed H.264 rendition and emits an Annex-B elementary -//! stream; [`Import`] is the importer (auto-detects either wire shape -//! from the leading bytes). +//! stream; [`Split`] does the byte-level framing for the Annex-B (avc3) +//! wire shape and [`Import`] is the pure frame publisher that resolves the +//! catalog. avc1 (length-prefixed NALU) has no stream framing; wrap one +//! access unit with `avc1_frame`. mod export; mod import; +mod split; pub use export::*; pub use import::*; +pub use split::*; use bytes::{Buf, BufMut, Bytes, BytesMut}; @@ -20,6 +24,49 @@ use bytes::{Buf, BufMut, Bytes, BytesMut}; const NAL_TYPE_SPS: u8 = 7; const NAL_TYPE_PPS: u8 = 8; +/// Wrap one avc1 (length-prefixed NALU) access unit as a single +/// [`Frame`](crate::container::Frame), with the keyframe flag set when it +/// carries an IDR slice (NAL type 5). +/// +/// avc1 is not a stream: each access unit arrives whole with its NALU +/// `length_size` known out-of-band from the avcC (`super::Avcc::parse(avcc).length_size`). +/// The payload is passed through verbatim. +pub(crate) fn avc1_frame( + data: &[u8], + length_size: usize, + pts: moq_net::Timestamp, +) -> crate::Result { + Ok(crate::container::Frame { + timestamp: pts, + payload: data.to_vec().into(), + keyframe: avc1_is_keyframe(data, length_size), + duration: None, + }) +} + +/// Detect whether an avc1-shaped (length-prefixed) buffer contains an IDR slice. +fn avc1_is_keyframe(data: &[u8], length_size: usize) -> bool { + let mut offset = 0; + while offset + length_size <= data.len() { + let nal_len = match length_size { + 1 => data[offset] as usize, + 2 => u16::from_be_bytes([data[offset], data[offset + 1]]) as usize, + 3 => u32::from_be_bytes([0, data[offset], data[offset + 1], data[offset + 2]]) as usize, + 4 => u32::from_be_bytes([data[offset], data[offset + 1], data[offset + 2], data[offset + 3]]) as usize, + _ => return false, + }; + offset += length_size; + if offset + nal_len > data.len() { + break; + } + if nal_len > 0 && data[offset] & 0x1f == 5 { + return true; // IDR slice + } + offset += nal_len; + } + false +} + /// H.264 parsing and transform errors. #[derive(Debug, Clone, thiserror::Error)] #[non_exhaustive] @@ -54,21 +101,15 @@ pub enum Error { #[error("forbidden zero bit is not zero")] ForbiddenZeroBit, - #[error("not initialized; call initialize() or with_mode() first")] + #[error("not initialized")] NotInitialized, - #[error("fixed track cannot be reconfigured")] - FixedTrackReconfigured, - #[error("avc3 track not created")] Avc3TrackNotCreated, #[error("missing timestamp")] MissingTimestamp, - #[error("decode_stream is avc3 only")] - StreamNotAvc3, - #[error("annexb: {0}")] Annexb(#[from] crate::codec::annexb::Error), } @@ -426,6 +467,32 @@ mod tests { buf.freeze() } + /// avc1: a length-prefixed access unit with an IDR slice wraps as one keyframe; + /// the payload is passed through verbatim. + #[test] + fn avc1_frame_keyframe() { + let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; + let mut au = BytesMut::new(); + au.extend_from_slice(&(idr.len() as u32).to_be_bytes()); + au.extend_from_slice(idr); + + let frame = avc1_frame(&au, 4, moq_net::Timestamp::from_micros(0).unwrap()).unwrap(); + assert!(frame.keyframe); + assert_eq!(frame.payload[4..], *idr); + } + + /// avc1: a length-prefixed access unit with a non-IDR slice is a delta frame. + #[test] + fn avc1_frame_delta() { + let pslice: &[u8] = &[0x61, 0xe0, 0x12, 0x34]; + let mut au = BytesMut::new(); + au.extend_from_slice(&(pslice.len() as u32).to_be_bytes()); + au.extend_from_slice(pslice); + + let frame = avc1_frame(&au, 4, moq_net::Timestamp::from_micros(0).unwrap()).unwrap(); + assert!(!frame.keyframe); + } + #[test] fn avc3_strips_sps_pps_and_builds_avcc() { let sps = &[0x67, 0x42, 0xc0, 0x1f, 0xde][..]; diff --git a/rs/moq-mux/src/codec/h264/split.rs b/rs/moq-mux/src/codec/h264/split.rs new file mode 100644 index 000000000..76431cc04 --- /dev/null +++ b/rs/moq-mux/src/codec/h264/split.rs @@ -0,0 +1,334 @@ +//! H.264 Annex-B stream splitter. +//! +//! [`Split`] turns a raw H.264 Annex-B byte stream (inline SPS/PPS, the "avc3" +//! wire shape) into [`crate::container::Frame`]s. It finds access-unit +//! boundaries, caches SPS/PPS and re-inserts them ahead of each keyframe so +//! every keyframe is self-contained, and stamps wall-clock timestamps when the +//! caller has none (stdin). +//! +//! It is deliberately dumb: framing and structural parsing only. It owns no +//! track, catalog, or codec config (no [`VideoConfig`](hang::catalog::VideoConfig)). +//! The importer parses the codec config out of the frames it emits. +//! +//! avc1 (length-prefixed NALU + out-of-band avcC) is not a stream and has no +//! splitter; wrap one access unit with `super::avc1_frame`. + +use bytes::{Buf, Bytes, BytesMut}; + +use super::Error; +use crate::Result; +use crate::codec::annexb::{NalIterator, START_CODE}; + +/// H.264 Annex-B stream splitter: bytes in, [`Frame`](crate::container::Frame)s out. +/// +/// Feed bytes via [`decode`](Self::decode) (unknown frame boundaries, e.g. +/// stdin); call [`flush`](Self::flush) to emit the final in-flight access unit. +/// SPS/PPS seen inline are cached and re-inserted ahead of each keyframe so each +/// keyframe is self-contained. +pub struct Split { + /// Bytes carried over between calls: complete NALs are parsed out on each + /// [`decode`](Self::decode), leaving the in-flight (final, not-yet-terminated) + /// NAL here until the next start code arrives or [`flush`](Self::flush) drains it. + tail: BytesMut, + current: Avc3Frame, + sps: Option, + pps: Option, + zero: Option, + pending: Vec, +} + +#[derive(Default)] +struct Avc3Frame { + chunks: BytesMut, + contains_idr: bool, + contains_slice: bool, + contains_sps: bool, + contains_pps: bool, +} + +impl Default for Split { + fn default() -> Self { + Self::new() + } +} + +impl Split { + /// A fresh splitter with an empty parameter-set cache. + pub fn new() -> Self { + Self { + tail: BytesMut::new(), + current: Avc3Frame::default(), + sps: None, + pps: None, + zero: None, + pending: Vec::new(), + } + } + + /// Decode a buffer where frame boundaries are unknown, returning the access + /// units it can complete. The leading start code of the *next* access unit is + /// what signals the previous one is complete, so the final NAL of the in-flight + /// access unit stays buffered until the next call (or [`flush`](Self::flush)). + /// The buffer is fully consumed. + pub fn decode>( + &mut self, + buf: &mut T, + pts: impl Into>, + ) -> Result> { + let pts = self.pts(pts.into())?; + while buf.has_remaining() { + let chunk = buf.chunk(); + self.tail.extend_from_slice(chunk); + let len = chunk.len(); + buf.advance(len); + } + // Iterate complete NALs out of `tail`, leaving the trailing (in-flight) NAL + // (with its start code) buffered for the next call or `flush`. + let nals = NalIterator::new(&mut self.tail); + let mut parsed = Vec::new(); + for nal in nals { + parsed.push(nal?); + } + for nal in parsed { + self.decode_nal(nal, pts)?; + } + Ok(std::mem::take(&mut self.pending)) + } + + /// Emit the in-flight access unit, if any. Call after the last + /// [`decode`](Self::decode) when a caller handed over a complete access unit + /// (or at end of stream) so the final NAL isn't left buffered. + pub fn flush(&mut self, pts: impl Into>) -> Result> { + let pts = self.pts(pts.into())?; + if let Some(nal) = NalIterator::new(&mut self.tail).flush()? { + self.decode_nal(nal, pts)?; + } + self.tail.clear(); + self.maybe_start_frame(pts)?; + Ok(std::mem::take(&mut self.pending)) + } + + fn decode_nal(&mut self, nal: Bytes, pts: moq_net::Timestamp) -> Result<()> { + let header = nal.first().ok_or(Error::NalTooShort)?; + let forbidden_zero_bit = (header >> 7) & 1; + if forbidden_zero_bit != 0 { + return Err(Error::ForbiddenZeroBit.into()); + } + + let nal_unit_type = header & 0b11111; + let nal_type = Avc3NalType::try_from(nal_unit_type).ok(); + + match nal_type { + Some(Avc3NalType::Sps) => { + self.maybe_start_frame(pts)?; + if self.sps.as_ref().is_some_and(|cached| cached != &nal) { + // SPS changed mid-stream. The cached PPS is tied to the old + // SPS and may already have been appended to current.chunks + // earlier in this AU; reset so the new SPS+PPS pair is the + // only parameter set we emit. + self.pps = None; + self.current.chunks.clear(); + self.current.contains_pps = false; + self.current.contains_sps = false; + } + self.sps = Some(nal.clone()); + self.current.contains_sps = true; + } + Some(Avc3NalType::Pps) => { + self.maybe_start_frame(pts)?; + self.pps = Some(nal.clone()); + self.current.contains_pps = true; + } + Some(Avc3NalType::Aud) | Some(Avc3NalType::Sei) => { + self.maybe_start_frame(pts)?; + } + Some(Avc3NalType::IdrSlice) => { + if !self.current.contains_sps + && let Some(sps) = self.sps.clone() + { + self.current.chunks.extend_from_slice(&START_CODE); + self.current.chunks.extend_from_slice(&sps); + self.current.contains_sps = true; + } + if !self.current.contains_pps + && let Some(pps) = self.pps.clone() + { + self.current.chunks.extend_from_slice(&START_CODE); + self.current.chunks.extend_from_slice(&pps); + self.current.contains_pps = true; + } + self.current.contains_idr = true; + self.current.contains_slice = true; + } + Some(Avc3NalType::NonIdrSlice) + | Some(Avc3NalType::DataPartitionA) + | Some(Avc3NalType::DataPartitionB) + | Some(Avc3NalType::DataPartitionC) => { + if nal.get(1).ok_or(Error::NalTooShort)? & 0x80 != 0 { + self.maybe_start_frame(pts)?; + } + self.current.contains_slice = true; + } + _ => {} + } + + tracing::trace!(kind = ?nal_type, "parsed NAL"); + + self.current.chunks.extend_from_slice(&START_CODE); + self.current.chunks.extend_from_slice(&nal); + Ok(()) + } + + fn maybe_start_frame(&mut self, pts: moq_net::Timestamp) -> Result<()> { + if !self.current.contains_slice { + return Ok(()); + } + let payload = std::mem::take(&mut self.current.chunks).freeze(); + let keyframe = self.current.contains_idr; + self.current.contains_idr = false; + self.current.contains_slice = false; + self.current.contains_sps = false; + self.current.contains_pps = false; + + self.pending.push(crate::container::Frame { + timestamp: pts, + payload, + keyframe, + duration: None, + }); + Ok(()) + } + + /// Drop any in-flight access unit. + /// + /// Pre-reset NALs would otherwise leak into a later frame with the wrong + /// timestamp. The parameter-set cache is kept so subsequent keyframes stay + /// self-contained. + pub fn reset(&mut self) { + self.current = Avc3Frame::default(); + self.tail.clear(); + } + + fn pts(&mut self, hint: Option) -> Result { + if let Some(pts) = hint { + return Ok(pts); + } + let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); + Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, num_enum::TryFromPrimitive)] +#[repr(u8)] +enum Avc3NalType { + Unspecified = 0, + NonIdrSlice = 1, + DataPartitionA = 2, + DataPartitionB = 3, + DataPartitionC = 4, + IdrSlice = 5, + Sei = 6, + Sps = 7, + Pps = 8, + Aud = 9, + EndOfSeq = 10, + EndOfStream = 11, + Filler = 12, + SpsExt = 13, + Prefix = 14, + SubsetSps = 15, + DepthParameterSet = 16, +} + +#[cfg(test)] +mod tests { + use super::*; + + const SC4: &[u8] = &[0, 0, 0, 1]; + + fn annexb(nals: &[&[u8]]) -> BytesMut { + let mut buf = BytesMut::new(); + for nal in nals { + buf.extend_from_slice(SC4); + buf.extend_from_slice(nal); + } + buf + } + + fn ts() -> moq_net::Timestamp { + moq_net::Timestamp::from_micros(0).unwrap() + } + + /// Decode one complete access unit handed over as a single buffer: `decode` + /// buffers it, `flush` emits it. + fn decode_one(split: &mut Split, buf: &mut BytesMut, pts: moq_net::Timestamp) -> Vec { + let mut frames = split.decode(buf, pts).unwrap(); + frames.extend(split.flush(pts).unwrap()); + frames + } + + /// A keyframe access unit fed as one buffer emits one self-contained frame: + /// SPS+PPS are packaged ahead of the IDR slice and `keyframe` is set. + #[tokio::test(start_paused = true)] + async fn decode_packages_keyframe() { + let sps: &[u8] = &[0x67, 0x42, 0xc0, 0x1f]; + let pps: &[u8] = &[0x68, 0xce, 0x3c, 0x80]; + let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; + + let mut split = Split::new(); + let frames = decode_one(&mut split, &mut annexb(&[sps, pps, idr]), ts()); + + assert_eq!(frames.len(), 1); + assert!(frames[0].keyframe); + // The payload carries SPS, PPS, then the IDR slice (each start-code prefixed). + assert_eq!(&frames[0].payload[..SC4.len()], SC4); + assert!(frames[0].payload.windows(sps.len()).any(|w| w == sps)); + assert!(frames[0].payload.windows(idr.len()).any(|w| w == idr)); + } + + /// Parameter sets fed up front (as the leading stream bytes) are cached and + /// re-inserted ahead of a later bare IDR, so the keyframe is self-contained + /// even when the stream never repeats its parameter sets inline. + #[tokio::test(start_paused = true)] + async fn params_then_bare_keyframe_self_contained() { + let sps: &[u8] = &[0x67, 0x42, 0xc0, 0x1f]; + let pps: &[u8] = &[0x68, 0xce, 0x3c, 0x80]; + let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; + + let mut split = Split::new(); + // The leading SPS/PPS carry no slice, so they complete no frame yet. + assert!(split.decode(&mut annexb(&[sps, pps]), ts()).unwrap().is_empty()); + + let frames = decode_one(&mut split, &mut annexb(&[idr]), ts()); + assert_eq!(frames.len(), 1); + assert!(frames[0].keyframe); + assert!(frames[0].payload.windows(sps.len()).any(|w| w == sps)); + assert!(frames[0].payload.windows(pps.len()).any(|w| w == pps)); + } + + /// In streaming mode an access unit completes only once the next one begins + /// (a slice with first_mb_in_slice set). A keyframe AU followed by a P-slice + /// of the next AU completes the keyframe; the P-slice's own AU stays buffered + /// until `flush`. + #[tokio::test(start_paused = true)] + async fn decode_emits_on_next_boundary() { + let sps: &[u8] = &[0x67, 0x42, 0xc0, 0x1f]; + let pps: &[u8] = &[0x68, 0xce, 0x3c, 0x80]; + let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; + // P-slice with first_mb_in_slice (byte 1 high bit) set, opening a new AU. + let pslice: &[u8] = &[0x61, 0xe0, 0x12, 0x34]; + // A trailing AUD so the P-slice is a *complete* NAL (it has a following + // start code), letting the keyframe boundary be detected during decode. + let aud: &[u8] = &[0x09, 0x10]; + + let mut split = Split::new(); + let frames = split.decode(&mut annexb(&[sps, pps, idr, pslice, aud]), ts()).unwrap(); + assert_eq!(frames.len(), 1); + assert!(frames[0].keyframe); + + // Flushing closes the buffered P-slice AU (the AUD rides along with it). + let tail = split.flush(ts()).unwrap(); + assert_eq!(tail.len(), 1); + assert!(!tail[0].keyframe); + } +} diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index 9790b184f..13d5e4268 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -1,87 +1,118 @@ +//! H.265 importer. +//! +//! Publishes H.265 frames (Annex-B, inline VPS/SPS/PPS, the "hev1" shape) on a +//! single moq track and resolves the catalog rendition. Only single-layer +//! streams are supported (VPS is cached but not parsed). +//! +//! The codec config is scanned out of the SPS the splitter packages into the +//! first keyframe (or seeded via [`initialize`](Import::initialize)). A keyframe +//! that can't be configured is an error; non-keyframes before the first config +//! are written through to the producer, which reports +//! [`MissingKeyframe`](crate::container::MissingKeyframe) for a mid-stream join. +//! Annex-B byte parsing lives in [`Split`](super::Split); this type is a pure frame publisher +//! that whoever owns the split drives via [`decode`](Import::decode). + +use bytes::Bytes; +use scuffle_h265::SpsNALUnit; + +use super::{Error, split::nal_unit_type}; +use crate::Result; use crate::catalog::hang::CatalogExt; -use crate::codec::annexb::{NalIterator, START_CODE}; +use crate::codec::annexb::NalIterator; +use crate::container::Frame; use crate::container::jitter::MinFrameDuration; -use bytes::{Buf, Bytes, BytesMut}; -use scuffle_h265::{NALUnitType, SpsNALUnit}; - -use super::Error; -use crate::Result; - -/// A decoder for H.265 with inline SPS/PPS. +/// A pure-publisher importer for H.265 with inline VPS/SPS/PPS. /// Only supports single layer streams (VPS is cached but not parsed). +/// +/// Build it with [`new`](Self::new), passing the track producer and the +/// [`catalog::Producer`](crate::catalog::Producer) it publishes into, and feed it +/// frames a [`Split`](super::Split) produced via [`decode`](Self::decode). The +/// catalog rendition fills in lazily once the first SPS is parsed. pub struct Import { - // Where new media tracks come from. - tracks: crate::track_provider::TrackProvider, - - // The catalog being produced. - catalog: crate::catalog::Producer, - - // The track being produced. - track: Option>, - - // Whether the track has been initialized. - // If it changes, then we'll reinitialize with a new track. + track: crate::container::Producer, + rendition: crate::catalog::VideoTrack, config: Option, - - // The current frame being built. - current: Frame, - - // Used to compute wall clock timestamps if needed. - zero: Option, - - // Cached parameter set NALs for re-insertion before keyframes. - vps: Option, - sps: Option, - pps: Option, - - // Tracks the minimum frame duration and updates the catalog `jitter` field. + last_sps: Option, jitter: MinFrameDuration, } impl Import { - pub fn new(broadcast: moq_net::BroadcastProducer, catalog: crate::catalog::Producer) -> Self { + /// Publish on an existing track producer, registering the rendition in `catalog`. + pub fn new(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { + let rendition = catalog.video_track(track.name()); Self { - tracks: crate::track_provider::TrackProvider::unique(broadcast, ".hev1"), - catalog, - track: None, + track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), + rendition, config: None, - current: Default::default(), - zero: None, - vps: None, - sps: None, - pps: None, + last_sps: None, jitter: MinFrameDuration::new(), } } - pub fn new_with_track(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { - Self { - tracks: crate::track_provider::TrackProvider::fixed(track), - catalog, - track: None, - config: None, - current: Default::default(), - zero: None, - vps: None, - sps: None, - pps: None, - jitter: MinFrameDuration::new(), + /// Resolve the codec config from VPS/SPS/PPS and other non-slice NALs. + /// + /// Resolves the config from any SPS in the buffer. Optional, since the importer + /// also self-initializes from the first keyframe. Takes a read-only slice: the + /// dispatcher-owned [`Split`](super::Split) is what consumes the stream (and seeds + /// its parameter-set cache). + pub fn initialize(&mut self, buf: &[u8]) -> Result<()> { + let mut scan = Bytes::copy_from_slice(buf); + let mut nals = NalIterator::new(&mut scan); + while let Some(nal) = nals.next().transpose()? { + if is_sps(&nal) { + self.configure_from_sps(&nal)?; + } + } + if let Some(nal) = nals.flush()? + && is_sps(&nal) + { + self.configure_from_sps(&nal)?; } + Ok(()) + } + + /// A watch-only handle to this track's subscriber demand. + pub fn demand(&self) -> moq_net::TrackDemand { + self.track.track().demand() + } + + /// True once the first SPS has populated the catalog. + pub fn is_initialized(&self) -> bool { + self.config.is_some() + } + + /// Finish the track, flushing the current group. + pub fn finish(&mut self) -> Result<()> { + self.track.finish()?; + Ok(()) } - fn init(&mut self, sps: &SpsNALUnit) -> Result<()> { + /// Close the current group and open the next one at `sequence`. + pub fn seek(&mut self, sequence: u64) -> Result<()> { + self.track.seek(sequence)?; + Ok(()) + } + + /// Resolve the config from an inline SPS, updating the rendition in place on a + /// change. + fn configure_from_sps(&mut self, sps_nal: &Bytes) -> Result<()> { + if self.last_sps.as_ref() == Some(sps_nal) { + return Ok(()); + } + + let sps = SpsNALUnit::parse(&mut &sps_nal[..]).map_err(|_| Error::SpsParse)?; let profile = &sps.rbsp.profile_tier_level.general_profile; let vui_data = sps.rbsp.vui_parameters.as_ref().map(VuiData::new).unwrap_or_default(); let mut config = hang::catalog::VideoConfig::new(hang::catalog::H265 { - in_band: true, // We only support `hev1` with inline SPS/PPS for now + in_band: true, // We only support `hev1` with inline VPS/SPS/PPS for now. profile_space: profile.profile_space, profile_idc: profile.profile_idc, profile_compatibility_flags: profile.profile_compatibility_flag.bits().to_be_bytes(), tier_flag: profile.tier_flag, level_idc: profile.level_idc.ok_or(Error::MissingLevelIdc)?, - constraint_flags: crate::codec::h265::pack_constraint_flags(profile), + constraint_flags: super::pack_constraint_flags(profile), }); config.coded_width = Some(sps.rbsp.cropped_width() as u32); config.coded_height = Some(sps.rbsp.cropped_height() as u32); @@ -90,305 +121,69 @@ impl Import { config.display_ratio_height = vui_data.display_ratio_height; config.container = hang::catalog::Container::Legacy; - if let Some(old) = &self.config - && old == &config - { - return Ok(()); - } - - let mut catalog = self.catalog.lock(); - - if self.track.is_some() && self.tracks.is_fixed() { - return Err(Error::FixedTrackReconfigured.into()); - } + self.last_sps = Some(sps_nal.clone()); - if let Some(track) = self.track.take() { - tracing::debug!(name = ?track.name(), "reinitializing track"); - catalog.video.renditions.remove(track.name()); + // A changed SPS just re-mirrors the rendition in place; there are no fixed + // tracks to reject a reconfiguration. + if self.config.as_ref() == Some(&config) { + return Ok(()); } - let track = self.tracks.create()?; - tracing::debug!(name = ?track.name(), ?config, "starting track"); - catalog - .video - .renditions - .insert(track.name().to_string(), config.clone()); - + tracing::debug!(name = ?self.track.name(), ?config, "starting track"); + self.rendition.set(config.clone()); self.config = Some(config); - self.track = - Some(crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy).with_lenient_start()); - - Ok(()) - } - - /// Initialize the decoder with SPS/PPS and other non-slice NALs. - pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { - let mut nals = NalIterator::new(buf); - - while let Some(nal) = nals.next().transpose()? { - self.decode_nal(nal, None)?; - } - - if let Some(nal) = nals.flush()? { - self.decode_nal(nal, None)?; - } - - Ok(()) - } - - /// Returns a reference to the underlying track producer. - pub fn track(&self) -> Result<&moq_net::TrackProducer> { - Ok(self.track.as_ref().ok_or(Error::NotInitialized)?.track()) - } - - /// Decode as much data as possible from the given buffer. - /// - /// Unlike [Self::decode_frame], this method needs the start code for the next frame. - /// This means it works for streaming media (ex. stdin) but adds a frame of latency. - /// - /// TODO: This currently associates PTS with the *previous* frame, as part of `maybe_start_frame`. - pub fn decode_stream>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - let pts = self.pts(pts)?; - - // Iterate over the NAL units in the buffer based on start codes. - let nals = NalIterator::new(buf); - - for nal in nals { - self.decode_nal(nal?, Some(pts))?; - } - - Ok(()) - } - - /// Decode all data in the buffer, assuming the buffer contains (the rest of) a frame. - /// - /// Unlike [Self::decode_stream], this is called when we know NAL boundaries. - /// This can avoid a frame of latency just waiting for the next frame's start code. - /// This can also be used when EOF is detected to flush the final frame. - /// - /// NOTE: The next decode will fail if it doesn't begin with a start code. - pub fn decode_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - let pts = self.pts(pts)?; - // Iterate over the NAL units in the buffer based on start codes. - let mut nals = NalIterator::new(buf); - - // Iterate over each NAL that is followed by a start code. - while let Some(nal) = nals.next().transpose()? { - self.decode_nal(nal, Some(pts))?; - } - - // Assume the rest of the buffer is a single NAL. - if let Some(nal) = nals.flush()? { - self.decode_nal(nal, Some(pts))?; - } - - // Flush the frame if we read a slice. - self.maybe_start_frame(Some(pts))?; - Ok(()) } - /// Decode a single NAL unit. Only reads the first header byte to extract nal_unit_type, - /// Ignores nuh_layer_id and nuh_temporal_id_plus1. - fn decode_nal(&mut self, nal: Bytes, pts: Option) -> Result<()> { - if nal.len() < 2 { - return Err(Error::NalTooShort.into()); - } - // u16 header: [forbidden_zero_bit(1) | nal_unit_type(6) | nuh_layer_id(6) | nuh_temporal_id_plus1(3)] - let header = nal.first().ok_or(Error::NalTooShort)?; - - let forbidden_zero_bit = (header >> 7) & 1; - if forbidden_zero_bit != 0 { - return Err(Error::ForbiddenZeroBit.into()); - } - - // Bits 1-6: nal_unit_type - let nal_unit_type = (header >> 1) & 0b111111; - let nal_type = NALUnitType::from(nal_unit_type); - - match nal_type { - NALUnitType::VpsNut => { - self.maybe_start_frame(pts)?; - - self.vps = Some(nal.clone()); - self.current.contains_vps = true; + /// Write split frames to the track, resolving the config from the first + /// keyframe's inline SPS and refining the catalog jitter as it goes. + fn write_frames(&mut self, frames: impl IntoIterator) -> Result<()> { + for frame in frames { + if frame.keyframe + && let Some(sps) = find_sps(&frame.payload) + { + self.configure_from_sps(&sps)?; } - NALUnitType::SpsNut => { - self.maybe_start_frame(pts)?; - - // Try to reinitialize the track if the SPS has changed. - let sps = SpsNALUnit::parse(&mut &nal[..]).map_err(|_| Error::SpsParse)?; - self.init(&sps)?; - - // SPS changed mid-AU. Cached VPS/PPS are tied to the old SPS - // and may already have been appended to current.chunks earlier - // in this AU; reset the AU so the new VPS+SPS+PPS triple is - // the only parameter set we emit. - if self.sps.as_ref().is_some_and(|cached| cached != &nal) { - self.pps = None; - self.current.chunks.clear(); - self.current.contains_vps = false; - self.current.contains_sps = false; - self.current.contains_pps = false; - } - self.sps = Some(nal.clone()); - self.current.contains_sps = true; + // A keyframe we still can't configure (no SPS) is undecodable. + if frame.keyframe && self.config.is_none() { + return Err(Error::MissingSps.into()); } - NALUnitType::PpsNut => { - self.maybe_start_frame(pts)?; - self.pps = Some(nal.clone()); - self.current.contains_pps = true; - } - NALUnitType::AudNut | NALUnitType::PrefixSeiNut | NALUnitType::SuffixSeiNut => { - self.maybe_start_frame(pts)?; - } - // Keyframe containing slices - NALUnitType::IdrWRadl - | NALUnitType::IdrNLp - | NALUnitType::BlaNLp - | NALUnitType::BlaWRadl - | NALUnitType::BlaWLp - | NALUnitType::CraNut => { - // Insert cached VPS/SPS/PPS before keyframes if not already present in this frame. - if !self.current.contains_vps - && let Some(vps) = &self.vps - { - self.current.chunks.extend_from_slice(&START_CODE); - self.current.chunks.extend_from_slice(vps); - self.current.contains_vps = true; - } - if !self.current.contains_sps - && let Some(sps) = &self.sps - { - self.current.chunks.extend_from_slice(&START_CODE); - self.current.chunks.extend_from_slice(sps); - self.current.contains_sps = true; - } - if !self.current.contains_pps - && let Some(pps) = &self.pps - { - self.current.chunks.extend_from_slice(&START_CODE); - self.current.chunks.extend_from_slice(pps); - self.current.contains_pps = true; - } + let pts = frame.timestamp; + // A pre-keyframe delta has no group to anchor it: the producer returns + // MissingKeyframe, which the caller (e.g. a TS mid-stream join) skips. + self.track.write(frame)?; - self.current.contains_idr = true; - self.current.contains_slice = true; + if let Some(jitter) = self.jitter.observe(pts) { + self.rendition.update(|c| c.jitter = Some(jitter)); } - // All other slice types (both N and R variants) - NALUnitType::TrailN - | NALUnitType::TrailR - | NALUnitType::TsaN - | NALUnitType::TsaR - | NALUnitType::StsaN - | NALUnitType::StsaR - | NALUnitType::RadlN - | NALUnitType::RadlR - | NALUnitType::RaslN - | NALUnitType::RaslR => { - // Check first_slice_segment_in_pic_flag (bit 7 of third byte, after 2-byte header) - if nal.get(2).ok_or(Error::NalTooShort)? & 0x80 != 0 { - self.maybe_start_frame(pts)?; - } - self.current.contains_slice = true; - } - _ => {} } - - // Replace the original start code with a canonical 4-byte start code (marginally easier - // for downstream players, e.g. MSE). - self.current.chunks.extend_from_slice(&START_CODE); - self.current.chunks.extend_from_slice(&nal); - Ok(()) } - fn maybe_start_frame(&mut self, pts: Option) -> Result<()> { - // If we haven't seen any slices, we shouldn't flush yet. - if !self.current.contains_slice { - return Ok(()); - } - - let track = self.track.as_mut().ok_or(Error::MissingSps)?; - let pts = pts.ok_or(Error::MissingTimestamp)?; - - let payload = std::mem::take(&mut self.current.chunks).freeze(); - - let frame = crate::container::Frame { - timestamp: pts, - payload, - keyframe: self.current.contains_idr, - duration: None, - }; - - track.write(frame)?; - - if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.lock().video.renditions.get_mut(track.name()) - { - c.jitter = Some(jitter); - } - - self.current.contains_idr = false; - self.current.contains_slice = false; - self.current.contains_vps = false; - self.current.contains_sps = false; - self.current.contains_pps = false; - - Ok(()) - } - - /// Finish the track, flushing the current group. - pub fn finish(&mut self) -> Result<()> { - let track = self.track.as_mut().ok_or(Error::NotInitialized)?; - track.finish()?; - Ok(()) + /// Publish split frames, resolving the config from the first keyframe's inline + /// SPS and refining the catalog jitter as it goes. + pub fn decode(&mut self, frames: impl IntoIterator) -> Result<()> { + self.write_frames(frames) } +} - /// Close the current group and open the next one at `sequence`. - /// - /// Any in-flight access unit is dropped. Pre-seek NALs would otherwise leak - /// into the post-seek group with the wrong timestamp. - pub fn seek(&mut self, sequence: u64) -> Result<()> { - self.current = Frame::default(); - let track = self.track.as_mut().ok_or(Error::NotInitialized)?; - track.seek(sequence)?; - Ok(()) - } - - pub fn is_initialized(&self) -> bool { - self.track.is_some() - } - - fn pts(&mut self, hint: Option) -> Result { - if let Some(pts) = hint { - return Ok(pts); - } - - let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); - Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) - } +fn is_sps(nal: &[u8]) -> bool { + nal.first() + .is_some_and(|h| nal_unit_type(*h) == scuffle_h265::NALUnitType::SpsNut) } -impl Drop for Import { - fn drop(&mut self) { - if let Some(track) = &self.track { - tracing::debug!(name = ?track.name(), "ending track"); - self.catalog.lock().video.renditions.remove(track.name()); +/// Find the first SPS NAL in an Annex-B payload, if any. +fn find_sps(payload: &[u8]) -> Option { + let mut buf = Bytes::copy_from_slice(payload); + let mut nals = NalIterator::new(&mut buf); + while let Some(Ok(nal)) = nals.next() { + if is_sps(&nal) { + return Some(nal); } } -} - -#[derive(Default)] -struct Frame { - chunks: BytesMut, - contains_idr: bool, - contains_slice: bool, - contains_vps: bool, - contains_sps: bool, - contains_pps: bool, + nals.flush().ok().flatten().filter(|nal| is_sps(nal)) } #[derive(Default)] diff --git a/rs/moq-mux/src/codec/h265/mod.rs b/rs/moq-mux/src/codec/h265/mod.rs index 828a5798d..7c9648df6 100644 --- a/rs/moq-mux/src/codec/h265/mod.rs +++ b/rs/moq-mux/src/codec/h265/mod.rs @@ -8,9 +8,11 @@ mod export; mod import; +mod split; pub use export::*; pub use import::*; +pub use split::*; use bytes::{Buf, BufMut, Bytes, BytesMut}; use scuffle_h265::{NALUnitType, SpsNALUnit}; @@ -40,9 +42,6 @@ pub enum Error { #[error("not initialized")] NotInitialized, - #[error("fixed track cannot be reconfigured")] - FixedTrackReconfigured, - #[error("expected SPS before any frames")] MissingSps, diff --git a/rs/moq-mux/src/codec/h265/split.rs b/rs/moq-mux/src/codec/h265/split.rs new file mode 100644 index 000000000..fc8180c29 --- /dev/null +++ b/rs/moq-mux/src/codec/h265/split.rs @@ -0,0 +1,331 @@ +//! H.265 Annex-B stream splitter. +//! +//! The H.265 analogue of [`crate::codec::h264::Split`]: turns a raw Annex-B byte +//! stream (inline VPS/SPS/PPS) into [`crate::container::Frame`]s. It finds +//! access-unit boundaries, caches VPS/SPS/PPS and re-inserts them ahead of each +//! keyframe so every keyframe is self-contained, and stamps wall-clock +//! timestamps when the caller has none (stdin). It owns no track, catalog, or +//! codec config. The importer parses the codec config out of the frames it +//! emits. + +use bytes::{Buf, Bytes, BytesMut}; +use scuffle_h265::NALUnitType; + +use super::Error; +use crate::Result; +use crate::codec::annexb::{NalIterator, START_CODE}; + +/// H.265 Annex-B stream splitter: bytes in, [`Frame`](crate::container::Frame)s out. +/// +/// Feed bytes via [`decode`](Self::decode) (unknown frame boundaries, e.g. +/// stdin); call [`flush`](Self::flush) to emit the final in-flight access unit. +/// VPS/SPS/PPS seen inline are cached and re-inserted ahead of each keyframe so +/// each keyframe is self-contained. +pub struct Split { + /// Bytes carried over between calls: complete NALs are parsed out on each + /// [`decode`](Self::decode), leaving the in-flight (final, not-yet-terminated) + /// NAL here until the next start code arrives or [`flush`](Self::flush) drains it. + tail: BytesMut, + current: Au, + vps: Option, + sps: Option, + pps: Option, + zero: Option, + pending: Vec, +} + +#[derive(Default)] +struct Au { + chunks: BytesMut, + contains_idr: bool, + contains_slice: bool, + contains_vps: bool, + contains_sps: bool, + contains_pps: bool, +} + +impl Default for Split { + fn default() -> Self { + Self::new() + } +} + +impl Split { + /// A fresh splitter with an empty parameter-set cache. + pub fn new() -> Self { + Self { + tail: BytesMut::new(), + current: Au::default(), + vps: None, + sps: None, + pps: None, + zero: None, + pending: Vec::new(), + } + } + + /// Decode a buffer where frame boundaries are unknown, returning the access + /// units it can complete. The leading start code of the *next* access unit is + /// what signals the previous one is complete, so the final NAL of the in-flight + /// access unit stays buffered until the next call (or [`flush`](Self::flush)). + /// The buffer is fully consumed. + pub fn decode>( + &mut self, + buf: &mut T, + pts: impl Into>, + ) -> Result> { + let pts = self.pts(pts.into())?; + while buf.has_remaining() { + let chunk = buf.chunk(); + self.tail.extend_from_slice(chunk); + let len = chunk.len(); + buf.advance(len); + } + // Iterate complete NALs out of `tail`, leaving the trailing (in-flight) NAL + // (with its start code) buffered for the next call or `flush`. + let nals = NalIterator::new(&mut self.tail); + let mut parsed = Vec::new(); + for nal in nals { + parsed.push(nal?); + } + for nal in parsed { + self.decode_nal(nal, pts)?; + } + Ok(std::mem::take(&mut self.pending)) + } + + /// Emit the in-flight access unit, if any. Call after the last + /// [`decode`](Self::decode) when a caller handed over a complete access unit + /// (or at end of stream) so the final NAL isn't left buffered. + pub fn flush(&mut self, pts: impl Into>) -> Result> { + let pts = self.pts(pts.into())?; + if let Some(nal) = NalIterator::new(&mut self.tail).flush()? { + self.decode_nal(nal, pts)?; + } + self.tail.clear(); + self.maybe_start_frame(pts)?; + Ok(std::mem::take(&mut self.pending)) + } + + /// Decode a single NAL unit. Only reads the first header byte to extract + /// nal_unit_type, ignoring nuh_layer_id and nuh_temporal_id_plus1. + fn decode_nal(&mut self, nal: Bytes, pts: moq_net::Timestamp) -> Result<()> { + if nal.len() < 2 { + return Err(Error::NalTooShort.into()); + } + // u16 header: [forbidden_zero_bit(1) | nal_unit_type(6) | nuh_layer_id(6) | nuh_temporal_id_plus1(3)] + let header = nal.first().ok_or(Error::NalTooShort)?; + if (header >> 7) & 1 != 0 { + return Err(Error::ForbiddenZeroBit.into()); + } + + let nal_type = nal_unit_type(*header); + + match nal_type { + NALUnitType::VpsNut => { + self.maybe_start_frame(pts)?; + self.vps = Some(nal.clone()); + self.current.contains_vps = true; + } + NALUnitType::SpsNut => { + self.maybe_start_frame(pts)?; + + // SPS changed mid-stream. Cached PPS is tied to the old SPS and + // may already have been appended to current.chunks earlier in + // this AU; reset so the new VPS+SPS+PPS triple is the only + // parameter set we emit. + if self.sps.as_ref().is_some_and(|cached| cached != &nal) { + self.pps = None; + self.current.chunks.clear(); + self.current.contains_vps = false; + self.current.contains_sps = false; + self.current.contains_pps = false; + } + + self.sps = Some(nal.clone()); + self.current.contains_sps = true; + } + NALUnitType::PpsNut => { + self.maybe_start_frame(pts)?; + self.pps = Some(nal.clone()); + self.current.contains_pps = true; + } + NALUnitType::AudNut | NALUnitType::PrefixSeiNut | NALUnitType::SuffixSeiNut => { + self.maybe_start_frame(pts)?; + } + // Keyframe containing slices. + NALUnitType::IdrWRadl + | NALUnitType::IdrNLp + | NALUnitType::BlaNLp + | NALUnitType::BlaWRadl + | NALUnitType::BlaWLp + | NALUnitType::CraNut => { + // Insert cached VPS/SPS/PPS before keyframes if not already present. + if !self.current.contains_vps + && let Some(vps) = self.vps.clone() + { + self.current.chunks.extend_from_slice(&START_CODE); + self.current.chunks.extend_from_slice(&vps); + self.current.contains_vps = true; + } + if !self.current.contains_sps + && let Some(sps) = self.sps.clone() + { + self.current.chunks.extend_from_slice(&START_CODE); + self.current.chunks.extend_from_slice(&sps); + self.current.contains_sps = true; + } + if !self.current.contains_pps + && let Some(pps) = self.pps.clone() + { + self.current.chunks.extend_from_slice(&START_CODE); + self.current.chunks.extend_from_slice(&pps); + self.current.contains_pps = true; + } + + self.current.contains_idr = true; + self.current.contains_slice = true; + } + // All other slice types (both N and R variants). + NALUnitType::TrailN + | NALUnitType::TrailR + | NALUnitType::TsaN + | NALUnitType::TsaR + | NALUnitType::StsaN + | NALUnitType::StsaR + | NALUnitType::RadlN + | NALUnitType::RadlR + | NALUnitType::RaslN + | NALUnitType::RaslR => { + // Check first_slice_segment_in_pic_flag (bit 7 of third byte, after 2-byte header). + if nal.get(2).ok_or(Error::NalTooShort)? & 0x80 != 0 { + self.maybe_start_frame(pts)?; + } + self.current.contains_slice = true; + } + _ => {} + } + + // Replace the original start code with a canonical 4-byte start code (marginally + // easier for downstream players, e.g. MSE). + self.current.chunks.extend_from_slice(&START_CODE); + self.current.chunks.extend_from_slice(&nal); + + Ok(()) + } + + fn maybe_start_frame(&mut self, pts: moq_net::Timestamp) -> Result<()> { + if !self.current.contains_slice { + return Ok(()); + } + + let payload = std::mem::take(&mut self.current.chunks).freeze(); + let keyframe = self.current.contains_idr; + self.current.contains_idr = false; + self.current.contains_slice = false; + self.current.contains_vps = false; + self.current.contains_sps = false; + self.current.contains_pps = false; + + self.pending.push(crate::container::Frame { + timestamp: pts, + payload, + keyframe, + duration: None, + }); + Ok(()) + } + + /// Drop any in-flight access unit. + /// + /// Pre-reset NALs would otherwise leak into a later frame with the wrong + /// timestamp. The parameter-set cache is kept so subsequent keyframes stay + /// self-contained. + pub fn reset(&mut self) { + self.current = Au::default(); + self.tail.clear(); + } + + fn pts(&mut self, hint: Option) -> Result { + if let Some(pts) = hint { + return Ok(pts); + } + let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); + Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) + } +} + +/// Extract the HEVC `nal_unit_type` from the first header byte (bits 1..=6). +pub(super) fn nal_unit_type(header: u8) -> NALUnitType { + NALUnitType::from((header >> 1) & 0b111111) +} + +#[cfg(test)] +mod tests { + use super::*; + + const SC4: &[u8] = &[0, 0, 0, 1]; + + // HEVC NAL headers: byte0 = nal_unit_type << 1 (forbidden bit 0, layer id 0). + const VPS: &[u8] = &[0x40, 0x01, 0x0c]; // type 32 + const SPS: &[u8] = &[0x42, 0x01, 0x01]; // type 33 + const PPS: &[u8] = &[0x44, 0x01, 0xc0]; // type 34 + const IDR: &[u8] = &[0x26, 0x01, 0x80, 0xaa]; // type 19 (IdrWRadl) + + fn annexb(nals: &[&[u8]]) -> BytesMut { + let mut buf = BytesMut::new(); + for nal in nals { + buf.extend_from_slice(SC4); + buf.extend_from_slice(nal); + } + buf + } + + fn ts() -> moq_net::Timestamp { + moq_net::Timestamp::from_micros(0).unwrap() + } + + fn contains(haystack: &[u8], needle: &[u8]) -> bool { + haystack.windows(needle.len()).any(|w| w == needle) + } + + /// Decode one complete access unit handed over as a single buffer: `decode` + /// buffers it, `flush` emits it. + fn decode_one(split: &mut Split, buf: &mut BytesMut, pts: moq_net::Timestamp) -> Vec { + let mut frames = split.decode(buf, pts).unwrap(); + frames.extend(split.flush(pts).unwrap()); + frames + } + + /// A keyframe access unit fed as one buffer emits one self-contained frame: + /// VPS+SPS+PPS are packaged ahead of the IDR slice and `keyframe` is set. + #[tokio::test(start_paused = true)] + async fn decode_packages_keyframe() { + let mut split = Split::new(); + let frames = decode_one(&mut split, &mut annexb(&[VPS, SPS, PPS, IDR]), ts()); + + assert_eq!(frames.len(), 1); + assert!(frames[0].keyframe); + assert!(contains(&frames[0].payload, VPS)); + assert!(contains(&frames[0].payload, SPS)); + assert!(contains(&frames[0].payload, PPS)); + assert!(contains(&frames[0].payload, IDR)); + } + + /// Parameter sets fed up front (as the leading stream bytes) are cached and + /// re-inserted ahead of a later bare IDR, so the keyframe is self-contained + /// even when the stream never repeats its parameter sets inline. + #[tokio::test(start_paused = true)] + async fn params_then_bare_keyframe_self_contained() { + let mut split = Split::new(); + // The leading VPS/SPS/PPS carry no slice, so they complete no frame yet. + assert!(split.decode(&mut annexb(&[VPS, SPS, PPS]), ts()).unwrap().is_empty()); + + let frames = decode_one(&mut split, &mut annexb(&[IDR]), ts()); + assert_eq!(frames.len(), 1); + assert!(frames[0].keyframe); + assert!(contains(&frames[0].payload, VPS)); + assert!(contains(&frames[0].payload, SPS)); + assert!(contains(&frames[0].payload, PPS)); + } +} diff --git a/rs/moq-mux/src/codec/legacy.rs b/rs/moq-mux/src/codec/legacy.rs index 7dfcdaca9..56b37abc7 100644 --- a/rs/moq-mux/src/codec/legacy.rs +++ b/rs/moq-mux/src/codec/legacy.rs @@ -7,10 +7,75 @@ //! codec contributes only a header parser and a [`Descriptor`]; this module //! owns the track lifecycle. -use bytes::{Buf, BytesMut}; use moq_net::Timestamp; use crate::catalog::hang::CatalogExt; +use crate::container::Frame; + +/// Legacy audio (MP2 / AC-3 / E-AC-3) header parsing errors. +#[derive(Debug, Clone, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + #[error("AC-3 header needs 7 bytes")] + Ac3HeaderTooShort, + + #[error("missing AC-3 sync word")] + Ac3MissingSyncWord, + + #[error("invalid AC-3 frame size code")] + Ac3InvalidFrameSizeCode, + + #[error("unsupported AC-3 bsid {0}")] + Ac3UnsupportedBsid(u8), + + #[error("reserved AC-3 sample-rate code")] + Ac3ReservedSampleRate, + + #[error("E-AC-3 header needs 6 bytes")] + Eac3HeaderTooShort, + + #[error("missing E-AC-3 sync word")] + Eac3MissingSyncWord, + + #[error("not an E-AC-3 bitstream (bsid {0})")] + Eac3NotEac3Bsid(u8), + + #[error("reserved E-AC-3 stream type")] + Eac3ReservedStreamType, + + #[error("E-AC-3 dependent substream (7.1+ layout) is not supported; only a single independent substream")] + Eac3DependentSubstream, + + #[error("E-AC-3 additional substream {0} is not supported; only a single independent substream")] + Eac3AdditionalSubstream(u8), + + #[error("E-AC-3 frame length {0} shorter than its header")] + Eac3FrameShorterThanHeader(usize), + + #[error("reserved E-AC-3 sample-rate code")] + Eac3ReservedSampleRate, + + #[error("MP2 header needs 4 bytes")] + Mp2HeaderTooShort, + + #[error("missing MP2 frame sync")] + Mp2MissingSync, + + #[error("reserved or MPEG-2.5 audio version")] + Mp2ReservedVersion, + + #[error("not MPEG Layer II")] + Mp2NotLayerII, + + #[error("reserved MP2 sample-rate index")] + Mp2ReservedSampleRate, + + #[error("free-format or invalid MP2 bitrate")] + Mp2InvalidBitrate, +} + +/// A Result type alias for legacy audio header parsing. +pub type Result = std::result::Result; /// A parsed legacy-audio frame header. #[derive(Debug)] @@ -33,7 +98,7 @@ pub(crate) struct Descriptor { /// Bytes needed to attempt a header parse. pub min_header_len: usize, /// Parse one frame header at the start of the slice. - pub parse: fn(&[u8]) -> anyhow::Result
, + pub parse: fn(&[u8]) -> Result
, } /// Catalog config for a legacy audio track. Both fields come from the frame @@ -49,25 +114,20 @@ pub(crate) struct Config { /// forwards it immediately. The audio is never decoded; the catalog carries the /// codec, sample rate and channel count read from the frame header. pub(crate) struct Import { - catalog: crate::catalog::Producer, track: crate::container::Producer, - zero: Option, + rendition: crate::catalog::AudioTrack, } impl Import { + /// Publish on an existing track, registering the rendition in `catalog`. Mint the + /// track at the descriptor's suffix and the microsecond + /// [`hang::container::TIMESCALE`] (e.g. via [`crate::import::unique_track`]). pub fn new( descriptor: &'static Descriptor, - mut broadcast: moq_net::BroadcastProducer, - mut catalog: crate::catalog::Producer, + track: moq_net::TrackProducer, + catalog: crate::catalog::Producer, config: Config, - ) -> anyhow::Result { - // The legacy container normalizes the per-frame timestamp to microseconds on - // the wire, so the track declares that timescale to match. - let track = broadcast.unique_track( - descriptor.track_suffix, - moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), - )?; - + ) -> Self { let mut audio_config = hang::catalog::AudioConfig::new(descriptor.codec.clone(), config.sample_rate, config.channel_count); audio_config.container = hang::catalog::Container::Legacy; @@ -76,69 +136,38 @@ impl Import { // cannot decode these codecs). Fill it only if a real consumer ever needs it. tracing::debug!(name = ?track.name(), config = ?audio_config, "starting track"); - catalog - .lock() - .audio - .renditions - .insert(track.name().to_string(), audio_config); - - Ok(Self { - catalog, + + let mut rendition = catalog.audio_track(track.name()); + rendition.set(audio_config); + + Self { track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), - zero: None, - }) + rendition, + } } /// Finish the track, flushing the current group. - pub fn finish(&mut self) -> anyhow::Result<()> { + pub fn finish(&mut self) -> crate::Result<()> { self.track.finish()?; Ok(()) } /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> anyhow::Result<()> { + pub fn seek(&mut self, sequence: u64) -> crate::Result<()> { self.track.seek(sequence)?; Ok(()) } /// Publish one whole frame as a hang frame in its own group. - pub fn decode(&mut self, buf: &mut T, pts: Option) -> anyhow::Result<()> { - let pts = self.pts(pts)?; - - let mut payload = BytesMut::with_capacity(buf.remaining()); - while buf.has_remaining() { - let chunk = buf.chunk(); - payload.extend_from_slice(chunk); - let len = chunk.len(); - buf.advance(len); - } - - let frame = crate::container::Frame { - timestamp: pts, + pub fn decode(&mut self, frame: &[u8], pts: Option) -> crate::Result<()> { + let timestamp = self.rendition.timestamp(pts)?; + self.track.write(Frame { + timestamp, duration: None, - payload: payload.freeze(), + payload: bytes::Bytes::copy_from_slice(frame), keyframe: true, - }; - - self.track.write(frame)?; + })?; self.track.finish_group()?; - Ok(()) } - - fn pts(&mut self, hint: Option) -> anyhow::Result { - if let Some(pts) = hint { - return Ok(pts); - } - - let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); - Ok(Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) - } -} - -impl Drop for Import { - fn drop(&mut self) { - tracing::debug!(name = ?self.track.name(), "ending track"); - self.catalog.lock().audio.renditions.remove(self.track.name()); - } } diff --git a/rs/moq-mux/src/codec/mp2.rs b/rs/moq-mux/src/codec/mp2.rs index cd8690baa..0b5610402 100644 --- a/rs/moq-mux/src/codec/mp2.rs +++ b/rs/moq-mux/src/codec/mp2.rs @@ -32,10 +32,14 @@ const SAMPLE_RATE: [[u32; 3]; 2] = [[44100, 48000, 32000], [22050, 24000, 16000] const SAMPLES_PER_FRAME: u64 = 1152; /// Parse a Layer II frame header from the start of `data` (needs >= 4 bytes). -pub(crate) fn parse_header(data: &[u8]) -> anyhow::Result { - anyhow::ensure!(data.len() >= 4, "MP2 header needs 4 bytes"); +pub(crate) fn parse_header(data: &[u8]) -> legacy::Result { + if data.len() < 4 { + return Err(legacy::Error::Mp2HeaderTooShort); + } // Frame sync: 11 bits set (0xFFE). - anyhow::ensure!(data[0] == 0xFF && (data[1] & 0xE0) == 0xE0, "missing MP2 frame sync"); + if !(data[0] == 0xFF && (data[1] & 0xE0) == 0xE0) { + return Err(legacy::Error::Mp2MissingSync); + } // 0b00 is the unofficial MPEG-2.5 extension: 13818-1 has no stream type for // it, so accepting it here would re-announce it as 0x04 on export and invent @@ -44,23 +48,29 @@ pub(crate) fn parse_header(data: &[u8]) -> anyhow::Result { let (version, sr_row) = match (data[1] >> 3) & 0x03 { 0b11 => (Version::Mpeg1, 0), 0b10 => (Version::Mpeg2, 1), - _ => anyhow::bail!("reserved or MPEG-2.5 audio version"), + _ => return Err(legacy::Error::Mp2ReservedVersion), }; // Layer field 0b10 is Layer II. - anyhow::ensure!((data[1] >> 1) & 0x03 == 0b10, "not MPEG Layer II"); + if (data[1] >> 1) & 0x03 != 0b10 { + return Err(legacy::Error::Mp2NotLayerII); + } let bitrate_index = (data[2] >> 4) & 0x0F; let sr_index = (data[2] >> 2) & 0x03; let padding = ((data[2] >> 1) & 0x01) as usize; - anyhow::ensure!(sr_index != 3, "reserved MP2 sample-rate index"); + if sr_index == 3 { + return Err(legacy::Error::Mp2ReservedSampleRate); + } let sample_rate = SAMPLE_RATE[sr_row][sr_index as usize]; let bitrate_kbps = match version { Version::Mpeg1 => BITRATE_MPEG1_L2[bitrate_index as usize], Version::Mpeg2 => BITRATE_MPEG2_L2[bitrate_index as usize], }; - anyhow::ensure!(bitrate_kbps != 0, "free-format or invalid MP2 bitrate"); + if bitrate_kbps == 0 { + return Err(legacy::Error::Mp2InvalidBitrate); + } // Layer II is always 1152 samples, so the frame is 144 * bitrate / sample_rate bytes. let len = (144 * bitrate_kbps * 1000 / sample_rate) as usize + padding; diff --git a/rs/moq-mux/src/codec/opus/import.rs b/rs/moq-mux/src/codec/opus/import.rs index 583c25c25..9fe97367f 100644 --- a/rs/moq-mux/src/codec/opus/import.rs +++ b/rs/moq-mux/src/codec/opus/import.rs @@ -1,72 +1,49 @@ -use bytes::{Buf, BytesMut}; - use super::Config; +use crate::catalog::hang::CatalogExt; +use crate::container::Frame; /// Opus importer. /// -/// Initialized from an OpusHead packet. Each input buffer passed to [`decode`](Self::decode) -/// is published as one hang frame in its own group, so the relay can forward each frame -/// without waiting for a group boundary. Opus' packet loss concealment handles drops. -/// Ogg framing is not supported, feed raw Opus packets. -pub struct Import { - catalog: crate::catalog::Producer, +/// Publishes raw Opus frames (no Ogg framing) to a single moq track. Build it with +/// [`new`](Self::new), passing the track producer and the +/// [`catalog::Producer`](crate::catalog::Producer) it publishes its rendition into. +/// +/// Each packet handed to [`decode`](Self::decode) is published in its own group so +/// the relay can forward it immediately without waiting for a group boundary; Opus' +/// packet loss concealment handles drops. +pub struct Import { track: crate::container::Producer, - zero: Option, + rendition: crate::catalog::AudioTrack, } -impl Import { +impl Import { + /// Publish on an existing track producer, registering the rendition in `catalog`. pub fn new( - broadcast: moq_net::BroadcastProducer, - catalog: crate::catalog::Producer, - config: Config, - ) -> crate::Result { - Self::new_with_source( - crate::track_provider::TrackProvider::unique(broadcast, ".opus"), - catalog, - config, - ) - } - - pub fn new_with_track( track: moq_net::TrackProducer, - catalog: crate::catalog::Producer, + catalog: crate::catalog::Producer, config: Config, ) -> crate::Result { - Self::new_with_source(crate::track_provider::TrackProvider::fixed(track), catalog, config) - } - - fn new_with_source( - mut tracks: crate::track_provider::TrackProvider, - mut catalog: crate::catalog::Producer, - config: Config, - ) -> crate::Result { - let track = tracks.create()?; - - let mut audio_config = hang::catalog::AudioConfig::new( + let mut audio = hang::catalog::AudioConfig::new( hang::catalog::AudioCodec::Opus, config.sample_rate, config.channel_count, ); - audio_config.container = hang::catalog::Container::Legacy; + audio.container = hang::catalog::Container::Legacy; - tracing::debug!(name = ?track.name(), config = ?audio_config, "starting track"); - catalog - .lock() - .audio - .renditions - .insert(track.name().to_string(), audio_config); + tracing::debug!(name = ?track.name(), config = ?audio, "starting track"); + + let mut rendition = catalog.audio_track(track.name()); + rendition.set(audio); Ok(Self { - catalog, track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), - zero: None, + rendition, }) } - /// Returns a reference to the underlying track producer, e.g. for - /// monitoring subscriber state via `used()`/`unused()`. - pub fn track(&self) -> &moq_net::TrackProducer { - self.track.track() + /// A watch-only handle to this track's subscriber demand. + pub fn demand(&self) -> moq_net::TrackDemand { + self.track.track().demand() } /// Finish the track, flushing the current group. @@ -81,46 +58,16 @@ impl Import { Ok(()) } - pub fn decode(&mut self, buf: &mut T, pts: Option) -> crate::Result<()> { - let pts = self.pts(pts)?; - - // Collect the input into a contiguous Bytes payload. - let mut payload = BytesMut::with_capacity(buf.remaining()); - while buf.has_remaining() { - let chunk = buf.chunk(); - payload.extend_from_slice(chunk); - let len = chunk.len(); - buf.advance(len); - } - - // Each frame is its own group so the relay can forward it immediately. - // Opus' packet loss concealment handles drops. - let frame = crate::container::Frame { - timestamp: pts, - payload: payload.freeze(), + /// Publish one Opus packet as its own group, stamping `pts` or a wall clock when absent. + pub fn decode(&mut self, frame: &[u8], pts: Option) -> crate::Result<()> { + let timestamp = self.rendition.timestamp(pts)?; + self.track.write(Frame { + timestamp, + payload: bytes::Bytes::copy_from_slice(frame), keyframe: true, duration: None, - }; - - self.track.write(frame)?; + })?; self.track.finish_group()?; - Ok(()) } - - fn pts(&mut self, hint: Option) -> crate::Result { - if let Some(pts) = hint { - return Ok(pts); - } - - let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); - Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) - } -} - -impl Drop for Import { - fn drop(&mut self) { - tracing::debug!(name = ?self.track.name(), "ending track"); - self.catalog.lock().audio.renditions.remove(self.track.name()); - } } diff --git a/rs/moq-mux/src/codec/vp8/import.rs b/rs/moq-mux/src/codec/vp8/import.rs index 67bdb698e..89e9be5b0 100644 --- a/rs/moq-mux/src/codec/vp8/import.rs +++ b/rs/moq-mux/src/codec/vp8/import.rs @@ -1,6 +1,7 @@ -use anyhow::Context; -use bytes::Buf; +use bytes::Bytes; +use crate::catalog::hang::CatalogExt; +use crate::container::Frame; use crate::container::jitter::MinFrameDuration; use super::FrameHeader; @@ -8,66 +9,50 @@ use super::FrameHeader; /// A frame-based importer for raw VP8. /// /// A VP8 elementary stream isn't self-delimiting, so the caller must pass whole -/// frames, one per [`decode_frame`](Self::decode_frame). The first key frame's -/// header supplies the catalog dimensions; the track is created lazily so the -/// importer can be constructed before any media arrives. -pub struct Import { - // Where new media tracks come from. - tracks: crate::track_provider::TrackProvider, +/// frames, one per [`decode`](Self::decode). The first key frame's header supplies +/// the catalog dimensions, so the rendition isn't published until then. Build it +/// with [`new`](Self::new), passing the track producer and the +/// [`catalog::Producer`](crate::catalog::Producer) it publishes into. +pub struct Import { + // The track being produced. + track: crate::container::Producer, - // The catalog being produced. - catalog: crate::catalog::Producer, - - // The track being produced, created on the first key frame. - track: Option>, + // This importer's catalog rendition, published on the first key frame. + rendition: crate::catalog::VideoTrack, // The resolved config, used to detect resolution changes. config: Option, - // Used to compute wall clock timestamps when the caller has none. - zero: Option, - // Tracks the minimum frame duration and updates the catalog `jitter` field. jitter: MinFrameDuration, } -impl Import { - pub fn new(broadcast: moq_net::BroadcastProducer, catalog: crate::catalog::Producer) -> Self { - Self { - tracks: crate::track_provider::TrackProvider::unique(broadcast, ".vp8"), - catalog, - track: None, - config: None, - zero: None, - jitter: MinFrameDuration::new(), - } - } - - pub fn new_with_track(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { +impl Import { + /// Publish on an existing track producer, registering the rendition in `catalog`. + pub fn new(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { + let rendition = catalog.video_track(track.name()); Self { - tracks: crate::track_provider::TrackProvider::fixed(track), - catalog, - track: None, + track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), + rendition, config: None, - zero: None, jitter: MinFrameDuration::new(), } } /// Initialize the importer. /// - /// VP8 has no out-of-band configuration record, so this is normally called - /// with an empty buffer (gstreamer / ffi pass `Bytes::new()`) and the track - /// is created lazily from the first key frame. If the caller does pass the - /// first frame here, it's decoded so nothing is dropped. - pub fn initialize>(&mut self, buf: &mut T) -> anyhow::Result<()> { - if buf.has_remaining() { - self.decode_frame(buf, None)?; + /// VP8 has no out-of-band configuration record, so this is normally called with + /// an empty slice (gstreamer / ffi pass `&[]`) and the catalog is filled from the + /// first key frame. If the caller does pass the first frame here, it's decoded so + /// nothing is dropped. + pub fn initialize(&mut self, buf: &[u8]) -> crate::Result<()> { + if !buf.is_empty() { + self.decode(buf, None)?; } Ok(()) } - fn init(&mut self, width: u16, height: u16) -> anyhow::Result<()> { + fn init(&mut self, width: u16, height: u16) -> crate::Result<()> { let mut config = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); config.coded_width = Some(width as u32); config.coded_height = Some(height as u32); @@ -77,109 +62,60 @@ impl Import { return Ok(()); } - if self.track.is_some() && self.tracks.is_fixed() { - anyhow::bail!("fixed track cannot be reconfigured"); - } - - if let Some(track) = self.track.take() { - tracing::debug!(name = ?track.name(), "reinitializing track"); - self.catalog.lock().video.renditions.remove(track.name()); - } - - let track = self.tracks.create()?; - tracing::debug!(name = ?track.name(), ?config, "starting track"); - self.catalog - .lock() - .video - .renditions - .insert(track.name().to_string(), config.clone()); - + tracing::debug!(name = ?self.track.name(), ?config, "starting track"); + self.rendition.set(config.clone()); self.config = Some(config); - self.track = Some(crate::container::Producer::new( - track, - crate::catalog::hang::Container::Legacy, - )); Ok(()) } /// Decode a single VP8 frame. - pub fn decode_frame>( - &mut self, - buf: &mut T, - pts: Option, - ) -> anyhow::Result<()> { - let payload = buf.copy_to_bytes(buf.remaining()); - anyhow::ensure!(!payload.is_empty(), "empty VP8 frame"); + pub fn decode(&mut self, frame: &[u8], pts: Option) -> crate::Result<()> { + if frame.is_empty() { + return Err(super::Error::EmptyFrame.into()); + } + let payload = Bytes::copy_from_slice(frame); let header = FrameHeader::parse(&payload)?; if let Some((width, height)) = header.dimensions { self.init(width, height)?; } - // Resolve the timestamp before borrowing `track` so `pts` doesn't hold a - // `&mut self` across the track write. - let pts = self.pts(pts)?; - let track = self - .track - .as_mut() - .context("expected a VP8 key frame before any interframe")?; - - track.write(crate::container::Frame { + let pts = self.rendition.timestamp(pts)?; + self.track.write(Frame { timestamp: pts, payload, keyframe: header.keyframe, duration: None, })?; - if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.lock().video.renditions.get_mut(track.name()) - { - c.jitter = Some(jitter); + if let Some(jitter) = self.jitter.observe(pts) { + self.rendition.update(|c| c.jitter = Some(jitter)); } Ok(()) } - /// Returns a reference to the underlying track producer. - pub fn track(&self) -> anyhow::Result<&moq_net::TrackProducer> { - Ok(self.track.as_ref().context("not initialized")?.track()) + /// A watch-only handle to this track's subscriber demand. + pub fn demand(&self) -> moq_net::TrackDemand { + self.track.track().demand() } /// Finish the track, flushing the current group. - pub fn finish(&mut self) -> anyhow::Result<()> { - let track = self.track.as_mut().context("not initialized")?; - track.finish()?; + pub fn finish(&mut self) -> crate::Result<()> { + self.track.finish()?; Ok(()) } /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> anyhow::Result<()> { - let track = self.track.as_mut().context("not initialized")?; - track.seek(sequence)?; + pub fn seek(&mut self, sequence: u64) -> crate::Result<()> { + self.track.seek(sequence)?; Ok(()) } + /// True once the first key frame has populated the catalog. pub fn is_initialized(&self) -> bool { - self.track.is_some() - } - - fn pts(&mut self, hint: Option) -> anyhow::Result { - if let Some(pts) = hint { - return Ok(pts); - } - - let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); - Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) - } -} - -impl Drop for Import { - fn drop(&mut self) { - if let Some(track) = self.track.take() { - tracing::debug!(name = ?track.name(), "ending track"); - self.catalog.lock().video.renditions.remove(track.name()); - } + self.config.is_some() } } @@ -189,26 +125,38 @@ mod tests { use moq_net::Timestamp; + fn setup() -> (moq_net::TrackProducer, crate::catalog::Producer) { + let mut broadcast = moq_net::BroadcastInfo::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); + let track = broadcast + .create_track( + "0.vp8", + moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), + ) + .unwrap(); + (track, catalog) + } + /// A 320x240 key frame followed by an interframe should create a single VP8 /// rendition with the right dimensions and emit both frames. #[tokio::test(start_paused = true)] async fn imports_keyframe_then_interframe() { - let mut broadcast = moq_net::BroadcastInfo::new().produce(); - let mut catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); - let mut import = super::Import::new(broadcast.clone(), catalog.clone()); + let (track, catalog) = setup(); + let mut import = super::Import::new(track, catalog.clone()); - // Empty init buffer: the track is created lazily on the first key frame. - import.initialize(&mut Bytes::new()).unwrap(); + // Empty init buffer: the catalog is filled on the first key frame. + import.initialize(&[]).unwrap(); assert!(!import.is_initialized()); + assert!(catalog.snapshot().video.renditions.is_empty()); let keyframe = Bytes::from_static(&[0x10, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x40, 0x01, 0xf0, 0x00]); import - .decode_frame(&mut keyframe.clone(), Some(Timestamp::from_micros(0).unwrap())) + .decode(&keyframe, Some(Timestamp::from_micros(0).unwrap())) .unwrap(); assert!(import.is_initialized()); - let name = import.track().unwrap().name().to_string(); - let config = catalog.lock().video.renditions.get(&name).cloned().unwrap(); + let snapshot = catalog.snapshot(); + let config = snapshot.video.renditions.get("0.vp8").unwrap(); assert_eq!(config.codec, hang::catalog::VideoCodec::VP8); assert_eq!(config.coded_width, Some(320)); assert_eq!(config.coded_height, Some(240)); @@ -216,24 +164,23 @@ mod tests { // Interframe: no start code or dimensions, but still a valid frame. let interframe = Bytes::from_static(&[0x31, 0x00, 0x00, 0xaa, 0xbb]); import - .decode_frame(&mut interframe.clone(), Some(Timestamp::from_micros(33_000).unwrap())) + .decode(&interframe, Some(Timestamp::from_micros(33_000).unwrap())) .unwrap(); import.finish().unwrap(); } - /// An interframe before any key frame has no dimensions, so the track can't - /// be created and the Producer rejects a non-keyframe first frame. + /// An interframe before any key frame has no dimensions, so the Producer + /// rejects a non-keyframe as the first frame in a group. #[tokio::test(start_paused = true)] async fn rejects_interframe_first() { - let mut broadcast = moq_net::BroadcastInfo::new().produce(); - let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); - let mut import = super::Import::new(broadcast.clone(), catalog); + let (track, catalog) = setup(); + let mut import = super::Import::new(track, catalog); - let mut interframe = Bytes::from_static(&[0x31, 0x00, 0x00, 0xaa, 0xbb]); + let interframe = Bytes::from_static(&[0x31, 0x00, 0x00, 0xaa, 0xbb]); assert!( import - .decode_frame(&mut interframe, Some(Timestamp::from_micros(0).unwrap())) + .decode(&interframe, Some(Timestamp::from_micros(0).unwrap())) .is_err() ); } diff --git a/rs/moq-mux/src/codec/vp8/mod.rs b/rs/moq-mux/src/codec/vp8/mod.rs index d48e3b51e..da9559aed 100644 --- a/rs/moq-mux/src/codec/vp8/mod.rs +++ b/rs/moq-mux/src/codec/vp8/mod.rs @@ -11,6 +11,26 @@ mod import; pub use import::*; +/// VP8 parsing errors. +#[derive(Debug, Clone, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + #[error("VP8 frame too short for tag")] + FrameTooShort, + + #[error("VP8 key frame too short for header")] + KeyframeHeaderTooShort, + + #[error("VP8 key frame start code mismatch")] + StartCodeMismatch, + + #[error("empty VP8 frame")] + EmptyFrame, +} + +/// A Result type alias for VP8 parsing. +pub type Result = std::result::Result; + /// Fields parsed from a VP8 frame tag (RFC 6386 §9.1) plus, for key frames, the /// key-frame header (§19.1). #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -27,8 +47,10 @@ impl FrameHeader { /// Reads the 3-byte frame tag, and on a key frame the 7-byte header that /// follows (start code + dimensions). Interframes carry neither a start code /// nor dimensions. - pub fn parse(data: &[u8]) -> anyhow::Result { - anyhow::ensure!(data.len() >= 3, "VP8 frame too short for tag"); + pub fn parse(data: &[u8]) -> Result { + if data.len() < 3 { + return Err(Error::FrameTooShort); + } // 24-bit little-endian frame tag. Bit 0 is frame_type: 0 = key frame. let tag = u32::from(data[0]) | (u32::from(data[1]) << 8) | (u32::from(data[2]) << 16); @@ -41,11 +63,12 @@ impl FrameHeader { }); } - anyhow::ensure!(data.len() >= 10, "VP8 key frame too short for header"); - anyhow::ensure!( - data[3] == 0x9d && data[4] == 0x01 && data[5] == 0x2a, - "VP8 key frame start code mismatch" - ); + if data.len() < 10 { + return Err(Error::KeyframeHeaderTooShort); + } + if !(data[3] == 0x9d && data[4] == 0x01 && data[5] == 0x2a) { + return Err(Error::StartCodeMismatch); + } // 14-bit dimensions; the top 2 bits of each field are the scaling factor. let width = (u16::from(data[6]) | (u16::from(data[7]) << 8)) & 0x3fff; diff --git a/rs/moq-mux/src/codec/vp9/import.rs b/rs/moq-mux/src/codec/vp9/import.rs index 7272a836e..9d982ed3e 100644 --- a/rs/moq-mux/src/codec/vp9/import.rs +++ b/rs/moq-mux/src/codec/vp9/import.rs @@ -1,6 +1,7 @@ -use anyhow::Context; -use bytes::Buf; +use bytes::Bytes; +use crate::catalog::hang::CatalogExt; +use crate::container::Frame; use crate::container::jitter::MinFrameDuration; use super::FrameHeader; @@ -8,66 +9,50 @@ use super::FrameHeader; /// A frame-based importer for raw VP9. /// /// Like VP8, a VP9 elementary stream isn't self-delimiting, so the caller must -/// pass whole frames (or superframes), one per -/// [`decode_frame`](Self::decode_frame). The first key frame's header supplies -/// the catalog config; the track is created lazily. -pub struct Import { - // Where new media tracks come from. - tracks: crate::track_provider::TrackProvider, +/// pass whole frames (or superframes), one per [`decode`](Self::decode). The first +/// key frame's header supplies the catalog config, so the rendition isn't published +/// until then. Build it with [`new`](Self::new), passing the track producer and the +/// [`catalog::Producer`](crate::catalog::Producer) it publishes into. +pub struct Import { + // The track being produced. + track: crate::container::Producer, - // The catalog being produced. - catalog: crate::catalog::Producer, - - // The track being produced, created on the first key frame. - track: Option>, + // This importer's catalog rendition, published on the first key frame. + rendition: crate::catalog::VideoTrack, // The resolved config, used to detect resolution / format changes. config: Option, - // Used to compute wall clock timestamps when the caller has none. - zero: Option, - // Tracks the minimum frame duration and updates the catalog `jitter` field. jitter: MinFrameDuration, } -impl Import { - pub fn new(broadcast: moq_net::BroadcastProducer, catalog: crate::catalog::Producer) -> Self { +impl Import { + /// Publish on an existing track producer, registering the rendition in `catalog`. + pub fn new(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { + let rendition = catalog.video_track(track.name()); Self { - tracks: crate::track_provider::TrackProvider::unique(broadcast, ".vp09"), - catalog, - track: None, + track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), + rendition, config: None, - zero: None, - jitter: MinFrameDuration::new(), - } - } - - pub fn new_with_track(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { - Self { - tracks: crate::track_provider::TrackProvider::fixed(track), - catalog, - track: None, - config: None, - zero: None, jitter: MinFrameDuration::new(), } } /// Initialize the importer. /// - /// VP9 has no out-of-band configuration record, so this is normally called - /// with an empty buffer (gstreamer / ffi pass `Bytes::new()`) and the track - /// is created lazily from the first key frame. If the caller does pass the - /// first frame here, it's decoded so nothing is dropped. - pub fn initialize>(&mut self, buf: &mut T) -> anyhow::Result<()> { - if buf.has_remaining() { - self.decode_frame(buf, None)?; + /// VP9 has no out-of-band configuration record, so this is normally called with + /// an empty slice (gstreamer / ffi pass `&[]`) and the catalog is filled from the + /// first key frame. If the caller does pass the first frame here, it's decoded so + /// nothing is dropped. + pub fn initialize(&mut self, buf: &[u8]) -> crate::Result<()> { + if !buf.is_empty() { + self.decode(buf, None)?; } Ok(()) } - fn init(&mut self, vp9: hang::catalog::VP9, width: u16, height: u16) -> anyhow::Result<()> { + fn init(&mut self, vp9: hang::catalog::VP9, width: u16, height: u16) -> crate::Result<()> { let mut config = hang::catalog::VideoConfig::new(vp9); config.coded_width = Some(width as u32); config.coded_height = Some(height as u32); @@ -77,109 +62,60 @@ impl Import { return Ok(()); } - if self.track.is_some() && self.tracks.is_fixed() { - anyhow::bail!("fixed track cannot be reconfigured"); - } - - if let Some(track) = self.track.take() { - tracing::debug!(name = ?track.name(), "reinitializing track"); - self.catalog.lock().video.renditions.remove(track.name()); - } - - let track = self.tracks.create()?; - tracing::debug!(name = ?track.name(), ?config, "starting track"); - self.catalog - .lock() - .video - .renditions - .insert(track.name().to_string(), config.clone()); - + tracing::debug!(name = ?self.track.name(), ?config, "starting track"); + self.rendition.set(config.clone()); self.config = Some(config); - self.track = Some(crate::container::Producer::new( - track, - crate::catalog::hang::Container::Legacy, - )); Ok(()) } /// Decode a single VP9 frame (or superframe). - pub fn decode_frame>( - &mut self, - buf: &mut T, - pts: Option, - ) -> anyhow::Result<()> { - let payload = buf.copy_to_bytes(buf.remaining()); - anyhow::ensure!(!payload.is_empty(), "empty VP9 frame"); + pub fn decode(&mut self, frame: &[u8], pts: Option) -> crate::Result<()> { + if frame.is_empty() { + return Err(super::Error::EmptyFrame.into()); + } + let payload = Bytes::copy_from_slice(frame); let header = FrameHeader::parse(&payload)?; if let Some(key) = header.key { self.init(key.to_catalog(), key.width, key.height)?; } - // Resolve the timestamp before borrowing `track` so `pts` doesn't hold a - // `&mut self` across the track write. - let pts = self.pts(pts)?; - let track = self - .track - .as_mut() - .context("expected a VP9 key frame before any interframe")?; - - track.write(crate::container::Frame { + let pts = self.rendition.timestamp(pts)?; + self.track.write(Frame { timestamp: pts, payload, keyframe: header.keyframe, duration: None, })?; - if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.lock().video.renditions.get_mut(track.name()) - { - c.jitter = Some(jitter); + if let Some(jitter) = self.jitter.observe(pts) { + self.rendition.update(|c| c.jitter = Some(jitter)); } Ok(()) } - /// Returns a reference to the underlying track producer. - pub fn track(&self) -> anyhow::Result<&moq_net::TrackProducer> { - Ok(self.track.as_ref().context("not initialized")?.track()) + /// A watch-only handle to this track's subscriber demand. + pub fn demand(&self) -> moq_net::TrackDemand { + self.track.track().demand() } /// Finish the track, flushing the current group. - pub fn finish(&mut self) -> anyhow::Result<()> { - let track = self.track.as_mut().context("not initialized")?; - track.finish()?; + pub fn finish(&mut self) -> crate::Result<()> { + self.track.finish()?; Ok(()) } /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> anyhow::Result<()> { - let track = self.track.as_mut().context("not initialized")?; - track.seek(sequence)?; + pub fn seek(&mut self, sequence: u64) -> crate::Result<()> { + self.track.seek(sequence)?; Ok(()) } + /// True once the first key frame has populated the catalog. pub fn is_initialized(&self) -> bool { - self.track.is_some() - } - - fn pts(&mut self, hint: Option) -> anyhow::Result { - if let Some(pts) = hint { - return Ok(pts); - } - - let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); - Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) - } -} - -impl Drop for Import { - fn drop(&mut self) { - if let Some(track) = self.track.take() { - tracing::debug!(name = ?track.name(), "ending track"); - self.catalog.lock().video.renditions.remove(track.name()); - } + self.config.is_some() } } @@ -192,35 +128,41 @@ mod tests { // profile 0, 8-bit, CS_BT_601, studio range, 4:2:0, 320x240. const KEYFRAME: &[u8] = &[0x82, 0x49, 0x83, 0x42, 0x20, 0x13, 0xf0, 0x0e, 0xf0, 0x00]; + fn setup() -> (moq_net::TrackProducer, crate::catalog::Producer) { + let mut broadcast = moq_net::BroadcastInfo::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); + let track = broadcast + .create_track( + "0.vp9", + moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), + ) + .unwrap(); + (track, catalog) + } + #[tokio::test(start_paused = true)] async fn imports_keyframe_then_interframe() { - let mut broadcast = moq_net::BroadcastInfo::new().produce(); - let mut catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); - let mut import = super::Import::new(broadcast.clone(), catalog.clone()); + let (track, catalog) = setup(); + let mut import = super::Import::new(track, catalog.clone()); - import.initialize(&mut Bytes::new()).unwrap(); + import.initialize(&[]).unwrap(); assert!(!import.is_initialized()); + assert!(catalog.snapshot().video.renditions.is_empty()); import - .decode_frame( - &mut Bytes::from_static(KEYFRAME), - Some(Timestamp::from_micros(0).unwrap()), - ) + .decode(KEYFRAME, Some(Timestamp::from_micros(0).unwrap())) .unwrap(); assert!(import.is_initialized()); - let name = import.track().unwrap().name().to_string(); - let config = catalog.lock().video.renditions.get(&name).cloned().unwrap(); + let snapshot = catalog.snapshot(); + let config = snapshot.video.renditions.get("0.vp9").unwrap(); assert!(matches!(config.codec, hang::catalog::VideoCodec::VP9(_))); assert_eq!(config.coded_width, Some(320)); assert_eq!(config.coded_height, Some(240)); // Interframe: marker(10) profile(00) show_existing(0) frame_type(1) = 0x84. import - .decode_frame( - &mut Bytes::from_static(&[0x84, 0x00, 0x00]), - Some(Timestamp::from_micros(33_000).unwrap()), - ) + .decode(&[0x84, 0x00, 0x00], Some(Timestamp::from_micros(33_000).unwrap())) .unwrap(); import.finish().unwrap(); @@ -228,14 +170,13 @@ mod tests { #[tokio::test(start_paused = true)] async fn rejects_interframe_first() { - let mut broadcast = moq_net::BroadcastInfo::new().produce(); - let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); - let mut import = super::Import::new(broadcast.clone(), catalog); + let (track, catalog) = setup(); + let mut import = super::Import::new(track, catalog); - let mut interframe = Bytes::from_static(&[0x84, 0x00, 0x00]); + let interframe = Bytes::from_static(&[0x84, 0x00, 0x00]); assert!( import - .decode_frame(&mut interframe, Some(Timestamp::from_micros(0).unwrap())) + .decode(&interframe, Some(Timestamp::from_micros(0).unwrap())) .is_err() ); } diff --git a/rs/moq-mux/src/codec/vp9/mod.rs b/rs/moq-mux/src/codec/vp9/mod.rs index 6a0c8efd0..f7193fb4d 100644 --- a/rs/moq-mux/src/codec/vp9/mod.rs +++ b/rs/moq-mux/src/codec/vp9/mod.rs @@ -14,6 +14,26 @@ pub use import::*; use hang::catalog::VP9; +/// VP9 parsing errors. +#[derive(Debug, Clone, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + #[error("invalid VP9 frame marker")] + InvalidFrameMarker, + + #[error("invalid VP9 sync code")] + InvalidSyncCode, + + #[error("VP9 header truncated")] + Truncated, + + #[error("empty VP9 frame")] + EmptyFrame, +} + +/// A Result type alias for VP9 parsing. +pub type Result = std::result::Result; + /// VP9 key-frame sync code (VP9 spec §6.2, `frame_sync_code`). const SYNC_CODE: u32 = 0x49_8342; @@ -46,10 +66,12 @@ impl FrameHeader { /// /// Reads only as far as the frame size (the last field we care about); /// everything after it is left untouched. - pub fn parse(data: &[u8]) -> anyhow::Result { + pub fn parse(data: &[u8]) -> Result { let mut r = BitReader::new(data); - anyhow::ensure!(r.read(2)? == 0b10, "invalid VP9 frame marker"); + if r.read(2)? != 0b10 { + return Err(Error::InvalidFrameMarker); + } let profile_low = r.read(1)?; let profile_high = r.read(1)?; @@ -77,7 +99,9 @@ impl FrameHeader { }); } - anyhow::ensure!(r.read(24)? == SYNC_CODE, "invalid VP9 sync code"); + if r.read(24)? != SYNC_CODE { + return Err(Error::InvalidSyncCode); + } // color_config (VP9 spec §6.2.2). let bit_depth = if profile >= 2 { @@ -227,11 +251,13 @@ impl<'a> BitReader<'a> { Self { data, bit: 0 } } - fn read(&mut self, n: u32) -> anyhow::Result { + fn read(&mut self, n: u32) -> Result { let mut value = 0; for _ in 0..n { let byte = self.bit / 8; - anyhow::ensure!(byte < self.data.len(), "VP9 header truncated"); + if byte >= self.data.len() { + return Err(Error::Truncated); + } let shift = 7 - (self.bit % 8); value = (value << 1) | u32::from((self.data[byte] >> shift) & 1); self.bit += 1; @@ -239,7 +265,7 @@ impl<'a> BitReader<'a> { Ok(value) } - fn skip(&mut self, n: u32) -> anyhow::Result<()> { + fn skip(&mut self, n: u32) -> Result<()> { self.read(n).map(|_| ()) } } diff --git a/rs/moq-mux/src/container/flv/import.rs b/rs/moq-mux/src/container/flv/import.rs index 107aa73e0..c2f6956df 100644 --- a/rs/moq-mux/src/container/flv/import.rs +++ b/rs/moq-mux/src/container/flv/import.rs @@ -178,13 +178,18 @@ impl Import { // FLV stores DTS in the tag; PTS is DTS plus the composition offset. let pts_ms = (timestamp as i64) + (composition_time as i64); anyhow::ensure!(pts_ms >= 0, "negative AVC presentation timestamp"); - stream.track.write(Frame { + // Live FLV can join mid-GOP: a leading delta before the first + // keyframe has no group to anchor, so the producer reports + // MissingKeyframe and we drop it rather than abort. + match stream.track.write(Frame { timestamp: Timestamp::from_millis(pts_ms as u64)?, duration: None, payload: Bytes::copy_from_slice(data), keyframe: frame_type == FRAME_TYPE_KEY, - })?; - Ok(()) + }) { + Ok(()) | Err(crate::Error::MissingKeyframe(_)) => Ok(()), + Err(e) => Err(e.into()), + } } // AVCPacketType 2 is "end of sequence"; nothing to emit. _ => Ok(()), @@ -252,9 +257,9 @@ impl Import { .renditions .insert(net_track.name().to_string(), config); self.video = Some(Stream { - // Live FLV can join mid-GOP; tolerate leading deltas before the first keyframe. - track: crate::container::Producer::new(net_track, crate::catalog::hang::Container::Legacy) - .with_lenient_start(), + // Leading deltas before the first keyframe are skipped at the write + // site (the producer reports MissingKeyframe), so a mid-GOP join works. + track: crate::container::Producer::new(net_track, crate::catalog::hang::Container::Legacy), description: Bytes::copy_from_slice(avcc_bytes), }); Ok(()) diff --git a/rs/moq-mux/src/container/fmp4/mod.rs b/rs/moq-mux/src/container/fmp4/mod.rs index a3e2281d4..3f4fc4623 100644 --- a/rs/moq-mux/src/container/fmp4/mod.rs +++ b/rs/moq-mux/src/container/fmp4/mod.rs @@ -36,6 +36,9 @@ pub enum Error { #[error("moq: {0}")] Moq(#[from] moq_net::Error), + #[error("missing keyframe: a group must open on a keyframe")] + MissingKeyframe(#[from] crate::container::MissingKeyframe), + #[error("timestamp overflow")] TimestampOverflow(#[from] moq_net::TimeOverflow), diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index 339679834..7b66aed4f 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -70,6 +70,16 @@ pub struct Frame { pub keyframe: bool, } +/// A non-keyframe frame arrived with no open group. +/// +/// A track must open with a keyframe (and so must the frame after +/// [`finish_group`](Producer::finish_group) / [`seek`](Producer::seek)). +/// [`Producer::write`] returns this so a caller joining mid-stream can skip +/// frames until the first keyframe instead of treating it as fatal. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("missing keyframe: a group must open on a keyframe")] +pub struct MissingKeyframe; + /// Encode and decode media frames over a moq-lite group. /// /// Implementors decide how many [`Frame`]s map onto one moq-lite frame: @@ -77,8 +87,9 @@ pub struct Frame { /// pack many samples into a single moof+mdat fragment. pub trait Container { /// Container-specific error. Must be convertible from [`moq_net::Error`] - /// so the IO layer's errors propagate cleanly. - type Error: std::error::Error + Send + Sync + Unpin + From; + /// (so IO errors propagate) and [`MissingKeyframe`] (so the producer can + /// reject a group that doesn't open on a keyframe). + type Error: std::error::Error + Send + Sync + Unpin + From + From; /// Encode one or more frames into a single moq-lite frame appended to `group`. fn write(&self, group: &mut moq_net::GroupProducer, frames: &[Frame]) -> Result<(), Self::Error>; diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index 411832b27..239654a49 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -37,12 +37,6 @@ pub struct Producer { /// Sequence to use for the next group opened by [`Self::write`]. /// Set by [`Self::seek`] and consumed on the next group creation. pending_sequence: Option, - - /// When set, a non-keyframe arriving with no open group is dropped instead of - /// erroring. Lets a track join mid-stream (the leading deltas before the first - /// keyframe have no group to anchor) without a protocol violation. Off by - /// default so a producer that simply never marks keyframes still fails loudly. - lenient_start: bool, } impl Producer { @@ -55,18 +49,9 @@ impl Producer { buffer: Vec::new(), latency: std::time::Duration::ZERO, pending_sequence: None, - lenient_start: false, } } - /// Tolerate joining a stream mid-flight: drop non-keyframes that arrive before - /// the first keyframe (which has no group to anchor) instead of erroring. Use - /// for live ingest, where the input may start mid-GOP. - pub fn with_lenient_start(mut self) -> Self { - self.lenient_start = true; - self - } - /// The underlying moq-lite track producer. Read-only; mutating it directly /// would sidestep group/keyframe invariants. pub fn track(&self) -> &moq_net::TrackProducer { @@ -88,8 +73,8 @@ impl Producer { /// Write a frame to the track. /// /// A keyframe closes any open group and starts a new one. A non-keyframe extends - /// the current group; if no group is open it's a protocol violation, unless - /// [`with_lenient_start`](Self::with_lenient_start) was set (then it's dropped). + /// the current group; if no group is open it returns [`MissingKeyframe`](super::MissingKeyframe), + /// so a caller joining mid-stream can skip frames until the first keyframe. pub fn write(&mut self, frame: Frame) -> Result<(), C::Error> { // Close the current group on an explicit keyframe, passing its timestamp so // the previous group's last frame can borrow it as a duration boundary. @@ -100,12 +85,9 @@ impl Producer { // Start a new group if needed; the first frame of a group must be a keyframe. if self.group.is_none() { if !frame.keyframe { - // Mid-stream join: no group yet and this delta can't anchor one. Drop it - // (wait for the first keyframe) when lenient; otherwise it's a violation. - if self.lenient_start { - return Ok(()); - } - return Err(moq_net::Error::ProtocolViolation.into()); + // No group yet and this delta can't anchor one. The caller (e.g. a + // mid-stream join) decides whether to skip until the first keyframe. + return Err(super::MissingKeyframe.into()); } self.group = Some(match self.pending_sequence.take() { Some(sequence) => self.inner.create_group(moq_net::Group { sequence })?, @@ -286,7 +268,7 @@ mod tests { assert_eq!(collect_groups(consumer).await, vec![2, 1]); } - /// Writing a non-keyframe with no open group is a protocol violation. + /// Writing a non-keyframe with no open group returns MissingKeyframe. #[test] fn first_frame_must_be_keyframe() { let track = moq_net::TrackProducer::new( @@ -296,7 +278,7 @@ mod tests { let mut producer = Producer::new(track, Container::Legacy); let err = producer.write(frame(0, false)).unwrap_err(); - assert!(matches!(err, crate::Error::Moq(moq_net::Error::ProtocolViolation))); + assert!(matches!(err, crate::Error::MissingKeyframe(_))); } /// Drain all groups from a finished track, returning their sequence numbers. diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index ea5d0a667..f883d6eb2 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -262,17 +262,21 @@ impl Import { let stream = match stream_type { StreamType::H264 => { - let import = - h264::Import::new(self.broadcast.clone(), self.catalog.clone()).with_mode(h264::Mode::Avc3)?; + let track = crate::import::unique_track(&mut self.broadcast, ".avc3")?; Stream::H264 { - import: Box::new(import), + split: h264::Split::new(), + import: Box::new(h264::Import::new(track, self.catalog.clone())), + unwrap: PtsUnwrap::default(), + } + } + StreamType::H265 => { + let track = crate::import::unique_track(&mut self.broadcast, ".hev1")?; + Stream::H265 { + split: h265::Split::new(), + import: Box::new(h265::Import::new(track, self.catalog.clone())), unwrap: PtsUnwrap::default(), } } - StreamType::H265 => Stream::H265 { - import: Box::new(h265::Import::new(self.broadcast.clone(), self.catalog.clone())), - unwrap: PtsUnwrap::default(), - }, // Only ADTS-framed AAC (0x0F). 0x11 is LATM/LOAS, which uses a different // framing and syncword, so it falls through to the ignored arm below. StreamType::AdtsAac => Stream::Aac(Box::new(AacStream { @@ -713,10 +717,12 @@ impl ScteReassembler { /// One elementary stream's codec importer plus PTS-unwrap state. enum Stream { H264 { + split: h264::Split, import: Box>, unwrap: PtsUnwrap, }, H265 { + split: h265::Split, import: Box>, unwrap: PtsUnwrap, }, @@ -731,13 +737,23 @@ enum Stream { impl Stream { fn write(&mut self, pending: Pending, burst: Option) -> anyhow::Result<()> { match self { - Stream::H264 { import, unwrap } => { + Stream::H264 { split, import, unwrap } => { let pts = unwrap_pts(unwrap, pending.pts)?; - Ok(import.decode_frame(&mut pending.data.as_slice(), pts)?) + skip_missing_keyframe((|| { + // Each PES is one access unit, so flush to emit it immediately. + let mut frames = split.decode(&mut pending.data.as_slice(), pts)?; + frames.extend(split.flush(pts)?); + import.decode(frames) + })()) } - Stream::H265 { import, unwrap } => { + Stream::H265 { split, import, unwrap } => { let pts = unwrap_pts(unwrap, pending.pts)?; - Ok(import.decode_frame(&mut pending.data.as_slice(), pts)?) + skip_missing_keyframe((|| { + // Each PES is one access unit, so flush to emit it immediately. + let mut frames = split.decode(&mut pending.data.as_slice(), pts)?; + frames.extend(split.flush(pts)?); + import.decode(frames) + })()) } Stream::Aac(stream) => stream.write(pending, burst), Stream::Legacy(stream) => stream.write(pending), @@ -747,8 +763,14 @@ impl Stream { fn seek(&mut self, sequence: u64) -> anyhow::Result<()> { match self { - Stream::H264 { import, .. } => Ok(import.seek(sequence)?), - Stream::H265 { import, .. } => Ok(import.seek(sequence)?), + Stream::H264 { split, import, .. } => { + split.reset(); + Ok(import.seek(sequence)?) + } + Stream::H265 { split, import, .. } => { + split.reset(); + Ok(import.seek(sequence)?) + } Stream::Aac(stream) => stream.seek(sequence), Stream::Legacy(stream) => stream.seek(sequence), Stream::Clock | Stream::Ignored => Ok(()), @@ -805,12 +827,10 @@ impl AacStream { // downstream consumers that need out-of-band config (fMP4/MKV export, // WebCodecs) can configure the decoder. TS itself carries it inline. let description = config.encode(); - let import = aac::Import::new(self.broadcast.clone(), self.catalog.clone(), config)?; - let name = import.track().name().to_string(); - if let Some(rendition) = self.catalog.lock().audio.renditions.get_mut(&name) { - rendition.description = Some(description); - } - self.import.insert(import) + let track = crate::import::unique_track(&mut self.broadcast, ".aac")?; + let mut aac = aac::Import::new(track, self.catalog.clone(), config)?; + aac.update_rendition(|rendition| rendition.description = Some(description)); + self.import.insert(aac) } }; @@ -825,8 +845,7 @@ impl AacStream { other => other, }; - let mut raw = &data[offset + header.header_len..end]; - import.decode(&mut raw, pts)?; + import.decode(&data[offset + header.header_len..end], pts)?; offset = end; index += 1; @@ -868,11 +887,8 @@ impl AacStream { } self.jitter = Some(jitter); - if let Some(import) = &self.import { - let name = import.track().name().to_string(); - if let Some(rendition) = self.catalog.lock().audio.renditions.get_mut(&name) { - rendition.jitter = Some(jitter.into()); - } + if let Some(import) = &mut self.import { + import.update_rendition(|rendition| rendition.jitter = Some(jitter.into())); } Ok(()) } @@ -958,14 +974,13 @@ impl LegacyStream { sample_rate: header.sample_rate, channel_count: header.channel_count, }; - let import = - legacy::Import::new(self.descriptor, self.broadcast.clone(), self.catalog.clone(), config)?; - self.import.insert(import) + let track = crate::import::unique_track(&mut self.broadcast, self.descriptor.track_suffix)?; + let legacy = legacy::Import::new(self.descriptor, track, self.catalog.clone(), config); + self.import.insert(legacy) } }; - let mut frame = &data[offset..end]; - import.decode(&mut frame, pts)?; + import.decode(&data[offset..end], pts)?; pts = match pts { // `pts` is a 90 kHz PES PTS; rescale the sample-rate advance to match @@ -1033,6 +1048,16 @@ fn pes_data_len(header: &PesHeader, pes_packet_len: u16) -> Option { /// Convert a raw 90 kHz PTS to a microsecond [`Timestamp`], unwrapping the /// 33-bit field. Returns `None` when the PES carried no PTS (the codec layer /// then falls back to a wall-clock timestamp). +/// Swallow a [`MissingKeyframe`](crate::container::MissingKeyframe) from a video +/// decode: a TS capture can join mid-GOP, so the deltas before the first keyframe +/// have no group to anchor and are simply dropped rather than aborting the demux. +fn skip_missing_keyframe(result: crate::Result<()>) -> anyhow::Result<()> { + match result { + Ok(()) | Err(crate::Error::MissingKeyframe(_)) => Ok(()), + Err(e) => Err(e.into()), + } +} + fn unwrap_pts(unwrap: &mut PtsUnwrap, pts: Option) -> anyhow::Result> { let Some(raw) = pts else { return Ok(None); diff --git a/rs/moq-mux/src/error.rs b/rs/moq-mux/src/error.rs index 08a2324db..37f48ea63 100644 --- a/rs/moq-mux/src/error.rs +++ b/rs/moq-mux/src/error.rs @@ -59,6 +59,18 @@ pub enum Error { #[error("av1: {0}")] Av1(#[from] crate::codec::av1::Error), + /// Error parsing VP8. + #[error("vp8: {0}")] + Vp8(#[from] crate::codec::vp8::Error), + + /// Error parsing VP9. + #[error("vp9: {0}")] + Vp9(#[from] crate::codec::vp9::Error), + + /// Error parsing legacy audio (MP2 / AC-3 / E-AC-3). + #[error("legacy: {0}")] + Legacy(#[from] crate::codec::legacy::Error), + /// Timestamp overflow when converting between timescales. #[error("timestamp overflow")] TimestampOverflow(#[from] moq_net::TimeOverflow), @@ -87,6 +99,11 @@ pub enum Error { #[error("{0} can contain multiple tracks")] MultipleTracks(&'static str), + /// A non-keyframe frame was received before any keyframe opened a group. + /// A track joining mid-stream should skip frames until the first keyframe. + #[error("{0}")] + MissingKeyframe(#[from] crate::container::MissingKeyframe), + /// Error from a muxer/demuxer that reports via `anyhow` (currently MPEG-TS). /// Boxed in an `Arc` so the enum stays `Clone` (`anyhow::Error` is not). #[error("{0}")] diff --git a/rs/moq-mux/src/import.rs b/rs/moq-mux/src/import.rs deleted file mode 100644 index 1704d4e95..000000000 --- a/rs/moq-mux/src/import.rs +++ /dev/null @@ -1,686 +0,0 @@ -//! Format dispatchers for callers who only have a format string. -//! -//! [`Framed`] is the entry point when the caller already has whole -//! frames (the typical case for files and reassembled network input). -//! [`Stream`] is for raw byte streams where frame boundaries have to -//! be inferred (piped Annex-B H.264, an fMP4 reader, …). Both pick a -//! concrete importer from a [`FramedFormat`] / [`StreamFormat`] string. -//! The concrete importers themselves live with their format under -//! [`crate::container`] or [`crate::codec`]. - -use std::{fmt, str::FromStr}; - -use bytes::Buf; - -use crate::Result; - -/// The supported framed formats (known frame boundaries). -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -#[non_exhaustive] -pub enum FramedFormat { - /// H264 with AVCC framing (length-prefixed NALUs, out-of-band SPS/PPS). - Avc1, - /// H264 with Annex B framing (start code prefixed, inline SPS/PPS). - Avc3, - /// fMP4/CMAF container. - Fmp4, - /// aka H265 with inline SPS/PPS - Hev1, - /// AV1 with inline sequence headers - Av01, - /// Raw AAC frames (not ADTS). - Aac, - /// Raw Opus frames (not Ogg). - Opus, - /// Matroska / WebM container. - Mkv, - /// MPEG-TS (transport stream) container. - Ts, - // New variants go at the end: this enum has no repr, so inserting in the - // middle would shift the implicit discriminants of everything after it. - /// VP8 (one frame per buffer; not self-delimiting). - Vp8, - /// VP9 (one frame per buffer; not self-delimiting). - Vp9, - /// FLV (Flash Video / RTMP) container. - Flv, -} - -impl FromStr for FramedFormat { - type Err = crate::Error; - - fn from_str(s: &str) -> std::result::Result { - match s { - "avc1" | "avcc" => Ok(FramedFormat::Avc1), - "avc3" | "h264" => Ok(FramedFormat::Avc3), - "hev1" => Ok(FramedFormat::Hev1), - "fmp4" | "cmaf" => Ok(FramedFormat::Fmp4), - "av01" | "av1" | "av1c" | "av1C" => Ok(FramedFormat::Av01), - "aac" => Ok(FramedFormat::Aac), - "opus" => Ok(FramedFormat::Opus), - "mkv" | "webm" | "matroska" => Ok(FramedFormat::Mkv), - "ts" | "mpegts" | "mpeg2ts" | "m2ts" => Ok(FramedFormat::Ts), - "vp8" | "vp08" => Ok(FramedFormat::Vp8), - "vp9" | "vp09" => Ok(FramedFormat::Vp9), - "flv" => Ok(FramedFormat::Flv), - _ => Err(crate::Error::UnknownFormat(s.to_string())), - } - } -} - -impl fmt::Display for FramedFormat { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match *self { - FramedFormat::Avc1 => write!(f, "avc1"), - FramedFormat::Avc3 => write!(f, "avc3"), - FramedFormat::Fmp4 => write!(f, "fmp4"), - FramedFormat::Hev1 => write!(f, "hev1"), - FramedFormat::Av01 => write!(f, "av01"), - FramedFormat::Aac => write!(f, "aac"), - FramedFormat::Opus => write!(f, "opus"), - FramedFormat::Mkv => write!(f, "mkv"), - FramedFormat::Ts => write!(f, "ts"), - FramedFormat::Vp8 => write!(f, "vp8"), - FramedFormat::Vp9 => write!(f, "vp9"), - FramedFormat::Flv => write!(f, "flv"), - } - } -} - -impl From for FramedFormat { - fn from(format: StreamFormat) -> Self { - match format { - StreamFormat::Avc3 => FramedFormat::Avc3, - StreamFormat::Fmp4 => FramedFormat::Fmp4, - StreamFormat::Hev1 => FramedFormat::Hev1, - StreamFormat::Av01 => FramedFormat::Av01, - StreamFormat::Mkv => FramedFormat::Mkv, - StreamFormat::Ts => FramedFormat::Ts, - StreamFormat::Flv => FramedFormat::Flv, - } - } -} - -enum FramedKind { - /// H.264 (both avc1 and avc3 wire shapes go through this importer; mode - /// is pinned by the caller's FramedFormat choice). - H264(crate::codec::h264::Import), - // Boxed because it's a large struct and clippy complains about the size. - Fmp4(Box), - Hev1(crate::codec::h265::Import), - Av01(crate::codec::av1::Import), - Vp8(crate::codec::vp8::Import), - Vp9(crate::codec::vp9::Import), - Aac(crate::codec::aac::Import), - Opus(crate::codec::opus::Import), - // Boxed for the same reason as Fmp4. - Mkv(Box), - // Boxed for the same reason as Fmp4. - Ts(Box), - // Boxed for the same reason as Fmp4. - Flv(Box), -} - -/// An importer for formats with known frame boundaries. -/// -/// This supports all formats and should be used when the caller knows the frame boundaries. -pub struct Framed { - decoder: FramedKind, -} - -impl Framed { - /// Create a new framed importer with the given format and initialization data. - /// - /// The buffer will be fully consumed, or an error will be returned. - pub fn new>( - broadcast: moq_net::BroadcastProducer, - catalog: crate::catalog::Producer, - format: FramedFormat, - buf: &mut T, - ) -> Result { - use crate::codec::h264::Mode as H264Mode; - let decoder = match format { - FramedFormat::Avc1 => { - let mut decoder = crate::codec::h264::Import::new(broadcast, catalog).with_mode(H264Mode::Avc1)?; - decoder.initialize(buf)?; - FramedKind::H264(decoder) - } - FramedFormat::Avc3 => { - let mut decoder = crate::codec::h264::Import::new(broadcast, catalog).with_mode(H264Mode::Avc3)?; - decoder.initialize(buf)?; - FramedKind::H264(decoder) - } - FramedFormat::Fmp4 => { - let mut decoder = Box::new(crate::container::fmp4::Import::new(broadcast, catalog)); - decoder.decode(buf)?; - FramedKind::Fmp4(decoder) - } - FramedFormat::Hev1 => { - let mut decoder = crate::codec::h265::Import::new(broadcast, catalog); - decoder.initialize(buf)?; - FramedKind::Hev1(decoder) - } - FramedFormat::Av01 => { - let mut decoder = crate::codec::av1::Import::new(broadcast, catalog); - decoder.initialize(buf)?; - FramedKind::Av01(decoder) - } - FramedFormat::Vp8 => { - let mut decoder = crate::codec::vp8::Import::new(broadcast, catalog); - decoder.initialize(buf)?; - FramedKind::Vp8(decoder) - } - FramedFormat::Vp9 => { - let mut decoder = crate::codec::vp9::Import::new(broadcast, catalog); - decoder.initialize(buf)?; - FramedKind::Vp9(decoder) - } - FramedFormat::Aac => { - let config = crate::codec::aac::Config::parse(buf)?; - FramedKind::Aac(crate::codec::aac::Import::new(broadcast, catalog, config)?) - } - FramedFormat::Opus => { - let config = crate::codec::opus::Config::parse(buf)?; - FramedKind::Opus(crate::codec::opus::Import::new(broadcast, catalog, config)?) - } - FramedFormat::Mkv => { - let mut decoder = Box::new(crate::container::mkv::Import::new(broadcast, catalog)); - decoder.decode(buf)?; - FramedKind::Mkv(decoder) - } - FramedFormat::Ts => { - let mut decoder = Box::new(crate::container::ts::Import::new(broadcast, catalog)); - decoder.decode(buf)?; - FramedKind::Ts(decoder) - } - FramedFormat::Flv => { - let mut decoder = Box::new(crate::container::flv::Import::new(broadcast, catalog)); - decoder.decode(buf)?; - FramedKind::Flv(decoder) - } - }; - - if buf.has_remaining() { - return Err(crate::Error::BufferNotConsumed); - } - - Ok(Self { decoder }) - } - - /// Create a new framed importer that publishes on an existing track. - /// - /// Only single-track formats are supported. Container formats that may - /// create multiple MoQ tracks need an explicit track mapping API. - pub fn new_with_track>( - track: moq_net::TrackProducer, - catalog: crate::catalog::Producer, - format: FramedFormat, - buf: &mut T, - ) -> anyhow::Result { - use crate::codec::h264::Mode as H264Mode; - let decoder = match format { - FramedFormat::Avc1 => { - let mut decoder = - crate::codec::h264::Import::new_with_track(track, catalog).with_mode(H264Mode::Avc1)?; - decoder.initialize(buf)?; - FramedKind::H264(decoder) - } - FramedFormat::Avc3 => { - let mut decoder = - crate::codec::h264::Import::new_with_track(track, catalog).with_mode(H264Mode::Avc3)?; - decoder.initialize(buf)?; - FramedKind::H264(decoder) - } - FramedFormat::Hev1 => { - let mut decoder = crate::codec::h265::Import::new_with_track(track, catalog); - decoder.initialize(buf)?; - FramedKind::Hev1(decoder) - } - FramedFormat::Av01 => { - let mut decoder = crate::codec::av1::Import::new_with_track(track, catalog); - decoder.initialize(buf)?; - FramedKind::Av01(decoder) - } - FramedFormat::Vp8 => { - let mut decoder = crate::codec::vp8::Import::new_with_track(track, catalog); - decoder.initialize(buf)?; - FramedKind::Vp8(decoder) - } - FramedFormat::Vp9 => { - let mut decoder = crate::codec::vp9::Import::new_with_track(track, catalog); - decoder.initialize(buf)?; - FramedKind::Vp9(decoder) - } - FramedFormat::Aac => { - let config = crate::codec::aac::Config::parse(buf)?; - FramedKind::Aac(crate::codec::aac::Import::new_with_track(track, catalog, config)?) - } - FramedFormat::Opus => { - let config = crate::codec::opus::Config::parse(buf)?; - FramedKind::Opus(crate::codec::opus::Import::new_with_track(track, catalog, config)?) - } - FramedFormat::Fmp4 | FramedFormat::Mkv | FramedFormat::Ts | FramedFormat::Flv => { - anyhow::bail!("{format} can publish multiple tracks") - } - }; - - anyhow::ensure!(!buf.has_remaining(), "buffer was not fully consumed"); - - Ok(Self { decoder }) - } - - /// Finish the decoder, flushing any buffered data. - pub fn finish(&mut self) -> Result<()> { - match self.decoder { - FramedKind::H264(ref mut decoder) => decoder.finish(), - FramedKind::Fmp4(ref mut decoder) => decoder.finish(), - FramedKind::Hev1(ref mut decoder) => decoder.finish(), - FramedKind::Av01(ref mut decoder) => decoder.finish(), - FramedKind::Vp8(ref mut decoder) => decoder.finish().map_err(Into::into), - FramedKind::Vp9(ref mut decoder) => decoder.finish().map_err(Into::into), - FramedKind::Aac(ref mut decoder) => decoder.finish(), - FramedKind::Opus(ref mut decoder) => decoder.finish(), - FramedKind::Mkv(ref mut decoder) => decoder.finish(), - FramedKind::Ts(ref mut decoder) => decoder.finish().map_err(Into::into), - FramedKind::Flv(ref mut decoder) => decoder.finish().map_err(Into::into), - } - } - - /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> Result<()> { - match self.decoder { - FramedKind::H264(ref mut decoder) => decoder.seek(sequence), - FramedKind::Fmp4(ref mut decoder) => decoder.seek(sequence), - FramedKind::Hev1(ref mut decoder) => decoder.seek(sequence), - FramedKind::Av01(ref mut decoder) => decoder.seek(sequence), - FramedKind::Vp8(ref mut decoder) => decoder.seek(sequence).map_err(Into::into), - FramedKind::Vp9(ref mut decoder) => decoder.seek(sequence).map_err(Into::into), - FramedKind::Aac(ref mut decoder) => decoder.seek(sequence), - FramedKind::Opus(ref mut decoder) => decoder.seek(sequence), - FramedKind::Mkv(ref mut decoder) => decoder.seek(sequence), - FramedKind::Ts(ref mut decoder) => decoder.seek(sequence).map_err(Into::into), - FramedKind::Flv(ref mut decoder) => decoder.seek(sequence).map_err(Into::into), - } - } - - /// Return the single track produced by this importer. - pub fn track(&self) -> Result<&moq_net::TrackProducer> { - match self.decoder { - FramedKind::H264(ref decoder) => decoder - .track() - .ok_or_else(|| crate::codec::h264::Error::Avc3TrackNotCreated.into()), - FramedKind::Fmp4(_) => Err(crate::Error::MultipleTracks("fmp4")), - FramedKind::Hev1(ref decoder) => decoder.track(), - FramedKind::Av01(ref decoder) => decoder.track(), - FramedKind::Vp8(ref decoder) => decoder.track().map_err(Into::into), - FramedKind::Vp9(ref decoder) => decoder.track().map_err(Into::into), - FramedKind::Aac(ref decoder) => Ok(decoder.track()), - FramedKind::Opus(ref decoder) => Ok(decoder.track()), - FramedKind::Mkv(_) => Err(crate::Error::MultipleTracks("mkv")), - FramedKind::Ts(_) => Err(crate::Error::MultipleTracks("ts")), - FramedKind::Flv(_) => Err(crate::Error::MultipleTracks("flv")), - } - } - - /// Decode a frame from the given buffer. - pub fn decode_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - match self.decoder { - FramedKind::H264(ref mut decoder) => decoder.decode_frame(buf, pts)?, - FramedKind::Fmp4(ref mut decoder) => decoder.decode(buf)?, - FramedKind::Hev1(ref mut decoder) => decoder.decode_frame(buf, pts)?, - FramedKind::Av01(ref mut decoder) => decoder.decode_frame(buf, pts)?, - FramedKind::Vp8(ref mut decoder) => decoder.decode_frame(buf, pts)?, - FramedKind::Vp9(ref mut decoder) => decoder.decode_frame(buf, pts)?, - FramedKind::Aac(ref mut decoder) => decoder.decode(buf, pts)?, - FramedKind::Opus(ref mut decoder) => decoder.decode(buf, pts)?, - FramedKind::Mkv(ref mut decoder) => { - let _ = pts; - decoder.decode(buf)?; - } - FramedKind::Ts(ref mut decoder) => { - let _ = pts; - decoder.decode(buf)?; - } - FramedKind::Flv(ref mut decoder) => { - let _ = pts; - decoder.decode(buf)?; - } - } - - if buf.has_remaining() { - return Err(crate::Error::BufferNotConsumed); - } - - Ok(()) - } -} - -// Lift an already-built codec importer into a `Framed` so callers that build -// their config out-of-band (e.g. moq-gst, which constructs `opus::Config` from -// gstreamer caps instead of an OpusHead buffer) can keep using `.into()`. -impl From for Framed { - fn from(opus: crate::codec::opus::Import) -> Self { - Self { - decoder: FramedKind::Opus(opus), - } - } -} - -impl From for Framed { - fn from(aac: crate::codec::aac::Import) -> Self { - Self { - decoder: FramedKind::Aac(aac), - } - } -} - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use bytes::Bytes; - - use super::*; - use moq_net::Timestamp; - - fn opus_head() -> Vec { - let mut head = Vec::with_capacity(19); - head.extend_from_slice(b"OpusHead"); - head.push(1); - head.push(2); - head.extend_from_slice(&0u16.to_le_bytes()); - head.extend_from_slice(&48000u32.to_le_bytes()); - head.extend_from_slice(&0u16.to_le_bytes()); - head.push(0); - head - } - - fn h264_init() -> Vec { - let mut init = Vec::new(); - init.extend_from_slice(&[0x00, 0x00, 0x00, 0x01]); - init.extend_from_slice(&[ - 0x67, 0x64, 0x00, 0x1f, 0xac, 0x24, 0x84, 0x01, 0x40, 0x16, 0xec, 0x04, 0x40, 0x00, 0x00, 0x03, 0x00, 0x40, - 0x00, 0x00, 0x0c, 0x23, 0xc6, 0x0c, 0x92, - ]); - init.extend_from_slice(&[0x00, 0x00, 0x00, 0x01]); - init.extend_from_slice(&[0x68, 0xee, 0x32, 0xc8, 0xb0]); - init - } - - fn new_broadcast() -> (moq_net::BroadcastProducer, crate::catalog::Producer) { - let mut broadcast = moq_net::BroadcastInfo::new().produce(); - let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); - (broadcast, catalog) - } - - #[tokio::test(start_paused = true)] - async fn fixed_track_opus_uses_existing_name_and_delivers_frames() { - let (mut broadcast, catalog) = new_broadcast(); - // Legacy-container codecs write micro-timestamped frames, so a caller-supplied - // fixed track must declare the matching timescale. - let track = broadcast - .create_track( - "requested-audio", - moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), - ) - .unwrap(); - let consumer = track.subscribe(None); - let init = opus_head(); - let mut init = init.as_slice(); - - let mut framed = Framed::new_with_track(track, catalog.clone(), FramedFormat::Opus, &mut init).unwrap(); - - assert_eq!(framed.track().unwrap().name(), "requested-audio"); - let snapshot = catalog.snapshot(); - assert!(snapshot.audio.renditions.contains_key("requested-audio")); - assert!(!snapshot.audio.renditions.contains_key("0.opus")); - - let mut media = crate::container::Consumer::new(consumer, crate::catalog::hang::Container::Legacy); - let payload = b"opus payload".to_vec(); - let mut frame = payload.as_slice(); - framed - .decode_frame(&mut frame, Some(Timestamp::from_micros(1_000).unwrap())) - .unwrap(); - - let frame = tokio::time::timeout(Duration::from_secs(1), media.read()) - .await - .unwrap() - .unwrap() - .unwrap(); - assert_eq!(frame.payload, payload); - assert_eq!(frame.timestamp, Timestamp::from_micros(1_000).unwrap()); - - framed.finish().unwrap(); - } - - #[tokio::test(start_paused = true)] - async fn fixed_track_h264_uses_existing_name_in_catalog() { - let (mut broadcast, catalog) = new_broadcast(); - let track = broadcast.create_track("camera", None).unwrap(); - let init = h264_init(); - let mut init = init.as_slice(); - - let framed = Framed::new_with_track(track, catalog.clone(), FramedFormat::Avc3, &mut init).unwrap(); - - assert_eq!(framed.track().unwrap().name(), "camera"); - let snapshot = catalog.snapshot(); - let video = snapshot.video.renditions.get("camera").unwrap(); - assert_eq!(video.coded_width, Some(1280)); - assert_eq!(video.coded_height, Some(720)); - assert!(!snapshot.video.renditions.contains_key("0.avc3")); - } - - #[test] - fn fixed_track_rejects_multi_track_formats() { - for format in [FramedFormat::Fmp4, FramedFormat::Mkv, FramedFormat::Ts] { - let (mut broadcast, catalog) = new_broadcast(); - let track = broadcast.create_track("media", None).unwrap(); - let mut init = Bytes::new(); - - let err = match Framed::new_with_track(track, catalog, format, &mut init) { - Ok(_) => panic!("multi-track format should be rejected"), - Err(err) => err, - }; - assert!(err.to_string().contains("multiple tracks")); - } - } - - #[tokio::test(start_paused = true)] - async fn fixed_track_reconfiguration_errors() { - let (mut broadcast, catalog) = new_broadcast(); - let track = broadcast - .create_track( - "video", - moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), - ) - .unwrap(); - let mut init = Bytes::new(); - let mut framed = Framed::new_with_track(track, catalog, FramedFormat::Vp8, &mut init).unwrap(); - - let mut first = Bytes::from_static(&[0x10, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x40, 0x01, 0xf0, 0x00]); - framed - .decode_frame(&mut first, Some(Timestamp::from_micros(0).unwrap())) - .unwrap(); - - let mut second = Bytes::from_static(&[0x10, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x80, 0x02, 0xe0, 0x01]); - let err = framed - .decode_frame(&mut second, Some(Timestamp::from_micros(33_000).unwrap())) - .unwrap_err(); - assert!(err.to_string().contains("fixed track cannot be reconfigured")); - } -} - -// -- stream dispatcher -- - -/// Formats that support stream decoding (unknown frame boundaries). -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -#[non_exhaustive] -pub enum StreamFormat { - /// aka H264 with inline SPS/PPS - Avc3, - /// fMP4/CMAF container. - Fmp4, - /// aka H265 with inline SPS/PPS - Hev1, - /// AV1 with inline sequence headers - Av01, - /// Matroska / WebM container. - Mkv, - /// MPEG-TS (transport stream) container. - Ts, - /// FLV (Flash Video / RTMP) container. - Flv, -} - -impl FromStr for StreamFormat { - type Err = crate::Error; - - fn from_str(s: &str) -> std::result::Result { - match s { - "avc3" | "h264" => Ok(StreamFormat::Avc3), - "hev1" => Ok(StreamFormat::Hev1), - "fmp4" | "cmaf" => Ok(StreamFormat::Fmp4), - "av01" | "av1" | "av1c" | "av1C" => Ok(StreamFormat::Av01), - "mkv" | "webm" | "matroska" => Ok(StreamFormat::Mkv), - "ts" | "mpegts" | "mpeg2ts" | "m2ts" => Ok(StreamFormat::Ts), - "flv" => Ok(StreamFormat::Flv), - _ => Err(crate::Error::UnknownFormat(s.to_string())), - } - } -} - -impl fmt::Display for StreamFormat { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match *self { - StreamFormat::Avc3 => write!(f, "avc3"), - StreamFormat::Fmp4 => write!(f, "fmp4"), - StreamFormat::Hev1 => write!(f, "hev1"), - StreamFormat::Av01 => write!(f, "av01"), - StreamFormat::Mkv => write!(f, "mkv"), - StreamFormat::Ts => write!(f, "ts"), - StreamFormat::Flv => write!(f, "flv"), - } - } -} - -enum StreamKind { - /// H.264 in avc3 wire shape (Annex-B with inline SPS/PPS). - Avc3(crate::codec::h264::Import), - // Boxed because it's a large struct and clippy complains about the size. - Fmp4(Box), - Hev1(crate::codec::h265::Import), - Av01(crate::codec::av1::Import), - // Boxed for the same reason as Fmp4. - Mkv(Box), - // Boxed for the same reason as Fmp4. - Ts(Box), - // Boxed for the same reason as Fmp4. - Flv(Box), -} - -/// An importer for formats that support stream decoding (unknown frame boundaries). -/// -/// This includes formats like H.264 (AVC3), H.265 (HEV1), and fMP4/CMAF. -/// Use this when the caller does not know the frame boundaries. -pub struct Stream { - decoder: StreamKind, -} - -impl Stream { - /// Create a new stream importer with the given format. - pub fn new( - broadcast: moq_net::BroadcastProducer, - catalog: crate::catalog::Producer, - format: StreamFormat, - ) -> Result { - use crate::codec::h264::Mode as H264Mode; - let decoder = match format { - StreamFormat::Avc3 => { - StreamKind::Avc3(crate::codec::h264::Import::new(broadcast, catalog).with_mode(H264Mode::Avc3)?) - } - StreamFormat::Fmp4 => StreamKind::Fmp4(Box::new(crate::container::fmp4::Import::new(broadcast, catalog))), - StreamFormat::Hev1 => StreamKind::Hev1(crate::codec::h265::Import::new(broadcast, catalog)), - StreamFormat::Av01 => StreamKind::Av01(crate::codec::av1::Import::new(broadcast, catalog)), - StreamFormat::Mkv => StreamKind::Mkv(Box::new(crate::container::mkv::Import::new(broadcast, catalog))), - StreamFormat::Ts => StreamKind::Ts(Box::new(crate::container::ts::Import::new(broadcast, catalog))), - StreamFormat::Flv => StreamKind::Flv(Box::new(crate::container::flv::Import::new(broadcast, catalog))), - }; - - Ok(Self { decoder }) - } - - /// Initialize the decoder with the given buffer and populate the broadcast. - /// - /// This is not required for self-describing formats like fMP4 or AVC3. - /// - /// The buffer will be fully consumed, or an error will be returned. - pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { - match self.decoder { - StreamKind::Avc3(ref mut decoder) => decoder.initialize(buf)?, - StreamKind::Fmp4(ref mut decoder) => decoder.decode(buf)?, - StreamKind::Hev1(ref mut decoder) => decoder.initialize(buf)?, - StreamKind::Av01(ref mut decoder) => decoder.initialize(buf)?, - StreamKind::Mkv(ref mut decoder) => decoder.decode(buf)?, - StreamKind::Ts(ref mut decoder) => decoder.decode(buf)?, - StreamKind::Flv(ref mut decoder) => decoder.decode(buf)?, - } - - if buf.has_remaining() { - return Err(crate::Error::BufferNotConsumed); - } - - Ok(()) - } - - /// Decode a stream of data from the given buffer. - pub fn decode_stream>(&mut self, buf: &mut T) -> Result<()> { - match self.decoder { - StreamKind::Avc3(ref mut decoder) => decoder.decode_stream(buf, None), - StreamKind::Fmp4(ref mut decoder) => decoder.decode(buf), - StreamKind::Hev1(ref mut decoder) => decoder.decode_stream(buf, None), - StreamKind::Av01(ref mut decoder) => decoder.decode_stream(buf, None), - StreamKind::Mkv(ref mut decoder) => decoder.decode(buf), - StreamKind::Ts(ref mut decoder) => decoder.decode(buf).map_err(Into::into), - StreamKind::Flv(ref mut decoder) => decoder.decode(buf).map_err(Into::into), - } - } - - /// Finish the decoder, flushing any buffered data. - pub fn finish(&mut self) -> Result<()> { - match self.decoder { - StreamKind::Avc3(ref mut decoder) => decoder.finish(), - StreamKind::Fmp4(ref mut decoder) => decoder.finish(), - StreamKind::Hev1(ref mut decoder) => decoder.finish(), - StreamKind::Av01(ref mut decoder) => decoder.finish(), - StreamKind::Mkv(ref mut decoder) => decoder.finish(), - StreamKind::Ts(ref mut decoder) => decoder.finish().map_err(Into::into), - StreamKind::Flv(ref mut decoder) => decoder.finish().map_err(Into::into), - } - } - - /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> Result<()> { - match self.decoder { - StreamKind::Avc3(ref mut decoder) => decoder.seek(sequence), - StreamKind::Fmp4(ref mut decoder) => decoder.seek(sequence), - StreamKind::Hev1(ref mut decoder) => decoder.seek(sequence), - StreamKind::Av01(ref mut decoder) => decoder.seek(sequence), - StreamKind::Mkv(ref mut decoder) => decoder.seek(sequence), - StreamKind::Ts(ref mut decoder) => decoder.seek(sequence).map_err(Into::into), - StreamKind::Flv(ref mut decoder) => decoder.seek(sequence).map_err(Into::into), - } - } - - /// Check if the decoder has read enough data to be initialized. - pub fn is_initialized(&self) -> bool { - match self.decoder { - StreamKind::Avc3(ref decoder) => decoder.is_initialized(), - StreamKind::Fmp4(ref decoder) => decoder.is_initialized(), - StreamKind::Hev1(ref decoder) => decoder.is_initialized(), - StreamKind::Av01(ref decoder) => decoder.is_initialized(), - StreamKind::Mkv(ref decoder) => decoder.is_initialized(), - StreamKind::Ts(ref decoder) => decoder.is_initialized(), - StreamKind::Flv(ref decoder) => decoder.is_initialized(), - } - } -} diff --git a/rs/moq-mux/src/import/mod.rs b/rs/moq-mux/src/import/mod.rs new file mode 100644 index 000000000..6db3057c6 --- /dev/null +++ b/rs/moq-mux/src/import/mod.rs @@ -0,0 +1,987 @@ +//! Import media into a moq broadcast. +//! +//! The format dispatchers are the front door for callers who only have a format +//! string. [`Framed`] is the entry point when the caller already has whole frames +//! (the typical case for files and reassembled network input). [`Stream`] is for +//! raw byte streams where frame boundaries have to be inferred (piped Annex-B +//! H.264, an fMP4 reader, …). Both pick a concrete importer from a +//! [`FramedFormat`] / [`StreamFormat`] string. The concrete importers live with +//! their format under [`crate::container`] or [`crate::codec`] and publish their +//! own catalog rendition (see [`crate::catalog::VideoTrack`] / +//! [`crate::catalog::AudioTrack`]). +//! +//! [`unique_track`] mints a track for the single-codec importers. + +mod track; +pub use track::*; + +use std::{fmt, str::FromStr}; + +use bytes::Buf; + +use crate::Result; + +/// The supported framed formats (known frame boundaries). +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum FramedFormat { + /// H264 with AVCC framing (length-prefixed NALUs, out-of-band SPS/PPS). + Avc1, + /// H264 with Annex B framing (start code prefixed, inline SPS/PPS). + Avc3, + /// fMP4/CMAF container. + Fmp4, + /// aka H265 with inline SPS/PPS + Hev1, + /// AV1 with inline sequence headers + Av01, + /// Raw AAC frames (not ADTS). + Aac, + /// Raw Opus frames (not Ogg). + Opus, + /// Matroska / WebM container. + Mkv, + /// MPEG-TS (transport stream) container. + Ts, + // New variants go at the end: this enum has no repr, so inserting in the + // middle would shift the implicit discriminants of everything after it. + /// VP8 (one frame per buffer; not self-delimiting). + Vp8, + /// VP9 (one frame per buffer; not self-delimiting). + Vp9, + /// FLV (Flash Video / RTMP) container. + Flv, +} + +impl FromStr for FramedFormat { + type Err = crate::Error; + + fn from_str(s: &str) -> std::result::Result { + match s { + "avc1" | "avcc" => Ok(FramedFormat::Avc1), + "avc3" | "h264" => Ok(FramedFormat::Avc3), + "hev1" => Ok(FramedFormat::Hev1), + "fmp4" | "cmaf" => Ok(FramedFormat::Fmp4), + "av01" | "av1" | "av1c" | "av1C" => Ok(FramedFormat::Av01), + "aac" => Ok(FramedFormat::Aac), + "opus" => Ok(FramedFormat::Opus), + "mkv" | "webm" | "matroska" => Ok(FramedFormat::Mkv), + "ts" | "mpegts" | "mpeg2ts" | "m2ts" => Ok(FramedFormat::Ts), + "vp8" | "vp08" => Ok(FramedFormat::Vp8), + "vp9" | "vp09" => Ok(FramedFormat::Vp9), + "flv" => Ok(FramedFormat::Flv), + _ => Err(crate::Error::UnknownFormat(s.to_string())), + } + } +} + +impl fmt::Display for FramedFormat { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + FramedFormat::Avc1 => write!(f, "avc1"), + FramedFormat::Avc3 => write!(f, "avc3"), + FramedFormat::Fmp4 => write!(f, "fmp4"), + FramedFormat::Hev1 => write!(f, "hev1"), + FramedFormat::Av01 => write!(f, "av01"), + FramedFormat::Aac => write!(f, "aac"), + FramedFormat::Opus => write!(f, "opus"), + FramedFormat::Mkv => write!(f, "mkv"), + FramedFormat::Ts => write!(f, "ts"), + FramedFormat::Vp8 => write!(f, "vp8"), + FramedFormat::Vp9 => write!(f, "vp9"), + FramedFormat::Flv => write!(f, "flv"), + } + } +} + +impl From for FramedFormat { + fn from(format: StreamFormat) -> Self { + match format { + StreamFormat::Avc3 => FramedFormat::Avc3, + StreamFormat::Fmp4 => FramedFormat::Fmp4, + StreamFormat::Hev1 => FramedFormat::Hev1, + StreamFormat::Av01 => FramedFormat::Av01, + StreamFormat::Mkv => FramedFormat::Mkv, + StreamFormat::Ts => FramedFormat::Ts, + StreamFormat::Flv => FramedFormat::Flv, + } + } +} + +enum FramedKind { + /// H.264 avc3 (Annex-B, inline SPS/PPS). The split owns byte parsing; the + /// import publishes. + Avc3 { + split: crate::codec::h264::Split, + import: crate::codec::h264::Import, + }, + /// H.264 avc1 (length-prefixed NALU, out-of-band avcC). No splitter: each + /// access unit is wrapped directly. `length_size` is the NALU length prefix + /// width read from the avcC. + Avc1 { + length_size: usize, + import: crate::codec::h264::Import, + }, + // Boxed because it's a large struct and clippy complains about the size. + Fmp4(Box), + Hev1 { + split: crate::codec::h265::Split, + import: crate::codec::h265::Import, + }, + Av01 { + split: crate::codec::av1::Split, + import: crate::codec::av1::Import, + }, + Vp8(crate::codec::vp8::Import), + Vp9(crate::codec::vp9::Import), + Aac(crate::codec::aac::Import), + Opus(crate::codec::opus::Import), + // Boxed for the same reason as Fmp4. + Mkv(Box), + // Boxed for the same reason as Fmp4. + Ts(Box), + // Boxed for the same reason as Fmp4. + Flv(Box), +} + +/// An importer for formats with known frame boundaries. +/// +/// This supports all formats and should be used when the caller knows the frame boundaries. +pub struct Framed { + decoder: FramedKind, +} + +/// Build an H.264 avc3 split + import pair, resolving the config from `init`. +/// +/// The import reads `init` for the codec config; the split then reads it as the +/// leading bytes of the stream (caching any inline SPS/PPS). Any frames in the +/// init buffer are published. +fn build_h264_avc3( + track: moq_net::TrackProducer, + catalog: crate::catalog::Producer, + init: &[u8], +) -> Result<(crate::codec::h264::Split, crate::codec::h264::Import)> { + let mut import = crate::codec::h264::Import::new(track, catalog); + import.initialize(init)?; + let mut split = crate::codec::h264::Split::new(); + let mut data = init; + let frames = split.decode(&mut data, None)?; + import.decode(frames)?; + Ok((split, import)) +} + +/// Build an H.264 avc1 import, resolving the config and the NALU length size from +/// the avcC. avc1 has no splitter: each access unit is wrapped directly via +/// [`crate::codec::h264::avc1_frame`]. +fn build_h264_avc1( + track: moq_net::TrackProducer, + catalog: crate::catalog::Producer, + init: &[u8], +) -> Result<(usize, crate::codec::h264::Import)> { + let mut import = crate::codec::h264::Import::new(track, catalog); + import.initialize(init)?; + let length_size = crate::codec::h264::Avcc::parse(init)?.length_size; + Ok((length_size, import)) +} + +/// Build an H.265 split + import pair, resolving the config from `init`. +fn build_h265( + track: moq_net::TrackProducer, + catalog: crate::catalog::Producer, + init: &[u8], +) -> Result<(crate::codec::h265::Split, crate::codec::h265::Import)> { + let mut import = crate::codec::h265::Import::new(track, catalog); + import.initialize(init)?; + let mut split = crate::codec::h265::Split::new(); + let mut data = init; + let frames = split.decode(&mut data, None)?; + import.decode(frames)?; + Ok((split, import)) +} + +/// Build an AV1 split + import pair, resolving the config from `init`. +fn build_av1( + track: moq_net::TrackProducer, + catalog: crate::catalog::Producer, + init: &[u8], +) -> Result<(crate::codec::av1::Split, crate::codec::av1::Import)> { + let mut import = crate::codec::av1::Import::new(track, catalog); + import.initialize(init)?; + let mut split = crate::codec::av1::Split::new(); + // av1C (leading 0x81, ISO/IEC 14496-15) is an out-of-band config record, not an + // OBU stream, so it's read for config (above) and dropped here. Raw OBUs are the + // leading bytes of the stream and feed the splitter. + let frames = if init.len() >= 16 && init[0] == 0x81 { + Vec::new() + } else { + let mut data = init; + split.decode(&mut data, None)? + }; + import.decode(frames)?; + Ok((split, import)) +} + +impl Framed { + /// Create a new framed importer with the given format and initialization data. + pub fn new( + mut broadcast: moq_net::BroadcastProducer, + catalog: crate::catalog::Producer, + format: FramedFormat, + init: &[u8], + ) -> Result { + let decoder = match format { + FramedFormat::Avc1 => { + let track = crate::import::unique_track(&mut broadcast, ".avc1")?; + let (length_size, import) = build_h264_avc1(track, catalog, init)?; + FramedKind::Avc1 { length_size, import } + } + FramedFormat::Avc3 => { + let track = crate::import::unique_track(&mut broadcast, ".avc3")?; + let (split, import) = build_h264_avc3(track, catalog, init)?; + FramedKind::Avc3 { split, import } + } + FramedFormat::Fmp4 => { + let mut decoder = Box::new(crate::container::fmp4::Import::new(broadcast, catalog)); + decoder.decode(&mut { init })?; + FramedKind::Fmp4(decoder) + } + FramedFormat::Hev1 => { + let track = crate::import::unique_track(&mut broadcast, ".hev1")?; + let (split, import) = build_h265(track, catalog, init)?; + FramedKind::Hev1 { split, import } + } + FramedFormat::Av01 => { + let track = crate::import::unique_track(&mut broadcast, ".av01")?; + let (split, import) = build_av1(track, catalog, init)?; + FramedKind::Av01 { split, import } + } + FramedFormat::Vp8 => { + let track = crate::import::unique_track(&mut broadcast, ".vp8")?; + let mut import = crate::codec::vp8::Import::new(track, catalog); + import.initialize(init)?; + FramedKind::Vp8(import) + } + FramedFormat::Vp9 => { + let track = crate::import::unique_track(&mut broadcast, ".vp09")?; + let mut import = crate::codec::vp9::Import::new(track, catalog); + import.initialize(init)?; + FramedKind::Vp9(import) + } + FramedFormat::Aac => { + let mut data = init; + let config = crate::codec::aac::Config::parse(&mut data)?; + let track = crate::import::unique_track(&mut broadcast, ".aac")?; + let import = crate::codec::aac::Import::new(track, catalog, config)?; + FramedKind::Aac(import) + } + FramedFormat::Opus => { + let mut data = init; + let config = crate::codec::opus::Config::parse(&mut data)?; + let track = crate::import::unique_track(&mut broadcast, ".opus")?; + let import = crate::codec::opus::Import::new(track, catalog, config)?; + FramedKind::Opus(import) + } + FramedFormat::Mkv => { + let mut decoder = Box::new(crate::container::mkv::Import::new(broadcast, catalog)); + decoder.decode(&mut { init })?; + FramedKind::Mkv(decoder) + } + FramedFormat::Ts => { + let mut decoder = Box::new(crate::container::ts::Import::new(broadcast, catalog)); + decoder.decode(&mut { init })?; + FramedKind::Ts(decoder) + } + FramedFormat::Flv => { + let mut decoder = Box::new(crate::container::flv::Import::new(broadcast, catalog)); + decoder.decode(&mut { init })?; + FramedKind::Flv(decoder) + } + }; + + Ok(Self { decoder }) + } + + /// Create a new framed importer that publishes on an existing track. + /// + /// Only single-track formats are supported. Container formats that may + /// create multiple MoQ tracks need an explicit track mapping API. + pub fn new_with_track( + track: moq_net::TrackProducer, + catalog: crate::catalog::Producer, + format: FramedFormat, + init: &[u8], + ) -> anyhow::Result { + let decoder = match format { + FramedFormat::Avc1 => { + let (length_size, import) = build_h264_avc1(track, catalog, init)?; + FramedKind::Avc1 { length_size, import } + } + FramedFormat::Avc3 => { + let (split, import) = build_h264_avc3(track, catalog, init)?; + FramedKind::Avc3 { split, import } + } + FramedFormat::Hev1 => { + let (split, import) = build_h265(track, catalog, init)?; + FramedKind::Hev1 { split, import } + } + FramedFormat::Av01 => { + let (split, import) = build_av1(track, catalog, init)?; + FramedKind::Av01 { split, import } + } + FramedFormat::Vp8 => { + let mut import = crate::codec::vp8::Import::new(track, catalog); + import.initialize(init)?; + FramedKind::Vp8(import) + } + FramedFormat::Vp9 => { + let mut import = crate::codec::vp9::Import::new(track, catalog); + import.initialize(init)?; + FramedKind::Vp9(import) + } + FramedFormat::Aac => { + let mut data = init; + let config = crate::codec::aac::Config::parse(&mut data)?; + let import = crate::codec::aac::Import::new(track, catalog, config)?; + FramedKind::Aac(import) + } + FramedFormat::Opus => { + let mut data = init; + let config = crate::codec::opus::Config::parse(&mut data)?; + let import = crate::codec::opus::Import::new(track, catalog, config)?; + FramedKind::Opus(import) + } + FramedFormat::Fmp4 | FramedFormat::Mkv | FramedFormat::Ts | FramedFormat::Flv => { + anyhow::bail!("{format} can publish multiple tracks") + } + }; + + Ok(Self { decoder }) + } + + /// Finish the decoder, flushing any buffered data. + pub fn finish(&mut self) -> Result<()> { + match self.decoder { + FramedKind::Avc3 { ref mut import, .. } => import.finish(), + FramedKind::Avc1 { ref mut import, .. } => import.finish(), + FramedKind::Fmp4(ref mut decoder) => decoder.finish(), + FramedKind::Hev1 { ref mut import, .. } => import.finish(), + FramedKind::Av01 { ref mut import, .. } => import.finish(), + FramedKind::Vp8(ref mut import) => import.finish(), + FramedKind::Vp9(ref mut import) => import.finish(), + FramedKind::Aac(ref mut import) => import.finish(), + FramedKind::Opus(ref mut import) => import.finish(), + FramedKind::Mkv(ref mut decoder) => decoder.finish(), + FramedKind::Ts(ref mut decoder) => decoder.finish().map_err(Into::into), + FramedKind::Flv(ref mut decoder) => decoder.finish().map_err(Into::into), + } + } + + /// Close the current group and open the next one at `sequence`. + pub fn seek(&mut self, sequence: u64) -> Result<()> { + match self.decoder { + FramedKind::Avc3 { + ref mut split, + ref mut import, + } => { + split.reset(); + import.seek(sequence) + } + FramedKind::Avc1 { ref mut import, .. } => import.seek(sequence), + FramedKind::Fmp4(ref mut decoder) => decoder.seek(sequence), + FramedKind::Hev1 { + ref mut split, + ref mut import, + } => { + split.reset(); + import.seek(sequence) + } + FramedKind::Av01 { + ref mut split, + ref mut import, + } => { + split.reset(); + import.seek(sequence) + } + FramedKind::Vp8(ref mut import) => import.seek(sequence), + FramedKind::Vp9(ref mut import) => import.seek(sequence), + FramedKind::Aac(ref mut import) => import.seek(sequence), + FramedKind::Opus(ref mut import) => import.seek(sequence), + FramedKind::Mkv(ref mut decoder) => decoder.seek(sequence), + FramedKind::Ts(ref mut decoder) => decoder.seek(sequence).map_err(Into::into), + FramedKind::Flv(ref mut decoder) => decoder.seek(sequence).map_err(Into::into), + } + } + + /// A watch-only handle to the single track's subscriber demand. + /// + /// Returns [`Error::MultipleTracks`](crate::Error::MultipleTracks) for container + /// formats that may publish more than one track. The handle can't publish frames + /// or close the track. + pub fn demand(&self) -> Result { + match self.decoder { + FramedKind::Avc3 { ref import, .. } => Ok(import.demand()), + FramedKind::Avc1 { ref import, .. } => Ok(import.demand()), + FramedKind::Fmp4(_) => Err(crate::Error::MultipleTracks("fmp4")), + FramedKind::Hev1 { ref import, .. } => Ok(import.demand()), + FramedKind::Av01 { ref import, .. } => Ok(import.demand()), + FramedKind::Vp8(ref import) => Ok(import.demand()), + FramedKind::Vp9(ref import) => Ok(import.demand()), + FramedKind::Aac(ref import) => Ok(import.demand()), + FramedKind::Opus(ref import) => Ok(import.demand()), + FramedKind::Mkv(_) => Err(crate::Error::MultipleTracks("mkv")), + FramedKind::Ts(_) => Err(crate::Error::MultipleTracks("ts")), + FramedKind::Flv(_) => Err(crate::Error::MultipleTracks("flv")), + } + } + + /// The name of the single track this importer publishes. + pub fn name(&self) -> Result { + Ok(self.demand()?.name().to_string()) + } + + /// Decode one whole frame for the single-track formats, or a chunk of container + /// bytes for the multi-track ones. + pub fn decode(&mut self, frame: &[u8], pts: Option) -> Result<()> { + match self.decoder { + FramedKind::Avc3 { + ref mut split, + ref mut import, + } => { + // Framed hands over one whole access unit per call, so flush to emit it + // rather than waiting for the next start code. + let mut data = frame; + let mut frames = split.decode(&mut data, pts)?; + frames.extend(split.flush(pts)?); + import.decode(frames)?; + } + FramedKind::Avc1 { + length_size, + ref mut import, + } => { + let pts = pts.ok_or(crate::codec::h264::Error::MissingTimestamp)?; + let frame = crate::codec::h264::avc1_frame(frame, length_size, pts)?; + import.decode([frame])?; + } + FramedKind::Fmp4(ref mut decoder) => decoder.decode(&mut { frame })?, + FramedKind::Hev1 { + ref mut split, + ref mut import, + } => { + let mut data = frame; + let mut frames = split.decode(&mut data, pts)?; + frames.extend(split.flush(pts)?); + import.decode(frames)?; + } + FramedKind::Av01 { + ref mut split, + ref mut import, + } => { + let mut data = frame; + let mut frames = split.decode(&mut data, pts)?; + frames.extend(split.flush(pts)?); + import.decode(frames)?; + } + FramedKind::Vp8(ref mut import) => import.decode(frame, pts)?, + FramedKind::Vp9(ref mut import) => import.decode(frame, pts)?, + FramedKind::Aac(ref mut import) => import.decode(frame, pts)?, + FramedKind::Opus(ref mut import) => import.decode(frame, pts)?, + FramedKind::Mkv(ref mut decoder) => decoder.decode(&mut { frame })?, + FramedKind::Ts(ref mut decoder) => decoder.decode(&mut { frame })?, + FramedKind::Flv(ref mut decoder) => decoder.decode(&mut { frame })?, + } + + Ok(()) + } +} + +// Lift an already-built opus importer into a `Framed` so callers that build their +// config out-of-band (e.g. moq-gst, which constructs `opus::Config` from gstreamer +// caps instead of an OpusHead buffer) can keep using `.into()`. +impl From for Framed { + fn from(opus: crate::codec::opus::Import) -> Self { + Self { + decoder: FramedKind::Opus(opus), + } + } +} + +impl From for Framed { + fn from(aac: crate::codec::aac::Import) -> Self { + Self { + decoder: FramedKind::Aac(aac), + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + use moq_net::Timestamp; + + fn opus_head() -> Vec { + let mut head = Vec::with_capacity(19); + head.extend_from_slice(b"OpusHead"); + head.push(1); + head.push(2); + head.extend_from_slice(&0u16.to_le_bytes()); + head.extend_from_slice(&48000u32.to_le_bytes()); + head.extend_from_slice(&0u16.to_le_bytes()); + head.push(0); + head + } + + fn h264_init() -> Vec { + let mut init = Vec::new(); + init.extend_from_slice(&[0x00, 0x00, 0x00, 0x01]); + init.extend_from_slice(&[ + 0x67, 0x64, 0x00, 0x1f, 0xac, 0x24, 0x84, 0x01, 0x40, 0x16, 0xec, 0x04, 0x40, 0x00, 0x00, 0x03, 0x00, 0x40, + 0x00, 0x00, 0x0c, 0x23, 0xc6, 0x0c, 0x92, + ]); + init.extend_from_slice(&[0x00, 0x00, 0x00, 0x01]); + init.extend_from_slice(&[0x68, 0xee, 0x32, 0xc8, 0xb0]); + init + } + + fn new_broadcast() -> (moq_net::BroadcastProducer, crate::catalog::Producer) { + let mut broadcast = moq_net::BroadcastInfo::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); + (broadcast, catalog) + } + + #[tokio::test(start_paused = true)] + async fn fixed_track_opus_uses_existing_name_and_delivers_frames() { + let (mut broadcast, catalog) = new_broadcast(); + // Legacy-container codecs write micro-timestamped frames, so a caller-supplied + // fixed track must declare the matching timescale. + let track = broadcast + .create_track( + "requested-audio", + moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), + ) + .unwrap(); + let consumer = track.subscribe(None); + + let mut framed = Framed::new_with_track(track, catalog.clone(), FramedFormat::Opus, &opus_head()).unwrap(); + + assert_eq!(framed.name().unwrap(), "requested-audio"); + let snapshot = catalog.snapshot(); + assert!(snapshot.audio.renditions.contains_key("requested-audio")); + assert!(!snapshot.audio.renditions.contains_key("0.opus")); + + let mut media = crate::container::Consumer::new(consumer, crate::catalog::hang::Container::Legacy); + let payload = b"opus payload".to_vec(); + framed + .decode(&payload, Some(Timestamp::from_micros(1_000).unwrap())) + .unwrap(); + + let frame = tokio::time::timeout(Duration::from_secs(1), media.read()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(frame.payload, payload); + assert_eq!(frame.timestamp, Timestamp::from_micros(1_000).unwrap()); + + framed.finish().unwrap(); + } + + #[tokio::test(start_paused = true)] + async fn unique_track_opus_attaches_catalog_and_retires_on_drop() { + let (broadcast, catalog) = new_broadcast(); + + // The broadcast path mints a unique track and attaches its catalog rendition. + let mut framed = Framed::new(broadcast, catalog.clone(), FramedFormat::Opus, &opus_head()).unwrap(); + + assert_eq!(framed.name().unwrap(), "0.opus"); + assert!(catalog.snapshot().audio.renditions.contains_key("0.opus")); + + framed + .decode(b"opus payload", Some(Timestamp::from_micros(2_000).unwrap())) + .unwrap(); + framed.finish().unwrap(); + + // Dropping the importer retires its rendition from the shared catalog. + drop(framed); + assert!(!catalog.snapshot().audio.renditions.contains_key("0.opus")); + } + + #[tokio::test(start_paused = true)] + async fn opus_import_delivers_frames() { + let (mut broadcast, catalog) = new_broadcast(); + let track = broadcast + .create_track( + "audio", + moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), + ) + .unwrap(); + let subscriber = track.subscribe(None); + + let config = crate::codec::opus::Config { + sample_rate: 48_000, + channel_count: 2, + }; + let mut import = crate::codec::opus::Import::new(track, catalog.clone(), config).unwrap(); + assert!(catalog.snapshot().audio.renditions.contains_key("audio")); + + let mut media = crate::container::Consumer::new(subscriber, crate::catalog::hang::Container::Legacy); + + let payload = b"opus payload".to_vec(); + import + .decode(&payload, Some(Timestamp::from_micros(1_000).unwrap())) + .unwrap(); + + let frame = tokio::time::timeout(Duration::from_secs(1), media.read()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(frame.payload, payload); + + import.finish().unwrap(); + } + + #[tokio::test(start_paused = true)] + async fn fixed_track_h264_uses_existing_name_in_catalog() { + let (mut broadcast, catalog) = new_broadcast(); + let track = broadcast.create_track("camera", None).unwrap(); + + let framed = Framed::new_with_track(track, catalog.clone(), FramedFormat::Avc3, &h264_init()).unwrap(); + + assert_eq!(framed.name().unwrap(), "camera"); + let snapshot = catalog.snapshot(); + let video = snapshot.video.renditions.get("camera").unwrap(); + assert_eq!(video.coded_width, Some(1280)); + assert_eq!(video.coded_height, Some(720)); + assert!(!snapshot.video.renditions.contains_key("0.avc3")); + } + + #[test] + fn fixed_track_rejects_multi_track_formats() { + for format in [FramedFormat::Fmp4, FramedFormat::Mkv, FramedFormat::Ts] { + let (mut broadcast, catalog) = new_broadcast(); + let track = broadcast.create_track("media", None).unwrap(); + + let err = match Framed::new_with_track(track, catalog, format, &[]) { + Ok(_) => panic!("multi-track format should be rejected"), + Err(err) => err, + }; + assert!(err.to_string().contains("multiple tracks")); + } + } + + /// A changed key frame just updates the rendition in place; there are no fixed + /// tracks to reject a reconfiguration, so the second key frame succeeds. + #[tokio::test(start_paused = true)] + async fn reconfiguration_updates_in_place() { + let (mut broadcast, catalog) = new_broadcast(); + let track = broadcast + .create_track( + "video", + moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), + ) + .unwrap(); + let mut framed = Framed::new_with_track(track, catalog, FramedFormat::Vp8, &[]).unwrap(); + + framed + .decode( + &[0x10, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x40, 0x01, 0xf0, 0x00], + Some(Timestamp::from_micros(0).unwrap()), + ) + .unwrap(); + + framed + .decode( + &[0x10, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x80, 0x02, 0xe0, 0x01], + Some(Timestamp::from_micros(33_000).unwrap()), + ) + .unwrap(); + } +} + +// -- stream dispatcher -- + +/// Formats that support stream decoding (unknown frame boundaries). +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum StreamFormat { + /// aka H264 with inline SPS/PPS + Avc3, + /// fMP4/CMAF container. + Fmp4, + /// aka H265 with inline SPS/PPS + Hev1, + /// AV1 with inline sequence headers + Av01, + /// Matroska / WebM container. + Mkv, + /// MPEG-TS (transport stream) container. + Ts, + /// FLV (Flash Video / RTMP) container. + Flv, +} + +impl FromStr for StreamFormat { + type Err = crate::Error; + + fn from_str(s: &str) -> std::result::Result { + match s { + "avc3" | "h264" => Ok(StreamFormat::Avc3), + "hev1" => Ok(StreamFormat::Hev1), + "fmp4" | "cmaf" => Ok(StreamFormat::Fmp4), + "av01" | "av1" | "av1c" | "av1C" => Ok(StreamFormat::Av01), + "mkv" | "webm" | "matroska" => Ok(StreamFormat::Mkv), + "ts" | "mpegts" | "mpeg2ts" | "m2ts" => Ok(StreamFormat::Ts), + "flv" => Ok(StreamFormat::Flv), + _ => Err(crate::Error::UnknownFormat(s.to_string())), + } + } +} + +impl fmt::Display for StreamFormat { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + StreamFormat::Avc3 => write!(f, "avc3"), + StreamFormat::Fmp4 => write!(f, "fmp4"), + StreamFormat::Hev1 => write!(f, "hev1"), + StreamFormat::Av01 => write!(f, "av01"), + StreamFormat::Mkv => write!(f, "mkv"), + StreamFormat::Ts => write!(f, "ts"), + StreamFormat::Flv => write!(f, "flv"), + } + } +} + +enum StreamKind { + /// H.264 in avc3 wire shape (Annex-B with inline SPS/PPS). The split owns + /// byte parsing; the import publishes. + Avc3 { + split: crate::codec::h264::Split, + import: crate::codec::h264::Import, + }, + // Boxed because it's a large struct and clippy complains about the size. + Fmp4(Box), + Hev1 { + split: crate::codec::h265::Split, + import: crate::codec::h265::Import, + }, + Av01 { + split: crate::codec::av1::Split, + import: crate::codec::av1::Import, + }, + // Boxed for the same reason as Fmp4. + Mkv(Box), + // Boxed for the same reason as Fmp4. + Ts(Box), + // Boxed for the same reason as Fmp4. + Flv(Box), +} + +/// An importer for formats that support stream decoding (unknown frame boundaries). +/// +/// This includes formats like H.264 (AVC3), H.265 (HEV1), and fMP4/CMAF. +/// Use this when the caller does not know the frame boundaries. +pub struct Stream { + decoder: StreamKind, +} + +impl Stream { + /// Create a new stream importer with the given format. + pub fn new( + mut broadcast: moq_net::BroadcastProducer, + catalog: crate::catalog::Producer, + format: StreamFormat, + ) -> Result { + let decoder = match format { + StreamFormat::Avc3 => { + let track = crate::import::unique_track(&mut broadcast, ".avc3")?; + StreamKind::Avc3 { + split: crate::codec::h264::Split::new(), + import: crate::codec::h264::Import::new(track, catalog), + } + } + StreamFormat::Fmp4 => StreamKind::Fmp4(Box::new(crate::container::fmp4::Import::new(broadcast, catalog))), + StreamFormat::Hev1 => { + let track = crate::import::unique_track(&mut broadcast, ".hev1")?; + StreamKind::Hev1 { + split: crate::codec::h265::Split::new(), + import: crate::codec::h265::Import::new(track, catalog), + } + } + StreamFormat::Av01 => { + let track = crate::import::unique_track(&mut broadcast, ".av01")?; + StreamKind::Av01 { + split: crate::codec::av1::Split::new(), + import: crate::codec::av1::Import::new(track, catalog), + } + } + StreamFormat::Mkv => StreamKind::Mkv(Box::new(crate::container::mkv::Import::new(broadcast, catalog))), + StreamFormat::Ts => StreamKind::Ts(Box::new(crate::container::ts::Import::new(broadcast, catalog))), + StreamFormat::Flv => StreamKind::Flv(Box::new(crate::container::flv::Import::new(broadcast, catalog))), + }; + + Ok(Self { decoder }) + } + + /// Initialize the decoder with the given buffer and populate the broadcast. + /// + /// This is not required for self-describing formats like fMP4 or AVC3. + /// + /// The buffer will be fully consumed, or an error will be returned. + pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { + match self.decoder { + StreamKind::Avc3 { + ref mut split, + ref mut import, + } => { + import.initialize(buf.as_ref())?; + let frames = split.decode(buf, None)?; + import.decode(frames)?; + } + StreamKind::Fmp4(ref mut decoder) => decoder.decode(buf)?, + StreamKind::Hev1 { + ref mut split, + ref mut import, + } => { + import.initialize(buf.as_ref())?; + let frames = split.decode(buf, None)?; + import.decode(frames)?; + } + StreamKind::Av01 { + ref mut split, + ref mut import, + } => { + import.initialize(buf.as_ref())?; + // av1C (leading 0x81) is an out-of-band config record, not an OBU + // stream; read for config above and dropped here. + let data = buf.as_ref(); + let frames = if data.len() >= 16 && data[0] == 0x81 { + buf.advance(buf.remaining()); + Vec::new() + } else { + split.decode(buf, None)? + }; + import.decode(frames)?; + } + StreamKind::Mkv(ref mut decoder) => decoder.decode(buf)?, + StreamKind::Ts(ref mut decoder) => decoder.decode(buf)?, + StreamKind::Flv(ref mut decoder) => decoder.decode(buf)?, + } + + if buf.has_remaining() { + return Err(crate::Error::BufferNotConsumed); + } + + Ok(()) + } + + /// Decode a stream of data from the given buffer. + pub fn decode_stream>(&mut self, buf: &mut T) -> Result<()> { + match self.decoder { + StreamKind::Avc3 { + ref mut split, + ref mut import, + } => { + let frames = split.decode(buf, None)?; + import.decode(frames) + } + StreamKind::Fmp4(ref mut decoder) => decoder.decode(buf), + StreamKind::Hev1 { + ref mut split, + ref mut import, + } => { + let frames = split.decode(buf, None)?; + import.decode(frames) + } + StreamKind::Av01 { + ref mut split, + ref mut import, + } => { + let frames = split.decode(buf, None)?; + import.decode(frames) + } + StreamKind::Mkv(ref mut decoder) => decoder.decode(buf), + StreamKind::Ts(ref mut decoder) => decoder.decode(buf).map_err(Into::into), + StreamKind::Flv(ref mut decoder) => decoder.decode(buf).map_err(Into::into), + } + } + + /// Finish the decoder, flushing any buffered data. + pub fn finish(&mut self) -> Result<()> { + match self.decoder { + StreamKind::Avc3 { + ref mut split, + ref mut import, + } => { + let tail = split.flush(None)?; + import.decode(tail)?; + import.finish() + } + StreamKind::Fmp4(ref mut decoder) => decoder.finish(), + StreamKind::Hev1 { + ref mut split, + ref mut import, + } => { + let tail = split.flush(None)?; + import.decode(tail)?; + import.finish() + } + StreamKind::Av01 { + ref mut split, + ref mut import, + } => { + let tail = split.flush(None)?; + import.decode(tail)?; + import.finish() + } + StreamKind::Mkv(ref mut decoder) => decoder.finish(), + StreamKind::Ts(ref mut decoder) => decoder.finish().map_err(Into::into), + StreamKind::Flv(ref mut decoder) => decoder.finish().map_err(Into::into), + } + } + + /// Close the current group and open the next one at `sequence`. + pub fn seek(&mut self, sequence: u64) -> Result<()> { + match self.decoder { + StreamKind::Avc3 { + ref mut split, + ref mut import, + } => { + split.reset(); + import.seek(sequence) + } + StreamKind::Fmp4(ref mut decoder) => decoder.seek(sequence), + StreamKind::Hev1 { + ref mut split, + ref mut import, + } => { + split.reset(); + import.seek(sequence) + } + StreamKind::Av01 { + ref mut split, + ref mut import, + } => { + split.reset(); + import.seek(sequence) + } + StreamKind::Mkv(ref mut decoder) => decoder.seek(sequence), + StreamKind::Ts(ref mut decoder) => decoder.seek(sequence).map_err(Into::into), + StreamKind::Flv(ref mut decoder) => decoder.seek(sequence).map_err(Into::into), + } + } + + /// Check if the decoder has read enough data to be initialized. + pub fn is_initialized(&self) -> bool { + match self.decoder { + StreamKind::Avc3 { ref import, .. } => import.is_initialized(), + StreamKind::Fmp4(ref decoder) => decoder.is_initialized(), + StreamKind::Hev1 { ref import, .. } => import.is_initialized(), + StreamKind::Av01 { ref import, .. } => import.is_initialized(), + StreamKind::Mkv(ref decoder) => decoder.is_initialized(), + StreamKind::Ts(ref decoder) => decoder.is_initialized(), + StreamKind::Flv(ref decoder) => decoder.is_initialized(), + } + } +} diff --git a/rs/moq-mux/src/import/track.rs b/rs/moq-mux/src/import/track.rs new file mode 100644 index 000000000..ea194a251 --- /dev/null +++ b/rs/moq-mux/src/import/track.rs @@ -0,0 +1,12 @@ +//! Track-minting helper shared by the import dispatchers and containers. + +/// Mint a fresh unique track for a legacy single-codec importer. +/// +/// Picks a unique name from `suffix` and sets the microsecond +/// [`hang::container::TIMESCALE`] that the legacy importers stamp their frames +/// with, so the relay gets timing without parsing the payload. Hand the result to +/// the importer's `new`. +pub fn unique_track(broadcast: &mut moq_net::BroadcastProducer, suffix: &str) -> crate::Result { + let info = moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE); + Ok(broadcast.unique_track(suffix, info)?) +} diff --git a/rs/moq-mux/src/lib.rs b/rs/moq-mux/src/lib.rs index 460237d32..6d1242ab5 100644 --- a/rs/moq-mux/src/lib.rs +++ b/rs/moq-mux/src/lib.rs @@ -21,7 +21,6 @@ pub mod codec; pub mod container; mod error; pub mod import; -mod track_provider; pub use clock::Clock; pub use error::*; diff --git a/rs/moq-mux/src/track_provider.rs b/rs/moq-mux/src/track_provider.rs deleted file mode 100644 index fde002678..000000000 --- a/rs/moq-mux/src/track_provider.rs +++ /dev/null @@ -1,39 +0,0 @@ -pub(crate) enum TrackProvider { - Unique { - broadcast: moq_net::BroadcastProducer, - suffix: &'static str, - }, - Fixed(moq_net::TrackProducer), -} - -impl TrackProvider { - pub(crate) fn unique(broadcast: moq_net::BroadcastProducer, suffix: &'static str) -> Self { - Self::Unique { broadcast, suffix } - } - - pub(crate) fn fixed(track: moq_net::TrackProducer) -> Self { - Self::Fixed(track) - } - - pub(crate) fn is_fixed(&self) -> bool { - matches!(self, Self::Fixed(_)) - } - - pub(crate) fn set_suffix(&mut self, next: &'static str) { - if let Self::Unique { suffix, .. } = self { - *suffix = next; - } - } - - pub(crate) fn create(&mut self) -> crate::Result { - match self { - Self::Unique { broadcast, suffix } => { - // Newly created tracks publish at the native container timescale so the - // relay gets timing without parsing the payload. - let info = moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE); - Ok(broadcast.unique_track(suffix, info)?) - } - Self::Fixed(track) => Ok(track.clone()), - } - } -} diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 01a8ffdc9..173df8668 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -639,6 +639,20 @@ impl TrackProducer { } } + /// Create a [`TrackDemand`]: a cloneable, watch-only handle to this track's + /// subscriber demand. + /// + /// Lets a publisher gate work (e.g. on-demand capture) on whether anyone is + /// subscribed, without the ability to publish frames or close the track. The + /// handle is weak, so holding one neither keeps the track alive nor pins its + /// cached groups. + pub fn demand(&self) -> TrackDemand { + TrackDemand { + name: self.name.clone(), + state: self.state.weak(), + } + } + /// Get a consumer handle for this in-process track. /// /// Unlike a wire subscription, the info is already known, so a subscription @@ -897,6 +911,49 @@ impl TrackWeak { } } +/// A cloneable, watch-only handle to a track's subscriber demand. +/// +/// Obtained from [`TrackProducer::demand`]. A publisher uses it to react to +/// whether anyone is subscribed (on-demand capture / encoding) without being able +/// to publish frames or close the track. It's a weak handle, so it neither keeps +/// the track alive nor pins its cached groups; once the owning [`TrackProducer`] +/// goes away, [`used`](Self::used) / [`unused`](Self::unused) report the track's +/// closure. +#[derive(Clone)] +pub struct TrackDemand { + name: Arc, + state: kio::Weak, +} + +impl TrackDemand { + /// The track name this handle is bound to. + pub fn name(&self) -> &str { + &self.name + } + + /// Block until there is at least one active consumer. + pub async fn used(&self) -> Result<()> { + self.state + .used() + .await + .map_err(|r| r.abort.clone().unwrap_or(Error::Dropped)) + } + + /// Block until there are no active consumers. + pub async fn unused(&self) -> Result<()> { + self.state + .unused() + .await + .map_err(|r| r.abort.clone().unwrap_or(Error::Dropped)) + } + + /// Block until the track is closed or aborted, returning the cause. + pub async fn closed(&self) -> Error { + self.state.closed().await; + self.state.read().abort.clone().unwrap_or(Error::Dropped) + } +} + /// A handle to a single track within a broadcast. /// /// Obtained from [`crate::BroadcastConsumer::track`]. Holding it sends nothing diff --git a/rs/moq-rtc/src/codec/h264.rs b/rs/moq-rtc/src/codec/h264.rs index 396fadf53..2d17f9a1f 100644 --- a/rs/moq-rtc/src/codec/h264.rs +++ b/rs/moq-rtc/src/codec/h264.rs @@ -10,14 +10,16 @@ use bytes::BytesMut; use crate::{Result, codec}; pub struct Bridge { + split: moq_mux::codec::h264::Split, import: moq_mux::codec::h264::Import, } impl Bridge { - pub fn new(broadcast: moq_net::BroadcastProducer, catalog: moq_mux::catalog::Producer) -> Result { - let import = - moq_mux::codec::h264::Import::new(broadcast, catalog).with_mode(moq_mux::codec::h264::Mode::Avc3)?; - Ok(Self { import }) + pub fn new(mut broadcast: moq_net::BroadcastProducer, catalog: moq_mux::catalog::Producer) -> Result { + let track = moq_mux::import::unique_track(&mut broadcast, ".avc3")?; + let import = moq_mux::codec::h264::Import::new(track, catalog); + let split = moq_mux::codec::h264::Split::new(); + Ok(Self { split, import }) } } @@ -26,7 +28,10 @@ impl codec::Bridge for Bridge { let pts = moq_net::Timestamp::from_micros(frame.timestamp_us) .map_err(|err| crate::Error::Other(anyhow::anyhow!("invalid timestamp: {err}")))?; let mut buf = BytesMut::from(frame.payload.as_ref()); - self.import.decode_frame(&mut buf, Some(pts))?; + // str0m hands over one whole access unit per frame, so flush to emit it. + let mut frames = self.split.decode(&mut buf, Some(pts))?; + frames.extend(self.split.flush(Some(pts))?); + self.import.decode(frames)?; Ok(()) } } diff --git a/rs/moq-rtc/src/codec/opus.rs b/rs/moq-rtc/src/codec/opus.rs index 9257b04b1..14fe510c2 100644 --- a/rs/moq-rtc/src/codec/opus.rs +++ b/rs/moq-rtc/src/codec/opus.rs @@ -11,7 +11,7 @@ pub struct Bridge { impl Bridge { pub fn new( - broadcast: moq_net::BroadcastProducer, + mut broadcast: moq_net::BroadcastProducer, catalog: moq_mux::catalog::Producer, sample_rate: u32, channel_count: u32, @@ -20,7 +20,8 @@ impl Bridge { sample_rate, channel_count, }; - let import = moq_mux::codec::opus::Import::new(broadcast, catalog, config)?; + let track = moq_mux::import::unique_track(&mut broadcast, ".opus")?; + let import = moq_mux::codec::opus::Import::new(track, catalog, config)?; Ok(Self { import }) } } @@ -29,8 +30,7 @@ impl codec::Bridge for Bridge { fn push(&mut self, frame: codec::Frame) -> Result<()> { let pts = moq_net::Timestamp::from_micros(frame.timestamp_us) .map_err(|err| crate::Error::Other(anyhow::anyhow!("invalid timestamp: {err}")))?; - let mut payload = frame.payload; - self.import.decode(&mut payload, Some(pts))?; + self.import.decode(&frame.payload, Some(pts))?; Ok(()) } } diff --git a/rs/moq-rtc/tests/bitstream.rs b/rs/moq-rtc/tests/bitstream.rs index f6c4c4fd1..ba9c52d29 100644 --- a/rs/moq-rtc/tests/bitstream.rs +++ b/rs/moq-rtc/tests/bitstream.rs @@ -2,7 +2,7 @@ //! //! The H.264 Annex-B -> AVCC conversion is provided by `moq_mux::codec::h264`, //! but the WHIP path depends on the importer parsing the SPS for the catalog -//! and accepting Annex-B input via `decode_frame`. These tests guard against +//! and accepting Annex-B input via the bridge. These tests guard against //! regressions in the contract the moq-rtc bridge depends on. use bytes::{Bytes, BytesMut}; diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index f573c1646..811622051 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -25,27 +25,32 @@ const DEFAULT_FRAMERATE: u32 = 30; /// trigger capture on demand. `moq_mux::codec::h264::Import` handles /// catalog registration and framing. pub struct Producer { + split: moq_mux::codec::h264::Split, import: moq_mux::codec::h264::Import, } impl Producer { - pub fn new(broadcast: moq_net::BroadcastProducer, catalog: moq_mux::catalog::Producer) -> Result { - let import = - moq_mux::codec::h264::Import::new(broadcast, catalog).with_mode(moq_mux::codec::h264::Mode::Avc3)?; - Ok(Self { import }) + pub fn new(mut broadcast: moq_net::BroadcastProducer, catalog: moq_mux::catalog::Producer) -> Result { + let track = moq_mux::import::unique_track(&mut broadcast, ".avc3")?; + let import = moq_mux::codec::h264::Import::new(track, catalog); + let split = moq_mux::codec::h264::Split::new(); + Ok(Self { split, import }) } - /// The underlying track producer, eagerly created by avc3 mode. Clone it - /// to watch subscription state via [`used`](moq_net::TrackProducer::used) / - /// [`unused`](moq_net::TrackProducer::unused). - pub fn track(&self) -> Option<&moq_net::TrackProducer> { - self.import.track() + /// A watch-only handle to the track's subscriber demand, created eagerly so + /// subscription state is observable before any frames arrive. Watch it via + /// [`used`](moq_net::TrackDemand::used) / [`unused`](moq_net::TrackDemand::unused). + pub fn demand(&self) -> moq_net::TrackDemand { + self.import.demand() } /// Publish already-encoded Annex-B packets at the given timestamp. pub fn publish(&mut self, packets: Vec, timestamp: Timestamp) -> Result<(), Error> { for mut packet in packets { - self.import.decode_frame(&mut packet, Some(timestamp))?; + // The encoder emits one whole access unit per packet, so flush to emit it. + let mut frames = self.split.decode(&mut packet, Some(timestamp))?; + frames.extend(self.split.flush(Some(timestamp))?); + self.import.decode(frames)?; } Ok(()) } @@ -94,10 +99,7 @@ pub async fn publish_capture( } let producer = Producer::new(broadcast, catalog)?; - let track = producer - .track() - .cloned() - .ok_or_else(|| Error::Codec(anyhow::anyhow!("avc3 track was not created")))?; + let demand = producer.demand(); let gate = Gate::new(); @@ -109,7 +111,7 @@ pub async fn publish_capture( // Surface a capture/encode failure (e.g. camera open) promptly. res = &mut worker => res.map_err(|e| Error::Codec(anyhow::anyhow!("capture task: {e}")))?, // The broadcast was dropped: stop the worker and wait for it to flush. - () = monitor_demand(&track, &gate) => { + () = monitor_demand(&demand, &gate) => { gate.close(); worker .await @@ -120,13 +122,13 @@ pub async fn publish_capture( /// Toggle the gate as viewers subscribe and unsubscribe. Returns once the /// track stops being announced (broadcast dropped / aborted). -async fn monitor_demand(track: &moq_net::TrackProducer, gate: &Gate) { +async fn monitor_demand(demand: &moq_net::TrackDemand, gate: &Gate) { loop { - match track.used().await { + match demand.used().await { Ok(()) => gate.set_active(true), Err(err) => return log_track_ended(err), } - match track.unused().await { + match demand.unused().await { Ok(()) => gate.set_active(false), Err(err) => return log_track_ended(err), }