From 078527623c1b7cf5d22fcf1f8d8367a41e32523a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 02:40:35 +0000 Subject: [PATCH 01/19] moq-mux: decouple the opus importer from the broadcast catalog First slice of the import refactor. The opus importer no longer holds a `catalog::Producer` and mutates a shared catalog with a `Drop` hook. It now produces frames on a single track and exposes its own standalone `hang::Catalog` rendition, which a new `publish` bridge merges into a broadcast catalog (removing it on drop). - `codec::opus::Import`: `new(TrackRequest, Config)` (on-demand) and `from_track(TrackProducer, Config)` (broadcast push / fixed track). `decode` now takes `impl IntoIterator`; `decode_buf` is the raw-packet convenience that stamps a wall clock when no timestamp is given. `catalog()` returns the standalone `hang::Catalog`. - `publish` module: `Renditions` trait, `Published` (merges renditions into a `catalog::Producer`, retires on drop, derefs to the importer), and `unique_track` (mints a legacy single-codec track at the microsecond timescale). - Rewire `import::Framed`, moq-gst, and moq-rtc through the bridge. Tests cover both construction paths (TrackRequest and broadcast unique_track), frame delivery, the catalog merge, and retire-on-drop. https://claude.ai/code/session_011S7pzcg2XsPP3AExuymac1 --- rs/moq-gst/src/sink/imp.rs | 4 +- rs/moq-mux/src/codec/opus/import.rs | 111 +++++++++++++--------------- rs/moq-mux/src/import.rs | 93 ++++++++++++++++++++--- rs/moq-mux/src/lib.rs | 1 + rs/moq-mux/src/publish.rs | 99 +++++++++++++++++++++++++ rs/moq-rtc/src/codec/opus.rs | 10 ++- 6 files changed, 245 insertions(+), 73 deletions(-) create mode 100644 rs/moq-mux/src/publish.rs diff --git a/rs/moq-gst/src/sink/imp.rs b/rs/moq-gst/src/sink/imp.rs index b5c7d03ac..2a9cf94fc 100644 --- a/rs/moq-gst/src/sink/imp.rs +++ b/rs/moq-gst/src/sink/imp.rs @@ -486,7 +486,9 @@ fn handle_caps(runtime: &mut RuntimeState, pad_name: String, caps: gst::Caps) -> sample_rate, channel_count, }; - moq_mux::codec::opus::Import::new(runtime.broadcast.clone(), runtime.catalog.clone(), config)?.into() + let track = moq_mux::publish::unique_track(&mut runtime.broadcast, ".opus")?; + let import = moq_mux::codec::opus::Import::from_track(track, config)?; + moq_mux::publish::Published::new(runtime.catalog.clone(), import).into() } other => anyhow::bail!("unsupported caps: {}", other), }; diff --git a/rs/moq-mux/src/codec/opus/import.rs b/rs/moq-mux/src/codec/opus/import.rs index 583c25c25..d00dd12ee 100644 --- a/rs/moq-mux/src/codec/opus/import.rs +++ b/rs/moq-mux/src/codec/opus/import.rs @@ -1,70 +1,61 @@ use bytes::{Buf, BytesMut}; use super::Config; +use crate::container::Frame; +use crate::publish::Renditions; /// Opus importer. /// -/// Initialized from an OpusHead packet. Each input buffer passed to [`decode`](Self::decode) -/// is published as one hang frame in its own group, so the relay can forward each frame -/// without waiting for a group boundary. Opus' packet loss concealment handles drops. -/// Ogg framing is not supported, feed raw Opus packets. +/// Publishes raw Opus frames (no Ogg framing) to a single moq track. Build it +/// from a [`moq_net::TrackRequest`] (the on-demand path, [`new`](Self::new)) or +/// an existing [`moq_net::TrackProducer`] ([`from_track`](Self::from_track)). +/// +/// Each input frame is published in its own group so the relay can forward it +/// immediately without waiting for a group boundary; Opus' packet loss +/// concealment handles drops. The catalog rendition this importer publishes is +/// available via [`catalog`](Self::catalog); attach it to a broadcast catalog +/// with [`crate::publish::Published`]. pub struct Import { - catalog: crate::catalog::Producer, track: crate::container::Producer, + catalog: hang::Catalog, zero: Option, } impl Import { - pub fn new( - broadcast: moq_net::BroadcastProducer, - catalog: crate::catalog::Producer, - config: Config, - ) -> crate::Result { - Self::new_with_source( - crate::track_provider::TrackProvider::unique(broadcast, ".opus"), - catalog, - config, - ) + /// Serve a track request, accepting it at the microsecond timescale. + pub fn new(request: moq_net::TrackRequest, config: Config) -> crate::Result { + let info = moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE); + Self::from_track(request.accept(info), config) } - pub fn new_with_track( - track: moq_net::TrackProducer, - catalog: crate::catalog::Producer, - config: Config, - ) -> crate::Result { - Self::new_with_source(crate::track_provider::TrackProvider::fixed(track), catalog, config) - } - - fn new_with_source( - mut tracks: crate::track_provider::TrackProvider, - mut catalog: crate::catalog::Producer, - config: Config, - ) -> crate::Result { - let track = tracks.create()?; - - let mut audio_config = hang::catalog::AudioConfig::new( + /// Publish on an existing track producer. + pub fn from_track(track: moq_net::TrackProducer, config: Config) -> crate::Result { + let mut audio = hang::catalog::AudioConfig::new( hang::catalog::AudioCodec::Opus, config.sample_rate, config.channel_count, ); - audio_config.container = hang::catalog::Container::Legacy; + audio.container = hang::catalog::Container::Legacy; - tracing::debug!(name = ?track.name(), config = ?audio_config, "starting track"); - catalog - .lock() - .audio - .renditions - .insert(track.name().to_string(), audio_config); + tracing::debug!(name = ?track.name(), config = ?audio, "starting track"); + + let mut catalog = hang::Catalog::default(); + catalog.audio.renditions.insert(track.name().to_string(), audio); Ok(Self { - catalog, track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), + catalog, zero: None, }) } - /// Returns a reference to the underlying track producer, e.g. for - /// monitoring subscriber state via `used()`/`unused()`. + /// The standalone catalog this importer publishes (one Opus audio rendition). + pub fn catalog(&self) -> &hang::Catalog { + &self.catalog + } + + /// The underlying track producer, e.g. for monitoring subscriber state via + /// `used()` / `unused()`. pub fn track(&self) -> &moq_net::TrackProducer { self.track.track() } @@ -81,10 +72,22 @@ impl Import { Ok(()) } - pub fn decode(&mut self, buf: &mut T, pts: Option) -> crate::Result<()> { - let pts = self.pts(pts)?; + /// Publish frames, each in its own group. + pub fn decode(&mut self, frames: impl IntoIterator) -> crate::Result<()> { + for frame in frames { + self.track.write(frame)?; + self.track.finish_group()?; + } + Ok(()) + } + + /// Publish one Opus packet from `buf`, stamping `pts` or a wall clock when absent. + /// + /// Convenience for callers that hand over raw packet bytes plus an optional + /// timestamp; it wraps the packet in a [`Frame`] and forwards to [`decode`](Self::decode). + pub fn decode_buf(&mut self, buf: &mut T, pts: Option) -> crate::Result<()> { + let timestamp = self.pts(pts)?; - // Collect the input into a contiguous Bytes payload. let mut payload = BytesMut::with_capacity(buf.remaining()); while buf.has_remaining() { let chunk = buf.chunk(); @@ -93,19 +96,12 @@ impl Import { buf.advance(len); } - // Each frame is its own group so the relay can forward it immediately. - // Opus' packet loss concealment handles drops. - let frame = crate::container::Frame { - timestamp: pts, + self.decode(std::iter::once(Frame { + timestamp, payload: payload.freeze(), keyframe: true, duration: None, - }; - - self.track.write(frame)?; - self.track.finish_group()?; - - Ok(()) + })) } fn pts(&mut self, hint: Option) -> crate::Result { @@ -118,9 +114,8 @@ impl Import { } } -impl Drop for Import { - fn drop(&mut self) { - tracing::debug!(name = ?self.track.name(), "ending track"); - self.catalog.lock().audio.renditions.remove(self.track.name()); +impl Renditions for Import { + fn renditions(&self) -> &hang::Catalog { + &self.catalog } } diff --git a/rs/moq-mux/src/import.rs b/rs/moq-mux/src/import.rs index c5b98b426..0c3af2106 100644 --- a/rs/moq-mux/src/import.rs +++ b/rs/moq-mux/src/import.rs @@ -107,7 +107,7 @@ enum FramedKind { Vp8(crate::codec::vp8::Import), Vp9(crate::codec::vp9::Import), Aac(crate::codec::aac::Import), - Opus(crate::codec::opus::Import), + Opus(crate::publish::Published), // Boxed for the same reason as Fmp4. Mkv(Box), // Boxed for the same reason as Fmp4. @@ -126,7 +126,7 @@ impl Framed { /// /// The buffer will be fully consumed, or an error will be returned. pub fn new>( - broadcast: moq_net::BroadcastProducer, + mut broadcast: moq_net::BroadcastProducer, catalog: crate::catalog::Producer, format: FramedFormat, buf: &mut T, @@ -174,7 +174,9 @@ impl Framed { } FramedFormat::Opus => { let config = crate::codec::opus::Config::parse(buf)?; - FramedKind::Opus(crate::codec::opus::Import::new(broadcast, catalog, config)?) + let track = crate::publish::unique_track(&mut broadcast, ".opus")?; + let import = crate::codec::opus::Import::from_track(track, config)?; + FramedKind::Opus(crate::publish::Published::new(catalog, import)) } FramedFormat::Mkv => { let mut decoder = Box::new(crate::container::mkv::Import::new(broadcast, catalog)); @@ -245,7 +247,8 @@ impl Framed { } FramedFormat::Opus => { let config = crate::codec::opus::Config::parse(buf)?; - FramedKind::Opus(crate::codec::opus::Import::new_with_track(track, catalog, config)?) + let import = crate::codec::opus::Import::from_track(track, config)?; + FramedKind::Opus(crate::publish::Published::new(catalog, import)) } FramedFormat::Fmp4 | FramedFormat::Mkv | FramedFormat::Ts => { anyhow::bail!("{format} can publish multiple tracks") @@ -317,7 +320,7 @@ impl Framed { FramedKind::Vp8(ref mut decoder) => decoder.decode_frame(buf, pts)?, FramedKind::Vp9(ref mut decoder) => decoder.decode_frame(buf, pts)?, FramedKind::Aac(ref mut decoder) => decoder.decode(buf, pts)?, - FramedKind::Opus(ref mut decoder) => decoder.decode(buf, pts)?, + FramedKind::Opus(ref mut decoder) => decoder.decode_buf(buf, pts)?, FramedKind::Mkv(ref mut decoder) => { let _ = pts; decoder.decode(buf)?; @@ -336,11 +339,11 @@ impl Framed { } } -// Lift an already-built codec importer into a `Framed` so callers that build -// their config out-of-band (e.g. moq-gst, which constructs `opus::Config` from -// gstreamer caps instead of an OpusHead buffer) can keep using `.into()`. -impl From for Framed { - fn from(opus: crate::codec::opus::Import) -> Self { +// Lift an already-built, catalog-attached opus importer into a `Framed` so callers +// that build their config out-of-band (e.g. moq-gst, which constructs `opus::Config` +// from gstreamer caps instead of an OpusHead buffer) can keep using `.into()`. +impl From> for Framed { + fn from(opus: crate::publish::Published) -> Self { Self { decoder: FramedKind::Opus(opus), } @@ -434,6 +437,76 @@ mod tests { framed.finish().unwrap(); } + #[tokio::test(start_paused = true)] + async fn unique_track_opus_delivers_frames_via_broadcast() { + let (broadcast, catalog) = new_broadcast(); + let init = opus_head(); + let mut init = init.as_slice(); + + // The broadcast path mints a unique track and attaches its catalog rendition. + let mut framed = Framed::new(broadcast, catalog.clone(), FramedFormat::Opus, &mut init).unwrap(); + + assert_eq!(framed.track().unwrap().name(), "0.opus"); + assert!(catalog.snapshot().audio.renditions.contains_key("0.opus")); + + // Frames published through the minted producer are delivered. + let subscriber = framed.track().unwrap().subscribe(None); + let mut media = crate::container::Consumer::new(subscriber, crate::catalog::hang::Container::Legacy); + + let payload = b"opus payload".to_vec(); + let mut frame = payload.as_slice(); + framed + .decode_frame(&mut frame, Some(Timestamp::from_micros(2_000).unwrap())) + .unwrap(); + + let frame = tokio::time::timeout(Duration::from_secs(1), media.read()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(frame.payload, payload); + assert_eq!(frame.timestamp, Timestamp::from_micros(2_000).unwrap()); + + framed.finish().unwrap(); + + // Dropping the importer retires its rendition from the shared catalog. + drop(framed); + assert!(!catalog.snapshot().audio.renditions.contains_key("0.opus")); + } + + #[tokio::test(start_paused = true)] + async fn opus_import_serves_track_request() { + // The on-demand path: build straight from a TrackRequest, no broadcast/catalog. + let request = moq_net::TrackRequest::new("audio"); + let config = crate::codec::opus::Config { + sample_rate: 48_000, + channel_count: 2, + }; + let mut import = crate::codec::opus::Import::new(request, config).unwrap(); + + assert_eq!(import.track().name(), "audio"); + assert!(import.catalog().audio.renditions.contains_key("audio")); + + // Accepting the request yields a working producer that delivers frames. + let subscriber = import.track().subscribe(None); + let mut media = crate::container::Consumer::new(subscriber, crate::catalog::hang::Container::Legacy); + + let payload = b"opus payload".to_vec(); + let mut buf = payload.as_slice(); + import + .decode_buf(&mut buf, Some(Timestamp::from_micros(1_000).unwrap())) + .unwrap(); + + let frame = tokio::time::timeout(Duration::from_secs(1), media.read()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(frame.payload, payload); + + import.finish().unwrap(); + } + #[tokio::test(start_paused = true)] async fn fixed_track_h264_uses_existing_name_in_catalog() { let (mut broadcast, catalog) = new_broadcast(); diff --git a/rs/moq-mux/src/lib.rs b/rs/moq-mux/src/lib.rs index 460237d32..fc79999cc 100644 --- a/rs/moq-mux/src/lib.rs +++ b/rs/moq-mux/src/lib.rs @@ -21,6 +21,7 @@ pub mod codec; pub mod container; mod error; pub mod import; +pub mod publish; mod track_provider; pub use clock::Clock; diff --git a/rs/moq-mux/src/publish.rs b/rs/moq-mux/src/publish.rs new file mode 100644 index 000000000..0ee3441cc --- /dev/null +++ b/rs/moq-mux/src/publish.rs @@ -0,0 +1,99 @@ +//! Bridge from the request-based single-track importers back to a broadcast catalog. +//! +//! A single-track importer (in [`crate::codec`]) produces frames on one track and +//! exposes the catalog renditions it publishes via [`Renditions`]. Most callers, +//! though, work with a whole [`moq_net::BroadcastProducer`] plus a shared +//! [`catalog::Producer`](crate::catalog::Producer). [`Published`] is the adapter: +//! it merges an importer's renditions into that catalog and removes them on drop. +//! +//! For the broadcast-push case, mint a track with +//! [`BroadcastProducer::unique_track`](moq_net::BroadcastProducer::unique_track) and +//! build the importer `from_track`. A [`moq_net::TrackRequest`] (from +//! [`BroadcastDynamic::requested_track`](moq_net::BroadcastDynamic::requested_track)) +//! is instead the on-demand path, fed directly to the importer's `new`. + +use std::ops::{Deref, DerefMut}; + +/// A single-track importer that exposes the catalog renditions it publishes. +/// +/// Implemented by the per-codec importers so [`Published`] can merge their +/// renditions into a broadcast catalog generically. +pub trait Renditions { + /// The standalone media catalog (video/audio renditions) this importer publishes. + fn renditions(&self) -> &hang::Catalog; +} + +/// Mint a fresh unique track for a legacy single-codec importer. +/// +/// Picks a unique name from `suffix` and sets the microsecond +/// [`hang::container::TIMESCALE`] that the legacy importers stamp their frames +/// with, so the relay gets timing without parsing the payload. Hand the result +/// to the importer's `from_track`. +pub fn unique_track(broadcast: &mut moq_net::BroadcastProducer, suffix: &str) -> crate::Result { + let info = moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE); + Ok(broadcast.unique_track(suffix, info)?) +} + +/// A single-track importer attached to a broadcast catalog. +/// +/// Merges the importer's [`Renditions`] into a [`catalog::Producer`](crate::catalog::Producer) +/// on creation and removes them on drop. Derefs to the inner importer, so all of +/// its methods (`decode`, `finish`, `seek`, ...) are available directly. +pub struct Published { + inner: I, + catalog: crate::catalog::Producer, + video: Vec, + audio: Vec, +} + +impl Published { + /// Merge `inner`'s renditions into `catalog`, publishing the update. + pub fn new(mut catalog: crate::catalog::Producer, inner: I) -> Self { + let media = inner.renditions(); + let video: Vec = media.video.renditions.keys().cloned().collect(); + let audio: Vec = media.audio.renditions.keys().cloned().collect(); + + { + let mut guard = catalog.lock(); + for (name, config) in &media.video.renditions { + guard.video.renditions.insert(name.clone(), config.clone()); + } + for (name, config) in &media.audio.renditions { + guard.audio.renditions.insert(name.clone(), config.clone()); + } + } + + Self { + inner, + catalog, + video, + audio, + } + } +} + +impl Deref for Published { + type Target = I; + + fn deref(&self) -> &I { + &self.inner + } +} + +impl DerefMut for Published { + fn deref_mut(&mut self) -> &mut I { + &mut self.inner + } +} + +impl Drop for Published { + fn drop(&mut self) { + let mut guard = self.catalog.lock(); + for name in &self.video { + guard.video.renditions.remove(name); + } + for name in &self.audio { + guard.audio.renditions.remove(name); + } + } +} diff --git a/rs/moq-rtc/src/codec/opus.rs b/rs/moq-rtc/src/codec/opus.rs index 9257b04b1..e234f68c6 100644 --- a/rs/moq-rtc/src/codec/opus.rs +++ b/rs/moq-rtc/src/codec/opus.rs @@ -6,12 +6,12 @@ use crate::{Result, codec}; pub struct Bridge { - import: moq_mux::codec::opus::Import, + import: moq_mux::publish::Published, } impl Bridge { pub fn new( - broadcast: moq_net::BroadcastProducer, + mut broadcast: moq_net::BroadcastProducer, catalog: moq_mux::catalog::Producer, sample_rate: u32, channel_count: u32, @@ -20,7 +20,9 @@ impl Bridge { sample_rate, channel_count, }; - let import = moq_mux::codec::opus::Import::new(broadcast, catalog, config)?; + let track = moq_mux::publish::unique_track(&mut broadcast, ".opus")?; + let import = moq_mux::codec::opus::Import::from_track(track, config)?; + let import = moq_mux::publish::Published::new(catalog, import); Ok(Self { import }) } } @@ -30,7 +32,7 @@ impl codec::Bridge for Bridge { let pts = moq_net::Timestamp::from_micros(frame.timestamp_us) .map_err(|err| crate::Error::Other(anyhow::anyhow!("invalid timestamp: {err}")))?; let mut payload = frame.payload; - self.import.decode(&mut payload, Some(pts))?; + self.import.decode_buf(&mut payload, Some(pts))?; Ok(()) } } From 23f3bcb222f47e9ff7f46501f1eed9b2da2d0bb3 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 03:26:25 +0000 Subject: [PATCH 02/19] moq-mux: decouple the H.264 importer from the broadcast catalog Second slice of the import refactor. The H.264 importer joins opus on the request-based core: it no longer holds a `catalog::Producer` or mutates a shared catalog from a `Drop` hook. It produces frames on a single track and exposes its own `hang::Catalog`, which the `publish` bridge mirrors into a broadcast catalog. Unlike opus, H.264's catalog is lazy (avcC for avc1, the first SPS for avc3) and refines over time (jitter), so `Published` gains a `sync()` that re-mirrors the importer's renditions and is a no-op when nothing changed. Callers invoke it after each decode. - `codec::h264::Import`: drop the `E`/`catalog::Producer`/`TrackProvider` coupling. `new(TrackRequest)` (on-demand) and `from_track(TrackProducer)` (broadcast push / fixed track), `with_mode`, lazy `catalog() -> Option`, `Renditions` impl. avc1 reconfiguration now errors instead of minting a new track (a single fixed track can't represent a new init segment); avc3 SPS changes still update the rendition in place. - `publish::Published`: generic over the catalog extension `E` (so it attaches to a container's extended catalog), plus `sync()` for lazy/updating catalogs. - The TS container builds its per-PID H.264 stream through `Published` (`unique_track` + `from_track`), syncing after each decode; external behavior is unchanged (byte-exact roundtrip tests guard it). - Rewire `import::{Framed,Stream}`, moq-cli, moq-video, moq-rtc through the bridge. Drop the now-unused `TrackProvider::set_suffix`. Also fixes a pre-existing missing `.await` in the moq-ffi tests (`dynamic_track_request_can_publish_media`) that broke `cargo check --all-targets` on dev, unrelated to this change. https://claude.ai/code/session_011S7pzcg2XsPP3AExuymac1 --- rs/moq-cli/src/publish.rs | 14 +- rs/moq-ffi/src/test.rs | 3 +- rs/moq-mux/src/codec/h264/import.rs | 180 ++++++++++---------------- rs/moq-mux/src/container/ts/import.rs | 12 +- rs/moq-mux/src/import.rs | 50 ++++--- rs/moq-mux/src/publish.rs | 145 ++++++++++++++++----- rs/moq-mux/src/track_provider.rs | 6 - rs/moq-rtc/src/codec/h264.rs | 10 +- rs/moq-video/src/encode/producer.rs | 23 ++-- 9 files changed, 246 insertions(+), 197 deletions(-) diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 9cb596073..a9998c317 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -78,7 +78,7 @@ pub struct CaptureArgs { } enum PublishDecoder { - Avc3(Box), + Avc3(Box>), Fmp4(Box), Ts(Box), Hls(Box), @@ -88,7 +88,11 @@ impl PublishDecoder { /// Decode a chunk of bytes from stdin (Avc3, Fmp4, or Ts). fn decode_buf(&mut self, buffer: &mut bytes::BytesMut) -> anyhow::Result<()> { match self { - Self::Avc3(d) => Ok(d.decode_stream(buffer, None)?), + Self::Avc3(d) => { + d.decode_stream(buffer, None)?; + d.sync(); + Ok(()) + } Self::Fmp4(d) => Ok(d.decode(buffer)?), Self::Ts(d) => Ok(d.decode(buffer)?), Self::Hls(_) => unreachable!(), @@ -126,8 +130,10 @@ impl Publish { let source = match format { PublishFormat::Avc3 => { - let avc3 = moq_mux::codec::h264::Import::new(broadcast.clone(), catalog.clone()) - .with_mode(moq_mux::codec::h264::Mode::Avc3)?; + let track = moq_mux::publish::unique_track(&mut broadcast, ".avc3")?; + let avc3 = + moq_mux::codec::h264::Import::from_track(track).with_mode(moq_mux::codec::h264::Mode::Avc3)?; + let avc3 = moq_mux::publish::Published::new(catalog.clone(), avc3); Source::Stream(PublishDecoder::Avc3(Box::new(avc3))) } PublishFormat::Fmp4 => { diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index 64e3b5930..36e1264a1 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -125,9 +125,10 @@ async fn dynamic_track_request_can_publish_media() { let broadcast = MoqBroadcastProducer::new().unwrap(); let dynamic = broadcast.dynamic().unwrap(); let consumer = broadcast.consume().unwrap(); - let catalog_consumer = consumer.subscribe_catalog().unwrap(); + let catalog_consumer = consumer.subscribe_catalog().await.unwrap(); let media_consumer = consumer .subscribe_media("requested-audio".into(), crate::media::Container::Legacy, 10_000) + .await .unwrap(); let track = tokio::time::timeout(TIMEOUT, dynamic.requested_track()) diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index bede4fdd3..c4ccd1e41 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -12,9 +12,9 @@ use tokio::io::{AsyncRead, AsyncReadExt}; use super::{Error, Sps}; use crate::Result; -use crate::catalog::hang::CatalogExt; use crate::codec::annexb::{NalIterator, START_CODE}; use crate::container::jitter::MinFrameDuration; +use crate::publish::Renditions; /// The wire shape an [`Import`] is processing. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -30,10 +30,16 @@ pub enum Mode { /// H.264 importer. Handles both avc1 (length-prefixed) and avc3 (Annex-B) /// input streams; the shape is detected from the first bytes the caller /// supplies, or forced explicitly via [`with_mode`](Self::with_mode). -pub struct Import { - tracks: crate::track_provider::TrackProvider, - catalog: crate::catalog::Producer, - track: Option>, +/// +/// Build it from a [`moq_net::TrackRequest`] ([`new`](Self::new), the on-demand +/// path) or an existing [`moq_net::TrackProducer`] ([`from_track`](Self::from_track), +/// the broadcast-push / fixed-track path). The catalog rendition fills in lazily +/// once the codec config is known (avcC for avc1, the first SPS for avc3); read it +/// via [`catalog`](Self::catalog) or attach the importer to a broadcast catalog +/// with [`crate::publish::Published`]. +pub struct Import { + track: crate::container::Producer, + catalog: hang::Catalog, config: Option, state: State, zero: Option, @@ -62,24 +68,18 @@ struct Avc3Frame { contains_pps: bool, } -impl Import { - pub fn new(broadcast: moq_net::BroadcastProducer, catalog: crate::catalog::Producer) -> Self { - Self { - tracks: crate::track_provider::TrackProvider::unique(broadcast, ".avc3"), - catalog, - track: None, - config: None, - state: State::Pending { mode_hint: None }, - zero: None, - jitter: MinFrameDuration::new(), - } +impl Import { + /// Serve a track request, accepting it at the microsecond timescale. + pub fn new(request: moq_net::TrackRequest) -> Self { + let info = moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE); + Self::from_track(request.accept(info)) } - pub fn new_with_track(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { + /// Publish on an existing track producer. + pub fn from_track(track: moq_net::TrackProducer) -> Self { Self { - tracks: crate::track_provider::TrackProvider::fixed(track), - catalog, - track: None, + track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy).with_lenient_start(), + catalog: hang::Catalog::default(), config: None, state: State::Pending { mode_hint: None }, zero: None, @@ -88,24 +88,15 @@ impl Import { } /// Pin the wire shape ahead of time; skips the leading-bytes auto-detect - /// inside [`initialize`](Self::initialize). Eagerly creates the broadcast - /// track for avc3 sources so the caller can observe subscriber state - /// (`used()` / `unused()`) before any frames arrive. + /// inside [`initialize`](Self::initialize). pub fn with_mode(mut self, mode: Mode) -> Result { match mode { Mode::Avc1 => { - self.tracks.set_suffix(".avc1"); self.state = State::Pending { mode_hint: Some(Mode::Avc1), }; } Mode::Avc3 => { - self.tracks.set_suffix(".avc3"); - let track = self.tracks.create()?; - self.track = Some( - crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy) - .with_lenient_start(), - ); self.state = State::Avc3 { current: Avc3Frame::default(), sps: None, @@ -116,12 +107,15 @@ impl Import { Ok(self) } - /// Returns a reference to the underlying track producer, e.g. for - /// monitoring subscriber state via `used()` / `unused()`. Available only - /// after the track has been created. i.e. after [`with_mode`](Self::with_mode) - /// for avc3 or after [`initialize`](Self::initialize) for avc1. - pub fn track(&self) -> Option<&moq_net::TrackProducer> { - self.track.as_ref().map(|t| t.track()) + /// The underlying track producer, e.g. for monitoring subscriber state via + /// `used()` / `unused()`. + pub fn track(&self) -> &moq_net::TrackProducer { + self.track.track() + } + + /// The standalone catalog once the codec config is known, else `None`. + pub fn catalog(&self) -> Option<&hang::Catalog> { + self.config.is_some().then_some(&self.catalog) } /// Initialize from the codec's leading bytes. @@ -164,7 +158,6 @@ impl Import { config.description = Some(Bytes::copy_from_slice(avcc_bytes)); config.container = hang::catalog::Container::Legacy; - self.tracks.set_suffix(".avc1"); self.swap_config(config)?; buf.advance(buf.remaining()); @@ -172,24 +165,14 @@ impl Import { } /// Initialize the avc3 path by parsing Annex-B NALs (SPS/PPS seed the - /// catalog rendition; the track is created eagerly on first SPS). + /// catalog rendition once the first SPS is parsed). fn initialize_avc3>(&mut self, buf: &mut T) -> Result<()> { - // Eager-create the track + state on first switch into Avc3 mode so - // callers can observe `used()` / `unused()` before any frames arrive. if !matches!(self.state, State::Avc3 { .. }) { self.state = State::Avc3 { current: Avc3Frame::default(), sps: None, pps: None, }; - if self.track.is_none() { - self.tracks.set_suffix(".avc3"); - let track = self.tracks.create()?; - self.track = Some( - crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy) - .with_lenient_start(), - ); - } } let mut nals = NalIterator::new(buf); @@ -203,8 +186,9 @@ impl Import { Ok(()) } + /// True once the codec config is known and the catalog rendition is published. pub fn is_initialized(&self) -> bool { - self.track.is_some() + self.config.is_some() } /// Decode from an asynchronous reader. avc3 only — for avc1, the caller @@ -252,9 +236,8 @@ impl Import { let data = buf.as_ref(); let pts = self.pts(pts)?; let keyframe = avc1_is_keyframe(data, length_size); - let track = self.track.as_mut().ok_or(Error::NotInitialized)?; - track.write(crate::container::Frame { + self.track.write(crate::container::Frame { timestamp: pts, payload: data.to_vec().into(), keyframe, @@ -262,7 +245,7 @@ impl Import { })?; if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.lock().video.renditions.get_mut(track.name()) + && let Some(c) = self.catalog.video.renditions.get_mut(self.track.name()) { c.jitter = Some(jitter); } @@ -389,16 +372,10 @@ impl Import { return Ok(()); } - // The avc3 track was created eagerly in initialize_avc3; just publish - // (or republish) the catalog rendition with the latest config. - let track_name = self - .track - .as_ref() - .ok_or(Error::Avc3TrackNotCreated)? - .name() - .to_string(); - let mut catalog = self.catalog.lock(); - catalog.video.renditions.insert(track_name, config.clone()); + // avc3 carries SPS inline, so a resolution change updates the rendition in + // place on the same track (no new init segment, unlike avc1). + let track_name = self.track.name().to_string(); + self.catalog.video.renditions.insert(track_name, config.clone()); self.config = Some(config); Ok(()) } @@ -418,8 +395,7 @@ impl Import { current.contains_sps = false; current.contains_pps = false; - let track = self.track.as_mut().ok_or(Error::Avc3TrackNotCreated)?; - track.write(crate::container::Frame { + self.track.write(crate::container::Frame { timestamp: pts, payload, keyframe, @@ -427,48 +403,36 @@ impl Import { })?; if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.lock().video.renditions.get_mut(track.name()) + && let Some(c) = self.catalog.video.renditions.get_mut(self.track.name()) { c.jitter = Some(jitter); } Ok(()) } - /// Replace the current track + catalog rendition with `config`. Used by - /// the avc1 path on every (re)initialization. + /// Register the avc1 codec config on the (fixed) track. + /// + /// The first config seeds the catalog rendition. A different avcC would mean + /// a new init segment, which the single fixed track can't represent, so a + /// reconfiguration is an error (mint a new track via a fresh importer). fn swap_config(&mut self, config: hang::catalog::VideoConfig) -> Result<()> { - if let Some(old) = &self.config - && old == &config - { - return Ok(()); - } - - let mut catalog = self.catalog.lock(); - if let Some(track) = self.track.take() { - if self.tracks.is_fixed() { - self.track = Some(track); - return Err(Error::FixedTrackReconfigured.into()); + if let Some(old) = &self.config { + if old == &config { + return Ok(()); } - tracing::debug!(name = ?track.name(), "reinitializing H.264 track"); - catalog.video.renditions.remove(track.name()); + return Err(Error::FixedTrackReconfigured.into()); } - let track = self.tracks.create()?; - tracing::debug!(name = ?track.name(), ?config, "starting H.264 track"); - catalog - .video - .renditions - .insert(track.name().to_string(), config.clone()); + let track_name = self.track.name().to_string(); + tracing::debug!(name = ?track_name, ?config, "starting H.264 track"); + self.catalog.video.renditions.insert(track_name, config.clone()); self.config = Some(config); - self.track = - Some(crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy).with_lenient_start()); Ok(()) } /// Finish the track, flushing any buffered data. pub fn finish(&mut self) -> Result<()> { - let track = self.track.as_mut().ok_or(Error::NotInitialized)?; - track.finish()?; + self.track.finish()?; Ok(()) } @@ -480,8 +444,7 @@ impl Import { if let State::Avc3 { current, .. } = &mut self.state { *current = Avc3Frame::default(); } - let track = self.track.as_mut().ok_or(Error::NotInitialized)?; - track.seek(sequence)?; + self.track.seek(sequence)?; Ok(()) } @@ -494,12 +457,9 @@ impl Import { } } -impl Drop for Import { - fn drop(&mut self) { - if let Some(track) = self.track.take() { - tracing::debug!(name = ?track.name(), "ending H.264 track"); - self.catalog.lock().video.renditions.remove(track.name()); - } +impl Renditions for Import { + fn renditions(&self) -> &hang::Catalog { + &self.catalog } } @@ -577,17 +537,13 @@ mod tests { avcc.extend_from_slice(&sps_nal); avcc.extend_from_slice(&[0x01, 0x00, 0x04, 0x68, 0xce, 0x3c, 0x80]); // num_pps + pps - let broadcast = moq_net::BroadcastInfo::new(); - let mut producer = broadcast.produce(); - let catalog = crate::catalog::Producer::new(&mut producer).unwrap(); - - let mut importer = Import::new(producer, catalog.clone()); + let mut importer = Import::new(moq_net::TrackRequest::new("0.avc1")); let mut buf = bytes::BytesMut::from(avcc.as_slice()); importer.initialize(&mut buf).expect("initialize avc1"); - let snapshot = catalog.snapshot(); - assert_eq!(snapshot.video.renditions.len(), 1); - let cfg = snapshot.video.renditions.values().next().unwrap(); + let catalog = importer.catalog().expect("config known after init"); + assert_eq!(catalog.video.renditions.len(), 1); + let cfg = catalog.video.renditions.values().next().unwrap(); let hang::catalog::VideoCodec::H264(h264) = &cfg.codec else { panic!("expected H.264 codec") }; @@ -613,16 +569,12 @@ mod tests { annexb.extend_from_slice(&[0, 0, 0, 1]); annexb.extend_from_slice(pps); - let broadcast = moq_net::BroadcastInfo::new(); - let mut producer = broadcast.produce(); - let catalog = crate::catalog::Producer::new(&mut producer).unwrap(); - - let mut importer = Import::new(producer, catalog.clone()); + let mut importer = Import::new(moq_net::TrackRequest::new("0.avc3")); importer.initialize(&mut annexb).expect("initialize avc3"); - let snapshot = catalog.snapshot(); - assert_eq!(snapshot.video.renditions.len(), 1); - let cfg = snapshot.video.renditions.values().next().unwrap(); + let catalog = importer.catalog().expect("config known after first SPS"); + assert_eq!(catalog.video.renditions.len(), 1); + let cfg = catalog.video.renditions.values().next().unwrap(); let hang::catalog::VideoCodec::H264(h264) = &cfg.codec else { panic!("expected H.264 codec") }; diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index ea5d0a667..399a56ba3 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -262,10 +262,10 @@ impl Import { let stream = match stream_type { StreamType::H264 => { - let import = - h264::Import::new(self.broadcast.clone(), self.catalog.clone()).with_mode(h264::Mode::Avc3)?; + let track = crate::publish::unique_track(&mut self.broadcast, ".avc3")?; + let import = h264::Import::from_track(track).with_mode(h264::Mode::Avc3)?; Stream::H264 { - import: Box::new(import), + import: Box::new(crate::publish::Published::new(self.catalog.clone(), import)), unwrap: PtsUnwrap::default(), } } @@ -713,7 +713,7 @@ impl ScteReassembler { /// One elementary stream's codec importer plus PTS-unwrap state. enum Stream { H264 { - import: Box>, + import: Box>, unwrap: PtsUnwrap, }, H265 { @@ -733,7 +733,9 @@ impl Stream { match self { Stream::H264 { import, unwrap } => { let pts = unwrap_pts(unwrap, pending.pts)?; - Ok(import.decode_frame(&mut pending.data.as_slice(), pts)?) + import.decode_frame(&mut pending.data.as_slice(), pts)?; + import.sync(); + Ok(()) } Stream::H265 { import, unwrap } => { let pts = unwrap_pts(unwrap, pending.pts)?; diff --git a/rs/moq-mux/src/import.rs b/rs/moq-mux/src/import.rs index 0c3af2106..2f385a4f4 100644 --- a/rs/moq-mux/src/import.rs +++ b/rs/moq-mux/src/import.rs @@ -99,7 +99,7 @@ impl From for FramedFormat { enum FramedKind { /// H.264 (both avc1 and avc3 wire shapes go through this importer; mode /// is pinned by the caller's FramedFormat choice). - H264(crate::codec::h264::Import), + H264(crate::publish::Published), // Boxed because it's a large struct and clippy complains about the size. Fmp4(Box), Hev1(crate::codec::h265::Import), @@ -134,14 +134,16 @@ impl Framed { use crate::codec::h264::Mode as H264Mode; let decoder = match format { FramedFormat::Avc1 => { - let mut decoder = crate::codec::h264::Import::new(broadcast, catalog).with_mode(H264Mode::Avc1)?; + let track = crate::publish::unique_track(&mut broadcast, ".avc1")?; + let mut decoder = crate::codec::h264::Import::from_track(track).with_mode(H264Mode::Avc1)?; decoder.initialize(buf)?; - FramedKind::H264(decoder) + FramedKind::H264(crate::publish::Published::new(catalog, decoder)) } FramedFormat::Avc3 => { - let mut decoder = crate::codec::h264::Import::new(broadcast, catalog).with_mode(H264Mode::Avc3)?; + let track = crate::publish::unique_track(&mut broadcast, ".avc3")?; + let mut decoder = crate::codec::h264::Import::from_track(track).with_mode(H264Mode::Avc3)?; decoder.initialize(buf)?; - FramedKind::H264(decoder) + FramedKind::H264(crate::publish::Published::new(catalog, decoder)) } FramedFormat::Fmp4 => { let mut decoder = Box::new(crate::container::fmp4::Import::new(broadcast, catalog)); @@ -210,16 +212,14 @@ impl Framed { use crate::codec::h264::Mode as H264Mode; let decoder = match format { FramedFormat::Avc1 => { - let mut decoder = - crate::codec::h264::Import::new_with_track(track, catalog).with_mode(H264Mode::Avc1)?; + let mut decoder = crate::codec::h264::Import::from_track(track).with_mode(H264Mode::Avc1)?; decoder.initialize(buf)?; - FramedKind::H264(decoder) + FramedKind::H264(crate::publish::Published::new(catalog, decoder)) } FramedFormat::Avc3 => { - let mut decoder = - crate::codec::h264::Import::new_with_track(track, catalog).with_mode(H264Mode::Avc3)?; + let mut decoder = crate::codec::h264::Import::from_track(track).with_mode(H264Mode::Avc3)?; decoder.initialize(buf)?; - FramedKind::H264(decoder) + FramedKind::H264(crate::publish::Published::new(catalog, decoder)) } FramedFormat::Hev1 => { let mut decoder = crate::codec::h265::Import::new_with_track(track, catalog); @@ -295,9 +295,7 @@ impl Framed { /// Return the single track produced by this importer. pub fn track(&self) -> Result<&moq_net::TrackProducer> { match self.decoder { - FramedKind::H264(ref decoder) => decoder - .track() - .ok_or_else(|| crate::codec::h264::Error::Avc3TrackNotCreated.into()), + FramedKind::H264(ref decoder) => Ok(decoder.track()), FramedKind::Fmp4(_) => Err(crate::Error::MultipleTracks("fmp4")), FramedKind::Hev1(ref decoder) => decoder.track(), FramedKind::Av01(ref decoder) => decoder.track(), @@ -313,7 +311,10 @@ impl Framed { /// Decode a frame from the given buffer. pub fn decode_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { match self.decoder { - FramedKind::H264(ref mut decoder) => decoder.decode_frame(buf, pts)?, + FramedKind::H264(ref mut decoder) => { + decoder.decode_frame(buf, pts)?; + decoder.sync(); + } FramedKind::Fmp4(ref mut decoder) => decoder.decode(buf)?, FramedKind::Hev1(ref mut decoder) => decoder.decode_frame(buf, pts)?, FramedKind::Av01(ref mut decoder) => decoder.decode_frame(buf, pts)?, @@ -615,7 +616,7 @@ impl fmt::Display for StreamFormat { enum StreamKind { /// H.264 in avc3 wire shape (Annex-B with inline SPS/PPS). - Avc3(crate::codec::h264::Import), + Avc3(crate::publish::Published), // Boxed because it's a large struct and clippy complains about the size. Fmp4(Box), Hev1(crate::codec::h265::Import), @@ -637,14 +638,16 @@ pub struct Stream { impl Stream { /// Create a new stream importer with the given format. pub fn new( - broadcast: moq_net::BroadcastProducer, + mut broadcast: moq_net::BroadcastProducer, catalog: crate::catalog::Producer, format: StreamFormat, ) -> Result { use crate::codec::h264::Mode as H264Mode; let decoder = match format { StreamFormat::Avc3 => { - StreamKind::Avc3(crate::codec::h264::Import::new(broadcast, catalog).with_mode(H264Mode::Avc3)?) + let track = crate::publish::unique_track(&mut broadcast, ".avc3")?; + let decoder = crate::codec::h264::Import::from_track(track).with_mode(H264Mode::Avc3)?; + StreamKind::Avc3(crate::publish::Published::new(catalog, decoder)) } StreamFormat::Fmp4 => StreamKind::Fmp4(Box::new(crate::container::fmp4::Import::new(broadcast, catalog))), StreamFormat::Hev1 => StreamKind::Hev1(crate::codec::h265::Import::new(broadcast, catalog)), @@ -663,7 +666,10 @@ impl Stream { /// The buffer will be fully consumed, or an error will be returned. pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { match self.decoder { - StreamKind::Avc3(ref mut decoder) => decoder.initialize(buf)?, + StreamKind::Avc3(ref mut decoder) => { + decoder.initialize(buf)?; + decoder.sync(); + } StreamKind::Fmp4(ref mut decoder) => decoder.decode(buf)?, StreamKind::Hev1(ref mut decoder) => decoder.initialize(buf)?, StreamKind::Av01(ref mut decoder) => decoder.initialize(buf)?, @@ -681,7 +687,11 @@ impl Stream { /// Decode a stream of data from the given buffer. pub fn decode_stream>(&mut self, buf: &mut T) -> Result<()> { match self.decoder { - StreamKind::Avc3(ref mut decoder) => decoder.decode_stream(buf, None), + StreamKind::Avc3(ref mut decoder) => { + decoder.decode_stream(buf, None)?; + decoder.sync(); + Ok(()) + } StreamKind::Fmp4(ref mut decoder) => decoder.decode(buf), StreamKind::Hev1(ref mut decoder) => decoder.decode_stream(buf, None), StreamKind::Av01(ref mut decoder) => decoder.decode_stream(buf, None), diff --git a/rs/moq-mux/src/publish.rs b/rs/moq-mux/src/publish.rs index 0ee3441cc..98853f56a 100644 --- a/rs/moq-mux/src/publish.rs +++ b/rs/moq-mux/src/publish.rs @@ -6,18 +6,25 @@ //! [`catalog::Producer`](crate::catalog::Producer). [`Published`] is the adapter: //! it merges an importer's renditions into that catalog and removes them on drop. //! -//! For the broadcast-push case, mint a track with -//! [`BroadcastProducer::unique_track`](moq_net::BroadcastProducer::unique_track) and -//! build the importer `from_track`. A [`moq_net::TrackRequest`] (from +//! For the broadcast-push case, mint a track with [`unique_track`] and build the +//! importer `from_track`. A [`moq_net::TrackRequest`] (from //! [`BroadcastDynamic::requested_track`](moq_net::BroadcastDynamic::requested_track)) //! is instead the on-demand path, fed directly to the importer's `new`. +//! +//! Some importers fill their catalog lazily (H.264 only knows its config once SPS +//! arrives) or refine it over time (jitter). Call [`Published::sync`] after +//! feeding such an importer so the new/changed renditions reach the catalog; it's +//! a cheap comparison when nothing changed. use std::ops::{Deref, DerefMut}; +use crate::catalog::hang::CatalogExt; + /// A single-track importer that exposes the catalog renditions it publishes. /// /// Implemented by the per-codec importers so [`Published`] can merge their -/// renditions into a broadcast catalog generically. +/// renditions into a broadcast catalog generically. The returned catalog may be +/// empty (and grow later) for importers that initialize lazily. pub trait Renditions { /// The standalone media catalog (video/audio renditions) this importer publishes. fn renditions(&self) -> &hang::Catalog; @@ -36,43 +43,71 @@ pub fn unique_track(broadcast: &mut moq_net::BroadcastProducer, suffix: &str) -> /// A single-track importer attached to a broadcast catalog. /// -/// Merges the importer's [`Renditions`] into a [`catalog::Producer`](crate::catalog::Producer) -/// on creation and removes them on drop. Derefs to the inner importer, so all of -/// its methods (`decode`, `finish`, `seek`, ...) are available directly. -pub struct Published { +/// Mirrors the importer's [`Renditions`] into a [`catalog::Producer`](crate::catalog::Producer) +/// and removes them on drop. Derefs to the inner importer, so all of its methods +/// (`decode`, `finish`, `seek`, ...) are available directly. Generic over the +/// catalog extension `E` so it can attach to an extended broadcast catalog (e.g. +/// the one a container holds). +pub struct Published { inner: I, - catalog: crate::catalog::Producer, - video: Vec, - audio: Vec, + catalog: crate::catalog::Producer, + /// The renditions we last mirrored into the catalog, so [`sync`](Self::sync) + /// can diff against the importer's current state and retire them on drop. + published: hang::Catalog, } -impl Published { - /// Merge `inner`'s renditions into `catalog`, publishing the update. - pub fn new(mut catalog: crate::catalog::Producer, inner: I) -> Self { - let media = inner.renditions(); - let video: Vec = media.video.renditions.keys().cloned().collect(); - let audio: Vec = media.audio.renditions.keys().cloned().collect(); +impl Published { + /// Attach `inner` to `catalog`, mirroring whatever renditions it already has. + pub fn new(catalog: crate::catalog::Producer, inner: I) -> Self { + let mut this = Self { + inner, + catalog, + published: hang::Catalog::default(), + }; + this.sync(); + this + } + + /// Re-mirror the importer's current renditions into the catalog. + /// + /// Call this after feeding an importer whose catalog appears or changes + /// lazily (H.264 once SPS is parsed, jitter refinement, ...). It's a cheap + /// comparison and touches the catalog only when something actually changed. + pub fn sync(&mut self) { + let current = self.inner.renditions(); + if self.published == *current { + return; + } { - let mut guard = catalog.lock(); - for (name, config) in &media.video.renditions { + let mut guard = self.catalog.lock(); + + // Retire renditions we published before that the importer dropped. + for name in self.published.video.renditions.keys() { + if !current.video.renditions.contains_key(name) { + guard.video.renditions.remove(name); + } + } + for name in self.published.audio.renditions.keys() { + if !current.audio.renditions.contains_key(name) { + guard.audio.renditions.remove(name); + } + } + + // Insert or update the current ones. + for (name, config) in ¤t.video.renditions { guard.video.renditions.insert(name.clone(), config.clone()); } - for (name, config) in &media.audio.renditions { + for (name, config) in ¤t.audio.renditions { guard.audio.renditions.insert(name.clone(), config.clone()); } } - Self { - inner, - catalog, - video, - audio, - } + self.published = current.clone(); } } -impl Deref for Published { +impl Deref for Published { type Target = I; fn deref(&self) -> &I { @@ -80,20 +115,68 @@ impl Deref for Published { } } -impl DerefMut for Published { +impl DerefMut for Published { fn deref_mut(&mut self) -> &mut I { &mut self.inner } } -impl Drop for Published { +impl Drop for Published { fn drop(&mut self) { + if self.published == hang::Catalog::default() { + return; + } + let mut guard = self.catalog.lock(); - for name in &self.video { + for name in self.published.video.renditions.keys() { guard.video.renditions.remove(name); } - for name in &self.audio { + for name in self.published.audio.renditions.keys() { guard.audio.renditions.remove(name); } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// An importer whose catalog we can mutate, to drive [`Published::sync`]. + struct Fake(hang::Catalog); + + impl Renditions for Fake { + fn renditions(&self) -> &hang::Catalog { + &self.0 + } + } + + fn video() -> hang::catalog::VideoConfig { + let mut config = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); + config.container = hang::catalog::Container::Legacy; + config + } + + #[tokio::test(start_paused = true)] + async fn sync_propagates_lazily_and_drop_retires() { + let mut broadcast = moq_net::BroadcastInfo::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); + + // Importer starts with an empty catalog (lazy init): nothing merged yet. + let mut published = Published::new(catalog.clone(), Fake(hang::Catalog::default())); + assert!(catalog.snapshot().video.renditions.is_empty()); + + // A rendition appears later; sync mirrors it into the broadcast catalog. + published.0.video.renditions.insert("v".to_string(), video()); + published.sync(); + assert!(catalog.snapshot().video.renditions.contains_key("v")); + + // An update to the same rendition propagates too. + published.0.video.renditions.get_mut("v").unwrap().bitrate = Some(1_000); + published.sync(); + assert_eq!(catalog.snapshot().video.renditions["v"].bitrate, Some(1_000)); + + // Dropping the wrapper retires the rendition. + drop(published); + assert!(catalog.snapshot().video.renditions.is_empty()); + } +} diff --git a/rs/moq-mux/src/track_provider.rs b/rs/moq-mux/src/track_provider.rs index fde002678..01ca3e585 100644 --- a/rs/moq-mux/src/track_provider.rs +++ b/rs/moq-mux/src/track_provider.rs @@ -19,12 +19,6 @@ impl TrackProvider { matches!(self, Self::Fixed(_)) } - pub(crate) fn set_suffix(&mut self, next: &'static str) { - if let Self::Unique { suffix, .. } = self { - *suffix = next; - } - } - pub(crate) fn create(&mut self) -> crate::Result { match self { Self::Unique { broadcast, suffix } => { diff --git a/rs/moq-rtc/src/codec/h264.rs b/rs/moq-rtc/src/codec/h264.rs index 396fadf53..b0111d287 100644 --- a/rs/moq-rtc/src/codec/h264.rs +++ b/rs/moq-rtc/src/codec/h264.rs @@ -10,13 +10,14 @@ use bytes::BytesMut; use crate::{Result, codec}; pub struct Bridge { - import: moq_mux::codec::h264::Import, + import: moq_mux::publish::Published, } impl Bridge { - pub fn new(broadcast: moq_net::BroadcastProducer, catalog: moq_mux::catalog::Producer) -> Result { - let import = - moq_mux::codec::h264::Import::new(broadcast, catalog).with_mode(moq_mux::codec::h264::Mode::Avc3)?; + pub fn new(mut broadcast: moq_net::BroadcastProducer, catalog: moq_mux::catalog::Producer) -> Result { + let track = moq_mux::publish::unique_track(&mut broadcast, ".avc3")?; + let import = moq_mux::codec::h264::Import::from_track(track).with_mode(moq_mux::codec::h264::Mode::Avc3)?; + let import = moq_mux::publish::Published::new(catalog, import); Ok(Self { import }) } } @@ -27,6 +28,7 @@ impl codec::Bridge for Bridge { .map_err(|err| crate::Error::Other(anyhow::anyhow!("invalid timestamp: {err}")))?; let mut buf = BytesMut::from(frame.payload.as_ref()); self.import.decode_frame(&mut buf, Some(pts))?; + self.import.sync(); Ok(()) } } diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index f573c1646..d1d0baffb 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -25,20 +25,21 @@ const DEFAULT_FRAMERATE: u32 = 30; /// trigger capture on demand. `moq_mux::codec::h264::Import` handles /// catalog registration and framing. pub struct Producer { - import: moq_mux::codec::h264::Import, + import: moq_mux::publish::Published, } impl Producer { - pub fn new(broadcast: moq_net::BroadcastProducer, catalog: moq_mux::catalog::Producer) -> Result { - let import = - moq_mux::codec::h264::Import::new(broadcast, catalog).with_mode(moq_mux::codec::h264::Mode::Avc3)?; + pub fn new(mut broadcast: moq_net::BroadcastProducer, catalog: moq_mux::catalog::Producer) -> Result { + let track = moq_mux::publish::unique_track(&mut broadcast, ".avc3")?; + let import = moq_mux::codec::h264::Import::from_track(track).with_mode(moq_mux::codec::h264::Mode::Avc3)?; + let import = moq_mux::publish::Published::new(catalog, import); Ok(Self { import }) } - /// The underlying track producer, eagerly created by avc3 mode. Clone it - /// to watch subscription state via [`used`](moq_net::TrackProducer::used) / - /// [`unused`](moq_net::TrackProducer::unused). - pub fn track(&self) -> Option<&moq_net::TrackProducer> { + /// The underlying track producer, created eagerly so subscription state is + /// observable before any frames arrive. Clone it to watch via + /// [`used`](moq_net::TrackProducer::used) / [`unused`](moq_net::TrackProducer::unused). + pub fn track(&self) -> &moq_net::TrackProducer { self.import.track() } @@ -47,6 +48,7 @@ impl Producer { for mut packet in packets { self.import.decode_frame(&mut packet, Some(timestamp))?; } + self.import.sync(); Ok(()) } @@ -94,10 +96,7 @@ pub async fn publish_capture( } let producer = Producer::new(broadcast, catalog)?; - let track = producer - .track() - .cloned() - .ok_or_else(|| Error::Codec(anyhow::anyhow!("avc3 track was not created")))?; + let track = producer.track().clone(); let gate = Gate::new(); From e3151896784c5cf5e4f9ea191bac5b43e15f4165 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 03:32:29 +0000 Subject: [PATCH 03/19] moq-boy: fix track() call after moq-video Producer::track() lost its Option moq-video's `Producer::track()` now returns `&TrackProducer` (the avc3 track is always created eagerly), so the `.expect(...)` no longer applies. This broke `cargo check --all-targets` on moq-boy. https://claude.ai/code/session_011S7pzcg2XsPP3AExuymac1 --- rs/moq-boy/src/video.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/moq-boy/src/video.rs b/rs/moq-boy/src/video.rs index 3eda1ddd2..17daf9e5d 100644 --- a/rs/moq-boy/src/video.rs +++ b/rs/moq-boy/src/video.rs @@ -36,7 +36,7 @@ impl VideoEncoder { pub fn spawn(broadcast: moq_net::BroadcastProducer, catalog: moq_mux::catalog::Producer) -> Self { let (tx, rx) = tokio::sync::mpsc::channel(4); let producer = moq_video::encode::Producer::new(broadcast, catalog).expect("failed to create avc3 producer"); - let track = producer.track().expect("avc3 track is eagerly created").clone(); + let track = producer.track().clone(); let force_keyframe = Arc::new(AtomicBool::new(false)); let encode_duration = Arc::new(AtomicU64::new(0)); From 433e8eaad8f429942f83724de15f20b9246474c6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 06:07:26 +0000 Subject: [PATCH 04/19] moq-mux: port the remaining importers off the broadcast catalog Completes the import reshape: h265, av1, vp8, vp9, aac, and the legacy audio importer (MP2/AC-3/E-AC-3) now follow the same request-based core as opus/h264. None hold a `catalog::Producer` or mutate a shared catalog from a `Drop` hook; each produces frames on a single track and reports its own `hang::Catalog`, attached to a broadcast catalog through `publish::Published`. - Each importer: `new(TrackRequest)` / `from_track(TrackProducer)`, a local catalog, lazy `catalog() -> Option` for the video codecs (config known on the first key frame / SPS) and eager `catalog() -> &hang::Catalog` for aac. Reconfiguration on the fixed track is an error (no new-track minting). - `import::{Framed,Stream}`: every codec arm mints via `unique_track` + `from_track` + `Published`, syncing the video codecs after each decode. - TS container: H.265, AAC, and legacy per-PID streams build through `Published`. AAC's synthesized `description` and audio-burst `jitter` are set via a `pub(crate) aac::Import::rendition_mut` + `sync`, since the rendition now lives in the importer's local catalog. External behavior is unchanged (the byte-exact roundtrip tests guard it). - Delete the now-unused `TrackProvider` (every codec mints via `unique_track`). All codec/TS importers now share one shape; `TrackRequest` is the on-demand entry point and `from_track` the broadcast-push one. https://claude.ai/code/session_011S7pzcg2XsPP3AExuymac1 --- rs/moq-mux/src/codec/aac/import.rs | 77 ++++++++-------- rs/moq-mux/src/codec/av1/import.rs | 116 +++++++++--------------- rs/moq-mux/src/codec/h265/import.rs | 119 +++++++++++-------------- rs/moq-mux/src/codec/legacy.rs | 45 ++++------ rs/moq-mux/src/codec/vp8/import.rs | 122 ++++++++++--------------- rs/moq-mux/src/codec/vp9/import.rs | 115 +++++++++--------------- rs/moq-mux/src/container/ts/import.rs | 45 ++++++---- rs/moq-mux/src/import.rs | 123 +++++++++++++++++--------- rs/moq-mux/src/lib.rs | 1 - rs/moq-mux/src/track_provider.rs | 33 ------- 10 files changed, 344 insertions(+), 452 deletions(-) delete mode 100644 rs/moq-mux/src/track_provider.rs diff --git a/rs/moq-mux/src/codec/aac/import.rs b/rs/moq-mux/src/codec/aac/import.rs index a956482bb..1699d73cb 100644 --- a/rs/moq-mux/src/codec/aac/import.rs +++ b/rs/moq-mux/src/codec/aac/import.rs @@ -1,48 +1,31 @@ use bytes::{Buf, BytesMut}; use super::Config; -use crate::catalog::hang::CatalogExt; +use crate::publish::Renditions; /// AAC importer. /// /// Initialized from an AudioSpecificConfig blob (variable-length, typically extracted from -/// an MP4 ESDS atom). Each input buffer passed to [`decode`](Self::decode) is published as -/// one hang frame in its own group, so the relay can forward each frame without waiting for -/// a group boundary. The codec's packet loss concealment handles drops. -pub struct Import { - catalog: crate::catalog::Producer, +/// an MP4 ESDS atom), so its catalog is known up front. Each input buffer passed to +/// [`decode`](Self::decode) is published as one hang frame in its own group, so the relay can +/// forward each frame without waiting for a group boundary. The codec's packet loss +/// concealment handles drops. Build it from a [`moq_net::TrackRequest`] ([`new`](Self::new)) +/// or an existing track ([`from_track`](Self::from_track)). +pub struct Import { track: crate::container::Producer, + catalog: hang::Catalog, zero: Option, } -impl Import { - pub fn new( - broadcast: moq_net::BroadcastProducer, - catalog: crate::catalog::Producer, - config: Config, - ) -> crate::Result { - Self::new_with_source( - crate::track_provider::TrackProvider::unique(broadcast, ".aac"), - catalog, - config, - ) - } - - pub fn new_with_track( - track: moq_net::TrackProducer, - catalog: crate::catalog::Producer, - config: Config, - ) -> crate::Result { - Self::new_with_source(crate::track_provider::TrackProvider::fixed(track), catalog, config) +impl Import { + /// Serve a track request, accepting it at the microsecond timescale. + pub fn new(request: moq_net::TrackRequest, config: Config) -> crate::Result { + let info = moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE); + Self::from_track(request.accept(info), config) } - fn new_with_source( - mut tracks: crate::track_provider::TrackProvider, - mut catalog: crate::catalog::Producer, - config: Config, - ) -> crate::Result { - let track = tracks.create()?; - + /// Publish on an existing track producer. + pub fn from_track(track: moq_net::TrackProducer, config: Config) -> crate::Result { let mut audio_config = hang::catalog::AudioConfig::new( hang::catalog::AAC { profile: config.profile, @@ -53,24 +36,35 @@ impl Import { audio_config.container = hang::catalog::Container::Legacy; tracing::debug!(name = ?track.name(), config = ?audio_config, "starting track"); - catalog - .lock() - .audio - .renditions - .insert(track.name().to_string(), audio_config); + + let mut catalog = hang::Catalog::default(); + catalog.audio.renditions.insert(track.name().to_string(), audio_config); Ok(Self { - catalog, track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), + catalog, zero: None, }) } + /// The standalone catalog this importer publishes (one AAC audio rendition). + pub fn catalog(&self) -> &hang::Catalog { + &self.catalog + } + /// Returns a reference to the underlying track producer. pub fn track(&self) -> &moq_net::TrackProducer { self.track.track() } + /// Mutable access to the single audio rendition, for callers that refine it + /// after construction (the TS importer sets the synthesized `description` and + /// an audio-burst `jitter`). Follow with [`crate::publish::Published::sync`]. + pub(crate) fn rendition_mut(&mut self) -> Option<&mut hang::catalog::AudioConfig> { + let name = self.track.name(); + self.catalog.audio.renditions.get_mut(name) + } + /// Finish the track, flushing the current group. pub fn finish(&mut self) -> crate::Result<()> { self.track.finish()?; @@ -120,9 +114,8 @@ impl Import { } } -impl Drop for Import { - fn drop(&mut self) { - tracing::debug!(name = ?self.track.name(), "ending track"); - self.catalog.lock().audio.renditions.remove(self.track.name()); +impl Renditions for Import { + fn renditions(&self) -> &hang::Catalog { + &self.catalog } } diff --git a/rs/moq-mux/src/codec/av1/import.rs b/rs/moq-mux/src/codec/av1/import.rs index 9076913d3..af6029b48 100644 --- a/rs/moq-mux/src/codec/av1/import.rs +++ b/rs/moq-mux/src/codec/av1/import.rs @@ -1,4 +1,5 @@ use crate::container::jitter::MinFrameDuration; +use crate::publish::Renditions; use bytes::BytesMut; use bytes::{Buf, Bytes}; @@ -9,14 +10,11 @@ use crate::Result; /// A decoder for AV1 with inline sequence headers. pub struct Import { - // Where new media tracks come from. - tracks: crate::track_provider::TrackProvider, - - // The catalog being produced. - catalog: crate::catalog::Producer, - // The track being produced. - track: Option>, + track: crate::container::Producer, + + // The standalone catalog, populated once the config is known. + catalog: hang::Catalog, // Whether the track has been initialized. config: Option, @@ -39,23 +37,17 @@ struct Frame { } impl Import { - pub fn new(broadcast: moq_net::BroadcastProducer, catalog: crate::catalog::Producer) -> Self { - Self { - tracks: crate::track_provider::TrackProvider::unique(broadcast, ".av01"), - catalog, - track: None, - config: None, - current: Default::default(), - zero: None, - jitter: MinFrameDuration::new(), - } + /// Serve a track request, accepting it at the microsecond timescale. + pub fn new(request: moq_net::TrackRequest) -> Self { + let info = moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE); + Self::from_track(request.accept(info)) } - pub fn new_with_track(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { + /// Publish on an existing track producer. + pub fn from_track(track: moq_net::TrackProducer) -> Self { Self { - tracks: crate::track_provider::TrackProvider::fixed(track), - catalog, - track: None, + track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), + catalog: hang::Catalog::default(), config: None, current: Default::default(), zero: None, @@ -101,28 +93,17 @@ impl Import { return Ok(()); } - if self.track.is_some() && self.tracks.is_fixed() { + if self.config.is_some() { return Err(Error::FixedTrackReconfigured.into()); } - if let Some(track) = self.track.take() { - tracing::debug!(name = ?track.name(), "reinitializing track"); - self.catalog.lock().video.renditions.remove(track.name()); - } - - let track = self.tracks.create()?; - tracing::debug!(name = ?track.name(), ?config, "starting track"); + tracing::debug!(name = ?self.track.name(), ?config, "starting track"); self.catalog - .lock() .video .renditions - .insert(track.name().to_string(), config.clone()); + .insert(self.track.name().to_string(), config.clone()); self.config = Some(config); - self.track = Some(crate::container::Producer::new( - track, - crate::catalog::hang::Container::Legacy, - )); Ok(()) } @@ -145,23 +126,17 @@ impl Import { }); config.container = hang::catalog::Container::Legacy; - if self.track.is_some() && self.tracks.is_fixed() { + if self.config.is_some() { return Err(Error::FixedTrackReconfigured.into()); } - let track = self.tracks.create()?; - tracing::debug!(name = ?track.name(), "starting track with minimal config"); + tracing::debug!(name = ?self.track.name(), "starting track with minimal config"); self.catalog - .lock() .video .renditions - .insert(track.name().to_string(), config.clone()); + .insert(self.track.name().to_string(), config.clone()); self.config = Some(config); - self.track = Some(crate::container::Producer::new( - track, - crate::catalog::hang::Container::Legacy, - )); Ok(()) } @@ -222,33 +197,28 @@ impl Import { return Ok(()); } - if self.track.is_some() && self.tracks.is_fixed() { + if self.config.is_some() { return Err(Error::FixedTrackReconfigured.into()); } - if let Some(track) = self.track.take() { - self.catalog.lock().video.renditions.remove(track.name()); - } - - let track = self.tracks.create()?; self.catalog - .lock() .video .renditions - .insert(track.name().to_string(), config.clone()); + .insert(self.track.name().to_string(), config.clone()); self.config = Some(config); - self.track = Some(crate::container::Producer::new( - track, - crate::catalog::hang::Container::Legacy, - )); Ok(()) } - /// Returns a reference to the underlying track producer. - pub fn track(&self) -> Result<&moq_net::TrackProducer> { - Ok(self.track.as_ref().ok_or(Error::NotInitialized)?.track()) + /// The standalone catalog once the config is known, else `None`. + pub fn catalog(&self) -> Option<&hang::Catalog> { + self.config.is_some().then_some(&self.catalog) + } + + /// The underlying track producer. + pub fn track(&self) -> &moq_net::TrackProducer { + self.track.track() } /// Decode as much data as possible from the given buffer. @@ -304,7 +274,7 @@ impl Import { } Err(_) => { // Use minimal config so stream can work (catalog won't have full info) - if self.track.is_none() { + if self.config.is_none() { tracing::debug!("Sequence header parsing failed, initializing with minimal config"); self.init_minimal()?; } @@ -372,7 +342,9 @@ impl Import { return Ok(()); } - let track = self.track.as_mut().ok_or(Error::MissingSequenceHeader)?; + if self.config.is_none() { + return Err(Error::MissingSequenceHeader.into()); + } let pts = pts.ok_or(Error::MissingTimestamp)?; let payload = std::mem::take(&mut self.current.chunks).freeze(); @@ -384,10 +356,10 @@ impl Import { duration: None, }; - track.write(frame)?; + self.track.write(frame)?; if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.lock().video.renditions.get_mut(track.name()) + && let Some(c) = self.catalog.video.renditions.get_mut(self.track.name()) { c.jitter = Some(jitter); } @@ -400,8 +372,7 @@ impl Import { /// Finish the track, flushing the current group. pub fn finish(&mut self) -> Result<()> { - let track = self.track.as_mut().ok_or(Error::NotInitialized)?; - track.finish()?; + self.track.finish()?; Ok(()) } @@ -411,13 +382,13 @@ impl Import { /// into the post-seek group with the wrong timestamp. pub fn seek(&mut self, sequence: u64) -> Result<()> { self.current = Frame::default(); - let track = self.track.as_mut().ok_or(Error::NotInitialized)?; - track.seek(sequence)?; + self.track.seek(sequence)?; Ok(()) } + /// True once the config is known and the catalog has been populated. pub fn is_initialized(&self) -> bool { - self.track.is_some() + self.config.is_some() } fn pts(&mut self, hint: Option) -> Result { @@ -430,12 +401,9 @@ impl Import { } } -impl Drop for Import { - fn drop(&mut self) { - if let Some(track) = self.track.take() { - tracing::debug!(name = ?track.name(), "ending track"); - self.catalog.lock().video.renditions.remove(track.name()); - } +impl Renditions for Import { + fn renditions(&self) -> &hang::Catalog { + &self.catalog } } diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index 9790b184f..943ddffa8 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -1,6 +1,6 @@ -use crate::catalog::hang::CatalogExt; use crate::codec::annexb::{NalIterator, START_CODE}; use crate::container::jitter::MinFrameDuration; +use crate::publish::Renditions; use bytes::{Buf, Bytes, BytesMut}; use scuffle_h265::{NALUnitType, SpsNALUnit}; @@ -10,18 +10,18 @@ use crate::Result; /// A decoder for H.265 with inline SPS/PPS. /// Only supports single layer streams (VPS is cached but not parsed). -pub struct Import { - // Where new media tracks come from. - tracks: crate::track_provider::TrackProvider, - - // The catalog being produced. - catalog: crate::catalog::Producer, - +/// +/// Build it from a [`moq_net::TrackRequest`] ([`new`](Self::new)) or an existing +/// track ([`from_track`](Self::from_track)). The catalog rendition fills in lazily +/// once the first SPS is parsed; read it via [`catalog`](Self::catalog). +pub struct Import { // The track being produced. - track: Option>, + track: crate::container::Producer, - // Whether the track has been initialized. - // If it changes, then we'll reinitialize with a new track. + // The standalone catalog, populated once the codec config is known. + catalog: hang::Catalog, + + // The resolved config; a change to it on a fixed track is an error. config: Option, // The current frame being built. @@ -39,27 +39,18 @@ pub struct Import { jitter: MinFrameDuration, } -impl Import { - pub fn new(broadcast: moq_net::BroadcastProducer, catalog: crate::catalog::Producer) -> Self { - Self { - tracks: crate::track_provider::TrackProvider::unique(broadcast, ".hev1"), - catalog, - track: None, - config: None, - current: Default::default(), - zero: None, - vps: None, - sps: None, - pps: None, - jitter: MinFrameDuration::new(), - } +impl Import { + /// Serve a track request, accepting it at the microsecond timescale. + pub fn new(request: moq_net::TrackRequest) -> Self { + let info = moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE); + Self::from_track(request.accept(info)) } - pub fn new_with_track(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { + /// Publish on an existing track producer. + pub fn from_track(track: moq_net::TrackProducer) -> Self { Self { - tracks: crate::track_provider::TrackProvider::fixed(track), - catalog, - track: None, + track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy).with_lenient_start(), + catalog: hang::Catalog::default(), config: None, current: Default::default(), zero: None, @@ -90,33 +81,19 @@ impl Import { config.display_ratio_height = vui_data.display_ratio_height; config.container = hang::catalog::Container::Legacy; - if let Some(old) = &self.config - && old == &config - { - return Ok(()); - } - - let mut catalog = self.catalog.lock(); - - if self.track.is_some() && self.tracks.is_fixed() { + if let Some(old) = &self.config { + if old == &config { + return Ok(()); + } + // A different SPS would need a new init segment, which the single fixed + // track can't represent. return Err(Error::FixedTrackReconfigured.into()); } - if let Some(track) = self.track.take() { - tracing::debug!(name = ?track.name(), "reinitializing track"); - catalog.video.renditions.remove(track.name()); - } - - let track = self.tracks.create()?; - tracing::debug!(name = ?track.name(), ?config, "starting track"); - catalog - .video - .renditions - .insert(track.name().to_string(), config.clone()); - + let track_name = self.track.name().to_string(); + tracing::debug!(name = ?track_name, ?config, "starting track"); + self.catalog.video.renditions.insert(track_name, config.clone()); self.config = Some(config); - self.track = - Some(crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy).with_lenient_start()); Ok(()) } @@ -136,9 +113,14 @@ impl Import { Ok(()) } - /// Returns a reference to the underlying track producer. - pub fn track(&self) -> Result<&moq_net::TrackProducer> { - Ok(self.track.as_ref().ok_or(Error::NotInitialized)?.track()) + /// The underlying track producer. + pub fn track(&self) -> &moq_net::TrackProducer { + self.track.track() + } + + /// The standalone catalog once the first SPS is parsed, else `None`. + pub fn catalog(&self) -> Option<&hang::Catalog> { + self.config.is_some().then_some(&self.catalog) } /// Decode as much data as possible from the given buffer. @@ -311,7 +293,10 @@ impl Import { return Ok(()); } - let track = self.track.as_mut().ok_or(Error::MissingSps)?; + // A slice before the first SPS has no catalog config to anchor it. + if self.config.is_none() { + return Err(Error::MissingSps.into()); + } let pts = pts.ok_or(Error::MissingTimestamp)?; let payload = std::mem::take(&mut self.current.chunks).freeze(); @@ -323,10 +308,10 @@ impl Import { duration: None, }; - track.write(frame)?; + self.track.write(frame)?; if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.lock().video.renditions.get_mut(track.name()) + && let Some(c) = self.catalog.video.renditions.get_mut(self.track.name()) { c.jitter = Some(jitter); } @@ -342,8 +327,7 @@ impl Import { /// Finish the track, flushing the current group. pub fn finish(&mut self) -> Result<()> { - let track = self.track.as_mut().ok_or(Error::NotInitialized)?; - track.finish()?; + self.track.finish()?; Ok(()) } @@ -353,13 +337,13 @@ impl Import { /// into the post-seek group with the wrong timestamp. pub fn seek(&mut self, sequence: u64) -> Result<()> { self.current = Frame::default(); - let track = self.track.as_mut().ok_or(Error::NotInitialized)?; - track.seek(sequence)?; + self.track.seek(sequence)?; Ok(()) } + /// True once the first SPS has populated the catalog. pub fn is_initialized(&self) -> bool { - self.track.is_some() + self.config.is_some() } fn pts(&mut self, hint: Option) -> Result { @@ -372,12 +356,9 @@ impl Import { } } -impl Drop for Import { - fn drop(&mut self) { - if let Some(track) = &self.track { - tracing::debug!(name = ?track.name(), "ending track"); - self.catalog.lock().video.renditions.remove(track.name()); - } +impl Renditions for Import { + fn renditions(&self) -> &hang::Catalog { + &self.catalog } } diff --git a/rs/moq-mux/src/codec/legacy.rs b/rs/moq-mux/src/codec/legacy.rs index 7dfcdaca9..ccaa51e61 100644 --- a/rs/moq-mux/src/codec/legacy.rs +++ b/rs/moq-mux/src/codec/legacy.rs @@ -10,7 +10,7 @@ use bytes::{Buf, BytesMut}; use moq_net::Timestamp; -use crate::catalog::hang::CatalogExt; +use crate::publish::Renditions; /// A parsed legacy-audio frame header. #[derive(Debug)] @@ -48,26 +48,16 @@ pub(crate) struct Config { /// Publishes each whole frame as one hang frame in its own group, so the relay /// forwards it immediately. The audio is never decoded; the catalog carries the /// codec, sample rate and channel count read from the frame header. -pub(crate) struct Import { - catalog: crate::catalog::Producer, +pub(crate) struct Import { track: crate::container::Producer, + catalog: hang::Catalog, zero: Option, } -impl Import { - pub fn new( - descriptor: &'static Descriptor, - mut broadcast: moq_net::BroadcastProducer, - mut catalog: crate::catalog::Producer, - config: Config, - ) -> anyhow::Result { - // The legacy container normalizes the per-frame timestamp to microseconds on - // the wire, so the track declares that timescale to match. - let track = broadcast.unique_track( - descriptor.track_suffix, - moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), - )?; - +impl Import { + /// Publish on an existing track. Mint it at the descriptor's suffix and the + /// microsecond [`hang::container::TIMESCALE`] (e.g. via [`crate::publish::unique_track`]). + pub fn from_track(descriptor: &'static Descriptor, track: moq_net::TrackProducer, config: Config) -> Self { let mut audio_config = hang::catalog::AudioConfig::new(descriptor.codec.clone(), config.sample_rate, config.channel_count); audio_config.container = hang::catalog::Container::Legacy; @@ -76,17 +66,15 @@ impl Import { // cannot decode these codecs). Fill it only if a real consumer ever needs it. tracing::debug!(name = ?track.name(), config = ?audio_config, "starting track"); - catalog - .lock() - .audio - .renditions - .insert(track.name().to_string(), audio_config); - Ok(Self { - catalog, + let mut catalog = hang::Catalog::default(); + catalog.audio.renditions.insert(track.name().to_string(), audio_config); + + Self { track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), + catalog, zero: None, - }) + } } /// Finish the track, flushing the current group. @@ -136,9 +124,8 @@ impl Import { } } -impl Drop for Import { - fn drop(&mut self) { - tracing::debug!(name = ?self.track.name(), "ending track"); - self.catalog.lock().audio.renditions.remove(self.track.name()); +impl Renditions for Import { + fn renditions(&self) -> &hang::Catalog { + &self.catalog } } diff --git a/rs/moq-mux/src/codec/vp8/import.rs b/rs/moq-mux/src/codec/vp8/import.rs index 67bdb698e..3261bf0b8 100644 --- a/rs/moq-mux/src/codec/vp8/import.rs +++ b/rs/moq-mux/src/codec/vp8/import.rs @@ -1,7 +1,7 @@ -use anyhow::Context; use bytes::Buf; use crate::container::jitter::MinFrameDuration; +use crate::publish::Renditions; use super::FrameHeader; @@ -9,17 +9,15 @@ use super::FrameHeader; /// /// A VP8 elementary stream isn't self-delimiting, so the caller must pass whole /// frames, one per [`decode_frame`](Self::decode_frame). The first key frame's -/// header supplies the catalog dimensions; the track is created lazily so the -/// importer can be constructed before any media arrives. +/// header supplies the catalog dimensions, so [`catalog`](Self::catalog) is +/// `None` until then. Build it from a [`moq_net::TrackRequest`] ([`new`](Self::new)) +/// or an existing track ([`from_track`](Self::from_track)). pub struct Import { - // Where new media tracks come from. - tracks: crate::track_provider::TrackProvider, + // The track being produced. + track: crate::container::Producer, - // The catalog being produced. - catalog: crate::catalog::Producer, - - // The track being produced, created on the first key frame. - track: Option>, + // The standalone catalog, populated on the first key frame. + catalog: hang::Catalog, // The resolved config, used to detect resolution changes. config: Option, @@ -32,22 +30,17 @@ pub struct Import { } impl Import { - pub fn new(broadcast: moq_net::BroadcastProducer, catalog: crate::catalog::Producer) -> Self { - Self { - tracks: crate::track_provider::TrackProvider::unique(broadcast, ".vp8"), - catalog, - track: None, - config: None, - zero: None, - jitter: MinFrameDuration::new(), - } + /// Serve a track request, accepting it at the microsecond timescale. + pub fn new(request: moq_net::TrackRequest) -> Self { + let info = moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE); + Self::from_track(request.accept(info)) } - pub fn new_with_track(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { + /// Publish on an existing track producer. + pub fn from_track(track: moq_net::TrackProducer) -> Self { Self { - tracks: crate::track_provider::TrackProvider::fixed(track), - catalog, - track: None, + track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), + catalog: hang::Catalog::default(), config: None, zero: None, jitter: MinFrameDuration::new(), @@ -57,9 +50,9 @@ impl Import { /// Initialize the importer. /// /// VP8 has no out-of-band configuration record, so this is normally called - /// with an empty buffer (gstreamer / ffi pass `Bytes::new()`) and the track - /// is created lazily from the first key frame. If the caller does pass the - /// first frame here, it's decoded so nothing is dropped. + /// with an empty buffer (gstreamer / ffi pass `Bytes::new()`) and the catalog + /// is filled from the first key frame. If the caller does pass the first frame + /// here, it's decoded so nothing is dropped. pub fn initialize>(&mut self, buf: &mut T) -> anyhow::Result<()> { if buf.has_remaining() { self.decode_frame(buf, None)?; @@ -77,28 +70,16 @@ impl Import { return Ok(()); } - if self.track.is_some() && self.tracks.is_fixed() { + if self.config.is_some() { anyhow::bail!("fixed track cannot be reconfigured"); } - if let Some(track) = self.track.take() { - tracing::debug!(name = ?track.name(), "reinitializing track"); - self.catalog.lock().video.renditions.remove(track.name()); - } - - let track = self.tracks.create()?; - tracing::debug!(name = ?track.name(), ?config, "starting track"); + tracing::debug!(name = ?self.track.name(), ?config, "starting track"); self.catalog - .lock() .video .renditions - .insert(track.name().to_string(), config.clone()); - + .insert(self.track.name().to_string(), config.clone()); self.config = Some(config); - self.track = Some(crate::container::Producer::new( - track, - crate::catalog::hang::Container::Legacy, - )); Ok(()) } @@ -117,15 +98,8 @@ impl Import { self.init(width, height)?; } - // Resolve the timestamp before borrowing `track` so `pts` doesn't hold a - // `&mut self` across the track write. let pts = self.pts(pts)?; - let track = self - .track - .as_mut() - .context("expected a VP8 key frame before any interframe")?; - - track.write(crate::container::Frame { + self.track.write(crate::container::Frame { timestamp: pts, payload, keyframe: header.keyframe, @@ -133,7 +107,7 @@ impl Import { })?; if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.lock().video.renditions.get_mut(track.name()) + && let Some(c) = self.catalog.video.renditions.get_mut(self.track.name()) { c.jitter = Some(jitter); } @@ -141,27 +115,31 @@ impl Import { Ok(()) } - /// Returns a reference to the underlying track producer. - pub fn track(&self) -> anyhow::Result<&moq_net::TrackProducer> { - Ok(self.track.as_ref().context("not initialized")?.track()) + /// The standalone catalog once the first key frame is seen, else `None`. + pub fn catalog(&self) -> Option<&hang::Catalog> { + self.config.is_some().then_some(&self.catalog) + } + + /// The underlying track producer. + pub fn track(&self) -> &moq_net::TrackProducer { + self.track.track() } /// Finish the track, flushing the current group. pub fn finish(&mut self) -> anyhow::Result<()> { - let track = self.track.as_mut().context("not initialized")?; - track.finish()?; + self.track.finish()?; Ok(()) } /// Close the current group and open the next one at `sequence`. pub fn seek(&mut self, sequence: u64) -> anyhow::Result<()> { - let track = self.track.as_mut().context("not initialized")?; - track.seek(sequence)?; + self.track.seek(sequence)?; Ok(()) } + /// True once the first key frame has populated the catalog. pub fn is_initialized(&self) -> bool { - self.track.is_some() + self.config.is_some() } fn pts(&mut self, hint: Option) -> anyhow::Result { @@ -174,12 +152,9 @@ impl Import { } } -impl Drop for Import { - fn drop(&mut self) { - if let Some(track) = self.track.take() { - tracing::debug!(name = ?track.name(), "ending track"); - self.catalog.lock().video.renditions.remove(track.name()); - } +impl Renditions for Import { + fn renditions(&self) -> &hang::Catalog { + &self.catalog } } @@ -193,13 +168,12 @@ mod tests { /// rendition with the right dimensions and emit both frames. #[tokio::test(start_paused = true)] async fn imports_keyframe_then_interframe() { - let mut broadcast = moq_net::BroadcastInfo::new().produce(); - let mut catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); - let mut import = super::Import::new(broadcast.clone(), catalog.clone()); + let mut import = super::Import::new(moq_net::TrackRequest::new("0.vp8")); - // Empty init buffer: the track is created lazily on the first key frame. + // Empty init buffer: the catalog is filled on the first key frame. import.initialize(&mut Bytes::new()).unwrap(); assert!(!import.is_initialized()); + assert!(import.catalog().is_none()); let keyframe = Bytes::from_static(&[0x10, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x40, 0x01, 0xf0, 0x00]); import @@ -207,8 +181,8 @@ mod tests { .unwrap(); assert!(import.is_initialized()); - let name = import.track().unwrap().name().to_string(); - let config = catalog.lock().video.renditions.get(&name).cloned().unwrap(); + let catalog = import.catalog().unwrap(); + let config = catalog.video.renditions.get(import.track().name()).unwrap(); assert_eq!(config.codec, hang::catalog::VideoCodec::VP8); assert_eq!(config.coded_width, Some(320)); assert_eq!(config.coded_height, Some(240)); @@ -222,13 +196,11 @@ mod tests { import.finish().unwrap(); } - /// An interframe before any key frame has no dimensions, so the track can't - /// be created and the Producer rejects a non-keyframe first frame. + /// An interframe before any key frame has no dimensions, so the Producer + /// rejects a non-keyframe as the first frame in a group. #[tokio::test(start_paused = true)] async fn rejects_interframe_first() { - let mut broadcast = moq_net::BroadcastInfo::new().produce(); - let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); - let mut import = super::Import::new(broadcast.clone(), catalog); + let mut import = super::Import::new(moq_net::TrackRequest::new("0.vp8")); let mut interframe = Bytes::from_static(&[0x31, 0x00, 0x00, 0xaa, 0xbb]); assert!( diff --git a/rs/moq-mux/src/codec/vp9/import.rs b/rs/moq-mux/src/codec/vp9/import.rs index 7272a836e..dd2b615f3 100644 --- a/rs/moq-mux/src/codec/vp9/import.rs +++ b/rs/moq-mux/src/codec/vp9/import.rs @@ -1,7 +1,7 @@ -use anyhow::Context; use bytes::Buf; use crate::container::jitter::MinFrameDuration; +use crate::publish::Renditions; use super::FrameHeader; @@ -10,16 +10,15 @@ use super::FrameHeader; /// Like VP8, a VP9 elementary stream isn't self-delimiting, so the caller must /// pass whole frames (or superframes), one per /// [`decode_frame`](Self::decode_frame). The first key frame's header supplies -/// the catalog config; the track is created lazily. +/// the catalog config, so [`catalog`](Self::catalog) is `None` until then. Build +/// it from a [`moq_net::TrackRequest`] ([`new`](Self::new)) or an existing track +/// ([`from_track`](Self::from_track)). pub struct Import { - // Where new media tracks come from. - tracks: crate::track_provider::TrackProvider, + // The track being produced. + track: crate::container::Producer, - // The catalog being produced. - catalog: crate::catalog::Producer, - - // The track being produced, created on the first key frame. - track: Option>, + // The standalone catalog, populated on the first key frame. + catalog: hang::Catalog, // The resolved config, used to detect resolution / format changes. config: Option, @@ -32,22 +31,17 @@ pub struct Import { } impl Import { - pub fn new(broadcast: moq_net::BroadcastProducer, catalog: crate::catalog::Producer) -> Self { - Self { - tracks: crate::track_provider::TrackProvider::unique(broadcast, ".vp09"), - catalog, - track: None, - config: None, - zero: None, - jitter: MinFrameDuration::new(), - } + /// Serve a track request, accepting it at the microsecond timescale. + pub fn new(request: moq_net::TrackRequest) -> Self { + let info = moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE); + Self::from_track(request.accept(info)) } - pub fn new_with_track(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { + /// Publish on an existing track producer. + pub fn from_track(track: moq_net::TrackProducer) -> Self { Self { - tracks: crate::track_provider::TrackProvider::fixed(track), - catalog, - track: None, + track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), + catalog: hang::Catalog::default(), config: None, zero: None, jitter: MinFrameDuration::new(), @@ -57,9 +51,9 @@ impl Import { /// Initialize the importer. /// /// VP9 has no out-of-band configuration record, so this is normally called - /// with an empty buffer (gstreamer / ffi pass `Bytes::new()`) and the track - /// is created lazily from the first key frame. If the caller does pass the - /// first frame here, it's decoded so nothing is dropped. + /// with an empty buffer (gstreamer / ffi pass `Bytes::new()`) and the catalog + /// is filled from the first key frame. If the caller does pass the first frame + /// here, it's decoded so nothing is dropped. pub fn initialize>(&mut self, buf: &mut T) -> anyhow::Result<()> { if buf.has_remaining() { self.decode_frame(buf, None)?; @@ -77,28 +71,16 @@ impl Import { return Ok(()); } - if self.track.is_some() && self.tracks.is_fixed() { + if self.config.is_some() { anyhow::bail!("fixed track cannot be reconfigured"); } - if let Some(track) = self.track.take() { - tracing::debug!(name = ?track.name(), "reinitializing track"); - self.catalog.lock().video.renditions.remove(track.name()); - } - - let track = self.tracks.create()?; - tracing::debug!(name = ?track.name(), ?config, "starting track"); + tracing::debug!(name = ?self.track.name(), ?config, "starting track"); self.catalog - .lock() .video .renditions - .insert(track.name().to_string(), config.clone()); - + .insert(self.track.name().to_string(), config.clone()); self.config = Some(config); - self.track = Some(crate::container::Producer::new( - track, - crate::catalog::hang::Container::Legacy, - )); Ok(()) } @@ -117,15 +99,8 @@ impl Import { self.init(key.to_catalog(), key.width, key.height)?; } - // Resolve the timestamp before borrowing `track` so `pts` doesn't hold a - // `&mut self` across the track write. let pts = self.pts(pts)?; - let track = self - .track - .as_mut() - .context("expected a VP9 key frame before any interframe")?; - - track.write(crate::container::Frame { + self.track.write(crate::container::Frame { timestamp: pts, payload, keyframe: header.keyframe, @@ -133,7 +108,7 @@ impl Import { })?; if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.lock().video.renditions.get_mut(track.name()) + && let Some(c) = self.catalog.video.renditions.get_mut(self.track.name()) { c.jitter = Some(jitter); } @@ -141,27 +116,31 @@ impl Import { Ok(()) } - /// Returns a reference to the underlying track producer. - pub fn track(&self) -> anyhow::Result<&moq_net::TrackProducer> { - Ok(self.track.as_ref().context("not initialized")?.track()) + /// The standalone catalog once the first key frame is seen, else `None`. + pub fn catalog(&self) -> Option<&hang::Catalog> { + self.config.is_some().then_some(&self.catalog) + } + + /// The underlying track producer. + pub fn track(&self) -> &moq_net::TrackProducer { + self.track.track() } /// Finish the track, flushing the current group. pub fn finish(&mut self) -> anyhow::Result<()> { - let track = self.track.as_mut().context("not initialized")?; - track.finish()?; + self.track.finish()?; Ok(()) } /// Close the current group and open the next one at `sequence`. pub fn seek(&mut self, sequence: u64) -> anyhow::Result<()> { - let track = self.track.as_mut().context("not initialized")?; - track.seek(sequence)?; + self.track.seek(sequence)?; Ok(()) } + /// True once the first key frame has populated the catalog. pub fn is_initialized(&self) -> bool { - self.track.is_some() + self.config.is_some() } fn pts(&mut self, hint: Option) -> anyhow::Result { @@ -174,12 +153,9 @@ impl Import { } } -impl Drop for Import { - fn drop(&mut self) { - if let Some(track) = self.track.take() { - tracing::debug!(name = ?track.name(), "ending track"); - self.catalog.lock().video.renditions.remove(track.name()); - } +impl Renditions for Import { + fn renditions(&self) -> &hang::Catalog { + &self.catalog } } @@ -194,12 +170,11 @@ mod tests { #[tokio::test(start_paused = true)] async fn imports_keyframe_then_interframe() { - let mut broadcast = moq_net::BroadcastInfo::new().produce(); - let mut catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); - let mut import = super::Import::new(broadcast.clone(), catalog.clone()); + let mut import = super::Import::new(moq_net::TrackRequest::new("0.vp9")); import.initialize(&mut Bytes::new()).unwrap(); assert!(!import.is_initialized()); + assert!(import.catalog().is_none()); import .decode_frame( @@ -209,8 +184,8 @@ mod tests { .unwrap(); assert!(import.is_initialized()); - let name = import.track().unwrap().name().to_string(); - let config = catalog.lock().video.renditions.get(&name).cloned().unwrap(); + let catalog = import.catalog().unwrap(); + let config = catalog.video.renditions.get(import.track().name()).unwrap(); assert!(matches!(config.codec, hang::catalog::VideoCodec::VP9(_))); assert_eq!(config.coded_width, Some(320)); assert_eq!(config.coded_height, Some(240)); @@ -228,9 +203,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn rejects_interframe_first() { - let mut broadcast = moq_net::BroadcastInfo::new().produce(); - let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); - let mut import = super::Import::new(broadcast.clone(), catalog); + let mut import = super::Import::new(moq_net::TrackRequest::new("0.vp9")); let mut interframe = Bytes::from_static(&[0x84, 0x00, 0x00]); assert!( diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 399a56ba3..e4d97d77f 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -269,10 +269,16 @@ impl Import { unwrap: PtsUnwrap::default(), } } - StreamType::H265 => Stream::H265 { - import: Box::new(h265::Import::new(self.broadcast.clone(), self.catalog.clone())), - unwrap: PtsUnwrap::default(), - }, + StreamType::H265 => { + let track = crate::publish::unique_track(&mut self.broadcast, ".hev1")?; + Stream::H265 { + import: Box::new(crate::publish::Published::new( + self.catalog.clone(), + h265::Import::from_track(track), + )), + unwrap: PtsUnwrap::default(), + } + } // Only ADTS-framed AAC (0x0F). 0x11 is LATM/LOAS, which uses a different // framing and syncword, so it falls through to the ignored arm below. StreamType::AdtsAac => Stream::Aac(Box::new(AacStream { @@ -717,7 +723,7 @@ enum Stream { unwrap: PtsUnwrap, }, H265 { - import: Box>, + import: Box>, unwrap: PtsUnwrap, }, Aac(Box>), @@ -739,7 +745,9 @@ impl Stream { } Stream::H265 { import, unwrap } => { let pts = unwrap_pts(unwrap, pending.pts)?; - Ok(import.decode_frame(&mut pending.data.as_slice(), pts)?) + import.decode_frame(&mut pending.data.as_slice(), pts)?; + import.sync(); + Ok(()) } Stream::Aac(stream) => stream.write(pending, burst), Stream::Legacy(stream) => stream.write(pending), @@ -772,7 +780,7 @@ impl Stream { /// (the sample rate and channel layout aren't in the PMT), so creation is /// deferred until the first frame arrives. struct AacStream { - import: Option>, + import: Option>, broadcast: moq_net::BroadcastProducer, catalog: crate::catalog::Producer, unwrap: PtsUnwrap, @@ -807,11 +815,13 @@ impl AacStream { // downstream consumers that need out-of-band config (fMP4/MKV export, // WebCodecs) can configure the decoder. TS itself carries it inline. let description = config.encode(); - let import = aac::Import::new(self.broadcast.clone(), self.catalog.clone(), config)?; - let name = import.track().name().to_string(); - if let Some(rendition) = self.catalog.lock().audio.renditions.get_mut(&name) { + let track = crate::publish::unique_track(&mut self.broadcast, ".aac")?; + let aac = aac::Import::from_track(track, config)?; + let mut import = crate::publish::Published::new(self.catalog.clone(), aac); + if let Some(rendition) = import.rendition_mut() { rendition.description = Some(description); } + import.sync(); self.import.insert(import) } }; @@ -870,11 +880,11 @@ impl AacStream { } self.jitter = Some(jitter); - if let Some(import) = &self.import { - let name = import.track().name().to_string(); - if let Some(rendition) = self.catalog.lock().audio.renditions.get_mut(&name) { + if let Some(import) = &mut self.import { + if let Some(rendition) = import.rendition_mut() { rendition.jitter = Some(jitter.into()); } + import.sync(); } Ok(()) } @@ -901,7 +911,7 @@ impl AacStream { /// players, which cannot decode these codecs. struct LegacyStream { descriptor: &'static legacy::Descriptor, - import: Option>, + import: Option>, broadcast: moq_net::BroadcastProducer, catalog: crate::catalog::Producer, unwrap: PtsUnwrap, @@ -960,9 +970,10 @@ impl LegacyStream { sample_rate: header.sample_rate, channel_count: header.channel_count, }; - let import = - legacy::Import::new(self.descriptor, self.broadcast.clone(), self.catalog.clone(), config)?; - self.import.insert(import) + let track = crate::publish::unique_track(&mut self.broadcast, self.descriptor.track_suffix)?; + let legacy = legacy::Import::from_track(self.descriptor, track, config); + self.import + .insert(crate::publish::Published::new(self.catalog.clone(), legacy)) } }; diff --git a/rs/moq-mux/src/import.rs b/rs/moq-mux/src/import.rs index 2f385a4f4..db95cb8ca 100644 --- a/rs/moq-mux/src/import.rs +++ b/rs/moq-mux/src/import.rs @@ -102,11 +102,11 @@ enum FramedKind { H264(crate::publish::Published), // Boxed because it's a large struct and clippy complains about the size. Fmp4(Box), - Hev1(crate::codec::h265::Import), - Av01(crate::codec::av1::Import), - Vp8(crate::codec::vp8::Import), - Vp9(crate::codec::vp9::Import), - Aac(crate::codec::aac::Import), + Hev1(crate::publish::Published), + Av01(crate::publish::Published), + Vp8(crate::publish::Published), + Vp9(crate::publish::Published), + Aac(crate::publish::Published), Opus(crate::publish::Published), // Boxed for the same reason as Fmp4. Mkv(Box), @@ -151,28 +151,34 @@ impl Framed { FramedKind::Fmp4(decoder) } FramedFormat::Hev1 => { - let mut decoder = crate::codec::h265::Import::new(broadcast, catalog); + let track = crate::publish::unique_track(&mut broadcast, ".hev1")?; + let mut decoder = crate::codec::h265::Import::from_track(track); decoder.initialize(buf)?; - FramedKind::Hev1(decoder) + FramedKind::Hev1(crate::publish::Published::new(catalog, decoder)) } FramedFormat::Av01 => { - let mut decoder = crate::codec::av1::Import::new(broadcast, catalog); + let track = crate::publish::unique_track(&mut broadcast, ".av01")?; + let mut decoder = crate::codec::av1::Import::from_track(track); decoder.initialize(buf)?; - FramedKind::Av01(decoder) + FramedKind::Av01(crate::publish::Published::new(catalog, decoder)) } FramedFormat::Vp8 => { - let mut decoder = crate::codec::vp8::Import::new(broadcast, catalog); + let track = crate::publish::unique_track(&mut broadcast, ".vp8")?; + let mut decoder = crate::codec::vp8::Import::from_track(track); decoder.initialize(buf)?; - FramedKind::Vp8(decoder) + FramedKind::Vp8(crate::publish::Published::new(catalog, decoder)) } FramedFormat::Vp9 => { - let mut decoder = crate::codec::vp9::Import::new(broadcast, catalog); + let track = crate::publish::unique_track(&mut broadcast, ".vp09")?; + let mut decoder = crate::codec::vp9::Import::from_track(track); decoder.initialize(buf)?; - FramedKind::Vp9(decoder) + FramedKind::Vp9(crate::publish::Published::new(catalog, decoder)) } FramedFormat::Aac => { let config = crate::codec::aac::Config::parse(buf)?; - FramedKind::Aac(crate::codec::aac::Import::new(broadcast, catalog, config)?) + let track = crate::publish::unique_track(&mut broadcast, ".aac")?; + let import = crate::codec::aac::Import::from_track(track, config)?; + FramedKind::Aac(crate::publish::Published::new(catalog, import)) } FramedFormat::Opus => { let config = crate::codec::opus::Config::parse(buf)?; @@ -222,28 +228,29 @@ impl Framed { FramedKind::H264(crate::publish::Published::new(catalog, decoder)) } FramedFormat::Hev1 => { - let mut decoder = crate::codec::h265::Import::new_with_track(track, catalog); + let mut decoder = crate::codec::h265::Import::from_track(track); decoder.initialize(buf)?; - FramedKind::Hev1(decoder) + FramedKind::Hev1(crate::publish::Published::new(catalog, decoder)) } FramedFormat::Av01 => { - let mut decoder = crate::codec::av1::Import::new_with_track(track, catalog); + let mut decoder = crate::codec::av1::Import::from_track(track); decoder.initialize(buf)?; - FramedKind::Av01(decoder) + FramedKind::Av01(crate::publish::Published::new(catalog, decoder)) } FramedFormat::Vp8 => { - let mut decoder = crate::codec::vp8::Import::new_with_track(track, catalog); + let mut decoder = crate::codec::vp8::Import::from_track(track); decoder.initialize(buf)?; - FramedKind::Vp8(decoder) + FramedKind::Vp8(crate::publish::Published::new(catalog, decoder)) } FramedFormat::Vp9 => { - let mut decoder = crate::codec::vp9::Import::new_with_track(track, catalog); + let mut decoder = crate::codec::vp9::Import::from_track(track); decoder.initialize(buf)?; - FramedKind::Vp9(decoder) + FramedKind::Vp9(crate::publish::Published::new(catalog, decoder)) } FramedFormat::Aac => { let config = crate::codec::aac::Config::parse(buf)?; - FramedKind::Aac(crate::codec::aac::Import::new_with_track(track, catalog, config)?) + let import = crate::codec::aac::Import::from_track(track, config)?; + FramedKind::Aac(crate::publish::Published::new(catalog, import)) } FramedFormat::Opus => { let config = crate::codec::opus::Config::parse(buf)?; @@ -297,10 +304,10 @@ impl Framed { match self.decoder { FramedKind::H264(ref decoder) => Ok(decoder.track()), FramedKind::Fmp4(_) => Err(crate::Error::MultipleTracks("fmp4")), - FramedKind::Hev1(ref decoder) => decoder.track(), - FramedKind::Av01(ref decoder) => decoder.track(), - FramedKind::Vp8(ref decoder) => decoder.track().map_err(Into::into), - FramedKind::Vp9(ref decoder) => decoder.track().map_err(Into::into), + FramedKind::Hev1(ref decoder) => Ok(decoder.track()), + FramedKind::Av01(ref decoder) => Ok(decoder.track()), + FramedKind::Vp8(ref decoder) => Ok(decoder.track()), + FramedKind::Vp9(ref decoder) => Ok(decoder.track()), FramedKind::Aac(ref decoder) => Ok(decoder.track()), FramedKind::Opus(ref decoder) => Ok(decoder.track()), FramedKind::Mkv(_) => Err(crate::Error::MultipleTracks("mkv")), @@ -316,10 +323,22 @@ impl Framed { decoder.sync(); } FramedKind::Fmp4(ref mut decoder) => decoder.decode(buf)?, - FramedKind::Hev1(ref mut decoder) => decoder.decode_frame(buf, pts)?, - FramedKind::Av01(ref mut decoder) => decoder.decode_frame(buf, pts)?, - FramedKind::Vp8(ref mut decoder) => decoder.decode_frame(buf, pts)?, - FramedKind::Vp9(ref mut decoder) => decoder.decode_frame(buf, pts)?, + FramedKind::Hev1(ref mut decoder) => { + decoder.decode_frame(buf, pts)?; + decoder.sync(); + } + FramedKind::Av01(ref mut decoder) => { + decoder.decode_frame(buf, pts)?; + decoder.sync(); + } + FramedKind::Vp8(ref mut decoder) => { + decoder.decode_frame(buf, pts)?; + decoder.sync(); + } + FramedKind::Vp9(ref mut decoder) => { + decoder.decode_frame(buf, pts)?; + decoder.sync(); + } FramedKind::Aac(ref mut decoder) => decoder.decode(buf, pts)?, FramedKind::Opus(ref mut decoder) => decoder.decode_buf(buf, pts)?, FramedKind::Mkv(ref mut decoder) => { @@ -351,8 +370,8 @@ impl From> for Framed { } } -impl From for Framed { - fn from(aac: crate::codec::aac::Import) -> Self { +impl From> for Framed { + fn from(aac: crate::publish::Published) -> Self { Self { decoder: FramedKind::Aac(aac), } @@ -619,8 +638,8 @@ enum StreamKind { Avc3(crate::publish::Published), // Boxed because it's a large struct and clippy complains about the size. Fmp4(Box), - Hev1(crate::codec::h265::Import), - Av01(crate::codec::av1::Import), + Hev1(crate::publish::Published), + Av01(crate::publish::Published), // Boxed for the same reason as Fmp4. Mkv(Box), // Boxed for the same reason as Fmp4. @@ -650,8 +669,16 @@ impl Stream { StreamKind::Avc3(crate::publish::Published::new(catalog, decoder)) } StreamFormat::Fmp4 => StreamKind::Fmp4(Box::new(crate::container::fmp4::Import::new(broadcast, catalog))), - StreamFormat::Hev1 => StreamKind::Hev1(crate::codec::h265::Import::new(broadcast, catalog)), - StreamFormat::Av01 => StreamKind::Av01(crate::codec::av1::Import::new(broadcast, catalog)), + StreamFormat::Hev1 => { + let track = crate::publish::unique_track(&mut broadcast, ".hev1")?; + let decoder = crate::codec::h265::Import::from_track(track); + StreamKind::Hev1(crate::publish::Published::new(catalog, decoder)) + } + StreamFormat::Av01 => { + let track = crate::publish::unique_track(&mut broadcast, ".av01")?; + let decoder = crate::codec::av1::Import::from_track(track); + StreamKind::Av01(crate::publish::Published::new(catalog, decoder)) + } StreamFormat::Mkv => StreamKind::Mkv(Box::new(crate::container::mkv::Import::new(broadcast, catalog))), StreamFormat::Ts => StreamKind::Ts(Box::new(crate::container::ts::Import::new(broadcast, catalog))), }; @@ -671,8 +698,14 @@ impl Stream { decoder.sync(); } StreamKind::Fmp4(ref mut decoder) => decoder.decode(buf)?, - StreamKind::Hev1(ref mut decoder) => decoder.initialize(buf)?, - StreamKind::Av01(ref mut decoder) => decoder.initialize(buf)?, + StreamKind::Hev1(ref mut decoder) => { + decoder.initialize(buf)?; + decoder.sync(); + } + StreamKind::Av01(ref mut decoder) => { + decoder.initialize(buf)?; + decoder.sync(); + } StreamKind::Mkv(ref mut decoder) => decoder.decode(buf)?, StreamKind::Ts(ref mut decoder) => decoder.decode(buf)?, } @@ -693,8 +726,16 @@ impl Stream { Ok(()) } StreamKind::Fmp4(ref mut decoder) => decoder.decode(buf), - StreamKind::Hev1(ref mut decoder) => decoder.decode_stream(buf, None), - StreamKind::Av01(ref mut decoder) => decoder.decode_stream(buf, None), + StreamKind::Hev1(ref mut decoder) => { + decoder.decode_stream(buf, None)?; + decoder.sync(); + Ok(()) + } + StreamKind::Av01(ref mut decoder) => { + decoder.decode_stream(buf, None)?; + decoder.sync(); + Ok(()) + } StreamKind::Mkv(ref mut decoder) => decoder.decode(buf), StreamKind::Ts(ref mut decoder) => decoder.decode(buf).map_err(Into::into), } diff --git a/rs/moq-mux/src/lib.rs b/rs/moq-mux/src/lib.rs index fc79999cc..72e5cbfc0 100644 --- a/rs/moq-mux/src/lib.rs +++ b/rs/moq-mux/src/lib.rs @@ -22,7 +22,6 @@ pub mod container; mod error; pub mod import; pub mod publish; -mod track_provider; pub use clock::Clock; pub use error::*; diff --git a/rs/moq-mux/src/track_provider.rs b/rs/moq-mux/src/track_provider.rs deleted file mode 100644 index 01ca3e585..000000000 --- a/rs/moq-mux/src/track_provider.rs +++ /dev/null @@ -1,33 +0,0 @@ -pub(crate) enum TrackProvider { - Unique { - broadcast: moq_net::BroadcastProducer, - suffix: &'static str, - }, - Fixed(moq_net::TrackProducer), -} - -impl TrackProvider { - pub(crate) fn unique(broadcast: moq_net::BroadcastProducer, suffix: &'static str) -> Self { - Self::Unique { broadcast, suffix } - } - - pub(crate) fn fixed(track: moq_net::TrackProducer) -> Self { - Self::Fixed(track) - } - - pub(crate) fn is_fixed(&self) -> bool { - matches!(self, Self::Fixed(_)) - } - - pub(crate) fn create(&mut self) -> crate::Result { - match self { - Self::Unique { broadcast, suffix } => { - // Newly created tracks publish at the native container timescale so the - // relay gets timing without parsing the payload. - let info = moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE); - Ok(broadcast.unique_track(suffix, info)?) - } - Self::Fixed(track) => Ok(track.clone()), - } - } -} From dc37cea4ae17f76d2fc186061da1868c61b15051 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 13:35:55 +0000 Subject: [PATCH 05/19] moq-mux: extract the H.264 byte->frame Split; add the FrameDecode path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First slice of the splitter / unified-decode work. - New `codec::h264::Split`: a standalone parser that turns H.264 bytes into `container::Frame`s plus a resolved `VideoConfig` (the NAL/AU assembly, avcC parsing, SPS/PPS cache, and Annex-B wall-clock all live here, independently testable). avc1 still errors on reconfiguration; avc3 still updates in place. - `codec::h264::Import` now drives `Split` internally for its byte APIs (`decode_frame` / `decode_stream` unchanged, so the Framed/Stream/TS dispatchers and external callers are untouched and don't regress) and pulls the resolved config into its local catalog. - New `publish::FrameDecode` trait (`decode(impl IntoIterator)`), the uniform decode entry point a caller drives with frames from its own `Split`. `Published::decode` wraps it and syncs, so the catalog re-mirror can't be forgotten — the footgun-free path. h264::Import implements it. The dispatchers still use the byte API + manual `sync()`; migrating them to `Split` + `Published::decode` (and resolving the avc1-vs-avc3 config flow on that path) is the next slice, then rolling the same split to the other codecs. https://claude.ai/code/session_011S7pzcg2XsPP3AExuymac1 --- rs/moq-mux/src/codec/h264/import.rs | 571 +++------------------------ rs/moq-mux/src/codec/h264/mod.rs | 2 + rs/moq-mux/src/codec/h264/split.rs | 592 ++++++++++++++++++++++++++++ rs/moq-mux/src/publish.rs | 44 +++ 4 files changed, 699 insertions(+), 510 deletions(-) create mode 100644 rs/moq-mux/src/codec/h264/split.rs diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index c4ccd1e41..b94b1aa83 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -1,31 +1,20 @@ -//! H.264 importer for both wire shapes. +//! H.264 importer. //! -//! [`Import`] accepts either length-prefixed NALU input with an -//! out-of-band [`AVCDecoderConfigurationRecord`](super::Avcc) (the "avc1" -//! shape) or Annex-B input with inline SPS/PPS (the "avc3" shape). The shape -//! is detected at [`initialize`](Import::initialize) time by looking for a -//! leading start code; callers that already know it can also force the -//! mode via [`with_mode`](Import::with_mode). - -use bytes::{Buf, Bytes, BytesMut}; +//! Publishes split H.264 frames on a single moq track and tracks the catalog +//! rendition. Byte parsing lives in [`Split`]; this type drives it for the +//! convenience byte APIs ([`decode_frame`](Import::decode_frame) / +//! [`decode_stream`](Import::decode_stream)) and writes the resulting frames, +//! and also implements [`FrameDecode`] so a caller that runs its own [`Split`] +//! can publish frames directly. + +use bytes::{Buf, BytesMut}; use tokio::io::{AsyncRead, AsyncReadExt}; -use super::{Error, Sps}; +use super::{Mode, Split}; use crate::Result; -use crate::codec::annexb::{NalIterator, START_CODE}; +use crate::container::Frame; use crate::container::jitter::MinFrameDuration; -use crate::publish::Renditions; - -/// The wire shape an [`Import`] is processing. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum Mode { - /// Length-prefixed NALU with out-of-band AVCDecoderConfigurationRecord - /// (catalog `H264 { inline: false }`, `description = avcC`). - Avc1, - /// Annex-B (start-code prefixed) with inline SPS/PPS - /// (catalog `H264 { inline: true }`, no description). - Avc3, -} +use crate::publish::{FrameDecode, Renditions}; /// H.264 importer. Handles both avc1 (length-prefixed) and avc3 (Annex-B) /// input streams; the shape is detected from the first bytes the caller @@ -38,36 +27,13 @@ pub enum Mode { /// via [`catalog`](Self::catalog) or attach the importer to a broadcast catalog /// with [`crate::publish::Published`]. pub struct Import { + split: Split, track: crate::container::Producer, catalog: hang::Catalog, config: Option, - state: State, - zero: Option, jitter: MinFrameDuration, } -enum State { - /// No bytes seen yet; mode pinned ahead of time or unknown. - Pending { mode_hint: Option }, - /// avc1 wire shape: length-prefixed NALU, codec config out-of-band. - Avc1 { length_size: usize }, - /// avc3 wire shape: Annex-B NALU, inline SPS/PPS. - Avc3 { - current: Avc3Frame, - sps: Option, - pps: Option, - }, -} - -#[derive(Default)] -struct Avc3Frame { - chunks: BytesMut, - contains_idr: bool, - contains_slice: bool, - contains_sps: bool, - contains_pps: bool, -} - impl Import { /// Serve a track request, accepting it at the microsecond timescale. pub fn new(request: moq_net::TrackRequest) -> Self { @@ -78,39 +44,25 @@ impl Import { /// Publish on an existing track producer. pub fn from_track(track: moq_net::TrackProducer) -> Self { Self { + split: Split::new(), track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy).with_lenient_start(), catalog: hang::Catalog::default(), config: None, - state: State::Pending { mode_hint: None }, - zero: None, jitter: MinFrameDuration::new(), } } - /// Pin the wire shape ahead of time; skips the leading-bytes auto-detect - /// inside [`initialize`](Self::initialize). + /// Pin the wire shape ahead of time; skips the leading-bytes auto-detect. pub fn with_mode(mut self, mode: Mode) -> Result { - match mode { - Mode::Avc1 => { - self.state = State::Pending { - mode_hint: Some(Mode::Avc1), - }; - } - Mode::Avc3 => { - self.state = State::Avc3 { - current: Avc3Frame::default(), - sps: None, - pps: None, - }; - } - } + self.split = Split::with_mode(mode)?; Ok(self) } - /// The underlying track producer, e.g. for monitoring subscriber state via - /// `used()` / `unused()`. - pub fn track(&self) -> &moq_net::TrackProducer { - self.track.track() + /// Initialize from the codec's leading bytes (avcC for avc1, SPS/PPS for avc3). + pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { + self.split.initialize(buf)?; + self.pull_config(); + Ok(()) } /// The standalone catalog once the codec config is known, else `None`. @@ -118,72 +70,9 @@ impl Import { self.config.is_some().then_some(&self.catalog) } - /// Initialize from the codec's leading bytes. - /// - /// - **avc1** (no leading start code): the buffer is parsed as an - /// `AVCDecoderConfigurationRecord` and stored as the catalog `description`. - /// - **avc3** (leading `0x00 0x00 0x01` or `0x00 0x00 0x00 0x01`): the buffer - /// is parsed as Annex-B NALs to seed the cached SPS/PPS. - /// - /// The buffer is fully consumed. - pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { - let mode = match &self.state { - State::Pending { mode_hint } => mode_hint.unwrap_or_else(|| detect_mode(buf.as_ref())), - State::Avc1 { .. } => Mode::Avc1, - State::Avc3 { .. } => Mode::Avc3, - }; - - match mode { - Mode::Avc1 => self.initialize_avc1(buf), - Mode::Avc3 => self.initialize_avc3(buf), - } - } - - /// Initialize the avc1 path from an `AVCDecoderConfigurationRecord` buffer. - fn initialize_avc1>(&mut self, buf: &mut T) -> Result<()> { - let avcc_bytes = buf.as_ref(); - let avcc = super::Avcc::parse(avcc_bytes)?; - self.state = State::Avc1 { - length_size: avcc.length_size, - }; - - let mut config = hang::catalog::VideoConfig::new(hang::catalog::H264 { - profile: avcc.profile, - constraints: avcc.constraints, - level: avcc.level, - inline: false, - }); - config.coded_width = avcc.coded_width; - config.coded_height = avcc.coded_height; - config.description = Some(Bytes::copy_from_slice(avcc_bytes)); - config.container = hang::catalog::Container::Legacy; - - self.swap_config(config)?; - buf.advance(buf.remaining()); - - Ok(()) - } - - /// Initialize the avc3 path by parsing Annex-B NALs (SPS/PPS seed the - /// catalog rendition once the first SPS is parsed). - fn initialize_avc3>(&mut self, buf: &mut T) -> Result<()> { - if !matches!(self.state, State::Avc3 { .. }) { - self.state = State::Avc3 { - current: Avc3Frame::default(), - sps: None, - pps: None, - }; - } - - let mut nals = NalIterator::new(buf); - while let Some(nal) = nals.next().transpose()? { - self.decode_nal(nal, None)?; - } - if let Some(nal) = nals.flush()? { - self.decode_nal(nal, None)?; - } - - Ok(()) + /// The underlying track producer. + pub fn track(&self) -> &moq_net::TrackProducer { + self.track.track() } /// True once the codec config is known and the catalog rendition is published. @@ -191,8 +80,7 @@ impl Import { self.config.is_some() } - /// Decode from an asynchronous reader. avc3 only — for avc1, the caller - /// already has framed buffers and uses [`decode_frame`](Self::decode_frame). + /// Decode from an asynchronous reader (avc3 streaming input). pub async fn decode_from(&mut self, reader: &mut T) -> Result<()> { let mut buffer = BytesMut::new(); while reader.read_buf(&mut buffer).await? > 0 { @@ -201,233 +89,18 @@ impl Import { Ok(()) } - /// Decode a buffer where frame boundaries are unknown (avc3 streaming - /// input). The leading start code of the *next* frame is what signals the - /// previous frame is done. - pub fn decode_stream>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - if !matches!(self.state, State::Avc3 { .. }) { - return Err(Error::StreamNotAvc3.into()); - } - let pts = self.pts(pts)?; - let nals = NalIterator::new(buf); - for nal in nals { - self.decode_nal(nal?, Some(pts))?; - } - Ok(()) - } - - /// Decode a buffer assumed to hold (the rest of) a single frame. - /// - /// - avc1: the buffer is written as one length-prefixed-NALU frame. - /// - avc3: NALs are parsed; any trailing NAL without a start code is - /// flushed as the last NAL of this frame. + /// Decode a buffer holding (the rest of) a single frame. pub fn decode_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - match &self.state { - State::Avc1 { .. } => self.decode_avc1(buf, pts), - State::Avc3 { .. } => self.decode_avc3_frame(buf, pts), - State::Pending { .. } => Err(Error::NotInitialized.into()), - } - } - - fn decode_avc1>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - let State::Avc1 { length_size } = self.state else { - unreachable!("checked by decode_frame") - }; - let data = buf.as_ref(); - let pts = self.pts(pts)?; - let keyframe = avc1_is_keyframe(data, length_size); - - self.track.write(crate::container::Frame { - timestamp: pts, - payload: data.to_vec().into(), - keyframe, - duration: None, - })?; - - if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.video.renditions.get_mut(self.track.name()) - { - c.jitter = Some(jitter); - } - - buf.advance(buf.remaining()); - Ok(()) - } - - fn decode_avc3_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - let pts = self.pts(pts)?; - let mut nals = NalIterator::new(buf); - while let Some(nal) = nals.next().transpose()? { - self.decode_nal(nal, Some(pts))?; - } - if let Some(nal) = nals.flush()? { - self.decode_nal(nal, Some(pts))?; - } - self.maybe_start_frame(Some(pts))?; - Ok(()) - } - - fn decode_nal(&mut self, nal: Bytes, pts: Option) -> Result<()> { - let header = nal.first().ok_or(Error::NalTooShort)?; - let forbidden_zero_bit = (header >> 7) & 1; - if forbidden_zero_bit != 0 { - return Err(Error::ForbiddenZeroBit.into()); - } - - let nal_unit_type = header & 0b11111; - let nal_type = Avc3NalType::try_from(nal_unit_type).ok(); - - match nal_type { - Some(Avc3NalType::Sps) => { - self.maybe_start_frame(pts)?; - let sps = Sps::parse(&nal)?; - self.init_from_sps(&sps)?; - let State::Avc3 { current, sps, pps } = &mut self.state else { - unreachable!("decode_nal is avc3 only") - }; - if sps.as_ref().is_some_and(|cached| cached != &nal) { - // SPS changed mid-AU. The cached PPS is tied to the old SPS - // and may already have been appended to current.chunks - // earlier in this AU; reset the AU so the new SPS+PPS pair - // is the only parameter set we emit. - *pps = None; - current.chunks.clear(); - current.contains_pps = false; - current.contains_sps = false; - } - *sps = Some(nal.clone()); - current.contains_sps = true; - } - Some(Avc3NalType::Pps) => { - self.maybe_start_frame(pts)?; - let State::Avc3 { current, pps, .. } = &mut self.state else { - unreachable!() - }; - *pps = Some(nal.clone()); - current.contains_pps = true; - } - Some(Avc3NalType::Aud) | Some(Avc3NalType::Sei) => { - self.maybe_start_frame(pts)?; - } - Some(Avc3NalType::IdrSlice) => { - let State::Avc3 { current, sps, pps } = &mut self.state else { - unreachable!() - }; - if !current.contains_sps - && let Some(sps) = sps.as_ref() - { - current.chunks.extend_from_slice(&START_CODE); - current.chunks.extend_from_slice(sps); - current.contains_sps = true; - } - if !current.contains_pps - && let Some(pps) = pps.as_ref() - { - current.chunks.extend_from_slice(&START_CODE); - current.chunks.extend_from_slice(pps); - current.contains_pps = true; - } - current.contains_idr = true; - current.contains_slice = true; - } - Some(Avc3NalType::NonIdrSlice) - | Some(Avc3NalType::DataPartitionA) - | Some(Avc3NalType::DataPartitionB) - | Some(Avc3NalType::DataPartitionC) => { - if nal.get(1).ok_or(Error::NalTooShort)? & 0x80 != 0 { - self.maybe_start_frame(pts)?; - } - let State::Avc3 { current, .. } = &mut self.state else { - unreachable!() - }; - current.contains_slice = true; - } - _ => {} - } - - tracing::trace!(kind = ?nal_type, "parsed NAL"); - - let State::Avc3 { current, .. } = &mut self.state else { - unreachable!() - }; - current.chunks.extend_from_slice(&START_CODE); - current.chunks.extend_from_slice(&nal); - Ok(()) - } - - fn init_from_sps(&mut self, sps: &Sps) -> Result<()> { - let mut config = hang::catalog::VideoConfig::new(hang::catalog::H264 { - profile: sps.profile, - constraints: sps.constraints, - level: sps.level, - inline: true, - }); - config.coded_width = Some(sps.coded_width); - config.coded_height = Some(sps.coded_height); - config.container = hang::catalog::Container::Legacy; - - if let Some(old) = &self.config - && old == &config - { - return Ok(()); - } - - // avc3 carries SPS inline, so a resolution change updates the rendition in - // place on the same track (no new init segment, unlike avc1). - let track_name = self.track.name().to_string(); - self.catalog.video.renditions.insert(track_name, config.clone()); - self.config = Some(config); - Ok(()) - } - - fn maybe_start_frame(&mut self, pts: Option) -> Result<()> { - let State::Avc3 { current, .. } = &mut self.state else { - return Ok(()); - }; - if !current.contains_slice { - return Ok(()); - } - let pts = pts.ok_or(Error::MissingTimestamp)?; - let payload = std::mem::take(&mut current.chunks).freeze(); - let keyframe = current.contains_idr; - current.contains_idr = false; - current.contains_slice = false; - current.contains_sps = false; - current.contains_pps = false; - - self.track.write(crate::container::Frame { - timestamp: pts, - payload, - keyframe, - duration: None, - })?; - - if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.video.renditions.get_mut(self.track.name()) - { - c.jitter = Some(jitter); - } - Ok(()) + let frames = self.split.decode_frame(buf, pts)?; + self.pull_config(); + self.write_frames(frames) } - /// Register the avc1 codec config on the (fixed) track. - /// - /// The first config seeds the catalog rendition. A different avcC would mean - /// a new init segment, which the single fixed track can't represent, so a - /// reconfiguration is an error (mint a new track via a fresh importer). - fn swap_config(&mut self, config: hang::catalog::VideoConfig) -> Result<()> { - if let Some(old) = &self.config { - if old == &config { - return Ok(()); - } - return Err(Error::FixedTrackReconfigured.into()); - } - - let track_name = self.track.name().to_string(); - tracing::debug!(name = ?track_name, ?config, "starting H.264 track"); - self.catalog.video.renditions.insert(track_name, config.clone()); - self.config = Some(config); - Ok(()) + /// Decode a buffer where frame boundaries are unknown (avc3 streaming input). + pub fn decode_stream>(&mut self, buf: &mut T, pts: Option) -> Result<()> { + let frames = self.split.decode_stream(buf, pts)?; + self.pull_config(); + self.write_frames(frames) } /// Finish the track, flushing any buffered data. @@ -441,168 +114,46 @@ impl Import { /// Any in-flight avc3 access unit is dropped. Pre-seek NALs would otherwise /// leak into the post-seek group with the wrong timestamp. pub fn seek(&mut self, sequence: u64) -> Result<()> { - if let State::Avc3 { current, .. } = &mut self.state { - *current = Avc3Frame::default(); - } + self.split.reset(); self.track.seek(sequence)?; Ok(()) } - fn pts(&mut self, hint: Option) -> Result { - if let Some(pts) = hint { - return Ok(pts); + /// Apply a newly-resolved config from the splitter to the catalog rendition. + fn pull_config(&mut self) { + if let Some(config) = self.split.take_config() { + self.catalog + .video + .renditions + .insert(self.track.name().to_string(), config.clone()); + self.config = Some(config); } - let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); - Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) } -} -impl Renditions for Import { - fn renditions(&self) -> &hang::Catalog { - &self.catalog - } -} + /// Write split frames to the track, refining the catalog jitter as it goes. + fn write_frames(&mut self, frames: impl IntoIterator) -> Result<()> { + for frame in frames { + let pts = frame.timestamp; + self.track.write(frame)?; -/// Detect the wire shape from leading bytes: a 3- or 4-byte Annex-B start -/// code means avc3, otherwise an AVCDecoderConfigurationRecord (avc1). -fn detect_mode(bytes: &[u8]) -> Mode { - let three_byte = matches!(bytes, [0, 0, 1, ..]); - let four_byte = matches!(bytes, [0, 0, 0, 1, ..]); - if three_byte || four_byte { - Mode::Avc3 - } else { - Mode::Avc1 - } -} - -/// Detect if an avc1-shaped (length-prefixed) buffer contains an IDR slice. -fn avc1_is_keyframe(data: &[u8], length_size: usize) -> bool { - let mut offset = 0; - while offset + length_size <= data.len() { - let nal_len = match length_size { - 1 => data[offset] as usize, - 2 => u16::from_be_bytes([data[offset], data[offset + 1]]) as usize, - 3 => u32::from_be_bytes([0, data[offset], data[offset + 1], data[offset + 2]]) as usize, - 4 => u32::from_be_bytes([data[offset], data[offset + 1], data[offset + 2], data[offset + 3]]) as usize, - _ => return false, - }; - offset += length_size; - if offset + nal_len > data.len() { - break; - } - if nal_len > 0 && data[offset] & 0x1f == 5 { - return true; // IDR slice + if let Some(jitter) = self.jitter.observe(pts) + && let Some(c) = self.catalog.video.renditions.get_mut(self.track.name()) + { + c.jitter = Some(jitter); + } } - offset += nal_len; + Ok(()) } - false } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn detect_mode_avc1_avcc_buffer() { - // AVCDecoderConfigurationRecord starts with configurationVersion = 1, profile, ... - // First byte is 0x01, definitely not a start code. - let avcc: &[u8] = &[ - 0x01, 0x42, 0xc0, 0x1f, 0xff, 0xe1, 0x00, 0x06, 0x67, 0x42, 0xc0, 0x1f, 0xde, 0xad, - ]; - assert_eq!(detect_mode(avcc), Mode::Avc1); - } - - #[test] - fn detect_mode_avc3_3byte_start_code() { - let nals: &[u8] = &[0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1f]; - assert_eq!(detect_mode(nals), Mode::Avc3); - } - - #[test] - fn detect_mode_avc3_4byte_start_code() { - let nals: &[u8] = &[0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1f]; - assert_eq!(detect_mode(nals), Mode::Avc3); - } - - /// Auto-detect routes an avcC initializer into the avc1 path and stores - /// it in the catalog `description`. - #[tokio::test(start_paused = true)] - async fn auto_detect_avc1_lands_in_catalog() { - // Minimal AVCDecoderConfigurationRecord: version(1) profile(0x42) compat(0xc0) level(0x1f) - // length_size_minus_one + 0xfc | 3 = 0xff - // reserved | num_sps = 0xe1 - // sps_len = 4, sps bytes (NAL header 0x67 + profile/level for parsing). - let sps_nal = [0x67, 0x42, 0xc0, 0x1f]; - let mut avcc = vec![0x01, 0x42, 0xc0, 0x1f, 0xff, 0xe1, 0x00, sps_nal.len() as u8]; - avcc.extend_from_slice(&sps_nal); - avcc.extend_from_slice(&[0x01, 0x00, 0x04, 0x68, 0xce, 0x3c, 0x80]); // num_pps + pps - - let mut importer = Import::new(moq_net::TrackRequest::new("0.avc1")); - let mut buf = bytes::BytesMut::from(avcc.as_slice()); - importer.initialize(&mut buf).expect("initialize avc1"); - - let catalog = importer.catalog().expect("config known after init"); - assert_eq!(catalog.video.renditions.len(), 1); - let cfg = catalog.video.renditions.values().next().unwrap(); - let hang::catalog::VideoCodec::H264(h264) = &cfg.codec else { - panic!("expected H.264 codec") - }; - assert!(!h264.inline, "avc1 source should land as inline=false"); - assert_eq!(h264.profile, 0x42); - assert_eq!(h264.level, 0x1f); - let desc = cfg.description.as_ref().expect("description set"); - assert_eq!(desc.as_ref(), avcc.as_slice()); - } - - /// Auto-detect routes an Annex-B initializer into the avc3 path; the - /// catalog rendition reports inline=true and no description. - #[tokio::test(start_paused = true)] - async fn auto_detect_avc3_lands_in_catalog() { - let sps: &[u8] = &[ - 0x67, 0x42, 0xc0, 0x1f, 0xda, 0x01, 0x40, 0x16, 0xe9, 0xb8, 0x08, 0x08, 0x0a, 0x00, 0x00, 0x07, 0xd0, 0x00, - 0x01, 0xd4, 0xc0, 0x80, - ]; - let pps: &[u8] = &[0x68, 0xce, 0x3c, 0x80]; - let mut annexb = bytes::BytesMut::new(); - annexb.extend_from_slice(&[0, 0, 0, 1]); - annexb.extend_from_slice(sps); - annexb.extend_from_slice(&[0, 0, 0, 1]); - annexb.extend_from_slice(pps); - - let mut importer = Import::new(moq_net::TrackRequest::new("0.avc3")); - importer.initialize(&mut annexb).expect("initialize avc3"); - - let catalog = importer.catalog().expect("config known after first SPS"); - assert_eq!(catalog.video.renditions.len(), 1); - let cfg = catalog.video.renditions.values().next().unwrap(); - let hang::catalog::VideoCodec::H264(h264) = &cfg.codec else { - panic!("expected H.264 codec") - }; - assert!(h264.inline, "avc3 source should land as inline=true"); - assert!(cfg.description.is_none(), "avc3 has no out-of-band description"); - assert_eq!(h264.profile, sps[1]); - assert_eq!(h264.level, sps[3]); +impl FrameDecode for Import { + fn decode>(&mut self, frames: I) -> Result<()> { + self.write_frames(frames) } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, num_enum::TryFromPrimitive)] -#[repr(u8)] -enum Avc3NalType { - Unspecified = 0, - NonIdrSlice = 1, - DataPartitionA = 2, - DataPartitionB = 3, - DataPartitionC = 4, - IdrSlice = 5, - Sei = 6, - Sps = 7, - Pps = 8, - Aud = 9, - EndOfSeq = 10, - EndOfStream = 11, - Filler = 12, - SpsExt = 13, - Prefix = 14, - SubsetSps = 15, - DepthParameterSet = 16, +impl Renditions for Import { + fn renditions(&self) -> &hang::Catalog { + &self.catalog + } } diff --git a/rs/moq-mux/src/codec/h264/mod.rs b/rs/moq-mux/src/codec/h264/mod.rs index c85bf64f4..636956b52 100644 --- a/rs/moq-mux/src/codec/h264/mod.rs +++ b/rs/moq-mux/src/codec/h264/mod.rs @@ -10,9 +10,11 @@ mod export; mod import; +mod split; pub use export::*; pub use import::*; +pub use split::*; use bytes::{Buf, BufMut, Bytes, BytesMut}; diff --git a/rs/moq-mux/src/codec/h264/split.rs b/rs/moq-mux/src/codec/h264/split.rs new file mode 100644 index 000000000..e827632b4 --- /dev/null +++ b/rs/moq-mux/src/codec/h264/split.rs @@ -0,0 +1,592 @@ +//! H.264 byte parser for both wire shapes. +//! +//! [`Split`] turns H.264 bytes into [`crate::container::Frame`]s plus a resolved +//! [`hang::catalog::VideoConfig`]. It accepts either length-prefixed NALU input +//! with an out-of-band [`AVCDecoderConfigurationRecord`](super::Avcc) (the "avc1" +//! shape) or Annex-B input with inline SPS/PPS (the "avc3" shape). The shape is +//! detected at [`initialize`](Split::initialize) time by looking for a leading +//! start code; callers that already know it can also force the mode via +//! [`with_mode`](Split::with_mode). +//! +//! Unlike a full importer, [`Split`] owns no track or catalog: it only parses, +//! emitting frames and surfacing the codec config via [`take_config`](Split::take_config). + +use bytes::{Buf, Bytes, BytesMut}; +use tokio::io::{AsyncRead, AsyncReadExt}; + +use super::{Error, Sps}; +use crate::Result; +use crate::codec::annexb::{NalIterator, START_CODE}; + +/// The wire shape a [`Split`] is processing. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Mode { + /// Length-prefixed NALU with out-of-band AVCDecoderConfigurationRecord + /// (catalog `H264 { inline: false }`, `description = avcC`). + Avc1, + /// Annex-B (start-code prefixed) with inline SPS/PPS + /// (catalog `H264 { inline: true }`, no description). + Avc3, +} + +/// H.264 byte parser. Handles both avc1 (length-prefixed) and avc3 (Annex-B) +/// input streams; the shape is detected from the first bytes the caller +/// supplies, or forced explicitly via [`with_mode`](Self::with_mode). +/// +/// Feed bytes via [`initialize`](Self::initialize), [`decode_frame`](Self::decode_frame), +/// [`decode_stream`](Self::decode_stream), or [`decode_from`](Self::decode_from); each +/// returns the [`Frame`](crate::container::Frame)s it produced. The resolved +/// [`hang::catalog::VideoConfig`] is exposed lazily once the codec config is known +/// (avcC for avc1, the first SPS for avc3) via [`take_config`](Self::take_config). +pub struct Split { + config: Option, + config_dirty: bool, + state: State, + zero: Option, + pending: Vec, +} + +enum State { + /// No bytes seen yet; mode pinned ahead of time or unknown. + Pending { mode_hint: Option }, + /// avc1 wire shape: length-prefixed NALU, codec config out-of-band. + Avc1 { length_size: usize }, + /// avc3 wire shape: Annex-B NALU, inline SPS/PPS. + Avc3 { + current: Avc3Frame, + sps: Option, + pps: Option, + }, +} + +#[derive(Default)] +struct Avc3Frame { + chunks: BytesMut, + contains_idr: bool, + contains_slice: bool, + contains_sps: bool, + contains_pps: bool, +} + +impl Default for Split { + fn default() -> Self { + Self::new() + } +} + +impl Split { + /// Auto-detect the wire shape from the first bytes supplied to + /// [`initialize`](Self::initialize). + pub fn new() -> Self { + Self { + config: None, + config_dirty: false, + state: State::Pending { mode_hint: None }, + zero: None, + pending: Vec::new(), + } + } + + /// Pin the wire shape ahead of time; skips the leading-bytes auto-detect + /// inside [`initialize`](Self::initialize). + pub fn with_mode(mode: Mode) -> Result { + let state = match mode { + Mode::Avc1 => State::Pending { + mode_hint: Some(Mode::Avc1), + }, + Mode::Avc3 => State::Avc3 { + current: Avc3Frame::default(), + sps: None, + pps: None, + }, + }; + Ok(Self { + config: None, + config_dirty: false, + state, + zero: None, + pending: Vec::new(), + }) + } + + /// Initialize from the codec's leading bytes. + /// + /// - **avc1** (no leading start code): the buffer is parsed as an + /// `AVCDecoderConfigurationRecord` and stored as the config `description`. + /// - **avc3** (leading `0x00 0x00 0x01` or `0x00 0x00 0x00 0x01`): the buffer + /// is parsed as Annex-B NALs to seed the cached SPS/PPS. + /// + /// The buffer is fully consumed. + pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { + let mode = match &self.state { + State::Pending { mode_hint } => mode_hint.unwrap_or_else(|| detect_mode(buf.as_ref())), + State::Avc1 { .. } => Mode::Avc1, + State::Avc3 { .. } => Mode::Avc3, + }; + + match mode { + Mode::Avc1 => self.initialize_avc1(buf), + Mode::Avc3 => self.initialize_avc3(buf), + } + } + + /// Initialize the avc1 path from an `AVCDecoderConfigurationRecord` buffer. + fn initialize_avc1>(&mut self, buf: &mut T) -> Result<()> { + let avcc_bytes = buf.as_ref(); + let avcc = super::Avcc::parse(avcc_bytes)?; + self.state = State::Avc1 { + length_size: avcc.length_size, + }; + + let mut config = hang::catalog::VideoConfig::new(hang::catalog::H264 { + profile: avcc.profile, + constraints: avcc.constraints, + level: avcc.level, + inline: false, + }); + config.coded_width = avcc.coded_width; + config.coded_height = avcc.coded_height; + config.description = Some(Bytes::copy_from_slice(avcc_bytes)); + config.container = hang::catalog::Container::Legacy; + + self.swap_config(config)?; + buf.advance(buf.remaining()); + + Ok(()) + } + + /// Initialize the avc3 path by parsing Annex-B NALs (SPS/PPS seed the + /// config once the first SPS is parsed). + fn initialize_avc3>(&mut self, buf: &mut T) -> Result<()> { + if !matches!(self.state, State::Avc3 { .. }) { + self.state = State::Avc3 { + current: Avc3Frame::default(), + sps: None, + pps: None, + }; + } + + let mut nals = NalIterator::new(buf); + while let Some(nal) = nals.next().transpose()? { + self.decode_nal(nal, None)?; + } + if let Some(nal) = nals.flush()? { + self.decode_nal(nal, None)?; + } + + Ok(()) + } + + /// True once the codec config has been resolved. + pub fn is_initialized(&self) -> bool { + self.config.is_some() + } + + /// The resolved config if it changed since the last call, else `None`. + /// + /// Set whenever a new SPS/avcC is parsed; returns `Some` once per change. + pub fn take_config(&mut self) -> Option { + if self.config_dirty { + self.config_dirty = false; + self.config.clone() + } else { + None + } + } + + /// Decode from an asynchronous reader, returning all frames produced. + /// + /// avc3 only. For avc1, the caller already has framed buffers and uses + /// [`decode_frame`](Self::decode_frame). + pub async fn decode_from(&mut self, reader: &mut T) -> Result> { + let mut frames = Vec::new(); + let mut buffer = BytesMut::new(); + while reader.read_buf(&mut buffer).await? > 0 { + frames.extend(self.decode_stream(&mut buffer, None)?); + } + Ok(frames) + } + + /// Decode a buffer where frame boundaries are unknown (avc3 streaming + /// input), returning the frames it produced. The leading start code of the + /// *next* frame is what signals the previous frame is done. + pub fn decode_stream>( + &mut self, + buf: &mut T, + pts: Option, + ) -> Result> { + if !matches!(self.state, State::Avc3 { .. }) { + return Err(Error::StreamNotAvc3.into()); + } + let pts = self.pts(pts)?; + let nals = NalIterator::new(buf); + for nal in nals { + self.decode_nal(nal?, Some(pts))?; + } + Ok(std::mem::take(&mut self.pending)) + } + + /// Decode a buffer assumed to hold (the rest of) a single frame, returning + /// the frames it produced. + /// + /// - avc1: the buffer is written as one length-prefixed-NALU frame. + /// - avc3: NALs are parsed; any trailing NAL without a start code is + /// flushed as the last NAL of this frame. + pub fn decode_frame>( + &mut self, + buf: &mut T, + pts: Option, + ) -> Result> { + match &self.state { + State::Avc1 { .. } => self.decode_avc1(buf, pts)?, + State::Avc3 { .. } => self.decode_avc3_frame(buf, pts)?, + State::Pending { .. } => return Err(Error::NotInitialized.into()), + } + Ok(std::mem::take(&mut self.pending)) + } + + fn decode_avc1>(&mut self, buf: &mut T, pts: Option) -> Result<()> { + let State::Avc1 { length_size } = self.state else { + unreachable!("checked by decode_frame") + }; + let data = buf.as_ref(); + let pts = self.pts(pts)?; + let keyframe = avc1_is_keyframe(data, length_size); + + self.pending.push(crate::container::Frame { + timestamp: pts, + payload: data.to_vec().into(), + keyframe, + duration: None, + }); + + buf.advance(buf.remaining()); + Ok(()) + } + + fn decode_avc3_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { + let pts = self.pts(pts)?; + let mut nals = NalIterator::new(buf); + while let Some(nal) = nals.next().transpose()? { + self.decode_nal(nal, Some(pts))?; + } + if let Some(nal) = nals.flush()? { + self.decode_nal(nal, Some(pts))?; + } + self.maybe_start_frame(Some(pts))?; + Ok(()) + } + + fn decode_nal(&mut self, nal: Bytes, pts: Option) -> Result<()> { + let header = nal.first().ok_or(Error::NalTooShort)?; + let forbidden_zero_bit = (header >> 7) & 1; + if forbidden_zero_bit != 0 { + return Err(Error::ForbiddenZeroBit.into()); + } + + let nal_unit_type = header & 0b11111; + let nal_type = Avc3NalType::try_from(nal_unit_type).ok(); + + match nal_type { + Some(Avc3NalType::Sps) => { + self.maybe_start_frame(pts)?; + let sps = Sps::parse(&nal)?; + self.init_from_sps(&sps)?; + let State::Avc3 { current, sps, pps } = &mut self.state else { + unreachable!("decode_nal is avc3 only") + }; + if sps.as_ref().is_some_and(|cached| cached != &nal) { + // SPS changed mid-AU. The cached PPS is tied to the old SPS + // and may already have been appended to current.chunks + // earlier in this AU; reset the AU so the new SPS+PPS pair + // is the only parameter set we emit. + *pps = None; + current.chunks.clear(); + current.contains_pps = false; + current.contains_sps = false; + } + *sps = Some(nal.clone()); + current.contains_sps = true; + } + Some(Avc3NalType::Pps) => { + self.maybe_start_frame(pts)?; + let State::Avc3 { current, pps, .. } = &mut self.state else { + unreachable!() + }; + *pps = Some(nal.clone()); + current.contains_pps = true; + } + Some(Avc3NalType::Aud) | Some(Avc3NalType::Sei) => { + self.maybe_start_frame(pts)?; + } + Some(Avc3NalType::IdrSlice) => { + let State::Avc3 { current, sps, pps } = &mut self.state else { + unreachable!() + }; + if !current.contains_sps + && let Some(sps) = sps.as_ref() + { + current.chunks.extend_from_slice(&START_CODE); + current.chunks.extend_from_slice(sps); + current.contains_sps = true; + } + if !current.contains_pps + && let Some(pps) = pps.as_ref() + { + current.chunks.extend_from_slice(&START_CODE); + current.chunks.extend_from_slice(pps); + current.contains_pps = true; + } + current.contains_idr = true; + current.contains_slice = true; + } + Some(Avc3NalType::NonIdrSlice) + | Some(Avc3NalType::DataPartitionA) + | Some(Avc3NalType::DataPartitionB) + | Some(Avc3NalType::DataPartitionC) => { + if nal.get(1).ok_or(Error::NalTooShort)? & 0x80 != 0 { + self.maybe_start_frame(pts)?; + } + let State::Avc3 { current, .. } = &mut self.state else { + unreachable!() + }; + current.contains_slice = true; + } + _ => {} + } + + tracing::trace!(kind = ?nal_type, "parsed NAL"); + + let State::Avc3 { current, .. } = &mut self.state else { + unreachable!() + }; + current.chunks.extend_from_slice(&START_CODE); + current.chunks.extend_from_slice(&nal); + Ok(()) + } + + fn init_from_sps(&mut self, sps: &Sps) -> Result<()> { + let mut config = hang::catalog::VideoConfig::new(hang::catalog::H264 { + profile: sps.profile, + constraints: sps.constraints, + level: sps.level, + inline: true, + }); + config.coded_width = Some(sps.coded_width); + config.coded_height = Some(sps.coded_height); + config.container = hang::catalog::Container::Legacy; + + if let Some(old) = &self.config + && old == &config + { + return Ok(()); + } + + // avc3 carries SPS inline, so a resolution change updates the config in + // place (no new init segment, unlike avc1). + self.config = Some(config); + self.config_dirty = true; + Ok(()) + } + + fn maybe_start_frame(&mut self, pts: Option) -> Result<()> { + let State::Avc3 { current, .. } = &mut self.state else { + return Ok(()); + }; + if !current.contains_slice { + return Ok(()); + } + let pts = pts.ok_or(Error::MissingTimestamp)?; + let payload = std::mem::take(&mut current.chunks).freeze(); + let keyframe = current.contains_idr; + current.contains_idr = false; + current.contains_slice = false; + current.contains_sps = false; + current.contains_pps = false; + + self.pending.push(crate::container::Frame { + timestamp: pts, + payload, + keyframe, + duration: None, + }); + Ok(()) + } + + /// Resolve the avc1 codec config. + /// + /// The first config is stored. A different avcC would mean a new init + /// segment, which a single fixed track can't represent, so a reconfiguration + /// is an error (mint a new track via a fresh parser). + fn swap_config(&mut self, config: hang::catalog::VideoConfig) -> Result<()> { + if let Some(old) = &self.config { + if old == &config { + return Ok(()); + } + return Err(Error::FixedTrackReconfigured.into()); + } + + tracing::debug!(?config, "starting H.264 track"); + self.config = Some(config); + self.config_dirty = true; + Ok(()) + } + + /// Drop any in-flight avc3 access unit. + /// + /// Pre-reset NALs would otherwise leak into a later frame with the wrong + /// timestamp. + pub fn reset(&mut self) { + if let State::Avc3 { current, .. } = &mut self.state { + *current = Avc3Frame::default(); + } + } + + fn pts(&mut self, hint: Option) -> Result { + if let Some(pts) = hint { + return Ok(pts); + } + let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); + Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) + } +} + +/// Detect the wire shape from leading bytes: a 3- or 4-byte Annex-B start +/// code means avc3, otherwise an AVCDecoderConfigurationRecord (avc1). +fn detect_mode(bytes: &[u8]) -> Mode { + let three_byte = matches!(bytes, [0, 0, 1, ..]); + let four_byte = matches!(bytes, [0, 0, 0, 1, ..]); + if three_byte || four_byte { + Mode::Avc3 + } else { + Mode::Avc1 + } +} + +/// Detect if an avc1-shaped (length-prefixed) buffer contains an IDR slice. +fn avc1_is_keyframe(data: &[u8], length_size: usize) -> bool { + let mut offset = 0; + while offset + length_size <= data.len() { + let nal_len = match length_size { + 1 => data[offset] as usize, + 2 => u16::from_be_bytes([data[offset], data[offset + 1]]) as usize, + 3 => u32::from_be_bytes([0, data[offset], data[offset + 1], data[offset + 2]]) as usize, + 4 => u32::from_be_bytes([data[offset], data[offset + 1], data[offset + 2], data[offset + 3]]) as usize, + _ => return false, + }; + offset += length_size; + if offset + nal_len > data.len() { + break; + } + if nal_len > 0 && data[offset] & 0x1f == 5 { + return true; // IDR slice + } + offset += nal_len; + } + false +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, num_enum::TryFromPrimitive)] +#[repr(u8)] +enum Avc3NalType { + Unspecified = 0, + NonIdrSlice = 1, + DataPartitionA = 2, + DataPartitionB = 3, + DataPartitionC = 4, + IdrSlice = 5, + Sei = 6, + Sps = 7, + Pps = 8, + Aud = 9, + EndOfSeq = 10, + EndOfStream = 11, + Filler = 12, + SpsExt = 13, + Prefix = 14, + SubsetSps = 15, + DepthParameterSet = 16, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detect_mode_avc1_avcc_buffer() { + // AVCDecoderConfigurationRecord starts with configurationVersion = 1, profile, ... + // First byte is 0x01, definitely not a start code. + let avcc: &[u8] = &[ + 0x01, 0x42, 0xc0, 0x1f, 0xff, 0xe1, 0x00, 0x06, 0x67, 0x42, 0xc0, 0x1f, 0xde, 0xad, + ]; + assert_eq!(detect_mode(avcc), Mode::Avc1); + } + + #[test] + fn detect_mode_avc3_3byte_start_code() { + let nals: &[u8] = &[0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1f]; + assert_eq!(detect_mode(nals), Mode::Avc3); + } + + #[test] + fn detect_mode_avc3_4byte_start_code() { + let nals: &[u8] = &[0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1f]; + assert_eq!(detect_mode(nals), Mode::Avc3); + } + + /// Auto-detect routes an avcC initializer into the avc1 path and resolves a + /// config with the avcC stored as `description`. + #[tokio::test(start_paused = true)] + async fn auto_detect_avc1_lands_in_catalog() { + // Minimal AVCDecoderConfigurationRecord: version(1) profile(0x42) compat(0xc0) level(0x1f) + // length_size_minus_one + 0xfc | 3 = 0xff + // reserved | num_sps = 0xe1 + // sps_len = 4, sps bytes (NAL header 0x67 + profile/level for parsing). + let sps_nal = [0x67, 0x42, 0xc0, 0x1f]; + let mut avcc = vec![0x01, 0x42, 0xc0, 0x1f, 0xff, 0xe1, 0x00, sps_nal.len() as u8]; + avcc.extend_from_slice(&sps_nal); + avcc.extend_from_slice(&[0x01, 0x00, 0x04, 0x68, 0xce, 0x3c, 0x80]); // num_pps + pps + + let mut split = Split::new(); + let mut buf = bytes::BytesMut::from(avcc.as_slice()); + split.initialize(&mut buf).expect("initialize avc1"); + + let cfg = split.take_config().expect("config known after init"); + let hang::catalog::VideoCodec::H264(h264) = &cfg.codec else { + panic!("expected H.264 codec") + }; + assert!(!h264.inline, "avc1 source should land as inline=false"); + assert_eq!(h264.profile, 0x42); + assert_eq!(h264.level, 0x1f); + let desc = cfg.description.as_ref().expect("description set"); + assert_eq!(desc.as_ref(), avcc.as_slice()); + } + + /// Auto-detect routes an Annex-B initializer into the avc3 path; the + /// resolved config reports inline=true and no description. + #[tokio::test(start_paused = true)] + async fn auto_detect_avc3_lands_in_catalog() { + let sps: &[u8] = &[ + 0x67, 0x42, 0xc0, 0x1f, 0xda, 0x01, 0x40, 0x16, 0xe9, 0xb8, 0x08, 0x08, 0x0a, 0x00, 0x00, 0x07, 0xd0, 0x00, + 0x01, 0xd4, 0xc0, 0x80, + ]; + let pps: &[u8] = &[0x68, 0xce, 0x3c, 0x80]; + let mut annexb = bytes::BytesMut::new(); + annexb.extend_from_slice(&[0, 0, 0, 1]); + annexb.extend_from_slice(sps); + annexb.extend_from_slice(&[0, 0, 0, 1]); + annexb.extend_from_slice(pps); + + let mut split = Split::new(); + split.initialize(&mut annexb).expect("initialize avc3"); + + let cfg = split.take_config().expect("config known after first SPS"); + let hang::catalog::VideoCodec::H264(h264) = &cfg.codec else { + panic!("expected H.264 codec") + }; + assert!(h264.inline, "avc3 source should land as inline=true"); + assert!(cfg.description.is_none(), "avc3 has no out-of-band description"); + assert_eq!(h264.profile, sps[1]); + assert_eq!(h264.level, sps[3]); + } +} diff --git a/rs/moq-mux/src/publish.rs b/rs/moq-mux/src/publish.rs index 98853f56a..32fe519c1 100644 --- a/rs/moq-mux/src/publish.rs +++ b/rs/moq-mux/src/publish.rs @@ -30,6 +30,17 @@ pub trait Renditions { fn renditions(&self) -> &hang::Catalog; } +/// A single-track importer that publishes already-split frames. +/// +/// The uniform decode entry point: callers split bytes into [`Frame`](crate::container::Frame)s +/// (a per-format splitter, e.g. [`crate::codec::h264::Split`]) and hand them over. +/// [`Published`] wraps this so the catalog re-mirror can't be forgotten (see +/// [`Published::decode`]). +pub trait FrameDecode { + /// Publish frames on this importer's track. + fn decode>(&mut self, frames: I) -> crate::Result<()>; +} + /// Mint a fresh unique track for a legacy single-codec importer. /// /// Picks a unique name from `suffix` and sets the microsecond @@ -107,6 +118,18 @@ impl Published { } } +impl Published { + /// Publish frames and re-mirror any catalog change in one call. + /// + /// This is the footgun-free path: it [`sync`](Self::sync)s after decoding, so + /// a lazily-resolved config or refined jitter always reaches the catalog. + pub fn decode>(&mut self, frames: It) -> crate::Result<()> { + self.inner.decode(frames)?; + self.sync(); + Ok(()) + } +} + impl Deref for Published { type Target = I; @@ -150,6 +173,14 @@ mod tests { } } + impl FrameDecode for Fake { + fn decode>(&mut self, _frames: I) -> crate::Result<()> { + // Simulate an importer resolving its config lazily while decoding. + self.0.video.renditions.insert("v".to_string(), video()); + Ok(()) + } + } + fn video() -> hang::catalog::VideoConfig { let mut config = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); config.container = hang::catalog::Container::Legacy; @@ -179,4 +210,17 @@ mod tests { drop(published); assert!(catalog.snapshot().video.renditions.is_empty()); } + + #[tokio::test(start_paused = true)] + async fn decode_auto_syncs_catalog() { + let mut broadcast = moq_net::BroadcastInfo::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); + + let mut published = Published::new(catalog.clone(), Fake(hang::Catalog::default())); + assert!(catalog.snapshot().video.renditions.is_empty()); + + // `decode` resolves the rendition and mirrors it — no manual `sync()`. + published.decode(std::iter::empty::()).unwrap(); + assert!(catalog.snapshot().video.renditions.contains_key("v")); + } } From f752b7bea3d8b6a8cb3ed209fc9a9b8d82805acb Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 17:24:18 +0000 Subject: [PATCH 06/19] moq-mux: make the H.264 Split dumb; move config into the importer Per review: the splitter shouldn't know what a codec config is. It just takes a single byte stream (no out-of-band init), finds access-unit boundaries, and packages SPS/PPS into each keyframe so every keyframe is self-contained. - `codec::h264::Split` is now an Annex-B stream assembler only: no avc1, no `VideoConfig`, no `take_config`. It exposes `decode_stream` (unknown boundaries), `decode_frame` (one access unit), `decode_from`, `seed` (prime the SPS/PPS cache from an out-of-band parameter-set buffer), and `reset`. Wall-clock timestamps for stdin live here. - `codec::h264::Import` owns all config, from exactly two sources: an avcC handed to `initialize` (avc1, required), or the SPS the splitter packages into the first keyframe, scanned out of the frame here (avc3, no init needed). It also owns the avc1 length-prefixed framed path; the stream splitter is Annex-B/avc3 only, matching "if you can init out of band you already know frame boundaries, so you don't need the stream splitter". - A keyframe that can't be configured (no inline SPS, no avcC/seed) is a hard error. A non-keyframe before the first config is tolerated: it's a mid-stream-join leftover that the producer's lenient start drops ahead of the first keyframe (preserves `survives_midstream_join` and the dirty TS joins). https://claude.ai/code/session_011S7pzcg2XsPP3AExuymac1 --- rs/moq-mux/src/codec/h264/import.rs | 468 ++++++++++++++++++++++-- rs/moq-mux/src/codec/h264/split.rs | 538 +++++++--------------------- 2 files changed, 572 insertions(+), 434 deletions(-) diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index b94b1aa83..289464315 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -1,24 +1,43 @@ //! H.264 importer. //! -//! Publishes split H.264 frames on a single moq track and tracks the catalog -//! rendition. Byte parsing lives in [`Split`]; this type drives it for the -//! convenience byte APIs ([`decode_frame`](Import::decode_frame) / -//! [`decode_stream`](Import::decode_stream)) and writes the resulting frames, -//! and also implements [`FrameDecode`] so a caller that runs its own [`Split`] -//! can publish frames directly. - -use bytes::{Buf, BytesMut}; +//! [`Import`] publishes H.264 frames on a single moq track and resolves the +//! catalog rendition. It accepts either length-prefixed NALU input with an +//! out-of-band [`AVCDecoderConfigurationRecord`](super::Avcc) (the "avc1" shape) +//! or Annex-B input with inline SPS/PPS (the "avc3" shape). The shape is detected +//! from the first bytes the caller supplies, or forced via +//! [`with_mode`](Import::with_mode). +//! +//! The codec config comes from exactly one of two places: an avcC handed to +//! [`initialize`](Import::initialize) (avc1), or the SPS that the splitter +//! packages into the first keyframe (avc3, scanned out of the frame here). A +//! keyframe that can't be configured from either is an error; non-keyframes +//! before the first config are tolerated (mid-stream joins). Annex-B byte +//! parsing lives in [`Split`]; this type drives it and adds the catalog. + +use bytes::{Buf, Bytes, BytesMut}; use tokio::io::{AsyncRead, AsyncReadExt}; -use super::{Mode, Split}; +use super::{Error, NAL_TYPE_SPS, Split, Sps}; use crate::Result; +use crate::codec::annexb::NalIterator; use crate::container::Frame; use crate::container::jitter::MinFrameDuration; use crate::publish::{FrameDecode, Renditions}; -/// H.264 importer. Handles both avc1 (length-prefixed) and avc3 (Annex-B) -/// input streams; the shape is detected from the first bytes the caller -/// supplies, or forced explicitly via [`with_mode`](Self::with_mode). +/// The wire shape an [`Import`] processes. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Mode { + /// Length-prefixed NALU with out-of-band AVCDecoderConfigurationRecord + /// (catalog `H264 { inline: false }`, `description = avcC`). + Avc1, + /// Annex-B (start-code prefixed) with inline SPS/PPS + /// (catalog `H264 { inline: true }`, no description). + Avc3, +} + +/// H.264 importer. Handles both avc1 (length-prefixed) and avc3 (Annex-B) input; +/// the shape is detected from the first bytes the caller supplies, or forced via +/// [`with_mode`](Self::with_mode). /// /// Build it from a [`moq_net::TrackRequest`] ([`new`](Self::new), the on-demand /// path) or an existing [`moq_net::TrackProducer`] ([`from_track`](Self::from_track), @@ -27,13 +46,24 @@ use crate::publish::{FrameDecode, Renditions}; /// via [`catalog`](Self::catalog) or attach the importer to a broadcast catalog /// with [`crate::publish::Published`]. pub struct Import { + shape: Shape, split: Split, track: crate::container::Producer, catalog: hang::Catalog, config: Option, + last_sps: Option, jitter: MinFrameDuration, } +enum Shape { + /// No bytes seen yet; mode pinned ahead of time or still unknown. + Pending { hint: Option }, + /// avc1: length-prefixed NALU, codec config out-of-band (avcC). + Avc1 { length_size: usize }, + /// avc3: Annex-B NALU, inline SPS/PPS. + Avc3, +} + impl Import { /// Serve a track request, accepting it at the microsecond timescale. pub fn new(request: moq_net::TrackRequest) -> Self { @@ -44,24 +74,93 @@ impl Import { /// Publish on an existing track producer. pub fn from_track(track: moq_net::TrackProducer) -> Self { Self { + shape: Shape::Pending { hint: None }, split: Split::new(), track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy).with_lenient_start(), catalog: hang::Catalog::default(), config: None, + last_sps: None, jitter: MinFrameDuration::new(), } } /// Pin the wire shape ahead of time; skips the leading-bytes auto-detect. + /// + /// avc1 still needs an [`initialize`](Self::initialize) with the avcC to + /// learn the NALU length size and codec config. pub fn with_mode(mut self, mode: Mode) -> Result { - self.split = Split::with_mode(mode)?; + self.shape = match mode { + Mode::Avc1 => Shape::Pending { hint: Some(Mode::Avc1) }, + Mode::Avc3 => Shape::Avc3, + }; Ok(self) } - /// Initialize from the codec's leading bytes (avcC for avc1, SPS/PPS for avc3). + /// Initialize from the codec's leading bytes. + /// + /// - **avc1** (no leading start code): the buffer is parsed as an + /// `AVCDecoderConfigurationRecord`, which resolves the config and is stored + /// as the catalog `description`. Required for avc1. + /// - **avc3** (leading start code): the buffer is parsed as Annex-B; any SPS + /// resolves the config and primes the splitter's parameter-set cache. + /// Optional, since avc3 also self-initializes from the first keyframe. + /// + /// The buffer is fully consumed. pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { - self.split.initialize(buf)?; - self.pull_config(); + let mode = match &self.shape { + Shape::Pending { hint } => hint.unwrap_or_else(|| detect_mode(buf.as_ref())), + Shape::Avc1 { .. } => Mode::Avc1, + Shape::Avc3 => Mode::Avc3, + }; + + match mode { + Mode::Avc1 => self.initialize_avc1(buf), + Mode::Avc3 => self.initialize_avc3(buf), + } + } + + fn initialize_avc1>(&mut self, buf: &mut T) -> Result<()> { + let avcc_bytes = buf.as_ref(); + let avcc = super::Avcc::parse(avcc_bytes)?; + self.shape = Shape::Avc1 { + length_size: avcc.length_size, + }; + + let mut config = hang::catalog::VideoConfig::new(hang::catalog::H264 { + profile: avcc.profile, + constraints: avcc.constraints, + level: avcc.level, + inline: false, + }); + config.coded_width = avcc.coded_width; + config.coded_height = avcc.coded_height; + config.description = Some(Bytes::copy_from_slice(avcc_bytes)); + config.container = hang::catalog::Container::Legacy; + + self.set_config(config)?; + buf.advance(buf.remaining()); + Ok(()) + } + + fn initialize_avc3>(&mut self, buf: &mut T) -> Result<()> { + self.shape = Shape::Avc3; + + // Resolve the config from any SPS in the seed buffer, then prime the + // splitter's cache so the first keyframe is self-contained. + let mut scan = Bytes::copy_from_slice(buf.as_ref()); + let mut nals = NalIterator::new(&mut scan); + while let Some(nal) = nals.next().transpose()? { + if is_sps(&nal) { + self.configure_from_sps(&nal)?; + } + } + if let Some(nal) = nals.flush()? + && is_sps(&nal) + { + self.configure_from_sps(&nal)?; + } + + self.split.seed(buf)?; Ok(()) } @@ -89,17 +188,44 @@ impl Import { Ok(()) } - /// Decode a buffer holding (the rest of) a single frame. + /// Decode a buffer holding one complete frame. + /// + /// - avc1: the buffer is one length-prefixed-NALU access unit. + /// - avc3: the buffer is one Annex-B access unit. pub fn decode_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - let frames = self.split.decode_frame(buf, pts)?; - self.pull_config(); - self.write_frames(frames) + match self.shape { + Shape::Avc1 { length_size } => { + let frame = read_avc1_frame(buf, length_size, pts)?; + self.write_frames([frame]) + } + Shape::Avc3 => { + let frames = self.split.decode_frame(buf, pts)?; + self.write_frames(frames) + } + Shape::Pending { hint } => match hint.unwrap_or_else(|| detect_mode(buf.as_ref())) { + Mode::Avc3 => { + self.shape = Shape::Avc3; + let frames = self.split.decode_frame(buf, pts)?; + self.write_frames(frames) + } + // avc1 needs the avcC (length size) from initialize() first. + Mode::Avc1 => Err(Error::NotInitialized.into()), + }, + } } /// Decode a buffer where frame boundaries are unknown (avc3 streaming input). pub fn decode_stream>(&mut self, buf: &mut T, pts: Option) -> Result<()> { + match self.shape { + Shape::Avc3 => {} + Shape::Pending { + hint: None | Some(Mode::Avc3), + } => self.shape = Shape::Avc3, + Shape::Avc1 { .. } | Shape::Pending { hint: Some(Mode::Avc1) } => { + return Err(Error::StreamNotAvc3.into()); + } + } let frames = self.split.decode_stream(buf, pts)?; - self.pull_config(); self.write_frames(frames) } @@ -119,20 +245,78 @@ impl Import { Ok(()) } - /// Apply a newly-resolved config from the splitter to the catalog rendition. - fn pull_config(&mut self) { - if let Some(config) = self.split.take_config() { - self.catalog - .video - .renditions - .insert(self.track.name().to_string(), config.clone()); - self.config = Some(config); + /// Resolve the avc3 config from an inline SPS, updating it in place. + /// + /// avc3 carries SPS inline, so a resolution change just updates the config + /// (no new init segment, unlike avc1). + fn configure_from_sps(&mut self, sps_nal: &Bytes) -> Result<()> { + if self.last_sps.as_ref() == Some(sps_nal) { + return Ok(()); + } + let sps = Sps::parse(sps_nal)?; + let mut config = hang::catalog::VideoConfig::new(hang::catalog::H264 { + profile: sps.profile, + constraints: sps.constraints, + level: sps.level, + inline: true, + }); + config.coded_width = Some(sps.coded_width); + config.coded_height = Some(sps.coded_height); + config.container = hang::catalog::Container::Legacy; + + self.last_sps = Some(sps_nal.clone()); + self.apply_config(config); + Ok(()) + } + + /// Resolve the avc1 config from an avcC. + /// + /// The first config is stored. A different avcC would mean a new init + /// segment, which a single fixed track can't represent, so a reconfiguration + /// is an error (mint a new track via a fresh importer). + fn set_config(&mut self, config: hang::catalog::VideoConfig) -> Result<()> { + if let Some(old) = &self.config { + if old == &config { + return Ok(()); + } + return Err(Error::FixedTrackReconfigured.into()); } + self.apply_config(config); + Ok(()) } - /// Write split frames to the track, refining the catalog jitter as it goes. + fn apply_config(&mut self, config: hang::catalog::VideoConfig) { + tracing::debug!(?config, "starting H.264 track"); + self.catalog + .video + .renditions + .insert(self.track.name().to_string(), config.clone()); + self.config = Some(config); + } + + /// Write split frames to the track, resolving the avc3 config from the first + /// keyframe's inline SPS and refining the catalog jitter as it goes. fn write_frames(&mut self, frames: impl IntoIterator) -> Result<()> { for frame in frames { + // avc1 config arrives out-of-band via initialize(); avc3 (and the + // not-yet-resolved Pending case) carries SPS inline on keyframes. + if !matches!(self.shape, Shape::Avc1 { .. }) + && frame.keyframe + && let Some(sps) = find_sps(&frame.payload) + { + self.configure_from_sps(&sps)?; + } + + if self.config.is_none() { + // A keyframe we still can't configure is undecodable, so bail + // loudly. A non-keyframe before config is a mid-stream-join + // leftover: write it and let the producer's lenient start drop it + // ahead of the first keyframe. + if frame.keyframe { + return Err(Error::NotInitialized.into()); + } + } + let pts = frame.timestamp; self.track.write(frame)?; @@ -157,3 +341,227 @@ impl Renditions for Import { &self.catalog } } + +/// Detect the wire shape from leading bytes: a 3- or 4-byte Annex-B start code +/// means avc3, otherwise an AVCDecoderConfigurationRecord (avc1). +fn detect_mode(bytes: &[u8]) -> Mode { + if matches!(bytes, [0, 0, 1, ..]) || matches!(bytes, [0, 0, 0, 1, ..]) { + Mode::Avc3 + } else { + Mode::Avc1 + } +} + +/// Build one avc1 frame from a length-prefixed-NALU buffer, scanning for an IDR +/// to set the keyframe flag. avc1 always carries timestamps, so a missing `pts` +/// is an error. +fn read_avc1_frame>( + buf: &mut T, + length_size: usize, + pts: Option, +) -> Result { + let data = buf.as_ref(); + let pts = pts.ok_or(Error::MissingTimestamp)?; + let keyframe = avc1_is_keyframe(data, length_size); + let frame = Frame { + timestamp: pts, + payload: data.to_vec().into(), + keyframe, + duration: None, + }; + buf.advance(buf.remaining()); + Ok(frame) +} + +fn is_sps(nal: &[u8]) -> bool { + nal.first().is_some_and(|h| h & 0x1f == NAL_TYPE_SPS) +} + +/// Find the first SPS NAL in an Annex-B payload, if any. +fn find_sps(payload: &[u8]) -> Option { + let mut buf = Bytes::copy_from_slice(payload); + let mut nals = NalIterator::new(&mut buf); + while let Some(Ok(nal)) = nals.next() { + if is_sps(&nal) { + return Some(nal); + } + } + nals.flush().ok().flatten().filter(|nal| is_sps(nal)) +} + +/// Detect whether an avc1-shaped (length-prefixed) buffer contains an IDR slice. +fn avc1_is_keyframe(data: &[u8], length_size: usize) -> bool { + let mut offset = 0; + while offset + length_size <= data.len() { + let nal_len = match length_size { + 1 => data[offset] as usize, + 2 => u16::from_be_bytes([data[offset], data[offset + 1]]) as usize, + 3 => u32::from_be_bytes([0, data[offset], data[offset + 1], data[offset + 2]]) as usize, + 4 => u32::from_be_bytes([data[offset], data[offset + 1], data[offset + 2], data[offset + 3]]) as usize, + _ => return false, + }; + offset += length_size; + if offset + nal_len > data.len() { + break; + } + if nal_len > 0 && data[offset] & 0x1f == 5 { + return true; // IDR slice + } + offset += nal_len; + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detect_mode_avc1_avcc_buffer() { + // AVCDecoderConfigurationRecord starts with configurationVersion = 1; + // the first byte is 0x01, never a start code. + let avcc: &[u8] = &[ + 0x01, 0x42, 0xc0, 0x1f, 0xff, 0xe1, 0x00, 0x06, 0x67, 0x42, 0xc0, 0x1f, 0xde, 0xad, + ]; + assert_eq!(detect_mode(avcc), Mode::Avc1); + } + + #[test] + fn detect_mode_avc3_3byte_start_code() { + assert_eq!(detect_mode(&[0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1f]), Mode::Avc3); + } + + #[test] + fn detect_mode_avc3_4byte_start_code() { + assert_eq!( + detect_mode(&[0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1f]), + Mode::Avc3 + ); + } + + /// An avcC initializer routes into the avc1 path and resolves a config with + /// the avcC stored as `description`. + #[tokio::test(start_paused = true)] + async fn initialize_avc1_lands_in_catalog() { + let sps_nal = [0x67, 0x42, 0xc0, 0x1f]; + let mut avcc = vec![0x01, 0x42, 0xc0, 0x1f, 0xff, 0xe1, 0x00, sps_nal.len() as u8]; + avcc.extend_from_slice(&sps_nal); + avcc.extend_from_slice(&[0x01, 0x00, 0x04, 0x68, 0xce, 0x3c, 0x80]); // num_pps + pps + + let mut broadcast = moq_net::BroadcastInfo::new().produce(); + let track = broadcast + .create_track( + "video", + moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), + ) + .unwrap(); + let mut import = Import::from_track(track); + let mut buf = bytes::BytesMut::from(avcc.as_slice()); + import.initialize(&mut buf).expect("initialize avc1"); + + let cfg = import + .catalog() + .expect("catalog known after init") + .video + .renditions + .get("video") + .expect("rendition"); + let hang::catalog::VideoCodec::H264(h264) = &cfg.codec else { + panic!("expected H.264 codec") + }; + assert!(!h264.inline, "avc1 source should land as inline=false"); + assert_eq!(h264.profile, 0x42); + assert_eq!(h264.level, 0x1f); + assert_eq!(cfg.description.as_ref().expect("description").as_ref(), avcc.as_slice()); + } + + /// An avc3 stream self-initializes: no `initialize`, the config is resolved + /// from the SPS the splitter packages into the first keyframe. + #[tokio::test(start_paused = true)] + async fn avc3_self_initializes_from_first_keyframe() { + let sps: &[u8] = &[ + 0x67, 0x42, 0xc0, 0x1f, 0xda, 0x01, 0x40, 0x16, 0xe9, 0xb8, 0x08, 0x08, 0x0a, 0x00, 0x00, 0x07, 0xd0, 0x00, + 0x01, 0xd4, 0xc0, 0x80, + ]; + let pps: &[u8] = &[0x68, 0xce, 0x3c, 0x80]; + let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; + + let mut annexb = bytes::BytesMut::new(); + for nal in [sps, pps, idr] { + annexb.extend_from_slice(&[0, 0, 0, 1]); + annexb.extend_from_slice(nal); + } + + let mut broadcast = moq_net::BroadcastInfo::new().produce(); + let track = broadcast + .create_track( + "video", + moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), + ) + .unwrap(); + let mut import = Import::from_track(track); + assert!(import.catalog().is_none(), "no config before any frame"); + + import + .decode_frame(&mut annexb, Some(moq_net::Timestamp::from_micros(0).unwrap())) + .expect("decode keyframe"); + + let cfg = import.catalog().expect("config after keyframe"); + let h264_cfg = cfg.video.renditions.get("video").expect("rendition"); + let hang::catalog::VideoCodec::H264(h264) = &h264_cfg.codec else { + panic!("expected H.264 codec") + }; + assert!(h264.inline, "avc3 source should land as inline=true"); + assert!(h264_cfg.description.is_none(), "avc3 has no out-of-band description"); + assert_eq!(h264.profile, sps[1]); + assert_eq!(h264.level, sps[3]); + } + + /// A keyframe that carries no SPS (and no avcC/seed to fall back on) is + /// undecodable, so it's a hard error rather than an uncatalogued frame. + #[tokio::test(start_paused = true)] + async fn keyframe_without_sps_errors() { + let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; // IDR slice, no inline SPS + let mut annexb = bytes::BytesMut::new(); + annexb.extend_from_slice(&[0, 0, 0, 1]); + annexb.extend_from_slice(idr); + + let mut broadcast = moq_net::BroadcastInfo::new().produce(); + let track = broadcast + .create_track( + "video", + moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), + ) + .unwrap(); + let mut import = Import::from_track(track); + + let err = import + .decode_frame(&mut annexb, Some(moq_net::Timestamp::from_micros(0).unwrap())) + .expect_err("an unconfigurable keyframe must error"); + assert!(matches!(err, crate::Error::H264(Error::NotInitialized)), "got {err:?}"); + } + + /// A non-keyframe before any config is a mid-stream-join leftover: it must + /// not abort the import (the producer's lenient start drops it downstream). + #[tokio::test(start_paused = true)] + async fn delta_before_init_is_tolerated() { + let pslice: &[u8] = &[0x61, 0xe0, 0x12, 0x34]; // non-IDR slice + let mut annexb = bytes::BytesMut::new(); + annexb.extend_from_slice(&[0, 0, 0, 1]); + annexb.extend_from_slice(pslice); + + let mut broadcast = moq_net::BroadcastInfo::new().produce(); + let track = broadcast + .create_track( + "video", + moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), + ) + .unwrap(); + let mut import = Import::from_track(track); + + import + .decode_frame(&mut annexb, Some(moq_net::Timestamp::from_micros(0).unwrap())) + .expect("a delta before init must be tolerated, not abort"); + assert!(import.catalog().is_none(), "no config yet, so no catalog"); + } +} diff --git a/rs/moq-mux/src/codec/h264/split.rs b/rs/moq-mux/src/codec/h264/split.rs index e827632b4..b2c4d2c81 100644 --- a/rs/moq-mux/src/codec/h264/split.rs +++ b/rs/moq-mux/src/codec/h264/split.rs @@ -1,64 +1,41 @@ -//! H.264 byte parser for both wire shapes. +//! H.264 Annex-B stream splitter. //! -//! [`Split`] turns H.264 bytes into [`crate::container::Frame`]s plus a resolved -//! [`hang::catalog::VideoConfig`]. It accepts either length-prefixed NALU input -//! with an out-of-band [`AVCDecoderConfigurationRecord`](super::Avcc) (the "avc1" -//! shape) or Annex-B input with inline SPS/PPS (the "avc3" shape). The shape is -//! detected at [`initialize`](Split::initialize) time by looking for a leading -//! start code; callers that already know it can also force the mode via -//! [`with_mode`](Split::with_mode). +//! [`Split`] turns a raw Annex-B byte stream (inline SPS/PPS, the "avc3" shape) +//! into [`crate::container::Frame`]s. It is deliberately dumb: it finds +//! access-unit boundaries, caches SPS/PPS and re-inserts them ahead of each +//! keyframe so every keyframe is self-contained, and stamps wall-clock +//! timestamps when the caller has none (stdin). It owns no track, catalog, or +//! codec config. The importer parses the codec config out of the frames it +//! emits. //! -//! Unlike a full importer, [`Split`] owns no track or catalog: it only parses, -//! emitting frames and surfacing the codec config via [`take_config`](Split::take_config). +//! There is no out-of-band initialization beyond optionally [seeding](Self::seed) +//! the parameter-set cache: a caller that can configure a decoder out of band +//! already knows frame boundaries, and would hand whole frames to the importer +//! rather than a byte stream. use bytes::{Buf, Bytes, BytesMut}; use tokio::io::{AsyncRead, AsyncReadExt}; -use super::{Error, Sps}; +use super::{Error, NAL_TYPE_PPS, NAL_TYPE_SPS}; use crate::Result; use crate::codec::annexb::{NalIterator, START_CODE}; -/// The wire shape a [`Split`] is processing. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum Mode { - /// Length-prefixed NALU with out-of-band AVCDecoderConfigurationRecord - /// (catalog `H264 { inline: false }`, `description = avcC`). - Avc1, - /// Annex-B (start-code prefixed) with inline SPS/PPS - /// (catalog `H264 { inline: true }`, no description). - Avc3, -} - -/// H.264 byte parser. Handles both avc1 (length-prefixed) and avc3 (Annex-B) -/// input streams; the shape is detected from the first bytes the caller -/// supplies, or forced explicitly via [`with_mode`](Self::with_mode). +/// H.264 Annex-B stream splitter: bytes in, [`Frame`](crate::container::Frame)s out. /// -/// Feed bytes via [`initialize`](Self::initialize), [`decode_frame`](Self::decode_frame), -/// [`decode_stream`](Self::decode_stream), or [`decode_from`](Self::decode_from); each -/// returns the [`Frame`](crate::container::Frame)s it produced. The resolved -/// [`hang::catalog::VideoConfig`] is exposed lazily once the codec config is known -/// (avcC for avc1, the first SPS for avc3) via [`take_config`](Self::take_config). +/// Feed bytes via [`decode_stream`](Self::decode_stream) (unknown frame +/// boundaries, e.g. stdin), [`decode_frame`](Self::decode_frame) (one complete +/// access unit per call), or [`decode_from`](Self::decode_from) (an async +/// reader). Each returns the frames it produced. SPS/PPS seen inline are cached +/// and re-inserted ahead of each keyframe; [`seed`](Self::seed) primes that +/// cache from an out-of-band parameter-set buffer. pub struct Split { - config: Option, - config_dirty: bool, - state: State, + current: Avc3Frame, + sps: Option, + pps: Option, zero: Option, pending: Vec, } -enum State { - /// No bytes seen yet; mode pinned ahead of time or unknown. - Pending { mode_hint: Option }, - /// avc1 wire shape: length-prefixed NALU, codec config out-of-band. - Avc1 { length_size: usize }, - /// avc3 wire shape: Annex-B NALU, inline SPS/PPS. - Avc3 { - current: Avc3Frame, - sps: Option, - pps: Option, - }, -} - #[derive(Default)] struct Avc3Frame { chunks: BytesMut, @@ -75,129 +52,40 @@ impl Default for Split { } impl Split { - /// Auto-detect the wire shape from the first bytes supplied to - /// [`initialize`](Self::initialize). + /// A fresh splitter with an empty parameter-set cache. pub fn new() -> Self { Self { - config: None, - config_dirty: false, - state: State::Pending { mode_hint: None }, - zero: None, - pending: Vec::new(), - } - } - - /// Pin the wire shape ahead of time; skips the leading-bytes auto-detect - /// inside [`initialize`](Self::initialize). - pub fn with_mode(mode: Mode) -> Result { - let state = match mode { - Mode::Avc1 => State::Pending { - mode_hint: Some(Mode::Avc1), - }, - Mode::Avc3 => State::Avc3 { - current: Avc3Frame::default(), - sps: None, - pps: None, - }, - }; - Ok(Self { - config: None, - config_dirty: false, - state, + current: Avc3Frame::default(), + sps: None, + pps: None, zero: None, pending: Vec::new(), - }) - } - - /// Initialize from the codec's leading bytes. - /// - /// - **avc1** (no leading start code): the buffer is parsed as an - /// `AVCDecoderConfigurationRecord` and stored as the config `description`. - /// - **avc3** (leading `0x00 0x00 0x01` or `0x00 0x00 0x00 0x01`): the buffer - /// is parsed as Annex-B NALs to seed the cached SPS/PPS. - /// - /// The buffer is fully consumed. - pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { - let mode = match &self.state { - State::Pending { mode_hint } => mode_hint.unwrap_or_else(|| detect_mode(buf.as_ref())), - State::Avc1 { .. } => Mode::Avc1, - State::Avc3 { .. } => Mode::Avc3, - }; - - match mode { - Mode::Avc1 => self.initialize_avc1(buf), - Mode::Avc3 => self.initialize_avc3(buf), } } - /// Initialize the avc1 path from an `AVCDecoderConfigurationRecord` buffer. - fn initialize_avc1>(&mut self, buf: &mut T) -> Result<()> { - let avcc_bytes = buf.as_ref(); - let avcc = super::Avcc::parse(avcc_bytes)?; - self.state = State::Avc1 { - length_size: avcc.length_size, - }; - - let mut config = hang::catalog::VideoConfig::new(hang::catalog::H264 { - profile: avcc.profile, - constraints: avcc.constraints, - level: avcc.level, - inline: false, - }); - config.coded_width = avcc.coded_width; - config.coded_height = avcc.coded_height; - config.description = Some(Bytes::copy_from_slice(avcc_bytes)); - config.container = hang::catalog::Container::Legacy; - - self.swap_config(config)?; - buf.advance(buf.remaining()); - - Ok(()) - } - - /// Initialize the avc3 path by parsing Annex-B NALs (SPS/PPS seed the - /// config once the first SPS is parsed). - fn initialize_avc3>(&mut self, buf: &mut T) -> Result<()> { - if !matches!(self.state, State::Avc3 { .. }) { - self.state = State::Avc3 { - current: Avc3Frame::default(), - sps: None, - pps: None, - }; - } - + /// Prime the SPS/PPS cache from an Annex-B parameter-set buffer, so the first + /// keyframe is self-contained even if the stream itself omits inline + /// parameter sets. Other NAL types in the buffer are ignored. + pub fn seed>(&mut self, buf: &mut T) -> Result<()> { let mut nals = NalIterator::new(buf); while let Some(nal) = nals.next().transpose()? { - self.decode_nal(nal, None)?; + self.cache_param(&nal); } if let Some(nal) = nals.flush()? { - self.decode_nal(nal, None)?; + self.cache_param(&nal); } - Ok(()) } - /// True once the codec config has been resolved. - pub fn is_initialized(&self) -> bool { - self.config.is_some() - } - - /// The resolved config if it changed since the last call, else `None`. - /// - /// Set whenever a new SPS/avcC is parsed; returns `Some` once per change. - pub fn take_config(&mut self) -> Option { - if self.config_dirty { - self.config_dirty = false; - self.config.clone() - } else { - None + fn cache_param(&mut self, nal: &Bytes) { + match nal.first().map(|h| h & 0x1f) { + Some(NAL_TYPE_SPS) => self.sps = Some(nal.clone()), + Some(NAL_TYPE_PPS) => self.pps = Some(nal.clone()), + _ => {} } } /// Decode from an asynchronous reader, returning all frames produced. - /// - /// avc3 only. For avc1, the caller already has framed buffers and uses - /// [`decode_frame`](Self::decode_frame). pub async fn decode_from(&mut self, reader: &mut T) -> Result> { let mut frames = Vec::new(); let mut buffer = BytesMut::new(); @@ -207,77 +95,44 @@ impl Split { Ok(frames) } - /// Decode a buffer where frame boundaries are unknown (avc3 streaming - /// input), returning the frames it produced. The leading start code of the - /// *next* frame is what signals the previous frame is done. + /// Decode a buffer where frame boundaries are unknown, returning the frames + /// it produced. The leading start code of the *next* access unit is what + /// signals the previous one is complete, so the final access unit stays + /// buffered until the next call (or [`decode_frame`](Self::decode_frame)). pub fn decode_stream>( &mut self, buf: &mut T, - pts: Option, + pts: impl Into>, ) -> Result> { - if !matches!(self.state, State::Avc3 { .. }) { - return Err(Error::StreamNotAvc3.into()); - } - let pts = self.pts(pts)?; + let pts = self.pts(pts.into())?; let nals = NalIterator::new(buf); for nal in nals { - self.decode_nal(nal?, Some(pts))?; + self.decode_nal(nal?, pts)?; } Ok(std::mem::take(&mut self.pending)) } - /// Decode a buffer assumed to hold (the rest of) a single frame, returning - /// the frames it produced. - /// - /// - avc1: the buffer is written as one length-prefixed-NALU frame. - /// - avc3: NALs are parsed; any trailing NAL without a start code is - /// flushed as the last NAL of this frame. + /// Decode a buffer holding one complete access unit, returning the frames it + /// produced (typically one). Any trailing NAL without a start code is the + /// last NAL of this access unit, and the unit is flushed before returning. pub fn decode_frame>( &mut self, buf: &mut T, - pts: Option, + pts: impl Into>, ) -> Result> { - match &self.state { - State::Avc1 { .. } => self.decode_avc1(buf, pts)?, - State::Avc3 { .. } => self.decode_avc3_frame(buf, pts)?, - State::Pending { .. } => return Err(Error::NotInitialized.into()), - } - Ok(std::mem::take(&mut self.pending)) - } - - fn decode_avc1>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - let State::Avc1 { length_size } = self.state else { - unreachable!("checked by decode_frame") - }; - let data = buf.as_ref(); - let pts = self.pts(pts)?; - let keyframe = avc1_is_keyframe(data, length_size); - - self.pending.push(crate::container::Frame { - timestamp: pts, - payload: data.to_vec().into(), - keyframe, - duration: None, - }); - - buf.advance(buf.remaining()); - Ok(()) - } - - fn decode_avc3_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - let pts = self.pts(pts)?; + let pts = self.pts(pts.into())?; let mut nals = NalIterator::new(buf); while let Some(nal) = nals.next().transpose()? { - self.decode_nal(nal, Some(pts))?; + self.decode_nal(nal, pts)?; } if let Some(nal) = nals.flush()? { - self.decode_nal(nal, Some(pts))?; + self.decode_nal(nal, pts)?; } - self.maybe_start_frame(Some(pts))?; - Ok(()) + self.maybe_start_frame(pts)?; + Ok(std::mem::take(&mut self.pending)) } - fn decode_nal(&mut self, nal: Bytes, pts: Option) -> Result<()> { + fn decode_nal(&mut self, nal: Bytes, pts: moq_net::Timestamp) -> Result<()> { let header = nal.first().ok_or(Error::NalTooShort)?; let forbidden_zero_bit = (header >> 7) & 1; if forbidden_zero_bit != 0 { @@ -290,55 +145,44 @@ impl Split { match nal_type { Some(Avc3NalType::Sps) => { self.maybe_start_frame(pts)?; - let sps = Sps::parse(&nal)?; - self.init_from_sps(&sps)?; - let State::Avc3 { current, sps, pps } = &mut self.state else { - unreachable!("decode_nal is avc3 only") - }; - if sps.as_ref().is_some_and(|cached| cached != &nal) { - // SPS changed mid-AU. The cached PPS is tied to the old SPS - // and may already have been appended to current.chunks - // earlier in this AU; reset the AU so the new SPS+PPS pair - // is the only parameter set we emit. - *pps = None; - current.chunks.clear(); - current.contains_pps = false; - current.contains_sps = false; + if self.sps.as_ref().is_some_and(|cached| cached != &nal) { + // SPS changed mid-stream. The cached PPS is tied to the old + // SPS and may already have been appended to current.chunks + // earlier in this AU; reset so the new SPS+PPS pair is the + // only parameter set we emit. + self.pps = None; + self.current.chunks.clear(); + self.current.contains_pps = false; + self.current.contains_sps = false; } - *sps = Some(nal.clone()); - current.contains_sps = true; + self.sps = Some(nal.clone()); + self.current.contains_sps = true; } Some(Avc3NalType::Pps) => { self.maybe_start_frame(pts)?; - let State::Avc3 { current, pps, .. } = &mut self.state else { - unreachable!() - }; - *pps = Some(nal.clone()); - current.contains_pps = true; + self.pps = Some(nal.clone()); + self.current.contains_pps = true; } Some(Avc3NalType::Aud) | Some(Avc3NalType::Sei) => { self.maybe_start_frame(pts)?; } Some(Avc3NalType::IdrSlice) => { - let State::Avc3 { current, sps, pps } = &mut self.state else { - unreachable!() - }; - if !current.contains_sps - && let Some(sps) = sps.as_ref() + if !self.current.contains_sps + && let Some(sps) = self.sps.clone() { - current.chunks.extend_from_slice(&START_CODE); - current.chunks.extend_from_slice(sps); - current.contains_sps = true; + self.current.chunks.extend_from_slice(&START_CODE); + self.current.chunks.extend_from_slice(&sps); + self.current.contains_sps = true; } - if !current.contains_pps - && let Some(pps) = pps.as_ref() + if !self.current.contains_pps + && let Some(pps) = self.pps.clone() { - current.chunks.extend_from_slice(&START_CODE); - current.chunks.extend_from_slice(pps); - current.contains_pps = true; + self.current.chunks.extend_from_slice(&START_CODE); + self.current.chunks.extend_from_slice(&pps); + self.current.contains_pps = true; } - current.contains_idr = true; - current.contains_slice = true; + self.current.contains_idr = true; + self.current.contains_slice = true; } Some(Avc3NalType::NonIdrSlice) | Some(Avc3NalType::DataPartitionA) @@ -347,62 +191,28 @@ impl Split { if nal.get(1).ok_or(Error::NalTooShort)? & 0x80 != 0 { self.maybe_start_frame(pts)?; } - let State::Avc3 { current, .. } = &mut self.state else { - unreachable!() - }; - current.contains_slice = true; + self.current.contains_slice = true; } _ => {} } tracing::trace!(kind = ?nal_type, "parsed NAL"); - let State::Avc3 { current, .. } = &mut self.state else { - unreachable!() - }; - current.chunks.extend_from_slice(&START_CODE); - current.chunks.extend_from_slice(&nal); - Ok(()) - } - - fn init_from_sps(&mut self, sps: &Sps) -> Result<()> { - let mut config = hang::catalog::VideoConfig::new(hang::catalog::H264 { - profile: sps.profile, - constraints: sps.constraints, - level: sps.level, - inline: true, - }); - config.coded_width = Some(sps.coded_width); - config.coded_height = Some(sps.coded_height); - config.container = hang::catalog::Container::Legacy; - - if let Some(old) = &self.config - && old == &config - { - return Ok(()); - } - - // avc3 carries SPS inline, so a resolution change updates the config in - // place (no new init segment, unlike avc1). - self.config = Some(config); - self.config_dirty = true; + self.current.chunks.extend_from_slice(&START_CODE); + self.current.chunks.extend_from_slice(&nal); Ok(()) } - fn maybe_start_frame(&mut self, pts: Option) -> Result<()> { - let State::Avc3 { current, .. } = &mut self.state else { - return Ok(()); - }; - if !current.contains_slice { + fn maybe_start_frame(&mut self, pts: moq_net::Timestamp) -> Result<()> { + if !self.current.contains_slice { return Ok(()); } - let pts = pts.ok_or(Error::MissingTimestamp)?; - let payload = std::mem::take(&mut current.chunks).freeze(); - let keyframe = current.contains_idr; - current.contains_idr = false; - current.contains_slice = false; - current.contains_sps = false; - current.contains_pps = false; + let payload = std::mem::take(&mut self.current.chunks).freeze(); + let keyframe = self.current.contains_idr; + self.current.contains_idr = false; + self.current.contains_slice = false; + self.current.contains_sps = false; + self.current.contains_pps = false; self.pending.push(crate::container::Frame { timestamp: pts, @@ -413,33 +223,13 @@ impl Split { Ok(()) } - /// Resolve the avc1 codec config. - /// - /// The first config is stored. A different avcC would mean a new init - /// segment, which a single fixed track can't represent, so a reconfiguration - /// is an error (mint a new track via a fresh parser). - fn swap_config(&mut self, config: hang::catalog::VideoConfig) -> Result<()> { - if let Some(old) = &self.config { - if old == &config { - return Ok(()); - } - return Err(Error::FixedTrackReconfigured.into()); - } - - tracing::debug!(?config, "starting H.264 track"); - self.config = Some(config); - self.config_dirty = true; - Ok(()) - } - - /// Drop any in-flight avc3 access unit. + /// Drop any in-flight access unit. /// /// Pre-reset NALs would otherwise leak into a later frame with the wrong - /// timestamp. + /// timestamp. The parameter-set cache is kept so subsequent keyframes stay + /// self-contained. pub fn reset(&mut self) { - if let State::Avc3 { current, .. } = &mut self.state { - *current = Avc3Frame::default(); - } + self.current = Avc3Frame::default(); } fn pts(&mut self, hint: Option) -> Result { @@ -451,41 +241,6 @@ impl Split { } } -/// Detect the wire shape from leading bytes: a 3- or 4-byte Annex-B start -/// code means avc3, otherwise an AVCDecoderConfigurationRecord (avc1). -fn detect_mode(bytes: &[u8]) -> Mode { - let three_byte = matches!(bytes, [0, 0, 1, ..]); - let four_byte = matches!(bytes, [0, 0, 0, 1, ..]); - if three_byte || four_byte { - Mode::Avc3 - } else { - Mode::Avc1 - } -} - -/// Detect if an avc1-shaped (length-prefixed) buffer contains an IDR slice. -fn avc1_is_keyframe(data: &[u8], length_size: usize) -> bool { - let mut offset = 0; - while offset + length_size <= data.len() { - let nal_len = match length_size { - 1 => data[offset] as usize, - 2 => u16::from_be_bytes([data[offset], data[offset + 1]]) as usize, - 3 => u32::from_be_bytes([0, data[offset], data[offset + 1], data[offset + 2]]) as usize, - 4 => u32::from_be_bytes([data[offset], data[offset + 1], data[offset + 2], data[offset + 3]]) as usize, - _ => return false, - }; - offset += length_size; - if offset + nal_len > data.len() { - break; - } - if nal_len > 0 && data[offset] & 0x1f == 5 { - return true; // IDR slice - } - offset += nal_len; - } - false -} - #[derive(Debug, Clone, Copy, PartialEq, Eq, num_enum::TryFromPrimitive)] #[repr(u8)] enum Avc3NalType { @@ -512,81 +267,56 @@ enum Avc3NalType { mod tests { use super::*; - #[test] - fn detect_mode_avc1_avcc_buffer() { - // AVCDecoderConfigurationRecord starts with configurationVersion = 1, profile, ... - // First byte is 0x01, definitely not a start code. - let avcc: &[u8] = &[ - 0x01, 0x42, 0xc0, 0x1f, 0xff, 0xe1, 0x00, 0x06, 0x67, 0x42, 0xc0, 0x1f, 0xde, 0xad, - ]; - assert_eq!(detect_mode(avcc), Mode::Avc1); - } - - #[test] - fn detect_mode_avc3_3byte_start_code() { - let nals: &[u8] = &[0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1f]; - assert_eq!(detect_mode(nals), Mode::Avc3); - } + const SC4: &[u8] = &[0, 0, 0, 1]; - #[test] - fn detect_mode_avc3_4byte_start_code() { - let nals: &[u8] = &[0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1f]; - assert_eq!(detect_mode(nals), Mode::Avc3); + fn annexb(nals: &[&[u8]]) -> BytesMut { + let mut buf = BytesMut::new(); + for nal in nals { + buf.extend_from_slice(SC4); + buf.extend_from_slice(nal); + } + buf } - /// Auto-detect routes an avcC initializer into the avc1 path and resolves a - /// config with the avcC stored as `description`. + /// A keyframe access unit fed as one buffer emits one self-contained frame: + /// SPS+PPS are packaged ahead of the IDR slice and `keyframe` is set. #[tokio::test(start_paused = true)] - async fn auto_detect_avc1_lands_in_catalog() { - // Minimal AVCDecoderConfigurationRecord: version(1) profile(0x42) compat(0xc0) level(0x1f) - // length_size_minus_one + 0xfc | 3 = 0xff - // reserved | num_sps = 0xe1 - // sps_len = 4, sps bytes (NAL header 0x67 + profile/level for parsing). - let sps_nal = [0x67, 0x42, 0xc0, 0x1f]; - let mut avcc = vec![0x01, 0x42, 0xc0, 0x1f, 0xff, 0xe1, 0x00, sps_nal.len() as u8]; - avcc.extend_from_slice(&sps_nal); - avcc.extend_from_slice(&[0x01, 0x00, 0x04, 0x68, 0xce, 0x3c, 0x80]); // num_pps + pps + async fn decode_frame_packages_keyframe() { + let sps: &[u8] = &[0x67, 0x42, 0xc0, 0x1f]; + let pps: &[u8] = &[0x68, 0xce, 0x3c, 0x80]; + let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; let mut split = Split::new(); - let mut buf = bytes::BytesMut::from(avcc.as_slice()); - split.initialize(&mut buf).expect("initialize avc1"); - - let cfg = split.take_config().expect("config known after init"); - let hang::catalog::VideoCodec::H264(h264) = &cfg.codec else { - panic!("expected H.264 codec") - }; - assert!(!h264.inline, "avc1 source should land as inline=false"); - assert_eq!(h264.profile, 0x42); - assert_eq!(h264.level, 0x1f); - let desc = cfg.description.as_ref().expect("description set"); - assert_eq!(desc.as_ref(), avcc.as_slice()); + let mut buf = annexb(&[sps, pps, idr]); + let frames = split + .decode_frame(&mut buf, moq_net::Timestamp::from_micros(0).unwrap()) + .unwrap(); + + assert_eq!(frames.len(), 1); + assert!(frames[0].keyframe); + // The payload carries SPS, PPS, then the IDR slice (each start-code prefixed). + assert_eq!(&frames[0].payload[..SC4.len()], SC4); + assert!(frames[0].payload.windows(sps.len()).any(|w| w == sps)); + assert!(frames[0].payload.windows(idr.len()).any(|w| w == idr)); } - /// Auto-detect routes an Annex-B initializer into the avc3 path; the - /// resolved config reports inline=true and no description. + /// A seeded splitter re-inserts the cached SPS/PPS ahead of a bare IDR slice, + /// even though the stream itself never carried inline parameter sets. #[tokio::test(start_paused = true)] - async fn auto_detect_avc3_lands_in_catalog() { - let sps: &[u8] = &[ - 0x67, 0x42, 0xc0, 0x1f, 0xda, 0x01, 0x40, 0x16, 0xe9, 0xb8, 0x08, 0x08, 0x0a, 0x00, 0x00, 0x07, 0xd0, 0x00, - 0x01, 0xd4, 0xc0, 0x80, - ]; + async fn seed_makes_bare_keyframe_self_contained() { + let sps: &[u8] = &[0x67, 0x42, 0xc0, 0x1f]; let pps: &[u8] = &[0x68, 0xce, 0x3c, 0x80]; - let mut annexb = bytes::BytesMut::new(); - annexb.extend_from_slice(&[0, 0, 0, 1]); - annexb.extend_from_slice(sps); - annexb.extend_from_slice(&[0, 0, 0, 1]); - annexb.extend_from_slice(pps); + let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; let mut split = Split::new(); - split.initialize(&mut annexb).expect("initialize avc3"); - - let cfg = split.take_config().expect("config known after first SPS"); - let hang::catalog::VideoCodec::H264(h264) = &cfg.codec else { - panic!("expected H.264 codec") - }; - assert!(h264.inline, "avc3 source should land as inline=true"); - assert!(cfg.description.is_none(), "avc3 has no out-of-band description"); - assert_eq!(h264.profile, sps[1]); - assert_eq!(h264.level, sps[3]); + split.seed(&mut annexb(&[sps, pps])).unwrap(); + + let frames = split + .decode_frame(&mut annexb(&[idr]), moq_net::Timestamp::from_micros(0).unwrap()) + .unwrap(); + assert_eq!(frames.len(), 1); + assert!(frames[0].keyframe); + assert!(frames[0].payload.windows(sps.len()).any(|w| w == sps)); + assert!(frames[0].payload.windows(pps.len()).any(|w| w == pps)); } } From ba7a40624950cbefc9342dcbbc6749f01a085d29 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 18:07:54 +0000 Subject: [PATCH 07/19] moq-mux: Published owns decode+sync; drop manual sync calls The lazy-catalog codecs all did `decode_frame(...)?; sync();` by hand, an easy-to-forget two-step. Make `Published` own the pairing so the catalog re-mirror can't be skipped. - New `Published::decoding(|inner| ...)`: runs a decode/edit on the inner importer and re-mirrors the catalog in one call. Generic over the closure's error so it wraps both the `crate::Result` and `anyhow::Result` importers. Pairs with the existing `Published::decode(frames)` (frames in hand) as the byte-path equivalent. - Convert every `decode + sync` site to `decoding`: the Framed and Stream dispatchers, the TS H.264/H.265 streams, moq-rtc, moq-video, and moq-cli. - TS AAC: set the rendition `description` on the importer before `Published::new` (its attach-time mirror covers it) and route the jitter refinement through `decoding`. - `Published::sync` is now private: the only way to decode is through a path that syncs, so the footgun is gone. https://claude.ai/code/session_011S7pzcg2XsPP3AExuymac1 --- rs/moq-cli/src/publish.rs | 3 +- rs/moq-mux/src/container/ts/import.rs | 24 +++++------ rs/moq-mux/src/import.rs | 58 +++++---------------------- rs/moq-mux/src/publish.rs | 52 +++++++++++++++++------- rs/moq-rtc/src/codec/h264.rs | 3 +- rs/moq-video/src/encode/producer.rs | 10 +++-- 6 files changed, 69 insertions(+), 81 deletions(-) diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index a9998c317..860e229a3 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -89,8 +89,7 @@ impl PublishDecoder { fn decode_buf(&mut self, buffer: &mut bytes::BytesMut) -> anyhow::Result<()> { match self { Self::Avc3(d) => { - d.decode_stream(buffer, None)?; - d.sync(); + d.decoding(|d| d.decode_stream(buffer, None))?; Ok(()) } Self::Fmp4(d) => Ok(d.decode(buffer)?), diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index e4d97d77f..48b231d4b 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -739,14 +739,12 @@ impl Stream { match self { Stream::H264 { import, unwrap } => { let pts = unwrap_pts(unwrap, pending.pts)?; - import.decode_frame(&mut pending.data.as_slice(), pts)?; - import.sync(); + import.decoding(|i| i.decode_frame(&mut pending.data.as_slice(), pts))?; Ok(()) } Stream::H265 { import, unwrap } => { let pts = unwrap_pts(unwrap, pending.pts)?; - import.decode_frame(&mut pending.data.as_slice(), pts)?; - import.sync(); + import.decoding(|i| i.decode_frame(&mut pending.data.as_slice(), pts))?; Ok(()) } Stream::Aac(stream) => stream.write(pending, burst), @@ -816,12 +814,12 @@ impl AacStream { // WebCodecs) can configure the decoder. TS itself carries it inline. let description = config.encode(); let track = crate::publish::unique_track(&mut self.broadcast, ".aac")?; - let aac = aac::Import::from_track(track, config)?; - let mut import = crate::publish::Published::new(self.catalog.clone(), aac); - if let Some(rendition) = import.rendition_mut() { + let mut aac = aac::Import::from_track(track, config)?; + if let Some(rendition) = aac.rendition_mut() { rendition.description = Some(description); } - import.sync(); + // Published::new mirrors the rendition (description included) on attach. + let import = crate::publish::Published::new(self.catalog.clone(), aac); self.import.insert(import) } }; @@ -881,10 +879,12 @@ impl AacStream { self.jitter = Some(jitter); if let Some(import) = &mut self.import { - if let Some(rendition) = import.rendition_mut() { - rendition.jitter = Some(jitter.into()); - } - import.sync(); + import.decoding(|i| { + if let Some(rendition) = i.rendition_mut() { + rendition.jitter = Some(jitter.into()); + } + crate::Result::Ok(()) + })?; } Ok(()) } diff --git a/rs/moq-mux/src/import.rs b/rs/moq-mux/src/import.rs index db95cb8ca..722b05b8e 100644 --- a/rs/moq-mux/src/import.rs +++ b/rs/moq-mux/src/import.rs @@ -318,27 +318,12 @@ impl Framed { /// Decode a frame from the given buffer. pub fn decode_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { match self.decoder { - FramedKind::H264(ref mut decoder) => { - decoder.decode_frame(buf, pts)?; - decoder.sync(); - } + FramedKind::H264(ref mut decoder) => decoder.decoding(|d| d.decode_frame(buf, pts))?, FramedKind::Fmp4(ref mut decoder) => decoder.decode(buf)?, - FramedKind::Hev1(ref mut decoder) => { - decoder.decode_frame(buf, pts)?; - decoder.sync(); - } - FramedKind::Av01(ref mut decoder) => { - decoder.decode_frame(buf, pts)?; - decoder.sync(); - } - FramedKind::Vp8(ref mut decoder) => { - decoder.decode_frame(buf, pts)?; - decoder.sync(); - } - FramedKind::Vp9(ref mut decoder) => { - decoder.decode_frame(buf, pts)?; - decoder.sync(); - } + FramedKind::Hev1(ref mut decoder) => decoder.decoding(|d| d.decode_frame(buf, pts))?, + FramedKind::Av01(ref mut decoder) => decoder.decoding(|d| d.decode_frame(buf, pts))?, + FramedKind::Vp8(ref mut decoder) => decoder.decoding(|d| d.decode_frame(buf, pts))?, + FramedKind::Vp9(ref mut decoder) => decoder.decoding(|d| d.decode_frame(buf, pts))?, FramedKind::Aac(ref mut decoder) => decoder.decode(buf, pts)?, FramedKind::Opus(ref mut decoder) => decoder.decode_buf(buf, pts)?, FramedKind::Mkv(ref mut decoder) => { @@ -693,19 +678,10 @@ impl Stream { /// The buffer will be fully consumed, or an error will be returned. pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { match self.decoder { - StreamKind::Avc3(ref mut decoder) => { - decoder.initialize(buf)?; - decoder.sync(); - } + StreamKind::Avc3(ref mut decoder) => decoder.decoding(|d| d.initialize(buf))?, StreamKind::Fmp4(ref mut decoder) => decoder.decode(buf)?, - StreamKind::Hev1(ref mut decoder) => { - decoder.initialize(buf)?; - decoder.sync(); - } - StreamKind::Av01(ref mut decoder) => { - decoder.initialize(buf)?; - decoder.sync(); - } + StreamKind::Hev1(ref mut decoder) => decoder.decoding(|d| d.initialize(buf))?, + StreamKind::Av01(ref mut decoder) => decoder.decoding(|d| d.initialize(buf))?, StreamKind::Mkv(ref mut decoder) => decoder.decode(buf)?, StreamKind::Ts(ref mut decoder) => decoder.decode(buf)?, } @@ -720,22 +696,10 @@ impl Stream { /// Decode a stream of data from the given buffer. pub fn decode_stream>(&mut self, buf: &mut T) -> Result<()> { match self.decoder { - StreamKind::Avc3(ref mut decoder) => { - decoder.decode_stream(buf, None)?; - decoder.sync(); - Ok(()) - } + StreamKind::Avc3(ref mut decoder) => decoder.decoding(|d| d.decode_stream(buf, None)), StreamKind::Fmp4(ref mut decoder) => decoder.decode(buf), - StreamKind::Hev1(ref mut decoder) => { - decoder.decode_stream(buf, None)?; - decoder.sync(); - Ok(()) - } - StreamKind::Av01(ref mut decoder) => { - decoder.decode_stream(buf, None)?; - decoder.sync(); - Ok(()) - } + StreamKind::Hev1(ref mut decoder) => decoder.decoding(|d| d.decode_stream(buf, None)), + StreamKind::Av01(ref mut decoder) => decoder.decoding(|d| d.decode_stream(buf, None)), StreamKind::Mkv(ref mut decoder) => decoder.decode(buf), StreamKind::Ts(ref mut decoder) => decoder.decode(buf).map_err(Into::into), } diff --git a/rs/moq-mux/src/publish.rs b/rs/moq-mux/src/publish.rs index 32fe519c1..813a2115c 100644 --- a/rs/moq-mux/src/publish.rs +++ b/rs/moq-mux/src/publish.rs @@ -12,9 +12,9 @@ //! is instead the on-demand path, fed directly to the importer's `new`. //! //! Some importers fill their catalog lazily (H.264 only knows its config once SPS -//! arrives) or refine it over time (jitter). Call [`Published::sync`] after -//! feeding such an importer so the new/changed renditions reach the catalog; it's -//! a cheap comparison when nothing changed. +//! arrives) or refine it over time (jitter). Feed them through +//! [`Published::decode`] or [`Published::decoding`], which re-mirror the catalog +//! automatically so new/changed renditions always reach it. use std::ops::{Deref, DerefMut}; @@ -81,10 +81,10 @@ impl Published { /// Re-mirror the importer's current renditions into the catalog. /// - /// Call this after feeding an importer whose catalog appears or changes - /// lazily (H.264 once SPS is parsed, jitter refinement, ...). It's a cheap - /// comparison and touches the catalog only when something actually changed. - pub fn sync(&mut self) { + /// Runs after each decode via [`decode`](Self::decode) / [`decoding`](Self::decoding), + /// so callers never invoke it directly. A cheap comparison that touches the + /// catalog only when a rendition actually appeared, changed, or was dropped. + fn sync(&mut self) { let current = self.inner.renditions(); if self.published == *current { return; @@ -116,13 +116,29 @@ impl Published { self.published = current.clone(); } + + /// Run a decode on the inner importer, then re-mirror the catalog. + /// + /// The footgun-free wrapper for the byte-decode entry points (the importer's + /// `decode_frame` / `decode_stream` / `initialize`): it re-mirrors the catalog + /// after the closure returns, so a lazily-resolved config or refined jitter + /// always reaches it. Prefer [`decode`](Self::decode) where the caller already + /// has split frames. + pub fn decoding(&mut self, decode: impl FnOnce(&mut I) -> Result) -> crate::Result + where + crate::Error: From, + { + let out = decode(&mut self.inner)?; + self.sync(); + Ok(out) + } } impl Published { /// Publish frames and re-mirror any catalog change in one call. /// - /// This is the footgun-free path: it [`sync`](Self::sync)s after decoding, so - /// a lazily-resolved config or refined jitter always reaches the catalog. + /// This is the footgun-free path: it re-mirrors the catalog after decoding, so + /// a lazily-resolved config or refined jitter always reaches it. pub fn decode>(&mut self, frames: It) -> crate::Result<()> { self.inner.decode(frames)?; self.sync(); @@ -196,14 +212,22 @@ mod tests { let mut published = Published::new(catalog.clone(), Fake(hang::Catalog::default())); assert!(catalog.snapshot().video.renditions.is_empty()); - // A rendition appears later; sync mirrors it into the broadcast catalog. - published.0.video.renditions.insert("v".to_string(), video()); - published.sync(); + // A rendition appears later; decoding mirrors it into the broadcast catalog. + published + .decoding(|i| { + i.0.video.renditions.insert("v".to_string(), video()); + crate::Result::Ok(()) + }) + .unwrap(); assert!(catalog.snapshot().video.renditions.contains_key("v")); // An update to the same rendition propagates too. - published.0.video.renditions.get_mut("v").unwrap().bitrate = Some(1_000); - published.sync(); + published + .decoding(|i| { + i.0.video.renditions.get_mut("v").unwrap().bitrate = Some(1_000); + crate::Result::Ok(()) + }) + .unwrap(); assert_eq!(catalog.snapshot().video.renditions["v"].bitrate, Some(1_000)); // Dropping the wrapper retires the rendition. diff --git a/rs/moq-rtc/src/codec/h264.rs b/rs/moq-rtc/src/codec/h264.rs index b0111d287..33cc5ec65 100644 --- a/rs/moq-rtc/src/codec/h264.rs +++ b/rs/moq-rtc/src/codec/h264.rs @@ -27,8 +27,7 @@ impl codec::Bridge for Bridge { let pts = moq_net::Timestamp::from_micros(frame.timestamp_us) .map_err(|err| crate::Error::Other(anyhow::anyhow!("invalid timestamp: {err}")))?; let mut buf = BytesMut::from(frame.payload.as_ref()); - self.import.decode_frame(&mut buf, Some(pts))?; - self.import.sync(); + self.import.decoding(|i| i.decode_frame(&mut buf, Some(pts)))?; Ok(()) } } diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index d1d0baffb..050a746c0 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -45,10 +45,12 @@ impl Producer { /// Publish already-encoded Annex-B packets at the given timestamp. pub fn publish(&mut self, packets: Vec, timestamp: Timestamp) -> Result<(), Error> { - for mut packet in packets { - self.import.decode_frame(&mut packet, Some(timestamp))?; - } - self.import.sync(); + self.import.decoding(|i| { + for mut packet in packets { + i.decode_frame(&mut packet, Some(timestamp))?; + } + Ok::<(), moq_mux::Error>(()) + })?; Ok(()) } From 7f10a4e288348263511bb0dfbb4989e24f9b4c5c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 18:26:58 +0000 Subject: [PATCH 08/19] moq-mux: replace lenient_start with a MissingKeyframe error `Producer::with_lenient_start` silently dropped non-keyframes that arrived before the first keyframe. Replace that implicit drop with an explicit `MissingKeyframe` error the producer returns, and let the one caller that wants to tolerate a mid-stream join (MPEG-TS) skip it. - New `container::MissingKeyframe` error: `Producer::write` returns it when a non-keyframe arrives with no open group (was a silent drop under lenient_start, or a generic ProtocolViolation without it). Wired into `crate::Error` and `fmp4::Error` via the `Container::Error` bound. - Drop `with_lenient_start` entirely. - Importers now surface MissingKeyframe for a pre-keyframe delta: h264 and h265 (and av1) write the delta through to the producer instead of pre-empting with a config error, erroring early only on an *unconfigurable keyframe* (NotInitialized / MissingSps / MissingSequenceHeader). vp8/vp9 already wrote straight through. - The TS importer wraps its H.264/H.265 decode in `skip_missing_keyframe`, so a capture that joins mid-GOP drops the leading deltas and resumes at the first keyframe (preserves survives_midstream_join + the dirty TS joins). https://claude.ai/code/session_011S7pzcg2XsPP3AExuymac1 --- rs/moq-mux/src/codec/av1/import.rs | 18 ++++++++++----- rs/moq-mux/src/codec/h264/import.rs | 11 +++++---- rs/moq-mux/src/codec/h265/import.rs | 28 ++++++++++++++--------- rs/moq-mux/src/container/fmp4/mod.rs | 3 +++ rs/moq-mux/src/container/mod.rs | 15 +++++++++++-- rs/moq-mux/src/container/producer.rs | 32 ++++++--------------------- rs/moq-mux/src/container/ts/import.rs | 16 ++++++++++---- rs/moq-mux/src/error.rs | 5 +++++ 8 files changed, 76 insertions(+), 52 deletions(-) diff --git a/rs/moq-mux/src/codec/av1/import.rs b/rs/moq-mux/src/codec/av1/import.rs index af6029b48..85884b8ef 100644 --- a/rs/moq-mux/src/codec/av1/import.rs +++ b/rs/moq-mux/src/codec/av1/import.rs @@ -342,20 +342,29 @@ impl Import { return Ok(()); } - if self.config.is_none() { + let pts = pts.ok_or(Error::MissingTimestamp)?; + let keyframe = self.current.contains_keyframe; + + // A keyframe we couldn't configure (no sequence header) is undecodable. + if keyframe && self.config.is_none() { return Err(Error::MissingSequenceHeader.into()); } - let pts = pts.ok_or(Error::MissingTimestamp)?; + // Take the payload and clear the per-frame state up front, so a + // MissingKeyframe from a pre-keyframe delta leaves a clean slate. let payload = std::mem::take(&mut self.current.chunks).freeze(); + self.current.contains_keyframe = false; + self.current.contains_frame = false; let frame = crate::container::Frame { timestamp: pts, payload, - keyframe: self.current.contains_keyframe, + keyframe, duration: None, }; + // A pre-keyframe delta has no group to anchor it: the producer returns + // MissingKeyframe, which a caller joining mid-stream skips. self.track.write(frame)?; if let Some(jitter) = self.jitter.observe(pts) @@ -364,9 +373,6 @@ impl Import { c.jitter = Some(jitter); } - self.current.contains_keyframe = false; - self.current.contains_frame = false; - Ok(()) } diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index 289464315..50d9ff127 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -76,7 +76,7 @@ impl Import { Self { shape: Shape::Pending { hint: None }, split: Split::new(), - track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy).with_lenient_start(), + track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), catalog: hang::Catalog::default(), config: None, last_sps: None, @@ -543,8 +543,10 @@ mod tests { /// A non-keyframe before any config is a mid-stream-join leftover: it must /// not abort the import (the producer's lenient start drops it downstream). + /// A non-keyframe before the first keyframe has no group to anchor it, so the + /// producer surfaces MissingKeyframe (which a mid-stream join skips). #[tokio::test(start_paused = true)] - async fn delta_before_init_is_tolerated() { + async fn delta_before_init_reports_missing_keyframe() { let pslice: &[u8] = &[0x61, 0xe0, 0x12, 0x34]; // non-IDR slice let mut annexb = bytes::BytesMut::new(); annexb.extend_from_slice(&[0, 0, 0, 1]); @@ -559,9 +561,10 @@ mod tests { .unwrap(); let mut import = Import::from_track(track); - import + let err = import .decode_frame(&mut annexb, Some(moq_net::Timestamp::from_micros(0).unwrap())) - .expect("a delta before init must be tolerated, not abort"); + .expect_err("a delta before any keyframe must report MissingKeyframe"); + assert!(matches!(err, crate::Error::MissingKeyframe(_)), "got {err:?}"); assert!(import.catalog().is_none(), "no config yet, so no catalog"); } } diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index 943ddffa8..e7f11f095 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -49,7 +49,7 @@ impl Import { /// Publish on an existing track producer. pub fn from_track(track: moq_net::TrackProducer) -> Self { Self { - track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy).with_lenient_start(), + track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), catalog: hang::Catalog::default(), config: None, current: Default::default(), @@ -293,21 +293,33 @@ impl Import { return Ok(()); } - // A slice before the first SPS has no catalog config to anchor it. - if self.config.is_none() { + let pts = pts.ok_or(Error::MissingTimestamp)?; + let keyframe = self.current.contains_idr; + + // A keyframe we couldn't configure (no SPS) is undecodable. + if keyframe && self.config.is_none() { return Err(Error::MissingSps.into()); } - let pts = pts.ok_or(Error::MissingTimestamp)?; + // Take the payload and clear the per-AU state up front, so a + // MissingKeyframe from a pre-keyframe delta (a mid-stream join) leaves a + // clean slate for the next access unit. let payload = std::mem::take(&mut self.current.chunks).freeze(); + self.current.contains_idr = false; + self.current.contains_slice = false; + self.current.contains_vps = false; + self.current.contains_sps = false; + self.current.contains_pps = false; let frame = crate::container::Frame { timestamp: pts, payload, - keyframe: self.current.contains_idr, + keyframe, duration: None, }; + // A pre-keyframe delta has no group to anchor it: the producer returns + // MissingKeyframe, which the caller (e.g. a TS mid-stream join) skips. self.track.write(frame)?; if let Some(jitter) = self.jitter.observe(pts) @@ -316,12 +328,6 @@ impl Import { c.jitter = Some(jitter); } - self.current.contains_idr = false; - self.current.contains_slice = false; - self.current.contains_vps = false; - self.current.contains_sps = false; - self.current.contains_pps = false; - Ok(()) } diff --git a/rs/moq-mux/src/container/fmp4/mod.rs b/rs/moq-mux/src/container/fmp4/mod.rs index a3e2281d4..3f4fc4623 100644 --- a/rs/moq-mux/src/container/fmp4/mod.rs +++ b/rs/moq-mux/src/container/fmp4/mod.rs @@ -36,6 +36,9 @@ pub enum Error { #[error("moq: {0}")] Moq(#[from] moq_net::Error), + #[error("missing keyframe: a group must open on a keyframe")] + MissingKeyframe(#[from] crate::container::MissingKeyframe), + #[error("timestamp overflow")] TimestampOverflow(#[from] moq_net::TimeOverflow), diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index 25b642a3b..7ee251170 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -69,6 +69,16 @@ pub struct Frame { pub keyframe: bool, } +/// A non-keyframe frame arrived with no open group. +/// +/// A track must open with a keyframe (and so must the frame after +/// [`finish_group`](Producer::finish_group) / [`seek`](Producer::seek)). +/// [`Producer::write`] returns this so a caller joining mid-stream can skip +/// frames until the first keyframe instead of treating it as fatal. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("missing keyframe: a group must open on a keyframe")] +pub struct MissingKeyframe; + /// Encode and decode media frames over a moq-lite group. /// /// Implementors decide how many [`Frame`]s map onto one moq-lite frame: @@ -76,8 +86,9 @@ pub struct Frame { /// pack many samples into a single moof+mdat fragment. pub trait Container { /// Container-specific error. Must be convertible from [`moq_net::Error`] - /// so the IO layer's errors propagate cleanly. - type Error: std::error::Error + Send + Sync + Unpin + From; + /// (so IO errors propagate) and [`MissingKeyframe`] (so the producer can + /// reject a group that doesn't open on a keyframe). + type Error: std::error::Error + Send + Sync + Unpin + From + From; /// Encode one or more frames into a single moq-lite frame appended to `group`. fn write(&self, group: &mut moq_net::GroupProducer, frames: &[Frame]) -> Result<(), Self::Error>; diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index 411832b27..239654a49 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -37,12 +37,6 @@ pub struct Producer { /// Sequence to use for the next group opened by [`Self::write`]. /// Set by [`Self::seek`] and consumed on the next group creation. pending_sequence: Option, - - /// When set, a non-keyframe arriving with no open group is dropped instead of - /// erroring. Lets a track join mid-stream (the leading deltas before the first - /// keyframe have no group to anchor) without a protocol violation. Off by - /// default so a producer that simply never marks keyframes still fails loudly. - lenient_start: bool, } impl Producer { @@ -55,18 +49,9 @@ impl Producer { buffer: Vec::new(), latency: std::time::Duration::ZERO, pending_sequence: None, - lenient_start: false, } } - /// Tolerate joining a stream mid-flight: drop non-keyframes that arrive before - /// the first keyframe (which has no group to anchor) instead of erroring. Use - /// for live ingest, where the input may start mid-GOP. - pub fn with_lenient_start(mut self) -> Self { - self.lenient_start = true; - self - } - /// The underlying moq-lite track producer. Read-only; mutating it directly /// would sidestep group/keyframe invariants. pub fn track(&self) -> &moq_net::TrackProducer { @@ -88,8 +73,8 @@ impl Producer { /// Write a frame to the track. /// /// A keyframe closes any open group and starts a new one. A non-keyframe extends - /// the current group; if no group is open it's a protocol violation, unless - /// [`with_lenient_start`](Self::with_lenient_start) was set (then it's dropped). + /// the current group; if no group is open it returns [`MissingKeyframe`](super::MissingKeyframe), + /// so a caller joining mid-stream can skip frames until the first keyframe. pub fn write(&mut self, frame: Frame) -> Result<(), C::Error> { // Close the current group on an explicit keyframe, passing its timestamp so // the previous group's last frame can borrow it as a duration boundary. @@ -100,12 +85,9 @@ impl Producer { // Start a new group if needed; the first frame of a group must be a keyframe. if self.group.is_none() { if !frame.keyframe { - // Mid-stream join: no group yet and this delta can't anchor one. Drop it - // (wait for the first keyframe) when lenient; otherwise it's a violation. - if self.lenient_start { - return Ok(()); - } - return Err(moq_net::Error::ProtocolViolation.into()); + // No group yet and this delta can't anchor one. The caller (e.g. a + // mid-stream join) decides whether to skip until the first keyframe. + return Err(super::MissingKeyframe.into()); } self.group = Some(match self.pending_sequence.take() { Some(sequence) => self.inner.create_group(moq_net::Group { sequence })?, @@ -286,7 +268,7 @@ mod tests { assert_eq!(collect_groups(consumer).await, vec![2, 1]); } - /// Writing a non-keyframe with no open group is a protocol violation. + /// Writing a non-keyframe with no open group returns MissingKeyframe. #[test] fn first_frame_must_be_keyframe() { let track = moq_net::TrackProducer::new( @@ -296,7 +278,7 @@ mod tests { let mut producer = Producer::new(track, Container::Legacy); let err = producer.write(frame(0, false)).unwrap_err(); - assert!(matches!(err, crate::Error::Moq(moq_net::Error::ProtocolViolation))); + assert!(matches!(err, crate::Error::MissingKeyframe(_))); } /// Drain all groups from a finished track, returning their sequence numbers. diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 48b231d4b..51a7bc6c9 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -739,13 +739,11 @@ impl Stream { match self { Stream::H264 { import, unwrap } => { let pts = unwrap_pts(unwrap, pending.pts)?; - import.decoding(|i| i.decode_frame(&mut pending.data.as_slice(), pts))?; - Ok(()) + skip_missing_keyframe(import.decoding(|i| i.decode_frame(&mut pending.data.as_slice(), pts))) } Stream::H265 { import, unwrap } => { let pts = unwrap_pts(unwrap, pending.pts)?; - import.decoding(|i| i.decode_frame(&mut pending.data.as_slice(), pts))?; - Ok(()) + skip_missing_keyframe(import.decoding(|i| i.decode_frame(&mut pending.data.as_slice(), pts))) } Stream::Aac(stream) => stream.write(pending, burst), Stream::Legacy(stream) => stream.write(pending), @@ -1046,6 +1044,16 @@ fn pes_data_len(header: &PesHeader, pes_packet_len: u16) -> Option { /// Convert a raw 90 kHz PTS to a microsecond [`Timestamp`], unwrapping the /// 33-bit field. Returns `None` when the PES carried no PTS (the codec layer /// then falls back to a wall-clock timestamp). +/// Swallow a [`MissingKeyframe`](crate::container::MissingKeyframe) from a video +/// decode: a TS capture can join mid-GOP, so the deltas before the first keyframe +/// have no group to anchor and are simply dropped rather than aborting the demux. +fn skip_missing_keyframe(result: crate::Result<()>) -> anyhow::Result<()> { + match result { + Ok(()) | Err(crate::Error::MissingKeyframe(_)) => Ok(()), + Err(e) => Err(e.into()), + } +} + fn unwrap_pts(unwrap: &mut PtsUnwrap, pts: Option) -> anyhow::Result> { let Some(raw) = pts else { return Ok(None); diff --git a/rs/moq-mux/src/error.rs b/rs/moq-mux/src/error.rs index 08a2324db..ba9e31dd2 100644 --- a/rs/moq-mux/src/error.rs +++ b/rs/moq-mux/src/error.rs @@ -87,6 +87,11 @@ pub enum Error { #[error("{0} can contain multiple tracks")] MultipleTracks(&'static str), + /// A non-keyframe frame was received before any keyframe opened a group. + /// A track joining mid-stream should skip frames until the first keyframe. + #[error("{0}")] + MissingKeyframe(#[from] crate::container::MissingKeyframe), + /// Error from a muxer/demuxer that reports via `anyhow` (currently MPEG-TS). /// Boxed in an `Arc` so the enum stays `Clone` (`anyhow::Error` is not). #[error("{0}")] From f6318218fb4537177e7c3ec27488cd9b7c75ecea Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 18:33:35 +0000 Subject: [PATCH 09/19] moq-mux: extract the H.265 byte->frame Split Mirror the h264 split: separate the Annex-B parsing from the publisher. - New `codec::h265::Split`: a dumb Annex-B stream assembler that finds access-unit boundaries, caches VPS/SPS/PPS and re-inserts them ahead of each keyframe, and stamps wall-clock timestamps for stdin. No track, catalog, or codec config. `decode_stream` / `decode_frame` / `decode_from` / `seed` / `reset`, like h264. - `codec::h265::Import` now drives the splitter and owns the config: it scans the SPS the splitter packages into the first keyframe (or a seed buffer via `initialize`) to fill the catalog, errors on an unconfigurable keyframe, and writes pre-keyframe deltas through to the producer (which reports MissingKeyframe for a mid-stream join). It also implements `FrameDecode` so a caller with its own splitter can publish frames. - Adds the first H.265 unit tests (the splitter packaging path). https://claude.ai/code/session_011S7pzcg2XsPP3AExuymac1 --- rs/moq-mux/src/codec/h265/import.rs | 436 ++++++++++------------------ rs/moq-mux/src/codec/h265/mod.rs | 2 + rs/moq-mux/src/codec/h265/split.rs | 345 ++++++++++++++++++++++ 3 files changed, 497 insertions(+), 286 deletions(-) create mode 100644 rs/moq-mux/src/codec/h265/split.rs diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index e7f11f095..8b7967270 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -1,41 +1,40 @@ -use crate::codec::annexb::{NalIterator, START_CODE}; -use crate::container::jitter::MinFrameDuration; -use crate::publish::Renditions; +//! H.265 importer. +//! +//! Publishes H.265 frames (Annex-B, inline VPS/SPS/PPS, the "hev1" shape) on a +//! single moq track and resolves the catalog rendition. Only single-layer +//! streams are supported (VPS is cached but not parsed). +//! +//! The codec config is scanned out of the SPS the splitter packages into the +//! first keyframe (or seeded via [`initialize`](Import::initialize)). A keyframe +//! that can't be configured is an error; non-keyframes before the first config +//! are written through to the producer, which reports +//! [`MissingKeyframe`](crate::container::MissingKeyframe) for a mid-stream join. +//! Annex-B byte parsing lives in [`Split`]; this type drives it and adds the +//! catalog. use bytes::{Buf, Bytes, BytesMut}; -use scuffle_h265::{NALUnitType, SpsNALUnit}; +use scuffle_h265::SpsNALUnit; +use tokio::io::{AsyncRead, AsyncReadExt}; -use super::Error; +use super::{Error, Split, split::nal_unit_type}; use crate::Result; +use crate::codec::annexb::NalIterator; +use crate::container::Frame; +use crate::container::jitter::MinFrameDuration; +use crate::publish::{FrameDecode, Renditions}; -/// A decoder for H.265 with inline SPS/PPS. +/// A decoder for H.265 with inline VPS/SPS/PPS. /// Only supports single layer streams (VPS is cached but not parsed). /// /// Build it from a [`moq_net::TrackRequest`] ([`new`](Self::new)) or an existing /// track ([`from_track`](Self::from_track)). The catalog rendition fills in lazily /// once the first SPS is parsed; read it via [`catalog`](Self::catalog). pub struct Import { - // The track being produced. + split: Split, track: crate::container::Producer, - - // The standalone catalog, populated once the codec config is known. catalog: hang::Catalog, - - // The resolved config; a change to it on a fixed track is an error. config: Option, - - // The current frame being built. - current: Frame, - - // Used to compute wall clock timestamps if needed. - zero: Option, - - // Cached parameter set NALs for re-insertion before keyframes. - vps: Option, - sps: Option, - pps: Option, - - // Tracks the minimum frame duration and updates the catalog `jitter` field. + last_sps: Option, jitter: MinFrameDuration, } @@ -49,67 +48,35 @@ impl Import { /// Publish on an existing track producer. pub fn from_track(track: moq_net::TrackProducer) -> Self { Self { + split: Split::new(), track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), catalog: hang::Catalog::default(), config: None, - current: Default::default(), - zero: None, - vps: None, - sps: None, - pps: None, + last_sps: None, jitter: MinFrameDuration::new(), } } - fn init(&mut self, sps: &SpsNALUnit) -> Result<()> { - let profile = &sps.rbsp.profile_tier_level.general_profile; - let vui_data = sps.rbsp.vui_parameters.as_ref().map(VuiData::new).unwrap_or_default(); - - let mut config = hang::catalog::VideoConfig::new(hang::catalog::H265 { - in_band: true, // We only support `hev1` with inline SPS/PPS for now - profile_space: profile.profile_space, - profile_idc: profile.profile_idc, - profile_compatibility_flags: profile.profile_compatibility_flag.bits().to_be_bytes(), - tier_flag: profile.tier_flag, - level_idc: profile.level_idc.ok_or(Error::MissingLevelIdc)?, - constraint_flags: crate::codec::h265::pack_constraint_flags(profile), - }); - config.coded_width = Some(sps.rbsp.cropped_width() as u32); - config.coded_height = Some(sps.rbsp.cropped_height() as u32); - config.framerate = vui_data.framerate; - config.display_ratio_width = vui_data.display_ratio_width; - config.display_ratio_height = vui_data.display_ratio_height; - config.container = hang::catalog::Container::Legacy; - - if let Some(old) = &self.config { - if old == &config { - return Ok(()); - } - // A different SPS would need a new init segment, which the single fixed - // track can't represent. - return Err(Error::FixedTrackReconfigured.into()); - } - - let track_name = self.track.name().to_string(); - tracing::debug!(name = ?track_name, ?config, "starting track"); - self.catalog.video.renditions.insert(track_name, config.clone()); - self.config = Some(config); - - Ok(()) - } - - /// Initialize the decoder with SPS/PPS and other non-slice NALs. + /// Initialize the decoder with VPS/SPS/PPS and other non-slice NALs. + /// + /// Resolves the config from any SPS in the buffer and primes the splitter's + /// parameter-set cache. Optional, since the importer also self-initializes + /// from the first keyframe. The buffer is fully consumed. pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { - let mut nals = NalIterator::new(buf); - + let mut scan = Bytes::copy_from_slice(buf.as_ref()); + let mut nals = NalIterator::new(&mut scan); while let Some(nal) = nals.next().transpose()? { - self.decode_nal(nal, None)?; + if is_sps(&nal) { + self.configure_from_sps(&nal)?; + } } - - if let Some(nal) = nals.flush()? { - self.decode_nal(nal, None)?; + if let Some(nal) = nals.flush()? + && is_sps(&nal) + { + self.configure_from_sps(&nal)?; } + self.split.seed(buf)?; Ok(()) } @@ -123,242 +90,132 @@ impl Import { self.config.is_some().then_some(&self.catalog) } - /// Decode as much data as possible from the given buffer. - /// - /// Unlike [Self::decode_frame], this method needs the start code for the next frame. - /// This means it works for streaming media (ex. stdin) but adds a frame of latency. - /// - /// TODO: This currently associates PTS with the *previous* frame, as part of `maybe_start_frame`. - pub fn decode_stream>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - let pts = self.pts(pts)?; - - // Iterate over the NAL units in the buffer based on start codes. - let nals = NalIterator::new(buf); + /// True once the first SPS has populated the catalog. + pub fn is_initialized(&self) -> bool { + self.config.is_some() + } - for nal in nals { - self.decode_nal(nal?, Some(pts))?; + /// Decode from an asynchronous reader (streaming input). + pub async fn decode_from(&mut self, reader: &mut T) -> Result<()> { + let mut buffer = BytesMut::new(); + while reader.read_buf(&mut buffer).await? > 0 { + self.decode_stream(&mut buffer, None)?; } - Ok(()) } - /// Decode all data in the buffer, assuming the buffer contains (the rest of) a frame. + /// Decode as much data as possible from the given buffer. /// - /// Unlike [Self::decode_stream], this is called when we know NAL boundaries. - /// This can avoid a frame of latency just waiting for the next frame's start code. - /// This can also be used when EOF is detected to flush the final frame. + /// Unlike [Self::decode_frame], this needs the start code of the next frame, + /// so it works for streaming media (e.g. stdin) but adds a frame of latency. + pub fn decode_stream>(&mut self, buf: &mut T, pts: Option) -> Result<()> { + let frames = self.split.decode_stream(buf, pts)?; + self.write_frames(frames) + } + + /// Decode all data in the buffer, assuming it holds (the rest of) one frame. /// - /// NOTE: The next decode will fail if it doesn't begin with a start code. + /// Unlike [Self::decode_stream], this is called when NAL boundaries are + /// known, avoiding a frame of latency. Also used at EOF to flush the final frame. pub fn decode_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - let pts = self.pts(pts)?; - // Iterate over the NAL units in the buffer based on start codes. - let mut nals = NalIterator::new(buf); - - // Iterate over each NAL that is followed by a start code. - while let Some(nal) = nals.next().transpose()? { - self.decode_nal(nal, Some(pts))?; - } - - // Assume the rest of the buffer is a single NAL. - if let Some(nal) = nals.flush()? { - self.decode_nal(nal, Some(pts))?; - } - - // Flush the frame if we read a slice. - self.maybe_start_frame(Some(pts))?; + let frames = self.split.decode_frame(buf, pts)?; + self.write_frames(frames) + } + /// Finish the track, flushing the current group. + pub fn finish(&mut self) -> Result<()> { + self.track.finish()?; Ok(()) } - /// Decode a single NAL unit. Only reads the first header byte to extract nal_unit_type, - /// Ignores nuh_layer_id and nuh_temporal_id_plus1. - fn decode_nal(&mut self, nal: Bytes, pts: Option) -> Result<()> { - if nal.len() < 2 { - return Err(Error::NalTooShort.into()); - } - // u16 header: [forbidden_zero_bit(1) | nal_unit_type(6) | nuh_layer_id(6) | nuh_temporal_id_plus1(3)] - let header = nal.first().ok_or(Error::NalTooShort)?; - - let forbidden_zero_bit = (header >> 7) & 1; - if forbidden_zero_bit != 0 { - return Err(Error::ForbiddenZeroBit.into()); - } - - // Bits 1-6: nal_unit_type - let nal_unit_type = (header >> 1) & 0b111111; - let nal_type = NALUnitType::from(nal_unit_type); - - match nal_type { - NALUnitType::VpsNut => { - self.maybe_start_frame(pts)?; - - self.vps = Some(nal.clone()); - self.current.contains_vps = true; - } - NALUnitType::SpsNut => { - self.maybe_start_frame(pts)?; - - // Try to reinitialize the track if the SPS has changed. - let sps = SpsNALUnit::parse(&mut &nal[..]).map_err(|_| Error::SpsParse)?; - self.init(&sps)?; - - // SPS changed mid-AU. Cached VPS/PPS are tied to the old SPS - // and may already have been appended to current.chunks earlier - // in this AU; reset the AU so the new VPS+SPS+PPS triple is - // the only parameter set we emit. - if self.sps.as_ref().is_some_and(|cached| cached != &nal) { - self.pps = None; - self.current.chunks.clear(); - self.current.contains_vps = false; - self.current.contains_sps = false; - self.current.contains_pps = false; - } - - self.sps = Some(nal.clone()); - self.current.contains_sps = true; - } - NALUnitType::PpsNut => { - self.maybe_start_frame(pts)?; - - self.pps = Some(nal.clone()); - self.current.contains_pps = true; - } - NALUnitType::AudNut | NALUnitType::PrefixSeiNut | NALUnitType::SuffixSeiNut => { - self.maybe_start_frame(pts)?; - } - // Keyframe containing slices - NALUnitType::IdrWRadl - | NALUnitType::IdrNLp - | NALUnitType::BlaNLp - | NALUnitType::BlaWRadl - | NALUnitType::BlaWLp - | NALUnitType::CraNut => { - // Insert cached VPS/SPS/PPS before keyframes if not already present in this frame. - if !self.current.contains_vps - && let Some(vps) = &self.vps - { - self.current.chunks.extend_from_slice(&START_CODE); - self.current.chunks.extend_from_slice(vps); - self.current.contains_vps = true; - } - if !self.current.contains_sps - && let Some(sps) = &self.sps - { - self.current.chunks.extend_from_slice(&START_CODE); - self.current.chunks.extend_from_slice(sps); - self.current.contains_sps = true; - } - if !self.current.contains_pps - && let Some(pps) = &self.pps - { - self.current.chunks.extend_from_slice(&START_CODE); - self.current.chunks.extend_from_slice(pps); - self.current.contains_pps = true; - } - - self.current.contains_idr = true; - self.current.contains_slice = true; - } - // All other slice types (both N and R variants) - NALUnitType::TrailN - | NALUnitType::TrailR - | NALUnitType::TsaN - | NALUnitType::TsaR - | NALUnitType::StsaN - | NALUnitType::StsaR - | NALUnitType::RadlN - | NALUnitType::RadlR - | NALUnitType::RaslN - | NALUnitType::RaslR => { - // Check first_slice_segment_in_pic_flag (bit 7 of third byte, after 2-byte header) - if nal.get(2).ok_or(Error::NalTooShort)? & 0x80 != 0 { - self.maybe_start_frame(pts)?; - } - self.current.contains_slice = true; - } - _ => {} - } - - // Replace the original start code with a canonical 4-byte start code (marginally easier - // for downstream players, e.g. MSE). - self.current.chunks.extend_from_slice(&START_CODE); - self.current.chunks.extend_from_slice(&nal); - + /// Close the current group and open the next one at `sequence`. + /// + /// Any in-flight access unit is dropped. Pre-seek NALs would otherwise leak + /// into the post-seek group with the wrong timestamp. + pub fn seek(&mut self, sequence: u64) -> Result<()> { + self.split.reset(); + self.track.seek(sequence)?; Ok(()) } - fn maybe_start_frame(&mut self, pts: Option) -> Result<()> { - // If we haven't seen any slices, we shouldn't flush yet. - if !self.current.contains_slice { + /// Resolve the config from an inline SPS. + /// + /// A different SPS would need a new init segment, which the single fixed + /// track can't represent, so a reconfiguration is an error. + fn configure_from_sps(&mut self, sps_nal: &Bytes) -> Result<()> { + if self.last_sps.as_ref() == Some(sps_nal) { return Ok(()); } - let pts = pts.ok_or(Error::MissingTimestamp)?; - let keyframe = self.current.contains_idr; - - // A keyframe we couldn't configure (no SPS) is undecodable. - if keyframe && self.config.is_none() { - return Err(Error::MissingSps.into()); - } + let sps = SpsNALUnit::parse(&mut &sps_nal[..]).map_err(|_| Error::SpsParse)?; + let profile = &sps.rbsp.profile_tier_level.general_profile; + let vui_data = sps.rbsp.vui_parameters.as_ref().map(VuiData::new).unwrap_or_default(); - // Take the payload and clear the per-AU state up front, so a - // MissingKeyframe from a pre-keyframe delta (a mid-stream join) leaves a - // clean slate for the next access unit. - let payload = std::mem::take(&mut self.current.chunks).freeze(); - self.current.contains_idr = false; - self.current.contains_slice = false; - self.current.contains_vps = false; - self.current.contains_sps = false; - self.current.contains_pps = false; - - let frame = crate::container::Frame { - timestamp: pts, - payload, - keyframe, - duration: None, - }; + let mut config = hang::catalog::VideoConfig::new(hang::catalog::H265 { + in_band: true, // We only support `hev1` with inline VPS/SPS/PPS for now. + profile_space: profile.profile_space, + profile_idc: profile.profile_idc, + profile_compatibility_flags: profile.profile_compatibility_flag.bits().to_be_bytes(), + tier_flag: profile.tier_flag, + level_idc: profile.level_idc.ok_or(Error::MissingLevelIdc)?, + constraint_flags: super::pack_constraint_flags(profile), + }); + config.coded_width = Some(sps.rbsp.cropped_width() as u32); + config.coded_height = Some(sps.rbsp.cropped_height() as u32); + config.framerate = vui_data.framerate; + config.display_ratio_width = vui_data.display_ratio_width; + config.display_ratio_height = vui_data.display_ratio_height; + config.container = hang::catalog::Container::Legacy; - // A pre-keyframe delta has no group to anchor it: the producer returns - // MissingKeyframe, which the caller (e.g. a TS mid-stream join) skips. - self.track.write(frame)?; + self.last_sps = Some(sps_nal.clone()); - if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.video.renditions.get_mut(self.track.name()) - { - c.jitter = Some(jitter); + if let Some(old) = &self.config { + if old == &config { + return Ok(()); + } + return Err(Error::FixedTrackReconfigured.into()); } + let track_name = self.track.name().to_string(); + tracing::debug!(name = ?track_name, ?config, "starting track"); + self.catalog.video.renditions.insert(track_name, config.clone()); + self.config = Some(config); Ok(()) } - /// Finish the track, flushing the current group. - pub fn finish(&mut self) -> Result<()> { - self.track.finish()?; - Ok(()) - } + /// Write split frames to the track, resolving the config from the first + /// keyframe's inline SPS and refining the catalog jitter as it goes. + fn write_frames(&mut self, frames: impl IntoIterator) -> Result<()> { + for frame in frames { + if frame.keyframe + && let Some(sps) = find_sps(&frame.payload) + { + self.configure_from_sps(&sps)?; + } - /// Close the current group and open the next one at `sequence`. - /// - /// Any in-flight access unit is dropped. Pre-seek NALs would otherwise leak - /// into the post-seek group with the wrong timestamp. - pub fn seek(&mut self, sequence: u64) -> Result<()> { - self.current = Frame::default(); - self.track.seek(sequence)?; - Ok(()) - } + // A keyframe we still can't configure (no SPS) is undecodable. + if frame.keyframe && self.config.is_none() { + return Err(Error::MissingSps.into()); + } - /// True once the first SPS has populated the catalog. - pub fn is_initialized(&self) -> bool { - self.config.is_some() - } + let pts = frame.timestamp; + // A pre-keyframe delta has no group to anchor it: the producer returns + // MissingKeyframe, which the caller (e.g. a TS mid-stream join) skips. + self.track.write(frame)?; - fn pts(&mut self, hint: Option) -> Result { - if let Some(pts) = hint { - return Ok(pts); + if let Some(jitter) = self.jitter.observe(pts) + && let Some(c) = self.catalog.video.renditions.get_mut(self.track.name()) + { + c.jitter = Some(jitter); + } } + Ok(()) + } +} - let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); - Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) +impl FrameDecode for Import { + fn decode>(&mut self, frames: I) -> Result<()> { + self.write_frames(frames) } } @@ -368,14 +225,21 @@ impl Renditions for Import { } } -#[derive(Default)] -struct Frame { - chunks: BytesMut, - contains_idr: bool, - contains_slice: bool, - contains_vps: bool, - contains_sps: bool, - contains_pps: bool, +fn is_sps(nal: &[u8]) -> bool { + nal.first() + .is_some_and(|h| nal_unit_type(*h) == scuffle_h265::NALUnitType::SpsNut) +} + +/// Find the first SPS NAL in an Annex-B payload, if any. +fn find_sps(payload: &[u8]) -> Option { + let mut buf = Bytes::copy_from_slice(payload); + let mut nals = NalIterator::new(&mut buf); + while let Some(Ok(nal)) = nals.next() { + if is_sps(&nal) { + return Some(nal); + } + } + nals.flush().ok().flatten().filter(|nal| is_sps(nal)) } #[derive(Default)] diff --git a/rs/moq-mux/src/codec/h265/mod.rs b/rs/moq-mux/src/codec/h265/mod.rs index 828a5798d..37e84ccd2 100644 --- a/rs/moq-mux/src/codec/h265/mod.rs +++ b/rs/moq-mux/src/codec/h265/mod.rs @@ -8,9 +8,11 @@ mod export; mod import; +mod split; pub use export::*; pub use import::*; +pub use split::*; use bytes::{Buf, BufMut, Bytes, BytesMut}; use scuffle_h265::{NALUnitType, SpsNALUnit}; diff --git a/rs/moq-mux/src/codec/h265/split.rs b/rs/moq-mux/src/codec/h265/split.rs new file mode 100644 index 000000000..038146c61 --- /dev/null +++ b/rs/moq-mux/src/codec/h265/split.rs @@ -0,0 +1,345 @@ +//! H.265 Annex-B stream splitter. +//! +//! The H.265 analogue of [`crate::codec::h264::Split`]: turns a raw Annex-B byte +//! stream (inline VPS/SPS/PPS) into [`crate::container::Frame`]s. It finds +//! access-unit boundaries, caches VPS/SPS/PPS and re-inserts them ahead of each +//! keyframe so every keyframe is self-contained, and stamps wall-clock +//! timestamps when the caller has none (stdin). It owns no track, catalog, or +//! codec config. The importer parses the codec config out of the frames it +//! emits. + +use bytes::{Buf, Bytes, BytesMut}; +use scuffle_h265::NALUnitType; +use tokio::io::{AsyncRead, AsyncReadExt}; + +use super::Error; +use crate::Result; +use crate::codec::annexb::{NalIterator, START_CODE}; + +/// H.265 Annex-B stream splitter: bytes in, [`Frame`](crate::container::Frame)s out. +/// +/// Feed bytes via [`decode_stream`](Self::decode_stream) (unknown frame +/// boundaries, e.g. stdin), [`decode_frame`](Self::decode_frame) (one complete +/// access unit per call), or [`decode_from`](Self::decode_from) (an async +/// reader). Each returns the frames it produced. VPS/SPS/PPS seen inline are +/// cached and re-inserted ahead of each keyframe; [`seed`](Self::seed) primes +/// that cache from an out-of-band parameter-set buffer. +pub struct Split { + current: Au, + vps: Option, + sps: Option, + pps: Option, + zero: Option, + pending: Vec, +} + +#[derive(Default)] +struct Au { + chunks: BytesMut, + contains_idr: bool, + contains_slice: bool, + contains_vps: bool, + contains_sps: bool, + contains_pps: bool, +} + +impl Default for Split { + fn default() -> Self { + Self::new() + } +} + +impl Split { + /// A fresh splitter with an empty parameter-set cache. + pub fn new() -> Self { + Self { + current: Au::default(), + vps: None, + sps: None, + pps: None, + zero: None, + pending: Vec::new(), + } + } + + /// Prime the VPS/SPS/PPS cache from an Annex-B parameter-set buffer, so the + /// first keyframe is self-contained even if the stream itself omits inline + /// parameter sets. Other NAL types in the buffer are ignored. + pub fn seed>(&mut self, buf: &mut T) -> Result<()> { + let mut nals = NalIterator::new(buf); + while let Some(nal) = nals.next().transpose()? { + self.cache_param(&nal); + } + if let Some(nal) = nals.flush()? { + self.cache_param(&nal); + } + Ok(()) + } + + fn cache_param(&mut self, nal: &Bytes) { + match nal.first().map(|h| nal_unit_type(*h)) { + Some(NALUnitType::VpsNut) => self.vps = Some(nal.clone()), + Some(NALUnitType::SpsNut) => self.sps = Some(nal.clone()), + Some(NALUnitType::PpsNut) => self.pps = Some(nal.clone()), + _ => {} + } + } + + /// Decode from an asynchronous reader, returning all frames produced. + pub async fn decode_from(&mut self, reader: &mut T) -> Result> { + let mut frames = Vec::new(); + let mut buffer = BytesMut::new(); + while reader.read_buf(&mut buffer).await? > 0 { + frames.extend(self.decode_stream(&mut buffer, None)?); + } + Ok(frames) + } + + /// Decode a buffer where frame boundaries are unknown, returning the frames + /// it produced. The leading start code of the *next* access unit is what + /// signals the previous one is complete, so the final access unit stays + /// buffered until the next call (or [`decode_frame`](Self::decode_frame)). + pub fn decode_stream>( + &mut self, + buf: &mut T, + pts: impl Into>, + ) -> Result> { + let pts = self.pts(pts.into())?; + let nals = NalIterator::new(buf); + for nal in nals { + self.decode_nal(nal?, pts)?; + } + Ok(std::mem::take(&mut self.pending)) + } + + /// Decode a buffer holding one complete access unit, returning the frames it + /// produced (typically one). Any trailing NAL without a start code is the + /// last NAL of this access unit, and the unit is flushed before returning. + pub fn decode_frame>( + &mut self, + buf: &mut T, + pts: impl Into>, + ) -> Result> { + let pts = self.pts(pts.into())?; + let mut nals = NalIterator::new(buf); + while let Some(nal) = nals.next().transpose()? { + self.decode_nal(nal, pts)?; + } + if let Some(nal) = nals.flush()? { + self.decode_nal(nal, pts)?; + } + self.maybe_start_frame(pts)?; + Ok(std::mem::take(&mut self.pending)) + } + + /// Decode a single NAL unit. Only reads the first header byte to extract + /// nal_unit_type, ignoring nuh_layer_id and nuh_temporal_id_plus1. + fn decode_nal(&mut self, nal: Bytes, pts: moq_net::Timestamp) -> Result<()> { + if nal.len() < 2 { + return Err(Error::NalTooShort.into()); + } + // u16 header: [forbidden_zero_bit(1) | nal_unit_type(6) | nuh_layer_id(6) | nuh_temporal_id_plus1(3)] + let header = nal.first().ok_or(Error::NalTooShort)?; + if (header >> 7) & 1 != 0 { + return Err(Error::ForbiddenZeroBit.into()); + } + + let nal_type = nal_unit_type(*header); + + match nal_type { + NALUnitType::VpsNut => { + self.maybe_start_frame(pts)?; + self.vps = Some(nal.clone()); + self.current.contains_vps = true; + } + NALUnitType::SpsNut => { + self.maybe_start_frame(pts)?; + + // SPS changed mid-stream. Cached PPS is tied to the old SPS and + // may already have been appended to current.chunks earlier in + // this AU; reset so the new VPS+SPS+PPS triple is the only + // parameter set we emit. + if self.sps.as_ref().is_some_and(|cached| cached != &nal) { + self.pps = None; + self.current.chunks.clear(); + self.current.contains_vps = false; + self.current.contains_sps = false; + self.current.contains_pps = false; + } + + self.sps = Some(nal.clone()); + self.current.contains_sps = true; + } + NALUnitType::PpsNut => { + self.maybe_start_frame(pts)?; + self.pps = Some(nal.clone()); + self.current.contains_pps = true; + } + NALUnitType::AudNut | NALUnitType::PrefixSeiNut | NALUnitType::SuffixSeiNut => { + self.maybe_start_frame(pts)?; + } + // Keyframe containing slices. + NALUnitType::IdrWRadl + | NALUnitType::IdrNLp + | NALUnitType::BlaNLp + | NALUnitType::BlaWRadl + | NALUnitType::BlaWLp + | NALUnitType::CraNut => { + // Insert cached VPS/SPS/PPS before keyframes if not already present. + if !self.current.contains_vps + && let Some(vps) = self.vps.clone() + { + self.current.chunks.extend_from_slice(&START_CODE); + self.current.chunks.extend_from_slice(&vps); + self.current.contains_vps = true; + } + if !self.current.contains_sps + && let Some(sps) = self.sps.clone() + { + self.current.chunks.extend_from_slice(&START_CODE); + self.current.chunks.extend_from_slice(&sps); + self.current.contains_sps = true; + } + if !self.current.contains_pps + && let Some(pps) = self.pps.clone() + { + self.current.chunks.extend_from_slice(&START_CODE); + self.current.chunks.extend_from_slice(&pps); + self.current.contains_pps = true; + } + + self.current.contains_idr = true; + self.current.contains_slice = true; + } + // All other slice types (both N and R variants). + NALUnitType::TrailN + | NALUnitType::TrailR + | NALUnitType::TsaN + | NALUnitType::TsaR + | NALUnitType::StsaN + | NALUnitType::StsaR + | NALUnitType::RadlN + | NALUnitType::RadlR + | NALUnitType::RaslN + | NALUnitType::RaslR => { + // Check first_slice_segment_in_pic_flag (bit 7 of third byte, after 2-byte header). + if nal.get(2).ok_or(Error::NalTooShort)? & 0x80 != 0 { + self.maybe_start_frame(pts)?; + } + self.current.contains_slice = true; + } + _ => {} + } + + // Replace the original start code with a canonical 4-byte start code (marginally + // easier for downstream players, e.g. MSE). + self.current.chunks.extend_from_slice(&START_CODE); + self.current.chunks.extend_from_slice(&nal); + + Ok(()) + } + + fn maybe_start_frame(&mut self, pts: moq_net::Timestamp) -> Result<()> { + if !self.current.contains_slice { + return Ok(()); + } + + let payload = std::mem::take(&mut self.current.chunks).freeze(); + let keyframe = self.current.contains_idr; + self.current.contains_idr = false; + self.current.contains_slice = false; + self.current.contains_vps = false; + self.current.contains_sps = false; + self.current.contains_pps = false; + + self.pending.push(crate::container::Frame { + timestamp: pts, + payload, + keyframe, + duration: None, + }); + Ok(()) + } + + /// Drop any in-flight access unit. + /// + /// Pre-reset NALs would otherwise leak into a later frame with the wrong + /// timestamp. The parameter-set cache is kept so subsequent keyframes stay + /// self-contained. + pub fn reset(&mut self) { + self.current = Au::default(); + } + + fn pts(&mut self, hint: Option) -> Result { + if let Some(pts) = hint { + return Ok(pts); + } + let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); + Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) + } +} + +/// Extract the HEVC `nal_unit_type` from the first header byte (bits 1..=6). +pub(super) fn nal_unit_type(header: u8) -> NALUnitType { + NALUnitType::from((header >> 1) & 0b111111) +} + +#[cfg(test)] +mod tests { + use super::*; + + const SC4: &[u8] = &[0, 0, 0, 1]; + + // HEVC NAL headers: byte0 = nal_unit_type << 1 (forbidden bit 0, layer id 0). + const VPS: &[u8] = &[0x40, 0x01, 0x0c]; // type 32 + const SPS: &[u8] = &[0x42, 0x01, 0x01]; // type 33 + const PPS: &[u8] = &[0x44, 0x01, 0xc0]; // type 34 + const IDR: &[u8] = &[0x26, 0x01, 0x80, 0xaa]; // type 19 (IdrWRadl) + + fn annexb(nals: &[&[u8]]) -> BytesMut { + let mut buf = BytesMut::new(); + for nal in nals { + buf.extend_from_slice(SC4); + buf.extend_from_slice(nal); + } + buf + } + + fn ts() -> moq_net::Timestamp { + moq_net::Timestamp::from_micros(0).unwrap() + } + + fn contains(haystack: &[u8], needle: &[u8]) -> bool { + haystack.windows(needle.len()).any(|w| w == needle) + } + + /// A keyframe access unit fed as one buffer emits one self-contained frame: + /// VPS+SPS+PPS are packaged ahead of the IDR slice and `keyframe` is set. + #[tokio::test(start_paused = true)] + async fn decode_frame_packages_keyframe() { + let mut split = Split::new(); + let frames = split.decode_frame(&mut annexb(&[VPS, SPS, PPS, IDR]), ts()).unwrap(); + + assert_eq!(frames.len(), 1); + assert!(frames[0].keyframe); + assert!(contains(&frames[0].payload, VPS)); + assert!(contains(&frames[0].payload, SPS)); + assert!(contains(&frames[0].payload, PPS)); + assert!(contains(&frames[0].payload, IDR)); + } + + /// A seeded splitter re-inserts the cached VPS/SPS/PPS ahead of a bare IDR, + /// even though the stream itself never carried inline parameter sets. + #[tokio::test(start_paused = true)] + async fn seed_makes_bare_keyframe_self_contained() { + let mut split = Split::new(); + split.seed(&mut annexb(&[VPS, SPS, PPS])).unwrap(); + + let frames = split.decode_frame(&mut annexb(&[IDR]), ts()).unwrap(); + assert_eq!(frames.len(), 1); + assert!(frames[0].keyframe); + assert!(contains(&frames[0].payload, VPS)); + assert!(contains(&frames[0].payload, SPS)); + assert!(contains(&frames[0].payload, PPS)); + } +} From 249ff25edb0600fbc6158708693488eee7292092 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 18:47:29 +0000 Subject: [PATCH 10/19] moq-mux: extract the AV1 byte->frame Split Round out the streaming codecs: separate AV1 OBU parsing from the publisher, matching h264/h265. - New `codec::av1::Split`: a dumb OBU stream assembler that finds temporal-unit boundaries, flags keyframes (a sequence header or a KEY_FRAME), and stamps wall-clock timestamps for stdin. No track, catalog, or codec config. AV1 carries the sequence header inline ahead of keyframes, so unlike H.264/H.265 there's nothing to cache or re-insert; `seed` just prefixes leading metadata OBUs onto the next frame. The `ObuIterator` moves here. `decode_stream` keeps the per-OBU wall-clock timestamps. - `codec::av1::Import` now drives the splitter and owns the config: it scans the sequence header the splitter packages into the first keyframe (or an av1C / seed buffer via `initialize`) to fill the catalog, falls back to a minimal config on a parse failure, errors on an unconfigurable keyframe, and writes pre-keyframe deltas through to the producer (MissingKeyframe for a mid-stream join). It also implements `FrameDecode`. - Adds the first AV1 splitter unit tests (boundary + keyframe detection). https://claude.ai/code/session_011S7pzcg2XsPP3AExuymac1 --- rs/moq-mux/src/codec/av1/import.rs | 533 ++++++++++------------------- rs/moq-mux/src/codec/av1/mod.rs | 2 + rs/moq-mux/src/codec/av1/split.rs | 349 +++++++++++++++++++ 3 files changed, 526 insertions(+), 358 deletions(-) create mode 100644 rs/moq-mux/src/codec/av1/split.rs diff --git a/rs/moq-mux/src/codec/av1/import.rs b/rs/moq-mux/src/codec/av1/import.rs index 85884b8ef..ce92ebc66 100644 --- a/rs/moq-mux/src/codec/av1/import.rs +++ b/rs/moq-mux/src/codec/av1/import.rs @@ -1,41 +1,41 @@ -use crate::container::jitter::MinFrameDuration; -use crate::publish::Renditions; - -use bytes::BytesMut; -use bytes::{Buf, Bytes}; +//! AV1 importer. +//! +//! Publishes raw AV1 (OBU-framed, inline sequence headers) on a single moq +//! track and resolves the catalog rendition. The codec config comes from the +//! sequence header the splitter packages into the first keyframe (scanned out of +//! the frame here), or from an av1C record handed to +//! [`initialize`](Import::initialize). A keyframe that can't be configured is an +//! error; non-keyframes before the first config are written through to the +//! producer, which reports [`MissingKeyframe`](crate::container::MissingKeyframe) +//! for a mid-stream join. OBU byte parsing lives in [`Split`]; this type drives +//! it and adds the catalog. + +use bytes::{Buf, Bytes, BytesMut}; use scuffle_av1::seq::SequenceHeaderObu; +use scuffle_av1::{ObuHeader, ObuType}; +use tokio::io::{AsyncRead, AsyncReadExt}; -use super::Error; +use super::split::ObuIterator; +use super::{Error, Split}; use crate::Result; +use crate::container::Frame; +use crate::container::jitter::MinFrameDuration; +use crate::publish::{FrameDecode, Renditions}; /// A decoder for AV1 with inline sequence headers. +/// +/// Build it from a [`moq_net::TrackRequest`] ([`new`](Self::new)) or an existing +/// track ([`from_track`](Self::from_track)). The catalog rendition fills in lazily +/// once the config is known; read it via [`catalog`](Self::catalog). pub struct Import { - // The track being produced. + split: Split, track: crate::container::Producer, - - // The standalone catalog, populated once the config is known. catalog: hang::Catalog, - - // Whether the track has been initialized. config: Option, - - // The current frame being built. - current: Frame, - - // Used to compute wall clock timestamps if needed. - zero: Option, - - // Tracks the minimum frame duration and updates the catalog `jitter` field. + last_seq: Option, jitter: MinFrameDuration, } -#[derive(Default)] -struct Frame { - chunks: BytesMut, - contains_keyframe: bool, - contains_frame: bool, -} - impl Import { /// Serve a track request, accepting it at the microsecond timescale. pub fn new(request: moq_net::TrackRequest) -> Self { @@ -46,15 +46,69 @@ impl Import { /// Publish on an existing track producer. pub fn from_track(track: moq_net::TrackProducer) -> Self { Self { + split: Split::new(), track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), catalog: hang::Catalog::default(), config: None, - current: Default::default(), - zero: None, + last_seq: None, jitter: MinFrameDuration::new(), } } + /// Initialize the decoder with a sequence header / av1C and other metadata. + /// + /// - **av1C** (leading `0x81` marker): the buffer is parsed as an + /// AV1CodecConfigurationRecord, which resolves the config. + /// - **raw OBUs**: any sequence header resolves the config and is fed into + /// the splitter so it prefixes the first keyframe. + /// + /// Optional, since the importer also self-initializes from the first keyframe. + /// The buffer is fully consumed. + pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { + let data = buf.as_ref(); + + // av1C box starts with 0x81 (marker=1, version=1) per ISO/IEC 14496-15. + if data.len() >= 16 && data[0] == 0x81 { + self.init_from_av1c(data)?; + buf.advance(buf.remaining()); + return Ok(()); + } + + // Raw OBUs: resolve the config from any sequence header, then feed the + // metadata OBUs into the splitter so they prefix the first keyframe. + if let Some(seq) = find_sequence_header(data) { + self.configure_from_seq(&seq)?; + } + self.split.seed(buf)?; + Ok(()) + } + + fn init_from_av1c(&mut self, data: &[u8]) -> Result<()> { + let seq_profile = (data[1] >> 5) & 0x07; + let seq_level_idx = data[1] & 0x1F; + let tier = ((data[2] >> 7) & 0x01) == 1; + let high_bitdepth = ((data[2] >> 6) & 0x01) == 1; + let twelve_bit = ((data[2] >> 5) & 0x01) == 1; + + // Resolution is unknown from av1C; it's filled when the first sequence header arrives. + let mut config = hang::catalog::VideoConfig::new(hang::catalog::AV1 { + profile: seq_profile, + level: seq_level_idx, + tier: if tier { 'H' } else { 'M' }, + bitdepth: super::bitdepth(twelve_bit, high_bitdepth), + mono_chrome: ((data[2] >> 4) & 0x01) == 1, + chroma_subsampling_x: ((data[2] >> 3) & 0x01) == 1, + chroma_subsampling_y: ((data[2] >> 2) & 0x01) == 1, + chroma_sample_position: data[2] & 0x03, + color_primaries: 1, + transfer_characteristics: 1, + matrix_coefficients: 1, + full_range: false, + }); + config.container = hang::catalog::Container::Legacy; + self.apply_config(config) + } + fn init(&mut self, seq_header: &SequenceHeaderObu) -> Result<()> { let mut config = hang::catalog::VideoConfig::new(hang::catalog::AV1 { profile: seq_header.seq_profile, @@ -86,35 +140,17 @@ impl Import { config.coded_width = Some(seq_header.max_frame_width as u32); config.coded_height = Some(seq_header.max_frame_height as u32); config.container = hang::catalog::Container::Legacy; - - if let Some(old) = &self.config - && old == &config - { - return Ok(()); - } - - if self.config.is_some() { - return Err(Error::FixedTrackReconfigured.into()); - } - - tracing::debug!(name = ?self.track.name(), ?config, "starting track"); - self.catalog - .video - .renditions - .insert(self.track.name().to_string(), config.clone()); - - self.config = Some(config); - - Ok(()) + self.apply_config(config) } - /// Initialize with minimal config if sequence header parsing fails + /// Minimal config when sequence-header parsing fails, so the stream can still + /// flow (the catalog just won't carry full codec info). fn init_minimal(&mut self) -> Result<()> { let mut config = hang::catalog::VideoConfig::new(hang::catalog::AV1 { - profile: 0, // Main profile - level: 0, // Unknown - tier: 'M', // Main tier - bitdepth: 8, // Assume 8-bit + profile: 0, + level: 0, + tier: 'M', + bitdepth: 8, mono_chrome: false, chroma_subsampling_x: true, // 4:2:0 chroma_subsampling_y: true, @@ -125,90 +161,48 @@ impl Import { full_range: false, }); config.container = hang::catalog::Container::Legacy; + self.apply_config(config) + } - if self.config.is_some() { + /// Store a resolved config. The first one is kept; a different one would need + /// a new init segment, which the single fixed track can't represent. + fn apply_config(&mut self, config: hang::catalog::VideoConfig) -> Result<()> { + if let Some(old) = &self.config { + if old == &config { + return Ok(()); + } return Err(Error::FixedTrackReconfigured.into()); } - tracing::debug!(name = ?self.track.name(), "starting track with minimal config"); + tracing::debug!(name = ?self.track.name(), ?config, "starting track"); self.catalog .video .renditions .insert(self.track.name().to_string(), config.clone()); - self.config = Some(config); - Ok(()) } - /// Initialize the decoder with sequence header and other metadata OBUs. - pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { - let data = buf.as_ref(); - - // Handle av1C format (MP4/container initialization) - // av1C box starts with 0x81 (marker=1, version=1) per ISO/IEC 14496-15 - if data.len() >= 4 && data[0] == 0x81 && data.len() >= 16 { - self.init_from_av1c(data)?; - buf.advance(data.len()); + /// Resolve the config from a sequence-header OBU, falling back to a minimal + /// config if it fails to parse. + fn configure_from_seq(&mut self, seq_obu: &Bytes) -> Result<()> { + if self.last_seq.as_ref() == Some(seq_obu) { return Ok(()); } + self.last_seq = Some(seq_obu.clone()); - // Handle raw OBU format - let mut obus = ObuIterator::new(buf); - while let Some(obu) = obus.next().transpose()? { - self.decode_obu(obu, None)?; - } + let mut reader = &seq_obu[..]; + let header = ObuHeader::parse(&mut reader)?; + let payload_offset = seq_obu.len() - reader.len(); - if let Some(obu) = obus.flush()? { - self.decode_obu(obu, None)?; - } - - Ok(()) - } - - fn init_from_av1c(&mut self, data: &[u8]) -> Result<()> { - // Parse av1C box structure - let seq_profile = (data[1] >> 5) & 0x07; - let seq_level_idx = data[1] & 0x1F; - let tier = ((data[2] >> 7) & 0x01) == 1; - let high_bitdepth = ((data[2] >> 6) & 0x01) == 1; - let twelve_bit = ((data[2] >> 5) & 0x01) == 1; - - // Resolution unknown from av1C - will be updated when first sequence header arrives - let mut config = hang::catalog::VideoConfig::new(hang::catalog::AV1 { - profile: seq_profile, - level: seq_level_idx, - tier: if tier { 'H' } else { 'M' }, - bitdepth: super::bitdepth(twelve_bit, high_bitdepth), - mono_chrome: ((data[2] >> 4) & 0x01) == 1, - chroma_subsampling_x: ((data[2] >> 3) & 0x01) == 1, - chroma_subsampling_y: ((data[2] >> 2) & 0x01) == 1, - chroma_sample_position: data[2] & 0x03, - color_primaries: 1, - transfer_characteristics: 1, - matrix_coefficients: 1, - full_range: false, - }); - config.container = hang::catalog::Container::Legacy; - - if let Some(old) = &self.config - && old == &config - { - return Ok(()); - } - - if self.config.is_some() { - return Err(Error::FixedTrackReconfigured.into()); + match SequenceHeaderObu::parse(header, &mut &seq_obu[payload_offset..]) { + Ok(seq_header) => self.init(&seq_header), + Err(_) if self.config.is_none() => { + tracing::debug!("sequence header parse failed, using minimal config"); + self.init_minimal() + } + Err(_) => Ok(()), } - - self.catalog - .video - .renditions - .insert(self.track.name().to_string(), config.clone()); - - self.config = Some(config); - - Ok(()) } /// The standalone catalog once the config is known, else `None`. @@ -221,159 +215,30 @@ impl Import { self.track.track() } - /// Decode as much data as possible from the given buffer. - pub fn decode_stream>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - let obus = ObuIterator::new(buf); - - for obu in obus { - // Generate PTS for each OBU to avoid reusing same timestamp - let pts = self.pts(pts)?; - self.decode_obu(obu?, Some(pts))?; - } - - Ok(()) + /// True once the config is known and the catalog has been populated. + pub fn is_initialized(&self) -> bool { + self.config.is_some() } - /// Decode all data in the buffer, assuming the buffer contains (the rest of) a frame. - pub fn decode_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - let pts = self.pts(pts)?; - let mut obus = ObuIterator::new(buf); - - while let Some(obu) = obus.next().transpose()? { - self.decode_obu(obu, Some(pts))?; + /// Decode from an asynchronous reader (streaming input). + pub async fn decode_from(&mut self, reader: &mut T) -> Result<()> { + let mut buffer = BytesMut::new(); + while reader.read_buf(&mut buffer).await? > 0 { + self.decode_stream(&mut buffer, None)?; } - - if let Some(obu) = obus.flush()? { - self.decode_obu(obu, Some(pts))?; - } - - self.maybe_start_frame(Some(pts))?; - Ok(()) } - fn decode_obu(&mut self, obu_data: Bytes, pts: Option) -> Result<()> { - if obu_data.is_empty() { - return Err(Error::ObuTooShort.into()); - } - - // Parse OBU header - this consumes header + extension + LEB128 size - let mut reader = &obu_data[..]; - let header = scuffle_av1::ObuHeader::parse(&mut reader)?; - - // Calculate payload offset by seeing how much the parser consumed - let payload_offset = obu_data.len() - reader.len(); - - // Match on the ObuType enum directly - use scuffle_av1::ObuType; - match header.obu_type { - ObuType::SequenceHeader => { - match SequenceHeaderObu::parse(header, &mut &obu_data[payload_offset..]) { - Ok(seq_header) => { - self.init(&seq_header)?; - } - Err(_) => { - // Use minimal config so stream can work (catalog won't have full info) - if self.config.is_none() { - tracing::debug!("Sequence header parsing failed, initializing with minimal config"); - self.init_minimal()?; - } - } - } - - self.current.contains_keyframe = true; - } - ObuType::TemporalDelimiter => { - self.maybe_start_frame(pts)?; - } - ObuType::FrameHeader | ObuType::Frame => { - let is_keyframe = if obu_data.len() > payload_offset { - let data = &obu_data[payload_offset..]; - if data.is_empty() { - false - } else { - let first_byte = data[0]; - - let show_existing_frame = (first_byte >> 7) & 1; - - if show_existing_frame == 1 { - self.current.contains_keyframe - } else { - let frame_type = (first_byte >> 5) & 0b11; - - frame_type == 0 - } - } - } else { - tracing::warn!( - "Frame OBU too short: {} bytes (payload_offset={})", - obu_data.len(), - payload_offset - ); - false - }; - - if is_keyframe || self.current.contains_keyframe { - self.current.contains_keyframe = true; - } - - self.current.contains_frame = true; - } - ObuType::Metadata => { - self.maybe_start_frame(pts)?; - } - ObuType::TileGroup | ObuType::TileList => { - self.current.contains_frame = true; - } - _ => { - // Other OBU types - just include them - } - } - - tracing::trace!(?header.obu_type, "parsed OBU"); - - self.current.chunks.extend_from_slice(&obu_data); - - Ok(()) + /// Decode as much data as possible from the given buffer. + pub fn decode_stream>(&mut self, buf: &mut T, pts: Option) -> Result<()> { + let frames = self.split.decode_stream(buf, pts)?; + self.write_frames(frames) } - fn maybe_start_frame(&mut self, pts: Option) -> Result<()> { - if !self.current.contains_frame { - return Ok(()); - } - - let pts = pts.ok_or(Error::MissingTimestamp)?; - let keyframe = self.current.contains_keyframe; - - // A keyframe we couldn't configure (no sequence header) is undecodable. - if keyframe && self.config.is_none() { - return Err(Error::MissingSequenceHeader.into()); - } - - // Take the payload and clear the per-frame state up front, so a - // MissingKeyframe from a pre-keyframe delta leaves a clean slate. - let payload = std::mem::take(&mut self.current.chunks).freeze(); - self.current.contains_keyframe = false; - self.current.contains_frame = false; - - let frame = crate::container::Frame { - timestamp: pts, - payload, - keyframe, - duration: None, - }; - - // A pre-keyframe delta has no group to anchor it: the producer returns - // MissingKeyframe, which a caller joining mid-stream skips. - self.track.write(frame)?; - - if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.video.renditions.get_mut(self.track.name()) - { - c.jitter = Some(jitter); - } - - Ok(()) + /// Decode all data in the buffer, assuming it holds (the rest of) one frame. + pub fn decode_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { + let frames = self.split.decode_frame(buf, pts)?; + self.write_frames(frames) } /// Finish the track, flushing the current group. @@ -384,26 +249,47 @@ impl Import { /// Close the current group and open the next one at `sequence`. /// - /// Any in-flight access unit is dropped. Pre-seek OBUs would otherwise leak + /// Any in-flight temporal unit is dropped. Pre-seek OBUs would otherwise leak /// into the post-seek group with the wrong timestamp. pub fn seek(&mut self, sequence: u64) -> Result<()> { - self.current = Frame::default(); + self.split.reset(); self.track.seek(sequence)?; Ok(()) } - /// True once the config is known and the catalog has been populated. - pub fn is_initialized(&self) -> bool { - self.config.is_some() - } + /// Write split frames to the track, resolving the config from the first + /// keyframe's inline sequence header and refining the catalog jitter. + fn write_frames(&mut self, frames: impl IntoIterator) -> Result<()> { + for frame in frames { + if frame.keyframe + && let Some(seq) = find_sequence_header(&frame.payload) + { + self.configure_from_seq(&seq)?; + } + + // A keyframe we couldn't configure (no sequence header) is undecodable. + if frame.keyframe && self.config.is_none() { + return Err(Error::MissingSequenceHeader.into()); + } + + let pts = frame.timestamp; + // A pre-keyframe delta has no group to anchor it: the producer returns + // MissingKeyframe, which a caller joining mid-stream skips. + self.track.write(frame)?; - fn pts(&mut self, hint: Option) -> Result { - if let Some(pts) = hint { - return Ok(pts); + if let Some(jitter) = self.jitter.observe(pts) + && let Some(c) = self.catalog.video.renditions.get_mut(self.track.name()) + { + c.jitter = Some(jitter); + } } + Ok(()) + } +} - let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); - Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) +impl FrameDecode for Import { + fn decode>(&mut self, frames: I) -> Result<()> { + self.write_frames(frames) } } @@ -413,90 +299,21 @@ impl Renditions for Import { } } -/// Iterator over AV1 Open Bitstream Units (OBUs) -struct ObuIterator<'a, T: Buf + AsRef<[u8]> + 'a> { - buf: &'a mut T, -} - -impl<'a, T: Buf + AsRef<[u8]> + 'a> ObuIterator<'a, T> { - pub fn new(buf: &'a mut T) -> Self { - Self { buf } - } - - pub fn flush(self) -> Result> { - let remaining = self.buf.remaining(); - if remaining == 0 { - return Ok(None); - } - - let obu = self.buf.copy_to_bytes(remaining); - Ok(Some(obu)) - } +fn is_sequence_header(obu: &[u8]) -> bool { + let mut reader = obu; + ObuHeader::parse(&mut reader) + .map(|h| h.obu_type == ObuType::SequenceHeader) + .unwrap_or(false) } -impl<'a, T: Buf + AsRef<[u8]> + 'a> Iterator for ObuIterator<'a, T> { - type Item = Result; - - fn next(&mut self) -> Option { - if self.buf.remaining() == 0 { - return None; - } - - // Parse OBU header to get size - let data = self.buf.as_ref(); - if data.is_empty() { - return None; - } - - // OBU header format: - // - obu_forbidden_bit (1) - // - obu_type (4) - // - obu_extension_flag (1) - // - obu_has_size_field (1) - // - obu_reserved_1bit (1) - - let header = data[0]; - let has_extension = (header >> 2) & 1 == 1; - let has_size = (header >> 1) & 1 == 1; - - if !has_size { - let remaining = self.buf.remaining(); - let obu = self.buf.copy_to_bytes(remaining); - return Some(Ok(obu)); - } - - // LEB128 size field starts after header byte and optional extension byte - let mut size: usize = 0; - let mut offset = if has_extension { 2 } else { 1 }; - let mut shift = 0; - - loop { - if offset >= data.len() { - return None; - } - - let byte = data[offset]; - offset += 1; - - size |= ((byte & 0x7F) as usize) << shift; - shift += 7; - - if byte & 0x80 == 0 { - break; - } - - if shift >= 56 { - return Some(Err(Error::ObuSizeTooLarge.into())); - } - } - - let total_size = offset + size; - - if total_size > self.buf.remaining() { - return None; +/// Find the first sequence-header OBU in a payload, if any. +fn find_sequence_header(payload: &[u8]) -> Option { + let mut buf = Bytes::copy_from_slice(payload); + let mut obus = ObuIterator::new(&mut buf); + while let Some(Ok(obu)) = obus.next() { + if is_sequence_header(&obu) { + return Some(obu); } - - let obu = self.buf.copy_to_bytes(total_size); - Some(Ok(obu)) } + obus.flush().ok().flatten().filter(|obu| is_sequence_header(obu)) } diff --git a/rs/moq-mux/src/codec/av1/mod.rs b/rs/moq-mux/src/codec/av1/mod.rs index a5f519564..075330326 100644 --- a/rs/moq-mux/src/codec/av1/mod.rs +++ b/rs/moq-mux/src/codec/av1/mod.rs @@ -5,8 +5,10 @@ //! raw AV1 bitstreams (OBU-framed) to a moq broadcast. mod import; +mod split; pub use import::*; +pub use split::*; use hang::catalog::AV1; diff --git a/rs/moq-mux/src/codec/av1/split.rs b/rs/moq-mux/src/codec/av1/split.rs new file mode 100644 index 000000000..4038785a0 --- /dev/null +++ b/rs/moq-mux/src/codec/av1/split.rs @@ -0,0 +1,349 @@ +//! AV1 OBU stream splitter. +//! +//! The AV1 analogue of [`crate::codec::h264::Split`]: turns a raw OBU byte +//! stream into [`crate::container::Frame`]s. It finds temporal-unit boundaries +//! and flags keyframes (a sequence header or a `KEY_FRAME`), and stamps +//! wall-clock timestamps when the caller has none (stdin). It owns no track, +//! catalog, or codec config. AV1 carries the sequence header inline ahead of +//! keyframes, so unlike H.264/H.265 there is nothing to cache or re-insert; the +//! importer parses the config out of the frames it emits. + +use bytes::{Buf, Bytes, BytesMut}; +use scuffle_av1::{ObuHeader, ObuType}; +use tokio::io::{AsyncRead, AsyncReadExt}; + +use super::Error; +use crate::Result; + +/// AV1 OBU stream splitter: bytes in, [`Frame`](crate::container::Frame)s out. +/// +/// Feed bytes via [`decode_stream`](Self::decode_stream) (unknown frame +/// boundaries, e.g. stdin), [`decode_frame`](Self::decode_frame) (one complete +/// temporal unit per call), or [`decode_from`](Self::decode_from) (an async +/// reader). Each returns the frames it produced. [`seed`](Self::seed) feeds +/// leading metadata OBUs (e.g. a sequence header) into the next frame. +pub struct Split { + current: Au, + zero: Option, + pending: Vec, +} + +#[derive(Default)] +struct Au { + chunks: BytesMut, + contains_keyframe: bool, + contains_frame: bool, +} + +impl Default for Split { + fn default() -> Self { + Self::new() + } +} + +impl Split { + /// A fresh splitter. + pub fn new() -> Self { + Self { + current: Au::default(), + zero: None, + pending: Vec::new(), + } + } + + /// Feed leading metadata OBUs (e.g. a sequence header) into the in-flight + /// access unit without completing a frame, so they prefix the next keyframe. + /// The buffer must not contain a completed frame (no timestamp is available). + pub fn seed>(&mut self, buf: &mut T) -> Result<()> { + let mut obus = ObuIterator::new(buf); + while let Some(obu) = obus.next().transpose()? { + self.decode_obu(obu, None)?; + } + if let Some(obu) = obus.flush()? { + self.decode_obu(obu, None)?; + } + Ok(()) + } + + /// Decode from an asynchronous reader, returning all frames produced. + pub async fn decode_from(&mut self, reader: &mut T) -> Result> { + let mut frames = Vec::new(); + let mut buffer = BytesMut::new(); + while reader.read_buf(&mut buffer).await? > 0 { + frames.extend(self.decode_stream(&mut buffer, None)?); + } + Ok(frames) + } + + /// Decode a buffer where frame boundaries are unknown, returning the frames + /// it produced. The final temporal unit stays buffered until the next call + /// (or [`decode_frame`](Self::decode_frame)). + pub fn decode_stream>( + &mut self, + buf: &mut T, + pts: Option, + ) -> Result> { + let obus = ObuIterator::new(buf); + for obu in obus { + // Resolve a timestamp per OBU so a wall-clock stream doesn't reuse one. + let pts = self.pts(pts)?; + self.decode_obu(obu?, Some(pts))?; + } + Ok(std::mem::take(&mut self.pending)) + } + + /// Decode a buffer holding one complete temporal unit, returning the frames + /// it produced. The unit is flushed before returning. + pub fn decode_frame>( + &mut self, + buf: &mut T, + pts: Option, + ) -> Result> { + let pts = self.pts(pts)?; + let mut obus = ObuIterator::new(buf); + while let Some(obu) = obus.next().transpose()? { + self.decode_obu(obu, Some(pts))?; + } + if let Some(obu) = obus.flush()? { + self.decode_obu(obu, Some(pts))?; + } + self.maybe_start_frame(Some(pts))?; + Ok(std::mem::take(&mut self.pending)) + } + + fn decode_obu(&mut self, obu_data: Bytes, pts: Option) -> Result<()> { + if obu_data.is_empty() { + return Err(Error::ObuTooShort.into()); + } + + // Parse the OBU header to learn the type; the payload offset is whatever + // the parser consumed (header + optional extension + LEB128 size). + let mut reader = &obu_data[..]; + let header = ObuHeader::parse(&mut reader)?; + let payload_offset = obu_data.len() - reader.len(); + + match header.obu_type { + // A sequence header anchors a keyframe; the importer parses the config. + ObuType::SequenceHeader => { + self.current.contains_keyframe = true; + } + ObuType::TemporalDelimiter => { + self.maybe_start_frame(pts)?; + } + ObuType::FrameHeader | ObuType::Frame => { + let is_keyframe = obu_data.get(payload_offset).is_some_and(|first_byte| { + let show_existing_frame = (first_byte >> 7) & 1; + if show_existing_frame == 1 { + self.current.contains_keyframe + } else { + let frame_type = (first_byte >> 5) & 0b11; + frame_type == 0 // KEY_FRAME + } + }); + + if is_keyframe { + self.current.contains_keyframe = true; + } + self.current.contains_frame = true; + } + ObuType::Metadata => { + self.maybe_start_frame(pts)?; + } + ObuType::TileGroup | ObuType::TileList => { + self.current.contains_frame = true; + } + _ => {} + } + + tracing::trace!(?header.obu_type, "parsed OBU"); + + self.current.chunks.extend_from_slice(&obu_data); + Ok(()) + } + + fn maybe_start_frame(&mut self, pts: Option) -> Result<()> { + if !self.current.contains_frame { + return Ok(()); + } + + let pts = pts.ok_or(Error::MissingTimestamp)?; + let keyframe = self.current.contains_keyframe; + let payload = std::mem::take(&mut self.current.chunks).freeze(); + self.current.contains_keyframe = false; + self.current.contains_frame = false; + + self.pending.push(crate::container::Frame { + timestamp: pts, + payload, + keyframe, + duration: None, + }); + Ok(()) + } + + /// Drop any in-flight temporal unit. + /// + /// Pre-reset OBUs would otherwise leak into a later frame with the wrong + /// timestamp. + pub fn reset(&mut self) { + self.current = Au::default(); + } + + fn pts(&mut self, hint: Option) -> Result { + if let Some(pts) = hint { + return Ok(pts); + } + let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); + Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) + } +} + +/// Iterator over AV1 Open Bitstream Units (OBUs). +pub(super) struct ObuIterator<'a, T: Buf + AsRef<[u8]> + 'a> { + buf: &'a mut T, +} + +impl<'a, T: Buf + AsRef<[u8]> + 'a> ObuIterator<'a, T> { + pub fn new(buf: &'a mut T) -> Self { + Self { buf } + } + + pub fn flush(self) -> Result> { + let remaining = self.buf.remaining(); + if remaining == 0 { + return Ok(None); + } + Ok(Some(self.buf.copy_to_bytes(remaining))) + } +} + +impl<'a, T: Buf + AsRef<[u8]> + 'a> Iterator for ObuIterator<'a, T> { + type Item = Result; + + fn next(&mut self) -> Option { + if self.buf.remaining() == 0 { + return None; + } + + let data = self.buf.as_ref(); + if data.is_empty() { + return None; + } + + // OBU header: forbidden(1) | type(4) | extension_flag(1) | has_size(1) | reserved(1) + let header = data[0]; + let has_extension = (header >> 2) & 1 == 1; + let has_size = (header >> 1) & 1 == 1; + + if !has_size { + let remaining = self.buf.remaining(); + return Some(Ok(self.buf.copy_to_bytes(remaining))); + } + + // LEB128 size field follows the header byte and optional extension byte. + let mut size: usize = 0; + let mut offset = if has_extension { 2 } else { 1 }; + let mut shift = 0; + + loop { + if offset >= data.len() { + return None; + } + + let byte = data[offset]; + offset += 1; + + size |= ((byte & 0x7F) as usize) << shift; + shift += 7; + + if byte & 0x80 == 0 { + break; + } + if shift >= 56 { + return Some(Err(Error::ObuSizeTooLarge.into())); + } + } + + let total_size = offset + size; + if total_size > self.buf.remaining() { + return None; + } + + Some(Ok(self.buf.copy_to_bytes(total_size))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // OBU header byte: forbidden(0) | type(4) | extension_flag(0) | has_size(1) | reserved(0). + fn obu(obu_type: u8, payload: &[u8]) -> Vec { + let mut o = vec![(obu_type << 3) | 0b010, payload.len() as u8]; + o.extend_from_slice(payload); + o + } + + fn td() -> Vec { + obu(2, &[]) // OBU_TEMPORAL_DELIMITER + } + fn seq_header() -> Vec { + obu(1, &[0xaa, 0xbb]) // OBU_SEQUENCE_HEADER (payload not parsed by the splitter) + } + fn key_frame() -> Vec { + obu(6, &[0x00, 0x11]) // OBU_FRAME, first byte: show_existing=0, frame_type=0 (KEY_FRAME) + } + fn inter_frame() -> Vec { + obu(6, &[0x20, 0x11]) // OBU_FRAME, first byte: frame_type=1 (INTER_FRAME) + } + + fn cat(parts: &[Vec]) -> BytesMut { + let mut buf = BytesMut::new(); + for p in parts { + buf.extend_from_slice(p); + } + buf + } + + fn ts() -> moq_net::Timestamp { + moq_net::Timestamp::from_micros(0).unwrap() + } + + /// A temporal unit with a sequence header + KEY_FRAME emits one keyframe. + #[tokio::test(start_paused = true)] + async fn decode_frame_keyframe() { + let mut split = Split::new(); + let frames = split + .decode_frame(&mut cat(&[td(), seq_header(), key_frame()]), Some(ts())) + .unwrap(); + assert_eq!(frames.len(), 1); + assert!(frames[0].keyframe); + } + + /// A frame with no sequence header and INTER frame_type is not a keyframe. + #[tokio::test(start_paused = true)] + async fn decode_frame_delta_is_not_keyframe() { + let mut split = Split::new(); + let frames = split + .decode_frame(&mut cat(&[td(), inter_frame()]), Some(ts())) + .unwrap(); + assert_eq!(frames.len(), 1); + assert!(!frames[0].keyframe); + } + + /// In streaming mode the next temporal delimiter closes the previous unit, so + /// the trailing one stays buffered. + #[tokio::test(start_paused = true)] + async fn decode_stream_emits_on_next_boundary() { + let mut split = Split::new(); + let frames = split + .decode_stream( + &mut cat(&[td(), seq_header(), key_frame(), td(), inter_frame()]), + Some(ts()), + ) + .unwrap(); + // Only the keyframe is complete; the inter frame waits for the next TD. + assert_eq!(frames.len(), 1); + assert!(frames[0].keyframe); + } +} From f4a278a964b6dda6112c7ed733ff63b62d71f295 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 18:56:55 +0000 Subject: [PATCH 11/19] moq-mux: fix stale lenient-start comments in the h264 importer The lenient-start drop was replaced by the producer's MissingKeyframe; update the two h264 comments that still described the old behavior. --- rs/moq-mux/src/codec/h264/import.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index 50d9ff127..be8783932 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -310,8 +310,8 @@ impl Import { if self.config.is_none() { // A keyframe we still can't configure is undecodable, so bail // loudly. A non-keyframe before config is a mid-stream-join - // leftover: write it and let the producer's lenient start drop it - // ahead of the first keyframe. + // leftover: write it through, and the producer reports + // MissingKeyframe (which a mid-stream join skips). if frame.keyframe { return Err(Error::NotInitialized.into()); } @@ -541,10 +541,9 @@ mod tests { assert!(matches!(err, crate::Error::H264(Error::NotInitialized)), "got {err:?}"); } - /// A non-keyframe before any config is a mid-stream-join leftover: it must - /// not abort the import (the producer's lenient start drops it downstream). /// A non-keyframe before the first keyframe has no group to anchor it, so the - /// producer surfaces MissingKeyframe (which a mid-stream join skips). + /// producer surfaces MissingKeyframe (which a mid-stream join skips). It must + /// not silently abort the import. #[tokio::test(start_paused = true)] async fn delta_before_init_reports_missing_keyframe() { let pslice: &[u8] = &[0x61, 0xe0, 0x12, 0x34]; // non-IDR slice From db6e812770d68b3878e66744041b0cdfaf3005ab Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 20:35:58 +0000 Subject: [PATCH 12/19] moq-mux: drop the fixed-track concept; config updates in place Review follow-up. There are no fixed tracks anymore, so a changed codec config just re-mirrors the catalog rendition instead of erroring. - Remove the `FixedTrackReconfigured` error variant from the h264, h265, and av1 error enums. - h264: drop `set_config`; avc1 (avcC) and avc3 (inline SPS) both resolve through one in-place `apply_config` that no-ops on an unchanged config. - h265 / av1: `configure_from_sps` / `apply_config` update the rendition in place on a change. - av1 `Split::decode_stream` / `decode_frame` take `impl Into>`, matching the h264 and h265 splitters. https://claude.ai/code/session_011S7pzcg2XsPP3AExuymac1 --- rs/moq-mux/src/codec/av1/import.rs | 26 +++++++++++++------------- rs/moq-mux/src/codec/av1/mod.rs | 2 -- rs/moq-mux/src/codec/av1/split.rs | 9 +++++---- rs/moq-mux/src/codec/h264/import.rs | 23 +++++++---------------- rs/moq-mux/src/codec/h264/mod.rs | 2 -- rs/moq-mux/src/codec/h265/import.rs | 15 ++++++--------- rs/moq-mux/src/codec/h265/mod.rs | 2 -- 7 files changed, 31 insertions(+), 48 deletions(-) diff --git a/rs/moq-mux/src/codec/av1/import.rs b/rs/moq-mux/src/codec/av1/import.rs index ce92ebc66..839bed1e4 100644 --- a/rs/moq-mux/src/codec/av1/import.rs +++ b/rs/moq-mux/src/codec/av1/import.rs @@ -106,7 +106,8 @@ impl Import { full_range: false, }); config.container = hang::catalog::Container::Legacy; - self.apply_config(config) + self.apply_config(config); + Ok(()) } fn init(&mut self, seq_header: &SequenceHeaderObu) -> Result<()> { @@ -140,7 +141,8 @@ impl Import { config.coded_width = Some(seq_header.max_frame_width as u32); config.coded_height = Some(seq_header.max_frame_height as u32); config.container = hang::catalog::Container::Legacy; - self.apply_config(config) + self.apply_config(config); + Ok(()) } /// Minimal config when sequence-header parsing fails, so the stream can still @@ -161,26 +163,24 @@ impl Import { full_range: false, }); config.container = hang::catalog::Container::Legacy; - self.apply_config(config) + self.apply_config(config); + Ok(()) } - /// Store a resolved config. The first one is kept; a different one would need - /// a new init segment, which the single fixed track can't represent. - fn apply_config(&mut self, config: hang::catalog::VideoConfig) -> Result<()> { - if let Some(old) = &self.config { - if old == &config { - return Ok(()); - } - return Err(Error::FixedTrackReconfigured.into()); + /// Apply a resolved config, updating the catalog rendition in place. + /// + /// A changed config just re-mirrors the rendition; there are no fixed tracks + /// to reject a reconfiguration. + fn apply_config(&mut self, config: hang::catalog::VideoConfig) { + if self.config.as_ref() == Some(&config) { + return; } - tracing::debug!(name = ?self.track.name(), ?config, "starting track"); self.catalog .video .renditions .insert(self.track.name().to_string(), config.clone()); self.config = Some(config); - Ok(()) } /// Resolve the config from a sequence-header OBU, falling back to a minimal diff --git a/rs/moq-mux/src/codec/av1/mod.rs b/rs/moq-mux/src/codec/av1/mod.rs index 075330326..1b08bef59 100644 --- a/rs/moq-mux/src/codec/av1/mod.rs +++ b/rs/moq-mux/src/codec/av1/mod.rs @@ -25,8 +25,6 @@ pub enum Error { #[error("not initialized")] NotInitialized, - #[error("fixed track cannot be reconfigured")] - FixedTrackReconfigured, #[error("expected sequence header before any frames")] MissingSequenceHeader, diff --git a/rs/moq-mux/src/codec/av1/split.rs b/rs/moq-mux/src/codec/av1/split.rs index 4038785a0..cc1a3b0c0 100644 --- a/rs/moq-mux/src/codec/av1/split.rs +++ b/rs/moq-mux/src/codec/av1/split.rs @@ -81,12 +81,13 @@ impl Split { pub fn decode_stream>( &mut self, buf: &mut T, - pts: Option, + pts: impl Into>, ) -> Result> { + let hint = pts.into(); let obus = ObuIterator::new(buf); for obu in obus { // Resolve a timestamp per OBU so a wall-clock stream doesn't reuse one. - let pts = self.pts(pts)?; + let pts = self.pts(hint)?; self.decode_obu(obu?, Some(pts))?; } Ok(std::mem::take(&mut self.pending)) @@ -97,9 +98,9 @@ impl Split { pub fn decode_frame>( &mut self, buf: &mut T, - pts: Option, + pts: impl Into>, ) -> Result> { - let pts = self.pts(pts)?; + let pts = self.pts(pts.into())?; let mut obus = ObuIterator::new(buf); while let Some(obu) = obus.next().transpose()? { self.decode_obu(obu, Some(pts))?; diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index be8783932..980123012 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -137,7 +137,7 @@ impl Import { config.description = Some(Bytes::copy_from_slice(avcc_bytes)); config.container = hang::catalog::Container::Legacy; - self.set_config(config)?; + self.apply_config(config); buf.advance(buf.remaining()); Ok(()) } @@ -269,23 +269,14 @@ impl Import { Ok(()) } - /// Resolve the avc1 config from an avcC. + /// Apply a resolved config, updating the catalog rendition in place. /// - /// The first config is stored. A different avcC would mean a new init - /// segment, which a single fixed track can't represent, so a reconfiguration - /// is an error (mint a new track via a fresh importer). - fn set_config(&mut self, config: hang::catalog::VideoConfig) -> Result<()> { - if let Some(old) = &self.config { - if old == &config { - return Ok(()); - } - return Err(Error::FixedTrackReconfigured.into()); - } - self.apply_config(config); - Ok(()) - } - + /// A changed config (new avcC, or a new inline SPS) just re-mirrors the + /// rendition; there are no fixed tracks to reject a reconfiguration. fn apply_config(&mut self, config: hang::catalog::VideoConfig) { + if self.config.as_ref() == Some(&config) { + return; + } tracing::debug!(?config, "starting H.264 track"); self.catalog .video diff --git a/rs/moq-mux/src/codec/h264/mod.rs b/rs/moq-mux/src/codec/h264/mod.rs index 636956b52..7521a25aa 100644 --- a/rs/moq-mux/src/codec/h264/mod.rs +++ b/rs/moq-mux/src/codec/h264/mod.rs @@ -59,8 +59,6 @@ pub enum Error { #[error("not initialized; call initialize() or with_mode() first")] NotInitialized, - #[error("fixed track cannot be reconfigured")] - FixedTrackReconfigured, #[error("avc3 track not created")] Avc3TrackNotCreated, diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index 8b7967270..ab6233747 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -138,10 +138,8 @@ impl Import { Ok(()) } - /// Resolve the config from an inline SPS. - /// - /// A different SPS would need a new init segment, which the single fixed - /// track can't represent, so a reconfiguration is an error. + /// Resolve the config from an inline SPS, updating the rendition in place on a + /// change. fn configure_from_sps(&mut self, sps_nal: &Bytes) -> Result<()> { if self.last_sps.as_ref() == Some(sps_nal) { return Ok(()); @@ -169,11 +167,10 @@ impl Import { self.last_sps = Some(sps_nal.clone()); - if let Some(old) = &self.config { - if old == &config { - return Ok(()); - } - return Err(Error::FixedTrackReconfigured.into()); + // A changed SPS just re-mirrors the rendition in place; there are no fixed + // tracks to reject a reconfiguration. + if self.config.as_ref() == Some(&config) { + return Ok(()); } let track_name = self.track.name().to_string(); diff --git a/rs/moq-mux/src/codec/h265/mod.rs b/rs/moq-mux/src/codec/h265/mod.rs index 37e84ccd2..9d3e777d5 100644 --- a/rs/moq-mux/src/codec/h265/mod.rs +++ b/rs/moq-mux/src/codec/h265/mod.rs @@ -42,8 +42,6 @@ pub enum Error { #[error("not initialized")] NotInitialized, - #[error("fixed track cannot be reconfigured")] - FixedTrackReconfigured, #[error("expected SPS before any frames")] MissingSps, From de2d20179ed732206e263502301c771309eb330d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 20:42:25 +0000 Subject: [PATCH 13/19] moq-mux: convert the importers off anyhow to thiserror Review follow-up: a library crate shouldn't surface anyhow. Port the single-track codec importers that still used `anyhow::Result` to the crate's thiserror-based `crate::Result`, mirroring h264/h265/av1. - New `vp8::Error`, `vp9::Error`, and a `legacy::Error` (covering the MP2/AC-3/E-AC-3 header parsers), each wired into `crate::Error` via `#[from]` (`Vp8`/`Vp9`/`Legacy`). - `FrameHeader::parse` (vp8/vp9), the vp9 `BitReader`, the `ac3`/`eac3`/`mp2` `parse_header`s, and the vp8/vp9/legacy importer methods all return the typed errors now; no `anyhow::ensure!`/`bail!` remain in these modules. - Dropped the vp8/vp9 "fixed track cannot be reconfigured" bail too, so they update the rendition in place like the other codecs. The dispatcher test that asserted the old error now asserts in-place reconfiguration. - With every importer on `crate::Result`, `Published::decoding` drops its generic error parameter and just takes a `crate::Result` closure. https://claude.ai/code/session_011S7pzcg2XsPP3AExuymac1 --- rs/moq-mux/src/codec/ac3.rs | 20 +++++--- rs/moq-mux/src/codec/av1/mod.rs | 1 - rs/moq-mux/src/codec/eac3.rs | 40 ++++++++++------ rs/moq-mux/src/codec/h264/mod.rs | 1 - rs/moq-mux/src/codec/h265/mod.rs | 1 - rs/moq-mux/src/codec/legacy.rs | 75 ++++++++++++++++++++++++++++-- rs/moq-mux/src/codec/mp2.rs | 24 +++++++--- rs/moq-mux/src/codec/vp8/import.rs | 20 ++++---- rs/moq-mux/src/codec/vp8/mod.rs | 37 ++++++++++++--- rs/moq-mux/src/codec/vp9/import.rs | 20 ++++---- rs/moq-mux/src/codec/vp9/mod.rs | 38 ++++++++++++--- rs/moq-mux/src/error.rs | 12 +++++ rs/moq-mux/src/import.rs | 17 +++---- rs/moq-mux/src/publish.rs | 5 +- 14 files changed, 228 insertions(+), 83 deletions(-) diff --git a/rs/moq-mux/src/codec/ac3.rs b/rs/moq-mux/src/codec/ac3.rs index fd4a34686..a2b792acd 100644 --- a/rs/moq-mux/src/codec/ac3.rs +++ b/rs/moq-mux/src/codec/ac3.rs @@ -25,18 +25,26 @@ const CHANNELS: [u32; 8] = [2, 1, 2, 3, 3, 4, 4, 5]; const SAMPLES_PER_FRAME: u64 = 1536; /// Parse an AC-3 sync frame header from the start of `data` (needs >= 7 bytes). -pub(crate) fn parse_header(data: &[u8]) -> anyhow::Result { - anyhow::ensure!(data.len() >= 7, "AC-3 header needs 7 bytes"); - anyhow::ensure!(data[0] == 0x0B && data[1] == 0x77, "missing AC-3 sync word"); +pub(crate) fn parse_header(data: &[u8]) -> legacy::Result { + if data.len() < 7 { + return Err(legacy::Error::Ac3HeaderTooShort); + } + if !(data[0] == 0x0B && data[1] == 0x77) { + return Err(legacy::Error::Ac3MissingSyncWord); + } let fscod = data[4] >> 6; let frmsizecod = (data[4] & 0x3F) as usize; - anyhow::ensure!(frmsizecod <= 37, "invalid AC-3 frame size code"); + if frmsizecod > 37 { + return Err(legacy::Error::Ac3InvalidFrameSizeCode); + } let bitrate_kbps = BITRATE[frmsizecod >> 1] as usize; // bsid > 8 is E-AC-3 or a low-sample-rate variant, neither parsed here. let bsid = data[5] >> 3; - anyhow::ensure!(bsid <= 8, "unsupported AC-3 bsid {bsid}"); + if bsid > 8 { + return Err(legacy::Error::Ac3UnsupportedBsid(bsid)); + } // At 44.1 kHz the frame doesn't divide evenly, so the low frmsizecod bit // selects the padded size (A/52 Table 5.18). @@ -44,7 +52,7 @@ pub(crate) fn parse_header(data: &[u8]) -> anyhow::Result { 0b00 => (48000, 4 * bitrate_kbps), 0b01 => (44100, 2 * (320 * bitrate_kbps / 147 + (frmsizecod & 1))), 0b10 => (32000, 6 * bitrate_kbps), - _ => anyhow::bail!("reserved AC-3 sample-rate code"), + _ => return Err(legacy::Error::Ac3ReservedSampleRate), }; // acmod decides which mix-level fields precede lfeon; skip them bit by bit. diff --git a/rs/moq-mux/src/codec/av1/mod.rs b/rs/moq-mux/src/codec/av1/mod.rs index 1b08bef59..8a602b81a 100644 --- a/rs/moq-mux/src/codec/av1/mod.rs +++ b/rs/moq-mux/src/codec/av1/mod.rs @@ -25,7 +25,6 @@ pub enum Error { #[error("not initialized")] NotInitialized, - #[error("expected sequence header before any frames")] MissingSequenceHeader, diff --git a/rs/moq-mux/src/codec/eac3.rs b/rs/moq-mux/src/codec/eac3.rs index 920766fcd..621e0bb4c 100644 --- a/rs/moq-mux/src/codec/eac3.rs +++ b/rs/moq-mux/src/codec/eac3.rs @@ -26,41 +26,51 @@ const BLOCKS: [u64; 4] = [1, 2, 3, 6]; const CHANNELS: [u32; 8] = [2, 1, 2, 3, 3, 4, 4, 5]; /// Parse an E-AC-3 sync frame header from the start of `data` (needs >= 6 bytes). -pub(crate) fn parse_header(data: &[u8]) -> anyhow::Result { - anyhow::ensure!(data.len() >= 6, "E-AC-3 header needs 6 bytes"); - anyhow::ensure!(data[0] == 0x0B && data[1] == 0x77, "missing E-AC-3 sync word"); +pub(crate) fn parse_header(data: &[u8]) -> legacy::Result { + if data.len() < 6 { + return Err(legacy::Error::Eac3HeaderTooShort); + } + if !(data[0] == 0x0B && data[1] == 0x77) { + return Err(legacy::Error::Eac3MissingSyncWord); + } // bsid 11..=16 is E-AC-3; plain AC-3 (bsid <= 8) is routed by stream_type and // never reaches this parser. let bsid = data[5] >> 3; - anyhow::ensure!((11..=16).contains(&bsid), "not an E-AC-3 bitstream (bsid {bsid})"); + if !(11..=16).contains(&bsid) { + return Err(legacy::Error::Eac3NotEac3Bsid(bsid)); + } // A dependent substream (strmtyp 1) extends a prior program to 7.1+, and a // nonzero substreamid carries additional programs in the same PID. Either // would make this track's channel count a lie, so they are rejected rather // than mis-described. let strmtyp = data[2] >> 6; - anyhow::ensure!(strmtyp != 3, "reserved E-AC-3 stream type"); - anyhow::ensure!( - strmtyp != 1, - "E-AC-3 dependent substream (7.1+ layout) is not supported; only a single independent substream" - ); + if strmtyp == 3 { + return Err(legacy::Error::Eac3ReservedStreamType); + } + if strmtyp == 1 { + return Err(legacy::Error::Eac3DependentSubstream); + } let substreamid = (data[2] >> 3) & 0x07; - anyhow::ensure!( - substreamid == 0, - "E-AC-3 additional substream {substreamid} is not supported; only a single independent substream" - ); + if substreamid != 0 { + return Err(legacy::Error::Eac3AdditionalSubstream(substreamid)); + } let frmsiz = (((data[2] & 0x07) as usize) << 8) | data[3] as usize; let len = (frmsiz + 1) * 2; // frmsiz is a raw field; corrupt input can declare a "frame" shorter than the // header just parsed, which would surface later as a confusing sync error. - anyhow::ensure!(len >= 6, "E-AC-3 frame length {len} shorter than its header"); + if len < 6 { + return Err(legacy::Error::Eac3FrameShorterThanHeader(len)); + } let fscod = data[4] >> 6; let (sample_rate, blocks) = if fscod == 0b11 { let fscod2 = (data[4] >> 4) & 0x03; - anyhow::ensure!(fscod2 != 3, "reserved E-AC-3 sample-rate code"); + if fscod2 == 3 { + return Err(legacy::Error::Eac3ReservedSampleRate); + } // Reduced rates always run 6 blocks. (SAMPLE_RATE_REDUCED[fscod2 as usize], 6) } else { diff --git a/rs/moq-mux/src/codec/h264/mod.rs b/rs/moq-mux/src/codec/h264/mod.rs index 7521a25aa..3d16f7459 100644 --- a/rs/moq-mux/src/codec/h264/mod.rs +++ b/rs/moq-mux/src/codec/h264/mod.rs @@ -59,7 +59,6 @@ pub enum Error { #[error("not initialized; call initialize() or with_mode() first")] NotInitialized, - #[error("avc3 track not created")] Avc3TrackNotCreated, diff --git a/rs/moq-mux/src/codec/h265/mod.rs b/rs/moq-mux/src/codec/h265/mod.rs index 9d3e777d5..7c9648df6 100644 --- a/rs/moq-mux/src/codec/h265/mod.rs +++ b/rs/moq-mux/src/codec/h265/mod.rs @@ -42,7 +42,6 @@ pub enum Error { #[error("not initialized")] NotInitialized, - #[error("expected SPS before any frames")] MissingSps, diff --git a/rs/moq-mux/src/codec/legacy.rs b/rs/moq-mux/src/codec/legacy.rs index ccaa51e61..3c9c09620 100644 --- a/rs/moq-mux/src/codec/legacy.rs +++ b/rs/moq-mux/src/codec/legacy.rs @@ -12,6 +12,71 @@ use moq_net::Timestamp; use crate::publish::Renditions; +/// Legacy audio (MP2 / AC-3 / E-AC-3) header parsing errors. +#[derive(Debug, Clone, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + #[error("AC-3 header needs 7 bytes")] + Ac3HeaderTooShort, + + #[error("missing AC-3 sync word")] + Ac3MissingSyncWord, + + #[error("invalid AC-3 frame size code")] + Ac3InvalidFrameSizeCode, + + #[error("unsupported AC-3 bsid {0}")] + Ac3UnsupportedBsid(u8), + + #[error("reserved AC-3 sample-rate code")] + Ac3ReservedSampleRate, + + #[error("E-AC-3 header needs 6 bytes")] + Eac3HeaderTooShort, + + #[error("missing E-AC-3 sync word")] + Eac3MissingSyncWord, + + #[error("not an E-AC-3 bitstream (bsid {0})")] + Eac3NotEac3Bsid(u8), + + #[error("reserved E-AC-3 stream type")] + Eac3ReservedStreamType, + + #[error("E-AC-3 dependent substream (7.1+ layout) is not supported; only a single independent substream")] + Eac3DependentSubstream, + + #[error("E-AC-3 additional substream {0} is not supported; only a single independent substream")] + Eac3AdditionalSubstream(u8), + + #[error("E-AC-3 frame length {0} shorter than its header")] + Eac3FrameShorterThanHeader(usize), + + #[error("reserved E-AC-3 sample-rate code")] + Eac3ReservedSampleRate, + + #[error("MP2 header needs 4 bytes")] + Mp2HeaderTooShort, + + #[error("missing MP2 frame sync")] + Mp2MissingSync, + + #[error("reserved or MPEG-2.5 audio version")] + Mp2ReservedVersion, + + #[error("not MPEG Layer II")] + Mp2NotLayerII, + + #[error("reserved MP2 sample-rate index")] + Mp2ReservedSampleRate, + + #[error("free-format or invalid MP2 bitrate")] + Mp2InvalidBitrate, +} + +/// A Result type alias for legacy audio header parsing. +pub type Result = std::result::Result; + /// A parsed legacy-audio frame header. #[derive(Debug)] pub(crate) struct Header { @@ -33,7 +98,7 @@ pub(crate) struct Descriptor { /// Bytes needed to attempt a header parse. pub min_header_len: usize, /// Parse one frame header at the start of the slice. - pub parse: fn(&[u8]) -> anyhow::Result
, + pub parse: fn(&[u8]) -> Result
, } /// Catalog config for a legacy audio track. Both fields come from the frame @@ -78,19 +143,19 @@ impl Import { } /// Finish the track, flushing the current group. - pub fn finish(&mut self) -> anyhow::Result<()> { + pub fn finish(&mut self) -> crate::Result<()> { self.track.finish()?; Ok(()) } /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> anyhow::Result<()> { + pub fn seek(&mut self, sequence: u64) -> crate::Result<()> { self.track.seek(sequence)?; Ok(()) } /// Publish one whole frame as a hang frame in its own group. - pub fn decode(&mut self, buf: &mut T, pts: Option) -> anyhow::Result<()> { + pub fn decode(&mut self, buf: &mut T, pts: Option) -> crate::Result<()> { let pts = self.pts(pts)?; let mut payload = BytesMut::with_capacity(buf.remaining()); @@ -114,7 +179,7 @@ impl Import { Ok(()) } - fn pts(&mut self, hint: Option) -> anyhow::Result { + fn pts(&mut self, hint: Option) -> crate::Result { if let Some(pts) = hint { return Ok(pts); } diff --git a/rs/moq-mux/src/codec/mp2.rs b/rs/moq-mux/src/codec/mp2.rs index cd8690baa..0b5610402 100644 --- a/rs/moq-mux/src/codec/mp2.rs +++ b/rs/moq-mux/src/codec/mp2.rs @@ -32,10 +32,14 @@ const SAMPLE_RATE: [[u32; 3]; 2] = [[44100, 48000, 32000], [22050, 24000, 16000] const SAMPLES_PER_FRAME: u64 = 1152; /// Parse a Layer II frame header from the start of `data` (needs >= 4 bytes). -pub(crate) fn parse_header(data: &[u8]) -> anyhow::Result { - anyhow::ensure!(data.len() >= 4, "MP2 header needs 4 bytes"); +pub(crate) fn parse_header(data: &[u8]) -> legacy::Result { + if data.len() < 4 { + return Err(legacy::Error::Mp2HeaderTooShort); + } // Frame sync: 11 bits set (0xFFE). - anyhow::ensure!(data[0] == 0xFF && (data[1] & 0xE0) == 0xE0, "missing MP2 frame sync"); + if !(data[0] == 0xFF && (data[1] & 0xE0) == 0xE0) { + return Err(legacy::Error::Mp2MissingSync); + } // 0b00 is the unofficial MPEG-2.5 extension: 13818-1 has no stream type for // it, so accepting it here would re-announce it as 0x04 on export and invent @@ -44,23 +48,29 @@ pub(crate) fn parse_header(data: &[u8]) -> anyhow::Result { let (version, sr_row) = match (data[1] >> 3) & 0x03 { 0b11 => (Version::Mpeg1, 0), 0b10 => (Version::Mpeg2, 1), - _ => anyhow::bail!("reserved or MPEG-2.5 audio version"), + _ => return Err(legacy::Error::Mp2ReservedVersion), }; // Layer field 0b10 is Layer II. - anyhow::ensure!((data[1] >> 1) & 0x03 == 0b10, "not MPEG Layer II"); + if (data[1] >> 1) & 0x03 != 0b10 { + return Err(legacy::Error::Mp2NotLayerII); + } let bitrate_index = (data[2] >> 4) & 0x0F; let sr_index = (data[2] >> 2) & 0x03; let padding = ((data[2] >> 1) & 0x01) as usize; - anyhow::ensure!(sr_index != 3, "reserved MP2 sample-rate index"); + if sr_index == 3 { + return Err(legacy::Error::Mp2ReservedSampleRate); + } let sample_rate = SAMPLE_RATE[sr_row][sr_index as usize]; let bitrate_kbps = match version { Version::Mpeg1 => BITRATE_MPEG1_L2[bitrate_index as usize], Version::Mpeg2 => BITRATE_MPEG2_L2[bitrate_index as usize], }; - anyhow::ensure!(bitrate_kbps != 0, "free-format or invalid MP2 bitrate"); + if bitrate_kbps == 0 { + return Err(legacy::Error::Mp2InvalidBitrate); + } // Layer II is always 1152 samples, so the frame is 144 * bitrate / sample_rate bytes. let len = (144 * bitrate_kbps * 1000 / sample_rate) as usize + padding; diff --git a/rs/moq-mux/src/codec/vp8/import.rs b/rs/moq-mux/src/codec/vp8/import.rs index 3261bf0b8..b4a21fb1f 100644 --- a/rs/moq-mux/src/codec/vp8/import.rs +++ b/rs/moq-mux/src/codec/vp8/import.rs @@ -53,14 +53,14 @@ impl Import { /// with an empty buffer (gstreamer / ffi pass `Bytes::new()`) and the catalog /// is filled from the first key frame. If the caller does pass the first frame /// here, it's decoded so nothing is dropped. - pub fn initialize>(&mut self, buf: &mut T) -> anyhow::Result<()> { + pub fn initialize>(&mut self, buf: &mut T) -> crate::Result<()> { if buf.has_remaining() { self.decode_frame(buf, None)?; } Ok(()) } - fn init(&mut self, width: u16, height: u16) -> anyhow::Result<()> { + fn init(&mut self, width: u16, height: u16) -> crate::Result<()> { let mut config = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); config.coded_width = Some(width as u32); config.coded_height = Some(height as u32); @@ -70,10 +70,6 @@ impl Import { return Ok(()); } - if self.config.is_some() { - anyhow::bail!("fixed track cannot be reconfigured"); - } - tracing::debug!(name = ?self.track.name(), ?config, "starting track"); self.catalog .video @@ -89,9 +85,11 @@ impl Import { &mut self, buf: &mut T, pts: Option, - ) -> anyhow::Result<()> { + ) -> crate::Result<()> { let payload = buf.copy_to_bytes(buf.remaining()); - anyhow::ensure!(!payload.is_empty(), "empty VP8 frame"); + if payload.is_empty() { + return Err(super::Error::EmptyFrame.into()); + } let header = FrameHeader::parse(&payload)?; if let Some((width, height)) = header.dimensions { @@ -126,13 +124,13 @@ impl Import { } /// Finish the track, flushing the current group. - pub fn finish(&mut self) -> anyhow::Result<()> { + pub fn finish(&mut self) -> crate::Result<()> { self.track.finish()?; Ok(()) } /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> anyhow::Result<()> { + pub fn seek(&mut self, sequence: u64) -> crate::Result<()> { self.track.seek(sequence)?; Ok(()) } @@ -142,7 +140,7 @@ impl Import { self.config.is_some() } - fn pts(&mut self, hint: Option) -> anyhow::Result { + fn pts(&mut self, hint: Option) -> crate::Result { if let Some(pts) = hint { return Ok(pts); } diff --git a/rs/moq-mux/src/codec/vp8/mod.rs b/rs/moq-mux/src/codec/vp8/mod.rs index d48e3b51e..da9559aed 100644 --- a/rs/moq-mux/src/codec/vp8/mod.rs +++ b/rs/moq-mux/src/codec/vp8/mod.rs @@ -11,6 +11,26 @@ mod import; pub use import::*; +/// VP8 parsing errors. +#[derive(Debug, Clone, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + #[error("VP8 frame too short for tag")] + FrameTooShort, + + #[error("VP8 key frame too short for header")] + KeyframeHeaderTooShort, + + #[error("VP8 key frame start code mismatch")] + StartCodeMismatch, + + #[error("empty VP8 frame")] + EmptyFrame, +} + +/// A Result type alias for VP8 parsing. +pub type Result = std::result::Result; + /// Fields parsed from a VP8 frame tag (RFC 6386 §9.1) plus, for key frames, the /// key-frame header (§19.1). #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -27,8 +47,10 @@ impl FrameHeader { /// Reads the 3-byte frame tag, and on a key frame the 7-byte header that /// follows (start code + dimensions). Interframes carry neither a start code /// nor dimensions. - pub fn parse(data: &[u8]) -> anyhow::Result { - anyhow::ensure!(data.len() >= 3, "VP8 frame too short for tag"); + pub fn parse(data: &[u8]) -> Result { + if data.len() < 3 { + return Err(Error::FrameTooShort); + } // 24-bit little-endian frame tag. Bit 0 is frame_type: 0 = key frame. let tag = u32::from(data[0]) | (u32::from(data[1]) << 8) | (u32::from(data[2]) << 16); @@ -41,11 +63,12 @@ impl FrameHeader { }); } - anyhow::ensure!(data.len() >= 10, "VP8 key frame too short for header"); - anyhow::ensure!( - data[3] == 0x9d && data[4] == 0x01 && data[5] == 0x2a, - "VP8 key frame start code mismatch" - ); + if data.len() < 10 { + return Err(Error::KeyframeHeaderTooShort); + } + if !(data[3] == 0x9d && data[4] == 0x01 && data[5] == 0x2a) { + return Err(Error::StartCodeMismatch); + } // 14-bit dimensions; the top 2 bits of each field are the scaling factor. let width = (u16::from(data[6]) | (u16::from(data[7]) << 8)) & 0x3fff; diff --git a/rs/moq-mux/src/codec/vp9/import.rs b/rs/moq-mux/src/codec/vp9/import.rs index dd2b615f3..d6cff8c2f 100644 --- a/rs/moq-mux/src/codec/vp9/import.rs +++ b/rs/moq-mux/src/codec/vp9/import.rs @@ -54,14 +54,14 @@ impl Import { /// with an empty buffer (gstreamer / ffi pass `Bytes::new()`) and the catalog /// is filled from the first key frame. If the caller does pass the first frame /// here, it's decoded so nothing is dropped. - pub fn initialize>(&mut self, buf: &mut T) -> anyhow::Result<()> { + pub fn initialize>(&mut self, buf: &mut T) -> crate::Result<()> { if buf.has_remaining() { self.decode_frame(buf, None)?; } Ok(()) } - fn init(&mut self, vp9: hang::catalog::VP9, width: u16, height: u16) -> anyhow::Result<()> { + fn init(&mut self, vp9: hang::catalog::VP9, width: u16, height: u16) -> crate::Result<()> { let mut config = hang::catalog::VideoConfig::new(vp9); config.coded_width = Some(width as u32); config.coded_height = Some(height as u32); @@ -71,10 +71,6 @@ impl Import { return Ok(()); } - if self.config.is_some() { - anyhow::bail!("fixed track cannot be reconfigured"); - } - tracing::debug!(name = ?self.track.name(), ?config, "starting track"); self.catalog .video @@ -90,9 +86,11 @@ impl Import { &mut self, buf: &mut T, pts: Option, - ) -> anyhow::Result<()> { + ) -> crate::Result<()> { let payload = buf.copy_to_bytes(buf.remaining()); - anyhow::ensure!(!payload.is_empty(), "empty VP9 frame"); + if payload.is_empty() { + return Err(super::Error::EmptyFrame.into()); + } let header = FrameHeader::parse(&payload)?; if let Some(key) = header.key { @@ -127,13 +125,13 @@ impl Import { } /// Finish the track, flushing the current group. - pub fn finish(&mut self) -> anyhow::Result<()> { + pub fn finish(&mut self) -> crate::Result<()> { self.track.finish()?; Ok(()) } /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> anyhow::Result<()> { + pub fn seek(&mut self, sequence: u64) -> crate::Result<()> { self.track.seek(sequence)?; Ok(()) } @@ -143,7 +141,7 @@ impl Import { self.config.is_some() } - fn pts(&mut self, hint: Option) -> anyhow::Result { + fn pts(&mut self, hint: Option) -> crate::Result { if let Some(pts) = hint { return Ok(pts); } diff --git a/rs/moq-mux/src/codec/vp9/mod.rs b/rs/moq-mux/src/codec/vp9/mod.rs index 6a0c8efd0..f7193fb4d 100644 --- a/rs/moq-mux/src/codec/vp9/mod.rs +++ b/rs/moq-mux/src/codec/vp9/mod.rs @@ -14,6 +14,26 @@ pub use import::*; use hang::catalog::VP9; +/// VP9 parsing errors. +#[derive(Debug, Clone, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + #[error("invalid VP9 frame marker")] + InvalidFrameMarker, + + #[error("invalid VP9 sync code")] + InvalidSyncCode, + + #[error("VP9 header truncated")] + Truncated, + + #[error("empty VP9 frame")] + EmptyFrame, +} + +/// A Result type alias for VP9 parsing. +pub type Result = std::result::Result; + /// VP9 key-frame sync code (VP9 spec §6.2, `frame_sync_code`). const SYNC_CODE: u32 = 0x49_8342; @@ -46,10 +66,12 @@ impl FrameHeader { /// /// Reads only as far as the frame size (the last field we care about); /// everything after it is left untouched. - pub fn parse(data: &[u8]) -> anyhow::Result { + pub fn parse(data: &[u8]) -> Result { let mut r = BitReader::new(data); - anyhow::ensure!(r.read(2)? == 0b10, "invalid VP9 frame marker"); + if r.read(2)? != 0b10 { + return Err(Error::InvalidFrameMarker); + } let profile_low = r.read(1)?; let profile_high = r.read(1)?; @@ -77,7 +99,9 @@ impl FrameHeader { }); } - anyhow::ensure!(r.read(24)? == SYNC_CODE, "invalid VP9 sync code"); + if r.read(24)? != SYNC_CODE { + return Err(Error::InvalidSyncCode); + } // color_config (VP9 spec §6.2.2). let bit_depth = if profile >= 2 { @@ -227,11 +251,13 @@ impl<'a> BitReader<'a> { Self { data, bit: 0 } } - fn read(&mut self, n: u32) -> anyhow::Result { + fn read(&mut self, n: u32) -> Result { let mut value = 0; for _ in 0..n { let byte = self.bit / 8; - anyhow::ensure!(byte < self.data.len(), "VP9 header truncated"); + if byte >= self.data.len() { + return Err(Error::Truncated); + } let shift = 7 - (self.bit % 8); value = (value << 1) | u32::from((self.data[byte] >> shift) & 1); self.bit += 1; @@ -239,7 +265,7 @@ impl<'a> BitReader<'a> { Ok(value) } - fn skip(&mut self, n: u32) -> anyhow::Result<()> { + fn skip(&mut self, n: u32) -> Result<()> { self.read(n).map(|_| ()) } } diff --git a/rs/moq-mux/src/error.rs b/rs/moq-mux/src/error.rs index ba9e31dd2..37f48ea63 100644 --- a/rs/moq-mux/src/error.rs +++ b/rs/moq-mux/src/error.rs @@ -59,6 +59,18 @@ pub enum Error { #[error("av1: {0}")] Av1(#[from] crate::codec::av1::Error), + /// Error parsing VP8. + #[error("vp8: {0}")] + Vp8(#[from] crate::codec::vp8::Error), + + /// Error parsing VP9. + #[error("vp9: {0}")] + Vp9(#[from] crate::codec::vp9::Error), + + /// Error parsing legacy audio (MP2 / AC-3 / E-AC-3). + #[error("legacy: {0}")] + Legacy(#[from] crate::codec::legacy::Error), + /// Timestamp overflow when converting between timescales. #[error("timestamp overflow")] TimestampOverflow(#[from] moq_net::TimeOverflow), diff --git a/rs/moq-mux/src/import.rs b/rs/moq-mux/src/import.rs index 458fd4a3b..7880aca96 100644 --- a/rs/moq-mux/src/import.rs +++ b/rs/moq-mux/src/import.rs @@ -286,8 +286,8 @@ impl Framed { FramedKind::Fmp4(ref mut decoder) => decoder.finish(), FramedKind::Hev1(ref mut decoder) => decoder.finish(), FramedKind::Av01(ref mut decoder) => decoder.finish(), - FramedKind::Vp8(ref mut decoder) => decoder.finish().map_err(Into::into), - FramedKind::Vp9(ref mut decoder) => decoder.finish().map_err(Into::into), + FramedKind::Vp8(ref mut decoder) => decoder.finish(), + FramedKind::Vp9(ref mut decoder) => decoder.finish(), FramedKind::Aac(ref mut decoder) => decoder.finish(), FramedKind::Opus(ref mut decoder) => decoder.finish(), FramedKind::Mkv(ref mut decoder) => decoder.finish(), @@ -303,8 +303,8 @@ impl Framed { FramedKind::Fmp4(ref mut decoder) => decoder.seek(sequence), FramedKind::Hev1(ref mut decoder) => decoder.seek(sequence), FramedKind::Av01(ref mut decoder) => decoder.seek(sequence), - FramedKind::Vp8(ref mut decoder) => decoder.seek(sequence).map_err(Into::into), - FramedKind::Vp9(ref mut decoder) => decoder.seek(sequence).map_err(Into::into), + FramedKind::Vp8(ref mut decoder) => decoder.seek(sequence), + FramedKind::Vp9(ref mut decoder) => decoder.seek(sequence), FramedKind::Aac(ref mut decoder) => decoder.seek(sequence), FramedKind::Opus(ref mut decoder) => decoder.seek(sequence), FramedKind::Mkv(ref mut decoder) => decoder.seek(sequence), @@ -563,8 +563,10 @@ mod tests { } } + /// A changed key frame just updates the rendition in place; there are no fixed + /// tracks to reject a reconfiguration, so the second key frame succeeds. #[tokio::test(start_paused = true)] - async fn fixed_track_reconfiguration_errors() { + async fn reconfiguration_updates_in_place() { let (mut broadcast, catalog) = new_broadcast(); let track = broadcast .create_track( @@ -581,10 +583,9 @@ mod tests { .unwrap(); let mut second = Bytes::from_static(&[0x10, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x80, 0x02, 0xe0, 0x01]); - let err = framed + framed .decode_frame(&mut second, Some(Timestamp::from_micros(33_000).unwrap())) - .unwrap_err(); - assert!(err.to_string().contains("fixed track cannot be reconfigured")); + .unwrap(); } } diff --git a/rs/moq-mux/src/publish.rs b/rs/moq-mux/src/publish.rs index 813a2115c..288d8f5ec 100644 --- a/rs/moq-mux/src/publish.rs +++ b/rs/moq-mux/src/publish.rs @@ -124,10 +124,7 @@ impl Published { /// after the closure returns, so a lazily-resolved config or refined jitter /// always reaches it. Prefer [`decode`](Self::decode) where the caller already /// has split frames. - pub fn decoding(&mut self, decode: impl FnOnce(&mut I) -> Result) -> crate::Result - where - crate::Error: From, - { + pub fn decoding(&mut self, decode: impl FnOnce(&mut I) -> crate::Result) -> crate::Result { let out = decode(&mut self.inner)?; self.sync(); Ok(out) From b71966f7c97efbb0abe3edcd097adbd2eed011c7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 17 Jun 2026 02:19:45 +0000 Subject: [PATCH 14/19] moq-mux: make the video importers pure frame publishers The importers owned a Split internally and exposed decode_frame/decode_stream; that duplicated the splitter and kept byte parsing in the publish layer. Move all byte parsing to dispatcher-owned splits so the importers only take frames. - `h264`/`h265`/`av1` `Import` lose their `split` field and the `decode_frame`/`decode_stream`/`decode_from` methods. They're pure publishers now: `decode(impl IntoIterator)` (FrameDecode) + config resolution (from the inline SPS/sequence-header in keyframes, or an avcC/av1C via `initialize`) + catalog + finish/seek/track. `initialize` resolves config without consuming the buffer; `seek` no longer resets a split. - `h264::Split` regains avc1: it's the sole h264 byte->frame engine for both wire shapes (framing + NALU length size only, config stays in the importer). `Mode`/`with_mode`/auto-detect moved here from the importer. - The Framed/Stream dispatchers, the TS container, moq-cli, moq-rtc, and moq-video now own a `Split` and drive `split.decode_X(buf) -> import.decode(frames)`. Small `build_h264`/`build_h265`/`build_av1` helpers in the dispatcher encode the "import reads config, split consumes" contract. 269 tests pass (incl. the byte-exact TS roundtrips and fMP4/MKV); the split gained avc1 unit tests. https://claude.ai/code/session_011S7pzcg2XsPP3AExuymac1 --- rs/moq-cli/src/publish.rs | 21 +- rs/moq-mux/src/codec/av1/import.rs | 55 +---- rs/moq-mux/src/codec/h264/import.rs | 339 ++++++-------------------- rs/moq-mux/src/codec/h264/mod.rs | 5 +- rs/moq-mux/src/codec/h264/split.rs | 317 ++++++++++++++++++++++-- rs/moq-mux/src/codec/h265/import.rs | 58 +---- rs/moq-mux/src/container/ts/import.rs | 30 ++- rs/moq-mux/src/import.rs | 309 ++++++++++++++++++----- rs/moq-rtc/src/codec/h264.rs | 9 +- rs/moq-video/src/encode/producer.rs | 16 +- 10 files changed, 693 insertions(+), 466 deletions(-) diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index a425038ef..5f6e05304 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -80,7 +80,10 @@ pub struct CaptureArgs { } enum PublishDecoder { - Avc3(Box>), + Avc3 { + split: moq_mux::codec::h264::Split, + import: Box>, + }, Fmp4(Box), Ts(Box), Flv(Box), @@ -91,8 +94,9 @@ impl PublishDecoder { /// Decode a chunk of bytes from stdin (Avc3, Fmp4, Ts, or Flv). fn decode_buf(&mut self, buffer: &mut bytes::BytesMut) -> anyhow::Result<()> { match self { - Self::Avc3(d) => { - d.decoding(|d| d.decode_stream(buffer, None))?; + Self::Avc3 { split, import } => { + let frames = split.decode_stream(buffer, None)?; + import.decode(frames)?; Ok(()) } Self::Fmp4(d) => Ok(d.decode(buffer)?), @@ -134,10 +138,13 @@ impl Publish { let source = match format { PublishFormat::Avc3 => { let track = moq_mux::publish::unique_track(&mut broadcast, ".avc3")?; - let avc3 = - moq_mux::codec::h264::Import::from_track(track).with_mode(moq_mux::codec::h264::Mode::Avc3)?; - let avc3 = moq_mux::publish::Published::new(catalog.clone(), avc3); - Source::Stream(PublishDecoder::Avc3(Box::new(avc3))) + let import = moq_mux::codec::h264::Import::from_track(track); + let import = moq_mux::publish::Published::new(catalog.clone(), import); + let split = moq_mux::codec::h264::Split::new().with_mode(moq_mux::codec::h264::Mode::Avc3); + Source::Stream(PublishDecoder::Avc3 { + split, + import: Box::new(import), + }) } PublishFormat::Fmp4 => { let fmp4 = fmp4::Import::new(broadcast.clone(), catalog.clone()); diff --git a/rs/moq-mux/src/codec/av1/import.rs b/rs/moq-mux/src/codec/av1/import.rs index 839bed1e4..b61cc2602 100644 --- a/rs/moq-mux/src/codec/av1/import.rs +++ b/rs/moq-mux/src/codec/av1/import.rs @@ -7,28 +7,28 @@ //! [`initialize`](Import::initialize). A keyframe that can't be configured is an //! error; non-keyframes before the first config are written through to the //! producer, which reports [`MissingKeyframe`](crate::container::MissingKeyframe) -//! for a mid-stream join. OBU byte parsing lives in [`Split`]; this type drives -//! it and adds the catalog. +//! for a mid-stream join. OBU byte parsing lives in [`Split`](super::Split); this type is a +//! pure frame publisher that whoever owns the split drives via the +//! [`FrameDecode`] trait. -use bytes::{Buf, Bytes, BytesMut}; +use bytes::{Buf, Bytes}; use scuffle_av1::seq::SequenceHeaderObu; use scuffle_av1::{ObuHeader, ObuType}; -use tokio::io::{AsyncRead, AsyncReadExt}; +use super::Error; use super::split::ObuIterator; -use super::{Error, Split}; use crate::Result; use crate::container::Frame; use crate::container::jitter::MinFrameDuration; use crate::publish::{FrameDecode, Renditions}; -/// A decoder for AV1 with inline sequence headers. +/// A pure-publisher importer for AV1 with inline sequence headers. /// /// Build it from a [`moq_net::TrackRequest`] ([`new`](Self::new)) or an existing -/// track ([`from_track`](Self::from_track)). The catalog rendition fills in lazily +/// track ([`from_track`](Self::from_track)), and feed it frames a [`Split`](super::Split) +/// produced via the [`FrameDecode`] impl. The catalog rendition fills in lazily /// once the config is known; read it via [`catalog`](Self::catalog). pub struct Import { - split: Split, track: crate::container::Producer, catalog: hang::Catalog, config: Option, @@ -46,7 +46,6 @@ impl Import { /// Publish on an existing track producer. pub fn from_track(track: moq_net::TrackProducer) -> Self { Self { - split: Split::new(), track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), catalog: hang::Catalog::default(), config: None, @@ -55,31 +54,28 @@ impl Import { } } - /// Initialize the decoder with a sequence header / av1C and other metadata. + /// Resolve the codec config from a sequence header / av1C and other metadata. /// /// - **av1C** (leading `0x81` marker): the buffer is parsed as an /// AV1CodecConfigurationRecord, which resolves the config. - /// - **raw OBUs**: any sequence header resolves the config and is fed into - /// the splitter so it prefixes the first keyframe. + /// - **raw OBUs**: any sequence header resolves the config. /// /// Optional, since the importer also self-initializes from the first keyframe. - /// The buffer is fully consumed. + /// The buffer is *not* consumed: the dispatcher-owned [`Split`](super::Split) + /// consumes it (seeding the sequence header so it prefixes the first keyframe). pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { let data = buf.as_ref(); // av1C box starts with 0x81 (marker=1, version=1) per ISO/IEC 14496-15. if data.len() >= 16 && data[0] == 0x81 { self.init_from_av1c(data)?; - buf.advance(buf.remaining()); return Ok(()); } - // Raw OBUs: resolve the config from any sequence header, then feed the - // metadata OBUs into the splitter so they prefix the first keyframe. + // Raw OBUs: resolve the config from any sequence header. if let Some(seq) = find_sequence_header(data) { self.configure_from_seq(&seq)?; } - self.split.seed(buf)?; Ok(()) } @@ -220,27 +216,6 @@ impl Import { self.config.is_some() } - /// Decode from an asynchronous reader (streaming input). - pub async fn decode_from(&mut self, reader: &mut T) -> Result<()> { - let mut buffer = BytesMut::new(); - while reader.read_buf(&mut buffer).await? > 0 { - self.decode_stream(&mut buffer, None)?; - } - Ok(()) - } - - /// Decode as much data as possible from the given buffer. - pub fn decode_stream>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - let frames = self.split.decode_stream(buf, pts)?; - self.write_frames(frames) - } - - /// Decode all data in the buffer, assuming it holds (the rest of) one frame. - pub fn decode_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - let frames = self.split.decode_frame(buf, pts)?; - self.write_frames(frames) - } - /// Finish the track, flushing the current group. pub fn finish(&mut self) -> Result<()> { self.track.finish()?; @@ -248,11 +223,7 @@ impl Import { } /// Close the current group and open the next one at `sequence`. - /// - /// Any in-flight temporal unit is dropped. Pre-seek OBUs would otherwise leak - /// into the post-seek group with the wrong timestamp. pub fn seek(&mut self, sequence: u64) -> Result<()> { - self.split.reset(); self.track.seek(sequence)?; Ok(()) } diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index 980123012..dff366ad9 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -1,53 +1,38 @@ //! H.264 importer. //! -//! [`Import`] publishes H.264 frames on a single moq track and resolves the -//! catalog rendition. It accepts either length-prefixed NALU input with an -//! out-of-band [`AVCDecoderConfigurationRecord`](super::Avcc) (the "avc1" shape) -//! or Annex-B input with inline SPS/PPS (the "avc3" shape). The shape is detected -//! from the first bytes the caller supplies, or forced via -//! [`with_mode`](Import::with_mode). +//! [`Import`] publishes already-split H.264 frames on a single moq track and +//! resolves the catalog rendition. It is a pure frame publisher: byte parsing +//! and framing live in [`Split`](super::Split), and whoever drives the import owns the split. +//! Frames arrive via the [`FrameDecode`] trait ([`decode`](FrameDecode::decode)). //! //! The codec config comes from exactly one of two places: an avcC handed to -//! [`initialize`](Import::initialize) (avc1), or the SPS that the splitter -//! packages into the first keyframe (avc3, scanned out of the frame here). A -//! keyframe that can't be configured from either is an error; non-keyframes -//! before the first config are tolerated (mid-stream joins). Annex-B byte -//! parsing lives in [`Split`]; this type drives it and adds the catalog. +//! [`initialize`](Import::initialize) (the "avc1" shape), or the SPS the splitter +//! packages into the first keyframe (the "avc3" shape, scanned out of the frame +//! here). A keyframe that can't be configured from either is an error; +//! non-keyframes before the first config are tolerated (mid-stream joins). -use bytes::{Buf, Bytes, BytesMut}; -use tokio::io::{AsyncRead, AsyncReadExt}; +use bytes::{Buf, Bytes}; -use super::{Error, NAL_TYPE_SPS, Split, Sps}; +use super::{Error, NAL_TYPE_SPS, Sps}; use crate::Result; use crate::codec::annexb::NalIterator; use crate::container::Frame; use crate::container::jitter::MinFrameDuration; use crate::publish::{FrameDecode, Renditions}; -/// The wire shape an [`Import`] processes. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum Mode { - /// Length-prefixed NALU with out-of-band AVCDecoderConfigurationRecord - /// (catalog `H264 { inline: false }`, `description = avcC`). - Avc1, - /// Annex-B (start-code prefixed) with inline SPS/PPS - /// (catalog `H264 { inline: true }`, no description). - Avc3, -} - -/// H.264 importer. Handles both avc1 (length-prefixed) and avc3 (Annex-B) input; -/// the shape is detected from the first bytes the caller supplies, or forced via -/// [`with_mode`](Self::with_mode). +/// H.264 importer: a pure frame publisher that resolves the catalog rendition. /// /// Build it from a [`moq_net::TrackRequest`] ([`new`](Self::new), the on-demand /// path) or an existing [`moq_net::TrackProducer`] ([`from_track`](Self::from_track), -/// the broadcast-push / fixed-track path). The catalog rendition fills in lazily -/// once the codec config is known (avcC for avc1, the first SPS for avc3); read it -/// via [`catalog`](Self::catalog) or attach the importer to a broadcast catalog -/// with [`crate::publish::Published`]. +/// the broadcast-push / fixed-track path). Feed it frames a [`Split`](super::Split) produced via +/// the [`FrameDecode`] impl. The catalog rendition fills in lazily once the codec +/// config is known (avcC via [`initialize`](Self::initialize) for avc1, the first +/// SPS for avc3); read it via [`catalog`](Self::catalog) or attach the importer to +/// a broadcast catalog with [`crate::publish::Published`]. pub struct Import { - shape: Shape, - split: Split, + /// True for the avc1 shape: the codec config is out-of-band (avcC), so + /// keyframes are not scanned for an inline SPS. + avc1: bool, track: crate::container::Producer, catalog: hang::Catalog, config: Option, @@ -55,15 +40,6 @@ pub struct Import { jitter: MinFrameDuration, } -enum Shape { - /// No bytes seen yet; mode pinned ahead of time or still unknown. - Pending { hint: Option }, - /// avc1: length-prefixed NALU, codec config out-of-band (avcC). - Avc1 { length_size: usize }, - /// avc3: Annex-B NALU, inline SPS/PPS. - Avc3, -} - impl Import { /// Serve a track request, accepting it at the microsecond timescale. pub fn new(request: moq_net::TrackRequest) -> Self { @@ -74,8 +50,7 @@ impl Import { /// Publish on an existing track producer. pub fn from_track(track: moq_net::TrackProducer) -> Self { Self { - shape: Shape::Pending { hint: None }, - split: Split::new(), + avc1: false, track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), catalog: hang::Catalog::default(), config: None, @@ -84,47 +59,29 @@ impl Import { } } - /// Pin the wire shape ahead of time; skips the leading-bytes auto-detect. - /// - /// avc1 still needs an [`initialize`](Self::initialize) with the avcC to - /// learn the NALU length size and codec config. - pub fn with_mode(mut self, mode: Mode) -> Result { - self.shape = match mode { - Mode::Avc1 => Shape::Pending { hint: Some(Mode::Avc1) }, - Mode::Avc3 => Shape::Avc3, - }; - Ok(self) - } - - /// Initialize from the codec's leading bytes. + /// Resolve the codec config from the codec's leading bytes. /// /// - **avc1** (no leading start code): the buffer is parsed as an /// `AVCDecoderConfigurationRecord`, which resolves the config and is stored /// as the catalog `description`. Required for avc1. /// - **avc3** (leading start code): the buffer is parsed as Annex-B; any SPS - /// resolves the config and primes the splitter's parameter-set cache. - /// Optional, since avc3 also self-initializes from the first keyframe. + /// resolves the config. Optional, since avc3 also self-initializes from the + /// first keyframe. /// - /// The buffer is fully consumed. + /// The buffer is *not* consumed: the dispatcher-owned [`Split`](super::Split) consumes it + /// (and reads the same avcC for the NALU length size). The shape is detected + /// from the leading bytes. pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { - let mode = match &self.shape { - Shape::Pending { hint } => hint.unwrap_or_else(|| detect_mode(buf.as_ref())), - Shape::Avc1 { .. } => Mode::Avc1, - Shape::Avc3 => Mode::Avc3, - }; - - match mode { - Mode::Avc1 => self.initialize_avc1(buf), - Mode::Avc3 => self.initialize_avc3(buf), + if detect_avc1(buf.as_ref()) { + self.initialize_avc1(buf.as_ref()) + } else { + self.initialize_avc3(buf.as_ref()) } } - fn initialize_avc1>(&mut self, buf: &mut T) -> Result<()> { - let avcc_bytes = buf.as_ref(); + fn initialize_avc1(&mut self, avcc_bytes: &[u8]) -> Result<()> { + self.avc1 = true; let avcc = super::Avcc::parse(avcc_bytes)?; - self.shape = Shape::Avc1 { - length_size: avcc.length_size, - }; let mut config = hang::catalog::VideoConfig::new(hang::catalog::H264 { profile: avcc.profile, @@ -138,16 +95,13 @@ impl Import { config.container = hang::catalog::Container::Legacy; self.apply_config(config); - buf.advance(buf.remaining()); Ok(()) } - fn initialize_avc3>(&mut self, buf: &mut T) -> Result<()> { - self.shape = Shape::Avc3; - - // Resolve the config from any SPS in the seed buffer, then prime the - // splitter's cache so the first keyframe is self-contained. - let mut scan = Bytes::copy_from_slice(buf.as_ref()); + fn initialize_avc3(&mut self, data: &[u8]) -> Result<()> { + // Resolve the config from any SPS in the seed buffer. Scan a clone so the + // caller's buffer is left intact for the splitter to consume. + let mut scan = Bytes::copy_from_slice(data); let mut nals = NalIterator::new(&mut scan); while let Some(nal) = nals.next().transpose()? { if is_sps(&nal) { @@ -159,8 +113,6 @@ impl Import { { self.configure_from_sps(&nal)?; } - - self.split.seed(buf)?; Ok(()) } @@ -179,56 +131,6 @@ impl Import { self.config.is_some() } - /// Decode from an asynchronous reader (avc3 streaming input). - pub async fn decode_from(&mut self, reader: &mut T) -> Result<()> { - let mut buffer = BytesMut::new(); - while reader.read_buf(&mut buffer).await? > 0 { - self.decode_stream(&mut buffer, None)?; - } - Ok(()) - } - - /// Decode a buffer holding one complete frame. - /// - /// - avc1: the buffer is one length-prefixed-NALU access unit. - /// - avc3: the buffer is one Annex-B access unit. - pub fn decode_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - match self.shape { - Shape::Avc1 { length_size } => { - let frame = read_avc1_frame(buf, length_size, pts)?; - self.write_frames([frame]) - } - Shape::Avc3 => { - let frames = self.split.decode_frame(buf, pts)?; - self.write_frames(frames) - } - Shape::Pending { hint } => match hint.unwrap_or_else(|| detect_mode(buf.as_ref())) { - Mode::Avc3 => { - self.shape = Shape::Avc3; - let frames = self.split.decode_frame(buf, pts)?; - self.write_frames(frames) - } - // avc1 needs the avcC (length size) from initialize() first. - Mode::Avc1 => Err(Error::NotInitialized.into()), - }, - } - } - - /// Decode a buffer where frame boundaries are unknown (avc3 streaming input). - pub fn decode_stream>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - match self.shape { - Shape::Avc3 => {} - Shape::Pending { - hint: None | Some(Mode::Avc3), - } => self.shape = Shape::Avc3, - Shape::Avc1 { .. } | Shape::Pending { hint: Some(Mode::Avc1) } => { - return Err(Error::StreamNotAvc3.into()); - } - } - let frames = self.split.decode_stream(buf, pts)?; - self.write_frames(frames) - } - /// Finish the track, flushing any buffered data. pub fn finish(&mut self) -> Result<()> { self.track.finish()?; @@ -236,11 +138,7 @@ impl Import { } /// Close the current group and open the next one at `sequence`. - /// - /// Any in-flight avc3 access unit is dropped. Pre-seek NALs would otherwise - /// leak into the post-seek group with the wrong timestamp. pub fn seek(&mut self, sequence: u64) -> Result<()> { - self.split.reset(); self.track.seek(sequence)?; Ok(()) } @@ -289,9 +187,9 @@ impl Import { /// keyframe's inline SPS and refining the catalog jitter as it goes. fn write_frames(&mut self, frames: impl IntoIterator) -> Result<()> { for frame in frames { - // avc1 config arrives out-of-band via initialize(); avc3 (and the - // not-yet-resolved Pending case) carries SPS inline on keyframes. - if !matches!(self.shape, Shape::Avc1 { .. }) + // avc1 config arrives out-of-band via initialize(); avc3 carries SPS + // inline on keyframes. + if !self.avc1 && frame.keyframe && let Some(sps) = find_sps(&frame.payload) { @@ -333,35 +231,10 @@ impl Renditions for Import { } } -/// Detect the wire shape from leading bytes: a 3- or 4-byte Annex-B start code -/// means avc3, otherwise an AVCDecoderConfigurationRecord (avc1). -fn detect_mode(bytes: &[u8]) -> Mode { - if matches!(bytes, [0, 0, 1, ..]) || matches!(bytes, [0, 0, 0, 1, ..]) { - Mode::Avc3 - } else { - Mode::Avc1 - } -} - -/// Build one avc1 frame from a length-prefixed-NALU buffer, scanning for an IDR -/// to set the keyframe flag. avc1 always carries timestamps, so a missing `pts` -/// is an error. -fn read_avc1_frame>( - buf: &mut T, - length_size: usize, - pts: Option, -) -> Result { - let data = buf.as_ref(); - let pts = pts.ok_or(Error::MissingTimestamp)?; - let keyframe = avc1_is_keyframe(data, length_size); - let frame = Frame { - timestamp: pts, - payload: data.to_vec().into(), - keyframe, - duration: None, - }; - buf.advance(buf.remaining()); - Ok(frame) +/// Detect the avc1 wire shape from leading bytes: a 3- or 4-byte Annex-B start +/// code means avc3, otherwise an AVCDecoderConfigurationRecord (avc1). +fn detect_avc1(bytes: &[u8]) -> bool { + !(matches!(bytes, [0, 0, 1, ..]) || matches!(bytes, [0, 0, 0, 1, ..])) } fn is_sps(nal: &[u8]) -> bool { @@ -380,58 +253,24 @@ fn find_sps(payload: &[u8]) -> Option { nals.flush().ok().flatten().filter(|nal| is_sps(nal)) } -/// Detect whether an avc1-shaped (length-prefixed) buffer contains an IDR slice. -fn avc1_is_keyframe(data: &[u8], length_size: usize) -> bool { - let mut offset = 0; - while offset + length_size <= data.len() { - let nal_len = match length_size { - 1 => data[offset] as usize, - 2 => u16::from_be_bytes([data[offset], data[offset + 1]]) as usize, - 3 => u32::from_be_bytes([0, data[offset], data[offset + 1], data[offset + 2]]) as usize, - 4 => u32::from_be_bytes([data[offset], data[offset + 1], data[offset + 2], data[offset + 3]]) as usize, - _ => return false, - }; - offset += length_size; - if offset + nal_len > data.len() { - break; - } - if nal_len > 0 && data[offset] & 0x1f == 5 { - return true; // IDR slice - } - offset += nal_len; - } - false -} - #[cfg(test)] mod tests { - use super::*; - - #[test] - fn detect_mode_avc1_avcc_buffer() { - // AVCDecoderConfigurationRecord starts with configurationVersion = 1; - // the first byte is 0x01, never a start code. - let avcc: &[u8] = &[ - 0x01, 0x42, 0xc0, 0x1f, 0xff, 0xe1, 0x00, 0x06, 0x67, 0x42, 0xc0, 0x1f, 0xde, 0xad, - ]; - assert_eq!(detect_mode(avcc), Mode::Avc1); - } + use bytes::BytesMut; - #[test] - fn detect_mode_avc3_3byte_start_code() { - assert_eq!(detect_mode(&[0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1f]), Mode::Avc3); - } + use super::*; + use crate::codec::h264::{Mode, Split}; - #[test] - fn detect_mode_avc3_4byte_start_code() { - assert_eq!( - detect_mode(&[0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1f]), - Mode::Avc3 - ); + fn track(name: &str) -> moq_net::TrackProducer { + let mut broadcast = moq_net::BroadcastInfo::new().produce(); + broadcast + .create_track( + name, + moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), + ) + .unwrap() } - /// An avcC initializer routes into the avc1 path and resolves a config with - /// the avcC stored as `description`. + /// An avcC initializer resolves a config with the avcC stored as `description`. #[tokio::test(start_paused = true)] async fn initialize_avc1_lands_in_catalog() { let sps_nal = [0x67, 0x42, 0xc0, 0x1f]; @@ -439,16 +278,11 @@ mod tests { avcc.extend_from_slice(&sps_nal); avcc.extend_from_slice(&[0x01, 0x00, 0x04, 0x68, 0xce, 0x3c, 0x80]); // num_pps + pps - let mut broadcast = moq_net::BroadcastInfo::new().produce(); - let track = broadcast - .create_track( - "video", - moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), - ) - .unwrap(); - let mut import = Import::from_track(track); + let mut import = Import::from_track(track("video")); + // initialize() must not consume the buffer (the split owns the consume). let mut buf = bytes::BytesMut::from(avcc.as_slice()); import.initialize(&mut buf).expect("initialize avc1"); + assert_eq!(buf.len(), avcc.len(), "initialize must not consume the buffer"); let cfg = import .catalog() @@ -466,8 +300,8 @@ mod tests { assert_eq!(cfg.description.as_ref().expect("description").as_ref(), avcc.as_slice()); } - /// An avc3 stream self-initializes: no `initialize`, the config is resolved - /// from the SPS the splitter packages into the first keyframe. + /// An avc3 stream self-initializes: the config is resolved from the SPS the + /// splitter packages into the first keyframe. #[tokio::test(start_paused = true)] async fn avc3_self_initializes_from_first_keyframe() { let sps: &[u8] = &[ @@ -477,25 +311,20 @@ mod tests { let pps: &[u8] = &[0x68, 0xce, 0x3c, 0x80]; let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; - let mut annexb = bytes::BytesMut::new(); + let mut annexb = BytesMut::new(); for nal in [sps, pps, idr] { annexb.extend_from_slice(&[0, 0, 0, 1]); annexb.extend_from_slice(nal); } - let mut broadcast = moq_net::BroadcastInfo::new().produce(); - let track = broadcast - .create_track( - "video", - moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), - ) - .unwrap(); - let mut import = Import::from_track(track); + let mut split = Split::new().with_mode(Mode::Avc3); + let mut import = Import::from_track(track("video")); assert!(import.catalog().is_none(), "no config before any frame"); - import - .decode_frame(&mut annexb, Some(moq_net::Timestamp::from_micros(0).unwrap())) - .expect("decode keyframe"); + let frames = split + .decode_frame(&mut annexb, moq_net::Timestamp::from_micros(0).unwrap()) + .expect("split keyframe"); + import.decode(frames).expect("decode keyframe"); let cfg = import.catalog().expect("config after keyframe"); let h264_cfg = cfg.video.renditions.get("video").expect("rendition"); @@ -513,21 +342,18 @@ mod tests { #[tokio::test(start_paused = true)] async fn keyframe_without_sps_errors() { let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; // IDR slice, no inline SPS - let mut annexb = bytes::BytesMut::new(); + let mut annexb = BytesMut::new(); annexb.extend_from_slice(&[0, 0, 0, 1]); annexb.extend_from_slice(idr); - let mut broadcast = moq_net::BroadcastInfo::new().produce(); - let track = broadcast - .create_track( - "video", - moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), - ) - .unwrap(); - let mut import = Import::from_track(track); + let mut split = Split::new().with_mode(Mode::Avc3); + let mut import = Import::from_track(track("video")); + let frames = split + .decode_frame(&mut annexb, moq_net::Timestamp::from_micros(0).unwrap()) + .expect("split keyframe"); let err = import - .decode_frame(&mut annexb, Some(moq_net::Timestamp::from_micros(0).unwrap())) + .decode(frames) .expect_err("an unconfigurable keyframe must error"); assert!(matches!(err, crate::Error::H264(Error::NotInitialized)), "got {err:?}"); } @@ -538,21 +364,18 @@ mod tests { #[tokio::test(start_paused = true)] async fn delta_before_init_reports_missing_keyframe() { let pslice: &[u8] = &[0x61, 0xe0, 0x12, 0x34]; // non-IDR slice - let mut annexb = bytes::BytesMut::new(); + let mut annexb = BytesMut::new(); annexb.extend_from_slice(&[0, 0, 0, 1]); annexb.extend_from_slice(pslice); - let mut broadcast = moq_net::BroadcastInfo::new().produce(); - let track = broadcast - .create_track( - "video", - moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), - ) - .unwrap(); - let mut import = Import::from_track(track); + let mut split = Split::new().with_mode(Mode::Avc3); + let mut import = Import::from_track(track("video")); + let frames = split + .decode_frame(&mut annexb, moq_net::Timestamp::from_micros(0).unwrap()) + .expect("split delta"); let err = import - .decode_frame(&mut annexb, Some(moq_net::Timestamp::from_micros(0).unwrap())) + .decode(frames) .expect_err("a delta before any keyframe must report MissingKeyframe"); assert!(matches!(err, crate::Error::MissingKeyframe(_)), "got {err:?}"); assert!(import.catalog().is_none(), "no config yet, so no catalog"); diff --git a/rs/moq-mux/src/codec/h264/mod.rs b/rs/moq-mux/src/codec/h264/mod.rs index 3d16f7459..ea0bf719f 100644 --- a/rs/moq-mux/src/codec/h264/mod.rs +++ b/rs/moq-mux/src/codec/h264/mod.rs @@ -5,8 +5,9 @@ //! (inline SPS/PPS) as length-prefixed NALU + out-of-band avcC, which is //! what every CMAF and MKV consumer expects. [`Export`] subscribes to a //! catalog-narrowed H.264 rendition and emits an Annex-B elementary -//! stream; [`Import`] is the importer (auto-detects either wire shape -//! from the leading bytes). +//! stream; [`Split`] does the byte-level framing (either wire shape, +//! auto-detected from the leading bytes) and [`Import`] is the pure frame +//! publisher that resolves the catalog. mod export; mod import; diff --git a/rs/moq-mux/src/codec/h264/split.rs b/rs/moq-mux/src/codec/h264/split.rs index b2c4d2c81..17eea99d6 100644 --- a/rs/moq-mux/src/codec/h264/split.rs +++ b/rs/moq-mux/src/codec/h264/split.rs @@ -1,17 +1,24 @@ -//! H.264 Annex-B stream splitter. +//! H.264 stream splitter. //! -//! [`Split`] turns a raw Annex-B byte stream (inline SPS/PPS, the "avc3" shape) -//! into [`crate::container::Frame`]s. It is deliberately dumb: it finds -//! access-unit boundaries, caches SPS/PPS and re-inserts them ahead of each -//! keyframe so every keyframe is self-contained, and stamps wall-clock +//! [`Split`] turns raw H.264 bytes into [`crate::container::Frame`]s. It handles +//! both wire shapes: +//! +//! - **avc3** (Annex-B, inline SPS/PPS): finds access-unit boundaries, caches +//! SPS/PPS and re-inserts them ahead of each keyframe so every keyframe is +//! self-contained. +//! - **avc1** (length-prefixed NALU, out-of-band avcC): one length-prefixed +//! access unit per [`decode_frame`](Self::decode_frame) call, with the keyframe +//! flag set by scanning for an IDR slice. +//! +//! It is deliberately dumb: framing and structural parsing only, plus wall-clock //! timestamps when the caller has none (stdin). It owns no track, catalog, or -//! codec config. The importer parses the codec config out of the frames it -//! emits. +//! codec config (no [`VideoConfig`](hang::catalog::VideoConfig)). The importer +//! parses the codec config out of the frames it emits. //! -//! There is no out-of-band initialization beyond optionally [seeding](Self::seed) -//! the parameter-set cache: a caller that can configure a decoder out of band -//! already knows frame boundaries, and would hand whole frames to the importer -//! rather than a byte stream. +//! The shape is auto-detected from the first bytes ([`decode_frame`](Self::decode_frame)), +//! or pinned ahead of time with [`with_mode`](Self::with_mode). avc1 needs an +//! [`initialize`](Self::initialize) with the avcC to learn the NALU length size; +//! avc3 optionally [seeds](Self::seed) its parameter-set cache. use bytes::{Buf, Bytes, BytesMut}; use tokio::io::{AsyncRead, AsyncReadExt}; @@ -20,15 +27,33 @@ use super::{Error, NAL_TYPE_PPS, NAL_TYPE_SPS}; use crate::Result; use crate::codec::annexb::{NalIterator, START_CODE}; -/// H.264 Annex-B stream splitter: bytes in, [`Frame`](crate::container::Frame)s out. +/// The wire shape a [`Split`] processes. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Mode { + /// Length-prefixed NALU with out-of-band AVCDecoderConfigurationRecord + /// (catalog `H264 { inline: false }`, `description = avcC`). + Avc1, + /// Annex-B (start-code prefixed) with inline SPS/PPS + /// (catalog `H264 { inline: true }`, no description). + Avc3, +} + +/// H.264 stream splitter: bytes in, [`Frame`](crate::container::Frame)s out. +/// +/// Handles both wire shapes (avc1 and avc3); the shape is detected from the +/// first bytes [`decode_frame`](Self::decode_frame) sees, or pinned via +/// [`with_mode`](Self::with_mode). /// -/// Feed bytes via [`decode_stream`](Self::decode_stream) (unknown frame -/// boundaries, e.g. stdin), [`decode_frame`](Self::decode_frame) (one complete -/// access unit per call), or [`decode_from`](Self::decode_from) (an async -/// reader). Each returns the frames it produced. SPS/PPS seen inline are cached -/// and re-inserted ahead of each keyframe; [`seed`](Self::seed) primes that -/// cache from an out-of-band parameter-set buffer. +/// Feed bytes via [`decode_stream`](Self::decode_stream) (avc3 only, unknown +/// frame boundaries, e.g. stdin), [`decode_frame`](Self::decode_frame) (one +/// complete access unit per call, either shape), or [`decode_from`](Self::decode_from) +/// (an async reader, avc3). Each returns the frames it produced. For avc3, +/// SPS/PPS seen inline are cached and re-inserted ahead of each keyframe; +/// [`seed`](Self::seed) primes that cache from an out-of-band parameter-set +/// buffer. For avc1, [`initialize`](Self::initialize) reads the avcC for the +/// NALU length size. pub struct Split { + shape: Shape, current: Avc3Frame, sps: Option, pps: Option, @@ -36,6 +61,18 @@ pub struct Split { pending: Vec, } +/// Internal wire-shape state. Distinct from the public [`Mode`] because avc1 +/// carries the resolved NALU length size, and the shape may still be pending +/// auto-detection. +enum Shape { + /// No bytes seen yet; mode pinned ahead of time or still unknown. + Pending { hint: Option }, + /// avc1: length-prefixed NALU. The NALU length size comes from the avcC. + Avc1 { length_size: usize }, + /// avc3: Annex-B NALU, inline SPS/PPS. + Avc3, +} + #[derive(Default)] struct Avc3Frame { chunks: BytesMut, @@ -52,9 +89,10 @@ impl Default for Split { } impl Split { - /// A fresh splitter with an empty parameter-set cache. + /// A fresh splitter with an empty parameter-set cache, wire shape unpinned. pub fn new() -> Self { Self { + shape: Shape::Pending { hint: None }, current: Avc3Frame::default(), sps: None, pps: None, @@ -63,10 +101,56 @@ impl Split { } } + /// Pin the wire shape ahead of time; skips the leading-bytes auto-detect. + /// + /// avc1 still needs an [`initialize`](Self::initialize) with the avcC to + /// learn the NALU length size. + pub fn with_mode(mut self, mode: Mode) -> Self { + self.shape = match mode { + Mode::Avc1 => Shape::Pending { hint: Some(Mode::Avc1) }, + Mode::Avc3 => Shape::Avc3, + }; + self + } + + /// Initialize from the codec's leading bytes. + /// + /// - **avc1**: the buffer is an `AVCDecoderConfigurationRecord`; only the + /// NALU length size is read out (the importer resolves the codec config). + /// Required for avc1 before [`decode_frame`](Self::decode_frame). + /// - **avc3**: the buffer is Annex-B; any SPS/PPS primes the parameter-set + /// cache so the first keyframe is self-contained (the same as [`seed`](Self::seed)). + /// + /// The buffer is fully consumed. + pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { + let mode = match &self.shape { + Shape::Pending { hint } => hint.unwrap_or_else(|| detect_mode(buf.as_ref())), + Shape::Avc1 { .. } => Mode::Avc1, + Shape::Avc3 => Mode::Avc3, + }; + + match mode { + Mode::Avc1 => { + let avcc = super::Avcc::parse(buf.as_ref())?; + self.shape = Shape::Avc1 { + length_size: avcc.length_size, + }; + buf.advance(buf.remaining()); + Ok(()) + } + Mode::Avc3 => { + self.shape = Shape::Avc3; + self.seed(buf) + } + } + } + /// Prime the SPS/PPS cache from an Annex-B parameter-set buffer, so the first /// keyframe is self-contained even if the stream itself omits inline - /// parameter sets. Other NAL types in the buffer are ignored. + /// parameter sets. Other NAL types in the buffer are ignored. avc3 only; + /// implies the avc3 shape. pub fn seed>(&mut self, buf: &mut T) -> Result<()> { + self.shape = Shape::Avc3; let mut nals = NalIterator::new(buf); while let Some(nal) = nals.next().transpose()? { self.cache_param(&nal); @@ -85,7 +169,7 @@ impl Split { } } - /// Decode from an asynchronous reader, returning all frames produced. + /// Decode from an asynchronous reader, returning all frames produced (avc3). pub async fn decode_from(&mut self, reader: &mut T) -> Result> { let mut frames = Vec::new(); let mut buffer = BytesMut::new(); @@ -96,14 +180,24 @@ impl Split { } /// Decode a buffer where frame boundaries are unknown, returning the frames - /// it produced. The leading start code of the *next* access unit is what - /// signals the previous one is complete, so the final access unit stays - /// buffered until the next call (or [`decode_frame`](Self::decode_frame)). + /// it produced. avc3 only (avc1 has no self-delimiting framing). The leading + /// start code of the *next* access unit is what signals the previous one is + /// complete, so the final access unit stays buffered until the next call (or + /// [`decode_frame`](Self::decode_frame)). pub fn decode_stream>( &mut self, buf: &mut T, pts: impl Into>, ) -> Result> { + match self.shape { + Shape::Avc3 => {} + Shape::Pending { + hint: None | Some(Mode::Avc3), + } => self.shape = Shape::Avc3, + Shape::Avc1 { .. } | Shape::Pending { hint: Some(Mode::Avc1) } => { + return Err(Error::StreamNotAvc3.into()); + } + } let pts = self.pts(pts.into())?; let nals = NalIterator::new(buf); for nal in nals { @@ -113,14 +207,42 @@ impl Split { } /// Decode a buffer holding one complete access unit, returning the frames it - /// produced (typically one). Any trailing NAL without a start code is the - /// last NAL of this access unit, and the unit is flushed before returning. + /// produced (typically one). + /// + /// - avc3: any trailing NAL without a start code is the last NAL of this + /// access unit, and the unit is flushed before returning. + /// - avc1: the buffer is one length-prefixed access unit, emitted as a single + /// frame with the keyframe flag set if it carries an IDR slice. A missing + /// `pts` is an error (avc1 always carries timestamps). pub fn decode_frame>( &mut self, buf: &mut T, pts: impl Into>, ) -> Result> { - let pts = self.pts(pts.into())?; + let pts = pts.into(); + match self.shape { + Shape::Avc1 { length_size } => { + let frame = read_avc1_frame(buf, length_size, pts)?; + Ok(vec![frame]) + } + Shape::Avc3 => self.decode_frame_avc3(buf, pts), + Shape::Pending { hint } => match hint.unwrap_or_else(|| detect_mode(buf.as_ref())) { + Mode::Avc3 => { + self.shape = Shape::Avc3; + self.decode_frame_avc3(buf, pts) + } + // avc1 needs the avcC (length size) from initialize() first. + Mode::Avc1 => Err(Error::NotInitialized.into()), + }, + } + } + + fn decode_frame_avc3>( + &mut self, + buf: &mut T, + pts: Option, + ) -> Result> { + let pts = self.pts(pts)?; let mut nals = NalIterator::new(buf); while let Some(nal) = nals.next().transpose()? { self.decode_nal(nal, pts)?; @@ -241,6 +363,60 @@ impl Split { } } +/// Detect the wire shape from leading bytes: a 3- or 4-byte Annex-B start code +/// means avc3, otherwise an AVCDecoderConfigurationRecord (avc1). +fn detect_mode(bytes: &[u8]) -> Mode { + if matches!(bytes, [0, 0, 1, ..]) || matches!(bytes, [0, 0, 0, 1, ..]) { + Mode::Avc3 + } else { + Mode::Avc1 + } +} + +/// Build one avc1 frame from a length-prefixed-NALU buffer, scanning for an IDR +/// to set the keyframe flag. avc1 always carries timestamps, so a missing `pts` +/// is an error. +fn read_avc1_frame>( + buf: &mut T, + length_size: usize, + pts: Option, +) -> Result { + let data = buf.as_ref(); + let pts = pts.ok_or(Error::MissingTimestamp)?; + let keyframe = avc1_is_keyframe(data, length_size); + let frame = crate::container::Frame { + timestamp: pts, + payload: data.to_vec().into(), + keyframe, + duration: None, + }; + buf.advance(buf.remaining()); + Ok(frame) +} + +/// Detect whether an avc1-shaped (length-prefixed) buffer contains an IDR slice. +fn avc1_is_keyframe(data: &[u8], length_size: usize) -> bool { + let mut offset = 0; + while offset + length_size <= data.len() { + let nal_len = match length_size { + 1 => data[offset] as usize, + 2 => u16::from_be_bytes([data[offset], data[offset + 1]]) as usize, + 3 => u32::from_be_bytes([0, data[offset], data[offset + 1], data[offset + 2]]) as usize, + 4 => u32::from_be_bytes([data[offset], data[offset + 1], data[offset + 2], data[offset + 3]]) as usize, + _ => return false, + }; + offset += length_size; + if offset + nal_len > data.len() { + break; + } + if nal_len > 0 && data[offset] & 0x1f == 5 { + return true; // IDR slice + } + offset += nal_len; + } + false +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, num_enum::TryFromPrimitive)] #[repr(u8)] enum Avc3NalType { @@ -278,6 +454,29 @@ mod tests { buf } + #[test] + fn detect_mode_avc1_avcc_buffer() { + // AVCDecoderConfigurationRecord starts with configurationVersion = 1; + // the first byte is 0x01, never a start code. + let avcc: &[u8] = &[ + 0x01, 0x42, 0xc0, 0x1f, 0xff, 0xe1, 0x00, 0x06, 0x67, 0x42, 0xc0, 0x1f, 0xde, 0xad, + ]; + assert_eq!(detect_mode(avcc), Mode::Avc1); + } + + #[test] + fn detect_mode_avc3_3byte_start_code() { + assert_eq!(detect_mode(&[0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1f]), Mode::Avc3); + } + + #[test] + fn detect_mode_avc3_4byte_start_code() { + assert_eq!( + detect_mode(&[0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1f]), + Mode::Avc3 + ); + } + /// A keyframe access unit fed as one buffer emits one self-contained frame: /// SPS+PPS are packaged ahead of the IDR slice and `keyframe` is set. #[tokio::test(start_paused = true)] @@ -319,4 +518,68 @@ mod tests { assert!(frames[0].payload.windows(sps.len()).any(|w| w == sps)); assert!(frames[0].payload.windows(pps.len()).any(|w| w == pps)); } + + /// avc1: a length-prefixed access unit with an IDR slice is emitted as one + /// keyframe; the payload is passed through verbatim. + #[tokio::test(start_paused = true)] + async fn avc1_decode_frame_keyframe() { + let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; + let mut au = BytesMut::new(); + au.extend_from_slice(&(idr.len() as u32).to_be_bytes()); + au.extend_from_slice(idr); + + // avcC with lengthSizeMinusOne = 3 (4-byte length prefix). + let avcc: &[u8] = &[0x01, 0x42, 0xc0, 0x1f, 0xff, 0xe1, 0x00, 0x04, 0x67, 0x42, 0xc0, 0x1f]; + let mut split = Split::new().with_mode(Mode::Avc1); + split.initialize(&mut avcc.to_vec().as_slice()).unwrap(); + + let frames = split + .decode_frame(&mut au, moq_net::Timestamp::from_micros(0).unwrap()) + .unwrap(); + assert_eq!(frames.len(), 1); + assert!(frames[0].keyframe); + assert_eq!(frames[0].payload[4..], *idr); + } + + /// avc1: a length-prefixed access unit with a non-IDR slice is a delta frame. + #[tokio::test(start_paused = true)] + async fn avc1_decode_frame_delta() { + let pslice: &[u8] = &[0x61, 0xe0, 0x12, 0x34]; + let mut au = BytesMut::new(); + au.extend_from_slice(&(pslice.len() as u32).to_be_bytes()); + au.extend_from_slice(pslice); + + let avcc: &[u8] = &[0x01, 0x42, 0xc0, 0x1f, 0xff, 0xe1, 0x00, 0x04, 0x67, 0x42, 0xc0, 0x1f]; + let mut split = Split::new().with_mode(Mode::Avc1); + split.initialize(&mut avcc.to_vec().as_slice()).unwrap(); + + let frames = split + .decode_frame(&mut au, moq_net::Timestamp::from_micros(0).unwrap()) + .unwrap(); + assert_eq!(frames.len(), 1); + assert!(!frames[0].keyframe); + } + + /// avc1 with no avcC initialize() yet can't know the length size, so a + /// decode is an error rather than a misparse. + #[tokio::test(start_paused = true)] + async fn avc1_decode_before_init_errors() { + let mut au = BytesMut::from(&[0x00, 0x00, 0x00, 0x04, 0x65, 0x88, 0x84, 0x21][..]); + let mut split = Split::new().with_mode(Mode::Avc1); + let err = split + .decode_frame(&mut au, moq_net::Timestamp::from_micros(0).unwrap()) + .expect_err("avc1 needs initialize() first"); + assert!(matches!(err, crate::Error::H264(Error::NotInitialized)), "got {err:?}"); + } + + /// decode_stream rejects avc1 (no self-delimiting framing). + #[tokio::test(start_paused = true)] + async fn avc1_decode_stream_errors() { + let mut split = Split::new().with_mode(Mode::Avc1); + let mut buf = BytesMut::from(&[0x00, 0x00, 0x00, 0x04, 0x65, 0x88, 0x84, 0x21][..]); + let err = split + .decode_stream(&mut buf, moq_net::Timestamp::from_micros(0).unwrap()) + .expect_err("decode_stream is avc3 only"); + assert!(matches!(err, crate::Error::H264(Error::StreamNotAvc3)), "got {err:?}"); + } } diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index ab6233747..f6c28e972 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -9,28 +9,27 @@ //! that can't be configured is an error; non-keyframes before the first config //! are written through to the producer, which reports //! [`MissingKeyframe`](crate::container::MissingKeyframe) for a mid-stream join. -//! Annex-B byte parsing lives in [`Split`]; this type drives it and adds the -//! catalog. +//! Annex-B byte parsing lives in [`Split`](super::Split); this type is a pure frame publisher +//! that whoever owns the split drives via the [`FrameDecode`] trait. -use bytes::{Buf, Bytes, BytesMut}; +use bytes::{Buf, Bytes}; use scuffle_h265::SpsNALUnit; -use tokio::io::{AsyncRead, AsyncReadExt}; -use super::{Error, Split, split::nal_unit_type}; +use super::{Error, split::nal_unit_type}; use crate::Result; use crate::codec::annexb::NalIterator; use crate::container::Frame; use crate::container::jitter::MinFrameDuration; use crate::publish::{FrameDecode, Renditions}; -/// A decoder for H.265 with inline VPS/SPS/PPS. +/// A pure-publisher importer for H.265 with inline VPS/SPS/PPS. /// Only supports single layer streams (VPS is cached but not parsed). /// /// Build it from a [`moq_net::TrackRequest`] ([`new`](Self::new)) or an existing -/// track ([`from_track`](Self::from_track)). The catalog rendition fills in lazily +/// track ([`from_track`](Self::from_track)), and feed it frames a [`Split`](super::Split) +/// produced via the [`FrameDecode`] impl. The catalog rendition fills in lazily /// once the first SPS is parsed; read it via [`catalog`](Self::catalog). pub struct Import { - split: Split, track: crate::container::Producer, catalog: hang::Catalog, config: Option, @@ -48,7 +47,6 @@ impl Import { /// Publish on an existing track producer. pub fn from_track(track: moq_net::TrackProducer) -> Self { Self { - split: Split::new(), track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), catalog: hang::Catalog::default(), config: None, @@ -57,11 +55,12 @@ impl Import { } } - /// Initialize the decoder with VPS/SPS/PPS and other non-slice NALs. + /// Resolve the codec config from VPS/SPS/PPS and other non-slice NALs. /// - /// Resolves the config from any SPS in the buffer and primes the splitter's - /// parameter-set cache. Optional, since the importer also self-initializes - /// from the first keyframe. The buffer is fully consumed. + /// Resolves the config from any SPS in the buffer. Optional, since the + /// importer also self-initializes from the first keyframe. The buffer is + /// *not* consumed: the dispatcher-owned [`Split`](super::Split) consumes it (and seeds its + /// parameter-set cache). pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { let mut scan = Bytes::copy_from_slice(buf.as_ref()); let mut nals = NalIterator::new(&mut scan); @@ -75,8 +74,6 @@ impl Import { { self.configure_from_sps(&nal)?; } - - self.split.seed(buf)?; Ok(()) } @@ -95,33 +92,6 @@ impl Import { self.config.is_some() } - /// Decode from an asynchronous reader (streaming input). - pub async fn decode_from(&mut self, reader: &mut T) -> Result<()> { - let mut buffer = BytesMut::new(); - while reader.read_buf(&mut buffer).await? > 0 { - self.decode_stream(&mut buffer, None)?; - } - Ok(()) - } - - /// Decode as much data as possible from the given buffer. - /// - /// Unlike [Self::decode_frame], this needs the start code of the next frame, - /// so it works for streaming media (e.g. stdin) but adds a frame of latency. - pub fn decode_stream>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - let frames = self.split.decode_stream(buf, pts)?; - self.write_frames(frames) - } - - /// Decode all data in the buffer, assuming it holds (the rest of) one frame. - /// - /// Unlike [Self::decode_stream], this is called when NAL boundaries are - /// known, avoiding a frame of latency. Also used at EOF to flush the final frame. - pub fn decode_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { - let frames = self.split.decode_frame(buf, pts)?; - self.write_frames(frames) - } - /// Finish the track, flushing the current group. pub fn finish(&mut self) -> Result<()> { self.track.finish()?; @@ -129,11 +99,7 @@ impl Import { } /// Close the current group and open the next one at `sequence`. - /// - /// Any in-flight access unit is dropped. Pre-seek NALs would otherwise leak - /// into the post-seek group with the wrong timestamp. pub fn seek(&mut self, sequence: u64) -> Result<()> { - self.split.reset(); self.track.seek(sequence)?; Ok(()) } diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 51a7bc6c9..e5ff37177 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -263,8 +263,9 @@ impl Import { let stream = match stream_type { StreamType::H264 => { let track = crate::publish::unique_track(&mut self.broadcast, ".avc3")?; - let import = h264::Import::from_track(track).with_mode(h264::Mode::Avc3)?; + let import = h264::Import::from_track(track); Stream::H264 { + split: h264::Split::new().with_mode(h264::Mode::Avc3), import: Box::new(crate::publish::Published::new(self.catalog.clone(), import)), unwrap: PtsUnwrap::default(), } @@ -272,6 +273,7 @@ impl Import { StreamType::H265 => { let track = crate::publish::unique_track(&mut self.broadcast, ".hev1")?; Stream::H265 { + split: h265::Split::new(), import: Box::new(crate::publish::Published::new( self.catalog.clone(), h265::Import::from_track(track), @@ -719,10 +721,12 @@ impl ScteReassembler { /// One elementary stream's codec importer plus PTS-unwrap state. enum Stream { H264 { + split: h264::Split, import: Box>, unwrap: PtsUnwrap, }, H265 { + split: h265::Split, import: Box>, unwrap: PtsUnwrap, }, @@ -737,13 +741,19 @@ enum Stream { impl Stream { fn write(&mut self, pending: Pending, burst: Option) -> anyhow::Result<()> { match self { - Stream::H264 { import, unwrap } => { + Stream::H264 { split, import, unwrap } => { let pts = unwrap_pts(unwrap, pending.pts)?; - skip_missing_keyframe(import.decoding(|i| i.decode_frame(&mut pending.data.as_slice(), pts))) + skip_missing_keyframe((|| { + let frames = split.decode_frame(&mut pending.data.as_slice(), pts)?; + import.decode(frames) + })()) } - Stream::H265 { import, unwrap } => { + Stream::H265 { split, import, unwrap } => { let pts = unwrap_pts(unwrap, pending.pts)?; - skip_missing_keyframe(import.decoding(|i| i.decode_frame(&mut pending.data.as_slice(), pts))) + skip_missing_keyframe((|| { + let frames = split.decode_frame(&mut pending.data.as_slice(), pts)?; + import.decode(frames) + })()) } Stream::Aac(stream) => stream.write(pending, burst), Stream::Legacy(stream) => stream.write(pending), @@ -753,8 +763,14 @@ impl Stream { fn seek(&mut self, sequence: u64) -> anyhow::Result<()> { match self { - Stream::H264 { import, .. } => Ok(import.seek(sequence)?), - Stream::H265 { import, .. } => Ok(import.seek(sequence)?), + Stream::H264 { split, import, .. } => { + split.reset(); + Ok(import.seek(sequence)?) + } + Stream::H265 { split, import, .. } => { + split.reset(); + Ok(import.seek(sequence)?) + } Stream::Aac(stream) => stream.seek(sequence), Stream::Legacy(stream) => stream.seek(sequence), Stream::Clock | Stream::Ignored => Ok(()), diff --git a/rs/moq-mux/src/import.rs b/rs/moq-mux/src/import.rs index 7880aca96..9aa651605 100644 --- a/rs/moq-mux/src/import.rs +++ b/rs/moq-mux/src/import.rs @@ -102,13 +102,22 @@ impl From for FramedFormat { } enum FramedKind { - /// H.264 (both avc1 and avc3 wire shapes go through this importer; mode - /// is pinned by the caller's FramedFormat choice). - H264(crate::publish::Published), + /// H.264 (both avc1 and avc3 wire shapes; the split is pinned by the caller's + /// FramedFormat choice). The split owns byte parsing; the import publishes. + H264 { + split: crate::codec::h264::Split, + import: crate::publish::Published, + }, // Boxed because it's a large struct and clippy complains about the size. Fmp4(Box), - Hev1(crate::publish::Published), - Av01(crate::publish::Published), + Hev1 { + split: crate::codec::h265::Split, + import: crate::publish::Published, + }, + Av01 { + split: crate::codec::av1::Split, + import: crate::publish::Published, + }, Vp8(crate::publish::Published), Vp9(crate::publish::Published), Aac(crate::publish::Published), @@ -128,6 +137,67 @@ pub struct Framed { decoder: FramedKind, } +/// Build an H.264 split + import pair, resolving the config and consuming `buf`. +/// +/// The import reads `buf` for the codec config without consuming it; the split +/// then consumes it (seeding its parameter-set cache for avc3, or reading the +/// NALU length size for avc1). +fn build_h264>( + track: moq_net::TrackProducer, + catalog: crate::catalog::Producer, + mode: crate::codec::h264::Mode, + buf: &mut T, +) -> Result<( + crate::codec::h264::Split, + crate::publish::Published, +)> { + let mut import = crate::codec::h264::Import::from_track(track); + import.initialize(buf)?; + let mut split = crate::codec::h264::Split::new().with_mode(mode); + split.initialize(buf)?; + Ok((split, crate::publish::Published::new(catalog, import))) +} + +/// Build an H.265 split + import pair, resolving the config and consuming `buf`. +fn build_h265>( + track: moq_net::TrackProducer, + catalog: crate::catalog::Producer, + buf: &mut T, +) -> Result<( + crate::codec::h265::Split, + crate::publish::Published, +)> { + let mut import = crate::codec::h265::Import::from_track(track); + import.initialize(buf)?; + let mut split = crate::codec::h265::Split::new(); + split.seed(buf)?; + Ok((split, crate::publish::Published::new(catalog, import))) +} + +/// Build an AV1 split + import pair, resolving the config and consuming `buf`. +fn build_av1>( + track: moq_net::TrackProducer, + catalog: crate::catalog::Producer, + buf: &mut T, +) -> Result<( + crate::codec::av1::Split, + crate::publish::Published, +)> { + let mut import = crate::codec::av1::Import::from_track(track); + import.initialize(buf)?; + let mut split = crate::codec::av1::Split::new(); + // av1C (leading 0x81, ISO/IEC 14496-15) is config-only and not fed to the + // splitter; raw OBUs seed it so the sequence header prefixes the first + // keyframe. Mirror the importer's av1C detection exactly. + let data = buf.as_ref(); + if data.len() >= 16 && data[0] == 0x81 { + buf.advance(buf.remaining()); + } else { + split.seed(buf)?; + } + Ok((split, crate::publish::Published::new(catalog, import))) +} + impl Framed { /// Create a new framed importer with the given format and initialization data. /// @@ -142,15 +212,13 @@ impl Framed { let decoder = match format { FramedFormat::Avc1 => { let track = crate::publish::unique_track(&mut broadcast, ".avc1")?; - let mut decoder = crate::codec::h264::Import::from_track(track).with_mode(H264Mode::Avc1)?; - decoder.initialize(buf)?; - FramedKind::H264(crate::publish::Published::new(catalog, decoder)) + let (split, import) = build_h264(track, catalog, H264Mode::Avc1, buf)?; + FramedKind::H264 { split, import } } FramedFormat::Avc3 => { let track = crate::publish::unique_track(&mut broadcast, ".avc3")?; - let mut decoder = crate::codec::h264::Import::from_track(track).with_mode(H264Mode::Avc3)?; - decoder.initialize(buf)?; - FramedKind::H264(crate::publish::Published::new(catalog, decoder)) + let (split, import) = build_h264(track, catalog, H264Mode::Avc3, buf)?; + FramedKind::H264 { split, import } } FramedFormat::Fmp4 => { let mut decoder = Box::new(crate::container::fmp4::Import::new(broadcast, catalog)); @@ -159,15 +227,13 @@ impl Framed { } FramedFormat::Hev1 => { let track = crate::publish::unique_track(&mut broadcast, ".hev1")?; - let mut decoder = crate::codec::h265::Import::from_track(track); - decoder.initialize(buf)?; - FramedKind::Hev1(crate::publish::Published::new(catalog, decoder)) + let (split, import) = build_h265(track, catalog, buf)?; + FramedKind::Hev1 { split, import } } FramedFormat::Av01 => { let track = crate::publish::unique_track(&mut broadcast, ".av01")?; - let mut decoder = crate::codec::av1::Import::from_track(track); - decoder.initialize(buf)?; - FramedKind::Av01(crate::publish::Published::new(catalog, decoder)) + let (split, import) = build_av1(track, catalog, buf)?; + FramedKind::Av01 { split, import } } FramedFormat::Vp8 => { let track = crate::publish::unique_track(&mut broadcast, ".vp8")?; @@ -230,24 +296,20 @@ impl Framed { use crate::codec::h264::Mode as H264Mode; let decoder = match format { FramedFormat::Avc1 => { - let mut decoder = crate::codec::h264::Import::from_track(track).with_mode(H264Mode::Avc1)?; - decoder.initialize(buf)?; - FramedKind::H264(crate::publish::Published::new(catalog, decoder)) + let (split, import) = build_h264(track, catalog, H264Mode::Avc1, buf)?; + FramedKind::H264 { split, import } } FramedFormat::Avc3 => { - let mut decoder = crate::codec::h264::Import::from_track(track).with_mode(H264Mode::Avc3)?; - decoder.initialize(buf)?; - FramedKind::H264(crate::publish::Published::new(catalog, decoder)) + let (split, import) = build_h264(track, catalog, H264Mode::Avc3, buf)?; + FramedKind::H264 { split, import } } FramedFormat::Hev1 => { - let mut decoder = crate::codec::h265::Import::from_track(track); - decoder.initialize(buf)?; - FramedKind::Hev1(crate::publish::Published::new(catalog, decoder)) + let (split, import) = build_h265(track, catalog, buf)?; + FramedKind::Hev1 { split, import } } FramedFormat::Av01 => { - let mut decoder = crate::codec::av1::Import::from_track(track); - decoder.initialize(buf)?; - FramedKind::Av01(crate::publish::Published::new(catalog, decoder)) + let (split, import) = build_av1(track, catalog, buf)?; + FramedKind::Av01 { split, import } } FramedFormat::Vp8 => { let mut decoder = crate::codec::vp8::Import::from_track(track); @@ -282,10 +344,10 @@ impl Framed { /// Finish the decoder, flushing any buffered data. pub fn finish(&mut self) -> Result<()> { match self.decoder { - FramedKind::H264(ref mut decoder) => decoder.finish(), + FramedKind::H264 { ref mut import, .. } => import.finish(), FramedKind::Fmp4(ref mut decoder) => decoder.finish(), - FramedKind::Hev1(ref mut decoder) => decoder.finish(), - FramedKind::Av01(ref mut decoder) => decoder.finish(), + FramedKind::Hev1 { ref mut import, .. } => import.finish(), + FramedKind::Av01 { ref mut import, .. } => import.finish(), FramedKind::Vp8(ref mut decoder) => decoder.finish(), FramedKind::Vp9(ref mut decoder) => decoder.finish(), FramedKind::Aac(ref mut decoder) => decoder.finish(), @@ -299,10 +361,28 @@ impl Framed { /// Close the current group and open the next one at `sequence`. pub fn seek(&mut self, sequence: u64) -> Result<()> { match self.decoder { - FramedKind::H264(ref mut decoder) => decoder.seek(sequence), + FramedKind::H264 { + ref mut split, + ref mut import, + } => { + split.reset(); + import.seek(sequence) + } FramedKind::Fmp4(ref mut decoder) => decoder.seek(sequence), - FramedKind::Hev1(ref mut decoder) => decoder.seek(sequence), - FramedKind::Av01(ref mut decoder) => decoder.seek(sequence), + FramedKind::Hev1 { + ref mut split, + ref mut import, + } => { + split.reset(); + import.seek(sequence) + } + FramedKind::Av01 { + ref mut split, + ref mut import, + } => { + split.reset(); + import.seek(sequence) + } FramedKind::Vp8(ref mut decoder) => decoder.seek(sequence), FramedKind::Vp9(ref mut decoder) => decoder.seek(sequence), FramedKind::Aac(ref mut decoder) => decoder.seek(sequence), @@ -316,10 +396,10 @@ impl Framed { /// Return the single track produced by this importer. pub fn track(&self) -> Result<&moq_net::TrackProducer> { match self.decoder { - FramedKind::H264(ref decoder) => Ok(decoder.track()), + FramedKind::H264 { ref import, .. } => Ok(import.track()), FramedKind::Fmp4(_) => Err(crate::Error::MultipleTracks("fmp4")), - FramedKind::Hev1(ref decoder) => Ok(decoder.track()), - FramedKind::Av01(ref decoder) => Ok(decoder.track()), + FramedKind::Hev1 { ref import, .. } => Ok(import.track()), + FramedKind::Av01 { ref import, .. } => Ok(import.track()), FramedKind::Vp8(ref decoder) => Ok(decoder.track()), FramedKind::Vp9(ref decoder) => Ok(decoder.track()), FramedKind::Aac(ref decoder) => Ok(decoder.track()), @@ -333,10 +413,28 @@ impl Framed { /// Decode a frame from the given buffer. pub fn decode_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { match self.decoder { - FramedKind::H264(ref mut decoder) => decoder.decoding(|d| d.decode_frame(buf, pts))?, + FramedKind::H264 { + ref mut split, + ref mut import, + } => { + let frames = split.decode_frame(buf, pts)?; + import.decode(frames)?; + } FramedKind::Fmp4(ref mut decoder) => decoder.decode(buf)?, - FramedKind::Hev1(ref mut decoder) => decoder.decoding(|d| d.decode_frame(buf, pts))?, - FramedKind::Av01(ref mut decoder) => decoder.decoding(|d| d.decode_frame(buf, pts))?, + FramedKind::Hev1 { + ref mut split, + ref mut import, + } => { + let frames = split.decode_frame(buf, pts)?; + import.decode(frames)?; + } + FramedKind::Av01 { + ref mut split, + ref mut import, + } => { + let frames = split.decode_frame(buf, pts)?; + import.decode(frames)?; + } FramedKind::Vp8(ref mut decoder) => decoder.decoding(|d| d.decode_frame(buf, pts))?, FramedKind::Vp9(ref mut decoder) => decoder.decoding(|d| d.decode_frame(buf, pts))?, FramedKind::Aac(ref mut decoder) => decoder.decode(buf, pts)?, @@ -643,12 +741,22 @@ impl fmt::Display for StreamFormat { } enum StreamKind { - /// H.264 in avc3 wire shape (Annex-B with inline SPS/PPS). - Avc3(crate::publish::Published), + /// H.264 in avc3 wire shape (Annex-B with inline SPS/PPS). The split owns + /// byte parsing; the import publishes. + Avc3 { + split: crate::codec::h264::Split, + import: crate::publish::Published, + }, // Boxed because it's a large struct and clippy complains about the size. Fmp4(Box), - Hev1(crate::publish::Published), - Av01(crate::publish::Published), + Hev1 { + split: crate::codec::h265::Split, + import: crate::publish::Published, + }, + Av01 { + split: crate::codec::av1::Split, + import: crate::publish::Published, + }, // Boxed for the same reason as Fmp4. Mkv(Box), // Boxed for the same reason as Fmp4. @@ -676,19 +784,29 @@ impl Stream { let decoder = match format { StreamFormat::Avc3 => { let track = crate::publish::unique_track(&mut broadcast, ".avc3")?; - let decoder = crate::codec::h264::Import::from_track(track).with_mode(H264Mode::Avc3)?; - StreamKind::Avc3(crate::publish::Published::new(catalog, decoder)) + let import = crate::codec::h264::Import::from_track(track); + let split = crate::codec::h264::Split::new().with_mode(H264Mode::Avc3); + StreamKind::Avc3 { + split, + import: crate::publish::Published::new(catalog, import), + } } StreamFormat::Fmp4 => StreamKind::Fmp4(Box::new(crate::container::fmp4::Import::new(broadcast, catalog))), StreamFormat::Hev1 => { let track = crate::publish::unique_track(&mut broadcast, ".hev1")?; - let decoder = crate::codec::h265::Import::from_track(track); - StreamKind::Hev1(crate::publish::Published::new(catalog, decoder)) + let import = crate::codec::h265::Import::from_track(track); + StreamKind::Hev1 { + split: crate::codec::h265::Split::new(), + import: crate::publish::Published::new(catalog, import), + } } StreamFormat::Av01 => { let track = crate::publish::unique_track(&mut broadcast, ".av01")?; - let decoder = crate::codec::av1::Import::from_track(track); - StreamKind::Av01(crate::publish::Published::new(catalog, decoder)) + let import = crate::codec::av1::Import::from_track(track); + StreamKind::Av01 { + split: crate::codec::av1::Split::new(), + import: crate::publish::Published::new(catalog, import), + } } StreamFormat::Mkv => StreamKind::Mkv(Box::new(crate::container::mkv::Import::new(broadcast, catalog))), StreamFormat::Ts => StreamKind::Ts(Box::new(crate::container::ts::Import::new(broadcast, catalog))), @@ -705,10 +823,33 @@ impl Stream { /// The buffer will be fully consumed, or an error will be returned. pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { match self.decoder { - StreamKind::Avc3(ref mut decoder) => decoder.decoding(|d| d.initialize(buf))?, + StreamKind::Avc3 { + ref mut split, + ref mut import, + } => { + import.decoding(|d| d.initialize(buf))?; + split.initialize(buf)?; + } StreamKind::Fmp4(ref mut decoder) => decoder.decode(buf)?, - StreamKind::Hev1(ref mut decoder) => decoder.decoding(|d| d.initialize(buf))?, - StreamKind::Av01(ref mut decoder) => decoder.decoding(|d| d.initialize(buf))?, + StreamKind::Hev1 { + ref mut split, + ref mut import, + } => { + import.decoding(|d| d.initialize(buf))?; + split.seed(buf)?; + } + StreamKind::Av01 { + ref mut split, + ref mut import, + } => { + import.decoding(|d| d.initialize(buf))?; + let data = buf.as_ref(); + if data.len() >= 16 && data[0] == 0x81 { + buf.advance(buf.remaining()); + } else { + split.seed(buf)?; + } + } StreamKind::Mkv(ref mut decoder) => decoder.decode(buf)?, StreamKind::Ts(ref mut decoder) => decoder.decode(buf)?, StreamKind::Flv(ref mut decoder) => decoder.decode(buf)?, @@ -724,10 +865,28 @@ impl Stream { /// Decode a stream of data from the given buffer. pub fn decode_stream>(&mut self, buf: &mut T) -> Result<()> { match self.decoder { - StreamKind::Avc3(ref mut decoder) => decoder.decoding(|d| d.decode_stream(buf, None)), + StreamKind::Avc3 { + ref mut split, + ref mut import, + } => { + let frames = split.decode_stream(buf, None)?; + import.decode(frames) + } StreamKind::Fmp4(ref mut decoder) => decoder.decode(buf), - StreamKind::Hev1(ref mut decoder) => decoder.decoding(|d| d.decode_stream(buf, None)), - StreamKind::Av01(ref mut decoder) => decoder.decoding(|d| d.decode_stream(buf, None)), + StreamKind::Hev1 { + ref mut split, + ref mut import, + } => { + let frames = split.decode_stream(buf, None)?; + import.decode(frames) + } + StreamKind::Av01 { + ref mut split, + ref mut import, + } => { + let frames = split.decode_stream(buf, None)?; + import.decode(frames) + } StreamKind::Mkv(ref mut decoder) => decoder.decode(buf), StreamKind::Ts(ref mut decoder) => decoder.decode(buf).map_err(Into::into), StreamKind::Flv(ref mut decoder) => decoder.decode(buf).map_err(Into::into), @@ -737,10 +896,10 @@ impl Stream { /// Finish the decoder, flushing any buffered data. pub fn finish(&mut self) -> Result<()> { match self.decoder { - StreamKind::Avc3(ref mut decoder) => decoder.finish(), + StreamKind::Avc3 { ref mut import, .. } => import.finish(), StreamKind::Fmp4(ref mut decoder) => decoder.finish(), - StreamKind::Hev1(ref mut decoder) => decoder.finish(), - StreamKind::Av01(ref mut decoder) => decoder.finish(), + StreamKind::Hev1 { ref mut import, .. } => import.finish(), + StreamKind::Av01 { ref mut import, .. } => import.finish(), StreamKind::Mkv(ref mut decoder) => decoder.finish(), StreamKind::Ts(ref mut decoder) => decoder.finish().map_err(Into::into), StreamKind::Flv(ref mut decoder) => decoder.finish().map_err(Into::into), @@ -750,10 +909,28 @@ impl Stream { /// Close the current group and open the next one at `sequence`. pub fn seek(&mut self, sequence: u64) -> Result<()> { match self.decoder { - StreamKind::Avc3(ref mut decoder) => decoder.seek(sequence), + StreamKind::Avc3 { + ref mut split, + ref mut import, + } => { + split.reset(); + import.seek(sequence) + } StreamKind::Fmp4(ref mut decoder) => decoder.seek(sequence), - StreamKind::Hev1(ref mut decoder) => decoder.seek(sequence), - StreamKind::Av01(ref mut decoder) => decoder.seek(sequence), + StreamKind::Hev1 { + ref mut split, + ref mut import, + } => { + split.reset(); + import.seek(sequence) + } + StreamKind::Av01 { + ref mut split, + ref mut import, + } => { + split.reset(); + import.seek(sequence) + } StreamKind::Mkv(ref mut decoder) => decoder.seek(sequence), StreamKind::Ts(ref mut decoder) => decoder.seek(sequence).map_err(Into::into), StreamKind::Flv(ref mut decoder) => decoder.seek(sequence).map_err(Into::into), @@ -763,10 +940,10 @@ impl Stream { /// Check if the decoder has read enough data to be initialized. pub fn is_initialized(&self) -> bool { match self.decoder { - StreamKind::Avc3(ref decoder) => decoder.is_initialized(), + StreamKind::Avc3 { ref import, .. } => import.is_initialized(), StreamKind::Fmp4(ref decoder) => decoder.is_initialized(), - StreamKind::Hev1(ref decoder) => decoder.is_initialized(), - StreamKind::Av01(ref decoder) => decoder.is_initialized(), + StreamKind::Hev1 { ref import, .. } => import.is_initialized(), + StreamKind::Av01 { ref import, .. } => import.is_initialized(), StreamKind::Mkv(ref decoder) => decoder.is_initialized(), StreamKind::Ts(ref decoder) => decoder.is_initialized(), StreamKind::Flv(ref decoder) => decoder.is_initialized(), diff --git a/rs/moq-rtc/src/codec/h264.rs b/rs/moq-rtc/src/codec/h264.rs index 33cc5ec65..03ce46c57 100644 --- a/rs/moq-rtc/src/codec/h264.rs +++ b/rs/moq-rtc/src/codec/h264.rs @@ -10,15 +10,17 @@ use bytes::BytesMut; use crate::{Result, codec}; pub struct Bridge { + split: moq_mux::codec::h264::Split, import: moq_mux::publish::Published, } impl Bridge { pub fn new(mut broadcast: moq_net::BroadcastProducer, catalog: moq_mux::catalog::Producer) -> Result { let track = moq_mux::publish::unique_track(&mut broadcast, ".avc3")?; - let import = moq_mux::codec::h264::Import::from_track(track).with_mode(moq_mux::codec::h264::Mode::Avc3)?; + let import = moq_mux::codec::h264::Import::from_track(track); let import = moq_mux::publish::Published::new(catalog, import); - Ok(Self { import }) + let split = moq_mux::codec::h264::Split::new().with_mode(moq_mux::codec::h264::Mode::Avc3); + Ok(Self { split, import }) } } @@ -27,7 +29,8 @@ impl codec::Bridge for Bridge { let pts = moq_net::Timestamp::from_micros(frame.timestamp_us) .map_err(|err| crate::Error::Other(anyhow::anyhow!("invalid timestamp: {err}")))?; let mut buf = BytesMut::from(frame.payload.as_ref()); - self.import.decoding(|i| i.decode_frame(&mut buf, Some(pts)))?; + let frames = self.split.decode_frame(&mut buf, Some(pts))?; + self.import.decode(frames)?; Ok(()) } } diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index 050a746c0..dda0f093a 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -25,15 +25,17 @@ const DEFAULT_FRAMERATE: u32 = 30; /// trigger capture on demand. `moq_mux::codec::h264::Import` handles /// catalog registration and framing. pub struct Producer { + split: moq_mux::codec::h264::Split, import: moq_mux::publish::Published, } impl Producer { pub fn new(mut broadcast: moq_net::BroadcastProducer, catalog: moq_mux::catalog::Producer) -> Result { let track = moq_mux::publish::unique_track(&mut broadcast, ".avc3")?; - let import = moq_mux::codec::h264::Import::from_track(track).with_mode(moq_mux::codec::h264::Mode::Avc3)?; + let import = moq_mux::codec::h264::Import::from_track(track); let import = moq_mux::publish::Published::new(catalog, import); - Ok(Self { import }) + let split = moq_mux::codec::h264::Split::new().with_mode(moq_mux::codec::h264::Mode::Avc3); + Ok(Self { split, import }) } /// The underlying track producer, created eagerly so subscription state is @@ -45,12 +47,10 @@ impl Producer { /// Publish already-encoded Annex-B packets at the given timestamp. pub fn publish(&mut self, packets: Vec, timestamp: Timestamp) -> Result<(), Error> { - self.import.decoding(|i| { - for mut packet in packets { - i.decode_frame(&mut packet, Some(timestamp))?; - } - Ok::<(), moq_mux::Error>(()) - })?; + for mut packet in packets { + let frames = self.split.decode_frame(&mut packet, Some(timestamp))?; + self.import.decode(frames)?; + } Ok(()) } From fe95e63329b7f1767e784302b8002bde3f76ec77 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 17 Jun 2026 02:49:10 +0000 Subject: [PATCH 15/19] moq-mux: slim Split to a pure stream splitter (decode + flush) A Split is for a raw byte stream (stdin / unknown boundaries); the known-boundary decode_frame didn't belong on it. - Rename `decode_stream` -> `decode` on all three splits, add `flush` (emit the in-flight access/temporal unit), and drop `decode_frame`. `decode_from` flushes at EOF. The Annex-B splits (h264/h265) keep a `tail` buffer so `decode` fully consumes the caller's buffer (Framed's contract) while retaining the trailing NAL across chunks; `flush` pulls it. - Drop avc1 from `h264::Split` entirely. avc1 (length-prefixed + out-of-band avcC) is not a stream and can't arrive over stdin, so `Mode`/`with_mode`/ `initialize`/`detect_mode` and the avc1 framing leave the splitter. avc1 becomes a free `h264::avc1_frame(data, length_size, pts)` helper. - Framed splits its H264 arm into `Avc1 { length_size, import }` (no split, wraps each AU via `avc1_frame`) and `Avc3 { split, import }`. Known-boundary callers (Framed avc3/hev1/av01, TS, moq-rtc, moq-video) do `decode + flush` per unit; stdin callers (Stream, moq-cli) flush the tail at finish/EOF. 265 tests pass (incl. TS byte-exact + fMP4/MKV roundtrips, moq-rtc bitstream). https://claude.ai/code/session_011S7pzcg2XsPP3AExuymac1 --- rs/moq-cli/src/publish.rs | 15 +- rs/moq-mux/src/codec/av1/split.rs | 70 ++--- rs/moq-mux/src/codec/h264/import.rs | 26 +- rs/moq-mux/src/codec/h264/mod.rs | 81 +++++- rs/moq-mux/src/codec/h264/split.rs | 387 ++++++-------------------- rs/moq-mux/src/codec/h265/split.rs | 81 ++++-- rs/moq-mux/src/container/ts/import.rs | 10 +- rs/moq-mux/src/import.rs | 127 ++++++--- rs/moq-rtc/src/codec/h264.rs | 6 +- rs/moq-rtc/tests/bitstream.rs | 2 +- rs/moq-video/src/encode/producer.rs | 6 +- 11 files changed, 378 insertions(+), 433 deletions(-) diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 5f6e05304..10c0ebc24 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -95,7 +95,7 @@ impl PublishDecoder { fn decode_buf(&mut self, buffer: &mut bytes::BytesMut) -> anyhow::Result<()> { match self { Self::Avc3 { split, import } => { - let frames = split.decode_stream(buffer, None)?; + let frames = split.decode(buffer, None)?; import.decode(frames)?; Ok(()) } @@ -105,6 +105,16 @@ impl PublishDecoder { Self::Hls(_) => unreachable!(), } } + + /// Flush any in-flight access unit at end of stream. The avc3 split holds the + /// final AU until the next start code, so stdin EOF must flush it. + fn finish(&mut self) -> anyhow::Result<()> { + if let Self::Avc3 { split, import } = self { + let tail = split.flush(None)?; + import.decode(tail)?; + } + Ok(()) + } } // Exactly one Source exists per process, so the size gap between the small @@ -140,7 +150,7 @@ impl Publish { let track = moq_mux::publish::unique_track(&mut broadcast, ".avc3")?; let import = moq_mux::codec::h264::Import::from_track(track); let import = moq_mux::publish::Published::new(catalog.clone(), import); - let split = moq_mux::codec::h264::Split::new().with_mode(moq_mux::codec::h264::Mode::Avc3); + let split = moq_mux::codec::h264::Split::new(); Source::Stream(PublishDecoder::Avc3 { split, import: Box::new(import), @@ -191,6 +201,7 @@ impl Publish { loop { let n = tokio::io::AsyncReadExt::read_buf(&mut stdin, &mut buffer).await?; if n == 0 { + decoder.finish()?; return Ok(()); } decoder.decode_buf(&mut buffer)?; diff --git a/rs/moq-mux/src/codec/av1/split.rs b/rs/moq-mux/src/codec/av1/split.rs index cc1a3b0c0..874af3a98 100644 --- a/rs/moq-mux/src/codec/av1/split.rs +++ b/rs/moq-mux/src/codec/av1/split.rs @@ -17,11 +17,11 @@ use crate::Result; /// AV1 OBU stream splitter: bytes in, [`Frame`](crate::container::Frame)s out. /// -/// Feed bytes via [`decode_stream`](Self::decode_stream) (unknown frame -/// boundaries, e.g. stdin), [`decode_frame`](Self::decode_frame) (one complete -/// temporal unit per call), or [`decode_from`](Self::decode_from) (an async -/// reader). Each returns the frames it produced. [`seed`](Self::seed) feeds -/// leading metadata OBUs (e.g. a sequence header) into the next frame. +/// Feed bytes via [`decode`](Self::decode) (unknown frame boundaries, e.g. +/// stdin) or [`decode_from`](Self::decode_from) (an async reader); call +/// [`flush`](Self::flush) to emit the final in-flight temporal unit. Each +/// returns the frames it produced. [`seed`](Self::seed) feeds leading metadata +/// OBUs (e.g. a sequence header) into the next frame. pub struct Split { current: Au, zero: Option, @@ -70,15 +70,16 @@ impl Split { let mut frames = Vec::new(); let mut buffer = BytesMut::new(); while reader.read_buf(&mut buffer).await? > 0 { - frames.extend(self.decode_stream(&mut buffer, None)?); + frames.extend(self.decode(&mut buffer, None)?); } + frames.extend(self.flush(None)?); Ok(frames) } - /// Decode a buffer where frame boundaries are unknown, returning the frames - /// it produced. The final temporal unit stays buffered until the next call - /// (or [`decode_frame`](Self::decode_frame)). - pub fn decode_stream>( + /// Decode a buffer where frame boundaries are unknown, returning the temporal + /// units it can complete. The final temporal unit stays buffered until the + /// next call (or [`flush`](Self::flush)). + pub fn decode>( &mut self, buf: &mut T, pts: impl Into>, @@ -93,21 +94,11 @@ impl Split { Ok(std::mem::take(&mut self.pending)) } - /// Decode a buffer holding one complete temporal unit, returning the frames - /// it produced. The unit is flushed before returning. - pub fn decode_frame>( - &mut self, - buf: &mut T, - pts: impl Into>, - ) -> Result> { + /// Emit the in-flight temporal unit, if any. Call after the last + /// [`decode`](Self::decode) when a caller handed over a complete temporal unit + /// (or at end of stream) so the final unit isn't left buffered. + pub fn flush(&mut self, pts: impl Into>) -> Result> { let pts = self.pts(pts.into())?; - let mut obus = ObuIterator::new(buf); - while let Some(obu) = obus.next().transpose()? { - self.decode_obu(obu, Some(pts))?; - } - if let Some(obu) = obus.flush()? { - self.decode_obu(obu, Some(pts))?; - } self.maybe_start_frame(Some(pts))?; Ok(std::mem::take(&mut self.pending)) } @@ -310,35 +301,39 @@ mod tests { moq_net::Timestamp::from_micros(0).unwrap() } + /// Decode one complete temporal unit handed over as a single buffer: `decode` + /// buffers it, `flush` emits it. + fn decode_one(split: &mut Split, buf: &mut BytesMut, pts: moq_net::Timestamp) -> Vec { + let mut frames = split.decode(buf, Some(pts)).unwrap(); + frames.extend(split.flush(Some(pts)).unwrap()); + frames + } + /// A temporal unit with a sequence header + KEY_FRAME emits one keyframe. #[tokio::test(start_paused = true)] - async fn decode_frame_keyframe() { + async fn decode_keyframe() { let mut split = Split::new(); - let frames = split - .decode_frame(&mut cat(&[td(), seq_header(), key_frame()]), Some(ts())) - .unwrap(); + let frames = decode_one(&mut split, &mut cat(&[td(), seq_header(), key_frame()]), ts()); assert_eq!(frames.len(), 1); assert!(frames[0].keyframe); } /// A frame with no sequence header and INTER frame_type is not a keyframe. #[tokio::test(start_paused = true)] - async fn decode_frame_delta_is_not_keyframe() { + async fn decode_delta_is_not_keyframe() { let mut split = Split::new(); - let frames = split - .decode_frame(&mut cat(&[td(), inter_frame()]), Some(ts())) - .unwrap(); + let frames = decode_one(&mut split, &mut cat(&[td(), inter_frame()]), ts()); assert_eq!(frames.len(), 1); assert!(!frames[0].keyframe); } /// In streaming mode the next temporal delimiter closes the previous unit, so - /// the trailing one stays buffered. + /// the trailing one stays buffered until `flush`. #[tokio::test(start_paused = true)] - async fn decode_stream_emits_on_next_boundary() { + async fn decode_emits_on_next_boundary() { let mut split = Split::new(); let frames = split - .decode_stream( + .decode( &mut cat(&[td(), seq_header(), key_frame(), td(), inter_frame()]), Some(ts()), ) @@ -346,5 +341,10 @@ mod tests { // Only the keyframe is complete; the inter frame waits for the next TD. assert_eq!(frames.len(), 1); assert!(frames[0].keyframe); + + // Flushing closes the buffered inter frame. + let tail = split.flush(Some(ts())).unwrap(); + assert_eq!(tail.len(), 1); + assert!(!tail[0].keyframe); } } diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index dff366ad9..ad0ee828e 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -258,7 +258,7 @@ mod tests { use bytes::BytesMut; use super::*; - use crate::codec::h264::{Mode, Split}; + use crate::codec::h264::Split; fn track(name: &str) -> moq_net::TrackProducer { let mut broadcast = moq_net::BroadcastInfo::new().produce(); @@ -317,13 +317,13 @@ mod tests { annexb.extend_from_slice(nal); } - let mut split = Split::new().with_mode(Mode::Avc3); + let mut split = Split::new(); let mut import = Import::from_track(track("video")); assert!(import.catalog().is_none(), "no config before any frame"); - let frames = split - .decode_frame(&mut annexb, moq_net::Timestamp::from_micros(0).unwrap()) - .expect("split keyframe"); + let pts = moq_net::Timestamp::from_micros(0).unwrap(); + let mut frames = split.decode(&mut annexb, pts).expect("split keyframe"); + frames.extend(split.flush(pts).expect("flush keyframe")); import.decode(frames).expect("decode keyframe"); let cfg = import.catalog().expect("config after keyframe"); @@ -346,12 +346,12 @@ mod tests { annexb.extend_from_slice(&[0, 0, 0, 1]); annexb.extend_from_slice(idr); - let mut split = Split::new().with_mode(Mode::Avc3); + let mut split = Split::new(); let mut import = Import::from_track(track("video")); - let frames = split - .decode_frame(&mut annexb, moq_net::Timestamp::from_micros(0).unwrap()) - .expect("split keyframe"); + let pts = moq_net::Timestamp::from_micros(0).unwrap(); + let mut frames = split.decode(&mut annexb, pts).expect("split keyframe"); + frames.extend(split.flush(pts).expect("flush keyframe")); let err = import .decode(frames) .expect_err("an unconfigurable keyframe must error"); @@ -368,12 +368,12 @@ mod tests { annexb.extend_from_slice(&[0, 0, 0, 1]); annexb.extend_from_slice(pslice); - let mut split = Split::new().with_mode(Mode::Avc3); + let mut split = Split::new(); let mut import = Import::from_track(track("video")); - let frames = split - .decode_frame(&mut annexb, moq_net::Timestamp::from_micros(0).unwrap()) - .expect("split delta"); + let pts = moq_net::Timestamp::from_micros(0).unwrap(); + let mut frames = split.decode(&mut annexb, pts).expect("split delta"); + frames.extend(split.flush(pts).expect("flush delta")); let err = import .decode(frames) .expect_err("a delta before any keyframe must report MissingKeyframe"); diff --git a/rs/moq-mux/src/codec/h264/mod.rs b/rs/moq-mux/src/codec/h264/mod.rs index ea0bf719f..c7e94fc2a 100644 --- a/rs/moq-mux/src/codec/h264/mod.rs +++ b/rs/moq-mux/src/codec/h264/mod.rs @@ -5,9 +5,10 @@ //! (inline SPS/PPS) as length-prefixed NALU + out-of-band avcC, which is //! what every CMAF and MKV consumer expects. [`Export`] subscribes to a //! catalog-narrowed H.264 rendition and emits an Annex-B elementary -//! stream; [`Split`] does the byte-level framing (either wire shape, -//! auto-detected from the leading bytes) and [`Import`] is the pure frame -//! publisher that resolves the catalog. +//! stream; [`Split`] does the byte-level framing for the Annex-B (avc3) +//! wire shape and [`Import`] is the pure frame publisher that resolves the +//! catalog. avc1 (length-prefixed NALU) has no stream framing; wrap one +//! access unit with `avc1_frame`. mod export; mod import; @@ -23,6 +24,49 @@ use bytes::{Buf, BufMut, Bytes, BytesMut}; const NAL_TYPE_SPS: u8 = 7; const NAL_TYPE_PPS: u8 = 8; +/// Wrap one avc1 (length-prefixed NALU) access unit as a single +/// [`Frame`](crate::container::Frame), with the keyframe flag set when it +/// carries an IDR slice (NAL type 5). +/// +/// avc1 is not a stream: each access unit arrives whole with its NALU +/// `length_size` known out-of-band from the avcC (`super::Avcc::parse(avcc).length_size`). +/// The payload is passed through verbatim. +pub(crate) fn avc1_frame( + data: &[u8], + length_size: usize, + pts: moq_net::Timestamp, +) -> crate::Result { + Ok(crate::container::Frame { + timestamp: pts, + payload: data.to_vec().into(), + keyframe: avc1_is_keyframe(data, length_size), + duration: None, + }) +} + +/// Detect whether an avc1-shaped (length-prefixed) buffer contains an IDR slice. +fn avc1_is_keyframe(data: &[u8], length_size: usize) -> bool { + let mut offset = 0; + while offset + length_size <= data.len() { + let nal_len = match length_size { + 1 => data[offset] as usize, + 2 => u16::from_be_bytes([data[offset], data[offset + 1]]) as usize, + 3 => u32::from_be_bytes([0, data[offset], data[offset + 1], data[offset + 2]]) as usize, + 4 => u32::from_be_bytes([data[offset], data[offset + 1], data[offset + 2], data[offset + 3]]) as usize, + _ => return false, + }; + offset += length_size; + if offset + nal_len > data.len() { + break; + } + if nal_len > 0 && data[offset] & 0x1f == 5 { + return true; // IDR slice + } + offset += nal_len; + } + false +} + /// H.264 parsing and transform errors. #[derive(Debug, Clone, thiserror::Error)] #[non_exhaustive] @@ -57,7 +101,7 @@ pub enum Error { #[error("forbidden zero bit is not zero")] ForbiddenZeroBit, - #[error("not initialized; call initialize() or with_mode() first")] + #[error("not initialized")] NotInitialized, #[error("avc3 track not created")] @@ -66,9 +110,6 @@ pub enum Error { #[error("missing timestamp")] MissingTimestamp, - #[error("decode_stream is avc3 only")] - StreamNotAvc3, - #[error("annexb: {0}")] Annexb(#[from] crate::codec::annexb::Error), } @@ -426,6 +467,32 @@ mod tests { buf.freeze() } + /// avc1: a length-prefixed access unit with an IDR slice wraps as one keyframe; + /// the payload is passed through verbatim. + #[test] + fn avc1_frame_keyframe() { + let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; + let mut au = BytesMut::new(); + au.extend_from_slice(&(idr.len() as u32).to_be_bytes()); + au.extend_from_slice(idr); + + let frame = avc1_frame(&au, 4, moq_net::Timestamp::from_micros(0).unwrap()).unwrap(); + assert!(frame.keyframe); + assert_eq!(frame.payload[4..], *idr); + } + + /// avc1: a length-prefixed access unit with a non-IDR slice is a delta frame. + #[test] + fn avc1_frame_delta() { + let pslice: &[u8] = &[0x61, 0xe0, 0x12, 0x34]; + let mut au = BytesMut::new(); + au.extend_from_slice(&(pslice.len() as u32).to_be_bytes()); + au.extend_from_slice(pslice); + + let frame = avc1_frame(&au, 4, moq_net::Timestamp::from_micros(0).unwrap()).unwrap(); + assert!(!frame.keyframe); + } + #[test] fn avc3_strips_sps_pps_and_builds_avcc() { let sps = &[0x67, 0x42, 0xc0, 0x1f, 0xde][..]; diff --git a/rs/moq-mux/src/codec/h264/split.rs b/rs/moq-mux/src/codec/h264/split.rs index 17eea99d6..3da2fda54 100644 --- a/rs/moq-mux/src/codec/h264/split.rs +++ b/rs/moq-mux/src/codec/h264/split.rs @@ -1,24 +1,17 @@ -//! H.264 stream splitter. +//! H.264 Annex-B stream splitter. //! -//! [`Split`] turns raw H.264 bytes into [`crate::container::Frame`]s. It handles -//! both wire shapes: +//! [`Split`] turns a raw H.264 Annex-B byte stream (inline SPS/PPS, the "avc3" +//! wire shape) into [`crate::container::Frame`]s. It finds access-unit +//! boundaries, caches SPS/PPS and re-inserts them ahead of each keyframe so +//! every keyframe is self-contained, and stamps wall-clock timestamps when the +//! caller has none (stdin). //! -//! - **avc3** (Annex-B, inline SPS/PPS): finds access-unit boundaries, caches -//! SPS/PPS and re-inserts them ahead of each keyframe so every keyframe is -//! self-contained. -//! - **avc1** (length-prefixed NALU, out-of-band avcC): one length-prefixed -//! access unit per [`decode_frame`](Self::decode_frame) call, with the keyframe -//! flag set by scanning for an IDR slice. +//! It is deliberately dumb: framing and structural parsing only. It owns no +//! track, catalog, or codec config (no [`VideoConfig`](hang::catalog::VideoConfig)). +//! The importer parses the codec config out of the frames it emits. //! -//! It is deliberately dumb: framing and structural parsing only, plus wall-clock -//! timestamps when the caller has none (stdin). It owns no track, catalog, or -//! codec config (no [`VideoConfig`](hang::catalog::VideoConfig)). The importer -//! parses the codec config out of the frames it emits. -//! -//! The shape is auto-detected from the first bytes ([`decode_frame`](Self::decode_frame)), -//! or pinned ahead of time with [`with_mode`](Self::with_mode). avc1 needs an -//! [`initialize`](Self::initialize) with the avcC to learn the NALU length size; -//! avc3 optionally [seeds](Self::seed) its parameter-set cache. +//! avc1 (length-prefixed NALU + out-of-band avcC) is not a stream and has no +//! splitter; wrap one access unit with `super::avc1_frame`. use bytes::{Buf, Bytes, BytesMut}; use tokio::io::{AsyncRead, AsyncReadExt}; @@ -27,33 +20,19 @@ use super::{Error, NAL_TYPE_PPS, NAL_TYPE_SPS}; use crate::Result; use crate::codec::annexb::{NalIterator, START_CODE}; -/// The wire shape a [`Split`] processes. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum Mode { - /// Length-prefixed NALU with out-of-band AVCDecoderConfigurationRecord - /// (catalog `H264 { inline: false }`, `description = avcC`). - Avc1, - /// Annex-B (start-code prefixed) with inline SPS/PPS - /// (catalog `H264 { inline: true }`, no description). - Avc3, -} - -/// H.264 stream splitter: bytes in, [`Frame`](crate::container::Frame)s out. +/// H.264 Annex-B stream splitter: bytes in, [`Frame`](crate::container::Frame)s out. /// -/// Handles both wire shapes (avc1 and avc3); the shape is detected from the -/// first bytes [`decode_frame`](Self::decode_frame) sees, or pinned via -/// [`with_mode`](Self::with_mode). -/// -/// Feed bytes via [`decode_stream`](Self::decode_stream) (avc3 only, unknown -/// frame boundaries, e.g. stdin), [`decode_frame`](Self::decode_frame) (one -/// complete access unit per call, either shape), or [`decode_from`](Self::decode_from) -/// (an async reader, avc3). Each returns the frames it produced. For avc3, -/// SPS/PPS seen inline are cached and re-inserted ahead of each keyframe; -/// [`seed`](Self::seed) primes that cache from an out-of-band parameter-set -/// buffer. For avc1, [`initialize`](Self::initialize) reads the avcC for the -/// NALU length size. +/// Feed bytes via [`decode`](Self::decode) (unknown frame boundaries, e.g. +/// stdin) or [`decode_from`](Self::decode_from) (an async reader); call +/// [`flush`](Self::flush) to emit the final in-flight access unit. Each returns +/// the frames it produced. SPS/PPS seen inline are cached and re-inserted ahead +/// of each keyframe; [`seed`](Self::seed) primes that cache from an out-of-band +/// parameter-set buffer. pub struct Split { - shape: Shape, + /// Bytes carried over between calls: complete NALs are parsed out on each + /// [`decode`](Self::decode), leaving the in-flight (final, not-yet-terminated) + /// NAL here until the next start code arrives or [`flush`](Self::flush) drains it. + tail: BytesMut, current: Avc3Frame, sps: Option, pps: Option, @@ -61,18 +40,6 @@ pub struct Split { pending: Vec, } -/// Internal wire-shape state. Distinct from the public [`Mode`] because avc1 -/// carries the resolved NALU length size, and the shape may still be pending -/// auto-detection. -enum Shape { - /// No bytes seen yet; mode pinned ahead of time or still unknown. - Pending { hint: Option }, - /// avc1: length-prefixed NALU. The NALU length size comes from the avcC. - Avc1 { length_size: usize }, - /// avc3: Annex-B NALU, inline SPS/PPS. - Avc3, -} - #[derive(Default)] struct Avc3Frame { chunks: BytesMut, @@ -89,10 +56,10 @@ impl Default for Split { } impl Split { - /// A fresh splitter with an empty parameter-set cache, wire shape unpinned. + /// A fresh splitter with an empty parameter-set cache. pub fn new() -> Self { Self { - shape: Shape::Pending { hint: None }, + tail: BytesMut::new(), current: Avc3Frame::default(), sps: None, pps: None, @@ -101,56 +68,10 @@ impl Split { } } - /// Pin the wire shape ahead of time; skips the leading-bytes auto-detect. - /// - /// avc1 still needs an [`initialize`](Self::initialize) with the avcC to - /// learn the NALU length size. - pub fn with_mode(mut self, mode: Mode) -> Self { - self.shape = match mode { - Mode::Avc1 => Shape::Pending { hint: Some(Mode::Avc1) }, - Mode::Avc3 => Shape::Avc3, - }; - self - } - - /// Initialize from the codec's leading bytes. - /// - /// - **avc1**: the buffer is an `AVCDecoderConfigurationRecord`; only the - /// NALU length size is read out (the importer resolves the codec config). - /// Required for avc1 before [`decode_frame`](Self::decode_frame). - /// - **avc3**: the buffer is Annex-B; any SPS/PPS primes the parameter-set - /// cache so the first keyframe is self-contained (the same as [`seed`](Self::seed)). - /// - /// The buffer is fully consumed. - pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { - let mode = match &self.shape { - Shape::Pending { hint } => hint.unwrap_or_else(|| detect_mode(buf.as_ref())), - Shape::Avc1 { .. } => Mode::Avc1, - Shape::Avc3 => Mode::Avc3, - }; - - match mode { - Mode::Avc1 => { - let avcc = super::Avcc::parse(buf.as_ref())?; - self.shape = Shape::Avc1 { - length_size: avcc.length_size, - }; - buf.advance(buf.remaining()); - Ok(()) - } - Mode::Avc3 => { - self.shape = Shape::Avc3; - self.seed(buf) - } - } - } - /// Prime the SPS/PPS cache from an Annex-B parameter-set buffer, so the first /// keyframe is self-contained even if the stream itself omits inline - /// parameter sets. Other NAL types in the buffer are ignored. avc3 only; - /// implies the avc3 shape. + /// parameter sets. Other NAL types in the buffer are ignored. pub fn seed>(&mut self, buf: &mut T) -> Result<()> { - self.shape = Shape::Avc3; let mut nals = NalIterator::new(buf); while let Some(nal) = nals.next().transpose()? { self.cache_param(&nal); @@ -169,87 +90,56 @@ impl Split { } } - /// Decode from an asynchronous reader, returning all frames produced (avc3). + /// Decode from an asynchronous reader, returning all frames produced. pub async fn decode_from(&mut self, reader: &mut T) -> Result> { let mut frames = Vec::new(); let mut buffer = BytesMut::new(); while reader.read_buf(&mut buffer).await? > 0 { - frames.extend(self.decode_stream(&mut buffer, None)?); + frames.extend(self.decode(&mut buffer, None)?); } + frames.extend(self.flush(None)?); Ok(frames) } - /// Decode a buffer where frame boundaries are unknown, returning the frames - /// it produced. avc3 only (avc1 has no self-delimiting framing). The leading - /// start code of the *next* access unit is what signals the previous one is - /// complete, so the final access unit stays buffered until the next call (or - /// [`decode_frame`](Self::decode_frame)). - pub fn decode_stream>( + /// Decode a buffer where frame boundaries are unknown, returning the access + /// units it can complete. The leading start code of the *next* access unit is + /// what signals the previous one is complete, so the final NAL of the in-flight + /// access unit stays buffered until the next call (or [`flush`](Self::flush)). + /// The buffer is fully consumed. + pub fn decode>( &mut self, buf: &mut T, pts: impl Into>, ) -> Result> { - match self.shape { - Shape::Avc3 => {} - Shape::Pending { - hint: None | Some(Mode::Avc3), - } => self.shape = Shape::Avc3, - Shape::Avc1 { .. } | Shape::Pending { hint: Some(Mode::Avc1) } => { - return Err(Error::StreamNotAvc3.into()); - } - } let pts = self.pts(pts.into())?; - let nals = NalIterator::new(buf); + while buf.has_remaining() { + let chunk = buf.chunk(); + self.tail.extend_from_slice(chunk); + let len = chunk.len(); + buf.advance(len); + } + // Iterate complete NALs out of `tail`, leaving the trailing (in-flight) NAL + // (with its start code) buffered for the next call or `flush`. + let nals = NalIterator::new(&mut self.tail); + let mut parsed = Vec::new(); for nal in nals { - self.decode_nal(nal?, pts)?; + parsed.push(nal?); } - Ok(std::mem::take(&mut self.pending)) - } - - /// Decode a buffer holding one complete access unit, returning the frames it - /// produced (typically one). - /// - /// - avc3: any trailing NAL without a start code is the last NAL of this - /// access unit, and the unit is flushed before returning. - /// - avc1: the buffer is one length-prefixed access unit, emitted as a single - /// frame with the keyframe flag set if it carries an IDR slice. A missing - /// `pts` is an error (avc1 always carries timestamps). - pub fn decode_frame>( - &mut self, - buf: &mut T, - pts: impl Into>, - ) -> Result> { - let pts = pts.into(); - match self.shape { - Shape::Avc1 { length_size } => { - let frame = read_avc1_frame(buf, length_size, pts)?; - Ok(vec![frame]) - } - Shape::Avc3 => self.decode_frame_avc3(buf, pts), - Shape::Pending { hint } => match hint.unwrap_or_else(|| detect_mode(buf.as_ref())) { - Mode::Avc3 => { - self.shape = Shape::Avc3; - self.decode_frame_avc3(buf, pts) - } - // avc1 needs the avcC (length size) from initialize() first. - Mode::Avc1 => Err(Error::NotInitialized.into()), - }, + for nal in parsed { + self.decode_nal(nal, pts)?; } + Ok(std::mem::take(&mut self.pending)) } - fn decode_frame_avc3>( - &mut self, - buf: &mut T, - pts: Option, - ) -> Result> { - let pts = self.pts(pts)?; - let mut nals = NalIterator::new(buf); - while let Some(nal) = nals.next().transpose()? { - self.decode_nal(nal, pts)?; - } - if let Some(nal) = nals.flush()? { + /// Emit the in-flight access unit, if any. Call after the last + /// [`decode`](Self::decode) when a caller handed over a complete access unit + /// (or at end of stream) so the final NAL isn't left buffered. + pub fn flush(&mut self, pts: impl Into>) -> Result> { + let pts = self.pts(pts.into())?; + if let Some(nal) = NalIterator::new(&mut self.tail).flush()? { self.decode_nal(nal, pts)?; } + self.tail.clear(); self.maybe_start_frame(pts)?; Ok(std::mem::take(&mut self.pending)) } @@ -352,6 +242,7 @@ impl Split { /// self-contained. pub fn reset(&mut self) { self.current = Avc3Frame::default(); + self.tail.clear(); } fn pts(&mut self, hint: Option) -> Result { @@ -363,60 +254,6 @@ impl Split { } } -/// Detect the wire shape from leading bytes: a 3- or 4-byte Annex-B start code -/// means avc3, otherwise an AVCDecoderConfigurationRecord (avc1). -fn detect_mode(bytes: &[u8]) -> Mode { - if matches!(bytes, [0, 0, 1, ..]) || matches!(bytes, [0, 0, 0, 1, ..]) { - Mode::Avc3 - } else { - Mode::Avc1 - } -} - -/// Build one avc1 frame from a length-prefixed-NALU buffer, scanning for an IDR -/// to set the keyframe flag. avc1 always carries timestamps, so a missing `pts` -/// is an error. -fn read_avc1_frame>( - buf: &mut T, - length_size: usize, - pts: Option, -) -> Result { - let data = buf.as_ref(); - let pts = pts.ok_or(Error::MissingTimestamp)?; - let keyframe = avc1_is_keyframe(data, length_size); - let frame = crate::container::Frame { - timestamp: pts, - payload: data.to_vec().into(), - keyframe, - duration: None, - }; - buf.advance(buf.remaining()); - Ok(frame) -} - -/// Detect whether an avc1-shaped (length-prefixed) buffer contains an IDR slice. -fn avc1_is_keyframe(data: &[u8], length_size: usize) -> bool { - let mut offset = 0; - while offset + length_size <= data.len() { - let nal_len = match length_size { - 1 => data[offset] as usize, - 2 => u16::from_be_bytes([data[offset], data[offset + 1]]) as usize, - 3 => u32::from_be_bytes([0, data[offset], data[offset + 1], data[offset + 2]]) as usize, - 4 => u32::from_be_bytes([data[offset], data[offset + 1], data[offset + 2], data[offset + 3]]) as usize, - _ => return false, - }; - offset += length_size; - if offset + nal_len > data.len() { - break; - } - if nal_len > 0 && data[offset] & 0x1f == 5 { - return true; // IDR slice - } - offset += nal_len; - } - false -} - #[derive(Debug, Clone, Copy, PartialEq, Eq, num_enum::TryFromPrimitive)] #[repr(u8)] enum Avc3NalType { @@ -454,42 +291,28 @@ mod tests { buf } - #[test] - fn detect_mode_avc1_avcc_buffer() { - // AVCDecoderConfigurationRecord starts with configurationVersion = 1; - // the first byte is 0x01, never a start code. - let avcc: &[u8] = &[ - 0x01, 0x42, 0xc0, 0x1f, 0xff, 0xe1, 0x00, 0x06, 0x67, 0x42, 0xc0, 0x1f, 0xde, 0xad, - ]; - assert_eq!(detect_mode(avcc), Mode::Avc1); + fn ts() -> moq_net::Timestamp { + moq_net::Timestamp::from_micros(0).unwrap() } - #[test] - fn detect_mode_avc3_3byte_start_code() { - assert_eq!(detect_mode(&[0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1f]), Mode::Avc3); - } - - #[test] - fn detect_mode_avc3_4byte_start_code() { - assert_eq!( - detect_mode(&[0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1f]), - Mode::Avc3 - ); + /// Decode one complete access unit handed over as a single buffer: `decode` + /// buffers it, `flush` emits it. + fn decode_one(split: &mut Split, buf: &mut BytesMut, pts: moq_net::Timestamp) -> Vec { + let mut frames = split.decode(buf, pts).unwrap(); + frames.extend(split.flush(pts).unwrap()); + frames } /// A keyframe access unit fed as one buffer emits one self-contained frame: /// SPS+PPS are packaged ahead of the IDR slice and `keyframe` is set. #[tokio::test(start_paused = true)] - async fn decode_frame_packages_keyframe() { + async fn decode_packages_keyframe() { let sps: &[u8] = &[0x67, 0x42, 0xc0, 0x1f]; let pps: &[u8] = &[0x68, 0xce, 0x3c, 0x80]; let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; let mut split = Split::new(); - let mut buf = annexb(&[sps, pps, idr]); - let frames = split - .decode_frame(&mut buf, moq_net::Timestamp::from_micros(0).unwrap()) - .unwrap(); + let frames = decode_one(&mut split, &mut annexb(&[sps, pps, idr]), ts()); assert_eq!(frames.len(), 1); assert!(frames[0].keyframe); @@ -510,76 +333,36 @@ mod tests { let mut split = Split::new(); split.seed(&mut annexb(&[sps, pps])).unwrap(); - let frames = split - .decode_frame(&mut annexb(&[idr]), moq_net::Timestamp::from_micros(0).unwrap()) - .unwrap(); + let frames = decode_one(&mut split, &mut annexb(&[idr]), ts()); assert_eq!(frames.len(), 1); assert!(frames[0].keyframe); assert!(frames[0].payload.windows(sps.len()).any(|w| w == sps)); assert!(frames[0].payload.windows(pps.len()).any(|w| w == pps)); } - /// avc1: a length-prefixed access unit with an IDR slice is emitted as one - /// keyframe; the payload is passed through verbatim. + /// In streaming mode an access unit completes only once the next one begins + /// (a slice with first_mb_in_slice set). A keyframe AU followed by a P-slice + /// of the next AU completes the keyframe; the P-slice's own AU stays buffered + /// until `flush`. #[tokio::test(start_paused = true)] - async fn avc1_decode_frame_keyframe() { + async fn decode_emits_on_next_boundary() { + let sps: &[u8] = &[0x67, 0x42, 0xc0, 0x1f]; + let pps: &[u8] = &[0x68, 0xce, 0x3c, 0x80]; let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; - let mut au = BytesMut::new(); - au.extend_from_slice(&(idr.len() as u32).to_be_bytes()); - au.extend_from_slice(idr); - - // avcC with lengthSizeMinusOne = 3 (4-byte length prefix). - let avcc: &[u8] = &[0x01, 0x42, 0xc0, 0x1f, 0xff, 0xe1, 0x00, 0x04, 0x67, 0x42, 0xc0, 0x1f]; - let mut split = Split::new().with_mode(Mode::Avc1); - split.initialize(&mut avcc.to_vec().as_slice()).unwrap(); - - let frames = split - .decode_frame(&mut au, moq_net::Timestamp::from_micros(0).unwrap()) - .unwrap(); - assert_eq!(frames.len(), 1); - assert!(frames[0].keyframe); - assert_eq!(frames[0].payload[4..], *idr); - } - - /// avc1: a length-prefixed access unit with a non-IDR slice is a delta frame. - #[tokio::test(start_paused = true)] - async fn avc1_decode_frame_delta() { + // P-slice with first_mb_in_slice (byte 1 high bit) set, opening a new AU. let pslice: &[u8] = &[0x61, 0xe0, 0x12, 0x34]; - let mut au = BytesMut::new(); - au.extend_from_slice(&(pslice.len() as u32).to_be_bytes()); - au.extend_from_slice(pslice); + // A trailing AUD so the P-slice is a *complete* NAL (it has a following + // start code), letting the keyframe boundary be detected during decode. + let aud: &[u8] = &[0x09, 0x10]; - let avcc: &[u8] = &[0x01, 0x42, 0xc0, 0x1f, 0xff, 0xe1, 0x00, 0x04, 0x67, 0x42, 0xc0, 0x1f]; - let mut split = Split::new().with_mode(Mode::Avc1); - split.initialize(&mut avcc.to_vec().as_slice()).unwrap(); - - let frames = split - .decode_frame(&mut au, moq_net::Timestamp::from_micros(0).unwrap()) - .unwrap(); + let mut split = Split::new(); + let frames = split.decode(&mut annexb(&[sps, pps, idr, pslice, aud]), ts()).unwrap(); assert_eq!(frames.len(), 1); - assert!(!frames[0].keyframe); - } - - /// avc1 with no avcC initialize() yet can't know the length size, so a - /// decode is an error rather than a misparse. - #[tokio::test(start_paused = true)] - async fn avc1_decode_before_init_errors() { - let mut au = BytesMut::from(&[0x00, 0x00, 0x00, 0x04, 0x65, 0x88, 0x84, 0x21][..]); - let mut split = Split::new().with_mode(Mode::Avc1); - let err = split - .decode_frame(&mut au, moq_net::Timestamp::from_micros(0).unwrap()) - .expect_err("avc1 needs initialize() first"); - assert!(matches!(err, crate::Error::H264(Error::NotInitialized)), "got {err:?}"); - } + assert!(frames[0].keyframe); - /// decode_stream rejects avc1 (no self-delimiting framing). - #[tokio::test(start_paused = true)] - async fn avc1_decode_stream_errors() { - let mut split = Split::new().with_mode(Mode::Avc1); - let mut buf = BytesMut::from(&[0x00, 0x00, 0x00, 0x04, 0x65, 0x88, 0x84, 0x21][..]); - let err = split - .decode_stream(&mut buf, moq_net::Timestamp::from_micros(0).unwrap()) - .expect_err("decode_stream is avc3 only"); - assert!(matches!(err, crate::Error::H264(Error::StreamNotAvc3)), "got {err:?}"); + // Flushing closes the buffered P-slice AU (the AUD rides along with it). + let tail = split.flush(ts()).unwrap(); + assert_eq!(tail.len(), 1); + assert!(!tail[0].keyframe); } } diff --git a/rs/moq-mux/src/codec/h265/split.rs b/rs/moq-mux/src/codec/h265/split.rs index 038146c61..ce4f509d5 100644 --- a/rs/moq-mux/src/codec/h265/split.rs +++ b/rs/moq-mux/src/codec/h265/split.rs @@ -18,13 +18,17 @@ use crate::codec::annexb::{NalIterator, START_CODE}; /// H.265 Annex-B stream splitter: bytes in, [`Frame`](crate::container::Frame)s out. /// -/// Feed bytes via [`decode_stream`](Self::decode_stream) (unknown frame -/// boundaries, e.g. stdin), [`decode_frame`](Self::decode_frame) (one complete -/// access unit per call), or [`decode_from`](Self::decode_from) (an async -/// reader). Each returns the frames it produced. VPS/SPS/PPS seen inline are -/// cached and re-inserted ahead of each keyframe; [`seed`](Self::seed) primes -/// that cache from an out-of-band parameter-set buffer. +/// Feed bytes via [`decode`](Self::decode) (unknown frame boundaries, e.g. +/// stdin) or [`decode_from`](Self::decode_from) (an async reader); call +/// [`flush`](Self::flush) to emit the final in-flight access unit. Each returns +/// the frames it produced. VPS/SPS/PPS seen inline are cached and re-inserted +/// ahead of each keyframe; [`seed`](Self::seed) primes that cache from an +/// out-of-band parameter-set buffer. pub struct Split { + /// Bytes carried over between calls: complete NALs are parsed out on each + /// [`decode`](Self::decode), leaving the in-flight (final, not-yet-terminated) + /// NAL here until the next start code arrives or [`flush`](Self::flush) drains it. + tail: BytesMut, current: Au, vps: Option, sps: Option, @@ -53,6 +57,7 @@ impl Split { /// A fresh splitter with an empty parameter-set cache. pub fn new() -> Self { Self { + tail: BytesMut::new(), current: Au::default(), vps: None, sps: None, @@ -90,44 +95,51 @@ impl Split { let mut frames = Vec::new(); let mut buffer = BytesMut::new(); while reader.read_buf(&mut buffer).await? > 0 { - frames.extend(self.decode_stream(&mut buffer, None)?); + frames.extend(self.decode(&mut buffer, None)?); } + frames.extend(self.flush(None)?); Ok(frames) } - /// Decode a buffer where frame boundaries are unknown, returning the frames - /// it produced. The leading start code of the *next* access unit is what - /// signals the previous one is complete, so the final access unit stays - /// buffered until the next call (or [`decode_frame`](Self::decode_frame)). - pub fn decode_stream>( + /// Decode a buffer where frame boundaries are unknown, returning the access + /// units it can complete. The leading start code of the *next* access unit is + /// what signals the previous one is complete, so the final NAL of the in-flight + /// access unit stays buffered until the next call (or [`flush`](Self::flush)). + /// The buffer is fully consumed. + pub fn decode>( &mut self, buf: &mut T, pts: impl Into>, ) -> Result> { let pts = self.pts(pts.into())?; - let nals = NalIterator::new(buf); + while buf.has_remaining() { + let chunk = buf.chunk(); + self.tail.extend_from_slice(chunk); + let len = chunk.len(); + buf.advance(len); + } + // Iterate complete NALs out of `tail`, leaving the trailing (in-flight) NAL + // (with its start code) buffered for the next call or `flush`. + let nals = NalIterator::new(&mut self.tail); + let mut parsed = Vec::new(); for nal in nals { - self.decode_nal(nal?, pts)?; + parsed.push(nal?); + } + for nal in parsed { + self.decode_nal(nal, pts)?; } Ok(std::mem::take(&mut self.pending)) } - /// Decode a buffer holding one complete access unit, returning the frames it - /// produced (typically one). Any trailing NAL without a start code is the - /// last NAL of this access unit, and the unit is flushed before returning. - pub fn decode_frame>( - &mut self, - buf: &mut T, - pts: impl Into>, - ) -> Result> { + /// Emit the in-flight access unit, if any. Call after the last + /// [`decode`](Self::decode) when a caller handed over a complete access unit + /// (or at end of stream) so the final NAL isn't left buffered. + pub fn flush(&mut self, pts: impl Into>) -> Result> { let pts = self.pts(pts.into())?; - let mut nals = NalIterator::new(buf); - while let Some(nal) = nals.next().transpose()? { - self.decode_nal(nal, pts)?; - } - if let Some(nal) = nals.flush()? { + if let Some(nal) = NalIterator::new(&mut self.tail).flush()? { self.decode_nal(nal, pts)?; } + self.tail.clear(); self.maybe_start_frame(pts)?; Ok(std::mem::take(&mut self.pending)) } @@ -268,6 +280,7 @@ impl Split { /// self-contained. pub fn reset(&mut self) { self.current = Au::default(); + self.tail.clear(); } fn pts(&mut self, hint: Option) -> Result { @@ -313,12 +326,20 @@ mod tests { haystack.windows(needle.len()).any(|w| w == needle) } + /// Decode one complete access unit handed over as a single buffer: `decode` + /// buffers it, `flush` emits it. + fn decode_one(split: &mut Split, buf: &mut BytesMut, pts: moq_net::Timestamp) -> Vec { + let mut frames = split.decode(buf, pts).unwrap(); + frames.extend(split.flush(pts).unwrap()); + frames + } + /// A keyframe access unit fed as one buffer emits one self-contained frame: /// VPS+SPS+PPS are packaged ahead of the IDR slice and `keyframe` is set. #[tokio::test(start_paused = true)] - async fn decode_frame_packages_keyframe() { + async fn decode_packages_keyframe() { let mut split = Split::new(); - let frames = split.decode_frame(&mut annexb(&[VPS, SPS, PPS, IDR]), ts()).unwrap(); + let frames = decode_one(&mut split, &mut annexb(&[VPS, SPS, PPS, IDR]), ts()); assert_eq!(frames.len(), 1); assert!(frames[0].keyframe); @@ -335,7 +356,7 @@ mod tests { let mut split = Split::new(); split.seed(&mut annexb(&[VPS, SPS, PPS])).unwrap(); - let frames = split.decode_frame(&mut annexb(&[IDR]), ts()).unwrap(); + let frames = decode_one(&mut split, &mut annexb(&[IDR]), ts()); assert_eq!(frames.len(), 1); assert!(frames[0].keyframe); assert!(contains(&frames[0].payload, VPS)); diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index e5ff37177..a3a6422f5 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -265,7 +265,7 @@ impl Import { let track = crate::publish::unique_track(&mut self.broadcast, ".avc3")?; let import = h264::Import::from_track(track); Stream::H264 { - split: h264::Split::new().with_mode(h264::Mode::Avc3), + split: h264::Split::new(), import: Box::new(crate::publish::Published::new(self.catalog.clone(), import)), unwrap: PtsUnwrap::default(), } @@ -744,14 +744,18 @@ impl Stream { Stream::H264 { split, import, unwrap } => { let pts = unwrap_pts(unwrap, pending.pts)?; skip_missing_keyframe((|| { - let frames = split.decode_frame(&mut pending.data.as_slice(), pts)?; + // Each PES is one access unit, so flush to emit it immediately. + let mut frames = split.decode(&mut pending.data.as_slice(), pts)?; + frames.extend(split.flush(pts)?); import.decode(frames) })()) } Stream::H265 { split, import, unwrap } => { let pts = unwrap_pts(unwrap, pending.pts)?; skip_missing_keyframe((|| { - let frames = split.decode_frame(&mut pending.data.as_slice(), pts)?; + // Each PES is one access unit, so flush to emit it immediately. + let mut frames = split.decode(&mut pending.data.as_slice(), pts)?; + frames.extend(split.flush(pts)?); import.decode(frames) })()) } diff --git a/rs/moq-mux/src/import.rs b/rs/moq-mux/src/import.rs index 9aa651605..591320a1b 100644 --- a/rs/moq-mux/src/import.rs +++ b/rs/moq-mux/src/import.rs @@ -102,12 +102,19 @@ impl From for FramedFormat { } enum FramedKind { - /// H.264 (both avc1 and avc3 wire shapes; the split is pinned by the caller's - /// FramedFormat choice). The split owns byte parsing; the import publishes. - H264 { + /// H.264 avc3 (Annex-B, inline SPS/PPS). The split owns byte parsing; the + /// import publishes. + Avc3 { split: crate::codec::h264::Split, import: crate::publish::Published, }, + /// H.264 avc1 (length-prefixed NALU, out-of-band avcC). No splitter: each + /// access unit is wrapped directly. `length_size` is the NALU length prefix + /// width read from the avcC. + Avc1 { + length_size: usize, + import: crate::publish::Published, + }, // Boxed because it's a large struct and clippy complains about the size. Fmp4(Box), Hev1 { @@ -137,15 +144,13 @@ pub struct Framed { decoder: FramedKind, } -/// Build an H.264 split + import pair, resolving the config and consuming `buf`. +/// Build an H.264 avc3 split + import pair, resolving the config and consuming `buf`. /// /// The import reads `buf` for the codec config without consuming it; the split -/// then consumes it (seeding its parameter-set cache for avc3, or reading the -/// NALU length size for avc1). -fn build_h264>( +/// then consumes it, seeding its parameter-set cache. +fn build_h264_avc3>( track: moq_net::TrackProducer, catalog: crate::catalog::Producer, - mode: crate::codec::h264::Mode, buf: &mut T, ) -> Result<( crate::codec::h264::Split, @@ -153,11 +158,26 @@ fn build_h264>( )> { let mut import = crate::codec::h264::Import::from_track(track); import.initialize(buf)?; - let mut split = crate::codec::h264::Split::new().with_mode(mode); - split.initialize(buf)?; + let mut split = crate::codec::h264::Split::new(); + split.seed(buf)?; Ok((split, crate::publish::Published::new(catalog, import))) } +/// Build an H.264 avc1 import, resolving the config and the NALU length size from +/// the avcC, and consuming `buf`. avc1 has no splitter: each access unit is +/// wrapped directly via [`crate::codec::h264::avc1_frame`]. +fn build_h264_avc1>( + track: moq_net::TrackProducer, + catalog: crate::catalog::Producer, + buf: &mut T, +) -> Result<(usize, crate::publish::Published)> { + let mut import = crate::codec::h264::Import::from_track(track); + import.initialize(buf)?; + let length_size = crate::codec::h264::Avcc::parse(buf.as_ref())?.length_size; + buf.advance(buf.remaining()); + Ok((length_size, crate::publish::Published::new(catalog, import))) +} + /// Build an H.265 split + import pair, resolving the config and consuming `buf`. fn build_h265>( track: moq_net::TrackProducer, @@ -208,17 +228,16 @@ impl Framed { format: FramedFormat, buf: &mut T, ) -> Result { - use crate::codec::h264::Mode as H264Mode; let decoder = match format { FramedFormat::Avc1 => { let track = crate::publish::unique_track(&mut broadcast, ".avc1")?; - let (split, import) = build_h264(track, catalog, H264Mode::Avc1, buf)?; - FramedKind::H264 { split, import } + let (length_size, import) = build_h264_avc1(track, catalog, buf)?; + FramedKind::Avc1 { length_size, import } } FramedFormat::Avc3 => { let track = crate::publish::unique_track(&mut broadcast, ".avc3")?; - let (split, import) = build_h264(track, catalog, H264Mode::Avc3, buf)?; - FramedKind::H264 { split, import } + let (split, import) = build_h264_avc3(track, catalog, buf)?; + FramedKind::Avc3 { split, import } } FramedFormat::Fmp4 => { let mut decoder = Box::new(crate::container::fmp4::Import::new(broadcast, catalog)); @@ -293,15 +312,14 @@ impl Framed { format: FramedFormat, buf: &mut T, ) -> anyhow::Result { - use crate::codec::h264::Mode as H264Mode; let decoder = match format { FramedFormat::Avc1 => { - let (split, import) = build_h264(track, catalog, H264Mode::Avc1, buf)?; - FramedKind::H264 { split, import } + let (length_size, import) = build_h264_avc1(track, catalog, buf)?; + FramedKind::Avc1 { length_size, import } } FramedFormat::Avc3 => { - let (split, import) = build_h264(track, catalog, H264Mode::Avc3, buf)?; - FramedKind::H264 { split, import } + let (split, import) = build_h264_avc3(track, catalog, buf)?; + FramedKind::Avc3 { split, import } } FramedFormat::Hev1 => { let (split, import) = build_h265(track, catalog, buf)?; @@ -344,7 +362,8 @@ impl Framed { /// Finish the decoder, flushing any buffered data. pub fn finish(&mut self) -> Result<()> { match self.decoder { - FramedKind::H264 { ref mut import, .. } => import.finish(), + FramedKind::Avc3 { ref mut import, .. } => import.finish(), + FramedKind::Avc1 { ref mut import, .. } => import.finish(), FramedKind::Fmp4(ref mut decoder) => decoder.finish(), FramedKind::Hev1 { ref mut import, .. } => import.finish(), FramedKind::Av01 { ref mut import, .. } => import.finish(), @@ -361,13 +380,14 @@ impl Framed { /// Close the current group and open the next one at `sequence`. pub fn seek(&mut self, sequence: u64) -> Result<()> { match self.decoder { - FramedKind::H264 { + FramedKind::Avc3 { ref mut split, ref mut import, } => { split.reset(); import.seek(sequence) } + FramedKind::Avc1 { ref mut import, .. } => import.seek(sequence), FramedKind::Fmp4(ref mut decoder) => decoder.seek(sequence), FramedKind::Hev1 { ref mut split, @@ -396,7 +416,8 @@ impl Framed { /// Return the single track produced by this importer. pub fn track(&self) -> Result<&moq_net::TrackProducer> { match self.decoder { - FramedKind::H264 { ref import, .. } => Ok(import.track()), + FramedKind::Avc3 { ref import, .. } => Ok(import.track()), + FramedKind::Avc1 { ref import, .. } => Ok(import.track()), FramedKind::Fmp4(_) => Err(crate::Error::MultipleTracks("fmp4")), FramedKind::Hev1 { ref import, .. } => Ok(import.track()), FramedKind::Av01 { ref import, .. } => Ok(import.track()), @@ -413,26 +434,40 @@ impl Framed { /// Decode a frame from the given buffer. pub fn decode_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { match self.decoder { - FramedKind::H264 { + FramedKind::Avc3 { ref mut split, ref mut import, } => { - let frames = split.decode_frame(buf, pts)?; + // Framed hands over one whole access unit per call, so flush to + // emit it rather than waiting for the next start code. + let mut frames = split.decode(buf, pts)?; + frames.extend(split.flush(pts)?); import.decode(frames)?; } + FramedKind::Avc1 { + length_size, + ref mut import, + } => { + let pts = pts.ok_or(crate::codec::h264::Error::MissingTimestamp)?; + let frame = crate::codec::h264::avc1_frame(buf.as_ref(), length_size, pts)?; + import.decode([frame])?; + buf.advance(buf.remaining()); + } FramedKind::Fmp4(ref mut decoder) => decoder.decode(buf)?, FramedKind::Hev1 { ref mut split, ref mut import, } => { - let frames = split.decode_frame(buf, pts)?; + let mut frames = split.decode(buf, pts)?; + frames.extend(split.flush(pts)?); import.decode(frames)?; } FramedKind::Av01 { ref mut split, ref mut import, } => { - let frames = split.decode_frame(buf, pts)?; + let mut frames = split.decode(buf, pts)?; + frames.extend(split.flush(pts)?); import.decode(frames)?; } FramedKind::Vp8(ref mut decoder) => decoder.decoding(|d| d.decode_frame(buf, pts))?, @@ -780,12 +815,11 @@ impl Stream { catalog: crate::catalog::Producer, format: StreamFormat, ) -> Result { - use crate::codec::h264::Mode as H264Mode; let decoder = match format { StreamFormat::Avc3 => { let track = crate::publish::unique_track(&mut broadcast, ".avc3")?; let import = crate::codec::h264::Import::from_track(track); - let split = crate::codec::h264::Split::new().with_mode(H264Mode::Avc3); + let split = crate::codec::h264::Split::new(); StreamKind::Avc3 { split, import: crate::publish::Published::new(catalog, import), @@ -828,7 +862,7 @@ impl Stream { ref mut import, } => { import.decoding(|d| d.initialize(buf))?; - split.initialize(buf)?; + split.seed(buf)?; } StreamKind::Fmp4(ref mut decoder) => decoder.decode(buf)?, StreamKind::Hev1 { @@ -869,7 +903,7 @@ impl Stream { ref mut split, ref mut import, } => { - let frames = split.decode_stream(buf, None)?; + let frames = split.decode(buf, None)?; import.decode(frames) } StreamKind::Fmp4(ref mut decoder) => decoder.decode(buf), @@ -877,14 +911,14 @@ impl Stream { ref mut split, ref mut import, } => { - let frames = split.decode_stream(buf, None)?; + let frames = split.decode(buf, None)?; import.decode(frames) } StreamKind::Av01 { ref mut split, ref mut import, } => { - let frames = split.decode_stream(buf, None)?; + let frames = split.decode(buf, None)?; import.decode(frames) } StreamKind::Mkv(ref mut decoder) => decoder.decode(buf), @@ -896,10 +930,31 @@ impl Stream { /// Finish the decoder, flushing any buffered data. pub fn finish(&mut self) -> Result<()> { match self.decoder { - StreamKind::Avc3 { ref mut import, .. } => import.finish(), + StreamKind::Avc3 { + ref mut split, + ref mut import, + } => { + let tail = split.flush(None)?; + import.decode(tail)?; + import.finish() + } StreamKind::Fmp4(ref mut decoder) => decoder.finish(), - StreamKind::Hev1 { ref mut import, .. } => import.finish(), - StreamKind::Av01 { ref mut import, .. } => import.finish(), + StreamKind::Hev1 { + ref mut split, + ref mut import, + } => { + let tail = split.flush(None)?; + import.decode(tail)?; + import.finish() + } + StreamKind::Av01 { + ref mut split, + ref mut import, + } => { + let tail = split.flush(None)?; + import.decode(tail)?; + import.finish() + } StreamKind::Mkv(ref mut decoder) => decoder.finish(), StreamKind::Ts(ref mut decoder) => decoder.finish().map_err(Into::into), StreamKind::Flv(ref mut decoder) => decoder.finish().map_err(Into::into), diff --git a/rs/moq-rtc/src/codec/h264.rs b/rs/moq-rtc/src/codec/h264.rs index 03ce46c57..55c36853d 100644 --- a/rs/moq-rtc/src/codec/h264.rs +++ b/rs/moq-rtc/src/codec/h264.rs @@ -19,7 +19,7 @@ impl Bridge { let track = moq_mux::publish::unique_track(&mut broadcast, ".avc3")?; let import = moq_mux::codec::h264::Import::from_track(track); let import = moq_mux::publish::Published::new(catalog, import); - let split = moq_mux::codec::h264::Split::new().with_mode(moq_mux::codec::h264::Mode::Avc3); + let split = moq_mux::codec::h264::Split::new(); Ok(Self { split, import }) } } @@ -29,7 +29,9 @@ impl codec::Bridge for Bridge { let pts = moq_net::Timestamp::from_micros(frame.timestamp_us) .map_err(|err| crate::Error::Other(anyhow::anyhow!("invalid timestamp: {err}")))?; let mut buf = BytesMut::from(frame.payload.as_ref()); - let frames = self.split.decode_frame(&mut buf, Some(pts))?; + // str0m hands over one whole access unit per frame, so flush to emit it. + let mut frames = self.split.decode(&mut buf, Some(pts))?; + frames.extend(self.split.flush(Some(pts))?); self.import.decode(frames)?; Ok(()) } diff --git a/rs/moq-rtc/tests/bitstream.rs b/rs/moq-rtc/tests/bitstream.rs index f6c4c4fd1..ba9c52d29 100644 --- a/rs/moq-rtc/tests/bitstream.rs +++ b/rs/moq-rtc/tests/bitstream.rs @@ -2,7 +2,7 @@ //! //! The H.264 Annex-B -> AVCC conversion is provided by `moq_mux::codec::h264`, //! but the WHIP path depends on the importer parsing the SPS for the catalog -//! and accepting Annex-B input via `decode_frame`. These tests guard against +//! and accepting Annex-B input via the bridge. These tests guard against //! regressions in the contract the moq-rtc bridge depends on. use bytes::{Bytes, BytesMut}; diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index dda0f093a..b08369eee 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -34,7 +34,7 @@ impl Producer { let track = moq_mux::publish::unique_track(&mut broadcast, ".avc3")?; let import = moq_mux::codec::h264::Import::from_track(track); let import = moq_mux::publish::Published::new(catalog, import); - let split = moq_mux::codec::h264::Split::new().with_mode(moq_mux::codec::h264::Mode::Avc3); + let split = moq_mux::codec::h264::Split::new(); Ok(Self { split, import }) } @@ -48,7 +48,9 @@ impl Producer { /// Publish already-encoded Annex-B packets at the given timestamp. pub fn publish(&mut self, packets: Vec, timestamp: Timestamp) -> Result<(), Error> { for mut packet in packets { - let frames = self.split.decode_frame(&mut packet, Some(timestamp))?; + // The encoder emits one whole access unit per packet, so flush to emit it. + let mut frames = self.split.decode(&mut packet, Some(timestamp))?; + frames.extend(self.split.flush(Some(timestamp))?); self.import.decode(frames)?; } Ok(()) From 1ee96588fb3edf3063fc1f717a7f7ebe36f76032 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 17 Jun 2026 02:51:45 +0000 Subject: [PATCH 16/19] moq-mux: drop the unused Split::decode_from No caller used the splitters' decode_from async helper (the container importers have their own). Remove it from h264/h265/av1 Split; a caller that wants to drive a reader loops decode() + flush() itself. --- rs/moq-mux/src/codec/av1/split.rs | 19 +++---------------- rs/moq-mux/src/codec/h264/split.rs | 20 +++----------------- rs/moq-mux/src/codec/h265/split.rs | 20 +++----------------- 3 files changed, 9 insertions(+), 50 deletions(-) diff --git a/rs/moq-mux/src/codec/av1/split.rs b/rs/moq-mux/src/codec/av1/split.rs index 874af3a98..3adbe3324 100644 --- a/rs/moq-mux/src/codec/av1/split.rs +++ b/rs/moq-mux/src/codec/av1/split.rs @@ -10,7 +10,6 @@ use bytes::{Buf, Bytes, BytesMut}; use scuffle_av1::{ObuHeader, ObuType}; -use tokio::io::{AsyncRead, AsyncReadExt}; use super::Error; use crate::Result; @@ -18,10 +17,9 @@ use crate::Result; /// AV1 OBU stream splitter: bytes in, [`Frame`](crate::container::Frame)s out. /// /// Feed bytes via [`decode`](Self::decode) (unknown frame boundaries, e.g. -/// stdin) or [`decode_from`](Self::decode_from) (an async reader); call -/// [`flush`](Self::flush) to emit the final in-flight temporal unit. Each -/// returns the frames it produced. [`seed`](Self::seed) feeds leading metadata -/// OBUs (e.g. a sequence header) into the next frame. +/// stdin); call [`flush`](Self::flush) to emit the final in-flight temporal +/// unit. [`seed`](Self::seed) feeds leading metadata OBUs (e.g. a sequence +/// header) into the next frame. pub struct Split { current: Au, zero: Option, @@ -65,17 +63,6 @@ impl Split { Ok(()) } - /// Decode from an asynchronous reader, returning all frames produced. - pub async fn decode_from(&mut self, reader: &mut T) -> Result> { - let mut frames = Vec::new(); - let mut buffer = BytesMut::new(); - while reader.read_buf(&mut buffer).await? > 0 { - frames.extend(self.decode(&mut buffer, None)?); - } - frames.extend(self.flush(None)?); - Ok(frames) - } - /// Decode a buffer where frame boundaries are unknown, returning the temporal /// units it can complete. The final temporal unit stays buffered until the /// next call (or [`flush`](Self::flush)). diff --git a/rs/moq-mux/src/codec/h264/split.rs b/rs/moq-mux/src/codec/h264/split.rs index 3da2fda54..2aff82435 100644 --- a/rs/moq-mux/src/codec/h264/split.rs +++ b/rs/moq-mux/src/codec/h264/split.rs @@ -14,7 +14,6 @@ //! splitter; wrap one access unit with `super::avc1_frame`. use bytes::{Buf, Bytes, BytesMut}; -use tokio::io::{AsyncRead, AsyncReadExt}; use super::{Error, NAL_TYPE_PPS, NAL_TYPE_SPS}; use crate::Result; @@ -23,11 +22,9 @@ use crate::codec::annexb::{NalIterator, START_CODE}; /// H.264 Annex-B stream splitter: bytes in, [`Frame`](crate::container::Frame)s out. /// /// Feed bytes via [`decode`](Self::decode) (unknown frame boundaries, e.g. -/// stdin) or [`decode_from`](Self::decode_from) (an async reader); call -/// [`flush`](Self::flush) to emit the final in-flight access unit. Each returns -/// the frames it produced. SPS/PPS seen inline are cached and re-inserted ahead -/// of each keyframe; [`seed`](Self::seed) primes that cache from an out-of-band -/// parameter-set buffer. +/// stdin); call [`flush`](Self::flush) to emit the final in-flight access unit. +/// SPS/PPS seen inline are cached and re-inserted ahead of each keyframe; +/// [`seed`](Self::seed) primes that cache from an out-of-band parameter-set buffer. pub struct Split { /// Bytes carried over between calls: complete NALs are parsed out on each /// [`decode`](Self::decode), leaving the in-flight (final, not-yet-terminated) @@ -90,17 +87,6 @@ impl Split { } } - /// Decode from an asynchronous reader, returning all frames produced. - pub async fn decode_from(&mut self, reader: &mut T) -> Result> { - let mut frames = Vec::new(); - let mut buffer = BytesMut::new(); - while reader.read_buf(&mut buffer).await? > 0 { - frames.extend(self.decode(&mut buffer, None)?); - } - frames.extend(self.flush(None)?); - Ok(frames) - } - /// Decode a buffer where frame boundaries are unknown, returning the access /// units it can complete. The leading start code of the *next* access unit is /// what signals the previous one is complete, so the final NAL of the in-flight diff --git a/rs/moq-mux/src/codec/h265/split.rs b/rs/moq-mux/src/codec/h265/split.rs index ce4f509d5..f282f205d 100644 --- a/rs/moq-mux/src/codec/h265/split.rs +++ b/rs/moq-mux/src/codec/h265/split.rs @@ -10,7 +10,6 @@ use bytes::{Buf, Bytes, BytesMut}; use scuffle_h265::NALUnitType; -use tokio::io::{AsyncRead, AsyncReadExt}; use super::Error; use crate::Result; @@ -19,11 +18,9 @@ use crate::codec::annexb::{NalIterator, START_CODE}; /// H.265 Annex-B stream splitter: bytes in, [`Frame`](crate::container::Frame)s out. /// /// Feed bytes via [`decode`](Self::decode) (unknown frame boundaries, e.g. -/// stdin) or [`decode_from`](Self::decode_from) (an async reader); call -/// [`flush`](Self::flush) to emit the final in-flight access unit. Each returns -/// the frames it produced. VPS/SPS/PPS seen inline are cached and re-inserted -/// ahead of each keyframe; [`seed`](Self::seed) primes that cache from an -/// out-of-band parameter-set buffer. +/// stdin); call [`flush`](Self::flush) to emit the final in-flight access unit. +/// VPS/SPS/PPS seen inline are cached and re-inserted ahead of each keyframe; +/// [`seed`](Self::seed) primes that cache from an out-of-band parameter-set buffer. pub struct Split { /// Bytes carried over between calls: complete NALs are parsed out on each /// [`decode`](Self::decode), leaving the in-flight (final, not-yet-terminated) @@ -90,17 +87,6 @@ impl Split { } } - /// Decode from an asynchronous reader, returning all frames produced. - pub async fn decode_from(&mut self, reader: &mut T) -> Result> { - let mut frames = Vec::new(); - let mut buffer = BytesMut::new(); - while reader.read_buf(&mut buffer).await? > 0 { - frames.extend(self.decode(&mut buffer, None)?); - } - frames.extend(self.flush(None)?); - Ok(frames) - } - /// Decode a buffer where frame boundaries are unknown, returning the access /// units it can complete. The leading start code of the *next* access unit is /// what signals the previous one is complete, so the final NAL of the in-flight From e58d075ff8af1e35acec273f59e58285ecb18d85 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 17 Jun 2026 03:07:37 +0000 Subject: [PATCH 17/19] moq-mux: remove Split::seed; feed init bytes as the stream A Split is just a byte stream; a dedicated "here are out-of-band parameter sets" hook (seed) contradicts that. Remove it from all three splits. The dispatchers' init buffer (the codec header passed to Framed::new / Stream::initialize) is now fed to the splitter via `decode` as the leading bytes of the stream: the importer still reads it for the catalog config, and the splitter caches any inline SPS/PPS the same way it would mid-stream, so a "parameter sets once up front, then bare keyframes" encoder still produces self-contained keyframes. av1C (the out-of-band 0x81 config record) is not an OBU stream, so it stays config-only and isn't fed to the splitter. The two seed unit tests now exercise the same property through `decode` (leading params + a later bare IDR -> self-contained keyframe). 265 tests pass; dependents (moq-cli/rtc/video/ffi/libmoq) build. https://claude.ai/code/session_011S7pzcg2XsPP3AExuymac1 --- rs/moq-mux/src/codec/av1/split.rs | 18 +---------- rs/moq-mux/src/codec/h264/split.rs | 38 ++++++----------------- rs/moq-mux/src/codec/h265/split.rs | 37 +++++----------------- rs/moq-mux/src/import.rs | 50 +++++++++++++++++++----------- 4 files changed, 50 insertions(+), 93 deletions(-) diff --git a/rs/moq-mux/src/codec/av1/split.rs b/rs/moq-mux/src/codec/av1/split.rs index 3adbe3324..2ee5186f3 100644 --- a/rs/moq-mux/src/codec/av1/split.rs +++ b/rs/moq-mux/src/codec/av1/split.rs @@ -17,9 +17,7 @@ use crate::Result; /// AV1 OBU stream splitter: bytes in, [`Frame`](crate::container::Frame)s out. /// /// Feed bytes via [`decode`](Self::decode) (unknown frame boundaries, e.g. -/// stdin); call [`flush`](Self::flush) to emit the final in-flight temporal -/// unit. [`seed`](Self::seed) feeds leading metadata OBUs (e.g. a sequence -/// header) into the next frame. +/// stdin); call [`flush`](Self::flush) to emit the final in-flight temporal unit. pub struct Split { current: Au, zero: Option, @@ -49,20 +47,6 @@ impl Split { } } - /// Feed leading metadata OBUs (e.g. a sequence header) into the in-flight - /// access unit without completing a frame, so they prefix the next keyframe. - /// The buffer must not contain a completed frame (no timestamp is available). - pub fn seed>(&mut self, buf: &mut T) -> Result<()> { - let mut obus = ObuIterator::new(buf); - while let Some(obu) = obus.next().transpose()? { - self.decode_obu(obu, None)?; - } - if let Some(obu) = obus.flush()? { - self.decode_obu(obu, None)?; - } - Ok(()) - } - /// Decode a buffer where frame boundaries are unknown, returning the temporal /// units it can complete. The final temporal unit stays buffered until the /// next call (or [`flush`](Self::flush)). diff --git a/rs/moq-mux/src/codec/h264/split.rs b/rs/moq-mux/src/codec/h264/split.rs index 2aff82435..76431cc04 100644 --- a/rs/moq-mux/src/codec/h264/split.rs +++ b/rs/moq-mux/src/codec/h264/split.rs @@ -15,7 +15,7 @@ use bytes::{Buf, Bytes, BytesMut}; -use super::{Error, NAL_TYPE_PPS, NAL_TYPE_SPS}; +use super::Error; use crate::Result; use crate::codec::annexb::{NalIterator, START_CODE}; @@ -23,8 +23,8 @@ use crate::codec::annexb::{NalIterator, START_CODE}; /// /// Feed bytes via [`decode`](Self::decode) (unknown frame boundaries, e.g. /// stdin); call [`flush`](Self::flush) to emit the final in-flight access unit. -/// SPS/PPS seen inline are cached and re-inserted ahead of each keyframe; -/// [`seed`](Self::seed) primes that cache from an out-of-band parameter-set buffer. +/// SPS/PPS seen inline are cached and re-inserted ahead of each keyframe so each +/// keyframe is self-contained. pub struct Split { /// Bytes carried over between calls: complete NALs are parsed out on each /// [`decode`](Self::decode), leaving the in-flight (final, not-yet-terminated) @@ -65,28 +65,6 @@ impl Split { } } - /// Prime the SPS/PPS cache from an Annex-B parameter-set buffer, so the first - /// keyframe is self-contained even if the stream itself omits inline - /// parameter sets. Other NAL types in the buffer are ignored. - pub fn seed>(&mut self, buf: &mut T) -> Result<()> { - let mut nals = NalIterator::new(buf); - while let Some(nal) = nals.next().transpose()? { - self.cache_param(&nal); - } - if let Some(nal) = nals.flush()? { - self.cache_param(&nal); - } - Ok(()) - } - - fn cache_param(&mut self, nal: &Bytes) { - match nal.first().map(|h| h & 0x1f) { - Some(NAL_TYPE_SPS) => self.sps = Some(nal.clone()), - Some(NAL_TYPE_PPS) => self.pps = Some(nal.clone()), - _ => {} - } - } - /// Decode a buffer where frame boundaries are unknown, returning the access /// units it can complete. The leading start code of the *next* access unit is /// what signals the previous one is complete, so the final NAL of the in-flight @@ -308,16 +286,18 @@ mod tests { assert!(frames[0].payload.windows(idr.len()).any(|w| w == idr)); } - /// A seeded splitter re-inserts the cached SPS/PPS ahead of a bare IDR slice, - /// even though the stream itself never carried inline parameter sets. + /// Parameter sets fed up front (as the leading stream bytes) are cached and + /// re-inserted ahead of a later bare IDR, so the keyframe is self-contained + /// even when the stream never repeats its parameter sets inline. #[tokio::test(start_paused = true)] - async fn seed_makes_bare_keyframe_self_contained() { + async fn params_then_bare_keyframe_self_contained() { let sps: &[u8] = &[0x67, 0x42, 0xc0, 0x1f]; let pps: &[u8] = &[0x68, 0xce, 0x3c, 0x80]; let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; let mut split = Split::new(); - split.seed(&mut annexb(&[sps, pps])).unwrap(); + // The leading SPS/PPS carry no slice, so they complete no frame yet. + assert!(split.decode(&mut annexb(&[sps, pps]), ts()).unwrap().is_empty()); let frames = decode_one(&mut split, &mut annexb(&[idr]), ts()); assert_eq!(frames.len(), 1); diff --git a/rs/moq-mux/src/codec/h265/split.rs b/rs/moq-mux/src/codec/h265/split.rs index f282f205d..fc8180c29 100644 --- a/rs/moq-mux/src/codec/h265/split.rs +++ b/rs/moq-mux/src/codec/h265/split.rs @@ -19,8 +19,8 @@ use crate::codec::annexb::{NalIterator, START_CODE}; /// /// Feed bytes via [`decode`](Self::decode) (unknown frame boundaries, e.g. /// stdin); call [`flush`](Self::flush) to emit the final in-flight access unit. -/// VPS/SPS/PPS seen inline are cached and re-inserted ahead of each keyframe; -/// [`seed`](Self::seed) primes that cache from an out-of-band parameter-set buffer. +/// VPS/SPS/PPS seen inline are cached and re-inserted ahead of each keyframe so +/// each keyframe is self-contained. pub struct Split { /// Bytes carried over between calls: complete NALs are parsed out on each /// [`decode`](Self::decode), leaving the in-flight (final, not-yet-terminated) @@ -64,29 +64,6 @@ impl Split { } } - /// Prime the VPS/SPS/PPS cache from an Annex-B parameter-set buffer, so the - /// first keyframe is self-contained even if the stream itself omits inline - /// parameter sets. Other NAL types in the buffer are ignored. - pub fn seed>(&mut self, buf: &mut T) -> Result<()> { - let mut nals = NalIterator::new(buf); - while let Some(nal) = nals.next().transpose()? { - self.cache_param(&nal); - } - if let Some(nal) = nals.flush()? { - self.cache_param(&nal); - } - Ok(()) - } - - fn cache_param(&mut self, nal: &Bytes) { - match nal.first().map(|h| nal_unit_type(*h)) { - Some(NALUnitType::VpsNut) => self.vps = Some(nal.clone()), - Some(NALUnitType::SpsNut) => self.sps = Some(nal.clone()), - Some(NALUnitType::PpsNut) => self.pps = Some(nal.clone()), - _ => {} - } - } - /// Decode a buffer where frame boundaries are unknown, returning the access /// units it can complete. The leading start code of the *next* access unit is /// what signals the previous one is complete, so the final NAL of the in-flight @@ -335,12 +312,14 @@ mod tests { assert!(contains(&frames[0].payload, IDR)); } - /// A seeded splitter re-inserts the cached VPS/SPS/PPS ahead of a bare IDR, - /// even though the stream itself never carried inline parameter sets. + /// Parameter sets fed up front (as the leading stream bytes) are cached and + /// re-inserted ahead of a later bare IDR, so the keyframe is self-contained + /// even when the stream never repeats its parameter sets inline. #[tokio::test(start_paused = true)] - async fn seed_makes_bare_keyframe_self_contained() { + async fn params_then_bare_keyframe_self_contained() { let mut split = Split::new(); - split.seed(&mut annexb(&[VPS, SPS, PPS])).unwrap(); + // The leading VPS/SPS/PPS carry no slice, so they complete no frame yet. + assert!(split.decode(&mut annexb(&[VPS, SPS, PPS]), ts()).unwrap().is_empty()); let frames = decode_one(&mut split, &mut annexb(&[IDR]), ts()); assert_eq!(frames.len(), 1); diff --git a/rs/moq-mux/src/import.rs b/rs/moq-mux/src/import.rs index 591320a1b..25808b74d 100644 --- a/rs/moq-mux/src/import.rs +++ b/rs/moq-mux/src/import.rs @@ -146,8 +146,9 @@ pub struct Framed { /// Build an H.264 avc3 split + import pair, resolving the config and consuming `buf`. /// -/// The import reads `buf` for the codec config without consuming it; the split -/// then consumes it, seeding its parameter-set cache. +/// The import reads `buf` for the codec config (without consuming it); the split +/// then consumes it as the leading bytes of the stream (caching any inline +/// SPS/PPS). Any frames in the init buffer are published. fn build_h264_avc3>( track: moq_net::TrackProducer, catalog: crate::catalog::Producer, @@ -159,8 +160,10 @@ fn build_h264_avc3>( let mut import = crate::codec::h264::Import::from_track(track); import.initialize(buf)?; let mut split = crate::codec::h264::Split::new(); - split.seed(buf)?; - Ok((split, crate::publish::Published::new(catalog, import))) + let frames = split.decode(buf, None)?; + let mut published = crate::publish::Published::new(catalog, import); + published.decode(frames)?; + Ok((split, published)) } /// Build an H.264 avc1 import, resolving the config and the NALU length size from @@ -190,8 +193,10 @@ fn build_h265>( let mut import = crate::codec::h265::Import::from_track(track); import.initialize(buf)?; let mut split = crate::codec::h265::Split::new(); - split.seed(buf)?; - Ok((split, crate::publish::Published::new(catalog, import))) + let frames = split.decode(buf, None)?; + let mut published = crate::publish::Published::new(catalog, import); + published.decode(frames)?; + Ok((split, published)) } /// Build an AV1 split + import pair, resolving the config and consuming `buf`. @@ -206,16 +211,19 @@ fn build_av1>( let mut import = crate::codec::av1::Import::from_track(track); import.initialize(buf)?; let mut split = crate::codec::av1::Split::new(); - // av1C (leading 0x81, ISO/IEC 14496-15) is config-only and not fed to the - // splitter; raw OBUs seed it so the sequence header prefixes the first - // keyframe. Mirror the importer's av1C detection exactly. + // av1C (leading 0x81, ISO/IEC 14496-15) is an out-of-band config record, not + // an OBU stream, so it's read for config (above) and dropped here. Raw OBUs + // are the leading bytes of the stream and feed the splitter. let data = buf.as_ref(); - if data.len() >= 16 && data[0] == 0x81 { + let frames = if data.len() >= 16 && data[0] == 0x81 { buf.advance(buf.remaining()); + Vec::new() } else { - split.seed(buf)?; - } - Ok((split, crate::publish::Published::new(catalog, import))) + split.decode(buf, None)? + }; + let mut published = crate::publish::Published::new(catalog, import); + published.decode(frames)?; + Ok((split, published)) } impl Framed { @@ -862,7 +870,8 @@ impl Stream { ref mut import, } => { import.decoding(|d| d.initialize(buf))?; - split.seed(buf)?; + let frames = split.decode(buf, None)?; + import.decode(frames)?; } StreamKind::Fmp4(ref mut decoder) => decoder.decode(buf)?, StreamKind::Hev1 { @@ -870,19 +879,24 @@ impl Stream { ref mut import, } => { import.decoding(|d| d.initialize(buf))?; - split.seed(buf)?; + let frames = split.decode(buf, None)?; + import.decode(frames)?; } StreamKind::Av01 { ref mut split, ref mut import, } => { import.decoding(|d| d.initialize(buf))?; + // av1C (leading 0x81) is an out-of-band config record, not an OBU + // stream; read for config above and dropped here. let data = buf.as_ref(); - if data.len() >= 16 && data[0] == 0x81 { + let frames = if data.len() >= 16 && data[0] == 0x81 { buf.advance(buf.remaining()); + Vec::new() } else { - split.seed(buf)?; - } + split.decode(buf, None)? + }; + import.decode(frames)?; } StreamKind::Mkv(ref mut decoder) => decoder.decode(buf)?, StreamKind::Ts(ref mut decoder) => decoder.decode(buf)?, From de7cfb33aeeebfb33204c129ede2d79bacfaeab5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 17 Jun 2026 09:35:39 -0700 Subject: [PATCH 18/19] moq-mux: fold publish into import; add moq_net::TrackDemand Merge the new `publish` module into `import` and rename `Published` to `import::Track`. The module is now a directory: `import/mod.rs` (the `Framed`/`Stream` format dispatchers) plus a private `import/track.rs` (the catalog-bridge `Track`, `Renditions`, `FrameDecode`, `unique_track`), re-exported flat. The `publish::Published` stutter is gone, and "import" already names the ingest direction (the mirror of `export`). Add `moq_net::TrackDemand`, a cloneable, weak-backed watch-only handle (`name`/`used`/`unused`/`closed`) obtained from `TrackProducer::demand()`. It can't publish or close the track and doesn't pin the group cache, so callers can gate on subscriber demand without holding a writable producer. `TrackProducer: Clone` is left intact for now. Encapsulate the high-level front door: `Framed` no longer hands out a `&TrackProducer`. The match is now a private `producer()` helper behind curated `name()` / `subscribe()` / `demand()` accessors. moq-ffi's `MediaProducer` holds a `TrackDemand` instead of a cloned producer. The low-level `Track` and codec importers keep their public `track()`, since their callers build the track themselves, so moq-video/boy/audio/rtc/cli need no changes. Co-Authored-By: Claude Opus 4.8 (1M context) --- rs/moq-cli/src/publish.rs | 6 +- rs/moq-ffi/src/producer.rs | 27 ++- rs/moq-gst/src/sink/imp.rs | 4 +- rs/moq-mux/src/codec/aac/import.rs | 4 +- rs/moq-mux/src/codec/av1/import.rs | 2 +- rs/moq-mux/src/codec/h264/import.rs | 4 +- rs/moq-mux/src/codec/h265/import.rs | 2 +- rs/moq-mux/src/codec/legacy.rs | 4 +- rs/moq-mux/src/codec/opus/import.rs | 4 +- rs/moq-mux/src/codec/vp8/import.rs | 2 +- rs/moq-mux/src/codec/vp9/import.rs | 2 +- rs/moq-mux/src/container/ts/import.rs | 24 +-- rs/moq-mux/src/{import.rs => import/mod.rs} | 156 +++++++++++------- .../src/{publish.rs => import/track.rs} | 28 ++-- rs/moq-mux/src/lib.rs | 1 - rs/moq-net/src/model/track.rs | 57 +++++++ rs/moq-rtc/src/codec/h264.rs | 6 +- rs/moq-rtc/src/codec/opus.rs | 6 +- rs/moq-video/src/encode/producer.rs | 6 +- 19 files changed, 219 insertions(+), 126 deletions(-) rename rs/moq-mux/src/{import.rs => import/mod.rs} (85%) rename rs/moq-mux/src/{publish.rs => import/track.rs} (89%) diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 10c0ebc24..b0348305d 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -82,7 +82,7 @@ pub struct CaptureArgs { enum PublishDecoder { Avc3 { split: moq_mux::codec::h264::Split, - import: Box>, + import: Box>, }, Fmp4(Box), Ts(Box), @@ -147,9 +147,9 @@ impl Publish { let source = match format { PublishFormat::Avc3 => { - let track = moq_mux::publish::unique_track(&mut broadcast, ".avc3")?; + let track = moq_mux::import::unique_track(&mut broadcast, ".avc3")?; let import = moq_mux::codec::h264::Import::from_track(track); - let import = moq_mux::publish::Published::new(catalog.clone(), import); + let import = moq_mux::import::Track::new(catalog.clone(), import); let split = moq_mux::codec::h264::Split::new(); Source::Stream(PublishDecoder::Avc3 { split, diff --git a/rs/moq-ffi/src/producer.rs b/rs/moq-ffi/src/producer.rs index 0dc04040f..a883515f4 100644 --- a/rs/moq-ffi/src/producer.rs +++ b/rs/moq-ffi/src/producer.rs @@ -16,7 +16,7 @@ pub(crate) struct BroadcastProducer { struct MediaProducer { decoder: moq_mux::import::Framed, - track: moq_net::TrackProducer, + demand: moq_net::TrackDemand, } struct MediaStreamProducer { @@ -135,13 +135,12 @@ impl MoqBroadcastProducer { return Err(MoqError::Codec("init failed: trailing bytes".into())); } - let track = decoder - .track() - .map_err(|err| MoqError::Codec(format!("track unavailable: {err}")))? - .clone(); + let demand = decoder + .demand() + .map_err(|err| MoqError::Codec(format!("track unavailable: {err}")))?; Ok(Arc::new(MoqMediaProducer { - inner: std::sync::Mutex::new(Some(MediaProducer { decoder, track })), + inner: std::sync::Mutex::new(Some(MediaProducer { decoder, demand })), })) } @@ -182,7 +181,7 @@ impl MoqBroadcastProducer { Ok(Arc::new(MoqMediaProducer { inner: std::sync::Mutex::new(Some(MediaProducer { decoder, - track: track_clone, + demand: track_clone.demand(), })), })) } @@ -393,20 +392,20 @@ impl MoqMediaProducer { let _guard = crate::ffi::RUNTIME.enter(); let guard = self.inner.lock().unwrap(); let media = guard.as_ref().ok_or_else(|| MoqError::Closed)?; - Ok(media.track.name().to_string()) + Ok(media.demand.name().to_string()) } /// Wait until this media track has at least one active consumer. pub async fn used(&self) -> Result<(), MoqError> { - let track = self + let demand = self .inner .lock() .unwrap() .as_ref() .ok_or(MoqError::Closed)? - .track + .demand .clone(); - match crate::ffi::RUNTIME.spawn(async move { track.used().await }).await { + match crate::ffi::RUNTIME.spawn(async move { demand.used().await }).await { Ok(result) => result.map_err(Into::into), Err(e) if e.is_cancelled() => Err(MoqError::Cancelled), Err(e) => Err(MoqError::Task(e)), @@ -415,15 +414,15 @@ impl MoqMediaProducer { /// Wait until this media track has no active consumers. pub async fn unused(&self) -> Result<(), MoqError> { - let track = self + let demand = self .inner .lock() .unwrap() .as_ref() .ok_or(MoqError::Closed)? - .track + .demand .clone(); - match crate::ffi::RUNTIME.spawn(async move { track.unused().await }).await { + match crate::ffi::RUNTIME.spawn(async move { demand.unused().await }).await { Ok(result) => result.map_err(Into::into), Err(e) if e.is_cancelled() => Err(MoqError::Cancelled), Err(e) => Err(MoqError::Task(e)), diff --git a/rs/moq-gst/src/sink/imp.rs b/rs/moq-gst/src/sink/imp.rs index 2a9cf94fc..3fc2c18fe 100644 --- a/rs/moq-gst/src/sink/imp.rs +++ b/rs/moq-gst/src/sink/imp.rs @@ -486,9 +486,9 @@ fn handle_caps(runtime: &mut RuntimeState, pad_name: String, caps: gst::Caps) -> sample_rate, channel_count, }; - let track = moq_mux::publish::unique_track(&mut runtime.broadcast, ".opus")?; + let track = moq_mux::import::unique_track(&mut runtime.broadcast, ".opus")?; let import = moq_mux::codec::opus::Import::from_track(track, config)?; - moq_mux::publish::Published::new(runtime.catalog.clone(), import).into() + moq_mux::import::Track::new(runtime.catalog.clone(), import).into() } other => anyhow::bail!("unsupported caps: {}", other), }; diff --git a/rs/moq-mux/src/codec/aac/import.rs b/rs/moq-mux/src/codec/aac/import.rs index 1699d73cb..480334fd3 100644 --- a/rs/moq-mux/src/codec/aac/import.rs +++ b/rs/moq-mux/src/codec/aac/import.rs @@ -1,7 +1,7 @@ use bytes::{Buf, BytesMut}; use super::Config; -use crate::publish::Renditions; +use crate::import::Renditions; /// AAC importer. /// @@ -59,7 +59,7 @@ impl Import { /// Mutable access to the single audio rendition, for callers that refine it /// after construction (the TS importer sets the synthesized `description` and - /// an audio-burst `jitter`). Follow with [`crate::publish::Published::sync`]. + /// an audio-burst `jitter`). Follow with [`crate::import::Track::sync`]. pub(crate) fn rendition_mut(&mut self) -> Option<&mut hang::catalog::AudioConfig> { let name = self.track.name(); self.catalog.audio.renditions.get_mut(name) diff --git a/rs/moq-mux/src/codec/av1/import.rs b/rs/moq-mux/src/codec/av1/import.rs index b61cc2602..f04769ced 100644 --- a/rs/moq-mux/src/codec/av1/import.rs +++ b/rs/moq-mux/src/codec/av1/import.rs @@ -20,7 +20,7 @@ use super::split::ObuIterator; use crate::Result; use crate::container::Frame; use crate::container::jitter::MinFrameDuration; -use crate::publish::{FrameDecode, Renditions}; +use crate::import::{FrameDecode, Renditions}; /// A pure-publisher importer for AV1 with inline sequence headers. /// diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index ad0ee828e..a23cfe595 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -18,7 +18,7 @@ use crate::Result; use crate::codec::annexb::NalIterator; use crate::container::Frame; use crate::container::jitter::MinFrameDuration; -use crate::publish::{FrameDecode, Renditions}; +use crate::import::{FrameDecode, Renditions}; /// H.264 importer: a pure frame publisher that resolves the catalog rendition. /// @@ -28,7 +28,7 @@ use crate::publish::{FrameDecode, Renditions}; /// the [`FrameDecode`] impl. The catalog rendition fills in lazily once the codec /// config is known (avcC via [`initialize`](Self::initialize) for avc1, the first /// SPS for avc3); read it via [`catalog`](Self::catalog) or attach the importer to -/// a broadcast catalog with [`crate::publish::Published`]. +/// a broadcast catalog with [`crate::import::Track`]. pub struct Import { /// True for the avc1 shape: the codec config is out-of-band (avcC), so /// keyframes are not scanned for an inline SPS. diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index f6c28e972..6178cd3c4 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -20,7 +20,7 @@ use crate::Result; use crate::codec::annexb::NalIterator; use crate::container::Frame; use crate::container::jitter::MinFrameDuration; -use crate::publish::{FrameDecode, Renditions}; +use crate::import::{FrameDecode, Renditions}; /// A pure-publisher importer for H.265 with inline VPS/SPS/PPS. /// Only supports single layer streams (VPS is cached but not parsed). diff --git a/rs/moq-mux/src/codec/legacy.rs b/rs/moq-mux/src/codec/legacy.rs index 3c9c09620..3c6af9da9 100644 --- a/rs/moq-mux/src/codec/legacy.rs +++ b/rs/moq-mux/src/codec/legacy.rs @@ -10,7 +10,7 @@ use bytes::{Buf, BytesMut}; use moq_net::Timestamp; -use crate::publish::Renditions; +use crate::import::Renditions; /// Legacy audio (MP2 / AC-3 / E-AC-3) header parsing errors. #[derive(Debug, Clone, thiserror::Error)] @@ -121,7 +121,7 @@ pub(crate) struct Import { impl Import { /// Publish on an existing track. Mint it at the descriptor's suffix and the - /// microsecond [`hang::container::TIMESCALE`] (e.g. via [`crate::publish::unique_track`]). + /// microsecond [`hang::container::TIMESCALE`] (e.g. via [`crate::import::unique_track`]). pub fn from_track(descriptor: &'static Descriptor, track: moq_net::TrackProducer, config: Config) -> Self { let mut audio_config = hang::catalog::AudioConfig::new(descriptor.codec.clone(), config.sample_rate, config.channel_count); diff --git a/rs/moq-mux/src/codec/opus/import.rs b/rs/moq-mux/src/codec/opus/import.rs index d00dd12ee..b7ebdde58 100644 --- a/rs/moq-mux/src/codec/opus/import.rs +++ b/rs/moq-mux/src/codec/opus/import.rs @@ -2,7 +2,7 @@ use bytes::{Buf, BytesMut}; use super::Config; use crate::container::Frame; -use crate::publish::Renditions; +use crate::import::Renditions; /// Opus importer. /// @@ -14,7 +14,7 @@ use crate::publish::Renditions; /// immediately without waiting for a group boundary; Opus' packet loss /// concealment handles drops. The catalog rendition this importer publishes is /// available via [`catalog`](Self::catalog); attach it to a broadcast catalog -/// with [`crate::publish::Published`]. +/// with [`crate::import::Track`]. pub struct Import { track: crate::container::Producer, catalog: hang::Catalog, diff --git a/rs/moq-mux/src/codec/vp8/import.rs b/rs/moq-mux/src/codec/vp8/import.rs index b4a21fb1f..210112ecb 100644 --- a/rs/moq-mux/src/codec/vp8/import.rs +++ b/rs/moq-mux/src/codec/vp8/import.rs @@ -1,7 +1,7 @@ use bytes::Buf; use crate::container::jitter::MinFrameDuration; -use crate::publish::Renditions; +use crate::import::Renditions; use super::FrameHeader; diff --git a/rs/moq-mux/src/codec/vp9/import.rs b/rs/moq-mux/src/codec/vp9/import.rs index d6cff8c2f..1b28e3ded 100644 --- a/rs/moq-mux/src/codec/vp9/import.rs +++ b/rs/moq-mux/src/codec/vp9/import.rs @@ -1,7 +1,7 @@ use bytes::Buf; use crate::container::jitter::MinFrameDuration; -use crate::publish::Renditions; +use crate::import::Renditions; use super::FrameHeader; diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index a3a6422f5..70e38c23c 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -262,19 +262,19 @@ impl Import { let stream = match stream_type { StreamType::H264 => { - let track = crate::publish::unique_track(&mut self.broadcast, ".avc3")?; + let track = crate::import::unique_track(&mut self.broadcast, ".avc3")?; let import = h264::Import::from_track(track); Stream::H264 { split: h264::Split::new(), - import: Box::new(crate::publish::Published::new(self.catalog.clone(), import)), + import: Box::new(crate::import::Track::new(self.catalog.clone(), import)), unwrap: PtsUnwrap::default(), } } StreamType::H265 => { - let track = crate::publish::unique_track(&mut self.broadcast, ".hev1")?; + let track = crate::import::unique_track(&mut self.broadcast, ".hev1")?; Stream::H265 { split: h265::Split::new(), - import: Box::new(crate::publish::Published::new( + import: Box::new(crate::import::Track::new( self.catalog.clone(), h265::Import::from_track(track), )), @@ -722,12 +722,12 @@ impl ScteReassembler { enum Stream { H264 { split: h264::Split, - import: Box>, + import: Box>, unwrap: PtsUnwrap, }, H265 { split: h265::Split, - import: Box>, + import: Box>, unwrap: PtsUnwrap, }, Aac(Box>), @@ -796,7 +796,7 @@ impl Stream { /// (the sample rate and channel layout aren't in the PMT), so creation is /// deferred until the first frame arrives. struct AacStream { - import: Option>, + import: Option>, broadcast: moq_net::BroadcastProducer, catalog: crate::catalog::Producer, unwrap: PtsUnwrap, @@ -831,13 +831,13 @@ impl AacStream { // downstream consumers that need out-of-band config (fMP4/MKV export, // WebCodecs) can configure the decoder. TS itself carries it inline. let description = config.encode(); - let track = crate::publish::unique_track(&mut self.broadcast, ".aac")?; + let track = crate::import::unique_track(&mut self.broadcast, ".aac")?; let mut aac = aac::Import::from_track(track, config)?; if let Some(rendition) = aac.rendition_mut() { rendition.description = Some(description); } // Published::new mirrors the rendition (description included) on attach. - let import = crate::publish::Published::new(self.catalog.clone(), aac); + let import = crate::import::Track::new(self.catalog.clone(), aac); self.import.insert(import) } }; @@ -929,7 +929,7 @@ impl AacStream { /// players, which cannot decode these codecs. struct LegacyStream { descriptor: &'static legacy::Descriptor, - import: Option>, + import: Option>, broadcast: moq_net::BroadcastProducer, catalog: crate::catalog::Producer, unwrap: PtsUnwrap, @@ -988,10 +988,10 @@ impl LegacyStream { sample_rate: header.sample_rate, channel_count: header.channel_count, }; - let track = crate::publish::unique_track(&mut self.broadcast, self.descriptor.track_suffix)?; + let track = crate::import::unique_track(&mut self.broadcast, self.descriptor.track_suffix)?; let legacy = legacy::Import::from_track(self.descriptor, track, config); self.import - .insert(crate::publish::Published::new(self.catalog.clone(), legacy)) + .insert(crate::import::Track::new(self.catalog.clone(), legacy)) } }; diff --git a/rs/moq-mux/src/import.rs b/rs/moq-mux/src/import/mod.rs similarity index 85% rename from rs/moq-mux/src/import.rs rename to rs/moq-mux/src/import/mod.rs index 25808b74d..838913a58 100644 --- a/rs/moq-mux/src/import.rs +++ b/rs/moq-mux/src/import/mod.rs @@ -1,12 +1,19 @@ -//! Format dispatchers for callers who only have a format string. +//! Import media into a moq broadcast. //! -//! [`Framed`] is the entry point when the caller already has whole -//! frames (the typical case for files and reassembled network input). -//! [`Stream`] is for raw byte streams where frame boundaries have to -//! be inferred (piped Annex-B H.264, an fMP4 reader, …). Both pick a -//! concrete importer from a [`FramedFormat`] / [`StreamFormat`] string. -//! The concrete importers themselves live with their format under -//! [`crate::container`] or [`crate::codec`]. +//! The format dispatchers are the front door for callers who only have a format +//! string. [`Framed`] is the entry point when the caller already has whole frames +//! (the typical case for files and reassembled network input). [`Stream`] is for +//! raw byte streams where frame boundaries have to be inferred (piped Annex-B +//! H.264, an fMP4 reader, …). Both pick a concrete importer from a +//! [`FramedFormat`] / [`StreamFormat`] string. The concrete importers themselves +//! live with their format under [`crate::container`] or [`crate::codec`]. +//! +//! Underneath, [`Track`] binds a bare single-track codec importer to a broadcast +//! catalog, mirroring its [`Renditions`] in and retiring them on drop; +//! [`FrameDecode`] is the contract for handing it already-split frames. + +mod track; +pub use track::*; use std::{fmt, str::FromStr}; @@ -106,29 +113,29 @@ enum FramedKind { /// import publishes. Avc3 { split: crate::codec::h264::Split, - import: crate::publish::Published, + import: crate::import::Track, }, /// H.264 avc1 (length-prefixed NALU, out-of-band avcC). No splitter: each /// access unit is wrapped directly. `length_size` is the NALU length prefix /// width read from the avcC. Avc1 { length_size: usize, - import: crate::publish::Published, + import: crate::import::Track, }, // Boxed because it's a large struct and clippy complains about the size. Fmp4(Box), Hev1 { split: crate::codec::h265::Split, - import: crate::publish::Published, + import: crate::import::Track, }, Av01 { split: crate::codec::av1::Split, - import: crate::publish::Published, + import: crate::import::Track, }, - Vp8(crate::publish::Published), - Vp9(crate::publish::Published), - Aac(crate::publish::Published), - Opus(crate::publish::Published), + Vp8(crate::import::Track), + Vp9(crate::import::Track), + Aac(crate::import::Track), + Opus(crate::import::Track), // Boxed for the same reason as Fmp4. Mkv(Box), // Boxed for the same reason as Fmp4. @@ -155,13 +162,13 @@ fn build_h264_avc3>( buf: &mut T, ) -> Result<( crate::codec::h264::Split, - crate::publish::Published, + crate::import::Track, )> { let mut import = crate::codec::h264::Import::from_track(track); import.initialize(buf)?; let mut split = crate::codec::h264::Split::new(); let frames = split.decode(buf, None)?; - let mut published = crate::publish::Published::new(catalog, import); + let mut published = crate::import::Track::new(catalog, import); published.decode(frames)?; Ok((split, published)) } @@ -173,12 +180,12 @@ fn build_h264_avc1>( track: moq_net::TrackProducer, catalog: crate::catalog::Producer, buf: &mut T, -) -> Result<(usize, crate::publish::Published)> { +) -> Result<(usize, crate::import::Track)> { let mut import = crate::codec::h264::Import::from_track(track); import.initialize(buf)?; let length_size = crate::codec::h264::Avcc::parse(buf.as_ref())?.length_size; buf.advance(buf.remaining()); - Ok((length_size, crate::publish::Published::new(catalog, import))) + Ok((length_size, crate::import::Track::new(catalog, import))) } /// Build an H.265 split + import pair, resolving the config and consuming `buf`. @@ -188,13 +195,13 @@ fn build_h265>( buf: &mut T, ) -> Result<( crate::codec::h265::Split, - crate::publish::Published, + crate::import::Track, )> { let mut import = crate::codec::h265::Import::from_track(track); import.initialize(buf)?; let mut split = crate::codec::h265::Split::new(); let frames = split.decode(buf, None)?; - let mut published = crate::publish::Published::new(catalog, import); + let mut published = crate::import::Track::new(catalog, import); published.decode(frames)?; Ok((split, published)) } @@ -206,7 +213,7 @@ fn build_av1>( buf: &mut T, ) -> Result<( crate::codec::av1::Split, - crate::publish::Published, + crate::import::Track, )> { let mut import = crate::codec::av1::Import::from_track(track); import.initialize(buf)?; @@ -221,7 +228,7 @@ fn build_av1>( } else { split.decode(buf, None)? }; - let mut published = crate::publish::Published::new(catalog, import); + let mut published = crate::import::Track::new(catalog, import); published.decode(frames)?; Ok((split, published)) } @@ -238,12 +245,12 @@ impl Framed { ) -> Result { let decoder = match format { FramedFormat::Avc1 => { - let track = crate::publish::unique_track(&mut broadcast, ".avc1")?; + let track = crate::import::unique_track(&mut broadcast, ".avc1")?; let (length_size, import) = build_h264_avc1(track, catalog, buf)?; FramedKind::Avc1 { length_size, import } } FramedFormat::Avc3 => { - let track = crate::publish::unique_track(&mut broadcast, ".avc3")?; + let track = crate::import::unique_track(&mut broadcast, ".avc3")?; let (split, import) = build_h264_avc3(track, catalog, buf)?; FramedKind::Avc3 { split, import } } @@ -253,38 +260,38 @@ impl Framed { FramedKind::Fmp4(decoder) } FramedFormat::Hev1 => { - let track = crate::publish::unique_track(&mut broadcast, ".hev1")?; + let track = crate::import::unique_track(&mut broadcast, ".hev1")?; let (split, import) = build_h265(track, catalog, buf)?; FramedKind::Hev1 { split, import } } FramedFormat::Av01 => { - let track = crate::publish::unique_track(&mut broadcast, ".av01")?; + let track = crate::import::unique_track(&mut broadcast, ".av01")?; let (split, import) = build_av1(track, catalog, buf)?; FramedKind::Av01 { split, import } } FramedFormat::Vp8 => { - let track = crate::publish::unique_track(&mut broadcast, ".vp8")?; + let track = crate::import::unique_track(&mut broadcast, ".vp8")?; let mut decoder = crate::codec::vp8::Import::from_track(track); decoder.initialize(buf)?; - FramedKind::Vp8(crate::publish::Published::new(catalog, decoder)) + FramedKind::Vp8(crate::import::Track::new(catalog, decoder)) } FramedFormat::Vp9 => { - let track = crate::publish::unique_track(&mut broadcast, ".vp09")?; + let track = crate::import::unique_track(&mut broadcast, ".vp09")?; let mut decoder = crate::codec::vp9::Import::from_track(track); decoder.initialize(buf)?; - FramedKind::Vp9(crate::publish::Published::new(catalog, decoder)) + FramedKind::Vp9(crate::import::Track::new(catalog, decoder)) } FramedFormat::Aac => { let config = crate::codec::aac::Config::parse(buf)?; - let track = crate::publish::unique_track(&mut broadcast, ".aac")?; + let track = crate::import::unique_track(&mut broadcast, ".aac")?; let import = crate::codec::aac::Import::from_track(track, config)?; - FramedKind::Aac(crate::publish::Published::new(catalog, import)) + FramedKind::Aac(crate::import::Track::new(catalog, import)) } FramedFormat::Opus => { let config = crate::codec::opus::Config::parse(buf)?; - let track = crate::publish::unique_track(&mut broadcast, ".opus")?; + let track = crate::import::unique_track(&mut broadcast, ".opus")?; let import = crate::codec::opus::Import::from_track(track, config)?; - FramedKind::Opus(crate::publish::Published::new(catalog, import)) + FramedKind::Opus(crate::import::Track::new(catalog, import)) } FramedFormat::Mkv => { let mut decoder = Box::new(crate::container::mkv::Import::new(broadcast, catalog)); @@ -340,22 +347,22 @@ impl Framed { FramedFormat::Vp8 => { let mut decoder = crate::codec::vp8::Import::from_track(track); decoder.initialize(buf)?; - FramedKind::Vp8(crate::publish::Published::new(catalog, decoder)) + FramedKind::Vp8(crate::import::Track::new(catalog, decoder)) } FramedFormat::Vp9 => { let mut decoder = crate::codec::vp9::Import::from_track(track); decoder.initialize(buf)?; - FramedKind::Vp9(crate::publish::Published::new(catalog, decoder)) + FramedKind::Vp9(crate::import::Track::new(catalog, decoder)) } FramedFormat::Aac => { let config = crate::codec::aac::Config::parse(buf)?; let import = crate::codec::aac::Import::from_track(track, config)?; - FramedKind::Aac(crate::publish::Published::new(catalog, import)) + FramedKind::Aac(crate::import::Track::new(catalog, import)) } FramedFormat::Opus => { let config = crate::codec::opus::Config::parse(buf)?; let import = crate::codec::opus::Import::from_track(track, config)?; - FramedKind::Opus(crate::publish::Published::new(catalog, import)) + FramedKind::Opus(crate::import::Track::new(catalog, import)) } FramedFormat::Fmp4 | FramedFormat::Mkv | FramedFormat::Ts | FramedFormat::Flv => { anyhow::bail!("{format} can publish multiple tracks") @@ -421,8 +428,11 @@ impl Framed { } } - /// Return the single track produced by this importer. - pub fn track(&self) -> Result<&moq_net::TrackProducer> { + /// The single track's producer. Private: callers get the curated, read-only + /// accessors below ([`name`](Self::name) / [`subscribe`](Self::subscribe) / + /// [`demand`](Self::demand)) so the importer keeps sole ownership of the + /// publishing handle. + fn producer(&self) -> Result<&moq_net::TrackProducer> { match self.decoder { FramedKind::Avc3 { ref import, .. } => Ok(import.track()), FramedKind::Avc1 { ref import, .. } => Ok(import.track()), @@ -439,6 +449,34 @@ impl Framed { } } + /// The name of the single track this importer publishes. + /// + /// Returns [`Error::MultipleTracks`](crate::Error::MultipleTracks) for container + /// formats that may publish more than one track. + pub fn name(&self) -> Result<&str> { + Ok(self.producer()?.name()) + } + + /// Subscribe to the single track this importer publishes. + /// + /// A read-only handle; it can't publish frames or close the track. Pass `None` + /// for [`Subscription::default`](moq_net::Subscription). + pub fn subscribe( + &self, + subscription: impl Into>, + ) -> Result { + Ok(self.producer()?.subscribe(subscription)) + } + + /// A cloneable, watch-only handle to the single track's subscriber demand. + /// + /// Lets the caller gate work on whether anyone is subscribed (via + /// [`used`](moq_net::TrackDemand::used) / [`unused`](moq_net::TrackDemand::unused)) + /// without the ability to publish or close the track. + pub fn demand(&self) -> Result { + Ok(self.producer()?.demand()) + } + /// Decode a frame from the given buffer. pub fn decode_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { match self.decoder { @@ -507,16 +545,16 @@ impl Framed { // Lift an already-built, catalog-attached opus importer into a `Framed` so callers // that build their config out-of-band (e.g. moq-gst, which constructs `opus::Config` // from gstreamer caps instead of an OpusHead buffer) can keep using `.into()`. -impl From> for Framed { - fn from(opus: crate::publish::Published) -> Self { +impl From> for Framed { + fn from(opus: crate::import::Track) -> Self { Self { decoder: FramedKind::Opus(opus), } } } -impl From> for Framed { - fn from(aac: crate::publish::Published) -> Self { +impl From> for Framed { + fn from(aac: crate::import::Track) -> Self { Self { decoder: FramedKind::Aac(aac), } @@ -579,7 +617,7 @@ mod tests { let mut framed = Framed::new_with_track(track, catalog.clone(), FramedFormat::Opus, &mut init).unwrap(); - assert_eq!(framed.track().unwrap().name(), "requested-audio"); + assert_eq!(framed.name().unwrap(), "requested-audio"); let snapshot = catalog.snapshot(); assert!(snapshot.audio.renditions.contains_key("requested-audio")); assert!(!snapshot.audio.renditions.contains_key("0.opus")); @@ -611,11 +649,11 @@ mod tests { // The broadcast path mints a unique track and attaches its catalog rendition. let mut framed = Framed::new(broadcast, catalog.clone(), FramedFormat::Opus, &mut init).unwrap(); - assert_eq!(framed.track().unwrap().name(), "0.opus"); + assert_eq!(framed.name().unwrap(), "0.opus"); assert!(catalog.snapshot().audio.renditions.contains_key("0.opus")); // Frames published through the minted producer are delivered. - let subscriber = framed.track().unwrap().subscribe(None); + let subscriber = framed.subscribe(None).unwrap(); let mut media = crate::container::Consumer::new(subscriber, crate::catalog::hang::Container::Legacy); let payload = b"opus payload".to_vec(); @@ -681,7 +719,7 @@ mod tests { let framed = Framed::new_with_track(track, catalog.clone(), FramedFormat::Avc3, &mut init).unwrap(); - assert_eq!(framed.track().unwrap().name(), "camera"); + assert_eq!(framed.name().unwrap(), "camera"); let snapshot = catalog.snapshot(); let video = snapshot.video.renditions.get("camera").unwrap(); assert_eq!(video.coded_width, Some(1280)); @@ -788,17 +826,17 @@ enum StreamKind { /// byte parsing; the import publishes. Avc3 { split: crate::codec::h264::Split, - import: crate::publish::Published, + import: crate::import::Track, }, // Boxed because it's a large struct and clippy complains about the size. Fmp4(Box), Hev1 { split: crate::codec::h265::Split, - import: crate::publish::Published, + import: crate::import::Track, }, Av01 { split: crate::codec::av1::Split, - import: crate::publish::Published, + import: crate::import::Track, }, // Boxed for the same reason as Fmp4. Mkv(Box), @@ -825,29 +863,29 @@ impl Stream { ) -> Result { let decoder = match format { StreamFormat::Avc3 => { - let track = crate::publish::unique_track(&mut broadcast, ".avc3")?; + let track = crate::import::unique_track(&mut broadcast, ".avc3")?; let import = crate::codec::h264::Import::from_track(track); let split = crate::codec::h264::Split::new(); StreamKind::Avc3 { split, - import: crate::publish::Published::new(catalog, import), + import: crate::import::Track::new(catalog, import), } } StreamFormat::Fmp4 => StreamKind::Fmp4(Box::new(crate::container::fmp4::Import::new(broadcast, catalog))), StreamFormat::Hev1 => { - let track = crate::publish::unique_track(&mut broadcast, ".hev1")?; + let track = crate::import::unique_track(&mut broadcast, ".hev1")?; let import = crate::codec::h265::Import::from_track(track); StreamKind::Hev1 { split: crate::codec::h265::Split::new(), - import: crate::publish::Published::new(catalog, import), + import: crate::import::Track::new(catalog, import), } } StreamFormat::Av01 => { - let track = crate::publish::unique_track(&mut broadcast, ".av01")?; + let track = crate::import::unique_track(&mut broadcast, ".av01")?; let import = crate::codec::av1::Import::from_track(track); StreamKind::Av01 { split: crate::codec::av1::Split::new(), - import: crate::publish::Published::new(catalog, import), + import: crate::import::Track::new(catalog, import), } } StreamFormat::Mkv => StreamKind::Mkv(Box::new(crate::container::mkv::Import::new(broadcast, catalog))), diff --git a/rs/moq-mux/src/publish.rs b/rs/moq-mux/src/import/track.rs similarity index 89% rename from rs/moq-mux/src/publish.rs rename to rs/moq-mux/src/import/track.rs index 288d8f5ec..44ea17a39 100644 --- a/rs/moq-mux/src/publish.rs +++ b/rs/moq-mux/src/import/track.rs @@ -3,7 +3,7 @@ //! A single-track importer (in [`crate::codec`]) produces frames on one track and //! exposes the catalog renditions it publishes via [`Renditions`]. Most callers, //! though, work with a whole [`moq_net::BroadcastProducer`] plus a shared -//! [`catalog::Producer`](crate::catalog::Producer). [`Published`] is the adapter: +//! [`catalog::Producer`](crate::catalog::Producer). [`Track`] is the adapter: //! it merges an importer's renditions into that catalog and removes them on drop. //! //! For the broadcast-push case, mint a track with [`unique_track`] and build the @@ -13,7 +13,7 @@ //! //! Some importers fill their catalog lazily (H.264 only knows its config once SPS //! arrives) or refine it over time (jitter). Feed them through -//! [`Published::decode`] or [`Published::decoding`], which re-mirror the catalog +//! [`Track::decode`] or [`Track::decoding`], which re-mirror the catalog //! automatically so new/changed renditions always reach it. use std::ops::{Deref, DerefMut}; @@ -22,7 +22,7 @@ use crate::catalog::hang::CatalogExt; /// A single-track importer that exposes the catalog renditions it publishes. /// -/// Implemented by the per-codec importers so [`Published`] can merge their +/// Implemented by the per-codec importers so [`Track`] can merge their /// renditions into a broadcast catalog generically. The returned catalog may be /// empty (and grow later) for importers that initialize lazily. pub trait Renditions { @@ -34,8 +34,8 @@ pub trait Renditions { /// /// The uniform decode entry point: callers split bytes into [`Frame`](crate::container::Frame)s /// (a per-format splitter, e.g. [`crate::codec::h264::Split`]) and hand them over. -/// [`Published`] wraps this so the catalog re-mirror can't be forgotten (see -/// [`Published::decode`]). +/// [`Track`] wraps this so the catalog re-mirror can't be forgotten (see +/// [`Track::decode`]). pub trait FrameDecode { /// Publish frames on this importer's track. fn decode>(&mut self, frames: I) -> crate::Result<()>; @@ -59,7 +59,7 @@ pub fn unique_track(broadcast: &mut moq_net::BroadcastProducer, suffix: &str) -> /// (`decode`, `finish`, `seek`, ...) are available directly. Generic over the /// catalog extension `E` so it can attach to an extended broadcast catalog (e.g. /// the one a container holds). -pub struct Published { +pub struct Track { inner: I, catalog: crate::catalog::Producer, /// The renditions we last mirrored into the catalog, so [`sync`](Self::sync) @@ -67,7 +67,7 @@ pub struct Published { published: hang::Catalog, } -impl Published { +impl Track { /// Attach `inner` to `catalog`, mirroring whatever renditions it already has. pub fn new(catalog: crate::catalog::Producer, inner: I) -> Self { let mut this = Self { @@ -131,7 +131,7 @@ impl Published { } } -impl Published { +impl Track { /// Publish frames and re-mirror any catalog change in one call. /// /// This is the footgun-free path: it re-mirrors the catalog after decoding, so @@ -143,7 +143,7 @@ impl Published { } } -impl Deref for Published { +impl Deref for Track { type Target = I; fn deref(&self) -> &I { @@ -151,13 +151,13 @@ impl Deref for Published { } } -impl DerefMut for Published { +impl DerefMut for Track { fn deref_mut(&mut self) -> &mut I { &mut self.inner } } -impl Drop for Published { +impl Drop for Track { fn drop(&mut self) { if self.published == hang::Catalog::default() { return; @@ -177,7 +177,7 @@ impl Drop for Published { mod tests { use super::*; - /// An importer whose catalog we can mutate, to drive [`Published::sync`]. + /// An importer whose catalog we can mutate, to drive [`Track::sync`]. struct Fake(hang::Catalog); impl Renditions for Fake { @@ -206,7 +206,7 @@ mod tests { let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); // Importer starts with an empty catalog (lazy init): nothing merged yet. - let mut published = Published::new(catalog.clone(), Fake(hang::Catalog::default())); + let mut published = Track::new(catalog.clone(), Fake(hang::Catalog::default())); assert!(catalog.snapshot().video.renditions.is_empty()); // A rendition appears later; decoding mirrors it into the broadcast catalog. @@ -237,7 +237,7 @@ mod tests { let mut broadcast = moq_net::BroadcastInfo::new().produce(); let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); - let mut published = Published::new(catalog.clone(), Fake(hang::Catalog::default())); + let mut published = Track::new(catalog.clone(), Fake(hang::Catalog::default())); assert!(catalog.snapshot().video.renditions.is_empty()); // `decode` resolves the rendition and mirrors it — no manual `sync()`. diff --git a/rs/moq-mux/src/lib.rs b/rs/moq-mux/src/lib.rs index 72e5cbfc0..6d1242ab5 100644 --- a/rs/moq-mux/src/lib.rs +++ b/rs/moq-mux/src/lib.rs @@ -21,7 +21,6 @@ pub mod codec; pub mod container; mod error; pub mod import; -pub mod publish; pub use clock::Clock; pub use error::*; diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 01a8ffdc9..173df8668 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -639,6 +639,20 @@ impl TrackProducer { } } + /// Create a [`TrackDemand`]: a cloneable, watch-only handle to this track's + /// subscriber demand. + /// + /// Lets a publisher gate work (e.g. on-demand capture) on whether anyone is + /// subscribed, without the ability to publish frames or close the track. The + /// handle is weak, so holding one neither keeps the track alive nor pins its + /// cached groups. + pub fn demand(&self) -> TrackDemand { + TrackDemand { + name: self.name.clone(), + state: self.state.weak(), + } + } + /// Get a consumer handle for this in-process track. /// /// Unlike a wire subscription, the info is already known, so a subscription @@ -897,6 +911,49 @@ impl TrackWeak { } } +/// A cloneable, watch-only handle to a track's subscriber demand. +/// +/// Obtained from [`TrackProducer::demand`]. A publisher uses it to react to +/// whether anyone is subscribed (on-demand capture / encoding) without being able +/// to publish frames or close the track. It's a weak handle, so it neither keeps +/// the track alive nor pins its cached groups; once the owning [`TrackProducer`] +/// goes away, [`used`](Self::used) / [`unused`](Self::unused) report the track's +/// closure. +#[derive(Clone)] +pub struct TrackDemand { + name: Arc, + state: kio::Weak, +} + +impl TrackDemand { + /// The track name this handle is bound to. + pub fn name(&self) -> &str { + &self.name + } + + /// Block until there is at least one active consumer. + pub async fn used(&self) -> Result<()> { + self.state + .used() + .await + .map_err(|r| r.abort.clone().unwrap_or(Error::Dropped)) + } + + /// Block until there are no active consumers. + pub async fn unused(&self) -> Result<()> { + self.state + .unused() + .await + .map_err(|r| r.abort.clone().unwrap_or(Error::Dropped)) + } + + /// Block until the track is closed or aborted, returning the cause. + pub async fn closed(&self) -> Error { + self.state.closed().await; + self.state.read().abort.clone().unwrap_or(Error::Dropped) + } +} + /// A handle to a single track within a broadcast. /// /// Obtained from [`crate::BroadcastConsumer::track`]. Holding it sends nothing diff --git a/rs/moq-rtc/src/codec/h264.rs b/rs/moq-rtc/src/codec/h264.rs index 55c36853d..4a09ce181 100644 --- a/rs/moq-rtc/src/codec/h264.rs +++ b/rs/moq-rtc/src/codec/h264.rs @@ -11,14 +11,14 @@ use crate::{Result, codec}; pub struct Bridge { split: moq_mux::codec::h264::Split, - import: moq_mux::publish::Published, + import: moq_mux::import::Track, } impl Bridge { pub fn new(mut broadcast: moq_net::BroadcastProducer, catalog: moq_mux::catalog::Producer) -> Result { - let track = moq_mux::publish::unique_track(&mut broadcast, ".avc3")?; + let track = moq_mux::import::unique_track(&mut broadcast, ".avc3")?; let import = moq_mux::codec::h264::Import::from_track(track); - let import = moq_mux::publish::Published::new(catalog, import); + let import = moq_mux::import::Track::new(catalog, import); let split = moq_mux::codec::h264::Split::new(); Ok(Self { split, import }) } diff --git a/rs/moq-rtc/src/codec/opus.rs b/rs/moq-rtc/src/codec/opus.rs index e234f68c6..f3def248e 100644 --- a/rs/moq-rtc/src/codec/opus.rs +++ b/rs/moq-rtc/src/codec/opus.rs @@ -6,7 +6,7 @@ use crate::{Result, codec}; pub struct Bridge { - import: moq_mux::publish::Published, + import: moq_mux::import::Track, } impl Bridge { @@ -20,9 +20,9 @@ impl Bridge { sample_rate, channel_count, }; - let track = moq_mux::publish::unique_track(&mut broadcast, ".opus")?; + let track = moq_mux::import::unique_track(&mut broadcast, ".opus")?; let import = moq_mux::codec::opus::Import::from_track(track, config)?; - let import = moq_mux::publish::Published::new(catalog, import); + let import = moq_mux::import::Track::new(catalog, import); Ok(Self { import }) } } diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index b08369eee..6ba4356c3 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -26,14 +26,14 @@ const DEFAULT_FRAMERATE: u32 = 30; /// catalog registration and framing. pub struct Producer { split: moq_mux::codec::h264::Split, - import: moq_mux::publish::Published, + import: moq_mux::import::Track, } impl Producer { pub fn new(mut broadcast: moq_net::BroadcastProducer, catalog: moq_mux::catalog::Producer) -> Result { - let track = moq_mux::publish::unique_track(&mut broadcast, ".avc3")?; + let track = moq_mux::import::unique_track(&mut broadcast, ".avc3")?; let import = moq_mux::codec::h264::Import::from_track(track); - let import = moq_mux::publish::Published::new(catalog, import); + let import = moq_mux::import::Track::new(catalog, import); let split = moq_mux::codec::h264::Split::new(); Ok(Self { split, import }) } From 5de839c239cefc28aad0328026095e7e90c18e8f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 17 Jun 2026 14:40:48 -0700 Subject: [PATCH 19/19] moq-mux: importers self-catalog; frame-only decode; shared clock Reshape the import layer so each codec importer owns its catalog rendition and deals only in whole frames, and so concurrently-produced tracks share one timeline. Catalog: add `catalog::VideoTrack` / `catalog::AudioTrack`, scoped handles (via `Producer::video_track` / `audio_track`) that publish one importer's rendition and retire it on drop. This replaces the `import::Track` wrapper plus the `Renditions` / `FrameDecode` traits, which are deleted; the `Published`-style mirror is gone. Importers: every codec importer is now `Import` built with a single `new(track, catalog, [config])` (the `TrackRequest` constructor and `from_track` are dropped; the on-demand path accepts the request at the call site). They expose `demand() -> TrackDemand` instead of handing out a `TrackProducer`, take whole frames as `&[u8]` (no more `Buf`), and the per-importer `decode_buf` / `pts` helpers collapse into one `decode`. `Framed::decode_frame` becomes `Framed::decode(&[u8], pts)`. Sync: the wall-clock fallback moves to a `Clock` owned by the shared `catalog::Producer`, so audio and video synthesizing timestamps anchor to the same epoch (`Producer::timestamp` / `VideoTrack`/`AudioTrack::timestamp`). Containers and consumers (ffi/cli/rtc/gst/video/boy) are updated to the new constructors and `demand()`. Follow-ups: split `Framed` into `Framed` + `FramedTrack`, and give the TS/MKV/FLV/fMP4 containers their own `Split` modules so they too deal in frames. Co-Authored-By: Claude Opus 4.8 (1M context) --- rs/libmoq/src/publish.rs | 18 +- rs/moq-boy/src/main.rs | 10 +- rs/moq-boy/src/video.rs | 8 +- rs/moq-cli/src/publish.rs | 5 +- rs/moq-ffi/src/producer.rs | 23 +- rs/moq-gst/src/sink/imp.rs | 38 +-- rs/moq-mux/src/catalog/mod.rs | 2 + rs/moq-mux/src/catalog/producer.rs | 33 ++ rs/moq-mux/src/catalog/tracks.rs | 118 +++++++ rs/moq-mux/src/codec/aac/import.rs | 105 ++---- rs/moq-mux/src/codec/av1/import.rs | 72 ++-- rs/moq-mux/src/codec/h264/import.rs | 139 ++++---- rs/moq-mux/src/codec/h265/import.rs | 79 ++--- rs/moq-mux/src/codec/legacy.rs | 69 ++-- rs/moq-mux/src/codec/opus/import.rs | 108 ++---- rs/moq-mux/src/codec/vp8/import.rs | 137 ++++---- rs/moq-mux/src/codec/vp9/import.rs | 144 ++++---- rs/moq-mux/src/container/ts/import.rs | 44 +-- rs/moq-mux/src/import/mod.rs | 458 +++++++++++--------------- rs/moq-mux/src/import/track.rs | 241 +------------- rs/moq-rtc/src/codec/h264.rs | 5 +- rs/moq-rtc/src/codec/opus.rs | 8 +- rs/moq-video/src/encode/producer.rs | 25 +- 23 files changed, 730 insertions(+), 1159 deletions(-) create mode 100644 rs/moq-mux/src/catalog/tracks.rs diff --git a/rs/libmoq/src/publish.rs b/rs/libmoq/src/publish.rs index b49cfb379..b5a514091 100644 --- a/rs/libmoq/src/publish.rs +++ b/rs/libmoq/src/publish.rs @@ -1,6 +1,5 @@ use std::str::FromStr; -use bytes::Buf; use moq_mux::import; use crate::{Error, Id, NonZeroSlab}; @@ -52,29 +51,20 @@ impl Publish { Ok(()) } - pub fn media_ordered(&mut self, broadcast: Id, format: &str, mut init: &[u8]) -> Result { + pub fn media_ordered(&mut self, broadcast: Id, format: &str, init: &[u8]) -> Result { let (broadcast, catalog) = self.broadcasts.get(broadcast).ok_or(Error::BroadcastNotFound)?; let format = import::FramedFormat::from_str(format).map_err(|_| Error::UnknownFormat(format.to_string()))?; - let decoder = import::Framed::new(broadcast.clone(), catalog.clone(), format, &mut init)?; + let decoder = import::Framed::new(broadcast.clone(), catalog.clone(), format, init)?; let id = self.media.insert(decoder)?; Ok(id) } - pub fn media_frame( - &mut self, - media: Id, - mut data: &[u8], - timestamp: hang::container::Timestamp, - ) -> Result<(), Error> { + pub fn media_frame(&mut self, media: Id, data: &[u8], timestamp: hang::container::Timestamp) -> Result<(), Error> { let media = self.media.get_mut(media).ok_or(Error::MediaNotFound)?; - media.decode_frame(&mut data, Some(timestamp))?; - - if data.has_remaining() { - return Err(Error::BufferNotConsumed); - } + media.decode(data, Some(timestamp))?; Ok(()) } diff --git a/rs/moq-boy/src/main.rs b/rs/moq-boy/src/main.rs index 03a610c07..f7c4e534d 100644 --- a/rs/moq-boy/src/main.rs +++ b/rs/moq-boy/src/main.rs @@ -90,8 +90,8 @@ pub struct Config { /// track monitors, and async tasks. struct Session { video_encoder: video::VideoEncoder, - video_track: moq_net::TrackProducer, - audio_track: moq_net::TrackProducer, + video_track: moq_net::TrackDemand, + audio_track: moq_net::TrackDemand, /// Whether anyone is subscribed to the video/audio tracks. video_active: AtomicBool, @@ -109,7 +109,7 @@ struct Session { impl Session { /// Monitor a single track's subscription state. /// Sets the flag when a viewer subscribes, clears it when all unsubscribe. - async fn run_track_monitor(&self, name: &str, track: &moq_net::TrackProducer, flag: &AtomicBool) { + async fn run_track_monitor(&self, name: &str, track: &moq_net::TrackDemand, flag: &AtomicBool) { loop { if track.used().await.is_err() { break; @@ -246,8 +246,8 @@ async fn run(config: &Config) -> Result<()> { let audio_encoder = audio::AudioEncoder::new(broadcast.clone(), catalog.clone(), 44100)?; - let video_track = video_encoder.track.clone(); - let audio_track = audio_encoder.track().clone(); + let video_track = video_encoder.demand.clone(); + let audio_track = audio_encoder.track().demand(); let status_publisher = status::StatusPublisher::new(&mut broadcast)?; diff --git a/rs/moq-boy/src/video.rs b/rs/moq-boy/src/video.rs index 17daf9e5d..104274aed 100644 --- a/rs/moq-boy/src/video.rs +++ b/rs/moq-boy/src/video.rs @@ -19,8 +19,8 @@ use crate::emulator::{HEIGHT, WIDTH}; /// Frames are submitted via `try_frame()` (non-blocking, drops if full). pub struct VideoEncoder { tx: tokio::sync::mpsc::Sender, - /// Clone of the video track producer, for monitoring used/unused. - pub track: moq_net::TrackProducer, + /// Watch-only handle to the video track, for monitoring used/unused. + pub demand: moq_net::TrackDemand, force_keyframe: Arc, /// Latest encode duration in microseconds. encode_duration: Arc, @@ -36,7 +36,7 @@ impl VideoEncoder { pub fn spawn(broadcast: moq_net::BroadcastProducer, catalog: moq_mux::catalog::Producer) -> Self { let (tx, rx) = tokio::sync::mpsc::channel(4); let producer = moq_video::encode::Producer::new(broadcast, catalog).expect("failed to create avc3 producer"); - let track = producer.track().clone(); + let demand = producer.demand(); let force_keyframe = Arc::new(AtomicBool::new(false)); let encode_duration = Arc::new(AtomicU64::new(0)); @@ -49,7 +49,7 @@ impl VideoEncoder { Self { tx, - track, + demand, force_keyframe, encode_duration, _thread: thread, diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index b0348305d..6bbed7692 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -82,7 +82,7 @@ pub struct CaptureArgs { enum PublishDecoder { Avc3 { split: moq_mux::codec::h264::Split, - import: Box>, + import: Box, }, Fmp4(Box), Ts(Box), @@ -148,8 +148,7 @@ impl Publish { let source = match format { PublishFormat::Avc3 => { let track = moq_mux::import::unique_track(&mut broadcast, ".avc3")?; - let import = moq_mux::codec::h264::Import::from_track(track); - let import = moq_mux::import::Track::new(catalog.clone(), import); + let import = moq_mux::codec::h264::Import::new(track, catalog.clone()); let split = moq_mux::codec::h264::Split::new(); Source::Stream(PublishDecoder::Avc3 { split, diff --git a/rs/moq-ffi/src/producer.rs b/rs/moq-ffi/src/producer.rs index a883515f4..839230c3d 100644 --- a/rs/moq-ffi/src/producer.rs +++ b/rs/moq-ffi/src/producer.rs @@ -1,8 +1,6 @@ use std::str::FromStr; use std::sync::Arc; -use bytes::Buf; - use crate::consumer::{MoqBroadcastConsumer, MoqGroupConsumer, MoqTrackConsumer}; use crate::error::MoqError; use crate::ffi::Task; @@ -127,14 +125,9 @@ impl MoqBroadcastProducer { let format = moq_mux::import::FramedFormat::from_str(&format) .map_err(|_| MoqError::Codec(format!("unknown format: {format}")))?; - let mut buf = init.as_slice(); - let decoder = moq_mux::import::Framed::new(state.broadcast.clone(), state.catalog.clone(), format, &mut buf) + let decoder = moq_mux::import::Framed::new(state.broadcast.clone(), state.catalog.clone(), format, &init) .map_err(|err| MoqError::Codec(format!("init failed: {err}")))?; - if buf.has_remaining() { - return Err(MoqError::Codec("init failed: trailing bytes".into())); - } - let demand = decoder .demand() .map_err(|err| MoqError::Codec(format!("track unavailable: {err}")))?; @@ -166,15 +159,10 @@ impl MoqBroadcastProducer { guard.as_ref().ok_or_else(|| MoqError::Closed)?.clone() }; - let mut buf = init.as_slice(); let decoder = - moq_mux::import::Framed::new_with_track(track_clone.clone(), state.catalog.clone(), format, &mut buf) + moq_mux::import::Framed::new_with_track(track_clone.clone(), state.catalog.clone(), format, &init) .map_err(|err| MoqError::Codec(format!("init failed: {err}")))?; - if buf.has_remaining() { - return Err(MoqError::Codec("init failed: trailing bytes".into())); - } - let mut guard = track.inner.lock().unwrap(); guard.take().ok_or_else(|| MoqError::Closed)?; @@ -438,16 +426,11 @@ impl MoqMediaProducer { let media = guard.as_mut().ok_or_else(|| MoqError::Closed)?; let timestamp = hang::container::Timestamp::from_micros(timestamp_us)?; - let mut data = payload.as_slice(); media .decoder - .decode_frame(&mut data, Some(timestamp)) + .decode(&payload, Some(timestamp)) .map_err(|err| MoqError::Codec(format!("decode failed: {err}")))?; - if data.has_remaining() { - return Err(MoqError::Codec("buffer was not fully consumed".into())); - } - Ok(()) } diff --git a/rs/moq-gst/src/sink/imp.rs b/rs/moq-gst/src/sink/imp.rs index 3fc2c18fe..ed57825ca 100644 --- a/rs/moq-gst/src/sink/imp.rs +++ b/rs/moq-gst/src/sink/imp.rs @@ -448,32 +448,32 @@ fn handle_caps(runtime: &mut RuntimeState, pad_name: String, caps: gst::Caps) -> let structure = caps.structure(0).context("empty caps")?; let decoder: moq_mux::import::Framed = match structure.name().as_str() { "video/x-h264" => { - let mut bytes = Bytes::new(); - new_decoder(runtime, moq_mux::import::FramedFormat::Avc3, &mut bytes)? + let bytes = Bytes::new(); + new_decoder(runtime, moq_mux::import::FramedFormat::Avc3, &bytes)? } "video/x-h265" => { - let mut bytes = Bytes::new(); - new_decoder(runtime, moq_mux::import::FramedFormat::Hev1, &mut bytes)? + let bytes = Bytes::new(); + new_decoder(runtime, moq_mux::import::FramedFormat::Hev1, &bytes)? } "video/x-av1" => { - let mut bytes = Bytes::new(); - new_decoder(runtime, moq_mux::import::FramedFormat::Av01, &mut bytes)? + let bytes = Bytes::new(); + new_decoder(runtime, moq_mux::import::FramedFormat::Av01, &bytes)? } "video/x-vp8" => { - let mut bytes = Bytes::new(); - new_decoder(runtime, moq_mux::import::FramedFormat::Vp8, &mut bytes)? + let bytes = Bytes::new(); + new_decoder(runtime, moq_mux::import::FramedFormat::Vp8, &bytes)? } "video/x-vp9" => { - let mut bytes = Bytes::new(); - new_decoder(runtime, moq_mux::import::FramedFormat::Vp9, &mut bytes)? + let bytes = Bytes::new(); + new_decoder(runtime, moq_mux::import::FramedFormat::Vp9, &bytes)? } "audio/mpeg" => { let codec_data = structure .get::("codec_data") .context("AAC caps missing codec_data")?; let map = codec_data.map_readable().context("failed to map codec_data")?; - let mut data = Bytes::copy_from_slice(map.as_slice()); - new_decoder(runtime, moq_mux::import::FramedFormat::Aac, &mut data)? + let data = Bytes::copy_from_slice(map.as_slice()); + new_decoder(runtime, moq_mux::import::FramedFormat::Aac, &data)? } "audio/x-opus" => { let channels: i32 = structure.get("channels").unwrap_or(2); @@ -487,8 +487,7 @@ fn handle_caps(runtime: &mut RuntimeState, pad_name: String, caps: gst::Caps) -> channel_count, }; let track = moq_mux::import::unique_track(&mut runtime.broadcast, ".opus")?; - let import = moq_mux::codec::opus::Import::from_track(track, config)?; - moq_mux::import::Track::new(runtime.catalog.clone(), import).into() + moq_mux::codec::opus::Import::new(track, runtime.catalog.clone(), config)?.into() } other => anyhow::bail!("unsupported caps: {}", other), }; @@ -506,18 +505,13 @@ fn handle_caps(runtime: &mut RuntimeState, pad_name: String, caps: gst::Caps) -> fn new_decoder( runtime: &mut RuntimeState, format: moq_mux::import::FramedFormat, - buf: &mut Bytes, + buf: &[u8], ) -> Result { let decoder = moq_mux::import::Framed::new(runtime.broadcast.clone(), runtime.catalog.clone(), format, buf)?; Ok(decoder) } -fn handle_buffer( - runtime: &mut RuntimeState, - pad_name: String, - mut data: Bytes, - pts: Option, -) -> Result<()> { +fn handle_buffer(runtime: &mut RuntimeState, pad_name: String, data: Bytes, pts: Option) -> Result<()> { let pad = runtime.pads.get_mut(&pad_name).context("pad not configured")?; let ts = pts.and_then(|pts| { @@ -526,5 +520,5 @@ fn handle_buffer( hang::container::Timestamp::from_micros(relative.nseconds() / 1000).ok() }); - pad.decoder.decode_frame(&mut data, ts).map_err(|e| anyhow::anyhow!(e)) + pad.decoder.decode(&data, ts).map_err(|e| anyhow::anyhow!(e)) } diff --git a/rs/moq-mux/src/catalog/mod.rs b/rs/moq-mux/src/catalog/mod.rs index 6ae373965..e0bb25aef 100644 --- a/rs/moq-mux/src/catalog/mod.rs +++ b/rs/moq-mux/src/catalog/mod.rs @@ -29,6 +29,7 @@ mod format; mod producer; mod stream; mod target; +mod tracks; pub use consumer::Consumer; pub use filter::{Filter, FilterAudio, FilterVideo}; @@ -36,3 +37,4 @@ pub use format::*; pub use producer::{Guard, Producer}; pub use stream::Stream; pub use target::{Target, TargetAudio, TargetVideo}; +pub use tracks::{AudioTrack, VideoTrack}; diff --git a/rs/moq-mux/src/catalog/producer.rs b/rs/moq-mux/src/catalog/producer.rs index 5f14f36d4..1a7c87b76 100644 --- a/rs/moq-mux/src/catalog/producer.rs +++ b/rs/moq-mux/src/catalog/producer.rs @@ -25,6 +25,11 @@ pub struct Producer { msf_track: moq_net::TrackProducer, current: Arc>>, + + /// Shared wall clock for the broadcast's tracks. Every importer on this catalog + /// gets a clone (a `Copy` of the same epoch), so timestamps they synthesize when + /// a caller has none land on one timeline and audio/video stay in sync. + clock: crate::Clock, } // Manual Clone so a producer is cheaply clonable regardless of whether `E` is. @@ -34,6 +39,7 @@ impl Clone for Producer { hang: self.hang.clone(), msf_track: self.msf_track.clone(), current: self.current.clone(), + clock: self.clock, } } } @@ -62,9 +68,22 @@ impl Producer { hang, msf_track, current: Arc::new(Mutex::new(catalog)), + clock: crate::Clock::new(), }) } + /// Resolve a timestamp, synthesizing one from the broadcast's shared + /// [`Clock`](crate::Clock) when the caller has none. + /// + /// Sharing the clock across the catalog's tracks keeps concurrently-produced + /// audio and video on a single timeline. + pub fn timestamp(&self, hint: Option) -> crate::Result { + match hint { + Some(pts) => Ok(pts), + None => Ok(moq_net::Timestamp::from_micros(self.clock.micros())?), + } + } + /// Get mutable access to the catalog, publishing it after any changes. pub fn lock(&mut self) -> Guard<'_, E> { Guard { @@ -80,6 +99,20 @@ impl Producer { self.current.lock().unwrap().clone() } + /// A handle for one importer to publish a video rendition, retired on drop. + /// + /// See [`VideoTrack`](super::VideoTrack). + pub fn video_track(&self, name: impl Into) -> super::VideoTrack { + super::VideoTrack::new(self.clone(), name) + } + + /// A handle for one importer to publish an audio rendition, retired on drop. + /// + /// See [`AudioTrack`](super::AudioTrack). + pub fn audio_track(&self, name: impl Into) -> super::AudioTrack { + super::AudioTrack::new(self.clone(), name) + } + /// Create a consumer for this catalog, receiving updates as they're published. pub fn consume(&self) -> Result, moq_net::Error> { Ok(Consumer::new(self.hang.consume())) diff --git a/rs/moq-mux/src/catalog/tracks.rs b/rs/moq-mux/src/catalog/tracks.rs new file mode 100644 index 000000000..5963a0a8b --- /dev/null +++ b/rs/moq-mux/src/catalog/tracks.rs @@ -0,0 +1,118 @@ +use super::Producer; +use super::hang::CatalogExt; + +/// A single video track's catalog rendition, retired on drop. +/// +/// Made via [`Producer::video_track`]. An importer holds one and publishes its +/// rendition through it ([`set`](Self::set), refined in place with +/// [`update`](Self::update)). When the importer drops, the rendition is removed +/// from the shared catalog, so the broadcast catalog stays out of the importer's +/// type while still being published into. +pub struct VideoTrack { + catalog: Producer, + name: String, + /// Whether a config has been published yet, so a lazily-configured importer + /// (e.g. H.264 before its SPS) can hold the handle without a catalog entry, and + /// drop without a spurious removal. + present: bool, +} + +impl VideoTrack { + pub(super) fn new(catalog: Producer, name: impl Into) -> Self { + Self { + catalog, + name: name.into(), + present: false, + } + } + + /// The track name this rendition is keyed by. + pub fn name(&self) -> &str { + &self.name + } + + /// Resolve a timestamp on the broadcast's shared clock (see [`Producer::timestamp`]). + pub fn timestamp(&self, hint: Option) -> crate::Result { + self.catalog.timestamp(hint) + } + + /// Insert or replace the rendition, publishing the catalog. + pub fn set(&mut self, config: hang::catalog::VideoConfig) { + self.catalog.lock().video.renditions.insert(self.name.clone(), config); + self.present = true; + } + + /// Refine the rendition in place (e.g. observed jitter), publishing if present. + pub fn update(&mut self, f: impl FnOnce(&mut hang::catalog::VideoConfig)) { + if !self.present { + return; + } + let mut guard = self.catalog.lock(); + if let Some(config) = guard.video.renditions.get_mut(&self.name) { + f(config); + } + } +} + +impl Drop for VideoTrack { + fn drop(&mut self) { + if self.present { + self.catalog.lock().video.renditions.remove(&self.name); + } + } +} + +/// A single audio track's catalog rendition, retired on drop. +/// +/// The audio counterpart of [`VideoTrack`]; made via [`Producer::audio_track`]. +pub struct AudioTrack { + catalog: Producer, + name: String, + present: bool, +} + +impl AudioTrack { + pub(super) fn new(catalog: Producer, name: impl Into) -> Self { + Self { + catalog, + name: name.into(), + present: false, + } + } + + /// The track name this rendition is keyed by. + pub fn name(&self) -> &str { + &self.name + } + + /// Resolve a timestamp on the broadcast's shared clock (see [`Producer::timestamp`]). + pub fn timestamp(&self, hint: Option) -> crate::Result { + self.catalog.timestamp(hint) + } + + /// Insert or replace the rendition, publishing the catalog. + pub fn set(&mut self, config: hang::catalog::AudioConfig) { + self.catalog.lock().audio.renditions.insert(self.name.clone(), config); + self.present = true; + } + + /// Refine the rendition in place (e.g. a synthesized description or jitter), + /// publishing if present. + pub fn update(&mut self, f: impl FnOnce(&mut hang::catalog::AudioConfig)) { + if !self.present { + return; + } + let mut guard = self.catalog.lock(); + if let Some(config) = guard.audio.renditions.get_mut(&self.name) { + f(config); + } + } +} + +impl Drop for AudioTrack { + fn drop(&mut self) { + if self.present { + self.catalog.lock().audio.renditions.remove(&self.name); + } + } +} diff --git a/rs/moq-mux/src/codec/aac/import.rs b/rs/moq-mux/src/codec/aac/import.rs index 480334fd3..f9e8ec002 100644 --- a/rs/moq-mux/src/codec/aac/import.rs +++ b/rs/moq-mux/src/codec/aac/import.rs @@ -1,31 +1,27 @@ -use bytes::{Buf, BytesMut}; - use super::Config; -use crate::import::Renditions; +use crate::catalog::hang::CatalogExt; +use crate::container::Frame; /// AAC importer. /// /// Initialized from an AudioSpecificConfig blob (variable-length, typically extracted from -/// an MP4 ESDS atom), so its catalog is known up front. Each input buffer passed to +/// an MP4 ESDS atom), so its catalog is known up front. Each packet passed to /// [`decode`](Self::decode) is published as one hang frame in its own group, so the relay can /// forward each frame without waiting for a group boundary. The codec's packet loss -/// concealment handles drops. Build it from a [`moq_net::TrackRequest`] ([`new`](Self::new)) -/// or an existing track ([`from_track`](Self::from_track)). -pub struct Import { +/// concealment handles drops. Build it with [`new`](Self::new), passing the track producer +/// and the [`catalog::Producer`](crate::catalog::Producer) it publishes its rendition into. +pub struct Import { track: crate::container::Producer, - catalog: hang::Catalog, - zero: Option, + rendition: crate::catalog::AudioTrack, } -impl Import { - /// Serve a track request, accepting it at the microsecond timescale. - pub fn new(request: moq_net::TrackRequest, config: Config) -> crate::Result { - let info = moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE); - Self::from_track(request.accept(info), config) - } - - /// Publish on an existing track producer. - pub fn from_track(track: moq_net::TrackProducer, config: Config) -> crate::Result { +impl Import { + /// Publish on an existing track producer, registering the rendition in `catalog`. + pub fn new( + track: moq_net::TrackProducer, + catalog: crate::catalog::Producer, + config: Config, + ) -> crate::Result { let mut audio_config = hang::catalog::AudioConfig::new( hang::catalog::AAC { profile: config.profile, @@ -37,32 +33,26 @@ impl Import { tracing::debug!(name = ?track.name(), config = ?audio_config, "starting track"); - let mut catalog = hang::Catalog::default(); - catalog.audio.renditions.insert(track.name().to_string(), audio_config); + let mut rendition = catalog.audio_track(track.name()); + rendition.set(audio_config); Ok(Self { track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), - catalog, - zero: None, + rendition, }) } - /// The standalone catalog this importer publishes (one AAC audio rendition). - pub fn catalog(&self) -> &hang::Catalog { - &self.catalog - } - - /// Returns a reference to the underlying track producer. - pub fn track(&self) -> &moq_net::TrackProducer { - self.track.track() + /// A watch-only handle to this track's subscriber demand. + pub fn demand(&self) -> moq_net::TrackDemand { + self.track.track().demand() } - /// Mutable access to the single audio rendition, for callers that refine it - /// after construction (the TS importer sets the synthesized `description` and - /// an audio-burst `jitter`). Follow with [`crate::import::Track::sync`]. - pub(crate) fn rendition_mut(&mut self) -> Option<&mut hang::catalog::AudioConfig> { - let name = self.track.name(); - self.catalog.audio.renditions.get_mut(name) + /// Refine the single audio rendition in place, republishing the catalog. + /// + /// The TS importer uses this to set the synthesized `description` and an + /// audio-burst `jitter` once it knows them. + pub(crate) fn update_rendition(&mut self, f: impl FnOnce(&mut hang::catalog::AudioConfig)) { + self.rendition.update(f); } /// Finish the track, flushing the current group. @@ -77,45 +67,16 @@ impl Import { Ok(()) } - pub fn decode(&mut self, buf: &mut T, pts: Option) -> crate::Result<()> { - let pts = self.pts(pts)?; - - // Collect the input into a contiguous Bytes payload. - let mut payload = BytesMut::with_capacity(buf.remaining()); - while buf.has_remaining() { - let chunk = buf.chunk(); - payload.extend_from_slice(chunk); - let len = chunk.len(); - buf.advance(len); - } - - // Each frame is its own group so the relay can forward it immediately. - // The codec's packet loss concealment handles drops. - let frame = crate::container::Frame { - timestamp: pts, - payload: payload.freeze(), + /// Publish one AAC packet as its own group, stamping `pts` or a wall clock when absent. + pub fn decode(&mut self, frame: &[u8], pts: Option) -> crate::Result<()> { + let timestamp = self.rendition.timestamp(pts)?; + self.track.write(Frame { + timestamp, + payload: bytes::Bytes::copy_from_slice(frame), keyframe: true, duration: None, - }; - - self.track.write(frame)?; + })?; self.track.finish_group()?; - Ok(()) } - - fn pts(&mut self, hint: Option) -> crate::Result { - if let Some(pts) = hint { - return Ok(pts); - } - - let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); - Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) - } -} - -impl Renditions for Import { - fn renditions(&self) -> &hang::Catalog { - &self.catalog - } } diff --git a/rs/moq-mux/src/codec/av1/import.rs b/rs/moq-mux/src/codec/av1/import.rs index f04769ced..2657d8c4c 100644 --- a/rs/moq-mux/src/codec/av1/import.rs +++ b/rs/moq-mux/src/codec/av1/import.rs @@ -8,46 +8,40 @@ //! error; non-keyframes before the first config are written through to the //! producer, which reports [`MissingKeyframe`](crate::container::MissingKeyframe) //! for a mid-stream join. OBU byte parsing lives in [`Split`](super::Split); this type is a -//! pure frame publisher that whoever owns the split drives via the -//! [`FrameDecode`] trait. +//! pure frame publisher that whoever owns the split drives via [`decode`](Import::decode). -use bytes::{Buf, Bytes}; +use bytes::Bytes; use scuffle_av1::seq::SequenceHeaderObu; use scuffle_av1::{ObuHeader, ObuType}; use super::Error; use super::split::ObuIterator; use crate::Result; +use crate::catalog::hang::CatalogExt; use crate::container::Frame; use crate::container::jitter::MinFrameDuration; -use crate::import::{FrameDecode, Renditions}; /// A pure-publisher importer for AV1 with inline sequence headers. /// -/// Build it from a [`moq_net::TrackRequest`] ([`new`](Self::new)) or an existing -/// track ([`from_track`](Self::from_track)), and feed it frames a [`Split`](super::Split) -/// produced via the [`FrameDecode`] impl. The catalog rendition fills in lazily -/// once the config is known; read it via [`catalog`](Self::catalog). -pub struct Import { +/// Build it with [`new`](Self::new), passing the track producer and the +/// [`catalog::Producer`](crate::catalog::Producer) it publishes into, and feed it +/// frames a [`Split`](super::Split) produced via [`decode`](Self::decode). The +/// catalog rendition fills in lazily once the config is known. +pub struct Import { track: crate::container::Producer, - catalog: hang::Catalog, + rendition: crate::catalog::VideoTrack, config: Option, last_seq: Option, jitter: MinFrameDuration, } -impl Import { - /// Serve a track request, accepting it at the microsecond timescale. - pub fn new(request: moq_net::TrackRequest) -> Self { - let info = moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE); - Self::from_track(request.accept(info)) - } - - /// Publish on an existing track producer. - pub fn from_track(track: moq_net::TrackProducer) -> Self { +impl Import { + /// Publish on an existing track producer, registering the rendition in `catalog`. + pub fn new(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { + let rendition = catalog.video_track(track.name()); Self { track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), - catalog: hang::Catalog::default(), + rendition, config: None, last_seq: None, jitter: MinFrameDuration::new(), @@ -63,8 +57,8 @@ impl Import { /// Optional, since the importer also self-initializes from the first keyframe. /// The buffer is *not* consumed: the dispatcher-owned [`Split`](super::Split) /// consumes it (seeding the sequence header so it prefixes the first keyframe). - pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { - let data = buf.as_ref(); + pub fn initialize(&mut self, buf: &[u8]) -> Result<()> { + let data = buf; // av1C box starts with 0x81 (marker=1, version=1) per ISO/IEC 14496-15. if data.len() >= 16 && data[0] == 0x81 { @@ -172,10 +166,7 @@ impl Import { return; } tracing::debug!(name = ?self.track.name(), ?config, "starting track"); - self.catalog - .video - .renditions - .insert(self.track.name().to_string(), config.clone()); + self.rendition.set(config.clone()); self.config = Some(config); } @@ -201,14 +192,9 @@ impl Import { } } - /// The standalone catalog once the config is known, else `None`. - pub fn catalog(&self) -> Option<&hang::Catalog> { - self.config.is_some().then_some(&self.catalog) - } - - /// The underlying track producer. - pub fn track(&self) -> &moq_net::TrackProducer { - self.track.track() + /// A watch-only handle to this track's subscriber demand. + pub fn demand(&self) -> moq_net::TrackDemand { + self.track.track().demand() } /// True once the config is known and the catalog has been populated. @@ -248,28 +234,20 @@ impl Import { // MissingKeyframe, which a caller joining mid-stream skips. self.track.write(frame)?; - if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.video.renditions.get_mut(self.track.name()) - { - c.jitter = Some(jitter); + if let Some(jitter) = self.jitter.observe(pts) { + self.rendition.update(|c| c.jitter = Some(jitter)); } } Ok(()) } -} -impl FrameDecode for Import { - fn decode>(&mut self, frames: I) -> Result<()> { + /// Publish split frames, resolving the config from the first keyframe's inline + /// sequence header and refining the catalog jitter. + pub fn decode(&mut self, frames: impl IntoIterator) -> Result<()> { self.write_frames(frames) } } -impl Renditions for Import { - fn renditions(&self) -> &hang::Catalog { - &self.catalog - } -} - fn is_sequence_header(obu: &[u8]) -> bool { let mut reader = obu; ObuHeader::parse(&mut reader) diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index a23cfe595..96bde5b2f 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -3,7 +3,7 @@ //! [`Import`] publishes already-split H.264 frames on a single moq track and //! resolves the catalog rendition. It is a pure frame publisher: byte parsing //! and framing live in [`Split`](super::Split), and whoever drives the import owns the split. -//! Frames arrive via the [`FrameDecode`] trait ([`decode`](FrameDecode::decode)). +//! Frames arrive via [`decode`](Import::decode). //! //! The codec config comes from exactly one of two places: an avcC handed to //! [`initialize`](Import::initialize) (the "avc1" shape), or the SPS the splitter @@ -11,48 +11,41 @@ //! here). A keyframe that can't be configured from either is an error; //! non-keyframes before the first config are tolerated (mid-stream joins). -use bytes::{Buf, Bytes}; +use bytes::Bytes; use super::{Error, NAL_TYPE_SPS, Sps}; use crate::Result; +use crate::catalog::hang::CatalogExt; use crate::codec::annexb::NalIterator; use crate::container::Frame; use crate::container::jitter::MinFrameDuration; -use crate::import::{FrameDecode, Renditions}; /// H.264 importer: a pure frame publisher that resolves the catalog rendition. /// -/// Build it from a [`moq_net::TrackRequest`] ([`new`](Self::new), the on-demand -/// path) or an existing [`moq_net::TrackProducer`] ([`from_track`](Self::from_track), -/// the broadcast-push / fixed-track path). Feed it frames a [`Split`](super::Split) produced via -/// the [`FrameDecode`] impl. The catalog rendition fills in lazily once the codec -/// config is known (avcC via [`initialize`](Self::initialize) for avc1, the first -/// SPS for avc3); read it via [`catalog`](Self::catalog) or attach the importer to -/// a broadcast catalog with [`crate::import::Track`]. -pub struct Import { +/// Build it with [`new`](Self::new), passing the track producer and the +/// [`catalog::Producer`](crate::catalog::Producer) it publishes its rendition into. +/// Feed it frames a [`Split`](super::Split) produced via [`decode`](Self::decode). +/// The catalog rendition fills in lazily once the codec config is known (avcC via +/// [`initialize`](Self::initialize) for avc1, the first SPS for avc3). +pub struct Import { /// True for the avc1 shape: the codec config is out-of-band (avcC), so /// keyframes are not scanned for an inline SPS. avc1: bool, track: crate::container::Producer, - catalog: hang::Catalog, + rendition: crate::catalog::VideoTrack, config: Option, last_sps: Option, jitter: MinFrameDuration, } -impl Import { - /// Serve a track request, accepting it at the microsecond timescale. - pub fn new(request: moq_net::TrackRequest) -> Self { - let info = moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE); - Self::from_track(request.accept(info)) - } - - /// Publish on an existing track producer. - pub fn from_track(track: moq_net::TrackProducer) -> Self { +impl Import { + /// Publish on an existing track producer, registering the rendition in `catalog`. + pub fn new(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { + let rendition = catalog.video_track(track.name()); Self { avc1: false, track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), - catalog: hang::Catalog::default(), + rendition, config: None, last_sps: None, jitter: MinFrameDuration::new(), @@ -61,21 +54,20 @@ impl Import { /// Resolve the codec config from the codec's leading bytes. /// - /// - **avc1** (no leading start code): the buffer is parsed as an - /// `AVCDecoderConfigurationRecord`, which resolves the config and is stored - /// as the catalog `description`. Required for avc1. - /// - **avc3** (leading start code): the buffer is parsed as Annex-B; any SPS - /// resolves the config. Optional, since avc3 also self-initializes from the - /// first keyframe. + /// - **avc1** (no leading start code): parsed as an `AVCDecoderConfigurationRecord`, + /// which resolves the config and is stored as the catalog `description`. Required + /// for avc1. + /// - **avc3** (leading start code): parsed as Annex-B; any SPS resolves the config. + /// Optional, since avc3 also self-initializes from the first keyframe. /// - /// The buffer is *not* consumed: the dispatcher-owned [`Split`](super::Split) consumes it - /// (and reads the same avcC for the NALU length size). The shape is detected - /// from the leading bytes. - pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { - if detect_avc1(buf.as_ref()) { - self.initialize_avc1(buf.as_ref()) + /// Takes a read-only slice: the dispatcher-owned [`Split`](super::Split) is what + /// consumes the stream (and reads the same avcC for the NALU length size). The + /// shape is detected from the leading bytes. + pub fn initialize(&mut self, buf: &[u8]) -> Result<()> { + if detect_avc1(buf) { + self.initialize_avc1(buf) } else { - self.initialize_avc3(buf.as_ref()) + self.initialize_avc3(buf) } } @@ -116,14 +108,9 @@ impl Import { Ok(()) } - /// The standalone catalog once the codec config is known, else `None`. - pub fn catalog(&self) -> Option<&hang::Catalog> { - self.config.is_some().then_some(&self.catalog) - } - - /// The underlying track producer. - pub fn track(&self) -> &moq_net::TrackProducer { - self.track.track() + /// A watch-only handle to this track's subscriber demand. + pub fn demand(&self) -> moq_net::TrackDemand { + self.track.track().demand() } /// True once the codec config is known and the catalog rendition is published. @@ -176,10 +163,7 @@ impl Import { return; } tracing::debug!(?config, "starting H.264 track"); - self.catalog - .video - .renditions - .insert(self.track.name().to_string(), config.clone()); + self.rendition.set(config.clone()); self.config = Some(config); } @@ -209,28 +193,20 @@ impl Import { let pts = frame.timestamp; self.track.write(frame)?; - if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.video.renditions.get_mut(self.track.name()) - { - c.jitter = Some(jitter); + if let Some(jitter) = self.jitter.observe(pts) { + self.rendition.update(|c| c.jitter = Some(jitter)); } } Ok(()) } -} -impl FrameDecode for Import { - fn decode>(&mut self, frames: I) -> Result<()> { + /// Publish split frames, resolving the avc3 config from the first keyframe's + /// inline SPS and refining the catalog jitter as it goes. + pub fn decode(&mut self, frames: impl IntoIterator) -> Result<()> { self.write_frames(frames) } } -impl Renditions for Import { - fn renditions(&self) -> &hang::Catalog { - &self.catalog - } -} - /// Detect the avc1 wire shape from leading bytes: a 3- or 4-byte Annex-B start /// code means avc3, otherwise an AVCDecoderConfigurationRecord (avc1). fn detect_avc1(bytes: &[u8]) -> bool { @@ -260,14 +236,16 @@ mod tests { use super::*; use crate::codec::h264::Split; - fn track(name: &str) -> moq_net::TrackProducer { + fn setup(name: &str) -> (moq_net::TrackProducer, crate::catalog::Producer) { let mut broadcast = moq_net::BroadcastInfo::new().produce(); - broadcast + let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); + let track = broadcast .create_track( name, moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), ) - .unwrap() + .unwrap(); + (track, catalog) } /// An avcC initializer resolves a config with the avcC stored as `description`. @@ -278,19 +256,15 @@ mod tests { avcc.extend_from_slice(&sps_nal); avcc.extend_from_slice(&[0x01, 0x00, 0x04, 0x68, 0xce, 0x3c, 0x80]); // num_pps + pps - let mut import = Import::from_track(track("video")); + let (track, catalog) = setup("video"); + let mut import = Import::new(track, catalog.clone()); // initialize() must not consume the buffer (the split owns the consume). let mut buf = bytes::BytesMut::from(avcc.as_slice()); import.initialize(&mut buf).expect("initialize avc1"); assert_eq!(buf.len(), avcc.len(), "initialize must not consume the buffer"); - let cfg = import - .catalog() - .expect("catalog known after init") - .video - .renditions - .get("video") - .expect("rendition"); + let snapshot = catalog.snapshot(); + let cfg = snapshot.video.renditions.get("video").expect("rendition"); let hang::catalog::VideoCodec::H264(h264) = &cfg.codec else { panic!("expected H.264 codec") }; @@ -318,16 +292,20 @@ mod tests { } let mut split = Split::new(); - let mut import = Import::from_track(track("video")); - assert!(import.catalog().is_none(), "no config before any frame"); + let (track, catalog) = setup("video"); + let mut import = Import::new(track, catalog.clone()); + assert!( + catalog.snapshot().video.renditions.is_empty(), + "no config before any frame" + ); let pts = moq_net::Timestamp::from_micros(0).unwrap(); let mut frames = split.decode(&mut annexb, pts).expect("split keyframe"); frames.extend(split.flush(pts).expect("flush keyframe")); import.decode(frames).expect("decode keyframe"); - let cfg = import.catalog().expect("config after keyframe"); - let h264_cfg = cfg.video.renditions.get("video").expect("rendition"); + let snapshot = catalog.snapshot(); + let h264_cfg = snapshot.video.renditions.get("video").expect("rendition"); let hang::catalog::VideoCodec::H264(h264) = &h264_cfg.codec else { panic!("expected H.264 codec") }; @@ -347,7 +325,8 @@ mod tests { annexb.extend_from_slice(idr); let mut split = Split::new(); - let mut import = Import::from_track(track("video")); + let (track, catalog) = setup("video"); + let mut import = Import::new(track, catalog); let pts = moq_net::Timestamp::from_micros(0).unwrap(); let mut frames = split.decode(&mut annexb, pts).expect("split keyframe"); @@ -369,7 +348,8 @@ mod tests { annexb.extend_from_slice(pslice); let mut split = Split::new(); - let mut import = Import::from_track(track("video")); + let (track, catalog) = setup("video"); + let mut import = Import::new(track, catalog.clone()); let pts = moq_net::Timestamp::from_micros(0).unwrap(); let mut frames = split.decode(&mut annexb, pts).expect("split delta"); @@ -378,6 +358,9 @@ mod tests { .decode(frames) .expect_err("a delta before any keyframe must report MissingKeyframe"); assert!(matches!(err, crate::Error::MissingKeyframe(_)), "got {err:?}"); - assert!(import.catalog().is_none(), "no config yet, so no catalog"); + assert!( + catalog.snapshot().video.renditions.is_empty(), + "no config yet, so no catalog" + ); } } diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index 6178cd3c4..13d5e4268 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -10,45 +10,40 @@ //! are written through to the producer, which reports //! [`MissingKeyframe`](crate::container::MissingKeyframe) for a mid-stream join. //! Annex-B byte parsing lives in [`Split`](super::Split); this type is a pure frame publisher -//! that whoever owns the split drives via the [`FrameDecode`] trait. +//! that whoever owns the split drives via [`decode`](Import::decode). -use bytes::{Buf, Bytes}; +use bytes::Bytes; use scuffle_h265::SpsNALUnit; use super::{Error, split::nal_unit_type}; use crate::Result; +use crate::catalog::hang::CatalogExt; use crate::codec::annexb::NalIterator; use crate::container::Frame; use crate::container::jitter::MinFrameDuration; -use crate::import::{FrameDecode, Renditions}; /// A pure-publisher importer for H.265 with inline VPS/SPS/PPS. /// Only supports single layer streams (VPS is cached but not parsed). /// -/// Build it from a [`moq_net::TrackRequest`] ([`new`](Self::new)) or an existing -/// track ([`from_track`](Self::from_track)), and feed it frames a [`Split`](super::Split) -/// produced via the [`FrameDecode`] impl. The catalog rendition fills in lazily -/// once the first SPS is parsed; read it via [`catalog`](Self::catalog). -pub struct Import { +/// Build it with [`new`](Self::new), passing the track producer and the +/// [`catalog::Producer`](crate::catalog::Producer) it publishes into, and feed it +/// frames a [`Split`](super::Split) produced via [`decode`](Self::decode). The +/// catalog rendition fills in lazily once the first SPS is parsed. +pub struct Import { track: crate::container::Producer, - catalog: hang::Catalog, + rendition: crate::catalog::VideoTrack, config: Option, last_sps: Option, jitter: MinFrameDuration, } -impl Import { - /// Serve a track request, accepting it at the microsecond timescale. - pub fn new(request: moq_net::TrackRequest) -> Self { - let info = moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE); - Self::from_track(request.accept(info)) - } - - /// Publish on an existing track producer. - pub fn from_track(track: moq_net::TrackProducer) -> Self { +impl Import { + /// Publish on an existing track producer, registering the rendition in `catalog`. + pub fn new(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { + let rendition = catalog.video_track(track.name()); Self { track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), - catalog: hang::Catalog::default(), + rendition, config: None, last_sps: None, jitter: MinFrameDuration::new(), @@ -57,12 +52,12 @@ impl Import { /// Resolve the codec config from VPS/SPS/PPS and other non-slice NALs. /// - /// Resolves the config from any SPS in the buffer. Optional, since the - /// importer also self-initializes from the first keyframe. The buffer is - /// *not* consumed: the dispatcher-owned [`Split`](super::Split) consumes it (and seeds its - /// parameter-set cache). - pub fn initialize>(&mut self, buf: &mut T) -> Result<()> { - let mut scan = Bytes::copy_from_slice(buf.as_ref()); + /// Resolves the config from any SPS in the buffer. Optional, since the importer + /// also self-initializes from the first keyframe. Takes a read-only slice: the + /// dispatcher-owned [`Split`](super::Split) is what consumes the stream (and seeds + /// its parameter-set cache). + pub fn initialize(&mut self, buf: &[u8]) -> Result<()> { + let mut scan = Bytes::copy_from_slice(buf); let mut nals = NalIterator::new(&mut scan); while let Some(nal) = nals.next().transpose()? { if is_sps(&nal) { @@ -77,14 +72,9 @@ impl Import { Ok(()) } - /// The underlying track producer. - pub fn track(&self) -> &moq_net::TrackProducer { - self.track.track() - } - - /// The standalone catalog once the first SPS is parsed, else `None`. - pub fn catalog(&self) -> Option<&hang::Catalog> { - self.config.is_some().then_some(&self.catalog) + /// A watch-only handle to this track's subscriber demand. + pub fn demand(&self) -> moq_net::TrackDemand { + self.track.track().demand() } /// True once the first SPS has populated the catalog. @@ -139,9 +129,8 @@ impl Import { return Ok(()); } - let track_name = self.track.name().to_string(); - tracing::debug!(name = ?track_name, ?config, "starting track"); - self.catalog.video.renditions.insert(track_name, config.clone()); + tracing::debug!(name = ?self.track.name(), ?config, "starting track"); + self.rendition.set(config.clone()); self.config = Some(config); Ok(()) } @@ -166,28 +155,20 @@ impl Import { // MissingKeyframe, which the caller (e.g. a TS mid-stream join) skips. self.track.write(frame)?; - if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.video.renditions.get_mut(self.track.name()) - { - c.jitter = Some(jitter); + if let Some(jitter) = self.jitter.observe(pts) { + self.rendition.update(|c| c.jitter = Some(jitter)); } } Ok(()) } -} -impl FrameDecode for Import { - fn decode>(&mut self, frames: I) -> Result<()> { + /// Publish split frames, resolving the config from the first keyframe's inline + /// SPS and refining the catalog jitter as it goes. + pub fn decode(&mut self, frames: impl IntoIterator) -> Result<()> { self.write_frames(frames) } } -impl Renditions for Import { - fn renditions(&self) -> &hang::Catalog { - &self.catalog - } -} - fn is_sps(nal: &[u8]) -> bool { nal.first() .is_some_and(|h| nal_unit_type(*h) == scuffle_h265::NALUnitType::SpsNut) diff --git a/rs/moq-mux/src/codec/legacy.rs b/rs/moq-mux/src/codec/legacy.rs index 3c6af9da9..56b37abc7 100644 --- a/rs/moq-mux/src/codec/legacy.rs +++ b/rs/moq-mux/src/codec/legacy.rs @@ -7,10 +7,10 @@ //! codec contributes only a header parser and a [`Descriptor`]; this module //! owns the track lifecycle. -use bytes::{Buf, BytesMut}; use moq_net::Timestamp; -use crate::import::Renditions; +use crate::catalog::hang::CatalogExt; +use crate::container::Frame; /// Legacy audio (MP2 / AC-3 / E-AC-3) header parsing errors. #[derive(Debug, Clone, thiserror::Error)] @@ -113,16 +113,21 @@ pub(crate) struct Config { /// Publishes each whole frame as one hang frame in its own group, so the relay /// forwards it immediately. The audio is never decoded; the catalog carries the /// codec, sample rate and channel count read from the frame header. -pub(crate) struct Import { +pub(crate) struct Import { track: crate::container::Producer, - catalog: hang::Catalog, - zero: Option, + rendition: crate::catalog::AudioTrack, } -impl Import { - /// Publish on an existing track. Mint it at the descriptor's suffix and the - /// microsecond [`hang::container::TIMESCALE`] (e.g. via [`crate::import::unique_track`]). - pub fn from_track(descriptor: &'static Descriptor, track: moq_net::TrackProducer, config: Config) -> Self { +impl Import { + /// Publish on an existing track, registering the rendition in `catalog`. Mint the + /// track at the descriptor's suffix and the microsecond + /// [`hang::container::TIMESCALE`] (e.g. via [`crate::import::unique_track`]). + pub fn new( + descriptor: &'static Descriptor, + track: moq_net::TrackProducer, + catalog: crate::catalog::Producer, + config: Config, + ) -> Self { let mut audio_config = hang::catalog::AudioConfig::new(descriptor.codec.clone(), config.sample_rate, config.channel_count); audio_config.container = hang::catalog::Container::Legacy; @@ -132,13 +137,12 @@ impl Import { tracing::debug!(name = ?track.name(), config = ?audio_config, "starting track"); - let mut catalog = hang::Catalog::default(); - catalog.audio.renditions.insert(track.name().to_string(), audio_config); + let mut rendition = catalog.audio_track(track.name()); + rendition.set(audio_config); Self { track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), - catalog, - zero: None, + rendition, } } @@ -155,42 +159,15 @@ impl Import { } /// Publish one whole frame as a hang frame in its own group. - pub fn decode(&mut self, buf: &mut T, pts: Option) -> crate::Result<()> { - let pts = self.pts(pts)?; - - let mut payload = BytesMut::with_capacity(buf.remaining()); - while buf.has_remaining() { - let chunk = buf.chunk(); - payload.extend_from_slice(chunk); - let len = chunk.len(); - buf.advance(len); - } - - let frame = crate::container::Frame { - timestamp: pts, + pub fn decode(&mut self, frame: &[u8], pts: Option) -> crate::Result<()> { + let timestamp = self.rendition.timestamp(pts)?; + self.track.write(Frame { + timestamp, duration: None, - payload: payload.freeze(), + payload: bytes::Bytes::copy_from_slice(frame), keyframe: true, - }; - - self.track.write(frame)?; + })?; self.track.finish_group()?; - Ok(()) } - - fn pts(&mut self, hint: Option) -> crate::Result { - if let Some(pts) = hint { - return Ok(pts); - } - - let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); - Ok(Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) - } -} - -impl Renditions for Import { - fn renditions(&self) -> &hang::Catalog { - &self.catalog - } } diff --git a/rs/moq-mux/src/codec/opus/import.rs b/rs/moq-mux/src/codec/opus/import.rs index b7ebdde58..9fe97367f 100644 --- a/rs/moq-mux/src/codec/opus/import.rs +++ b/rs/moq-mux/src/codec/opus/import.rs @@ -1,35 +1,28 @@ -use bytes::{Buf, BytesMut}; - use super::Config; +use crate::catalog::hang::CatalogExt; use crate::container::Frame; -use crate::import::Renditions; /// Opus importer. /// -/// Publishes raw Opus frames (no Ogg framing) to a single moq track. Build it -/// from a [`moq_net::TrackRequest`] (the on-demand path, [`new`](Self::new)) or -/// an existing [`moq_net::TrackProducer`] ([`from_track`](Self::from_track)). +/// Publishes raw Opus frames (no Ogg framing) to a single moq track. Build it with +/// [`new`](Self::new), passing the track producer and the +/// [`catalog::Producer`](crate::catalog::Producer) it publishes its rendition into. /// -/// Each input frame is published in its own group so the relay can forward it -/// immediately without waiting for a group boundary; Opus' packet loss -/// concealment handles drops. The catalog rendition this importer publishes is -/// available via [`catalog`](Self::catalog); attach it to a broadcast catalog -/// with [`crate::import::Track`]. -pub struct Import { +/// Each packet handed to [`decode`](Self::decode) is published in its own group so +/// the relay can forward it immediately without waiting for a group boundary; Opus' +/// packet loss concealment handles drops. +pub struct Import { track: crate::container::Producer, - catalog: hang::Catalog, - zero: Option, + rendition: crate::catalog::AudioTrack, } -impl Import { - /// Serve a track request, accepting it at the microsecond timescale. - pub fn new(request: moq_net::TrackRequest, config: Config) -> crate::Result { - let info = moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE); - Self::from_track(request.accept(info), config) - } - - /// Publish on an existing track producer. - pub fn from_track(track: moq_net::TrackProducer, config: Config) -> crate::Result { +impl Import { + /// Publish on an existing track producer, registering the rendition in `catalog`. + pub fn new( + track: moq_net::TrackProducer, + catalog: crate::catalog::Producer, + config: Config, + ) -> crate::Result { let mut audio = hang::catalog::AudioConfig::new( hang::catalog::AudioCodec::Opus, config.sample_rate, @@ -39,25 +32,18 @@ impl Import { tracing::debug!(name = ?track.name(), config = ?audio, "starting track"); - let mut catalog = hang::Catalog::default(); - catalog.audio.renditions.insert(track.name().to_string(), audio); + let mut rendition = catalog.audio_track(track.name()); + rendition.set(audio); Ok(Self { track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), - catalog, - zero: None, + rendition, }) } - /// The standalone catalog this importer publishes (one Opus audio rendition). - pub fn catalog(&self) -> &hang::Catalog { - &self.catalog - } - - /// The underlying track producer, e.g. for monitoring subscriber state via - /// `used()` / `unused()`. - pub fn track(&self) -> &moq_net::TrackProducer { - self.track.track() + /// A watch-only handle to this track's subscriber demand. + pub fn demand(&self) -> moq_net::TrackDemand { + self.track.track().demand() } /// Finish the track, flushing the current group. @@ -72,50 +58,16 @@ impl Import { Ok(()) } - /// Publish frames, each in its own group. - pub fn decode(&mut self, frames: impl IntoIterator) -> crate::Result<()> { - for frame in frames { - self.track.write(frame)?; - self.track.finish_group()?; - } - Ok(()) - } - - /// Publish one Opus packet from `buf`, stamping `pts` or a wall clock when absent. - /// - /// Convenience for callers that hand over raw packet bytes plus an optional - /// timestamp; it wraps the packet in a [`Frame`] and forwards to [`decode`](Self::decode). - pub fn decode_buf(&mut self, buf: &mut T, pts: Option) -> crate::Result<()> { - let timestamp = self.pts(pts)?; - - let mut payload = BytesMut::with_capacity(buf.remaining()); - while buf.has_remaining() { - let chunk = buf.chunk(); - payload.extend_from_slice(chunk); - let len = chunk.len(); - buf.advance(len); - } - - self.decode(std::iter::once(Frame { + /// Publish one Opus packet as its own group, stamping `pts` or a wall clock when absent. + pub fn decode(&mut self, frame: &[u8], pts: Option) -> crate::Result<()> { + let timestamp = self.rendition.timestamp(pts)?; + self.track.write(Frame { timestamp, - payload: payload.freeze(), + payload: bytes::Bytes::copy_from_slice(frame), keyframe: true, duration: None, - })) - } - - fn pts(&mut self, hint: Option) -> crate::Result { - if let Some(pts) = hint { - return Ok(pts); - } - - let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); - Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) - } -} - -impl Renditions for Import { - fn renditions(&self) -> &hang::Catalog { - &self.catalog + })?; + self.track.finish_group()?; + Ok(()) } } diff --git a/rs/moq-mux/src/codec/vp8/import.rs b/rs/moq-mux/src/codec/vp8/import.rs index 210112ecb..89e9be5b0 100644 --- a/rs/moq-mux/src/codec/vp8/import.rs +++ b/rs/moq-mux/src/codec/vp8/import.rs @@ -1,61 +1,53 @@ -use bytes::Buf; +use bytes::Bytes; +use crate::catalog::hang::CatalogExt; +use crate::container::Frame; use crate::container::jitter::MinFrameDuration; -use crate::import::Renditions; use super::FrameHeader; /// A frame-based importer for raw VP8. /// /// A VP8 elementary stream isn't self-delimiting, so the caller must pass whole -/// frames, one per [`decode_frame`](Self::decode_frame). The first key frame's -/// header supplies the catalog dimensions, so [`catalog`](Self::catalog) is -/// `None` until then. Build it from a [`moq_net::TrackRequest`] ([`new`](Self::new)) -/// or an existing track ([`from_track`](Self::from_track)). -pub struct Import { +/// frames, one per [`decode`](Self::decode). The first key frame's header supplies +/// the catalog dimensions, so the rendition isn't published until then. Build it +/// with [`new`](Self::new), passing the track producer and the +/// [`catalog::Producer`](crate::catalog::Producer) it publishes into. +pub struct Import { // The track being produced. track: crate::container::Producer, - // The standalone catalog, populated on the first key frame. - catalog: hang::Catalog, + // This importer's catalog rendition, published on the first key frame. + rendition: crate::catalog::VideoTrack, // The resolved config, used to detect resolution changes. config: Option, - // Used to compute wall clock timestamps when the caller has none. - zero: Option, - // Tracks the minimum frame duration and updates the catalog `jitter` field. jitter: MinFrameDuration, } -impl Import { - /// Serve a track request, accepting it at the microsecond timescale. - pub fn new(request: moq_net::TrackRequest) -> Self { - let info = moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE); - Self::from_track(request.accept(info)) - } - - /// Publish on an existing track producer. - pub fn from_track(track: moq_net::TrackProducer) -> Self { +impl Import { + /// Publish on an existing track producer, registering the rendition in `catalog`. + pub fn new(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { + let rendition = catalog.video_track(track.name()); Self { track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), - catalog: hang::Catalog::default(), + rendition, config: None, - zero: None, jitter: MinFrameDuration::new(), } } /// Initialize the importer. /// - /// VP8 has no out-of-band configuration record, so this is normally called - /// with an empty buffer (gstreamer / ffi pass `Bytes::new()`) and the catalog - /// is filled from the first key frame. If the caller does pass the first frame - /// here, it's decoded so nothing is dropped. - pub fn initialize>(&mut self, buf: &mut T) -> crate::Result<()> { - if buf.has_remaining() { - self.decode_frame(buf, None)?; + /// VP8 has no out-of-band configuration record, so this is normally called with + /// an empty slice (gstreamer / ffi pass `&[]`) and the catalog is filled from the + /// first key frame. If the caller does pass the first frame here, it's decoded so + /// nothing is dropped. + pub fn initialize(&mut self, buf: &[u8]) -> crate::Result<()> { + if !buf.is_empty() { + self.decode(buf, None)?; } Ok(()) } @@ -71,56 +63,42 @@ impl Import { } tracing::debug!(name = ?self.track.name(), ?config, "starting track"); - self.catalog - .video - .renditions - .insert(self.track.name().to_string(), config.clone()); + self.rendition.set(config.clone()); self.config = Some(config); Ok(()) } /// Decode a single VP8 frame. - pub fn decode_frame>( - &mut self, - buf: &mut T, - pts: Option, - ) -> crate::Result<()> { - let payload = buf.copy_to_bytes(buf.remaining()); - if payload.is_empty() { + pub fn decode(&mut self, frame: &[u8], pts: Option) -> crate::Result<()> { + if frame.is_empty() { return Err(super::Error::EmptyFrame.into()); } + let payload = Bytes::copy_from_slice(frame); let header = FrameHeader::parse(&payload)?; if let Some((width, height)) = header.dimensions { self.init(width, height)?; } - let pts = self.pts(pts)?; - self.track.write(crate::container::Frame { + let pts = self.rendition.timestamp(pts)?; + self.track.write(Frame { timestamp: pts, payload, keyframe: header.keyframe, duration: None, })?; - if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.video.renditions.get_mut(self.track.name()) - { - c.jitter = Some(jitter); + if let Some(jitter) = self.jitter.observe(pts) { + self.rendition.update(|c| c.jitter = Some(jitter)); } Ok(()) } - /// The standalone catalog once the first key frame is seen, else `None`. - pub fn catalog(&self) -> Option<&hang::Catalog> { - self.config.is_some().then_some(&self.catalog) - } - - /// The underlying track producer. - pub fn track(&self) -> &moq_net::TrackProducer { - self.track.track() + /// A watch-only handle to this track's subscriber demand. + pub fn demand(&self) -> moq_net::TrackDemand { + self.track.track().demand() } /// Finish the track, flushing the current group. @@ -139,21 +117,6 @@ impl Import { pub fn is_initialized(&self) -> bool { self.config.is_some() } - - fn pts(&mut self, hint: Option) -> crate::Result { - if let Some(pts) = hint { - return Ok(pts); - } - - let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); - Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) - } -} - -impl Renditions for Import { - fn renditions(&self) -> &hang::Catalog { - &self.catalog - } } #[cfg(test)] @@ -162,25 +125,38 @@ mod tests { use moq_net::Timestamp; + fn setup() -> (moq_net::TrackProducer, crate::catalog::Producer) { + let mut broadcast = moq_net::BroadcastInfo::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); + let track = broadcast + .create_track( + "0.vp8", + moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), + ) + .unwrap(); + (track, catalog) + } + /// A 320x240 key frame followed by an interframe should create a single VP8 /// rendition with the right dimensions and emit both frames. #[tokio::test(start_paused = true)] async fn imports_keyframe_then_interframe() { - let mut import = super::Import::new(moq_net::TrackRequest::new("0.vp8")); + let (track, catalog) = setup(); + let mut import = super::Import::new(track, catalog.clone()); // Empty init buffer: the catalog is filled on the first key frame. - import.initialize(&mut Bytes::new()).unwrap(); + import.initialize(&[]).unwrap(); assert!(!import.is_initialized()); - assert!(import.catalog().is_none()); + assert!(catalog.snapshot().video.renditions.is_empty()); let keyframe = Bytes::from_static(&[0x10, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x40, 0x01, 0xf0, 0x00]); import - .decode_frame(&mut keyframe.clone(), Some(Timestamp::from_micros(0).unwrap())) + .decode(&keyframe, Some(Timestamp::from_micros(0).unwrap())) .unwrap(); assert!(import.is_initialized()); - let catalog = import.catalog().unwrap(); - let config = catalog.video.renditions.get(import.track().name()).unwrap(); + let snapshot = catalog.snapshot(); + let config = snapshot.video.renditions.get("0.vp8").unwrap(); assert_eq!(config.codec, hang::catalog::VideoCodec::VP8); assert_eq!(config.coded_width, Some(320)); assert_eq!(config.coded_height, Some(240)); @@ -188,7 +164,7 @@ mod tests { // Interframe: no start code or dimensions, but still a valid frame. let interframe = Bytes::from_static(&[0x31, 0x00, 0x00, 0xaa, 0xbb]); import - .decode_frame(&mut interframe.clone(), Some(Timestamp::from_micros(33_000).unwrap())) + .decode(&interframe, Some(Timestamp::from_micros(33_000).unwrap())) .unwrap(); import.finish().unwrap(); @@ -198,12 +174,13 @@ mod tests { /// rejects a non-keyframe as the first frame in a group. #[tokio::test(start_paused = true)] async fn rejects_interframe_first() { - let mut import = super::Import::new(moq_net::TrackRequest::new("0.vp8")); + let (track, catalog) = setup(); + let mut import = super::Import::new(track, catalog); - let mut interframe = Bytes::from_static(&[0x31, 0x00, 0x00, 0xaa, 0xbb]); + let interframe = Bytes::from_static(&[0x31, 0x00, 0x00, 0xaa, 0xbb]); assert!( import - .decode_frame(&mut interframe, Some(Timestamp::from_micros(0).unwrap())) + .decode(&interframe, Some(Timestamp::from_micros(0).unwrap())) .is_err() ); } diff --git a/rs/moq-mux/src/codec/vp9/import.rs b/rs/moq-mux/src/codec/vp9/import.rs index 1b28e3ded..9d982ed3e 100644 --- a/rs/moq-mux/src/codec/vp9/import.rs +++ b/rs/moq-mux/src/codec/vp9/import.rs @@ -1,62 +1,53 @@ -use bytes::Buf; +use bytes::Bytes; +use crate::catalog::hang::CatalogExt; +use crate::container::Frame; use crate::container::jitter::MinFrameDuration; -use crate::import::Renditions; use super::FrameHeader; /// A frame-based importer for raw VP9. /// /// Like VP8, a VP9 elementary stream isn't self-delimiting, so the caller must -/// pass whole frames (or superframes), one per -/// [`decode_frame`](Self::decode_frame). The first key frame's header supplies -/// the catalog config, so [`catalog`](Self::catalog) is `None` until then. Build -/// it from a [`moq_net::TrackRequest`] ([`new`](Self::new)) or an existing track -/// ([`from_track`](Self::from_track)). -pub struct Import { +/// pass whole frames (or superframes), one per [`decode`](Self::decode). The first +/// key frame's header supplies the catalog config, so the rendition isn't published +/// until then. Build it with [`new`](Self::new), passing the track producer and the +/// [`catalog::Producer`](crate::catalog::Producer) it publishes into. +pub struct Import { // The track being produced. track: crate::container::Producer, - // The standalone catalog, populated on the first key frame. - catalog: hang::Catalog, + // This importer's catalog rendition, published on the first key frame. + rendition: crate::catalog::VideoTrack, // The resolved config, used to detect resolution / format changes. config: Option, - // Used to compute wall clock timestamps when the caller has none. - zero: Option, - // Tracks the minimum frame duration and updates the catalog `jitter` field. jitter: MinFrameDuration, } -impl Import { - /// Serve a track request, accepting it at the microsecond timescale. - pub fn new(request: moq_net::TrackRequest) -> Self { - let info = moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE); - Self::from_track(request.accept(info)) - } - - /// Publish on an existing track producer. - pub fn from_track(track: moq_net::TrackProducer) -> Self { +impl Import { + /// Publish on an existing track producer, registering the rendition in `catalog`. + pub fn new(track: moq_net::TrackProducer, catalog: crate::catalog::Producer) -> Self { + let rendition = catalog.video_track(track.name()); Self { track: crate::container::Producer::new(track, crate::catalog::hang::Container::Legacy), - catalog: hang::Catalog::default(), + rendition, config: None, - zero: None, jitter: MinFrameDuration::new(), } } /// Initialize the importer. /// - /// VP9 has no out-of-band configuration record, so this is normally called - /// with an empty buffer (gstreamer / ffi pass `Bytes::new()`) and the catalog - /// is filled from the first key frame. If the caller does pass the first frame - /// here, it's decoded so nothing is dropped. - pub fn initialize>(&mut self, buf: &mut T) -> crate::Result<()> { - if buf.has_remaining() { - self.decode_frame(buf, None)?; + /// VP9 has no out-of-band configuration record, so this is normally called with + /// an empty slice (gstreamer / ffi pass `&[]`) and the catalog is filled from the + /// first key frame. If the caller does pass the first frame here, it's decoded so + /// nothing is dropped. + pub fn initialize(&mut self, buf: &[u8]) -> crate::Result<()> { + if !buf.is_empty() { + self.decode(buf, None)?; } Ok(()) } @@ -72,56 +63,42 @@ impl Import { } tracing::debug!(name = ?self.track.name(), ?config, "starting track"); - self.catalog - .video - .renditions - .insert(self.track.name().to_string(), config.clone()); + self.rendition.set(config.clone()); self.config = Some(config); Ok(()) } /// Decode a single VP9 frame (or superframe). - pub fn decode_frame>( - &mut self, - buf: &mut T, - pts: Option, - ) -> crate::Result<()> { - let payload = buf.copy_to_bytes(buf.remaining()); - if payload.is_empty() { + pub fn decode(&mut self, frame: &[u8], pts: Option) -> crate::Result<()> { + if frame.is_empty() { return Err(super::Error::EmptyFrame.into()); } + let payload = Bytes::copy_from_slice(frame); let header = FrameHeader::parse(&payload)?; if let Some(key) = header.key { self.init(key.to_catalog(), key.width, key.height)?; } - let pts = self.pts(pts)?; - self.track.write(crate::container::Frame { + let pts = self.rendition.timestamp(pts)?; + self.track.write(Frame { timestamp: pts, payload, keyframe: header.keyframe, duration: None, })?; - if let Some(jitter) = self.jitter.observe(pts) - && let Some(c) = self.catalog.video.renditions.get_mut(self.track.name()) - { - c.jitter = Some(jitter); + if let Some(jitter) = self.jitter.observe(pts) { + self.rendition.update(|c| c.jitter = Some(jitter)); } Ok(()) } - /// The standalone catalog once the first key frame is seen, else `None`. - pub fn catalog(&self) -> Option<&hang::Catalog> { - self.config.is_some().then_some(&self.catalog) - } - - /// The underlying track producer. - pub fn track(&self) -> &moq_net::TrackProducer { - self.track.track() + /// A watch-only handle to this track's subscriber demand. + pub fn demand(&self) -> moq_net::TrackDemand { + self.track.track().demand() } /// Finish the track, flushing the current group. @@ -140,21 +117,6 @@ impl Import { pub fn is_initialized(&self) -> bool { self.config.is_some() } - - fn pts(&mut self, hint: Option) -> crate::Result { - if let Some(pts) = hint { - return Ok(pts); - } - - let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); - Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) - } -} - -impl Renditions for Import { - fn renditions(&self) -> &hang::Catalog { - &self.catalog - } } #[cfg(test)] @@ -166,34 +128,41 @@ mod tests { // profile 0, 8-bit, CS_BT_601, studio range, 4:2:0, 320x240. const KEYFRAME: &[u8] = &[0x82, 0x49, 0x83, 0x42, 0x20, 0x13, 0xf0, 0x0e, 0xf0, 0x00]; + fn setup() -> (moq_net::TrackProducer, crate::catalog::Producer) { + let mut broadcast = moq_net::BroadcastInfo::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); + let track = broadcast + .create_track( + "0.vp9", + moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), + ) + .unwrap(); + (track, catalog) + } + #[tokio::test(start_paused = true)] async fn imports_keyframe_then_interframe() { - let mut import = super::Import::new(moq_net::TrackRequest::new("0.vp9")); + let (track, catalog) = setup(); + let mut import = super::Import::new(track, catalog.clone()); - import.initialize(&mut Bytes::new()).unwrap(); + import.initialize(&[]).unwrap(); assert!(!import.is_initialized()); - assert!(import.catalog().is_none()); + assert!(catalog.snapshot().video.renditions.is_empty()); import - .decode_frame( - &mut Bytes::from_static(KEYFRAME), - Some(Timestamp::from_micros(0).unwrap()), - ) + .decode(KEYFRAME, Some(Timestamp::from_micros(0).unwrap())) .unwrap(); assert!(import.is_initialized()); - let catalog = import.catalog().unwrap(); - let config = catalog.video.renditions.get(import.track().name()).unwrap(); + let snapshot = catalog.snapshot(); + let config = snapshot.video.renditions.get("0.vp9").unwrap(); assert!(matches!(config.codec, hang::catalog::VideoCodec::VP9(_))); assert_eq!(config.coded_width, Some(320)); assert_eq!(config.coded_height, Some(240)); // Interframe: marker(10) profile(00) show_existing(0) frame_type(1) = 0x84. import - .decode_frame( - &mut Bytes::from_static(&[0x84, 0x00, 0x00]), - Some(Timestamp::from_micros(33_000).unwrap()), - ) + .decode(&[0x84, 0x00, 0x00], Some(Timestamp::from_micros(33_000).unwrap())) .unwrap(); import.finish().unwrap(); @@ -201,12 +170,13 @@ mod tests { #[tokio::test(start_paused = true)] async fn rejects_interframe_first() { - let mut import = super::Import::new(moq_net::TrackRequest::new("0.vp9")); + let (track, catalog) = setup(); + let mut import = super::Import::new(track, catalog); - let mut interframe = Bytes::from_static(&[0x84, 0x00, 0x00]); + let interframe = Bytes::from_static(&[0x84, 0x00, 0x00]); assert!( import - .decode_frame(&mut interframe, Some(Timestamp::from_micros(0).unwrap())) + .decode(&interframe, Some(Timestamp::from_micros(0).unwrap())) .is_err() ); } diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 70e38c23c..f883d6eb2 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -263,10 +263,9 @@ impl Import { let stream = match stream_type { StreamType::H264 => { let track = crate::import::unique_track(&mut self.broadcast, ".avc3")?; - let import = h264::Import::from_track(track); Stream::H264 { split: h264::Split::new(), - import: Box::new(crate::import::Track::new(self.catalog.clone(), import)), + import: Box::new(h264::Import::new(track, self.catalog.clone())), unwrap: PtsUnwrap::default(), } } @@ -274,10 +273,7 @@ impl Import { let track = crate::import::unique_track(&mut self.broadcast, ".hev1")?; Stream::H265 { split: h265::Split::new(), - import: Box::new(crate::import::Track::new( - self.catalog.clone(), - h265::Import::from_track(track), - )), + import: Box::new(h265::Import::new(track, self.catalog.clone())), unwrap: PtsUnwrap::default(), } } @@ -722,12 +718,12 @@ impl ScteReassembler { enum Stream { H264 { split: h264::Split, - import: Box>, + import: Box>, unwrap: PtsUnwrap, }, H265 { split: h265::Split, - import: Box>, + import: Box>, unwrap: PtsUnwrap, }, Aac(Box>), @@ -796,7 +792,7 @@ impl Stream { /// (the sample rate and channel layout aren't in the PMT), so creation is /// deferred until the first frame arrives. struct AacStream { - import: Option>, + import: Option>, broadcast: moq_net::BroadcastProducer, catalog: crate::catalog::Producer, unwrap: PtsUnwrap, @@ -832,13 +828,9 @@ impl AacStream { // WebCodecs) can configure the decoder. TS itself carries it inline. let description = config.encode(); let track = crate::import::unique_track(&mut self.broadcast, ".aac")?; - let mut aac = aac::Import::from_track(track, config)?; - if let Some(rendition) = aac.rendition_mut() { - rendition.description = Some(description); - } - // Published::new mirrors the rendition (description included) on attach. - let import = crate::import::Track::new(self.catalog.clone(), aac); - self.import.insert(import) + let mut aac = aac::Import::new(track, self.catalog.clone(), config)?; + aac.update_rendition(|rendition| rendition.description = Some(description)); + self.import.insert(aac) } }; @@ -853,8 +845,7 @@ impl AacStream { other => other, }; - let mut raw = &data[offset + header.header_len..end]; - import.decode(&mut raw, pts)?; + import.decode(&data[offset + header.header_len..end], pts)?; offset = end; index += 1; @@ -897,12 +888,7 @@ impl AacStream { self.jitter = Some(jitter); if let Some(import) = &mut self.import { - import.decoding(|i| { - if let Some(rendition) = i.rendition_mut() { - rendition.jitter = Some(jitter.into()); - } - crate::Result::Ok(()) - })?; + import.update_rendition(|rendition| rendition.jitter = Some(jitter.into())); } Ok(()) } @@ -929,7 +915,7 @@ impl AacStream { /// players, which cannot decode these codecs. struct LegacyStream { descriptor: &'static legacy::Descriptor, - import: Option>, + import: Option>, broadcast: moq_net::BroadcastProducer, catalog: crate::catalog::Producer, unwrap: PtsUnwrap, @@ -989,14 +975,12 @@ impl LegacyStream { channel_count: header.channel_count, }; let track = crate::import::unique_track(&mut self.broadcast, self.descriptor.track_suffix)?; - let legacy = legacy::Import::from_track(self.descriptor, track, config); - self.import - .insert(crate::import::Track::new(self.catalog.clone(), legacy)) + let legacy = legacy::Import::new(self.descriptor, track, self.catalog.clone(), config); + self.import.insert(legacy) } }; - let mut frame = &data[offset..end]; - import.decode(&mut frame, pts)?; + import.decode(&data[offset..end], pts)?; pts = match pts { // `pts` is a 90 kHz PES PTS; rescale the sample-rate advance to match diff --git a/rs/moq-mux/src/import/mod.rs b/rs/moq-mux/src/import/mod.rs index 838913a58..6db3057c6 100644 --- a/rs/moq-mux/src/import/mod.rs +++ b/rs/moq-mux/src/import/mod.rs @@ -5,12 +5,12 @@ //! (the typical case for files and reassembled network input). [`Stream`] is for //! raw byte streams where frame boundaries have to be inferred (piped Annex-B //! H.264, an fMP4 reader, …). Both pick a concrete importer from a -//! [`FramedFormat`] / [`StreamFormat`] string. The concrete importers themselves -//! live with their format under [`crate::container`] or [`crate::codec`]. +//! [`FramedFormat`] / [`StreamFormat`] string. The concrete importers live with +//! their format under [`crate::container`] or [`crate::codec`] and publish their +//! own catalog rendition (see [`crate::catalog::VideoTrack`] / +//! [`crate::catalog::AudioTrack`]). //! -//! Underneath, [`Track`] binds a bare single-track codec importer to a broadcast -//! catalog, mirroring its [`Renditions`] in and retiring them on drop; -//! [`FrameDecode`] is the contract for handing it already-split frames. +//! [`unique_track`] mints a track for the single-codec importers. mod track; pub use track::*; @@ -113,29 +113,29 @@ enum FramedKind { /// import publishes. Avc3 { split: crate::codec::h264::Split, - import: crate::import::Track, + import: crate::codec::h264::Import, }, /// H.264 avc1 (length-prefixed NALU, out-of-band avcC). No splitter: each /// access unit is wrapped directly. `length_size` is the NALU length prefix /// width read from the avcC. Avc1 { length_size: usize, - import: crate::import::Track, + import: crate::codec::h264::Import, }, // Boxed because it's a large struct and clippy complains about the size. Fmp4(Box), Hev1 { split: crate::codec::h265::Split, - import: crate::import::Track, + import: crate::codec::h265::Import, }, Av01 { split: crate::codec::av1::Split, - import: crate::import::Track, + import: crate::codec::av1::Import, }, - Vp8(crate::import::Track), - Vp9(crate::import::Track), - Aac(crate::import::Track), - Opus(crate::import::Track), + Vp8(crate::codec::vp8::Import), + Vp9(crate::codec::vp9::Import), + Aac(crate::codec::aac::Import), + Opus(crate::codec::opus::Import), // Boxed for the same reason as Fmp4. Mkv(Box), // Boxed for the same reason as Fmp4. @@ -151,169 +151,153 @@ pub struct Framed { decoder: FramedKind, } -/// Build an H.264 avc3 split + import pair, resolving the config and consuming `buf`. +/// Build an H.264 avc3 split + import pair, resolving the config from `init`. /// -/// The import reads `buf` for the codec config (without consuming it); the split -/// then consumes it as the leading bytes of the stream (caching any inline -/// SPS/PPS). Any frames in the init buffer are published. -fn build_h264_avc3>( +/// The import reads `init` for the codec config; the split then reads it as the +/// leading bytes of the stream (caching any inline SPS/PPS). Any frames in the +/// init buffer are published. +fn build_h264_avc3( track: moq_net::TrackProducer, catalog: crate::catalog::Producer, - buf: &mut T, -) -> Result<( - crate::codec::h264::Split, - crate::import::Track, -)> { - let mut import = crate::codec::h264::Import::from_track(track); - import.initialize(buf)?; + init: &[u8], +) -> Result<(crate::codec::h264::Split, crate::codec::h264::Import)> { + let mut import = crate::codec::h264::Import::new(track, catalog); + import.initialize(init)?; let mut split = crate::codec::h264::Split::new(); - let frames = split.decode(buf, None)?; - let mut published = crate::import::Track::new(catalog, import); - published.decode(frames)?; - Ok((split, published)) + let mut data = init; + let frames = split.decode(&mut data, None)?; + import.decode(frames)?; + Ok((split, import)) } /// Build an H.264 avc1 import, resolving the config and the NALU length size from -/// the avcC, and consuming `buf`. avc1 has no splitter: each access unit is -/// wrapped directly via [`crate::codec::h264::avc1_frame`]. -fn build_h264_avc1>( +/// the avcC. avc1 has no splitter: each access unit is wrapped directly via +/// [`crate::codec::h264::avc1_frame`]. +fn build_h264_avc1( track: moq_net::TrackProducer, catalog: crate::catalog::Producer, - buf: &mut T, -) -> Result<(usize, crate::import::Track)> { - let mut import = crate::codec::h264::Import::from_track(track); - import.initialize(buf)?; - let length_size = crate::codec::h264::Avcc::parse(buf.as_ref())?.length_size; - buf.advance(buf.remaining()); - Ok((length_size, crate::import::Track::new(catalog, import))) + init: &[u8], +) -> Result<(usize, crate::codec::h264::Import)> { + let mut import = crate::codec::h264::Import::new(track, catalog); + import.initialize(init)?; + let length_size = crate::codec::h264::Avcc::parse(init)?.length_size; + Ok((length_size, import)) } -/// Build an H.265 split + import pair, resolving the config and consuming `buf`. -fn build_h265>( +/// Build an H.265 split + import pair, resolving the config from `init`. +fn build_h265( track: moq_net::TrackProducer, catalog: crate::catalog::Producer, - buf: &mut T, -) -> Result<( - crate::codec::h265::Split, - crate::import::Track, -)> { - let mut import = crate::codec::h265::Import::from_track(track); - import.initialize(buf)?; + init: &[u8], +) -> Result<(crate::codec::h265::Split, crate::codec::h265::Import)> { + let mut import = crate::codec::h265::Import::new(track, catalog); + import.initialize(init)?; let mut split = crate::codec::h265::Split::new(); - let frames = split.decode(buf, None)?; - let mut published = crate::import::Track::new(catalog, import); - published.decode(frames)?; - Ok((split, published)) + let mut data = init; + let frames = split.decode(&mut data, None)?; + import.decode(frames)?; + Ok((split, import)) } -/// Build an AV1 split + import pair, resolving the config and consuming `buf`. -fn build_av1>( +/// Build an AV1 split + import pair, resolving the config from `init`. +fn build_av1( track: moq_net::TrackProducer, catalog: crate::catalog::Producer, - buf: &mut T, -) -> Result<( - crate::codec::av1::Split, - crate::import::Track, -)> { - let mut import = crate::codec::av1::Import::from_track(track); - import.initialize(buf)?; + init: &[u8], +) -> Result<(crate::codec::av1::Split, crate::codec::av1::Import)> { + let mut import = crate::codec::av1::Import::new(track, catalog); + import.initialize(init)?; let mut split = crate::codec::av1::Split::new(); - // av1C (leading 0x81, ISO/IEC 14496-15) is an out-of-band config record, not - // an OBU stream, so it's read for config (above) and dropped here. Raw OBUs - // are the leading bytes of the stream and feed the splitter. - let data = buf.as_ref(); - let frames = if data.len() >= 16 && data[0] == 0x81 { - buf.advance(buf.remaining()); + // av1C (leading 0x81, ISO/IEC 14496-15) is an out-of-band config record, not an + // OBU stream, so it's read for config (above) and dropped here. Raw OBUs are the + // leading bytes of the stream and feed the splitter. + let frames = if init.len() >= 16 && init[0] == 0x81 { Vec::new() } else { - split.decode(buf, None)? + let mut data = init; + split.decode(&mut data, None)? }; - let mut published = crate::import::Track::new(catalog, import); - published.decode(frames)?; - Ok((split, published)) + import.decode(frames)?; + Ok((split, import)) } impl Framed { /// Create a new framed importer with the given format and initialization data. - /// - /// The buffer will be fully consumed, or an error will be returned. - pub fn new>( + pub fn new( mut broadcast: moq_net::BroadcastProducer, catalog: crate::catalog::Producer, format: FramedFormat, - buf: &mut T, + init: &[u8], ) -> Result { let decoder = match format { FramedFormat::Avc1 => { let track = crate::import::unique_track(&mut broadcast, ".avc1")?; - let (length_size, import) = build_h264_avc1(track, catalog, buf)?; + let (length_size, import) = build_h264_avc1(track, catalog, init)?; FramedKind::Avc1 { length_size, import } } FramedFormat::Avc3 => { let track = crate::import::unique_track(&mut broadcast, ".avc3")?; - let (split, import) = build_h264_avc3(track, catalog, buf)?; + let (split, import) = build_h264_avc3(track, catalog, init)?; FramedKind::Avc3 { split, import } } FramedFormat::Fmp4 => { let mut decoder = Box::new(crate::container::fmp4::Import::new(broadcast, catalog)); - decoder.decode(buf)?; + decoder.decode(&mut { init })?; FramedKind::Fmp4(decoder) } FramedFormat::Hev1 => { let track = crate::import::unique_track(&mut broadcast, ".hev1")?; - let (split, import) = build_h265(track, catalog, buf)?; + let (split, import) = build_h265(track, catalog, init)?; FramedKind::Hev1 { split, import } } FramedFormat::Av01 => { let track = crate::import::unique_track(&mut broadcast, ".av01")?; - let (split, import) = build_av1(track, catalog, buf)?; + let (split, import) = build_av1(track, catalog, init)?; FramedKind::Av01 { split, import } } FramedFormat::Vp8 => { let track = crate::import::unique_track(&mut broadcast, ".vp8")?; - let mut decoder = crate::codec::vp8::Import::from_track(track); - decoder.initialize(buf)?; - FramedKind::Vp8(crate::import::Track::new(catalog, decoder)) + let mut import = crate::codec::vp8::Import::new(track, catalog); + import.initialize(init)?; + FramedKind::Vp8(import) } FramedFormat::Vp9 => { let track = crate::import::unique_track(&mut broadcast, ".vp09")?; - let mut decoder = crate::codec::vp9::Import::from_track(track); - decoder.initialize(buf)?; - FramedKind::Vp9(crate::import::Track::new(catalog, decoder)) + let mut import = crate::codec::vp9::Import::new(track, catalog); + import.initialize(init)?; + FramedKind::Vp9(import) } FramedFormat::Aac => { - let config = crate::codec::aac::Config::parse(buf)?; + let mut data = init; + let config = crate::codec::aac::Config::parse(&mut data)?; let track = crate::import::unique_track(&mut broadcast, ".aac")?; - let import = crate::codec::aac::Import::from_track(track, config)?; - FramedKind::Aac(crate::import::Track::new(catalog, import)) + let import = crate::codec::aac::Import::new(track, catalog, config)?; + FramedKind::Aac(import) } FramedFormat::Opus => { - let config = crate::codec::opus::Config::parse(buf)?; + let mut data = init; + let config = crate::codec::opus::Config::parse(&mut data)?; let track = crate::import::unique_track(&mut broadcast, ".opus")?; - let import = crate::codec::opus::Import::from_track(track, config)?; - FramedKind::Opus(crate::import::Track::new(catalog, import)) + let import = crate::codec::opus::Import::new(track, catalog, config)?; + FramedKind::Opus(import) } FramedFormat::Mkv => { let mut decoder = Box::new(crate::container::mkv::Import::new(broadcast, catalog)); - decoder.decode(buf)?; + decoder.decode(&mut { init })?; FramedKind::Mkv(decoder) } FramedFormat::Ts => { let mut decoder = Box::new(crate::container::ts::Import::new(broadcast, catalog)); - decoder.decode(buf)?; + decoder.decode(&mut { init })?; FramedKind::Ts(decoder) } FramedFormat::Flv => { let mut decoder = Box::new(crate::container::flv::Import::new(broadcast, catalog)); - decoder.decode(buf)?; + decoder.decode(&mut { init })?; FramedKind::Flv(decoder) } }; - if buf.has_remaining() { - return Err(crate::Error::BufferNotConsumed); - } - Ok(Self { decoder }) } @@ -321,56 +305,56 @@ impl Framed { /// /// Only single-track formats are supported. Container formats that may /// create multiple MoQ tracks need an explicit track mapping API. - pub fn new_with_track>( + pub fn new_with_track( track: moq_net::TrackProducer, catalog: crate::catalog::Producer, format: FramedFormat, - buf: &mut T, + init: &[u8], ) -> anyhow::Result { let decoder = match format { FramedFormat::Avc1 => { - let (length_size, import) = build_h264_avc1(track, catalog, buf)?; + let (length_size, import) = build_h264_avc1(track, catalog, init)?; FramedKind::Avc1 { length_size, import } } FramedFormat::Avc3 => { - let (split, import) = build_h264_avc3(track, catalog, buf)?; + let (split, import) = build_h264_avc3(track, catalog, init)?; FramedKind::Avc3 { split, import } } FramedFormat::Hev1 => { - let (split, import) = build_h265(track, catalog, buf)?; + let (split, import) = build_h265(track, catalog, init)?; FramedKind::Hev1 { split, import } } FramedFormat::Av01 => { - let (split, import) = build_av1(track, catalog, buf)?; + let (split, import) = build_av1(track, catalog, init)?; FramedKind::Av01 { split, import } } FramedFormat::Vp8 => { - let mut decoder = crate::codec::vp8::Import::from_track(track); - decoder.initialize(buf)?; - FramedKind::Vp8(crate::import::Track::new(catalog, decoder)) + let mut import = crate::codec::vp8::Import::new(track, catalog); + import.initialize(init)?; + FramedKind::Vp8(import) } FramedFormat::Vp9 => { - let mut decoder = crate::codec::vp9::Import::from_track(track); - decoder.initialize(buf)?; - FramedKind::Vp9(crate::import::Track::new(catalog, decoder)) + let mut import = crate::codec::vp9::Import::new(track, catalog); + import.initialize(init)?; + FramedKind::Vp9(import) } FramedFormat::Aac => { - let config = crate::codec::aac::Config::parse(buf)?; - let import = crate::codec::aac::Import::from_track(track, config)?; - FramedKind::Aac(crate::import::Track::new(catalog, import)) + let mut data = init; + let config = crate::codec::aac::Config::parse(&mut data)?; + let import = crate::codec::aac::Import::new(track, catalog, config)?; + FramedKind::Aac(import) } FramedFormat::Opus => { - let config = crate::codec::opus::Config::parse(buf)?; - let import = crate::codec::opus::Import::from_track(track, config)?; - FramedKind::Opus(crate::import::Track::new(catalog, import)) + let mut data = init; + let config = crate::codec::opus::Config::parse(&mut data)?; + let import = crate::codec::opus::Import::new(track, catalog, config)?; + FramedKind::Opus(import) } FramedFormat::Fmp4 | FramedFormat::Mkv | FramedFormat::Ts | FramedFormat::Flv => { anyhow::bail!("{format} can publish multiple tracks") } }; - anyhow::ensure!(!buf.has_remaining(), "buffer was not fully consumed"); - Ok(Self { decoder }) } @@ -382,10 +366,10 @@ impl Framed { FramedKind::Fmp4(ref mut decoder) => decoder.finish(), FramedKind::Hev1 { ref mut import, .. } => import.finish(), FramedKind::Av01 { ref mut import, .. } => import.finish(), - FramedKind::Vp8(ref mut decoder) => decoder.finish(), - FramedKind::Vp9(ref mut decoder) => decoder.finish(), - FramedKind::Aac(ref mut decoder) => decoder.finish(), - FramedKind::Opus(ref mut decoder) => decoder.finish(), + FramedKind::Vp8(ref mut import) => import.finish(), + FramedKind::Vp9(ref mut import) => import.finish(), + FramedKind::Aac(ref mut import) => import.finish(), + FramedKind::Opus(ref mut import) => import.finish(), FramedKind::Mkv(ref mut decoder) => decoder.finish(), FramedKind::Ts(ref mut decoder) => decoder.finish().map_err(Into::into), FramedKind::Flv(ref mut decoder) => decoder.finish().map_err(Into::into), @@ -418,31 +402,32 @@ impl Framed { split.reset(); import.seek(sequence) } - FramedKind::Vp8(ref mut decoder) => decoder.seek(sequence), - FramedKind::Vp9(ref mut decoder) => decoder.seek(sequence), - FramedKind::Aac(ref mut decoder) => decoder.seek(sequence), - FramedKind::Opus(ref mut decoder) => decoder.seek(sequence), + FramedKind::Vp8(ref mut import) => import.seek(sequence), + FramedKind::Vp9(ref mut import) => import.seek(sequence), + FramedKind::Aac(ref mut import) => import.seek(sequence), + FramedKind::Opus(ref mut import) => import.seek(sequence), FramedKind::Mkv(ref mut decoder) => decoder.seek(sequence), FramedKind::Ts(ref mut decoder) => decoder.seek(sequence).map_err(Into::into), FramedKind::Flv(ref mut decoder) => decoder.seek(sequence).map_err(Into::into), } } - /// The single track's producer. Private: callers get the curated, read-only - /// accessors below ([`name`](Self::name) / [`subscribe`](Self::subscribe) / - /// [`demand`](Self::demand)) so the importer keeps sole ownership of the - /// publishing handle. - fn producer(&self) -> Result<&moq_net::TrackProducer> { + /// A watch-only handle to the single track's subscriber demand. + /// + /// Returns [`Error::MultipleTracks`](crate::Error::MultipleTracks) for container + /// formats that may publish more than one track. The handle can't publish frames + /// or close the track. + pub fn demand(&self) -> Result { match self.decoder { - FramedKind::Avc3 { ref import, .. } => Ok(import.track()), - FramedKind::Avc1 { ref import, .. } => Ok(import.track()), + FramedKind::Avc3 { ref import, .. } => Ok(import.demand()), + FramedKind::Avc1 { ref import, .. } => Ok(import.demand()), FramedKind::Fmp4(_) => Err(crate::Error::MultipleTracks("fmp4")), - FramedKind::Hev1 { ref import, .. } => Ok(import.track()), - FramedKind::Av01 { ref import, .. } => Ok(import.track()), - FramedKind::Vp8(ref decoder) => Ok(decoder.track()), - FramedKind::Vp9(ref decoder) => Ok(decoder.track()), - FramedKind::Aac(ref decoder) => Ok(decoder.track()), - FramedKind::Opus(ref decoder) => Ok(decoder.track()), + FramedKind::Hev1 { ref import, .. } => Ok(import.demand()), + FramedKind::Av01 { ref import, .. } => Ok(import.demand()), + FramedKind::Vp8(ref import) => Ok(import.demand()), + FramedKind::Vp9(ref import) => Ok(import.demand()), + FramedKind::Aac(ref import) => Ok(import.demand()), + FramedKind::Opus(ref import) => Ok(import.demand()), FramedKind::Mkv(_) => Err(crate::Error::MultipleTracks("mkv")), FramedKind::Ts(_) => Err(crate::Error::MultipleTracks("ts")), FramedKind::Flv(_) => Err(crate::Error::MultipleTracks("flv")), @@ -450,43 +435,22 @@ impl Framed { } /// The name of the single track this importer publishes. - /// - /// Returns [`Error::MultipleTracks`](crate::Error::MultipleTracks) for container - /// formats that may publish more than one track. - pub fn name(&self) -> Result<&str> { - Ok(self.producer()?.name()) + pub fn name(&self) -> Result { + Ok(self.demand()?.name().to_string()) } - /// Subscribe to the single track this importer publishes. - /// - /// A read-only handle; it can't publish frames or close the track. Pass `None` - /// for [`Subscription::default`](moq_net::Subscription). - pub fn subscribe( - &self, - subscription: impl Into>, - ) -> Result { - Ok(self.producer()?.subscribe(subscription)) - } - - /// A cloneable, watch-only handle to the single track's subscriber demand. - /// - /// Lets the caller gate work on whether anyone is subscribed (via - /// [`used`](moq_net::TrackDemand::used) / [`unused`](moq_net::TrackDemand::unused)) - /// without the ability to publish or close the track. - pub fn demand(&self) -> Result { - Ok(self.producer()?.demand()) - } - - /// Decode a frame from the given buffer. - pub fn decode_frame>(&mut self, buf: &mut T, pts: Option) -> Result<()> { + /// Decode one whole frame for the single-track formats, or a chunk of container + /// bytes for the multi-track ones. + pub fn decode(&mut self, frame: &[u8], pts: Option) -> Result<()> { match self.decoder { FramedKind::Avc3 { ref mut split, ref mut import, } => { - // Framed hands over one whole access unit per call, so flush to - // emit it rather than waiting for the next start code. - let mut frames = split.decode(buf, pts)?; + // Framed hands over one whole access unit per call, so flush to emit it + // rather than waiting for the next start code. + let mut data = frame; + let mut frames = split.decode(&mut data, pts)?; frames.extend(split.flush(pts)?); import.decode(frames)?; } @@ -495,16 +459,16 @@ impl Framed { ref mut import, } => { let pts = pts.ok_or(crate::codec::h264::Error::MissingTimestamp)?; - let frame = crate::codec::h264::avc1_frame(buf.as_ref(), length_size, pts)?; + let frame = crate::codec::h264::avc1_frame(frame, length_size, pts)?; import.decode([frame])?; - buf.advance(buf.remaining()); } - FramedKind::Fmp4(ref mut decoder) => decoder.decode(buf)?, + FramedKind::Fmp4(ref mut decoder) => decoder.decode(&mut { frame })?, FramedKind::Hev1 { ref mut split, ref mut import, } => { - let mut frames = split.decode(buf, pts)?; + let mut data = frame; + let mut frames = split.decode(&mut data, pts)?; frames.extend(split.flush(pts)?); import.decode(frames)?; } @@ -512,49 +476,37 @@ impl Framed { ref mut split, ref mut import, } => { - let mut frames = split.decode(buf, pts)?; + let mut data = frame; + let mut frames = split.decode(&mut data, pts)?; frames.extend(split.flush(pts)?); import.decode(frames)?; } - FramedKind::Vp8(ref mut decoder) => decoder.decoding(|d| d.decode_frame(buf, pts))?, - FramedKind::Vp9(ref mut decoder) => decoder.decoding(|d| d.decode_frame(buf, pts))?, - FramedKind::Aac(ref mut decoder) => decoder.decode(buf, pts)?, - FramedKind::Opus(ref mut decoder) => decoder.decode_buf(buf, pts)?, - FramedKind::Mkv(ref mut decoder) => { - let _ = pts; - decoder.decode(buf)?; - } - FramedKind::Ts(ref mut decoder) => { - let _ = pts; - decoder.decode(buf)?; - } - FramedKind::Flv(ref mut decoder) => { - let _ = pts; - decoder.decode(buf)?; - } - } - - if buf.has_remaining() { - return Err(crate::Error::BufferNotConsumed); + FramedKind::Vp8(ref mut import) => import.decode(frame, pts)?, + FramedKind::Vp9(ref mut import) => import.decode(frame, pts)?, + FramedKind::Aac(ref mut import) => import.decode(frame, pts)?, + FramedKind::Opus(ref mut import) => import.decode(frame, pts)?, + FramedKind::Mkv(ref mut decoder) => decoder.decode(&mut { frame })?, + FramedKind::Ts(ref mut decoder) => decoder.decode(&mut { frame })?, + FramedKind::Flv(ref mut decoder) => decoder.decode(&mut { frame })?, } Ok(()) } } -// Lift an already-built, catalog-attached opus importer into a `Framed` so callers -// that build their config out-of-band (e.g. moq-gst, which constructs `opus::Config` -// from gstreamer caps instead of an OpusHead buffer) can keep using `.into()`. -impl From> for Framed { - fn from(opus: crate::import::Track) -> Self { +// Lift an already-built opus importer into a `Framed` so callers that build their +// config out-of-band (e.g. moq-gst, which constructs `opus::Config` from gstreamer +// caps instead of an OpusHead buffer) can keep using `.into()`. +impl From for Framed { + fn from(opus: crate::codec::opus::Import) -> Self { Self { decoder: FramedKind::Opus(opus), } } } -impl From> for Framed { - fn from(aac: crate::import::Track) -> Self { +impl From for Framed { + fn from(aac: crate::codec::aac::Import) -> Self { Self { decoder: FramedKind::Aac(aac), } @@ -565,8 +517,6 @@ impl From> for Framed { mod tests { use std::time::Duration; - use bytes::Bytes; - use super::*; use moq_net::Timestamp; @@ -612,10 +562,8 @@ mod tests { ) .unwrap(); let consumer = track.subscribe(None); - let init = opus_head(); - let mut init = init.as_slice(); - let mut framed = Framed::new_with_track(track, catalog.clone(), FramedFormat::Opus, &mut init).unwrap(); + let mut framed = Framed::new_with_track(track, catalog.clone(), FramedFormat::Opus, &opus_head()).unwrap(); assert_eq!(framed.name().unwrap(), "requested-audio"); let snapshot = catalog.snapshot(); @@ -624,9 +572,8 @@ mod tests { let mut media = crate::container::Consumer::new(consumer, crate::catalog::hang::Container::Legacy); let payload = b"opus payload".to_vec(); - let mut frame = payload.as_slice(); framed - .decode_frame(&mut frame, Some(Timestamp::from_micros(1_000).unwrap())) + .decode(&payload, Some(Timestamp::from_micros(1_000).unwrap())) .unwrap(); let frame = tokio::time::timeout(Duration::from_secs(1), media.read()) @@ -641,35 +588,18 @@ mod tests { } #[tokio::test(start_paused = true)] - async fn unique_track_opus_delivers_frames_via_broadcast() { + async fn unique_track_opus_attaches_catalog_and_retires_on_drop() { let (broadcast, catalog) = new_broadcast(); - let init = opus_head(); - let mut init = init.as_slice(); // The broadcast path mints a unique track and attaches its catalog rendition. - let mut framed = Framed::new(broadcast, catalog.clone(), FramedFormat::Opus, &mut init).unwrap(); + let mut framed = Framed::new(broadcast, catalog.clone(), FramedFormat::Opus, &opus_head()).unwrap(); assert_eq!(framed.name().unwrap(), "0.opus"); assert!(catalog.snapshot().audio.renditions.contains_key("0.opus")); - // Frames published through the minted producer are delivered. - let subscriber = framed.subscribe(None).unwrap(); - let mut media = crate::container::Consumer::new(subscriber, crate::catalog::hang::Container::Legacy); - - let payload = b"opus payload".to_vec(); - let mut frame = payload.as_slice(); framed - .decode_frame(&mut frame, Some(Timestamp::from_micros(2_000).unwrap())) - .unwrap(); - - let frame = tokio::time::timeout(Duration::from_secs(1), media.read()) - .await - .unwrap() - .unwrap() + .decode(b"opus payload", Some(Timestamp::from_micros(2_000).unwrap())) .unwrap(); - assert_eq!(frame.payload, payload); - assert_eq!(frame.timestamp, Timestamp::from_micros(2_000).unwrap()); - framed.finish().unwrap(); // Dropping the importer retires its rendition from the shared catalog. @@ -678,26 +608,28 @@ mod tests { } #[tokio::test(start_paused = true)] - async fn opus_import_serves_track_request() { - // The on-demand path: build straight from a TrackRequest, no broadcast/catalog. - let request = moq_net::TrackRequest::new("audio"); + async fn opus_import_delivers_frames() { + let (mut broadcast, catalog) = new_broadcast(); + let track = broadcast + .create_track( + "audio", + moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), + ) + .unwrap(); + let subscriber = track.subscribe(None); + let config = crate::codec::opus::Config { sample_rate: 48_000, channel_count: 2, }; - let mut import = crate::codec::opus::Import::new(request, config).unwrap(); + let mut import = crate::codec::opus::Import::new(track, catalog.clone(), config).unwrap(); + assert!(catalog.snapshot().audio.renditions.contains_key("audio")); - assert_eq!(import.track().name(), "audio"); - assert!(import.catalog().audio.renditions.contains_key("audio")); - - // Accepting the request yields a working producer that delivers frames. - let subscriber = import.track().subscribe(None); let mut media = crate::container::Consumer::new(subscriber, crate::catalog::hang::Container::Legacy); let payload = b"opus payload".to_vec(); - let mut buf = payload.as_slice(); import - .decode_buf(&mut buf, Some(Timestamp::from_micros(1_000).unwrap())) + .decode(&payload, Some(Timestamp::from_micros(1_000).unwrap())) .unwrap(); let frame = tokio::time::timeout(Duration::from_secs(1), media.read()) @@ -714,10 +646,8 @@ mod tests { async fn fixed_track_h264_uses_existing_name_in_catalog() { let (mut broadcast, catalog) = new_broadcast(); let track = broadcast.create_track("camera", None).unwrap(); - let init = h264_init(); - let mut init = init.as_slice(); - let framed = Framed::new_with_track(track, catalog.clone(), FramedFormat::Avc3, &mut init).unwrap(); + let framed = Framed::new_with_track(track, catalog.clone(), FramedFormat::Avc3, &h264_init()).unwrap(); assert_eq!(framed.name().unwrap(), "camera"); let snapshot = catalog.snapshot(); @@ -732,9 +662,8 @@ mod tests { for format in [FramedFormat::Fmp4, FramedFormat::Mkv, FramedFormat::Ts] { let (mut broadcast, catalog) = new_broadcast(); let track = broadcast.create_track("media", None).unwrap(); - let mut init = Bytes::new(); - let err = match Framed::new_with_track(track, catalog, format, &mut init) { + let err = match Framed::new_with_track(track, catalog, format, &[]) { Ok(_) => panic!("multi-track format should be rejected"), Err(err) => err, }; @@ -753,17 +682,20 @@ mod tests { moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE), ) .unwrap(); - let mut init = Bytes::new(); - let mut framed = Framed::new_with_track(track, catalog, FramedFormat::Vp8, &mut init).unwrap(); + let mut framed = Framed::new_with_track(track, catalog, FramedFormat::Vp8, &[]).unwrap(); - let mut first = Bytes::from_static(&[0x10, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x40, 0x01, 0xf0, 0x00]); framed - .decode_frame(&mut first, Some(Timestamp::from_micros(0).unwrap())) + .decode( + &[0x10, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x40, 0x01, 0xf0, 0x00], + Some(Timestamp::from_micros(0).unwrap()), + ) .unwrap(); - let mut second = Bytes::from_static(&[0x10, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x80, 0x02, 0xe0, 0x01]); framed - .decode_frame(&mut second, Some(Timestamp::from_micros(33_000).unwrap())) + .decode( + &[0x10, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x80, 0x02, 0xe0, 0x01], + Some(Timestamp::from_micros(33_000).unwrap()), + ) .unwrap(); } } @@ -826,17 +758,17 @@ enum StreamKind { /// byte parsing; the import publishes. Avc3 { split: crate::codec::h264::Split, - import: crate::import::Track, + import: crate::codec::h264::Import, }, // Boxed because it's a large struct and clippy complains about the size. Fmp4(Box), Hev1 { split: crate::codec::h265::Split, - import: crate::import::Track, + import: crate::codec::h265::Import, }, Av01 { split: crate::codec::av1::Split, - import: crate::import::Track, + import: crate::codec::av1::Import, }, // Boxed for the same reason as Fmp4. Mkv(Box), @@ -864,28 +796,24 @@ impl Stream { let decoder = match format { StreamFormat::Avc3 => { let track = crate::import::unique_track(&mut broadcast, ".avc3")?; - let import = crate::codec::h264::Import::from_track(track); - let split = crate::codec::h264::Split::new(); StreamKind::Avc3 { - split, - import: crate::import::Track::new(catalog, import), + split: crate::codec::h264::Split::new(), + import: crate::codec::h264::Import::new(track, catalog), } } StreamFormat::Fmp4 => StreamKind::Fmp4(Box::new(crate::container::fmp4::Import::new(broadcast, catalog))), StreamFormat::Hev1 => { let track = crate::import::unique_track(&mut broadcast, ".hev1")?; - let import = crate::codec::h265::Import::from_track(track); StreamKind::Hev1 { split: crate::codec::h265::Split::new(), - import: crate::import::Track::new(catalog, import), + import: crate::codec::h265::Import::new(track, catalog), } } StreamFormat::Av01 => { let track = crate::import::unique_track(&mut broadcast, ".av01")?; - let import = crate::codec::av1::Import::from_track(track); StreamKind::Av01 { split: crate::codec::av1::Split::new(), - import: crate::import::Track::new(catalog, import), + import: crate::codec::av1::Import::new(track, catalog), } } StreamFormat::Mkv => StreamKind::Mkv(Box::new(crate::container::mkv::Import::new(broadcast, catalog))), @@ -907,7 +835,7 @@ impl Stream { ref mut split, ref mut import, } => { - import.decoding(|d| d.initialize(buf))?; + import.initialize(buf.as_ref())?; let frames = split.decode(buf, None)?; import.decode(frames)?; } @@ -916,7 +844,7 @@ impl Stream { ref mut split, ref mut import, } => { - import.decoding(|d| d.initialize(buf))?; + import.initialize(buf.as_ref())?; let frames = split.decode(buf, None)?; import.decode(frames)?; } @@ -924,7 +852,7 @@ impl Stream { ref mut split, ref mut import, } => { - import.decoding(|d| d.initialize(buf))?; + import.initialize(buf.as_ref())?; // av1C (leading 0x81) is an out-of-band config record, not an OBU // stream; read for config above and dropped here. let data = buf.as_ref(); diff --git a/rs/moq-mux/src/import/track.rs b/rs/moq-mux/src/import/track.rs index 44ea17a39..ea194a251 100644 --- a/rs/moq-mux/src/import/track.rs +++ b/rs/moq-mux/src/import/track.rs @@ -1,247 +1,12 @@ -//! Bridge from the request-based single-track importers back to a broadcast catalog. -//! -//! A single-track importer (in [`crate::codec`]) produces frames on one track and -//! exposes the catalog renditions it publishes via [`Renditions`]. Most callers, -//! though, work with a whole [`moq_net::BroadcastProducer`] plus a shared -//! [`catalog::Producer`](crate::catalog::Producer). [`Track`] is the adapter: -//! it merges an importer's renditions into that catalog and removes them on drop. -//! -//! For the broadcast-push case, mint a track with [`unique_track`] and build the -//! importer `from_track`. A [`moq_net::TrackRequest`] (from -//! [`BroadcastDynamic::requested_track`](moq_net::BroadcastDynamic::requested_track)) -//! is instead the on-demand path, fed directly to the importer's `new`. -//! -//! Some importers fill their catalog lazily (H.264 only knows its config once SPS -//! arrives) or refine it over time (jitter). Feed them through -//! [`Track::decode`] or [`Track::decoding`], which re-mirror the catalog -//! automatically so new/changed renditions always reach it. - -use std::ops::{Deref, DerefMut}; - -use crate::catalog::hang::CatalogExt; - -/// A single-track importer that exposes the catalog renditions it publishes. -/// -/// Implemented by the per-codec importers so [`Track`] can merge their -/// renditions into a broadcast catalog generically. The returned catalog may be -/// empty (and grow later) for importers that initialize lazily. -pub trait Renditions { - /// The standalone media catalog (video/audio renditions) this importer publishes. - fn renditions(&self) -> &hang::Catalog; -} - -/// A single-track importer that publishes already-split frames. -/// -/// The uniform decode entry point: callers split bytes into [`Frame`](crate::container::Frame)s -/// (a per-format splitter, e.g. [`crate::codec::h264::Split`]) and hand them over. -/// [`Track`] wraps this so the catalog re-mirror can't be forgotten (see -/// [`Track::decode`]). -pub trait FrameDecode { - /// Publish frames on this importer's track. - fn decode>(&mut self, frames: I) -> crate::Result<()>; -} +//! Track-minting helper shared by the import dispatchers and containers. /// Mint a fresh unique track for a legacy single-codec importer. /// /// Picks a unique name from `suffix` and sets the microsecond /// [`hang::container::TIMESCALE`] that the legacy importers stamp their frames -/// with, so the relay gets timing without parsing the payload. Hand the result -/// to the importer's `from_track`. +/// with, so the relay gets timing without parsing the payload. Hand the result to +/// the importer's `new`. pub fn unique_track(broadcast: &mut moq_net::BroadcastProducer, suffix: &str) -> crate::Result { let info = moq_net::TrackInfo::default().with_timescale(hang::container::TIMESCALE); Ok(broadcast.unique_track(suffix, info)?) } - -/// A single-track importer attached to a broadcast catalog. -/// -/// Mirrors the importer's [`Renditions`] into a [`catalog::Producer`](crate::catalog::Producer) -/// and removes them on drop. Derefs to the inner importer, so all of its methods -/// (`decode`, `finish`, `seek`, ...) are available directly. Generic over the -/// catalog extension `E` so it can attach to an extended broadcast catalog (e.g. -/// the one a container holds). -pub struct Track { - inner: I, - catalog: crate::catalog::Producer, - /// The renditions we last mirrored into the catalog, so [`sync`](Self::sync) - /// can diff against the importer's current state and retire them on drop. - published: hang::Catalog, -} - -impl Track { - /// Attach `inner` to `catalog`, mirroring whatever renditions it already has. - pub fn new(catalog: crate::catalog::Producer, inner: I) -> Self { - let mut this = Self { - inner, - catalog, - published: hang::Catalog::default(), - }; - this.sync(); - this - } - - /// Re-mirror the importer's current renditions into the catalog. - /// - /// Runs after each decode via [`decode`](Self::decode) / [`decoding`](Self::decoding), - /// so callers never invoke it directly. A cheap comparison that touches the - /// catalog only when a rendition actually appeared, changed, or was dropped. - fn sync(&mut self) { - let current = self.inner.renditions(); - if self.published == *current { - return; - } - - { - let mut guard = self.catalog.lock(); - - // Retire renditions we published before that the importer dropped. - for name in self.published.video.renditions.keys() { - if !current.video.renditions.contains_key(name) { - guard.video.renditions.remove(name); - } - } - for name in self.published.audio.renditions.keys() { - if !current.audio.renditions.contains_key(name) { - guard.audio.renditions.remove(name); - } - } - - // Insert or update the current ones. - for (name, config) in ¤t.video.renditions { - guard.video.renditions.insert(name.clone(), config.clone()); - } - for (name, config) in ¤t.audio.renditions { - guard.audio.renditions.insert(name.clone(), config.clone()); - } - } - - self.published = current.clone(); - } - - /// Run a decode on the inner importer, then re-mirror the catalog. - /// - /// The footgun-free wrapper for the byte-decode entry points (the importer's - /// `decode_frame` / `decode_stream` / `initialize`): it re-mirrors the catalog - /// after the closure returns, so a lazily-resolved config or refined jitter - /// always reaches it. Prefer [`decode`](Self::decode) where the caller already - /// has split frames. - pub fn decoding(&mut self, decode: impl FnOnce(&mut I) -> crate::Result) -> crate::Result { - let out = decode(&mut self.inner)?; - self.sync(); - Ok(out) - } -} - -impl Track { - /// Publish frames and re-mirror any catalog change in one call. - /// - /// This is the footgun-free path: it re-mirrors the catalog after decoding, so - /// a lazily-resolved config or refined jitter always reaches it. - pub fn decode>(&mut self, frames: It) -> crate::Result<()> { - self.inner.decode(frames)?; - self.sync(); - Ok(()) - } -} - -impl Deref for Track { - type Target = I; - - fn deref(&self) -> &I { - &self.inner - } -} - -impl DerefMut for Track { - fn deref_mut(&mut self) -> &mut I { - &mut self.inner - } -} - -impl Drop for Track { - fn drop(&mut self) { - if self.published == hang::Catalog::default() { - return; - } - - let mut guard = self.catalog.lock(); - for name in self.published.video.renditions.keys() { - guard.video.renditions.remove(name); - } - for name in self.published.audio.renditions.keys() { - guard.audio.renditions.remove(name); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// An importer whose catalog we can mutate, to drive [`Track::sync`]. - struct Fake(hang::Catalog); - - impl Renditions for Fake { - fn renditions(&self) -> &hang::Catalog { - &self.0 - } - } - - impl FrameDecode for Fake { - fn decode>(&mut self, _frames: I) -> crate::Result<()> { - // Simulate an importer resolving its config lazily while decoding. - self.0.video.renditions.insert("v".to_string(), video()); - Ok(()) - } - } - - fn video() -> hang::catalog::VideoConfig { - let mut config = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); - config.container = hang::catalog::Container::Legacy; - config - } - - #[tokio::test(start_paused = true)] - async fn sync_propagates_lazily_and_drop_retires() { - let mut broadcast = moq_net::BroadcastInfo::new().produce(); - let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); - - // Importer starts with an empty catalog (lazy init): nothing merged yet. - let mut published = Track::new(catalog.clone(), Fake(hang::Catalog::default())); - assert!(catalog.snapshot().video.renditions.is_empty()); - - // A rendition appears later; decoding mirrors it into the broadcast catalog. - published - .decoding(|i| { - i.0.video.renditions.insert("v".to_string(), video()); - crate::Result::Ok(()) - }) - .unwrap(); - assert!(catalog.snapshot().video.renditions.contains_key("v")); - - // An update to the same rendition propagates too. - published - .decoding(|i| { - i.0.video.renditions.get_mut("v").unwrap().bitrate = Some(1_000); - crate::Result::Ok(()) - }) - .unwrap(); - assert_eq!(catalog.snapshot().video.renditions["v"].bitrate, Some(1_000)); - - // Dropping the wrapper retires the rendition. - drop(published); - assert!(catalog.snapshot().video.renditions.is_empty()); - } - - #[tokio::test(start_paused = true)] - async fn decode_auto_syncs_catalog() { - let mut broadcast = moq_net::BroadcastInfo::new().produce(); - let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); - - let mut published = Track::new(catalog.clone(), Fake(hang::Catalog::default())); - assert!(catalog.snapshot().video.renditions.is_empty()); - - // `decode` resolves the rendition and mirrors it — no manual `sync()`. - published.decode(std::iter::empty::()).unwrap(); - assert!(catalog.snapshot().video.renditions.contains_key("v")); - } -} diff --git a/rs/moq-rtc/src/codec/h264.rs b/rs/moq-rtc/src/codec/h264.rs index 4a09ce181..2d17f9a1f 100644 --- a/rs/moq-rtc/src/codec/h264.rs +++ b/rs/moq-rtc/src/codec/h264.rs @@ -11,14 +11,13 @@ use crate::{Result, codec}; pub struct Bridge { split: moq_mux::codec::h264::Split, - import: moq_mux::import::Track, + import: moq_mux::codec::h264::Import, } impl Bridge { pub fn new(mut broadcast: moq_net::BroadcastProducer, catalog: moq_mux::catalog::Producer) -> Result { let track = moq_mux::import::unique_track(&mut broadcast, ".avc3")?; - let import = moq_mux::codec::h264::Import::from_track(track); - let import = moq_mux::import::Track::new(catalog, import); + let import = moq_mux::codec::h264::Import::new(track, catalog); let split = moq_mux::codec::h264::Split::new(); Ok(Self { split, import }) } diff --git a/rs/moq-rtc/src/codec/opus.rs b/rs/moq-rtc/src/codec/opus.rs index f3def248e..14fe510c2 100644 --- a/rs/moq-rtc/src/codec/opus.rs +++ b/rs/moq-rtc/src/codec/opus.rs @@ -6,7 +6,7 @@ use crate::{Result, codec}; pub struct Bridge { - import: moq_mux::import::Track, + import: moq_mux::codec::opus::Import, } impl Bridge { @@ -21,8 +21,7 @@ impl Bridge { channel_count, }; let track = moq_mux::import::unique_track(&mut broadcast, ".opus")?; - let import = moq_mux::codec::opus::Import::from_track(track, config)?; - let import = moq_mux::import::Track::new(catalog, import); + let import = moq_mux::codec::opus::Import::new(track, catalog, config)?; Ok(Self { import }) } } @@ -31,8 +30,7 @@ impl codec::Bridge for Bridge { fn push(&mut self, frame: codec::Frame) -> Result<()> { let pts = moq_net::Timestamp::from_micros(frame.timestamp_us) .map_err(|err| crate::Error::Other(anyhow::anyhow!("invalid timestamp: {err}")))?; - let mut payload = frame.payload; - self.import.decode_buf(&mut payload, Some(pts))?; + self.import.decode(&frame.payload, Some(pts))?; Ok(()) } } diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index 6ba4356c3..811622051 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -26,23 +26,22 @@ const DEFAULT_FRAMERATE: u32 = 30; /// catalog registration and framing. pub struct Producer { split: moq_mux::codec::h264::Split, - import: moq_mux::import::Track, + import: moq_mux::codec::h264::Import, } impl Producer { pub fn new(mut broadcast: moq_net::BroadcastProducer, catalog: moq_mux::catalog::Producer) -> Result { let track = moq_mux::import::unique_track(&mut broadcast, ".avc3")?; - let import = moq_mux::codec::h264::Import::from_track(track); - let import = moq_mux::import::Track::new(catalog, import); + let import = moq_mux::codec::h264::Import::new(track, catalog); let split = moq_mux::codec::h264::Split::new(); Ok(Self { split, import }) } - /// The underlying track producer, created eagerly so subscription state is - /// observable before any frames arrive. Clone it to watch via - /// [`used`](moq_net::TrackProducer::used) / [`unused`](moq_net::TrackProducer::unused). - pub fn track(&self) -> &moq_net::TrackProducer { - self.import.track() + /// A watch-only handle to the track's subscriber demand, created eagerly so + /// subscription state is observable before any frames arrive. Watch it via + /// [`used`](moq_net::TrackDemand::used) / [`unused`](moq_net::TrackDemand::unused). + pub fn demand(&self) -> moq_net::TrackDemand { + self.import.demand() } /// Publish already-encoded Annex-B packets at the given timestamp. @@ -100,7 +99,7 @@ pub async fn publish_capture( } let producer = Producer::new(broadcast, catalog)?; - let track = producer.track().clone(); + let demand = producer.demand(); let gate = Gate::new(); @@ -112,7 +111,7 @@ pub async fn publish_capture( // Surface a capture/encode failure (e.g. camera open) promptly. res = &mut worker => res.map_err(|e| Error::Codec(anyhow::anyhow!("capture task: {e}")))?, // The broadcast was dropped: stop the worker and wait for it to flush. - () = monitor_demand(&track, &gate) => { + () = monitor_demand(&demand, &gate) => { gate.close(); worker .await @@ -123,13 +122,13 @@ pub async fn publish_capture( /// Toggle the gate as viewers subscribe and unsubscribe. Returns once the /// track stops being announced (broadcast dropped / aborted). -async fn monitor_demand(track: &moq_net::TrackProducer, gate: &Gate) { +async fn monitor_demand(demand: &moq_net::TrackDemand, gate: &Gate) { loop { - match track.used().await { + match demand.used().await { Ok(()) => gate.set_active(true), Err(err) => return log_track_ended(err), } - match track.unused().await { + match demand.unused().await { Ok(()) => gate.set_active(false), Err(err) => return log_track_ended(err), }