From b5dcd7785ddc219389e455880c20c0c3bc5d8eef Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Mon, 24 Aug 2026 17:56:54 -0700 Subject: [PATCH] Make iOS simulator interactive bitrate an optional cap, uncapped by default --- .devin/skills/ios-simulator-video/SKILL.md | 35 +++++-- packages/accessibility-cli/src/lib.rs | 4 +- .../src/platform/ios_simulator/session.rs | 12 +++ packages/accessibility-core/src/video/mod.rs | 15 +-- .../src/macos/encoder.rs | 91 ++++++++++++++----- packages/accessibility-serve/src/session.rs | 5 + 6 files changed, 123 insertions(+), 39 deletions(-) diff --git a/.devin/skills/ios-simulator-video/SKILL.md b/.devin/skills/ios-simulator-video/SKILL.md index 5b5a13a..0775240 100644 --- a/.devin/skills/ios-simulator-video/SKILL.md +++ b/.devin/skills/ios-simulator-video/SKILL.md @@ -39,13 +39,35 @@ So "chunky, slow and janky" was a single root cause, not three. The fix is resolution, not bitrate. A phone framebuffer is roughly fifteen times the pixels the browser actually displays, so the long edge is capped at -1280 by default (`--max-dimension`, or `--native-resolution` to disable) and -the bitrate is derived from the encode resolution at ~0.15 bpp rather than -being a fixed number. Same stimulus, same bandwidth: +1280 by default (`--max-dimension`, or `--native-resolution` to disable). +Historically the bitrate was then derived from the encode resolution at +~0.15 bpp; measured against a fixed cap at the same stimulus and bandwidth: native 3.16 MP @ 6 Mbps 4.99 Mbps 0.0297 bpp 588x1280 @ derived 4.95 Mbps 0.1338 bpp +## Uncapped low-latency mode starves itself + +`--bitrate` is now an optional cap: omitting it sets **no** `AverageBitRate`. +The session creates fine without one, but do not assume VideoToolbox spends +generously when uncapped — it does the opposite. Measured on a 1206x2622 +device encoding at 588x1280, on a screen fully covered by CSS animation, +25-second windows: + + uncapped (low latency) 0.37-0.48 Mbps 0.029-0.030 bpp 2.7 KB/frame 16-23 fps + 6 Mbps (low latency) 5.84 Mbps 0.1302 bpp 12.0 KB/frame 59.6 fps + 24 Mbps (low latency) 8.80 Mbps 0.1965 bpp 18.1 KB/frame 59.5 fps + uncapped, low latency OFF 4.37 Mbps 0.2272 bpp 20.9 KB/frame 25.6 fps + +With low-latency rate control and no `AverageBitRate`, the encoder picks an +internal budget under half a megabit and drops most frames staying inside it — +the capped runs at the same stimulus sustained ~60 fps. Turning low latency +off while staying uncapped restores quality (0.23 bpp) but halves the frame +rate and re-adds ~300ms of decoder buffering. So an explicit generous cap is +what actually delivers both fps and quality; uncapped exists for callers that +want VideoToolbox's own choice and to keep a fixed number out of the default +path. + Raise `--max-dimension` if viewing in a large or retina window; the default trades sharpness for bits on the assumption of a normal-sized preview. @@ -78,9 +100,10 @@ So the two settings are mutually exclusive, and `Tuning` pairs them into a single choice rather than letting the useless combination be expressed: - `Interactive { bitrate }` — low-latency rate control and `MaxFrameDelayCount` - 0, spending a bitrate derived from the encode resolution. Omitting - low-latency costs roughly 300ms of decoder buffering, so this is what any - live viewer wants. + 0. `bitrate` is an optional cap: `Some` sets `AverageBitRate`, `None` sets + no rate-control property at all (see the uncapped measurements above). + Omitting low-latency costs roughly 300ms of decoder buffering, so this is + what any live viewer wants. - `Recording { quality }` — no low-latency constraint, so the quality target is honoured and bits go where the picture needs them. Latency is unbounded in principle. diff --git a/packages/accessibility-cli/src/lib.rs b/packages/accessibility-cli/src/lib.rs index 0e23f84..4e34e35 100644 --- a/packages/accessibility-cli/src/lib.rs +++ b/packages/accessibility-cli/src/lib.rs @@ -1889,8 +1889,8 @@ pub struct ServeSimArgs { #[arg(long, default_value = "interactive")] pub tuning: String, - /// Target bitrate in bits per second, for interactive tuning. Defaults to - /// a value derived from the encode resolution. + /// Bitrate cap in bits per second, for interactive tuning. Defaults to + /// uncapped: no cap is set and VideoToolbox chooses. #[arg(long, conflicts_with = "quality")] pub bitrate: Option, diff --git a/packages/accessibility-core/src/platform/ios_simulator/session.rs b/packages/accessibility-core/src/platform/ios_simulator/session.rs index 9c2ed5e..424f336 100644 --- a/packages/accessibility-core/src/platform/ios_simulator/session.rs +++ b/packages/accessibility-core/src/platform/ios_simulator/session.rs @@ -62,6 +62,10 @@ pub struct StatsReport { pub keyframe_requests: u64, pub lag_events: u64, pub subscribers: usize, + /// Bitrate cap the encoder was asked to target, in bits per second. + /// `None` means uncapped: no `AverageBitRate` was set and VideoToolbox + /// chose the spend reflected in `mbps`. + pub target_bitrate: Option, /// Frames written to the current recording, or `None` when idle. pub recording_frames: Option, /// Capture resolution. @@ -92,6 +96,8 @@ pub struct SimSession { latest_parameter_set: Arc>>, stats: Arc, started: Instant, + /// Bitrate cap the stream encoder targets, `None` for uncapped. + target_bitrate: Option, input: std::sync::mpsc::Sender, input_capabilities: InputCapabilities, ax: mpsc::UnboundedSender, @@ -141,6 +147,10 @@ impl SimSession { }) }; + let target_bitrate = match config.tuning { + crate::video::Tuning::Interactive { bitrate } => bitrate, + crate::video::Tuning::Recording { .. } => None, + }; let (capture, resolved_udid) = start_capture(udid, &config, sink)?; let (input, input_capabilities) = spawn_input_worker(&resolved_udid)?; let ax = spawn_ax_worker(&resolved_udid)?; @@ -153,6 +163,7 @@ impl SimSession { latest_parameter_set, stats, started: Instant::now(), + target_bitrate, input, input_capabilities, ax, @@ -215,6 +226,7 @@ impl SimSession { keyframe_requests: self.stats.keyframe_requests.load(Ordering::Relaxed), lag_events: self.stats.lag_events.load(Ordering::Relaxed), subscribers: self.frames.receiver_count(), + target_bitrate: self.target_bitrate, recording_frames: self.capture.recording_frames(), width: geometry.width, height: geometry.height, diff --git a/packages/accessibility-core/src/video/mod.rs b/packages/accessibility-core/src/video/mod.rs index d04ad7f..5562040 100644 --- a/packages/accessibility-core/src/video/mod.rs +++ b/packages/accessibility-core/src/video/mod.rs @@ -26,9 +26,12 @@ use bytes::Bytes; /// combination that silently does nothing impossible to ask for. #[derive(Debug, Clone, Copy, PartialEq)] pub enum Tuning { - /// Live interactive streaming. Lowest latency; spends a fixed bitrate. + /// Live interactive streaming. Lowest latency. Interactive { - /// Target bitrate, or `None` to derive one from the encode resolution. + /// Bitrate cap in bits per second, or `None` for uncapped: the + /// encoder chooses how to spend bits. On iOS this maps to + /// VideoToolbox `AverageBitRate`; Android derives a bitrate when + /// unset. bitrate: Option, }, /// Recording and offline capture. Targets a constant quality from 0 to 1, @@ -106,10 +109,10 @@ pub struct VideoConfig { pub codec: VideoCodec, pub nal_format: NalFormat, pub fps: u32, - /// What to optimize for. Deriving the bitrate is usually right: a fixed - /// value that suits a phone framebuffer is wildly wrong for a watch, and - /// too low a value does not just soften the image, it makes the encoder - /// drop frames. + /// What to optimize for. Leaving the bitrate uncapped is usually right: + /// a fixed value that suits a phone framebuffer is wildly wrong for a + /// watch, and too low a value does not just soften the image, it makes + /// the encoder drop frames. pub tuning: Tuning, /// Longest edge to encode at; the source is scaled down to fit. /// diff --git a/packages/accessibility-ios-sys/src/macos/encoder.rs b/packages/accessibility-ios-sys/src/macos/encoder.rs index e419afa..6b495dc 100644 --- a/packages/accessibility-ios-sys/src/macos/encoder.rs +++ b/packages/accessibility-ios-sys/src/macos/encoder.rs @@ -62,15 +62,6 @@ pub struct EncodedChunk { pub captured_at: Instant, } -/// Bits per pixel per frame to aim for when no explicit bitrate is given. -/// -/// Screen content needs roughly 0.10-0.20 bpp to avoid visible blocking on -/// motion. Below that VideoToolbox does not merely soften the picture: with -/// low-latency rate control it starts *dropping frames* to stay inside its -/// per-frame budget, so starving the encoder costs frame rate as well as -/// quality. -const TARGET_BITS_PER_PIXEL: f64 = 0.15; - /// Longest edge to encode at when no limit is given. /// /// A phone framebuffer is far larger than the browser ever displays it — an @@ -91,9 +82,12 @@ const DEFAULT_MAX_DIMENSION: u32 = 1280; pub enum Tuning { /// Live interactive streaming. Low-latency rate control and no frame /// delay, which costs perhaps 300ms of decoder buffering if omitted. - /// Spends a fixed bitrate; quality varies with how busy the screen is. Interactive { - /// Target bitrate, or `None` to derive one from the encode resolution. + /// Bitrate cap in bits per second, or `None` for uncapped: no + /// `AverageBitRate` is set and VideoToolbox chooses how to spend + /// bits. `Some` sets `AverageBitRate`, which low-latency rate + /// control enforces per frame — starving it costs frame rate, not + /// just quality. bitrate: Option, }, /// Recording and offline capture. Drops the low-latency constraint so the @@ -112,6 +106,30 @@ impl Tuning { } } +/// The single rate-control property a tuning sets on the session, if any. +/// +/// Resolved ahead of the CF calls so the translation is testable without +/// VideoToolbox. +#[derive(Debug, Clone, Copy, PartialEq)] +enum RateControl { + /// No rate-control property at all; VideoToolbox chooses. + Uncapped, + /// `AverageBitRate`, in bits per second. + AverageBitRate(i32), + /// `Quality`, from 0 to 1. + Quality(f64), +} + +fn rate_control(tuning: Tuning) -> RateControl { + match tuning { + Tuning::Interactive { bitrate: None } => RateControl::Uncapped, + Tuning::Interactive { + bitrate: Some(bitrate), + } => RateControl::AverageBitRate(bitrate as i32), + Tuning::Recording { quality } => RateControl::Quality(quality.clamp(0.0, 1.0)), + } +} + #[derive(Debug, Clone, Copy)] pub struct EncoderConfig { pub fps: u32, @@ -154,14 +172,6 @@ impl EncoderConfig { even((height as f64 * scale).round() as i32), ) } - - /// Bitrate for interactive tuning, derived from the encode size if unset. - fn resolved_bitrate(&self, bitrate: Option, width: i32, height: i32) -> u32 { - bitrate.unwrap_or_else(|| { - let pixels = (width as f64) * (height as f64); - (pixels * self.fps as f64 * TARGET_BITS_PER_PIXEL) as u32 - }) - } } fn even(value: i32) -> i32 { @@ -373,13 +383,13 @@ impl H264Encoder { "MaxKeyFrameIntervalDuration", self.config.keyframe_interval_secs as f64, ); - match self.config.tuning { - Tuning::Interactive { bitrate } => { - let bitrate = self.config.resolved_bitrate(bitrate, width, height); - set_i32(&session, "AverageBitRate", bitrate as i32); + match rate_control(self.config.tuning) { + RateControl::Uncapped => {} + RateControl::AverageBitRate(bitrate) => { + set_i32(&session, "AverageBitRate", bitrate); } - Tuning::Recording { quality } => { - set_f64(&session, "Quality", quality.clamp(0.0, 1.0)); + RateControl::Quality(quality) => { + set_f64(&session, "Quality", quality); } } @@ -589,3 +599,34 @@ fn set_string(session: &VTCompressionSession, key: &str, value: &str) { fn set_f64(session: &VTCompressionSession, key: &str, value: f64) { set_property(session, key, CFNumber::new_f64(value).as_ref()); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn interactive_without_bitrate_sets_no_rate_control() { + assert_eq!( + rate_control(Tuning::Interactive { bitrate: None }), + RateControl::Uncapped + ); + } + + #[test] + fn interactive_with_bitrate_sets_average_bitrate() { + assert_eq!( + rate_control(Tuning::Interactive { + bitrate: Some(6_000_000) + }), + RateControl::AverageBitRate(6_000_000) + ); + } + + #[test] + fn recording_sets_clamped_quality() { + assert_eq!( + rate_control(Tuning::Recording { quality: 1.7 }), + RateControl::Quality(1.0) + ); + } +} diff --git a/packages/accessibility-serve/src/session.rs b/packages/accessibility-serve/src/session.rs index 5a4940d..73287fd 100644 --- a/packages/accessibility-serve/src/session.rs +++ b/packages/accessibility-serve/src/session.rs @@ -54,6 +54,9 @@ pub struct StatsReport { pub keyframe_requests: u64, pub lag_events: u64, pub subscribers: usize, + /// Bitrate cap the encoder targets, in bits per second. `None` means + /// uncapped (or the platform does not report one). + pub target_bitrate: Option, pub recording_frames: Option, pub width: u32, pub height: u32, @@ -260,6 +263,7 @@ fn from_android_stats(stats: android_session::StatsReport) -> StatsReport { keyframe_requests: stats.keyframe_requests, lag_events: stats.lag_events, subscribers: stats.subscribers, + target_bitrate: None, recording_frames: stats.recording_frames, width: stats.width, height: stats.height, @@ -302,6 +306,7 @@ fn from_ios_stats(stats: ios_session::StatsReport) -> StatsReport { keyframe_requests: stats.keyframe_requests, lag_events: stats.lag_events, subscribers: stats.subscribers, + target_bitrate: stats.target_bitrate, recording_frames: stats.recording_frames, width: stats.width, height: stats.height,