Make iOS simulator interactive bitrate an optional cap, uncapped by default

.devin/skills/ios-simulator-video/SKILL.md

@@ -39,13 +39,35 @@ So "chunky, slow and janky" was a single root cause, not three.
 
 The fix is resolution, not bitrate. A phone framebuffer is roughly fifteen
 times the pixels the browser actually displays, so the long edge is capped at
-1280 by default (`--max-dimension`, or `--native-resolution` to disable) and
-the bitrate is derived from the encode resolution at ~0.15 bpp rather than
-being a fixed number. Same stimulus, same bandwidth:
+1280 by default (`--max-dimension`, or `--native-resolution` to disable).
+Historically the bitrate was then derived from the encode resolution at
+~0.15 bpp; measured against a fixed cap at the same stimulus and bandwidth:
 
     native 3.16 MP @ 6 Mbps   4.99 Mbps   0.0297 bpp
     588x1280 @ derived        4.95 Mbps   0.1338 bpp
 
+## Uncapped low-latency mode starves itself
+
+`--bitrate` is now an optional cap: omitting it sets **no** `AverageBitRate`.
+The session creates fine without one, but do not assume VideoToolbox spends
+generously when uncapped — it does the opposite. Measured on a 1206x2622
+device encoding at 588x1280, on a screen fully covered by CSS animation,
+25-second windows:
+
+    uncapped (low latency)      0.37-0.48 Mbps   0.029-0.030 bpp    2.7 KB/frame   16-23 fps
+    6 Mbps   (low latency)      5.84 Mbps        0.1302 bpp        12.0 KB/frame   59.6 fps
+    24 Mbps  (low latency)      8.80 Mbps        0.1965 bpp        18.1 KB/frame   59.5 fps
+    uncapped, low latency OFF   4.37 Mbps        0.2272 bpp        20.9 KB/frame   25.6 fps
+
+With low-latency rate control and no `AverageBitRate`, the encoder picks an
+internal budget under half a megabit and drops most frames staying inside it —
+the capped runs at the same stimulus sustained ~60 fps. Turning low latency
+off while staying uncapped restores quality (0.23 bpp) but halves the frame
+rate and re-adds ~300ms of decoder buffering. So an explicit generous cap is
+what actually delivers both fps and quality; uncapped exists for callers that
+want VideoToolbox's own choice and to keep a fixed number out of the default
+path.
+
 Raise `--max-dimension` if viewing in a large or retina window; the default
 trades sharpness for bits on the assumption of a normal-sized preview.
 
@@ -78,9 +100,10 @@ So the two settings are mutually exclusive, and `Tuning` pairs them into a
 single choice rather than letting the useless combination be expressed:
 
 - `Interactive { bitrate }` — low-latency rate control and `MaxFrameDelayCount`
-  0, spending a bitrate derived from the encode resolution. Omitting
-  low-latency costs roughly 300ms of decoder buffering, so this is what any
-  live viewer wants.
+  0. `bitrate` is an optional cap: `Some` sets `AverageBitRate`, `None` sets
+  no rate-control property at all (see the uncapped measurements above).
+  Omitting low-latency costs roughly 300ms of decoder buffering, so this is
+  what any live viewer wants.
 - `Recording { quality }` — no low-latency constraint, so the quality target is
   honoured and bits go where the picture needs them. Latency is unbounded in
   principle.

packages/accessibility-cli/src/lib.rs

@@ -1889,8 +1889,8 @@ pub struct ServeSimArgs {
     #[arg(long, default_value = "interactive")]
     pub tuning: String,
 
-    /// Target bitrate in bits per second, for interactive tuning. Defaults to
-    /// a value derived from the encode resolution.
+    /// Bitrate cap in bits per second, for interactive tuning. Defaults to
+    /// uncapped: no cap is set and VideoToolbox chooses.
     #[arg(long, conflicts_with = "quality")]
     pub bitrate: Option<u32>,
 

packages/accessibility-core/src/platform/ios_simulator/session.rs

@@ -62,6 +62,10 @@ pub struct StatsReport {
     pub keyframe_requests: u64,
     pub lag_events: u64,
     pub subscribers: usize,
+    /// Bitrate cap the encoder was asked to target, in bits per second.
+    /// `None` means uncapped: no `AverageBitRate` was set and VideoToolbox
+    /// chose the spend reflected in `mbps`.
+    pub target_bitrate: Option<u32>,
     /// Frames written to the current recording, or `None` when idle.
     pub recording_frames: Option<u64>,
     /// Capture resolution.
@@ -92,6 +96,8 @@ pub struct SimSession {
     latest_parameter_set: Arc<std::sync::Mutex<Option<EncodedFrame>>>,
     stats: Arc<StreamStats>,
     started: Instant,
+    /// Bitrate cap the stream encoder targets, `None` for uncapped.
+    target_bitrate: Option<u32>,
     input: std::sync::mpsc::Sender<InputCommand>,
     input_capabilities: InputCapabilities,
     ax: mpsc::UnboundedSender<AxCommand>,
@@ -141,6 +147,10 @@ impl SimSession {
             })
         };
 
+        let target_bitrate = match config.tuning {
+            crate::video::Tuning::Interactive { bitrate } => bitrate,
+            crate::video::Tuning::Recording { .. } => None,
+        };
         let (capture, resolved_udid) = start_capture(udid, &config, sink)?;
         let (input, input_capabilities) = spawn_input_worker(&resolved_udid)?;
         let ax = spawn_ax_worker(&resolved_udid)?;
@@ -153,6 +163,7 @@ impl SimSession {
             latest_parameter_set,
             stats,
             started: Instant::now(),
+            target_bitrate,
             input,
             input_capabilities,
             ax,
@@ -215,6 +226,7 @@ impl SimSession {
             keyframe_requests: self.stats.keyframe_requests.load(Ordering::Relaxed),
             lag_events: self.stats.lag_events.load(Ordering::Relaxed),
             subscribers: self.frames.receiver_count(),
+            target_bitrate: self.target_bitrate,
             recording_frames: self.capture.recording_frames(),
             width: geometry.width,
             height: geometry.height,

packages/accessibility-core/src/video/mod.rs

@@ -26,9 +26,12 @@ use bytes::Bytes;
 /// combination that silently does nothing impossible to ask for.
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub enum Tuning {
-    /// Live interactive streaming. Lowest latency; spends a fixed bitrate.
+    /// Live interactive streaming. Lowest latency.
     Interactive {
-        /// Target bitrate, or `None` to derive one from the encode resolution.
+        /// Bitrate cap in bits per second, or `None` for uncapped: the
+        /// encoder chooses how to spend bits. On iOS this maps to
+        /// VideoToolbox `AverageBitRate`; Android derives a bitrate when
+        /// unset.
         bitrate: Option<u32>,
     },
     /// Recording and offline capture. Targets a constant quality from 0 to 1,
@@ -106,10 +109,10 @@ pub struct VideoConfig {
     pub codec: VideoCodec,
     pub nal_format: NalFormat,
     pub fps: u32,
-    /// What to optimize for. Deriving the bitrate is usually right: a fixed
-    /// value that suits a phone framebuffer is wildly wrong for a watch, and
-    /// too low a value does not just soften the image, it makes the encoder
-    /// drop frames.
+    /// What to optimize for. Leaving the bitrate uncapped is usually right:
+    /// a fixed value that suits a phone framebuffer is wildly wrong for a
+    /// watch, and too low a value does not just soften the image, it makes
+    /// the encoder drop frames.
     pub tuning: Tuning,
     /// Longest edge to encode at; the source is scaled down to fit.
     ///

packages/accessibility-ios-sys/src/macos/encoder.rs

@@ -62,15 +62,6 @@ pub struct EncodedChunk {
     pub captured_at: Instant,
 }
 
-/// Bits per pixel per frame to aim for when no explicit bitrate is given.
-///
-/// Screen content needs roughly 0.10-0.20 bpp to avoid visible blocking on
-/// motion. Below that VideoToolbox does not merely soften the picture: with
-/// low-latency rate control it starts *dropping frames* to stay inside its
-/// per-frame budget, so starving the encoder costs frame rate as well as
-/// quality.
-const TARGET_BITS_PER_PIXEL: f64 = 0.15;
-
 /// Longest edge to encode at when no limit is given.
 ///
 /// A phone framebuffer is far larger than the browser ever displays it — an
@@ -91,9 +82,12 @@ const DEFAULT_MAX_DIMENSION: u32 = 1280;
 pub enum Tuning {
     /// Live interactive streaming. Low-latency rate control and no frame
     /// delay, which costs perhaps 300ms of decoder buffering if omitted.
-    /// Spends a fixed bitrate; quality varies with how busy the screen is.
     Interactive {
-        /// Target bitrate, or `None` to derive one from the encode resolution.
+        /// Bitrate cap in bits per second, or `None` for uncapped: no
+        /// `AverageBitRate` is set and VideoToolbox chooses how to spend
+        /// bits. `Some` sets `AverageBitRate`, which low-latency rate
+        /// control enforces per frame — starving it costs frame rate, not
+        /// just quality.
         bitrate: Option<u32>,
     },
     /// Recording and offline capture. Drops the low-latency constraint so the
@@ -112,6 +106,30 @@ impl Tuning {
     }
 }
 
+/// The single rate-control property a tuning sets on the session, if any.
+///
+/// Resolved ahead of the CF calls so the translation is testable without
+/// VideoToolbox.
+#[derive(Debug, Clone, Copy, PartialEq)]
+enum RateControl {
+    /// No rate-control property at all; VideoToolbox chooses.
+    Uncapped,
+    /// `AverageBitRate`, in bits per second.
+    AverageBitRate(i32),
+    /// `Quality`, from 0 to 1.
+    Quality(f64),
+}
+
+fn rate_control(tuning: Tuning) -> RateControl {
+    match tuning {
+        Tuning::Interactive { bitrate: None } => RateControl::Uncapped,
+        Tuning::Interactive {
+            bitrate: Some(bitrate),
+        } => RateControl::AverageBitRate(bitrate as i32),
+        Tuning::Recording { quality } => RateControl::Quality(quality.clamp(0.0, 1.0)),
+    }
+}
+
 #[derive(Debug, Clone, Copy)]
 pub struct EncoderConfig {
     pub fps: u32,
@@ -154,14 +172,6 @@ impl EncoderConfig {
             even((height as f64 * scale).round() as i32),
         )
     }
-
-    /// Bitrate for interactive tuning, derived from the encode size if unset.
-    fn resolved_bitrate(&self, bitrate: Option<u32>, width: i32, height: i32) -> u32 {
-        bitrate.unwrap_or_else(|| {
-            let pixels = (width as f64) * (height as f64);
-            (pixels * self.fps as f64 * TARGET_BITS_PER_PIXEL) as u32
-        })
-    }
 }
 
 fn even(value: i32) -> i32 {
@@ -373,13 +383,13 @@ impl H264Encoder {
             "MaxKeyFrameIntervalDuration",
             self.config.keyframe_interval_secs as f64,
         );
-        match self.config.tuning {
-            Tuning::Interactive { bitrate } => {
-                let bitrate = self.config.resolved_bitrate(bitrate, width, height);
-                set_i32(&session, "AverageBitRate", bitrate as i32);
+        match rate_control(self.config.tuning) {
+            RateControl::Uncapped => {}
+            RateControl::AverageBitRate(bitrate) => {
+                set_i32(&session, "AverageBitRate", bitrate);
             }
-            Tuning::Recording { quality } => {
-                set_f64(&session, "Quality", quality.clamp(0.0, 1.0));
+            RateControl::Quality(quality) => {
+                set_f64(&session, "Quality", quality);
             }
         }
 
@@ -589,3 +599,34 @@ fn set_string(session: &VTCompressionSession, key: &str, value: &str) {
 fn set_f64(session: &VTCompressionSession, key: &str, value: f64) {
     set_property(session, key, CFNumber::new_f64(value).as_ref());
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn interactive_without_bitrate_sets_no_rate_control() {
+        assert_eq!(
+            rate_control(Tuning::Interactive { bitrate: None }),
+            RateControl::Uncapped
+        );
+    }
+
+    #[test]
+    fn interactive_with_bitrate_sets_average_bitrate() {
+        assert_eq!(
+            rate_control(Tuning::Interactive {
+                bitrate: Some(6_000_000)
+            }),
+            RateControl::AverageBitRate(6_000_000)
+        );
+    }
+
+    #[test]
+    fn recording_sets_clamped_quality() {
+        assert_eq!(
+            rate_control(Tuning::Recording { quality: 1.7 }),
+            RateControl::Quality(1.0)
+        );
+    }
+}

packages/accessibility-serve/src/session.rs

@@ -54,6 +54,9 @@ pub struct StatsReport {
     pub keyframe_requests: u64,
     pub lag_events: u64,
     pub subscribers: usize,
+    /// Bitrate cap the encoder targets, in bits per second. `None` means
+    /// uncapped (or the platform does not report one).
+    pub target_bitrate: Option<u32>,
     pub recording_frames: Option<u64>,
     pub width: u32,
     pub height: u32,
@@ -260,6 +263,7 @@ fn from_android_stats(stats: android_session::StatsReport) -> StatsReport {
         keyframe_requests: stats.keyframe_requests,
         lag_events: stats.lag_events,
         subscribers: stats.subscribers,
+        target_bitrate: None,
         recording_frames: stats.recording_frames,
         width: stats.width,
         height: stats.height,
@@ -302,6 +306,7 @@ fn from_ios_stats(stats: ios_session::StatsReport) -> StatsReport {
         keyframe_requests: stats.keyframe_requests,
         lag_events: stats.lag_events,
         subscribers: stats.subscribers,
+        target_bitrate: stats.target_bitrate,
         recording_frames: stats.recording_frames,
         width: stats.width,
         height: stats.height,