feat: synchronize PTY terminal interactions

js/.changeset/tidy-tuis-wait.md

@@ -0,0 +1,7 @@
+---
+'command-stream': minor
+---
+
+Add regex readiness markers and per-interaction output-idle waits to PTY terminal
+capture, enabling deterministic end-to-end TUI navigation with raw key
+sequences.

js/README.md

@@ -415,8 +415,10 @@ const capture = await captureTerminal({
   cols: 80,
   // rows defaults to 30: an 80-column 4:3 terminal with the default cell ratio
   interactions: [
-    { after: 'Choose an option', key: 'DOWN' },
-    { after: 'Second option', key: 'ENTER' },
+    // Readiness accepts a string or RegExp. idleMilliseconds restarts whenever
+    // output arrives, so the key is sent only after the repaint goes quiet.
+    { after: /Choose an option/, idleMilliseconds: 50, key: 'DOWN' },
+    { after: 'Second option', idleMilliseconds: 50, key: 'ENTER' },
     { after: 'Your name', text: 'Ada', key: 'ENTER' },
     { after: 'Dashboard', resize: { cols: 120, rows: 40 } },
   ],
@@ -436,6 +438,26 @@ console.log(capture.transcript);
 console.log(`${capture.frames.length} distinct settled states`);
 ```
 
+Unlike `interactive: true`, `captureTerminal()` allocates a PTY without
+forwarding the session exclusively to the parent terminal. The child sees a
+TTY and raw-mode input while the returned `output`, `frames`, `transcript`, and
+`asciicast` remain available for assertions. To inspect output as it arrives,
+pass `onTrace(event)` and handle events whose `type` is `"output"`.
+
+Named keys include `UP`, `DOWN`, `LEFT`, `RIGHT`, `ENTER`, `TAB`, `ESCAPE`,
+`BACKSPACE`, `CTRL_C`, and `CTRL_D`. The `key` value may also be any raw
+sequence, such as `'\x1b[6~'` for Page Down; `text` is likewise written
+verbatim to the PTY master. Initial `cols`, `rows`, and `TERM` come from the
+capture options and `env`, and interaction `resize` values update the PTY and
+deliver the platform's resize notification to the child.
+
+Each interaction can use `after: 'literal text'` or `after: /pattern/` as its
+readiness condition. Add `idleMilliseconds` to require that no PTY output
+arrive for that duration before applying the interaction; new output restarts
+the idle wait. The complete, runnable
+[`tui-e2e.mjs`](examples/tui-e2e.mjs) example navigates a raw-mode menu and
+asserts on its captured output.
+
 The artifact directory contains an unrolled `transcript.txt`, machine-readable
 `frames.json`, an asciicast v2 `session.cast`, a final `snapshot.svg`, a
 self-contained animated `recording.svg`, and `recording.gif`. Frames retain

js/examples/tui-e2e.mjs

@@ -0,0 +1,42 @@
+import assert from 'node:assert/strict';
+
+import { captureTerminal } from 'command-stream';
+
+const menu = String.raw`
+process.stdin.setRawMode(true);
+let selected = 0;
+const choices = ['local', 'web_search'];
+const draw = () => {
+  process.stdout.write(
+    '\x1b[2J\x1b[HChoose a tool\n' +
+    choices.map((choice, index) => (index === selected ? '> ' : '  ') + choice).join('\n')
+  );
+};
+process.stdin.on('data', (data) => {
+  if (data.equals(Buffer.from('\x1b[B'))) {
+    selected = Math.min(selected + 1, choices.length - 1);
+    draw();
+  } else if (data.equals(Buffer.from('\r'))) {
+    process.stdout.write('\x1b[2J\x1b[Htool:' + choices[selected] + '\nresult:done');
+    process.exit(0);
+  }
+});
+draw();
+`;
+
+const capture = await captureTerminal({
+  file: process.execPath,
+  args: ['-e', menu],
+  cols: 80,
+  rows: 24,
+  env: { ...process.env, TERM: 'xterm-256color' },
+  interactions: [
+    { after: /Choose a tool/, idleMilliseconds: 50, key: '\x1b[B' },
+    { after: /> web_search/, idleMilliseconds: 50, key: 'ENTER' },
+  ],
+  artifactDirectory: 'artifacts/tui-e2e',
+});
+
+assert.match(capture.output, /tool:web_search/);
+assert.match(capture.output, /result:done/);
+console.log(capture.transcript);

js/src/terminal-capture.mjs

@@ -251,8 +251,16 @@ const createCaptureRecorder = (asciicast, onTrace) => {
   };
 };
 
-const interactionAfter = (interaction, output) =>
-  interaction.after === undefined || output.includes(interaction.after);
+const interactionAfter = (interaction, output) => {
+  if (interaction.after === undefined) {
+    return true;
+  }
+  if (interaction.after instanceof RegExp) {
+    interaction.after.lastIndex = 0;
+    return interaction.after.test(output);
+  }
+  return output.includes(interaction.after);
+};
 
 const applyInteraction = ({ interaction, process, terminal, record }) => {
   if (interaction.text !== undefined) {
@@ -273,6 +281,70 @@ const applyInteraction = ({ interaction, process, terminal, record }) => {
   }
 };
 
+const createInteractionCoordinator = ({
+  interactions,
+  output,
+  outputWriter,
+  appendFrame,
+  trace,
+  process,
+  terminal,
+  record,
+}) => {
+  let index = 0;
+  let timer;
+  let scheduled = false;
+  const applyNext = () => {
+    scheduled = true;
+    trace('interaction-scheduled', { interactionIndex: index });
+    outputWriter.after(() => {
+      appendFrame();
+      trace('interaction-applied', { interactionIndex: index });
+      applyInteraction({
+        interaction: interactions[index],
+        process,
+        terminal,
+        record,
+      });
+      index += 1;
+      scheduled = false;
+      advance();
+    });
+  };
+  const advance = () => {
+    if (
+      scheduled ||
+      index >= interactions.length ||
+      !interactionAfter(interactions[index], output())
+    ) {
+      return;
+    }
+    const idleMilliseconds = interactions[index].idleMilliseconds ?? 0;
+    if (idleMilliseconds > 0) {
+      scheduled = true;
+      timer = setTimeout(() => {
+        timer = undefined;
+        scheduled = false;
+        applyNext();
+      }, idleMilliseconds);
+      return;
+    }
+    applyNext();
+  };
+  return {
+    advance,
+    close: () => clearTimeout(timer),
+    count: () => index,
+    outputArrived: () => {
+      if (timer !== undefined) {
+        clearTimeout(timer);
+        timer = undefined;
+        scheduled = false;
+      }
+    },
+  };
+};
+
 const startTerminal = async ({ file, args, cwd, env, cols, rows }) => {
   const { Terminal } = await runtimeDependencies();
   const environment = captureEnvironment({
@@ -384,10 +456,8 @@ export const captureTerminal = async ({
   } = createCaptureRecorder(asciicast, onTrace);
   const frames = [];
   let output = '';
-  let interactionIndex = 0;
   let settleTimer, stopTimer;
   let stopMarkerSeen = false;
-  let interactionScheduled = false;
   let captureError;
   const appendFrame = () => {
     const frame = terminalFrame(terminal, elapsed);
@@ -400,31 +470,16 @@ export const captureTerminal = async ({
     settleTimer = setTimeout(appendFrame, settleMilliseconds);
   };
   const outputWriter = createTerminalOutputWriter(terminal, appendFrame);
-  const advanceInteractions = () => {
-    if (
-      interactionScheduled ||
-      interactionIndex >= interactions.length ||
-      !interactionAfter(interactions[interactionIndex], output)
-    ) {
-      return;
-    }
-
-    interactionScheduled = true;
-    traceCapture('interaction-scheduled', { interactionIndex });
-    outputWriter.after(() => {
-      appendFrame();
-      traceCapture('interaction-applied', { interactionIndex });
-      applyInteraction({
-        interaction: interactions[interactionIndex],
-        process: child,
-        terminal,
-        record,
-      });
-      interactionIndex += 1;
-      interactionScheduled = false;
-      advanceInteractions();
-    });
-  };
+  const interactionCoordinator = createInteractionCoordinator({
+    interactions,
+    output: () => output,
+    outputWriter,
+    appendFrame,
+    trace: traceCapture,
+    process: child,
+    terminal,
+    record,
+  });
 
   const completion = new Promise((resolve) => {
     const timeout = setTimeout(() => {
@@ -436,12 +491,13 @@ export const captureTerminal = async ({
     }, timeoutMilliseconds);
 
     child.onData((data) => {
+      interactionCoordinator.outputArrived();
       traceCapture('output', { data });
       output += data;
       record('o', data);
       outputWriter.write(data);
       outputWriter.after(settle);
-      advanceInteractions();
+      interactionCoordinator.advance();
 
       if (stopMarker && output.includes(stopMarker) && !stopMarkerSeen) {
         stopMarkerSeen = true;
@@ -456,6 +512,7 @@ export const captureTerminal = async ({
       traceCapture('exit', { exitCode, signal });
       clearTimeout(timeout);
       clearTimeout(stopTimer);
+      interactionCoordinator.close();
       captureError ??= error;
       resolve({ exitCode, signal });
     });
@@ -486,7 +543,7 @@ export const captureTerminal = async ({
     output,
     transcript,
     frames,
-    interactionCount: interactionIndex,
+    interactionCount: interactionCoordinator.count(),
     asciicast,
   });
   if (captureError) {

js/tests/terminal-capture.test.mjs

@@ -188,6 +188,44 @@ describe('PTY terminal capture', () => {
     }
   });
 
+  test('waits for regex readiness and output quiescence before sending a raw key sequence', async () => {
+    const trace = [];
+    const capture = await captureTerminal({
+      file: process.execPath,
+      args: [
+        '-e',
+        [
+          "process.stdin.setRawMode(true); process.stdout.write('boot:1');",
+          "setTimeout(() => process.stdout.write('\\rboot:ready-42'), 30);",
+          "process.stdin.once('data', data => {",
+          "  process.stdout.write(`\\rkey:${data.toString('hex')}`);",
+          '  process.exit(0);',
+          '});',
+        ].join(''),
+      ],
+      interactions: [
+        {
+          after: /boot:ready-\d+/,
+          idleMilliseconds: 50,
+          key: '\x1b[B',
+        },
+      ],
+      timeoutMilliseconds: 2_000,
+      onTrace: (event) => trace.push(event),
+    });
+
+    expect(capture.output).toContain('key:1b5b42');
+    expect(capture.interactionCount).toBe(1);
+
+    const finalOutput = trace.findLast(
+      ({ type, data }) => type === 'output' && data.includes('boot:ready-42')
+    );
+    const interaction = trace.findLast(
+      ({ type }) => type === 'interaction-applied'
+    );
+    expect(interaction.time - finalOutput.time).toBeGreaterThanOrEqual(0.04);
+  });
+
   test('retains a state when a later repaint is split across output chunks', async () => {
     const capture = await captureTerminal({
       file: process.execPath,

rust/README.md

@@ -102,6 +102,8 @@ let capture = capture_terminal(TerminalCaptureOptions {
     file: "codex".into(),
     args: vec!["--no-alt-screen".into()],
     interactions: vec![TerminalInteraction {
+        after_regex: Some("Ready: .+".into()),
+        idle_duration: std::time::Duration::from_millis(50),
         text: Some("Inspect the failing test".into()),
         key: Some(TerminalKey::Enter),
         ..TerminalInteraction::default()
@@ -121,6 +123,11 @@ artifact directory receives `transcript.txt`, `frames.json`, `session.cast`,
 partial capture and those diagnostic files. Use `capture_terminal_async` from
 an async application.
 
+Interactions can wait for literal output with `after`, regex output with
+`after_regex`, and output quiescence with `idle_duration`. Named
+`TerminalKey` variants cover arrows, Enter, Tab, Escape, Backspace, Ctrl-C, and
+Ctrl-D; use `TerminalKey::Raw` for any other escape sequence.
+
 ## Features
 
 - Shell parser for pipelines, command lists, logical operators, and redirection.

rust/changelog.d/20260725_174_tui_synchronization.md

@@ -0,0 +1,8 @@
+---
+bump: minor
+---
+
+### Added
+
+- Add regex readiness markers and per-interaction output-idle waits to PTY
+  terminal capture.

rust/src/terminal/capture.rs

@@ -4,6 +4,7 @@ use super::types::{
     TerminalCaptureOptions, TerminalCursor, TerminalFrame, TerminalInteraction, TerminalResize,
 };
 use portable_pty::{native_pty_system, CommandBuilder, MasterPty, PtySize};
+use regex::Regex;
 use std::collections::HashMap;
 use std::io::{Read, Write};
 use std::sync::mpsc;
@@ -220,6 +221,24 @@ pub fn capture_terminal(
             None,
         ));
     }
+    let interaction_regexes = options
+        .interactions
+        .iter()
+        .map(|interaction| {
+            interaction
+                .after_regex
+                .as_ref()
+                .map(|pattern| {
+                    Regex::new(pattern).map_err(|error| {
+                        TerminalCaptureError::new(
+                            format!("invalid terminal interaction regex: {error}"),
+                            None,
+                        )
+                    })
+                })
+                .transpose()
+        })
+        .collect::<Result<Vec<_>, _>>()?;
     let pty = native_pty_system()
         .openpty(PtySize {
             rows: options.rows,
@@ -316,6 +335,18 @@ pub fn capture_terminal(
             {
                 break;
             }
+            if interaction_regexes[interaction_index]
+                .as_ref()
+                .is_some_and(|pattern| !pattern.is_match(&output))
+            {
+                break;
+            }
+            if interaction.idle_duration > Duration::ZERO
+                && last_output.map_or_else(|| started.elapsed(), |instant| instant.elapsed())
+                    < interaction.idle_duration
+            {
+                break;
+            }
             append_frame(&mut frames, &parser, started);
             apply_interaction(
                 interaction,