Merge branch 'staging-v4' of github.com:simstudioai/sim into staging-v4

Author: icecrasher321

apps/desktop/src/main/browser-agent/driver.ts

@@ -119,8 +119,22 @@ function pushPageState(contents: WebContents): void {
   driverCallbacks?.onPageState(pageStateFor(contents, active.id))
 }
 
+let lastTabsStateFingerprint: string | null = null
+
+/**
+ * Pushes the tab list to the renderer, skipping a push identical to the last.
+ *
+ * Every tab's load and title events call this, background tabs included, and a
+ * site that rewrites its title on a timer fires them continuously — each an
+ * unconditional broadcast that rebuilt the whole strip. The fingerprint drops
+ * the repeats, matching what the terminal side already does with emitTabs.
+ */
 function pushTabsState(): void {
-  driverCallbacks?.onTabsState(session.getTabsState())
+  const state = session.getTabsState()
+  const fingerprint = JSON.stringify(state)
+  if (fingerprint === lastTabsStateFingerprint) return
+  lastTabsStateFingerprint = fingerprint
+  driverCallbacks?.onTabsState(state)
 }
 
 /** Instruments a fresh tab: CDP dialog/chooser handling + page-state pushes. */

apps/desktop/src/main/browser-agent/panel.ts

@@ -283,6 +283,23 @@ function resetOcclusion(): void {
  * the one caller that captures an already-hidden page (a tab switched while the
  * panel is occluded), where it stops Chromium promoting it back for the shot.
  */
+/** Widest the placeholder snapshot needs to be; it sits behind a transient overlay. */
+const SNAPSHOT_MAX_WIDTH = 1024
+const SNAPSHOT_JPEG_QUALITY = 70
+
+/**
+ * Turns a captured frame into a compact JPEG data URL. Resize is a native
+ * operation and JPEG encode runs in native code too, so both are far cheaper
+ * than a full-resolution PNG `toDataURL`, which encodes synchronously on the
+ * event loop.
+ */
+function encodeSnapshot(image: Electron.NativeImage): string {
+  const { width } = image.getSize()
+  const scaled = width > SNAPSHOT_MAX_WIDTH ? image.resize({ width: SNAPSHOT_MAX_WIDTH }) : image
+  const jpeg = scaled.toJPEG(SNAPSHOT_JPEG_QUALITY)
+  return `data:image/jpeg;base64,${jpeg.toString('base64')}`
+}
+
 function capturePanelSnapshot(onSettled?: () => void): void {
   const active = host.activeTab()
   const win = panelWindow()
@@ -301,7 +318,13 @@ function capturePanelSnapshot(onSettled?: () => void): void {
       // picture of the page, so it goes to the window still showing the
       // browser or nowhere at all.
       if (panelWindow() !== win || win.isDestroyed()) return
-      const snapshot: BrowserPanelSnapshot = { dataUrl: image.toDataURL(), tabId }
+      // Downscale and JPEG-encode before crossing IPC. capturePage returns a
+      // device-pixel PNG — on a retina half-window that is millions of pixels,
+      // and toDataURL's PNG encode is synchronous on the main process, so a
+      // full-size encode stalls every window's input for the frame. This is a
+      // placeholder shown under a transient overlay, so a downscaled JPEG is
+      // indistinguishable and an order of magnitude cheaper to encode and send.
+      const snapshot: BrowserPanelSnapshot = { dataUrl: encodeSnapshot(image), tabId }
       win.webContents.send('browser-agent:panel-snapshot', snapshot)
     })
     .catch((error) => {

apps/desktop/src/main/browser-agent/session.test.ts

@@ -235,15 +235,42 @@ describe('browser-agent session', () => {
     expect(session.closeFocusedTab()).toBe(true)
   })
 
-  it('only disables hidden-page throttling while browser automation is active', () => {
-    const tab = session.ensureTab()
-    const contents = (tab.view as unknown as MockView).webContents
+  it('unthrottles only the active tab while automation is active', () => {
+    const active = session.ensureTab()
+    const activeContents = (active.view as unknown as MockView).webContents
+    const background = session.addTab()
+    const backgroundContents = (background.view as unknown as MockView).webContents
+    // addTab activated the second tab; put focus back on the first.
+    session.switchTab(active.id)
+    activeContents.setBackgroundThrottling.mockClear()
+    backgroundContents.setBackgroundThrottling.mockClear()
 
     session.setAutomationActive(true)
-    expect(contents.setBackgroundThrottling).toHaveBeenLastCalledWith(false)
+    // The waking is scoped to the active tab; the background tab stays throttled.
+    expect(activeContents.setBackgroundThrottling).toHaveBeenLastCalledWith(false)
+    expect(backgroundContents.setBackgroundThrottling).toHaveBeenLastCalledWith(true)
 
     session.setAutomationActive(false)
-    expect(contents.setBackgroundThrottling).toHaveBeenLastCalledWith(true)
+    expect(activeContents.setBackgroundThrottling).toHaveBeenLastCalledWith(true)
+  })
+
+  it('moves the automation exemption to whichever tab becomes active', () => {
+    const first = session.ensureTab()
+    const second = session.addTab()
+    session.switchTab(first.id)
+    const firstContents = (first.view as unknown as MockView).webContents
+    const secondContents = (second.view as unknown as MockView).webContents
+
+    session.setAutomationActive(true)
+    firstContents.setBackgroundThrottling.mockClear()
+    secondContents.setBackgroundThrottling.mockClear()
+
+    session.switchTab(second.id)
+
+    // The old active tab is re-throttled, the new one exempted — otherwise a
+    // mid-tool switch would strand the wake on a tab the agent left behind.
+    expect(firstContents.setBackgroundThrottling).toHaveBeenLastCalledWith(true)
+    expect(secondContents.setBackgroundThrottling).toHaveBeenLastCalledWith(false)
   })
 
   it('updates the native backdrop when Sim changes browser theme', () => {
@@ -725,7 +752,7 @@ describe('browser-agent session', () => {
     expect(view.setVisible).not.toHaveBeenCalledWith(false)
     await vi.waitFor(() => {
       expect(win.webContents.send).toHaveBeenCalledWith('browser-agent:panel-snapshot', {
-        dataUrl: 'data:image/png;base64,c2lt',
+        dataUrl: 'data:image/jpeg;base64,c2lt',
         tabId: tab.id,
       })
     })

apps/desktop/src/main/browser-agent/session.ts

@@ -291,9 +291,10 @@ function createTabView(): WebContentsView {
       sandbox: true,
       webSecurity: true,
       webviewTag: false,
-      // Visible pages remain full speed. Hidden pages may be throttled unless
-      // a browser tool is actively waiting on them.
-      backgroundThrottling: !automationActive,
+      // Throttled by default: a hidden tab should idle. The one exception is
+      // the active tab while a tool waits on it, applied explicitly by
+      // applyActiveTabThrottling — never blanket across every tab.
+      backgroundThrottling: true,
       spellcheck: false,
     },
   })
@@ -383,15 +384,33 @@ export function hasSession(): boolean {
 }
 
 /**
- * Keeps hidden pages responsive during an agent action, then returns them to
- * Chromium's normal background throttling so they cannot contend with Sim.
+ * Keeps the ACTIVE tab responsive during an agent action, then returns it to
+ * normal background throttling.
+ *
+ * Only the active tab, deliberately. The agent drives one tab at a time — the
+ * active one — possibly while the panel is hidden and even that view is
+ * detached, so it is the only tab that must not be throttled mid-tool. Waking
+ * every tab, as this once did, meant an agent run kept all N-1 background
+ * renderers at full speed for the length of the run, which is the browser
+ * side of the multi-tab lag. Nothing depends on a background tab staying
+ * awake: switching to one activates it (and re-applies this) before any tool
+ * touches it, and network loading is not throttled anyway.
  */
 export function setAutomationActive(active: boolean): void {
   automationActive = active
+  applyActiveTabThrottling()
+}
+
+/**
+ * Unthrottles the active tab while automation is active, and throttles every
+ * other tab. Call after anything that changes which tab is active, so the
+ * exemption follows the active tab rather than being stranded on the old one.
+ */
+function applyActiveTabThrottling(): void {
   for (const tab of tabs) {
-    if (!tab.view.webContents.isDestroyed()) {
-      tab.view.webContents.setBackgroundThrottling(!active)
-    }
+    if (tab.view.webContents.isDestroyed()) continue
+    const exempt = automationActive && tab.id === activeTabId
+    tab.view.webContents.setBackgroundThrottling(!exempt)
   }
 }
 
@@ -482,6 +501,7 @@ function addTabInternal({
   }
   if (activate || activeTabId === null) {
     activeTabId = tab.id
+    applyActiveTabThrottling()
     layout()
     if (transferBrowserFocus) focusedBrowserTabId = tab.id
     if (notify) events?.onActiveTabChanged(tab.view.webContents)
@@ -563,6 +583,9 @@ export function switchTab(tabId: string): AgentTab {
   const transferBrowserFocus =
     focusedBrowserTabId !== null || tabs.some((entry) => entry.view.webContents.isFocused())
   activeTabId = tab.id
+  // The automation exemption follows the active tab, so a mid-tool switch
+  // unthrottles the new one and re-throttles the old.
+  applyActiveTabThrottling()
   layout()
   if (transferBrowserFocus) focusedBrowserTabId = tab.id
   events?.onActiveTabChanged(tab.view.webContents)

apps/desktop/src/main/terminal/index.ts

@@ -38,13 +38,15 @@ import {
   awaitRun,
   capturePane,
   closeRunWindow,
+  isTmuxUnavailable,
   killPane,
   listPanes,
   resolveAttachment,
   sendKey,
   sendText,
   startRun,
   TMUX_KEY_NAMES,
+  type TmuxAttachment,
 } from '@/main/terminal/tmux'
 
 const logger = createLogger('DesktopTerminal')
@@ -72,6 +74,9 @@ const CWD_POLL_MS = 1_000
 /** Pause between keys sent to a tmux pane, matching the pty keystroke gap. */
 const TMUX_KEY_GAP_MS = 150
 
+/** How long a resolved tmux attachment is reused before re-resolving. */
+const TMUX_ATTACHMENT_TTL_MS = 3_000
+
 /** How often a handoff checks whether the user or the command has finished. */
 const HANDOFF_POLL_MS = 500
 
@@ -160,6 +165,8 @@ export class TerminalService {
   private readonly recentlyClosedCwds: string[] = []
   /** Terminals handed to the user; the value is whether they have handed back. */
   private readonly handoffs = new Map<string, boolean>()
+  /** Recently resolved tmux attachments, by terminal id, to avoid re-spawning. */
+  private readonly tmuxCache = new Map<string, { at: number; attachment: TmuxAttachment | null }>()
 
   constructor(private readonly options: TerminalServiceOptions = {}) {}
 
@@ -281,6 +288,7 @@ export class TerminalService {
     const index = order.indexOf(terminalId)
     session.dispose()
     this.sessions.delete(terminalId)
+    this.tmuxCache.delete(terminalId)
 
     if (this.sessions.size === 0) {
       this.spawn(this.resolveCwd(closedCwd), cols, rows)
@@ -357,6 +365,7 @@ export class TerminalService {
     this.stopCwdWatch()
     for (const session of this.sessions.values()) session.dispose()
     this.sessions.clear()
+    this.tmuxCache.clear()
     this.activeId = null
   }
 
@@ -403,9 +412,9 @@ export class TerminalService {
     }
 
     const session = this.requireSession(args)
-    // Resolved once per call: a tab either has tmux attached or it does not,
-    // and every operation below behaves differently depending on which.
-    const tmux = await resolveAttachment(session.pid, session.env)
+    // A tab either has tmux attached or it does not, and every operation below
+    // behaves differently depending on which.
+    const tmux = await this.resolveTmux(session)
 
     switch (operation) {
       case 'cwd':
@@ -458,7 +467,7 @@ export class TerminalService {
       case 'read': {
         const requested = Number(args.lines)
         const lines = Number.isFinite(requested) && requested > 0 ? requested : 200
-        if (!tmux) return session.readScrollback(lines)
+        if (!tmux) return await session.readScrollback(lines)
         const target = await this.resolvePane(tmux.session, args, session)
         const captured = await capturePane(target, lines, session.env)
         if (!captured.ok) {
@@ -516,8 +525,8 @@ export class TerminalService {
     const terminalId = session.terminalId
     this.handoffs.set(terminalId, false)
 
-    const settled = (handedBack: boolean): TerminalHandoffResult => {
-      const view = session.readScrollback(INPUT_SCREEN_LINES)
+    const settled = async (handedBack: boolean): Promise<TerminalHandoffResult> => {
+      const view = await session.readScrollback(INPUT_SCREEN_LINES)
       return {
         terminalId,
         reason,
@@ -538,13 +547,13 @@ export class TerminalService {
         }
         // The command finishing is the real end of the handoff, whether or not
         // the user pressed anything: it means the prompt got answered.
-        if (!session.isBusy) return settled(this.handoffs.get(terminalId) === true)
+        if (!session.isBusy) return await settled(this.handoffs.get(terminalId) === true)
         if (this.handoffs.get(terminalId) === true) {
           handedBackAt ??= Date.now()
-          if (Date.now() - handedBackAt >= HANDOFF_SETTLE_MS) return settled(true)
+          if (Date.now() - handedBackAt >= HANDOFF_SETTLE_MS) return await settled(true)
         }
       }
-      return settled(this.handoffs.get(terminalId) === true)
+      return await settled(this.handoffs.get(terminalId) === true)
     } finally {
       this.handoffs.delete(terminalId)
     }
@@ -555,6 +564,26 @@ export class TerminalService {
     if (this.handoffs.has(terminalId)) this.handoffs.set(terminalId, true)
   }
 
+  /**
+   * The tmux session attached in a terminal, cached briefly.
+   *
+   * Resolving it spawns `tmux list-clients` and a whole-machine `ps` — a real
+   * cost to pay on every tool call, when a shell's attachment does not change
+   * between calls a second apart. A short TTL keeps the common burst of
+   * operations (run, then poll with read, then read again) to one resolution,
+   * while staying fresh enough to notice the user starting or leaving tmux.
+   */
+  private async resolveTmux(session: TerminalSession): Promise<TmuxAttachment | null> {
+    if (isTmuxUnavailable()) return null
+    const cached = this.tmuxCache.get(session.terminalId)
+    if (cached && Date.now() - cached.at < TMUX_ATTACHMENT_TTL_MS) {
+      return cached.attachment
+    }
+    const attachment = await resolveAttachment(session.pid, session.env)
+    this.tmuxCache.set(session.terminalId, { at: Date.now(), attachment })
+    return attachment
+  }
+
   /** The pane a call names, or the session's active one. */
   private async resolvePane(
     session: string,
@@ -631,12 +660,12 @@ export class TerminalService {
     if (keys.length > 0) {
       await session.pressKeys(keys)
       await delay(INPUT_ECHO_MS)
-      return { sent: keys.join(', '), ...session.readScrollback(INPUT_SCREEN_LINES) }
+      return { sent: keys.join(', '), ...(await session.readScrollback(INPUT_SCREEN_LINES)) }
     }
     if (typeof args.text === 'string') {
       await session.type(args.text)
       await delay(INPUT_ECHO_MS)
-      return { sent: args.text, ...session.readScrollback(INPUT_SCREEN_LINES) }
+      return { sent: args.text, ...(await session.readScrollback(INPUT_SCREEN_LINES)) }
     }
     throw new TerminalError('INVALID_REQUEST', 'input needs `text`, `key`, or `keys`.')
   }

apps/desktop/src/main/terminal/selection.test.ts

@@ -94,3 +94,42 @@ describe('findSelectedRow', () => {
     expect(findSelectedRow(term.buffer.active)).toBeNull()
   })
 })
+
+describe('replaying retained bytes into a fresh emulator', () => {
+  /**
+   * The screen is now rendered by replaying the retained byte stream into an
+   * emulator built for the read, rather than keeping one fed forever. Two
+   * things have to hold for that to be equivalent.
+   */
+  it('has the whole stream parsed by the time the write callback fires', async () => {
+    // xterm parses asynchronously in 12ms slices, so reading the buffer right
+    // after write() would catch it half-parsed. The callback is the signal.
+    const term = new Terminal({ cols: 40, rows: 8, allowProposedApi: true })
+    const lines = Array.from({ length: 200 }, (_, i) => `line ${i}`)
+
+    await new Promise<void>((resolve) => term.write(`${lines.join('\r\n')}\r\n`, resolve))
+
+    const buffer = term.buffer.active
+    const rendered: string[] = []
+    for (let row = 0; row < buffer.length; row++) {
+      const text = buffer.getLine(row)?.translateToString(true) ?? ''
+      if (text.trim()) rendered.push(text.trim())
+    }
+    expect(rendered.at(-1)).toBe('line 199')
+    expect(rendered).toHaveLength(200)
+  })
+
+  it('reconstructs the final screen of a program that redraws in place', async () => {
+    // A TUI overwrites the same rows, so a replay must end on the last frame
+    // rather than showing every frame stacked up — the reason a raw byte dump
+    // cannot be handed to the model directly.
+    const term = new Terminal({ cols: 40, rows: 8, allowProposedApi: true })
+    const frames = ['Loading   ', 'Loading.  ', 'Loading.. ', 'Done!     ']
+    const stream = frames.map((frame) => `\u001b[H\u001b[2K${frame}`).join('')
+
+    await new Promise<void>((resolve) => term.write(stream, resolve))
+
+    const firstRow = term.buffer.active.getLine(0)?.translateToString(true).trim()
+    expect(firstRow).toBe('Done!')
+  })
+})