feat(copilot): attach browser and terminal tab context

Author: Sg312

apps/desktop/src/main/terminal/session.ts

@@ -445,7 +445,11 @@ export class TerminalSession {
       // data the agent reads — it must stay true the instant a command starts.
       title: directory || 'Terminal',
       cwd: this.cwd,
-      running: this.foregroundCommand,
+      // Never an empty string. A shell can start a command without announcing
+      // its text, and "" would read as "nothing is running" to everything
+      // downstream while the terminal is in fact busy — the agent would treat
+      // it as free and the tab would render a blank label.
+      running: this.foregroundCommand === null ? null : this.foregroundCommand.trim() || 'command',
       interactive: this.altScreen,
       active,
     }

apps/sim/app/workspace/[workspaceId]/home/components/chat-context-kind-registry/chat-context-kind-registry.tsx

@@ -1,11 +1,13 @@
 import type { ReactNode } from 'react'
 import {
   Calendar,
+  Cursor,
   Database,
   Folder as FolderIcon,
   Library,
   Table as TableIcon,
   Task,
+  TerminalWindow,
   Workflow,
 } from '@sim/emcn/icons'
 import { AgentSkillsIcon, McpIcon } from '@/components/icons'
@@ -53,6 +55,14 @@ function renderIntegrationTile({ context, className }: RenderIconArgs): ReactNod
  * without an icon.
  */
 export const CHAT_CONTEXT_KIND_REGISTRY: Record<ChatContextKind, ChatContextKindConfig> = {
+  browser_tab: {
+    label: 'Browser tab',
+    renderIcon: ({ className }) => <Cursor className={className} />,
+  },
+  terminal_tab: {
+    label: 'Terminal',
+    renderIcon: ({ className }) => <TerminalWindow className={className} />,
+  },
   workflow: { label: 'Workflow', renderIcon: renderWorkflowIcon },
   current_workflow: { label: 'Current workflow', renderIcon: renderWorkflowIcon },
   workflow_block: { label: 'Block', renderIcon: renderWorkflowIcon },

apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-tab-strip.tsx

@@ -1,9 +1,16 @@
 'use client'
 
-import { type MouseEvent as ReactMouseEvent, useCallback, useMemo, useState } from 'react'
+import {
+  type DragEvent as ReactDragEvent,
+  type MouseEvent as ReactMouseEvent,
+  useCallback,
+  useMemo,
+  useState,
+} from 'react'
 import { type BrowserTabState, MAX_BROWSER_TABS } from '@sim/browser-protocol'
 import { TabStrip, type TabStripItem } from '@sim/emcn'
 import { Link, Loader } from '@sim/emcn/icons'
+import { SIM_RESOURCE_DRAG_TYPE } from '@/lib/copilot/resource-types'
 import { faviconUrl } from '@/lib/core/utils/favicon'
 import { ContextMenu } from '@/app/workspace/[workspaceId]/w/components/sidebar/components/workflow-list/components/context-menu/context-menu'
 import { useContextMenu } from '@/app/workspace/[workspaceId]/w/components/sidebar/hooks'
@@ -98,6 +105,22 @@ export function BrowserTabStrip({
     [tabs, activeTabId]
   )
 
+  // Dragging a tab into the chat attaches it as context. `copyMove` because
+  // the strip already set `move` for its own reordering, and a drop target
+  // asking for `copy` is refused outright unless copying is allowed too.
+  const startTabDrag = useCallback(
+    (event: ReactDragEvent<HTMLDivElement>, tabId: string) => {
+      const tab = tabs.find((entry) => entry.tabId === tabId)
+      if (!tab) return
+      event.dataTransfer.effectAllowed = 'copyMove'
+      event.dataTransfer.setData(
+        SIM_RESOURCE_DRAG_TYPE,
+        JSON.stringify({ type: 'browser', id: tab.tabId, title: tabTitle(tab) })
+      )
+    },
+    [tabs]
+  )
+
   const openTabContextMenu = useCallback(
     (event: ReactMouseEvent<HTMLDivElement>, tabId: string) => {
       window.getSelection()?.removeAllRanges()
@@ -114,6 +137,7 @@ export function BrowserTabStrip({
       onClose={onCloseTab}
       onNew={onNewTab}
       onTabContextMenu={openTabContextMenu}
+      onTabDragStart={startTabDrag}
       maxTabs={MAX_BROWSER_TABS}
       {...(reorderingSupported ? { onReorder: onReorderTab } : {})}
     >

apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/terminal-session/terminal-session.tsx

@@ -1,6 +1,7 @@
 'use client'
 
 import {
+  type DragEvent as ReactDragEvent,
   type MouseEvent as ReactMouseEvent,
   useCallback,
   useEffect,
@@ -19,6 +20,7 @@ import { Terminal } from '@xterm/xterm'
 import { useTheme } from 'next-themes'
 import '@xterm/xterm/css/xterm.css'
 import { MAX_TERMINALS, type TerminalTabState } from '@sim/terminal-protocol'
+import { SIM_RESOURCE_DRAG_TYPE } from '@/lib/copilot/resource-types'
 import { TERMINAL_SESSION_RESOURCE_ID } from '@/lib/copilot/resources/types'
 import {
   closeTerminal,
@@ -50,10 +52,27 @@ const logger = createLogger('TerminalSession')
  */
 const COMMAND_SETTLE_MS = 1_000
 
+/** Full working directory, plus whatever the shell is running in it. */
+function terminalTooltip(tab: TerminalTabState): string {
+  const where = tab.cwd ?? 'Terminal'
+  return tab.running ? `${where} — ${tab.running}` : where
+}
+
 function sameIds(a: ReadonlySet<string>, b: ReadonlySet<string>): boolean {
   return a.size === b.size && [...a].every((id) => b.has(id))
 }
 
+/**
+ * Whether a tab should be named after what it is running rather than where it
+ * is. A full-screen program is named the moment it appears: the delay exists
+ * to stop `ls` flickering the label, and an editor or coding agent is not a
+ * transient command — it holds the terminal until it is quit, so there is
+ * nothing to wait out.
+ */
+function namesItsCommand(tab: TerminalTabState, settled: ReadonlySet<string>): boolean {
+  return Boolean(tab.running) && (tab.interactive || settled.has(tab.terminalId))
+}
+
 /**
  * The terminals whose command has been running long enough to show. Returns a
  * stable set, so a tab strip that would render identically does not re-render.
@@ -513,12 +532,14 @@ export function TerminalSession() {
   const items = useMemo<TabStripItem[]>(
     () =>
       tabs.map((tab) => {
-        // A command only reaches the tab once it has run long enough to be
-        // worth naming; until then the tab stays as its directory.
-        const naming = settledCommands.has(tab.terminalId) ? tab.running : null
+        const naming = namesItsCommand(tab, settledCommands) ? tab.running : null
         return {
           id: tab.terminalId,
           title: naming ?? tab.title,
+          // The label is a basename, and the tab may be running something it
+          // is not naming yet, so hovering gives the whole picture: where the
+          // shell is, and what it is doing there.
+          tooltip: terminalTooltip(tab),
           icon:
             naming && !tab.interactive ? (
               <Loader className='size-[12px] shrink-0 animate-spin text-[var(--text-icon)]' />
@@ -559,6 +580,22 @@ export function TerminalSession() {
     void openTerminal(cwd ?? undefined)
   }, [])
 
+  // Dragging a terminal tab into the chat attaches it as context. This strip
+  // has no reordering, so supplying this is also what makes its tabs
+  // draggable at all.
+  const startTabDrag = useCallback(
+    (event: ReactDragEvent<HTMLDivElement>, terminalId: string) => {
+      const tab = tabs.find((entry) => entry.terminalId === terminalId)
+      if (!tab) return
+      event.dataTransfer.effectAllowed = 'copy'
+      event.dataTransfer.setData(
+        SIM_RESOURCE_DRAG_TYPE,
+        JSON.stringify({ type: 'terminal', id: tab.terminalId, title: tab.title })
+      )
+    },
+    [tabs]
+  )
+
   const openTabContextMenu = useCallback(
     (event: ReactMouseEvent<HTMLDivElement>, terminalId: string) => {
       window.getSelection()?.removeAllRanges()
@@ -576,6 +613,7 @@ export function TerminalSession() {
           onSelect={handleSwitch}
           onNew={handleNew}
           onTabContextMenu={openTabContextMenu}
+          onTabDragStart={startTabDrag}
           maxTabs={MAX_TERMINALS}
           newTabLabel='New terminal'
           onClose={handleClose}

apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/constants.ts

@@ -99,12 +99,16 @@ export const SPEECH_RECOGNITION_LANG = 'en-US'
  * so adding a new resource type fails compilation here until a conversion is
  * supplied — preventing silent drift between the two taxonomies.
  */
-// `browser` and `terminal` are live desktop panels, not addressable workspace
-// entities, so neither can become chat context.
+// A dragged `browser`/`terminal` resource is one TAB, and its `id` is that
+// tab's id — the panel itself is a singleton with nothing to point at. Both
+// become pointers the agent resolves with its own tools rather than content
+// captured here, so what it reads is the tab as it stands when it looks.
 const RESOURCE_TO_CONTEXT: Record<
-  Exclude<MothershipResourceType, 'browser' | 'terminal'>,
+  MothershipResourceType,
   (resource: MothershipResource) => ChatContext
 > = {
+  browser: (r) => ({ kind: 'browser_tab', tabId: r.id, label: r.title }),
+  terminal: (r) => ({ kind: 'terminal_tab', terminalId: r.id, label: r.title }),
   workflow: (r) => ({ kind: 'workflow', workflowId: r.id, label: r.title }),
   knowledgebase: (r) => ({ kind: 'knowledge', knowledgeId: r.id, label: r.title }),
   table: (r) => ({ kind: 'table', tableId: r.id, label: r.title }),
@@ -119,8 +123,5 @@ const RESOURCE_TO_CONTEXT: Record<
 }
 
 export function mapResourceToContext(resource: MothershipResource): ChatContext {
-  if (resource.type === 'browser' || resource.type === 'terminal') {
-    throw new Error(`Live ${resource.type} sessions cannot be attached as chat context`)
-  }
   return RESOURCE_TO_CONTEXT[resource.type](resource)
 }

apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/resource-context.test.ts

@@ -0,0 +1,39 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { mapResourceToContext } from '@/app/workspace/[workspaceId]/home/components/user-input/components/constants'
+import type { MothershipResource } from '@/app/workspace/[workspaceId]/home/types'
+
+function resource(partial: Partial<MothershipResource> & Pick<MothershipResource, 'type'>) {
+  return { id: 'id-1', title: 'Something', ...partial } as MothershipResource
+}
+
+describe('mapResourceToContext', () => {
+  it('turns a dragged browser tab into a pointer at that tab', () => {
+    // The id is the TAB's, not the panel's: the panel is a singleton and
+    // pointing at it would not say which page the user meant.
+    const context = mapResourceToContext(
+      resource({ type: 'browser', id: 'tab-7', title: 'Pull requests' })
+    )
+
+    expect(context).toEqual({ kind: 'browser_tab', tabId: 'tab-7', label: 'Pull requests' })
+  })
+
+  it('turns a dragged terminal tab into a pointer at that shell', () => {
+    const context = mapResourceToContext(resource({ type: 'terminal', id: '3', title: 'sim' }))
+
+    expect(context).toEqual({ kind: 'terminal_tab', terminalId: '3', label: 'sim' })
+  })
+
+  it('still maps the ordinary workspace resources', () => {
+    expect(
+      mapResourceToContext(resource({ type: 'workflow', id: 'wf-1', title: 'Deploy' }))
+    ).toEqual({ kind: 'workflow', workflowId: 'wf-1', label: 'Deploy' })
+    expect(mapResourceToContext(resource({ type: 'table', id: 't-1', title: 'Leads' }))).toEqual({
+      kind: 'table',
+      tableId: 't-1',
+      label: 'Leads',
+    })
+  })
+})

apps/sim/lib/copilot/chat/post.ts

@@ -152,6 +152,8 @@ const ChatContextSchema = z.object({
     'integration',
     'skill',
     'mcp',
+    'browser_tab',
+    'terminal_tab',
   ]),
   label: z.string(),
   chatId: z.string().optional(),
@@ -167,6 +169,8 @@ const ChatContextSchema = z.object({
   skillId: z.string().optional(),
   serverId: z.string().optional(),
   scheduleId: z.string().optional(),
+  tabId: z.string().optional(),
+  terminalId: z.string().optional(),
 })
 
 const ChatMessageSchema = z.object({

apps/sim/lib/copilot/chat/process-contents.ts

@@ -143,6 +143,24 @@ export async function processContextsServer(
           currentWorkspaceId
         )
       }
+      // Tabs resolve to a pointer, not their contents. The agent has tools
+      // that read a live tab, and by the time it acts the page may have
+      // navigated or the shell scrolled on — so naming the tab it should look
+      // at beats pasting a snapshot that was true when the message was sent.
+      if (ctx.kind === 'browser_tab' && ctx.tabId) {
+        return {
+          type: 'browser_tab',
+          tag: ctx.label ? `@${ctx.label}` : '@',
+          content: `The user pointed at an open browser tab: "${ctx.label}" (tabId ${ctx.tabId}). Act on THIS tab — switch to it with browser_switch_tab and read it with browser_snapshot rather than assuming which tab they meant.`,
+        }
+      }
+      if (ctx.kind === 'terminal_tab' && ctx.terminalId) {
+        return {
+          type: 'terminal_tab',
+          tag: ctx.label ? `@${ctx.label}` : '@',
+          content: `The user pointed at an open terminal: "${ctx.label}" (terminalId ${ctx.terminalId}). Act on THIS terminal — pass that terminalId to the terminal tool, and read its screen before assuming what is in it.`,
+        }
+      }
       if (ctx.kind === 'workflow_block' && ctx.workflowId && ctx.blockId) {
         return await processWorkflowBlockFromDb(
           ctx.workflowId,

apps/sim/stores/panel/types.ts

@@ -29,6 +29,14 @@ export type ChatContext =
   | { kind: 'filefolder'; fileFolderId: string; label: string }
   | { kind: 'scheduledtask'; scheduleId: string; label: string }
   | { kind: 'docs'; label: string }
+  /**
+   * A tab in the desktop browser or terminal panel, dragged into the input to
+   * say "this one". A pointer rather than a snapshot: the agent reads the tab
+   * with its own tools, so what it sees is the live state at the moment it
+   * looks rather than whatever was on screen when the message was sent.
+   */
+  | { kind: 'browser_tab'; tabId: string; label: string }
+  | { kind: 'terminal_tab'; terminalId: string; label: string }
   | { kind: 'slash_command'; command: string; label: string }
   | { kind: 'integration'; blockType: string; label: string }
   | { kind: 'skill'; skillId: string; label: string }

packages/emcn/src/components/tab-strip/tab-strip.test.ts

@@ -43,3 +43,38 @@ describe('tabDropIndex', () => {
     expect(tabDropIndex(plain, 'b', 1)).toBeNull()
   })
 })
+
+describe('tab tooltips', () => {
+  /** Mirrors the render condition: tooltip text, and whether it is shown. */
+  function tooltipFor(tab: TabStripItem, titleTruncated: boolean) {
+    const shown = Boolean(tab.tooltip || tab.pinned || titleTruncated)
+    return shown ? (tab.tooltip ?? tab.title) : null
+  }
+
+  it('prefers the fuller detail a tab supplies over its label', () => {
+    // A terminal labelled with a basename can say where it actually is.
+    const tab: TabStripItem = { id: '1', title: 'sim', tooltip: '/Users/me/code/sim — bun test' }
+
+    expect(tooltipFor(tab, false)).toBe('/Users/me/code/sim — bun test')
+  })
+
+  it('shows that detail even when the label fits', () => {
+    // It says something the tab cannot, so there is always a reason to hover.
+    const tab: TabStripItem = { id: '1', title: 'sim', tooltip: '/Users/me/code/sim' }
+
+    expect(tooltipFor(tab, false)).not.toBeNull()
+  })
+
+  it('falls back to the title, and only when the title is clipped', () => {
+    const tab: TabStripItem = { id: '1', title: 'a very long tab title' }
+
+    expect(tooltipFor(tab, true)).toBe('a very long tab title')
+    expect(tooltipFor(tab, false)).toBeNull()
+  })
+
+  it('still explains a pinned tab, which renders with no label at all', () => {
+    const tab: TabStripItem = { id: '1', title: 'GitHub', pinned: true }
+
+    expect(tooltipFor(tab, false)).toBe('GitHub')
+  })
+})