Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,17 @@ KEY_ENCRYPTION_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
# one to leave alone until somebody has decided otherwise: the trail is append-only and nothing else
# can remove a row, so this is the only way it ever shrinks.
# AUDIT_RETENTION_DAYS=365
# Two names for one number, and they have to agree.
#
# The server reads PORT (server/src/index.ts). scripts/start.sh reads SERVER_PORT, because it also
# has to know where the app should proxy and which port to report free -- and docs/configuration.md
# documents SERVER_PORT as the setting. Only PORT shipped here, so moving the server by editing this
# line left the script still looking at 3001: it found whatever else was there, accepted the first
# 200 as proof, and failed several stages later parsing that stranger's HTML as JSON.
#
# Change both, or neither.
PORT=3001
SERVER_PORT=3001
TENANT_PACKAGE_DIR=../examples/fintech
# What this deployment calls itself, when more than one shares an Intelligence project. A copy of a
# deployment made for development uses the same project key, and threads are listed per Bot with
Expand Down Expand Up @@ -230,8 +240,13 @@ COMPUTER_TOKEN=
#
# This is attribution, not anonymity, and it is not a boundary by itself: it gives a security team a
# per-Bot address for network rules alongside AGENT_COMPUTER_POLICY.
# EGRESS_PROXY_DEFAULT=http://user:password@proxy.internal:8080
# EGRESS_PROXY_SALES_BOT=http://sales.proxy.internal:8080
#
# These go in `egress.env` beside this file, NOT here. The names are per-Bot, so Compose cannot
# list them the way it lists every variable below, and it hands a container only what it is told to.
# In `.env` they reach no process and the browser goes out directly with nothing saying so.
#
# EGRESS_PROXY_DEFAULT=http://user:password@proxy.internal:8080
# EGRESS_PROXY_SALES_BOT=http://sales.proxy.internal:8080


# The managed coworker AG-UI endpoint. Optional: use an HTTP(S) URL, and set MANAGED_AGENT_TOKEN
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ jobs:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0
- uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
with:
version: v3.19.0
# For the coherence check below, which is a Bun script like everything else here.
Expand Down
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ docs/plans/
.env
.env.*
!.env.example
# Per-Bot egress proxies. Carries credentials in the URL, like .env does.
egress.env
node_modules/
**/dist/
app/src/lib/generated/application-config.ts
Expand Down
251 changes: 251 additions & 0 deletions CHANGELOG.md

Large diffs are not rendered by default.

27 changes: 27 additions & 0 deletions agent-computer/src/authorisation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,30 @@ export function offeredToken(headers: Headers, url: URL): string {
export function isOpenPath(pathname: string): boolean {
return pathname === "/health";
}

/**
* Which paths act on the computer, and so are refused while a person holds the wheel.
*
* One list, asked once per request, rather than a check inside each handler. The shell is the reason:
* `/exec` arrived after the wheel existed and was never given the guard the page paths had, so a Bot
* could keep running commands and writing files underneath somebody who had taken the browser at a
* login wall. A per-handler check is exactly the thing the next endpoint forgets, which is how that
* happened; a list the dispatcher consults is one an endpoint has to be added to.
*
* Reading is not acting. `/files/read` and `/files/list` stay open so a Bot that has been stopped can
* still read its own notes and explain what it was doing, which is the answer the person handing the
* wheel back usually wants.
*/
const ACTING_PATHS = new Set([
"/navigate",
"/click",
"/type",
"/key",
"/scroll",
"/exec",
"/files/write",
]);

export function actsOnTheComputer(pathname: string): boolean {
return ACTING_PATHS.has(pathname);
}
37 changes: 26 additions & 11 deletions agent-computer/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
import { serve } from "bun";
import type { Page } from "playwright";
import { parseAriaSnapshot, type SnapshotElement } from "./aria-snapshot";
import { isOpenPath, matchesToken, offeredToken } from "./authorisation";
import {
actsOnTheComputer,
isOpenPath,
matchesToken,
offeredToken,
} from "./authorisation";
import { isPlainBotId } from "./bot-id";
import {
type Control,
Expand Down Expand Up @@ -485,6 +490,26 @@ serve<StreamData>({
}
const session = sessionFor(botId);

/*
* The wheel, asked once for everything that acts.
*
* Refused here rather than inside each handler because the handler that forgets is the whole
* defect: the shell shipped without this check and ran commands underneath a person who had taken
* the browser at a login wall. `actsOnTheComputer` is the list, and a new acting endpoint is
* refused by being added to it rather than by remembering to repeat this.
*/
if (actsOnTheComputer(url.pathname)) {
try {
session.control.assertBotMayAct();
} catch (error) {
// A person holding the wheel is not a failure of the action; the Bot should wait and say so.
if (error instanceof ControlError) {
return json({ error: error.message, humanHasControl: true }, 409);
}
throw error;
}
}

if (url.pathname === "/stream") {
/*
* The socket carries the Bot in the query because it cannot do it in a header. Every other call here names
Expand Down Expand Up @@ -696,7 +721,6 @@ serve<StreamData>({

const startedAt = Date.now();
try {
session.control.assertBotMayAct();
const target = await currentPage(botId);
await target.goto(body.url, {
waitUntil: "domcontentloaded",
Expand All @@ -715,10 +739,6 @@ serve<StreamData>({
elapsedMs: Date.now() - startedAt,
});
} catch (error) {
// A person holding the wheel is not a failed navigation; the Bot should wait.
if (error instanceof ControlError) {
return json({ error: error.message, humanHasControl: true }, 409);
}
// The page is the Bot's working surface, so a failed navigation is reported rather than
// thrown: the transcript needs to say what happened, and the browser stays usable.
return json(
Expand Down Expand Up @@ -893,7 +913,6 @@ serve<StreamData>({

const startedAt = Date.now();
try {
session.control.assertBotMayAct();
const target = await currentPage(botId);
const detail = await performAction(
session,
Expand Down Expand Up @@ -932,10 +951,6 @@ serve<StreamData>({
if (error instanceof StaleSnapshotError) {
return json({ error: error.message, stale: true }, 409);
}
// 409 as well, and for the same reason: nothing is broken, the caller simply has to wait.
if (error instanceof ControlError) {
return json({ error: error.message, humanHasControl: true }, 409);
}
return json({ error: describe(error, "The action failed.") }, 502);
}
}
Expand Down
54 changes: 53 additions & 1 deletion agent-computer/tests/authorisation.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
import { describe, expect, test } from "bun:test";
import { isOpenPath, matchesToken, offeredToken } from "../src/authorisation";
import {
actsOnTheComputer,
isOpenPath,
matchesToken,
offeredToken,
} from "../src/authorisation";

/**
* The check that stands in front of a Bot's browser.
Expand Down Expand Up @@ -88,3 +93,50 @@ describe("what an unauthenticated caller may reach", () => {
}
});
});

/**
* Which paths the wheel stops.
*
* A person takes the wheel at a login wall precisely because they no longer want the Bot acting, and
* `control.ts` states the property outright: "While a person holds control every acting call from the
* Bot is refused". That was true of the page from the start and untrue of the shell, which arrived
* later (#62) and was never wired to the wheel, so a Bot could run a command and rewrite the
* workspace underneath somebody mid-sign-in.
*
* The list lives here, beside the other path decision, rather than in `index.ts`, for the reason the
* header of this file gives: a decision next to `chromium.launch()` cannot be tested without Chrome.
*
* Reading is not acting. `/files/read` and `/files/list` stay open so a Bot waiting to be handed the
* wheel back can still say what it was doing.
*/
describe("what the wheel stops while a person is driving", () => {
test("every path that acts on the computer, the shell and a workspace write included", () => {
for (const path of [
"/navigate",
"/click",
"/type",
"/key",
"/scroll",
"/exec",
"/files/write",
]) {
expect(actsOnTheComputer(path)).toBeTrue();
}
});

test("reading, looking and the handover itself are not acting", () => {
for (const path of [
"/files/read",
"/files/list",
"/snapshot",
"/screenshot",
"/health",
"/control",
"/control/take",
"/control/release",
"/stream",
]) {
expect(actsOnTheComputer(path)).toBeFalse();
}
});
});
39 changes: 38 additions & 1 deletion app/src/components/app-sidebar/app-sidebar.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,12 @@ import {
useQuery,
useQueryClient,
} from "@tanstack/react-query";
import { Link, type LinkOptions, useNavigate } from "@tanstack/react-router";
import {
Link,
type LinkOptions,
useNavigate,
useParams,
} from "@tanstack/react-router";
import { AnimatePresence, motion, useReducedMotion } from "motion/react";
import type * as React from "react";
import { useState } from "react";
Expand Down Expand Up @@ -123,6 +128,30 @@ export function pinnedFirst(channels: ChannelSummary[]): ChannelSummary[] {
return [...channels].sort((a, b) => Number(b.pinned) - Number(a.pinned));
}

/**
* Whether a Bot has said something this member has not had on screen yet.
*
* A Bot's message, and only a Bot's: your own message carries a null agent id and reading your own
* words needs no marker. ISO-8601 strings compare correctly as strings, which is the same bet the
* server's recency sort already makes.
*/
export function hasUnseenActivity(channel: ChannelSummary): boolean {
if (channel.lastMessageAgentId === null || channel.lastMessageAt === null) {
return false;
}
return (
channel.lastReadAt === null || channel.lastMessageAt > channel.lastReadAt
);
}

/** Unseen activity somewhere you are not looking. The open channel never shows the dot. */
export function isUnread(
channel: ChannelSummary,
openChannelId: string | undefined,
): boolean {
return channel.id !== openChannelId && hasUnseenActivity(channel);
}

/**
* A roster row that can animate.
*
Expand All @@ -138,6 +167,13 @@ function ChannelRow({
animateOrder: boolean;
}) {
const shouldReduceMotion = useReducedMotion();
// Whether this row is unread, as a boolean, for the same reason `Channel` computes `isOpen`
// that way: navigating re-renders the rows whose answer changed, not the whole roster.
const unread = useParams({
strict: false,
select: (params) =>
isUnread(channel, (params as { channelId?: string }).channelId),
});
return (
<motion.div
animate={{ opacity: 1, transform: "translateY(0px)" }}
Expand All @@ -160,6 +196,7 @@ function ChannelRow({
: undefined
}
pinned={channel.pinned}
unread={unread}
/>
</motion.div>
);
Expand Down
12 changes: 11 additions & 1 deletion app/src/components/app-sidebar/channel.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,15 @@ export const Channel = memo(function Channel({
lastMessage,
lastMessageAt,
pinned,
unread,
}: {
channelId: string;
participantIds: string[];
name: string;
lastMessage?: string;
lastMessageAt?: string;
pinned: boolean;
unread: boolean;
}) {
const queryClient = useQueryClient();
const navigate = useNavigate();
Expand Down Expand Up @@ -112,7 +114,11 @@ export const Channel = memo(function Channel({
</div>
<div className="flex-col min-w-0 flex-1">
<div className="flex flex-row items-center justify-between gap-2">
<span className="text-[14px] tracking-[-1%] truncate">
<span
className={`text-[14px] tracking-[-1%] truncate ${
unread ? "font-medium" : ""
}`}
>
{name}
</span>
<div className="text-[12px] text-muted-foreground/70">
Expand All @@ -123,6 +129,10 @@ export const Channel = memo(function Channel({
<span className="min-w-0 flex-1 truncate text-[12px] leading-4 text-muted-foreground">
{lastMessage}
</span>
{unread ? (
/* State about the message beats state about the row, so it sits first. */
<span className="size-2 shrink-0 rounded-full bg-primary" />
) : null}
{pinned ? (
<IconPinFilled className="size-3 shrink-0 text-muted-foreground/70" />
) : null}
Expand Down
56 changes: 54 additions & 2 deletions app/src/lib/channels/mutations.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
import { mutationOptions, type QueryClient } from "@tanstack/react-query";
import {
mutationOptions,
type InfiniteData,
type QueryClient,
} from "@tanstack/react-query";
import { client, tryClient } from "@/lib/client";
import { type AgentChannel, channelKeys } from "./queries";
import { type AgentChannel, type ChannelPage, channelKeys } from "./queries";

/**
* Start a new channel with one or more coworkers.
Expand Down Expand Up @@ -66,6 +70,54 @@ export function setChannelPinnedMutationOptions(queryClient: QueryClient) {
});
}

/**
* Stamp a channel read for this member, patching the cache before the wire answers.
*
* Patched in onMutate rather than refetched on success: the dot must clear the instant the channel
* opens, not a round-trip later. No rollback on failure and no invalidation — a mark-read that did
* not land is a dot that returns on the next refetch, which is the truth reasserting itself, and a
* refetch here would race the socket's own patches for nothing.
*/
export function markChannelReadMutationOptions(queryClient: QueryClient) {
return mutationOptions({
mutationFn: async (channelId: string) => {
await client(`/api/channels/${channelId}/read`, {
method: "PUT",
fallback: "Could not mark this channel read",
});
},
onMutate: (channelId) => {
const now = new Date().toISOString();
queryClient.setQueryData(
channelKeys.list(),
(data: InfiniteData<ChannelPage> | undefined) =>
data && {
...data,
pages: data.pages.map((page) => ({
...page,
channels: page.channels.map((row) =>
row.id === channelId
? {
...row,
/*
* The later of now and the row's own lastMessageAt: lastMessageAt comes from
* another clock, and a marker stamped "now" by a clock running behind it
* would leave the row still reading as unseen — and the dot still lit.
*/
lastReadAt:
row.lastMessageAt && row.lastMessageAt > now
? row.lastMessageAt
: now,
}
: row,
),
})),
},
);
},
});
}

/** Soft-delete a channel for everyone in it. The server keeps the transcript; the roster forgets. */
export function deleteChannelMutationOptions(queryClient: QueryClient) {
return mutationOptions({
Expand Down
Loading