From e8bde27fc1a897531e36446ee21fa63d0956007a Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Sun, 7 Jun 2026 22:19:45 +0100 Subject: [PATCH 1/8] Add collaborative markdown docs for agents --- AGENTS_MARKDOWN_DOCS_PLAN.md | 729 ++++++++++++++++++ packages/agents-runtime/src/client.ts | 4 + packages/agents-runtime/src/create-handler.ts | 25 + packages/agents-runtime/src/entity-schema.ts | 38 + packages/agents-runtime/src/index.ts | 6 + .../agents-runtime/src/manifest-helpers.ts | 28 + packages/agents-runtime/src/process-wake.ts | 20 + .../src/runtime-server-client.ts | 157 ++++ packages/agents-runtime/src/tools.ts | 1 + .../agents-runtime/src/tools/markdown-docs.ts | 202 +++++ packages/agents-runtime/src/types.ts | 26 + .../test/markdown-docs-tools.test.ts | 111 +++ packages/agents-server-ui/package.json | 9 + .../src/components/EntityContextDrawer.tsx | 27 + .../src/components/EntityTimeline.tsx | 72 +- .../views/MarkdownDocumentView.module.css | 94 +++ .../components/views/MarkdownDocumentView.tsx | 198 +++++ .../src/components/workspace/Workspace.tsx | 35 +- .../src/lib/workspace/registerViews.ts | 14 +- packages/agents-server/package.json | 4 +- packages/agents-server/src/entity-manager.ts | 376 ++++++++- .../agents-server/src/markdown-documents.ts | 176 +++++ .../src/routing/durable-streams-router.ts | 152 ++++ .../src/routing/entities-router.ts | 142 ++++ packages/agents-server/src/stream-client.ts | 43 ++ ...ic-agents-manager-write-validation.test.ts | 110 +++ .../test/electric-agents-routes.test.ts | 88 +++ packages/agents/skills/markdown-docs.md | 94 +++ packages/agents/src/agents/horton.ts | 15 +- packages/agents/src/bootstrap.ts | 10 +- .../agents/test/horton-system-prompt.test.ts | 18 + .../test/horton-tool-composition.test.ts | 15 +- pnpm-lock.yaml | 318 +++++--- 33 files changed, 3230 insertions(+), 127 deletions(-) create mode 100644 AGENTS_MARKDOWN_DOCS_PLAN.md create mode 100644 packages/agents-runtime/src/tools/markdown-docs.ts create mode 100644 packages/agents-runtime/test/markdown-docs-tools.test.ts create mode 100644 packages/agents-server-ui/src/components/views/MarkdownDocumentView.module.css create mode 100644 packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx create mode 100644 packages/agents-server/src/markdown-documents.ts create mode 100644 packages/agents/skills/markdown-docs.md diff --git a/AGENTS_MARKDOWN_DOCS_PLAN.md b/AGENTS_MARKDOWN_DOCS_PLAN.md new file mode 100644 index 0000000000..1bb1965867 --- /dev/null +++ b/AGENTS_MARKDOWN_DOCS_PLAN.md @@ -0,0 +1,729 @@ +# Agents Markdown Docs Implementation Plan + +## Goal + +Add first-class collaborative markdown documents to Electric Agents. + +Agents should be able to create a markdown document, add it to the entity +manifest, read it, and edit it with file-like replacement tools. Users should be +able to click the manifest entry, open a CodeMirror markdown editor in the +workspace, edit concurrently with other users, and see agent/user presence. + +The first implementation intentionally does not require streaming tool calls or +runtime-level interception of assistant text. Streaming edits can be added after +the document model, auth, UI, and non-streaming tools are working. + +## MVP Scope + +### In scope + +- A new manifest entry kind for collaborative markdown documents. +- One durable Yjs document stream per document, using + `@durable-streams/y-durable-streams`. +- A CodeMirror markdown editor bound to `Y.Text`. +- User presence through Yjs awareness. +- Agent presence during document tools, including status and edit location. +- Agent tools for create/read/write/exact text replacement. +- Unified diff results from write/edit tools, matching the current file tool + behavior. +- Explicit server auth for document Yjs stream paths. +- Forking support so forked entities receive forked document streams. + +### Out of scope for MVP + +- Token-by-token agent edits. +- Streaming tool arguments. +- Runtime routing of assistant text into documents. +- Rich-text CRDTs such as ProseMirror fragments. +- Markdown preview/render mode. +- Comments or suggestions inside docs. +- Document history UI beyond the Yjs/Durable Streams backing log. + +## Core Design + +### Manifest Entry + +Add a new manifest entry kind rather than encoding docs as attachments. + +Attachments are immutable, closed streams with byte length and sha256 semantics. +Markdown docs are mutable CRDT-backed resources, so they should be first-class +manifest entries with their own lifecycle and fork/auth behavior. + +Proposed manifest shape: + +```ts +type ManifestDocumentEntry = { + key?: string + kind: 'document' + id: string + title: string + provider: 'y-durable-streams' + docId: string + docPath: string + streamPath: string + contentMimeType: 'text/markdown' + transportMimeType: 'application/vnd.electric-agents.markdown-yjs' + yTextName: 'markdown' + createdAt: string + createdBy?: string + updatedAt?: string + meta?: Record +} +``` + +Recommended manifest key: + +```ts +document:${id} +``` + +Recommended stream path: + +```ts +/docs/agents/${entityType}/${instanceId}/documents/${id} +``` + +`docId` is the value passed to `YjsProvider`. + +`docPath` is the provider-facing stable document path and should not have a +leading slash: + +```ts +agents/${entityType}/${instanceId}/documents/${id} +``` + +`streamPath` is the Durable Streams document stream path used for auth, forking, +and debugging: + +```ts +/docs/${docPath} +``` + +This shape follows the `y-durable-streams` URL contract. The provider requests: + +```ts +{baseUrl}/docs/{docPath}?{queryParams} +``` + +For the agents server, use: + +```ts +baseUrl = agentsServerUrl +docId = docPath +``` + +Do not set `baseUrl` to the raw `streamPath`; the provider appends `/docs/...` +itself. + +### Yjs Document Model + +Use a plain Yjs text type: + +```ts +const ytext = ydoc.getText('markdown') +``` + +This keeps the MVP simple: + +- The stored CRDT is binary Yjs updates. +- The logical document content is markdown text. +- CodeMirror can bind directly to `Y.Text`. +- Agent tools can operate on `ytext.toString()` and commit Yjs transactions. + +### Mime Types + +Use two concepts: + +- `contentMimeType: 'text/markdown'` for what users and tools are editing. +- `transportMimeType: 'application/vnd.electric-agents.markdown-yjs'` for what + is stored in the durable stream. + +Do not label the durable stream itself as `text/markdown`; its bytes are Yjs +updates/snapshots. + +## Implementation Areas + +### 1. Runtime Types and Manifest Schema + +Files: + +- `packages/agents-runtime/src/entity-schema.ts` +- `packages/agents-runtime/src/types.ts` +- `packages/agents-runtime/src/manifest-helpers.ts` +- `packages/agents-server-ui/src/lib/ElectricAgentsProvider.tsx` + +Tasks: + +- Add `ManifestDocumentEntryValue`. +- Extend the manifest zod union with `kind: 'document'`. +- Export `ManifestDocumentEntry`. +- Add `manifestDocumentKey(id: string)`. +- Update UI-side manifest parsing/types to accept `kind: 'document'`. + +Acceptance: + +- Entity state can contain a `manifest` event with `kind: 'document'`. +- Existing manifest consumers continue to parse older entries. + +### 2. Server Document API + +Files: + +- `packages/agents-server/src/entity-manager.ts` +- `packages/agents-server/src/routing/entities-router.ts` +- `packages/agents-runtime/src/runtime-server-client.ts` +- `packages/agents-runtime/src/types.ts` + +Tasks: + +- Add document validation helpers: + - document id cannot be empty, start with `.`, or contain `/`. + - title should be non-empty and bounded. +- Add `createDocument(entityUrl, req)`: + - create durable Yjs backing stream if needed. + - initialize the Yjs document with optional markdown text. + - write the document manifest entry. + - return `{ txid, document }`. +- Add `getDocument(entityUrl, id)`. +- Add `readDocument(entityUrl, id)` returning current markdown text. +- Add `writeDocument(entityUrl, id, content)` replacing the whole `Y.Text`. +- Add `editDocument(entityUrl, id, old_string, new_string, replace_all?)`. +- Add HTTP routes under entity API: + - `POST /:type/:instanceId/documents` + - `GET /:type/:instanceId/documents/:documentId` + - `GET /:type/:instanceId/documents/:documentId/content` + - `PUT /:type/:instanceId/documents/:documentId/content` + - `PATCH /:type/:instanceId/documents/:documentId/content` + +Open implementation choice: + +- Preferred for MVP: put create/read/write/edit on the server API and expose + them through `RuntimeServerClient`. This keeps auth, fork locks, and manifest + writes in one place. +- Avoid direct runtime-tool writes to `YjsProvider` in the first cut. That is + faster to prototype, but it spreads auth, fork locks, and stream path rules + into the runtime. + +Acceptance: + +- Creating a doc appends a manifest row. +- Reading returns markdown text from the Yjs doc. +- Writing/editing produces Yjs updates, not manifest content mutations. +- Server rejects operations when the entity is stopped or fork-write-locked. + +### 3. Durable Stream Yjs Integration + +Files: + +- `packages/agents-server/package.json` +- `packages/agents-server-ui/package.json` +- `packages/agents-runtime/package.json` if runtime tools manipulate Yjs locally. + +Dependencies: + +- `@durable-streams/y-durable-streams` +- `yjs` +- `y-protocols` +- `lib0` +- UI only: + - `codemirror` + - `@codemirror/state` + - `@codemirror/view` + - `@codemirror/lang-markdown` + - `y-codemirror.next` + +Tasks: + +- Use `YjsProvider` for browser/editor connections. +- On server create/write/edit, either: + - use `YjsProvider` server-side and wait for sync, or + - use the y-durable-streams server utilities if exposed by the package. +- Always destroy providers after tool/server operations. +- For initial content, create a `Y.Doc`, set `getText('markdown')`, and persist + through the provider. +- Keep the Yjs mount constants in one shared server module: + - `docPathForDocument(entityUrl, documentId)` + - `documentStreamPathForDocPath(docPath)` + - `entityUrlFromYjsDocumentPath(path)` + - `entityUrlFromYjsAwarenessPath(path)` + +Acceptance: + +- A browser editor and server operation converge on the same markdown text. +- New editor clients load through snapshot discovery and then live updates. + +### 4. Durable Stream Auth + +Files: + +- `packages/agents-server/src/routing/durable-streams-router.ts` +- `packages/agents-server/src/routing/stream-append.ts` + +Tasks: + +- Add document path recognition for provider document requests: + +```ts +function entityUrlFromYjsDocumentPath(path: string): string | null { + const match = path.match( + /^\/docs\/agents\/([^/]+)\/([^/]+)\/documents\/[^/]+(?:\/.*)?$/ + ) + if (!match) return null + return `/${match[1]}/${match[2]}` +} +``` + +- Authorize `GET`/`HEAD` document stream access with entity read permission. +- Authorize `POST`/`PUT` document stream writes with entity write/manage rules + or a dedicated document write permission rule. +- Inspect the installed `@durable-streams/y-durable-streams` package and add an + equivalent `entityUrlFromYjsAwarenessPath(path)` for the exact awareness URL + pattern used by the provider. +- Add route tests using real provider URL shapes for: + - snapshot discovery and snapshot load. + - live update reads. + - local edit writes. + - awareness reads/writes. +- Reject direct writes to document paths during fork locks. + +Important: + +The current durable-stream proxy explicitly guards entity streams, attachment +streams, and shared-state streams. Unknown paths intentionally pass through. +Document and document-awareness paths must not remain in that pass-through +bucket. + +Acceptance: + +- Unauthorized users cannot read, write, or observe awareness for document + streams. +- Authorized users can edit through CodeMirror. +- Fork locks prevent concurrent writes while the subtree is being forked. + +### 5. Forking + +Files: + +- `packages/agents-server/src/entity-manager.ts` + +Tasks: + +- Collect document stream paths from document manifest entries during fork + snapshot reads. +- Lock document stream paths during fork, like shared-state streams. +- Fork each document durable stream from source to fork destination. +- Remap document manifest entries: + - `streamPath` + - `docPath` + - `docId` + - possibly `key` if document ids are rewritten. +- Keep document ids stable within a fork unless collisions require suffixing. + +Acceptance: + +- Forked entity opens an independent copy of each document. +- Editing a forked doc does not change the source entity's doc. +- Pointer forks include only document manifest entries visible at the fork point. + +### 6. Runtime Tool Surface + +Files: + +- `packages/agents-runtime/src/tools/documents.ts` +- `packages/agents-runtime/src/tools.ts` +- `packages/agents-runtime/src/types.ts` +- `packages/agents-runtime/src/process-wake.ts` +- `packages/agents/src/bootstrap.ts` + +Tasks: + +- Add framework document tool factory. +- Extend `ProcessWakeConfig.createElectricTools` context with + `principal?: RuntimePrincipal`, and pass `config.principal` through from + `processWake`. Document tools need this for agent awareness state. +- Extend `ProcessWakeConfig.createElectricTools` context with document methods + backed by `RuntimeServerClient`: + - `createMarkdownDocument` + - `readMarkdownDocument` + - `writeMarkdownDocument` + - `editMarkdownDocument` +- Add default built-in tools in `packages/agents/src/bootstrap.ts`, alongside + event-source tools. +- Keep worker exposure explicit if desired. Horton already includes + `ctx.electricTools`; Worker currently gets only selected tools. + +Tool shapes: + +```ts +create_markdown_doc({ + title: string, + content?: string +}) +``` + +```ts +read_markdown_doc({ + docId: string, +}) +``` + +```ts +write_markdown_doc({ + docId: string, + content: string, +}) +``` + +```ts +edit_markdown_doc({ + docId: string, + old_string: string, + new_string: string, + replace_all?: boolean +}) +``` + +Tool behavior should mirror file tools: + +- `read_markdown_doc`, `create_markdown_doc`, and `write_markdown_doc` mark the + document as read in a per-wake read set. +- `edit_markdown_doc` must reject edits unless the document has been read or + written in the same wake. +- `old_string` must occur exactly once unless `replace_all` is true. +- Return a useful error when not found or ambiguous. +- Return `details.diff` using `createTwoFilesPatch`. +- Return replacement counts and byte/char counts. + +Acceptance: + +- An agent can create a doc and then read/edit it with file-like tools. +- Tool call UI shows a diff for document edits without special casing if + possible. + +### 7. Agent Presence During Tools + +Files: + +- `packages/agents-runtime/src/tools/documents.ts` +- server-side document service module, if split from `entity-manager.ts` + +Tasks: + +- When a document tool edits content: + - connect to the Yjs provider with an `Awareness` instance. + - set local awareness state from the principal passed through + `createElectricTools`, or from an agent principal derived by the server: + +```ts +{ + user: { + principalUrl, + role: 'agent', + name, + color, + status: 'editing' + } +} +``` + +- Before applying a replacement, set the agent selection/cursor near the + replacement range. +- Apply the Yjs transaction. +- Move cursor to the end of the replacement. +- Set status back to `idle` or destroy provider so awareness removal is + broadcast. + +Acceptance: + +- While an agent edit tool is running, open editors see the agent presence. +- For quick edits this may be brief; that is acceptable for MVP. + +### 8. UI: Document Manifest Rows + +Files: + +- `packages/agents-server-ui/src/components/EntityTimeline.tsx` +- `packages/agents-server-ui/src/lib/attachments.ts` or a new + `documents.ts` + +Tasks: + +- Add `isDocumentManifest`. +- Display document rows as `Document`. +- Use the title as primary text. +- Show `text/markdown`, provider, and created metadata. +- Add an open action. +- Use workspace helper: + +```ts +workspace.helpers.openEntity(entityUrl, { + viewId: 'markdown-doc', + viewParams: { docId: manifest.id }, +}) +``` + +Acceptance: + +- Document manifests are not hidden as attachments. +- Clicking a document opens the editor view. + +### 9. UI: CodeMirror Markdown Editor View + +Files: + +- `packages/agents-server-ui/src/lib/workspace/registerViews.ts` +- `packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx` +- new CSS module for the editor view. + +Tasks: + +- Register entity view: + +```ts +registerView({ + kind: 'entity', + id: 'markdown-doc', + label: 'Docs', + icon: FileText, + Component: MarkdownDocumentView, +}) +``` + +- Resolve `docId` from `viewParams`. +- Find document manifest from entity DB. +- Construct `Y.Doc`, `Awareness`, and `YjsProvider`. +- Use `baseUrl` pointing at the agents server durable-stream proxy. +- Bind CodeMirror to `ydoc.getText('markdown')`. +- Set local user awareness from `useCurrentPrincipal()`. +- Pass configured auth/principal headers to `YjsProvider.headers`, matching the + rest of the agents UI request path. +- Render presence bar from awareness states. +- Destroy CodeMirror view/provider on unmount. + +Acceptance: + +- Two browser windows can concurrently edit one doc. +- Remote cursor/presence appears. +- Agent tool edits appear live in open editors. +- The editor survives tile split/open/close cycles. + +### 10. Tests + +Unit and integration tests should be added at the layer being changed. + +Runtime: + +- Manifest schema accepts document entries. +- Document tool exact replacement behavior matches file edit behavior. +- Diff details are returned. + +Server: + +- Create document writes manifest. +- Read/write/edit round trip through Yjs. +- Unauthorized durable stream document access is rejected. +- Forked docs are independent. + +UI: + +- Manifest row labels and open action. +- View registration. +- Editor view mounts with missing/invalid doc id states. + +## Deferred Streaming Edit Work + +The repo already has enough evidence for a later streaming path: + +- `@mariozechner/pi-ai` emits `toolcall_start`, `toolcall_delta`, and + `toolcall_end` provider events. +- `@mariozechner/pi-agent-core` forwards those as `message_update` while the + assistant message is streaming. +- `packages/agents-runtime/src/pi-adapter.ts` currently only handles + `text_delta` in `message_update`. +- `packages/agents-runtime/src/outbound-bridge.ts` currently persists tool calls + only at `tool_execution_start` and final completion. + +Later streaming options: + +1. Surface tool argument deltas through the outbound bridge and persist partial + args in the `toolCalls` collection. +2. Add a streaming document insertion tool whose string argument can be consumed + incrementally. +3. Or add a runtime-level text routing mode. This is more invasive and should + remain separate from the MVP. + +This plan intentionally chooses non-streaming exact replacements first because +it avoids changing agent execution semantics. + +## Open Questions + +Resolve these inside the single PR before enabling the feature: + +- Should document tools be enabled for all built-in agents by default, or only + for Horton initially? +- Should workers be able to receive document tools by name in their spawn args? +- Should document stream write permission be tied to entity `manage`, entity + `write`, or a new permission? +- Should document ids remain stable across forks, or be suffixed like shared + state ids? + +## Single PR Implementation Phases + +Implement this as one PR. The phases below are sequencing for development and +review inside the branch, not separate merge boundaries. The PR should not be +merged with document creation/editing enabled until schema, server API, +auth, forking, tools, UI, presence, and tests are all complete. + +### Phase 0: Provider Path Spike + +Goal: remove uncertainty before changing product code. + +Tasks: + +- Inspect the installed `@durable-streams/y-durable-streams` package. +- Confirm the exact document request URLs for: + - snapshot discovery. + - snapshot load. + - live update reads. + - local edit writes. +- Confirm the exact awareness request URLs and methods. +- Capture helper names and URL examples in code comments/tests, not as + free-floating assumptions. + +Exit criteria: + +- The implementation has concrete helpers for document and awareness path + recognition. +- Route tests use real provider-shaped URLs. + +### Phase 1: Schema and Shared Types + +Tasks: + +- Add `ManifestDocumentEntryValue`. +- Extend the manifest schema union. +- Export document manifest types. +- Add `manifestDocumentKey(id)`. +- Add shared document path helpers. +- Update UI manifest parsing/types. + +Exit criteria: + +- Existing entity streams still load. +- A synthetic document manifest row parses in runtime and UI tests. + +### Phase 2: Server Document Service + +Tasks: + +- Add document id/title validation. +- Add create/get/read/write/edit document methods. +- Store initial markdown as `Y.Text('markdown')`. +- Return unified diffs from write/edit operations. +- Add entity API routes and `RuntimeServerClient` methods. +- Keep document writes server-mediated for MVP. + +Exit criteria: + +- Server tests can create, read, write, and exact-replace a markdown doc. +- Edit errors match the file edit tool behavior for missing/ambiguous strings. + +### Phase 3: Auth and Fork Safety + +Tasks: + +- Authorize `/docs/agents/...` document paths. +- Authorize the matching y-durable-streams awareness paths. +- Reject unauthorized document reads/writes/presence. +- Lock document streams during fork work. +- Clone document streams during fork. +- Remap `streamPath`, `docPath`, and `docId` in forked manifest entries. + +Exit criteria: + +- Unauthorized users cannot read/write doc streams or awareness streams. +- Forked entities edit independent document streams. +- Pointer forks include only document manifests visible at the fork point. + +### Phase 4: Runtime Tools + +Tasks: + +- Add document methods and `principal` to `createElectricTools` context. +- Add `create_markdown_doc`, `read_markdown_doc`, `write_markdown_doc`, and + `edit_markdown_doc`. +- Maintain a per-wake read set. +- Require read/write/create before exact edit in the same wake. +- Add document tools to the built-in electric tool bundle. +- Decide and document Worker exposure in the same PR. + +Exit criteria: + +- Horton can create/read/write/edit a doc through tools. +- Tool results include `details.diff`. +- Tool behavior mirrors file tools closely enough that the existing tool UI is + usable. + +### Phase 5: UI Manifest and Editor + +Tasks: + +- Add document manifest row rendering. +- Add open action using `viewId: 'markdown-doc'` and + `viewParams: { docId }`. +- Register the `markdown-doc` entity view. +- Add CodeMirror markdown editor bound to `ydoc.getText('markdown')`. +- Pass auth/principal headers to `YjsProvider`. +- Handle missing/invalid doc ids and provider errors. +- Destroy CodeMirror/Yjs resources on unmount. + +Exit criteria: + +- Clicking a document manifest opens the editor. +- Two editor tiles/windows can edit the same doc concurrently. +- Agent tool edits appear in open editors. + +### Phase 6: Presence + +Tasks: + +- Set user awareness from `useCurrentPrincipal()`. +- Render presence states in the editor. +- Set agent awareness while document tools are running. +- Show agent status and cursor/edit location for replacements. + +Exit criteria: + +- Users see other active users in the document. +- Users see agent presence while an agent edit tool is applying a change. + +### Phase 7: Verification + +Tasks: + +- Run runtime tests. +- Run server tests. +- Run UI tests. +- Run package typechecks. +- Manually verify the desktop flow: + - agent creates a document. + - manifest entry appears. + - user opens it in a tile. + - user edits it. + - agent edits it with exact replacement. + - two windows/tiles see concurrent updates and presence. + - forked entity receives an independent document. + +Suggested commands after `pnpm install` from repo root: + +```sh +pnpm --filter @electric-ax/agents-runtime test +pnpm --filter @electric-ax/agents-server test +pnpm --filter @electric-ax/agents-server-ui test +pnpm --filter @electric-ax/agents-runtime typecheck +pnpm --filter @electric-ax/agents-server typecheck +pnpm --filter @electric-ax/agents-server-ui typecheck +``` + +Streaming edits should be a later design/implementation after the single PR +lands and the non-streaming collaborative document workflow is stable. diff --git a/packages/agents-runtime/src/client.ts b/packages/agents-runtime/src/client.ts index 0d467fd7f1..8f3ba99052 100644 --- a/packages/agents-runtime/src/client.ts +++ b/packages/agents-runtime/src/client.ts @@ -23,7 +23,10 @@ export { export { appendPathToUrl } from './url' export { getEntityAttachmentStreamPath, + getEntityMarkdownDocumentPath, + getEntityMarkdownDocumentUrlPath, manifestAttachmentKey, + manifestMarkdownDocumentKey, } from './manifest-helpers' export { buildSections, buildTimelineEntries } from './use-chat' export { COMPOSER_INPUT_MESSAGE_TYPE } from './composer-input' @@ -46,6 +49,7 @@ export type { AttachmentSubjectType, Manifest, ManifestAttachmentEntry, + ManifestDocumentEntry, } from './entity-schema' export type { AttachmentCreateInput, diff --git a/packages/agents-runtime/src/create-handler.ts b/packages/agents-runtime/src/create-handler.ts index c4008862d4..a8d2a26d72 100644 --- a/packages/agents-runtime/src/create-handler.ts +++ b/packages/agents-runtime/src/create-handler.ts @@ -19,6 +19,7 @@ import type { AgentTool, EntityStreamDBWithActions, HeadersProvider, + ManifestDocumentEntry, ProcessWakeConfig, WakeNotification, WebhookNotification, @@ -98,6 +99,30 @@ export interface RuntimeRouterConfig { unsubscribeFromEventSource: (opts: { id: string }) => Promise<{ txid: string }> + createMarkdownDocument: (opts: { + id?: string + title: string + content?: string + meta?: Record + }) => Promise<{ txid: string; document: ManifestDocumentEntry }> + readMarkdownDocument: (opts: { + id: string + }) => Promise<{ document: ManifestDocumentEntry; content: string }> + writeMarkdownDocument: (opts: { id: string; content: string }) => Promise<{ + txid: string + document: ManifestDocumentEntry + content: string + }> + editMarkdownDocument: (opts: { + id: string + oldString: string + newString: string + replaceAll?: boolean + }) => Promise<{ + txid: string + document: ManifestDocumentEntry + content: string + }> }) => Array | Promise> /** * Optional observer for background wake failures. Return true to mark the diff --git a/packages/agents-runtime/src/entity-schema.ts b/packages/agents-runtime/src/entity-schema.ts index 7d70d3cef2..159a27cc6a 100644 --- a/packages/agents-runtime/src/entity-schema.ts +++ b/packages/agents-runtime/src/entity-schema.ts @@ -311,6 +311,20 @@ type ManifestAttachmentEntryValue = { error?: string meta?: Record } +type ManifestDocumentEntryValue = { + key?: string + kind: `document` + id: string + docPath: string + streamPath: string + mimeType: `application/vnd.electric-agents.markdown-yjs` + contentMimeType: `text/markdown` + title: string + createdAt: string + createdBy?: string + updatedAt?: string + meta?: Record +} type ContextEntryAttrsValue = Record type ManifestContextEntryValue = { key?: string @@ -713,6 +727,7 @@ function createManifestSchema(): Schema< | ManifestSharedStateEntryValue | ManifestEffectEntryValue | ManifestAttachmentEntryValue + | ManifestDocumentEntryValue | ManifestContextEntryValue | ManifestCronScheduleEntryValue | ManifestFutureSendScheduleEntryValue @@ -778,6 +793,21 @@ function createManifestSchema(): Schema< error: z.string().optional(), meta: createAttachmentMetaSchema().optional(), }), + z.object({ + key: z.string().optional(), + ...timelineOrderField, + kind: z.literal(`document`), + id: z.string(), + docPath: z.string(), + streamPath: z.string(), + mimeType: z.literal(`application/vnd.electric-agents.markdown-yjs`), + contentMimeType: z.literal(`text/markdown`), + title: z.string(), + createdAt: z.string(), + createdBy: z.string().optional(), + updatedAt: z.string().optional(), + meta: createAttachmentMetaSchema().optional(), + }), z.object({ key: z.string().optional(), ...timelineOrderField, @@ -824,6 +854,7 @@ function createManifestSchema(): Schema< | ManifestSharedStateEntryValue | ManifestEffectEntryValue | ManifestAttachmentEntryValue + | ManifestDocumentEntryValue | ManifestContextEntryValue | ManifestCronScheduleEntryValue | ManifestFutureSendScheduleEntryValue @@ -875,6 +906,8 @@ export type AttachmentRole = AttachmentRoleValue export type AttachmentSubject = AttachmentSubjectValue export type ManifestAttachmentEntry = SequencedPersistedRow +export type ManifestDocumentEntry = + SequencedPersistedRow export type ManifestContextEntry = SequencedPersistedRow export type ManifestCronScheduleEntry = @@ -887,6 +920,7 @@ type ManifestUnion = | ManifestSharedStateEntry | ManifestEffectEntry | ManifestAttachmentEntry + | ManifestDocumentEntry | ManifestContextEntry | ManifestCronScheduleEntry | ManifestFutureSendScheduleEntry @@ -910,6 +944,10 @@ export type Manifest = ManifestUnion & { createdBy?: string error?: string meta?: Record + docPath?: string + contentMimeType?: `text/markdown` + title?: string + updatedAt?: string name?: string attrs?: ContextEntryAttrs content?: string diff --git a/packages/agents-runtime/src/index.ts b/packages/agents-runtime/src/index.ts index 3275e31be3..d84ce039c9 100644 --- a/packages/agents-runtime/src/index.ts +++ b/packages/agents-runtime/src/index.ts @@ -7,6 +7,7 @@ export type { ManifestAttachmentEntry, ManifestChildEntry, ManifestContextEntry, + ManifestDocumentEntry, ManifestEntry, ManifestEffectEntry, ManifestSourceEntry, @@ -70,6 +71,7 @@ export type { GeneratedStateActions, HandlerActions, ManifestContextEntry as ManifestContextRow, + ManifestDocumentEntry as ManifestDocumentRow, SchemaInput, SchemaOutput, SourceConfig, @@ -113,6 +115,7 @@ export type { AttachmentSubject, AttachmentSubjectType, ManifestContextEntry as ManifestContextEntryRow, + ManifestDocumentEntry as ManifestDocumentEntryRow, ReplayWatermark, WakeConfigValue, } from './entity-schema' @@ -120,7 +123,10 @@ export type { export { createEntityStreamDB } from './entity-stream-db' export { getEntityAttachmentStreamPath, + getEntityMarkdownDocumentPath, + getEntityMarkdownDocumentUrlPath, manifestAttachmentKey, + manifestMarkdownDocumentKey, } from './manifest-helpers' export { COMPOSER_INPUT_MESSAGE_TYPE, diff --git a/packages/agents-runtime/src/manifest-helpers.ts b/packages/agents-runtime/src/manifest-helpers.ts index 599cd6fa38..bce492b98c 100644 --- a/packages/agents-runtime/src/manifest-helpers.ts +++ b/packages/agents-runtime/src/manifest-helpers.ts @@ -16,9 +16,37 @@ export function manifestAttachmentKey(id: string): string { return `attachment:${id}` } +export function manifestMarkdownDocumentKey(id: string): string { + return `document:${id}` +} + export function getEntityAttachmentStreamPath( entityUrl: string, attachmentId: string ): string { return `${entityUrl.replace(/\/+$/, ``)}/attachments/${attachmentId}` } + +export function getEntityMarkdownDocumentPath( + entityUrl: string, + documentId: string +): string { + const segments = entityUrl.replace(/^\/+|\/+$/g, ``).split(`/`) + if (segments.length !== 2 || !segments[0] || !segments[1]) { + throw new Error( + `Invalid entity URL for markdown document path: ${entityUrl}` + ) + } + return `agents/${segments[0]}/${segments[1]}/documents/${documentId}` +} + +export function getEntityMarkdownDocumentUrlPath( + service: string, + entityUrl: string, + documentId: string +): string { + return `/v1/yjs/${encodeURIComponent(service)}/docs/${getEntityMarkdownDocumentPath( + entityUrl, + documentId + )}` +} diff --git a/packages/agents-runtime/src/process-wake.ts b/packages/agents-runtime/src/process-wake.ts index 3dd1cc60ec..8d1a224d81 100644 --- a/packages/agents-runtime/src/process-wake.ts +++ b/packages/agents-runtime/src/process-wake.ts @@ -1981,6 +1981,26 @@ export async function processWake( entityUrl, ...opts, }), + createMarkdownDocument: (opts) => + serverClient.createMarkdownDocument({ + entityUrl, + ...opts, + }), + readMarkdownDocument: (opts) => + serverClient.readMarkdownDocument({ + entityUrl, + ...opts, + }), + writeMarkdownDocument: (opts) => + serverClient.writeMarkdownDocument({ + entityUrl, + ...opts, + }), + editMarkdownDocument: (opts) => + serverClient.editMarkdownDocument({ + entityUrl, + ...opts, + }), }) : [] diff --git a/packages/agents-runtime/src/runtime-server-client.ts b/packages/agents-runtime/src/runtime-server-client.ts index bf0791e8b4..20d6476056 100644 --- a/packages/agents-runtime/src/runtime-server-client.ts +++ b/packages/agents-runtime/src/runtime-server-client.ts @@ -6,6 +6,7 @@ import type { ClaimTokenHeader, HeadersProvider, ManifestAttachmentEntry, + ManifestDocumentEntry, } from './types' import type { EntitySignal } from './entity-schema' import type { @@ -124,6 +125,37 @@ export interface RuntimeServerClient { entityUrl: string id: string }) => Promise + createMarkdownDocument: (options: { + entityUrl: string + id?: string + title: string + content?: string + meta?: Record + }) => Promise<{ txid: string; document: ManifestDocumentEntry }> + readMarkdownDocument: (options: { + entityUrl: string + id: string + }) => Promise<{ document: ManifestDocumentEntry; content: string }> + writeMarkdownDocument: (options: { + entityUrl: string + id: string + content: string + }) => Promise<{ + txid: string + document: ManifestDocumentEntry + content: string + }> + editMarkdownDocument: (options: { + entityUrl: string + id: string + oldString: string + newString: string + replaceAll?: boolean + }) => Promise<{ + txid: string + document: ManifestDocumentEntry + content: string + }> spawnEntity: (options: SpawnEntityOptions) => Promise getEntity: (entityUrl: string) => Promise ensureSharedStateStream: ( @@ -397,6 +429,127 @@ export function createRuntimeServerClient( return new Uint8Array(await response.arrayBuffer()) } + const createMarkdownDocument = async ({ + entityUrl, + id, + title, + content, + meta, + }: { + entityUrl: string + id?: string + title: string + content?: string + meta?: Record + }): Promise<{ txid: string; document: ManifestDocumentEntry }> => { + const response = await request(`${entityRpcPath(entityUrl)}/documents`, { + method: `POST`, + headers: { 'content-type': `application/json` }, + body: JSON.stringify({ id, title, content, meta }), + }) + if (!response.ok) { + throw new Error( + `create markdown document on ${entityUrl} failed (${response.status}): ${await readErrorText(response)}` + ) + } + return (await response.json()) as { + txid: string + document: ManifestDocumentEntry + } + } + + const readMarkdownDocument = async ({ + entityUrl, + id, + }: { + entityUrl: string + id: string + }): Promise<{ document: ManifestDocumentEntry; content: string }> => { + const response = await request( + `${entityRpcPath(entityUrl)}/documents/${encodeURIComponent(id)}`, + { method: `GET` } + ) + if (!response.ok) { + throw new Error( + `read markdown document ${id} on ${entityUrl} failed (${response.status}): ${await readErrorText(response)}` + ) + } + return (await response.json()) as { + document: ManifestDocumentEntry + content: string + } + } + + const writeMarkdownDocument = async ({ + entityUrl, + id, + content, + }: { + entityUrl: string + id: string + content: string + }): Promise<{ + txid: string + document: ManifestDocumentEntry + content: string + }> => { + const response = await request( + `${entityRpcPath(entityUrl)}/documents/${encodeURIComponent(id)}`, + { + method: `PUT`, + headers: { 'content-type': `application/json` }, + body: JSON.stringify({ content }), + } + ) + if (!response.ok) { + throw new Error( + `write markdown document ${id} on ${entityUrl} failed (${response.status}): ${await readErrorText(response)}` + ) + } + return (await response.json()) as { + txid: string + document: ManifestDocumentEntry + content: string + } + } + + const editMarkdownDocument = async ({ + entityUrl, + id, + oldString, + newString, + replaceAll, + }: { + entityUrl: string + id: string + oldString: string + newString: string + replaceAll?: boolean + }): Promise<{ + txid: string + document: ManifestDocumentEntry + content: string + }> => { + const response = await request( + `${entityRpcPath(entityUrl)}/documents/${encodeURIComponent(id)}`, + { + method: `PATCH`, + headers: { 'content-type': `application/json` }, + body: JSON.stringify({ oldString, newString, replaceAll }), + } + ) + if (!response.ok) { + throw new Error( + `edit markdown document ${id} on ${entityUrl} failed (${response.status}): ${await readErrorText(response)}` + ) + } + return (await response.json()) as { + txid: string + document: ManifestDocumentEntry + content: string + } + } + const getEntity = async (entityUrl: string): Promise => { const response = await request(entityRpcPath(entityUrl), { method: `GET` }) if (!response.ok) { @@ -798,6 +951,10 @@ export function createRuntimeServerClient( sendEntityMessage, createAttachment, readAttachment, + createMarkdownDocument, + readMarkdownDocument, + writeMarkdownDocument, + editMarkdownDocument, spawnEntity, getEntity, ensureSharedStateStream, diff --git a/packages/agents-runtime/src/tools.ts b/packages/agents-runtime/src/tools.ts index aa0f8741f8..449be9d226 100644 --- a/packages/agents-runtime/src/tools.ts +++ b/packages/agents-runtime/src/tools.ts @@ -7,3 +7,4 @@ export { createFetchUrlTool } from './tools/fetch-url' export { createScheduleTools } from './tools/schedules' export { createEventSourceTools } from './tools/event-sources' export { createSendTool } from './tools/send' +export { createMarkdownDocumentTools } from './tools/markdown-docs' diff --git a/packages/agents-runtime/src/tools/markdown-docs.ts b/packages/agents-runtime/src/tools/markdown-docs.ts new file mode 100644 index 0000000000..b2487ad54d --- /dev/null +++ b/packages/agents-runtime/src/tools/markdown-docs.ts @@ -0,0 +1,202 @@ +import { createTwoFilesPatch } from 'diff' +import { Type } from '@sinclair/typebox' +import type { AgentTool, ProcessWakeConfig } from '../types' + +type ElectricToolContext = Parameters< + NonNullable +>[0] + +function docLabel(id: string): string { + return `markdown-doc:${id}` +} + +export function createMarkdownDocumentTools( + context: ElectricToolContext +): Array { + const readDocs = new Map() + + return [ + { + name: `create_markdown_doc`, + label: `Create Markdown Doc`, + description: `Create a collaborative markdown document, persist it as Yjs updates, and add it to this entity's manifest so users can open it in the app. This is not a filesystem file.`, + parameters: Type.Object({ + title: Type.String({ description: `Document title shown in the UI.` }), + content: Type.Optional( + Type.String({ description: `Initial markdown content.` }) + ), + id: Type.Optional( + Type.String({ + description: `Optional stable document id. Use letters, numbers, hyphens, or underscores.`, + }) + ), + }), + execute: async (_toolCallId, params) => { + const { id, title, content } = params as { + id?: string + title: string + content?: string + } + const result = await context.createMarkdownDocument({ + id, + title, + content, + }) + readDocs.set(result.document.id, content ?? ``) + return { + content: [ + { + type: `text` as const, + text: `Created markdown document ${result.document.id}: ${result.document.title}`, + }, + ], + details: { document: result.document, txid: result.txid }, + } + }, + }, + { + name: `read_markdown_doc`, + label: `Read Markdown Doc`, + description: `Read the current plain markdown content from a collaborative app document, not from the filesystem.`, + parameters: Type.Object({ + id: Type.String({ description: `Document id.` }), + }), + execute: async (_toolCallId, params) => { + const { id } = params as { id: string } + const result = await context.readMarkdownDocument({ id }) + readDocs.set(id, result.content) + return { + content: [ + { + type: `text` as const, + text: result.content, + }, + ], + details: { + document: result.document, + bytes: new TextEncoder().encode(result.content).length, + }, + } + }, + }, + { + name: `write_markdown_doc`, + label: `Write Markdown Doc`, + description: `Replace the full content of a collaborative app markdown document. This does not write a filesystem file.`, + parameters: Type.Object({ + id: Type.String({ description: `Document id.` }), + content: Type.String({ description: `Full markdown content.` }), + }), + execute: async (_toolCallId, params) => { + const { id, content } = params as { id: string; content: string } + const before = + readDocs.get(id) ?? + (await context.readMarkdownDocument({ id })).content + const result = await context.writeMarkdownDocument({ id, content }) + readDocs.set(id, content) + const diff = createTwoFilesPatch( + docLabel(id), + docLabel(id), + before, + content, + undefined, + undefined, + { context: 3 } + ) + return { + content: [ + { + type: `text` as const, + text: `Wrote markdown document ${id}`, + }, + ], + details: { document: result.document, txid: result.txid, diff }, + } + }, + }, + { + name: `edit_markdown_doc`, + label: `Edit Markdown Doc`, + description: `Replace text in a collaborative app markdown document, not a filesystem file. The document must be read with read_markdown_doc earlier in this wake. By default old_string must occur exactly once; set replace_all to true to replace every occurrence.`, + parameters: Type.Object({ + id: Type.String({ description: `Document id.` }), + old_string: Type.String({ + description: `Literal markdown text to find. Must be unique unless replace_all is true.`, + }), + new_string: Type.String({ description: `Replacement markdown text.` }), + replace_all: Type.Optional( + Type.Boolean({ description: `Replace every occurrence.` }) + ), + }), + execute: async (_toolCallId, params) => { + const { id, old_string, new_string, replace_all } = params as { + id: string + old_string: string + new_string: string + replace_all?: boolean + } + const before = readDocs.get(id) + if (before === undefined) { + return { + content: [ + { + type: `text` as const, + text: `Document ${id} has not been read in this wake; call read_markdown_doc first.`, + }, + ], + details: { replacements: 0 }, + } + } + + const matches = before.split(old_string).length - 1 + if (matches === 0) { + return { + content: [ + { type: `text` as const, text: `Error: old_string not found` }, + ], + details: { replacements: 0 }, + } + } + if (!replace_all && matches > 1) { + return { + content: [ + { + type: `text` as const, + text: `Error: found ${matches} matches for old_string; pass replace_all=true or provide a more specific old_string.`, + }, + ], + details: { replacements: 0 }, + } + } + + const result = await context.editMarkdownDocument({ + id, + oldString: old_string, + newString: new_string, + replaceAll: replace_all, + }) + readDocs.set(id, result.content) + const diff = createTwoFilesPatch( + docLabel(id), + docLabel(id), + before, + result.content, + undefined, + undefined, + { context: 3 } + ) + return { + content: [ + { + type: `text` as const, + text: `Edited markdown document ${id}: ${matches} replacement${ + matches === 1 ? `` : `s` + }`, + }, + ], + details: { replacements: matches, document: result.document, diff }, + } + }, + }, + ] +} diff --git a/packages/agents-runtime/src/types.ts b/packages/agents-runtime/src/types.ts index 6ab10402f5..e935c01909 100644 --- a/packages/agents-runtime/src/types.ts +++ b/packages/agents-runtime/src/types.ts @@ -40,6 +40,7 @@ import type { ManifestAttachmentEntry as EntityManifestAttachmentEntry, ManifestChildEntry as EntityManifestChildEntry, ManifestContextEntry as EntityManifestContextEntry, + ManifestDocumentEntry as EntityManifestDocumentEntry, ManifestCronScheduleEntry as EntityManifestCronScheduleEntry, ManifestEffectEntry as EntityManifestEffectEntry, ManifestFutureSendScheduleEntry as EntityManifestFutureSendScheduleEntry, @@ -317,6 +318,7 @@ export type ManifestEntry = EntityManifest export type ManifestAttachmentEntry = EntityManifestAttachmentEntry export type ManifestChildEntry = EntityManifestChildEntry export type ManifestContextEntry = EntityManifestContextEntry +export type ManifestDocumentEntry = EntityManifestDocumentEntry export type ManifestCronScheduleEntry = EntityManifestCronScheduleEntry export type ManifestEffectEntry = EntityManifestEffectEntry export type ManifestFutureSendScheduleEntry = @@ -741,6 +743,30 @@ export interface ProcessWakeConfig { unsubscribeFromEventSource: (opts: { id: string }) => Promise<{ txid: string }> + createMarkdownDocument: (opts: { + id?: string + title: string + content?: string + meta?: Record + }) => Promise<{ txid: string; document: ManifestDocumentEntry }> + readMarkdownDocument: (opts: { + id: string + }) => Promise<{ document: ManifestDocumentEntry; content: string }> + writeMarkdownDocument: (opts: { id: string; content: string }) => Promise<{ + txid: string + document: ManifestDocumentEntry + content: string + }> + editMarkdownDocument: (opts: { + id: string + oldString: string + newString: string + replaceAll?: boolean + }) => Promise<{ + txid: string + document: ManifestDocumentEntry + content: string + }> }) => Array | Promise> /** Optional shutdown signal to end idle waits during host teardown. */ shutdownSignal?: AbortSignal diff --git a/packages/agents-runtime/test/markdown-docs-tools.test.ts b/packages/agents-runtime/test/markdown-docs-tools.test.ts new file mode 100644 index 0000000000..37b1f3a19a --- /dev/null +++ b/packages/agents-runtime/test/markdown-docs-tools.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it, vi } from 'vitest' +import { createMarkdownDocumentTools } from '../src/tools/markdown-docs' + +function createToolContext() { + const document = { + key: `document:notes`, + kind: `document`, + id: `notes`, + docPath: `agents/chat/session/documents/notes`, + streamPath: `/v1/yjs/default/docs/agents/chat/session/documents/notes`, + mimeType: `application/vnd.electric-agents.markdown-yjs`, + contentMimeType: `text/markdown`, + title: `Notes`, + createdAt: `2026-06-07T00:00:00.000Z`, + } as const + let content = `# Notes\n\nFirst line\n` + return { + context: { + entityUrl: `/chat/session`, + entityType: `chat`, + args: {}, + db: { collections: { manifests: { toArray: [] } } }, + events: [], + createMarkdownDocument: vi.fn( + async (opts: { id?: string; title: string; content?: string }) => { + content = opts.content ?? `` + return { + txid: `tx-create`, + document: { + ...document, + id: opts.id ?? document.id, + title: opts.title, + }, + } + } + ), + readMarkdownDocument: vi.fn(async () => ({ document, content })), + writeMarkdownDocument: vi.fn( + async (opts: { id: string; content: string }) => { + content = opts.content + return { txid: `tx-write`, document, content } + } + ), + editMarkdownDocument: vi.fn( + async (opts: { + oldString: string + newString: string + replaceAll?: boolean + }) => { + content = opts.replaceAll + ? content.split(opts.oldString).join(opts.newString) + : content.replace(opts.oldString, opts.newString) + return { txid: `tx-edit`, document, content } + } + ), + upsertCronSchedule: vi.fn(), + upsertFutureSendSchedule: vi.fn(), + deleteSchedule: vi.fn(), + listEventSources: vi.fn(), + subscribeToEventSource: vi.fn(), + unsubscribeFromEventSource: vi.fn(), + } as any, + getContent: () => content, + } +} + +describe(`markdown document tools`, () => { + it(`requires read_markdown_doc before edit_markdown_doc`, async () => { + const { context } = createToolContext() + const edit = createMarkdownDocumentTools(context).find( + (tool) => tool.name === `edit_markdown_doc` + )! + + const result = await edit.execute(`tool-edit`, { + id: `notes`, + old_string: `First`, + new_string: `Second`, + }) + + expect(context.editMarkdownDocument).not.toHaveBeenCalled() + expect(result.details).toMatchObject({ replacements: 0 }) + expect(result.content[0]).toMatchObject({ + type: `text`, + text: expect.stringContaining(`read_markdown_doc first`), + }) + }) + + it(`edits a read document and returns a diff`, async () => { + const { context, getContent } = createToolContext() + const tools = createMarkdownDocumentTools(context) + const read = tools.find((tool) => tool.name === `read_markdown_doc`)! + const edit = tools.find((tool) => tool.name === `edit_markdown_doc`)! + + await read.execute(`tool-read`, { id: `notes` }) + const result = await edit.execute(`tool-edit`, { + id: `notes`, + old_string: `First line`, + new_string: `Second line`, + }) + + expect(context.editMarkdownDocument).toHaveBeenCalledWith({ + id: `notes`, + oldString: `First line`, + newString: `Second line`, + replaceAll: undefined, + }) + expect(getContent()).toContain(`Second line`) + expect(result.details).toMatchObject({ replacements: 1 }) + expect(String((result.details as any).diff)).toContain(`Second line`) + }) +}) diff --git a/packages/agents-server-ui/package.json b/packages/agents-server-ui/package.json index 75939f01e0..9690d55efb 100644 --- a/packages/agents-server-ui/package.json +++ b/packages/agents-server-ui/package.json @@ -15,8 +15,12 @@ }, "dependencies": { "@base-ui/react": "^1.4.1", + "@codemirror/lang-markdown": "^6.5.0", + "@codemirror/state": "^6.6.0", + "@codemirror/view": "^6.43.0", "@durable-streams/client": "^0.2.6", "@durable-streams/state": "^0.3.1", + "@durable-streams/y-durable-streams": "0.2.7", "@electric-ax/agents-runtime": "workspace:*", "@handlewithcare/react-prosemirror": "^3.0.6", "@streamdown/math": "^1.0.2", @@ -26,8 +30,10 @@ "@tanstack/react-router": "^1.167.4", "@tanstack/react-table": "^8.21.3", "@tanstack/react-virtual": "^3.13.23", + "codemirror": "^6.0.1", "fractional-indexing": "^3.2.0", "katex": "^0.16.45", + "lib0": "^0.2.99", "lucide-react": "^0.561.0", "mermaid": "^11.14.0", "nanoid": "^3.3.11", @@ -41,6 +47,9 @@ "react-reconciler": "0.32.0", "shiki": "^4.0.2", "streamdown": "^2.5.0", + "y-codemirror.next": "0.3.5", + "y-protocols": "^1.0.6", + "yjs": "^13.6.26", "zod": "^3.25.76" }, "devDependencies": { diff --git a/packages/agents-server-ui/src/components/EntityContextDrawer.tsx b/packages/agents-server-ui/src/components/EntityContextDrawer.tsx index 7f93db305d..d25656820d 100644 --- a/packages/agents-server-ui/src/components/EntityContextDrawer.tsx +++ b/packages/agents-server-ui/src/components/EntityContextDrawer.tsx @@ -50,6 +50,7 @@ type DrawerEntry = action: | { kind: `entity`; url: string } | { kind: `state`; sourceId: string } + | { kind: `document`; id: string } | { kind: `inspect` } entity: DrawerEntity | null } @@ -218,11 +219,21 @@ export function EntityContextDrawer({ }) } + const openDocument = (documentId: string, side = false): void => { + helpers.openEntity(entity.url, { + viewId: `markdown-doc`, + viewParams: { doc: documentId }, + ...(side ? { target: { tileId, position: `split-right` as const } } : {}), + }) + } + const handleEntry = (entry: DrawerEntry): void => { if (entry.action.kind === `entity`) { openEntity(entry.action.url) } else if (entry.action.kind === `state`) { openStateInspector(entry.action.sourceId) + } else if (entry.action.kind === `document`) { + openDocument(entry.action.id) } else { setInspectTarget({ title: entry.title, value: entry.manifest }) } @@ -233,6 +244,8 @@ export function EntityContextDrawer({ openEntity(entry.action.url, true) } else if (entry.action.kind === `state`) { openStateInspector(entry.action.sourceId, true) + } else if (entry.action.kind === `document`) { + openDocument(entry.action.id, true) } } @@ -565,6 +578,8 @@ function manifestKindLabel(manifest: Manifest): string { return `Effect` case `attachment`: return `Attachment` + case `document`: + return `Markdown document` case `context`: return `Context` case `schedule`: @@ -682,6 +697,18 @@ function createManifestEntry( entity: null, } + case `document`: + return { + key: manifest.key, + groupKey: `document`, + groupLabel: `Documents`, + title: manifest.title, + meta: manifest.id, + manifest, + action: { kind: `document`, id: manifest.id }, + entity: null, + } + case `context`: return { key: manifest.key, diff --git a/packages/agents-server-ui/src/components/EntityTimeline.tsx b/packages/agents-server-ui/src/components/EntityTimeline.tsx index 214b806066..4b5fa2ef52 100644 --- a/packages/agents-server-ui/src/components/EntityTimeline.tsx +++ b/packages/agents-server-ui/src/components/EntityTimeline.tsx @@ -21,8 +21,10 @@ import { Database, ExternalLink, FileJson, + FileText, GitBranch, Radio, + SplitSquareHorizontal, } from 'lucide-react' import { loadTimelineRowHeights, @@ -573,6 +575,7 @@ function isTimelineFindMatch( function ManifestTimelineRow({ manifest, entityUrl, + tileId, entityStatus, }: { manifest: Manifest @@ -584,6 +587,8 @@ function ManifestTimelineRow({ const navigate = useNavigate() const entityTarget = getManifestEntityUrl(manifest) const stateSourceId = getManifestStateSourceId(manifest) + const documentId = manifest.kind === `document` ? manifest.id : null + const splitTargetTileId = tileId ?? workspace?.helpers.activeTileId ?? null const isEntity = entityTarget !== null const title = manifestTitle(manifest) const meta = manifestMeta(manifest) @@ -612,13 +617,62 @@ function ManifestTimelineRow({ }) }, [entityUrl, stateSourceId, workspace]) + const openDocument = useCallback(() => { + if (!entityUrl || !documentId || !workspace) return + workspace.helpers.openEntity(entityUrl, { + viewId: `markdown-doc`, + viewParams: { doc: documentId }, + }) + }, [documentId, entityUrl, workspace]) + + const splitDocumentRight = useCallback(() => { + if (!entityUrl || !documentId || !workspace) return + if (!splitTargetTileId) return + workspace.helpers.openEntity(entityUrl, { + viewId: `markdown-doc`, + viewParams: { doc: documentId }, + target: { tileId: splitTargetTileId, position: `split-right` }, + }) + }, [documentId, entityUrl, splitTargetTileId, workspace]) + const statusBadge = entityStatus ? ( {entityStatus} ) : null - const openAction = stateSourceId ? ( + const openAction = documentId ? ( + <> + + + + + + + + + + + + ) : stateSourceId ? ( - {isEntity || stateSourceId ? ( + {isEntity || stateSourceId || documentId ? ( details ) : ( <> @@ -726,6 +780,8 @@ function manifestKindLabel(manifest: Manifest): string { return `Effect` case `attachment`: return `Attachment` + case `document`: + return `Markdown document` case `context`: return `Context` case `schedule`: @@ -742,6 +798,7 @@ function manifestTitle(manifest: Manifest): string { case `shared-state`: case `effect`: case `attachment`: + case `document`: case `context`: case `schedule`: return manifest.id @@ -760,6 +817,8 @@ function manifestMeta(manifest: Manifest): string { return manifest.function_ref case `attachment`: return `${manifest.mimeType} ยท ${manifest.status}` + case `document`: + return manifest.title case `context`: return `${Object.keys(manifest.attrs).length} attrs` case `schedule`: @@ -808,6 +867,12 @@ function manifestDetails( value: `${manifest.subject.type}:${manifest.subject.key}`, }, ] + case `document`: + return [ + { label: `Title`, value: manifest.title }, + { label: `MIME`, value: manifest.contentMimeType }, + { label: `Path`, value: manifest.docPath }, + ] case `context`: return [ { label: `Name`, value: manifest.name }, @@ -831,6 +896,7 @@ function manifestIcon(manifest: Manifest) { if (getManifestStateSourceId(manifest)) return Database if (getManifestEntityUrl(manifest)) return GitBranch if (manifest.kind === `schedule`) return Radio + if (manifest.kind === `document`) return FileText if (manifest.kind === `attachment`) return FileJson return FileJson } diff --git a/packages/agents-server-ui/src/components/views/MarkdownDocumentView.module.css b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.module.css new file mode 100644 index 0000000000..bbb7dacced --- /dev/null +++ b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.module.css @@ -0,0 +1,94 @@ +.root { + display: flex; + min-height: 0; + height: 100%; + flex-direction: column; + background: var(--color-panel, #fff); +} + +.bar { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + padding: 8px 12px; + border-bottom: 1px solid var(--color-border, #d7dce2); + background: var(--color-bg-subtle, #f7f8fa); +} + +.title { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + font-size: 13px; + font-weight: 600; +} + +.status { + flex: 0 0 auto; + color: var(--color-text-muted, #67717f); + font-size: 12px; +} + +.editor { + min-height: 0; + flex: 1; +} + +.editor :global(.cm-editor) { + height: 100%; + font-size: 14px; +} + +.editor :global(.cm-scroller) { + font-family: + ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, 'Liberation Mono', + 'Courier New', monospace; +} + +.presence { + display: flex; + align-items: center; + gap: 6px; + min-width: 0; +} + +.presenceDot { + width: 8px; + height: 8px; + flex: 0 0 auto; + border-radius: 999px; +} + +.empty { + padding: 16px; + color: var(--color-text-muted, #67717f); + font-size: 13px; +} + +:global(.yRemoteSelection) { + opacity: 0.35; +} + +:global(.yRemoteSelectionHead) { + position: absolute; + box-sizing: border-box; + height: 1.2em; + border-left: 2px solid; +} + +:global(.yRemoteSelectionHead)::after { + position: absolute; + top: -1.05em; + left: -2px; + z-index: 10; + padding: 1px 4px; + border-radius: 3px; + color: white; + content: attr(data-user-name); + font-family: system-ui, sans-serif; + font-size: 10px; + line-height: 1.2; + white-space: nowrap; +} diff --git a/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx new file mode 100644 index 0000000000..1eb8ccf936 --- /dev/null +++ b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx @@ -0,0 +1,198 @@ +import { useEffect, useMemo, useRef, useState } from 'react' +import { markdown } from '@codemirror/lang-markdown' +import { EditorState } from '@codemirror/state' +import { EditorView, basicSetup } from 'codemirror' +import { keymap } from '@codemirror/view' +import { YjsProvider } from '@durable-streams/y-durable-streams' +import { yCollab, yUndoManagerKeymap } from 'y-codemirror.next' +import { Awareness } from 'y-protocols/awareness' +import * as Y from 'yjs' +import { useCurrentPrincipal } from '../../hooks/useCurrentPrincipal' +import { getConfiguredServerHeaders, serverFetch } from '../../lib/auth-fetch' +import { principalKeyFromInput } from '../../lib/principals' +import styles from './MarkdownDocumentView.module.css' +import type { EntityViewProps } from '../../lib/workspace/viewRegistry' +import type { ManifestDocumentEntry } from '@electric-ax/agents-runtime/client' + +type DocumentResponse = { + document: ManifestDocumentEntry + content: string +} + +function entityApiUrl(baseUrl: string, entityUrl: string, suffix: string): URL { + const url = new URL(baseUrl) + url.pathname = `${url.pathname.replace(/\/+$/, ``)}/_electric/entities${entityUrl}${suffix}` + return url +} + +function colorFor(value: string): { color: string; light: string } { + const colors = [ + [`#2563eb`, `#2563eb33`], + [`#059669`, `#05966933`], + [`#dc2626`, `#dc262633`], + [`#7c3aed`, `#7c3aed33`], + [`#c2410c`, `#c2410c33`], + [`#0f766e`, `#0f766e33`], + ] as const + let hash = 0 + for (let i = 0; i < value.length; i += 1) { + hash = (hash * 31 + value.charCodeAt(i)) >>> 0 + } + const [color, light] = colors[hash % colors.length]! + return { color, light } +} + +function providerBaseUrl(baseUrl: string, streamPath: string): string { + const docsIndex = streamPath.indexOf(`/docs/`) + const prefix = docsIndex >= 0 ? streamPath.slice(0, docsIndex) : streamPath + const url = new URL(baseUrl) + url.pathname = `${url.pathname.replace(/\/+$/, ``)}${prefix}` + return url.toString().replace(/\/+$/, ``) +} + +export function MarkdownDocumentView({ + baseUrl, + entityUrl, + viewParams, +}: EntityViewProps): React.ReactElement { + const documentId = viewParams?.doc ?? null + const editorRef = useRef(null) + const editorViewRef = useRef(null) + const [documentEntry, setDocumentEntry] = + useState(null) + const [status, setStatus] = useState< + `loading` | `connecting` | `connected` | `disconnected` | `error` + >(`loading`) + const [remoteUsers, setRemoteUsers] = useState>([]) + const { principal } = useCurrentPrincipal() + + useEffect(() => { + let cancelled = false + setDocumentEntry(null) + setStatus(documentId ? `loading` : `error`) + if (!documentId) return + const url = entityApiUrl( + baseUrl, + entityUrl, + `/documents/${encodeURIComponent(documentId)}` + ) + serverFetch(url, { headers: { accept: `application/json` } }) + .then(async (response) => { + if (!response.ok) { + throw new Error(`Document request failed (${response.status})`) + } + return (await response.json()) as DocumentResponse + }) + .then((result) => { + if (!cancelled) setDocumentEntry(result.document) + }) + .catch(() => { + if (!cancelled) setStatus(`error`) + }) + return () => { + cancelled = true + } + }, [baseUrl, entityUrl, documentId]) + + const principalLabel = useMemo( + () => principalKeyFromInput(principal) ?? principal, + [principal] + ) + + useEffect(() => { + if (!editorRef.current || !documentEntry) return + + const ydoc = new Y.Doc() + const awareness = new Awareness(ydoc) + const userColor = colorFor(principalLabel) + awareness.setLocalStateField(`user`, { + name: principalLabel, + color: userColor.color, + colorLight: userColor.light, + }) + + const docUrl = new URL( + `${providerBaseUrl(baseUrl, documentEntry.streamPath)}/docs/${documentEntry.docPath}` + ) + const provider = new YjsProvider({ + doc: ydoc, + baseUrl: providerBaseUrl(baseUrl, documentEntry.streamPath), + docId: documentEntry.docPath, + awareness, + headers: getConfiguredServerHeaders(docUrl), + liveMode: `sse`, + }) + const ytext = ydoc.getText(`markdown`) + const state = EditorState.create({ + doc: ytext.toString(), + extensions: [ + keymap.of([...yUndoManagerKeymap]), + basicSetup, + markdown(), + EditorView.lineWrapping, + yCollab(ytext, awareness), + ], + }) + const view = new EditorView({ state, parent: editorRef.current }) + editorViewRef.current = view + + const updateRemoteUsers = (): void => { + const names: Array = [] + awareness.getStates().forEach((state, clientId) => { + if (clientId === awareness.clientID) return + const user = (state as { user?: { name?: string } }).user + if (user?.name) names.push(user.name) + }) + setRemoteUsers(names) + } + const statusHandler = (next: typeof status): void => setStatus(next) + provider.on(`status`, statusHandler) + awareness.on(`change`, updateRemoteUsers) + provider.connect() + setStatus(`connecting`) + + return () => { + provider.off(`status`, statusHandler) + awareness.off(`change`, updateRemoteUsers) + provider.destroy() + editorViewRef.current?.destroy() + editorViewRef.current = null + ydoc.destroy() + setRemoteUsers([]) + } + }, [baseUrl, documentEntry, principalLabel]) + + if (!documentId) { + return
No document selected.
+ } + + return ( +
+
+
+ {documentEntry?.title ?? `Markdown document`} +
+
+ {status} + {remoteUsers.slice(0, 3).map((name) => { + const color = colorFor(name) + return ( + + + {name} + + ) + })} +
+
+ {status === `error` ? ( +
Document could not be opened.
+ ) : ( +
+ )} +
+ ) +} diff --git a/packages/agents-server-ui/src/components/workspace/Workspace.tsx b/packages/agents-server-ui/src/components/workspace/Workspace.tsx index 0937b89957..61ac255f76 100644 --- a/packages/agents-server-ui/src/components/workspace/Workspace.tsx +++ b/packages/agents-server-ui/src/components/workspace/Workspace.tsx @@ -15,6 +15,7 @@ import type { LocalRuntimeStatus, ServerConnectionStatus, } from '../../lib/server-connection' +import type { TileViewParams } from '../../lib/workspace/types' import type { ViewId } from '../../lib/workspace/viewRegistry' /** @@ -37,6 +38,7 @@ export function Workspace(): React.ReactElement { const search = useSearch({ strict: false }) as { view?: string source?: string + doc?: string layout?: string } const navigate = useNavigate() @@ -44,6 +46,7 @@ export function Workspace(): React.ReactElement { const entityUrl = splat ? `/${splat}` : null const requestedViewId = (search.view as ViewId | undefined) ?? null const requestedSource = (search.source as string | undefined) ?? null + const requestedDoc = (search.doc as string | undefined) ?? null const layoutParam = (search.layout as string | undefined) ?? null // ---- ?layout= import ------------------------------------------- @@ -73,6 +76,7 @@ export function Workspace(): React.ReactElement { search: { ...(requestedViewId ? { view: requestedViewId } : {}), ...(requestedSource ? { source: requestedSource } : {}), + ...(requestedDoc ? { doc: requestedDoc } : {}), }, replace: true, }) @@ -140,14 +144,21 @@ export function Workspace(): React.ReactElement { const availableViews = entity ? listViews(entity) : [] const defaultViewId = availableViews[0]?.id ?? `chat` const desiredViewId = - requestedViewId && availableViews.some((v) => v.id === requestedViewId) + requestedViewId === `markdown-doc` && requestedDoc ? requestedViewId - : defaultViewId - const desiredViewParams = + : requestedViewId && + availableViews.some((v) => v.id === requestedViewId) + ? requestedViewId + : defaultViewId + const desiredViewParams: TileViewParams | undefined = desiredViewId === `state-explorer` && requestedSource ? { source: requestedSource } - : undefined - const key = `${entityUrl}::${desiredViewId}::${requestedSource ?? ``}` + : desiredViewId === `markdown-doc` && requestedDoc + ? { doc: requestedDoc } + : undefined + const key = `${entityUrl}::${desiredViewId}::${viewParamsKey( + desiredViewParams + )}` if (lastSyncedKey.current === key) return const tiles = listTiles(workspace.root) @@ -157,7 +168,7 @@ export function Workspace(): React.ReactElement { (t) => t.entityUrl === entityUrl && t.viewId === desiredViewId && - (desiredViewParams?.source ?? ``) === (t.viewParams?.source ?? ``) + viewParamsKey(desiredViewParams) === viewParamsKey(t.viewParams) ) if (exactMatch) { helpers.setActiveTile(exactMatch.id) @@ -189,6 +200,7 @@ export function Workspace(): React.ReactElement { entityUrl, requestedViewId, requestedSource, + requestedDoc, entity, workspace.root, helpers, @@ -209,7 +221,7 @@ export function Workspace(): React.ReactElement { const expectedKey = tile.entityUrl === null ? `::${tile.viewId}` - : `${tile.entityUrl}::${tile.viewId}::${tile.viewParams?.source ?? ``}` + : `${tile.entityUrl}::${tile.viewId}::${viewParamsKey(tile.viewParams)}` if (lastSyncedKey.current === expectedKey) return lastSyncedKey.current = expectedKey if (tile.entityUrl === null) { @@ -222,6 +234,7 @@ export function Workspace(): React.ReactElement { search: { ...(tile.viewId === `chat` ? {} : { view: tile.viewId }), ...(tile.viewParams?.source ? { source: tile.viewParams.source } : {}), + ...(tile.viewParams?.doc ? { doc: tile.viewParams.doc } : {}), }, replace: true, }) @@ -306,6 +319,14 @@ export function Workspace(): React.ReactElement { ) } +function viewParamsKey(params: Record | undefined): string { + if (!params) return `` + return Object.entries(params) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([key, value]) => `${key}=${value}`) + .join(`&`) +} + function getRemoteStatus( connected: boolean, status: ServerConnectionStatus | undefined diff --git a/packages/agents-server-ui/src/lib/workspace/registerViews.ts b/packages/agents-server-ui/src/lib/workspace/registerViews.ts index e887aeb9bb..05cd984167 100644 --- a/packages/agents-server-ui/src/lib/workspace/registerViews.ts +++ b/packages/agents-server-ui/src/lib/workspace/registerViews.ts @@ -1,8 +1,9 @@ -import { Database, MessageSquare, SquarePen } from 'lucide-react' +import { Database, FileText, MessageSquare, SquarePen } from 'lucide-react' import { registerView } from './viewRegistry' import { NEW_SESSION_VIEW_ID } from './types' import { ChatView } from '../../components/views/ChatView' import { StateExplorerView } from '../../components/views/StateExplorerView' +import { MarkdownDocumentView } from '../../components/views/MarkdownDocumentView' import { NewSessionView } from '../../components/views/NewSessionView' /** @@ -32,6 +33,17 @@ registerView({ Component: StateExplorerView, }) +registerView({ + kind: `entity`, + id: `markdown-doc`, + label: `Markdown Doc`, + shortLabel: `Doc`, + icon: FileText, + description: `Collaborative markdown document editor`, + isAvailable: () => false, + Component: MarkdownDocumentView, +}) + /** * Standalone view: "new session". Doesn't belong to an entity, so it * never appears in the per-entity view-switcher. The workspace mounts diff --git a/packages/agents-server/package.json b/packages/agents-server/package.json index 361c8378b3..48c7ebc68b 100644 --- a/packages/agents-server/package.json +++ b/packages/agents-server/package.json @@ -59,11 +59,13 @@ "drizzle-orm": "^0.44.0", "fastq": "^1.20.1", "itty-router": "^5.0.23", + "lib0": "^0.2.99", "lmdb": "^3.5.1", "pino": "^10.3.1", "pino-pretty": "^13.0.0", "postgres": "^3.4.0", - "undici": "^7.24.7" + "undici": "^7.24.7", + "yjs": "^13.6.26" }, "devDependencies": { "@electric-ax/agents": "workspace:*", diff --git a/packages/agents-server/src/entity-manager.ts b/packages/agents-server/src/entity-manager.ts index c16b4cd08e..adfc9cc655 100644 --- a/packages/agents-server/src/entity-manager.ts +++ b/packages/agents-server/src/entity-manager.ts @@ -1,5 +1,6 @@ import { createHash, randomUUID } from 'node:crypto' import fastq from 'fastq' +import * as Y from 'yjs' import { COMPOSER_INPUT_MESSAGE_TYPE, assertTags, @@ -8,7 +9,9 @@ import { getSharedStateStreamPath, getNextCronFireAt, eventSourceSubscriptionManifestKey, + getEntityMarkdownDocumentUrlPath, manifestChildKey, + manifestMarkdownDocumentKey, manifestSharedStateKey, manifestSourceKey, resolveCronScheduleSpec, @@ -52,6 +55,17 @@ import { } from './manifest-side-effects.js' import { DEFAULT_TENANT_ID } from './tenant.js' import { ATTR, withSpan } from './tracing.js' +import { + MARKDOWN_DOCUMENT_CONTENT_MIME, + MARKDOWN_DOCUMENT_TRANSPORT_MIME, + assertMarkdownDocumentMatchesEntity, + frameYjsUpdate, + getMarkdownDocumentDocPath, + getMarkdownDocumentUpdateStreamPath, + markdownText, + readMarkdownYDoc, + replaceMarkdownText, +} from './markdown-documents.js' import type { queueAsPromised } from 'fastq' import type { SchedulerClient } from './scheduler.js' import type { WakeEvalResult, WakeRegistry } from './wake-registry.js' @@ -157,6 +171,41 @@ type ManifestAttachmentEntry = { meta?: Record } +export interface CreateMarkdownDocumentRequest { + id?: string + title: string + content?: string + createdBy?: string + meta?: Record +} + +export interface UpdateMarkdownDocumentRequest { + content: string + updatedBy?: string +} + +export interface EditMarkdownDocumentRequest { + oldString: string + newString: string + replaceAll?: boolean + updatedBy?: string +} + +export type ManifestMarkdownDocumentEntry = { + key: string + kind: `document` + id: string + docPath: string + streamPath: string + mimeType: typeof MARKDOWN_DOCUMENT_TRANSPORT_MIME + contentMimeType: typeof MARKDOWN_DOCUMENT_CONTENT_MIME + title: string + createdAt: string + createdBy?: string + updatedAt?: string + meta?: Record +} + function createInitialQueuePosition(date: Date): string { return `${String(date.getTime()).padStart(16, `0`)}:a0` } @@ -187,6 +236,7 @@ type ForkStateSnapshot = { childStatusesByEntity: Map>> replayWatermarksByEntity: Map>> sharedStateIds: Set + markdownDocumentDocPaths: Set } type ForkResult = { @@ -215,6 +265,16 @@ function manifestAttachmentKey(id: string): string { return `attachment:${id}` } +function validateMarkdownDocumentId(id: string): void { + if (!id || !/^[A-Za-z0-9_-]+$/.test(id)) { + throw new ElectricAgentsError( + ErrCodeInvalidRequest, + `document id must contain only letters, numbers, underscores, or hyphens`, + 400 + ) + } +} + function getEntityAttachmentStreamPath( entityUrl: string, attachmentId: string @@ -970,6 +1030,7 @@ export class EntityManager { childStatuses: Map> replayWatermarks: Map> sharedStateIds: Set + markdownDocumentDocPaths: Set } | undefined if (opts.forkPointer) { @@ -987,8 +1048,13 @@ export class EntityManager { const filteredEvents = flat.slice(0, target) const rootManifests = this.reduceStateRows(filteredEvents, `manifest`) const sharedStateIds = new Set() + const markdownDocumentDocPaths = new Set() for (const manifest of rootManifests.values()) { this.collectSharedStateIds(manifest, sharedStateIds) + this.collectMarkdownDocumentDocPaths( + manifest, + markdownDocumentDocPaths + ) } preFilteredRoot = { manifests: rootManifests, @@ -998,6 +1064,7 @@ export class EntityManager { `replay_watermark` ), sharedStateIds, + markdownDocumentDocPaths, } } @@ -1046,6 +1113,9 @@ export class EntityManager { for (const id of preFilteredRoot.sharedStateIds) { snapshot.sharedStateIds.add(id) } + for (const docPath of preFilteredRoot.markdownDocumentDocPaths) { + snapshot.markdownDocumentDocPaths.add(docPath) + } } const suffix = randomUUID().slice(0, 8) @@ -1073,7 +1143,14 @@ export class EntityManager { ) this.addForkLocks( this.forkWriteLockedStreams, - [...snapshot.sharedStateIds].map((id) => getSharedStateStreamPath(id)), + [ + ...[...snapshot.sharedStateIds].map((id) => + getSharedStateStreamPath(id) + ), + ...[...snapshot.markdownDocumentDocPaths].map((docPath) => + getMarkdownDocumentUpdateStreamPath(this.tenantId, docPath) + ), + ], writeStreamLocks ) @@ -1124,6 +1201,34 @@ export class EntityManager { } } + for (const plan of entityPlans) { + const manifests = + snapshot.manifestsByEntity.get(plan.source.url) ?? new Map() + for (const manifest of manifests.values()) { + if ( + manifest.kind !== `document` || + typeof manifest.docPath !== `string` || + typeof manifest.id !== `string` + ) { + continue + } + const forkDocPath = getMarkdownDocumentDocPath( + plan.fork.url, + manifest.id + ) + const sourcePath = getMarkdownDocumentUpdateStreamPath( + this.tenantId, + manifest.docPath + ) + const forkPath = getMarkdownDocumentUpdateStreamPath( + this.tenantId, + forkDocPath + ) + await this.streamClient.fork(forkPath, sourcePath) + createdStreams.push(forkPath) + } + } + for (const plan of entityPlans) { const reconciliation = this.buildForkReconciliation( plan, @@ -1557,6 +1662,7 @@ export class EntityManager { Map> >() const sharedStateIds = new Set() + const markdownDocumentDocPaths = new Set() for (const entity of entitiesToFork) { const events = await this.streamClient.readJson>( @@ -1572,6 +1678,7 @@ export class EntityManager { for (const manifest of manifests.values()) { this.collectSharedStateIds(manifest, sharedStateIds) + this.collectMarkdownDocumentDocPaths(manifest, markdownDocumentDocPaths) } } @@ -1580,6 +1687,7 @@ export class EntityManager { childStatusesByEntity, replayWatermarksByEntity, sharedStateIds, + markdownDocumentDocPaths, } } @@ -1631,6 +1739,16 @@ export class EntityManager { } } + private collectMarkdownDocumentDocPaths( + manifest: Record, + docPaths: Set + ): void { + if (manifest.kind !== `document` || typeof manifest.docPath !== `string`) { + return + } + docPaths.add(manifest.docPath) + } + private async buildForkEntityUrlMap( entitiesToFork: Array, opts: { suffix: string; rootUrl: string; rootInstanceId?: string } @@ -1979,6 +2097,29 @@ export class EntityManager { } } + if ( + next.kind === `document` && + typeof next.docPath === `string` && + typeof next.id === `string` + ) { + for (const [sourceUrl, forkUrl] of entityUrlMap) { + const expectedSourceDocPath = getMarkdownDocumentDocPath( + sourceUrl, + next.id + ) + if (next.docPath !== expectedSourceDocPath) { + continue + } + next.docPath = getMarkdownDocumentDocPath(forkUrl, next.id) + next.streamPath = getEntityMarkdownDocumentUrlPath( + this.tenantId, + forkUrl, + next.id + ) + return { key, value: next, changed: true } + } + } + if (next.kind === `schedule` && next.scheduleType === `future_send`) { let changed = false if (typeof next.targetUrl === `string`) { @@ -2526,6 +2667,239 @@ export class EntityManager { return { txid } } + // ========================================================================== + // Markdown Documents + // ========================================================================== + + isMarkdownDocumentUpdateStreamPath(path: string): boolean { + return /^\/yjs\/[^/]+\/docs\/agents\/[^/]+\/[^/]+\/documents\/[A-Za-z0-9_-]+\/\.updates$/.test( + path + ) + } + + async createMarkdownDocument( + entityUrl: string, + req: CreateMarkdownDocumentRequest + ): Promise<{ txid: string; document: ManifestMarkdownDocumentEntry }> { + const entity = await this.registry.getEntity(entityUrl) + if (!entity) { + throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404) + } + if (rejectsNormalWrites(entity.status)) { + throw new ElectricAgentsError( + ErrCodeNotRunning, + `Entity is not accepting writes`, + 409 + ) + } + if (this.isForkWorkLockedEntity(entityUrl)) { + this.assertEntityNotForkWorkLocked(entityUrl) + } + + const id = req.id ?? randomUUID() + validateMarkdownDocumentId(id) + + const docPath = getMarkdownDocumentDocPath(entityUrl, id) + const updateStreamPath = getMarkdownDocumentUpdateStreamPath( + this.tenantId, + docPath + ) + const now = new Date().toISOString() + const txid = randomUUID() + const document: ManifestMarkdownDocumentEntry = { + key: manifestMarkdownDocumentKey(id), + kind: `document`, + id, + docPath, + streamPath: getEntityMarkdownDocumentUrlPath( + this.tenantId, + entityUrl, + id + ), + mimeType: MARKDOWN_DOCUMENT_TRANSPORT_MIME, + contentMimeType: MARKDOWN_DOCUMENT_CONTENT_MIME, + title: req.title.trim() || `Untitled document`, + createdAt: now, + ...(req.createdBy ? { createdBy: req.createdBy } : {}), + ...(req.meta ? { meta: req.meta } : {}), + } + + let streamCreated = false + try { + await this.streamClient.create(updateStreamPath, { + contentType: `application/octet-stream`, + }) + streamCreated = true + if (req.content) { + const doc = new Y.Doc() + const update = replaceMarkdownText(doc, req.content) + await this.streamClient.appendBytes( + updateStreamPath, + frameYjsUpdate(update), + { + producerId: `agent-doc-create-${id}`, + epoch: 0, + seq: 0, + } + ) + } + await this.writeManifestEntry( + entityUrl, + document.key, + `upsert`, + document as unknown as Record, + { txid } + ) + } catch (error) { + if (streamCreated) { + await this.streamClient.delete(updateStreamPath).catch(() => undefined) + } + if (!streamCreated && isStreamCreateConflict(error)) { + throw new ElectricAgentsError( + ErrCodeInvalidRequest, + `Document already exists at id "${id}"`, + 409 + ) + } + throw error + } + + return { txid, document } + } + + async getMarkdownDocument( + entityUrl: string, + id: string + ): Promise { + validateMarkdownDocumentId(id) + const entity = await this.registry.getEntity(entityUrl) + if (!entity) { + throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404) + } + const events = await this.streamClient.readJson>( + entity.streams.main + ) + const manifest = this.reduceStateRows(events, `manifest`).get( + manifestMarkdownDocumentKey(id) + ) + if (!manifest || manifest.kind !== `document`) return null + assertMarkdownDocumentMatchesEntity(entity, manifest.docPath as string) + return manifest as unknown as ManifestMarkdownDocumentEntry + } + + async readMarkdownDocument( + entityUrl: string, + id: string + ): Promise<{ + document: ManifestMarkdownDocumentEntry + content: string + }> { + const document = await this.getMarkdownDocument(entityUrl, id) + if (!document) { + throw new ElectricAgentsError(ErrCodeNotFound, `Document not found`, 404) + } + const doc = await readMarkdownYDoc( + this.streamClient, + getMarkdownDocumentUpdateStreamPath(this.tenantId, document.docPath) + ) + return { document, content: markdownText(doc).toString() } + } + + async writeMarkdownDocument( + entityUrl: string, + id: string, + req: UpdateMarkdownDocumentRequest + ): Promise<{ + txid: string + document: ManifestMarkdownDocumentEntry + content: string + }> { + const entity = await this.registry.getEntity(entityUrl) + if (!entity) { + throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404) + } + if (rejectsNormalWrites(entity.status)) { + throw new ElectricAgentsError( + ErrCodeNotRunning, + `Entity is not accepting writes`, + 409 + ) + } + if (this.isForkWorkLockedEntity(entityUrl)) { + this.assertEntityNotForkWorkLocked(entityUrl) + } + const current = await this.readMarkdownDocument(entityUrl, id) + const updateStreamPath = getMarkdownDocumentUpdateStreamPath( + this.tenantId, + current.document.docPath + ) + const doc = await readMarkdownYDoc(this.streamClient, updateStreamPath) + const update = replaceMarkdownText(doc, req.content) + await this.streamClient.appendBytes( + updateStreamPath, + frameYjsUpdate(update), + { + producerId: `agent-doc-write-${id}`, + epoch: Date.now(), + seq: 0, + } + ) + const txid = randomUUID() + const nextDocument: ManifestMarkdownDocumentEntry = { + ...current.document, + updatedAt: new Date().toISOString(), + } + await this.writeManifestEntry( + entityUrl, + nextDocument.key, + `upsert`, + nextDocument as unknown as Record, + { txid } + ) + return { txid, document: nextDocument, content: req.content } + } + + async editMarkdownDocument( + entityUrl: string, + id: string, + req: EditMarkdownDocumentRequest + ): Promise<{ + txid: string + document: ManifestMarkdownDocumentEntry + content: string + }> { + if (req.oldString === ``) { + throw new ElectricAgentsError( + ErrCodeInvalidRequest, + `oldString must not be empty`, + 400 + ) + } + const current = await this.readMarkdownDocument(entityUrl, id) + const matches = current.content.split(req.oldString).length - 1 + if (matches === 0) { + throw new ElectricAgentsError( + ErrCodeInvalidRequest, + `oldString was not found in document`, + 400 + ) + } + if (!req.replaceAll && matches > 1) { + throw new ElectricAgentsError( + ErrCodeInvalidRequest, + `oldString appears multiple times; set replaceAll to replace all matches`, + 400 + ) + } + const content = req.replaceAll + ? current.content.split(req.oldString).join(req.newString) + : current.content.replace(req.oldString, req.newString) + return await this.writeMarkdownDocument(entityUrl, id, { + content, + updatedBy: req.updatedBy, + }) + } + // ========================================================================== // Tag Updates // ========================================================================== diff --git a/packages/agents-server/src/markdown-documents.ts b/packages/agents-server/src/markdown-documents.ts new file mode 100644 index 0000000000..838187de5c --- /dev/null +++ b/packages/agents-server/src/markdown-documents.ts @@ -0,0 +1,176 @@ +import * as decoding from 'lib0/decoding' +import * as encoding from 'lib0/encoding' +import * as Y from 'yjs' +import type { ElectricAgentsEntity } from './electric-agents-types.js' +import type { StreamClient } from './stream-client.js' + +export const MARKDOWN_DOCUMENT_TRANSPORT_MIME = + `application/vnd.electric-agents.markdown-yjs` as const +export const MARKDOWN_DOCUMENT_CONTENT_MIME = `text/markdown` as const +export const MARKDOWN_DOCUMENT_TEXT_NAME = `markdown` as const + +export interface ParsedMarkdownDocumentPath { + entityType: string + instanceId: string + entityUrl: string + documentId: string +} + +export function getMarkdownDocumentDocPath( + entityUrl: string, + documentId: string +): string { + const match = entityUrl.match(/^\/([^/]+)\/([^/]+)$/) + if (!match) { + throw new Error(`Invalid entity URL for markdown document: ${entityUrl}`) + } + return `agents/${match[1]}/${match[2]}/documents/${documentId}` +} + +export function getMarkdownDocumentUrlPath( + service: string, + entityUrl: string, + documentId: string +): string { + return `/v1/yjs/${encodeURIComponent(service)}/docs/${getMarkdownDocumentDocPath( + entityUrl, + documentId + )}` +} + +export function getMarkdownDocumentUpdateStreamPath( + service: string, + docPath: string +): string { + return `/yjs/${service}/docs/${docPath}/.updates` +} + +export function getMarkdownDocumentAwarenessStreamPath( + service: string, + docPath: string, + name: string +): string { + return `/yjs/${service}/docs/${docPath}/.awareness/${name}` +} + +export function getMarkdownDocumentIndexStreamPath( + service: string, + docPath: string +): string { + return `/yjs/${service}/docs/${docPath}/.index` +} + +export function getMarkdownDocumentSnapshotStreamPath( + service: string, + docPath: string, + snapshotKey: string +): string { + return `/yjs/${service}/docs/${docPath}/.snapshots/${snapshotKey}` +} + +export function parseMarkdownDocumentDocPath( + docPath: string +): ParsedMarkdownDocumentPath | null { + const match = docPath.match( + /^agents\/([^/]+)\/([^/]+)\/documents\/([A-Za-z0-9_-]+)$/ + ) + if (!match) return null + return { + entityType: match[1]!, + instanceId: match[2]!, + entityUrl: `/${match[1]}/${match[2]}`, + documentId: match[3]!, + } +} + +export function parseYjsDocumentRoutePath( + path: string +): { service: string; docPath: string } | null { + const match = path.match(/^\/v1\/yjs\/([^/]+)\/docs\/(.+)$/) + if (!match) return null + let docPath: string + try { + docPath = decodeURIComponent(match[2]!) + } catch { + return null + } + if ( + docPath.includes(`..`) || + docPath.split(`/`).some((segment) => segment === `.` || segment === ``) + ) { + return null + } + return { service: match[1]!, docPath } +} + +export function frameYjsUpdate(update: Uint8Array): Uint8Array { + const encoder = encoding.createEncoder() + encoding.writeVarUint8Array(encoder, update) + return encoding.toUint8Array(encoder) +} + +export function applyFramedYjsUpdates(doc: Y.Doc, data: Uint8Array): void { + if (data.length === 0) return + const decoder = decoding.createDecoder(data) + while (decoding.hasContent(decoder)) { + Y.applyUpdate(doc, decoding.readVarUint8Array(decoder), `server`) + } +} + +export async function readMarkdownYDoc( + streamClient: StreamClient, + updateStreamPath: string +): Promise { + const doc = new Y.Doc() + const result = await streamClient.read(updateStreamPath) + for (const message of result.messages) { + applyFramedYjsUpdates(doc, message.data) + } + return doc +} + +export function markdownText(doc: Y.Doc): Y.Text { + return doc.getText(MARKDOWN_DOCUMENT_TEXT_NAME) +} + +export function replaceMarkdownText(doc: Y.Doc, content: string): Uint8Array { + const before = Y.encodeStateVector(doc) + const text = markdownText(doc) + doc.transact(() => { + text.delete(0, text.length) + if (content.length > 0) text.insert(0, content) + }, `server`) + return Y.encodeStateAsUpdate(doc, before) +} + +export function entityUrlFromYjsDocumentRoutePath(path: string): string | null { + const route = parseYjsDocumentRoutePath(path) + if (!route) return null + return parseMarkdownDocumentDocPath(route.docPath)?.entityUrl ?? null +} + +export function parseMarkdownDocumentStreamPath( + path: string +): { service: string; docPath: string; entityUrl: string } | null { + const match = path.match( + /^\/yjs\/([^/]+)\/docs\/(.+)\/\.(updates|index|awareness|snapshots)(?:\/.*)?$/ + ) + if (!match) return null + const parsed = parseMarkdownDocumentDocPath(match[2]!) + if (!parsed) return null + return { + service: match[1]!, + docPath: match[2]!, + entityUrl: parsed.entityUrl, + } +} + +export function assertMarkdownDocumentMatchesEntity( + entity: ElectricAgentsEntity, + docPath: string +): void { + const parsed = parseMarkdownDocumentDocPath(docPath) + if (!parsed || parsed.entityUrl !== entity.url) { + throw new Error(`Markdown document path does not belong to ${entity.url}`) + } +} diff --git a/packages/agents-server/src/routing/durable-streams-router.ts b/packages/agents-server/src/routing/durable-streams-router.ts index 781ddfca21..a30ac9b74f 100644 --- a/packages/agents-server/src/routing/durable-streams-router.ts +++ b/packages/agents-server/src/routing/durable-streams-router.ts @@ -13,7 +13,9 @@ import { } from '../electric-agents-http.js' import { subscriptionWebhooks } from '../db/schema.js' import { + ErrCodeInvalidRequest, ErrCodeNotFound, + ErrCodeNotRunning, ErrCodeUnauthorized, } from '../electric-agents-types.js' import { @@ -29,6 +31,13 @@ import { webhookSigningMetadata, } from '../webhook-signing.js' import { resolveDurableStreamsRoutingAdapter } from './durable-streams-routing-adapter.js' +import { + getMarkdownDocumentAwarenessStreamPath, + getMarkdownDocumentUpdateStreamPath, + parseMarkdownDocumentDocPath, + parseMarkdownDocumentStreamPath, + parseYjsDocumentRoutePath, +} from '../markdown-documents.js' import type { IRequest, RouterType } from 'itty-router' import type { TenantContext } from './context.js' import type { DurableStreamsRoutingAdapter } from './durable-streams-routing-adapter.js' @@ -100,6 +109,7 @@ for (const action of subscriptionControlActions) { durableStreamsRouter.get(`/__ds/jwks.json`, webhookJwks) durableStreamsRouter.all(`/__ds`, controlPassThrough) durableStreamsRouter.all(`/__ds/*`, controlPassThrough) +durableStreamsRouter.all(`/v1/yjs/:service/docs/:docPath+`, yjsDocumentRoute) durableStreamsRouter.post(`*`, streamAppend) durableStreamsRouter.all(`*`, proxyPassThrough) @@ -617,6 +627,111 @@ async function streamAppend( ) } +async function yjsDocumentRoute( + request: IRequest, + ctx: TenantContext +): Promise { + const url = new URL(request.url) + const route = parseYjsDocumentRoutePath(url.pathname) + if (!route || route.service !== ctx.service) { + return apiError(400, ErrCodeInvalidRequest, `Invalid Yjs document path`) + } + const parsed = parseMarkdownDocumentDocPath(route.docPath) + if (!parsed) { + return apiError( + 400, + ErrCodeInvalidRequest, + `Invalid markdown document path` + ) + } + const entity = await ctx.entityManager.registry.getEntity(parsed.entityUrl) + if (!entity) { + return apiError(404, ErrCodeNotFound, `Entity not found`) + } + + const method = request.method.toUpperCase() + const awarenessName = url.searchParams.get(`awareness`) + const isAwareness = awarenessName !== null + const permission = + method === `GET` || method === `HEAD` || isAwareness ? `read` : `write` + if (!(await canAccessEntity(ctx, entity, permission, request as Request))) { + return apiError( + 401, + ErrCodeUnauthorized, + `Principal is not allowed to ${permission} ${entity.url}` + ) + } + + if ( + !isAwareness && + (method === `PUT` || method === `POST` || method === `DELETE`) && + ctx.entityManager.isForkWorkLockedEntity(entity.url) + ) { + return apiError(409, ErrCodeNotRunning, `Entity subtree is being forked`) + } + + if (!isAwareness && url.searchParams.get(`offset`) === `snapshot`) { + const redirect = new URL(request.url) + redirect.searchParams.set(`offset`, `-1`) + return new Response(null, { + status: 307, + headers: { + location: `${redirect.pathname}${redirect.search}`, + 'cache-control': `private, max-age=5`, + }, + }) + } + + const offset = url.searchParams.get(`offset`) + if (!isAwareness && offset?.endsWith(`_snapshot`)) { + return apiError(404, ErrCodeNotFound, `Snapshot not found`) + } + + const streamPath = isAwareness + ? getMarkdownDocumentAwarenessStreamPath( + ctx.service, + route.docPath, + awarenessName || `default` + ) + : getMarkdownDocumentUpdateStreamPath(ctx.service, route.docPath) + + if (method === `PUT`) { + const upstream = await forwardToDurableStreams( + ctx, + request, + undefined, + `stream`, + rewriteRequestUrlPath(request.url, streamPath) + ) + if (!isAwareness && (upstream.ok || upstream.status === 409)) { + await ctx.streamClient + .create( + getMarkdownDocumentAwarenessStreamPath( + ctx.service, + route.docPath, + `default` + ), + { contentType: `application/octet-stream` } + ) + .catch(() => undefined) + } + return responseFromUpstream(upstream) + } + + if (method === `GET` || method === `HEAD` || method === `POST`) { + const upstream = await forwardToDurableStreams( + ctx, + request, + undefined, + `stream`, + rewriteRequestUrlPath(request.url, streamPath) + ) + return responseFromUpstream(upstream) + } + + return apiError(400, ErrCodeInvalidRequest, `Unsupported Yjs document method`) +} + async function proxyPassThrough( request: IRequest, ctx: TenantContext @@ -640,6 +755,12 @@ async function proxyPassThrough( } } +function rewriteRequestUrlPath(requestUrl: string, path: string): string { + const url = new URL(requestUrl) + url.pathname = path + return url.toString() +} + async function authorizeDurableStreamAccess( request: IRequest, ctx: TenantContext @@ -685,6 +806,37 @@ async function authorizeDurableStreamAccess( } const sharedStateId = sharedStateIdFromPath(streamPath) + const markdownDocumentStream = parseMarkdownDocumentStreamPath(streamPath) + if (markdownDocumentStream) { + if (markdownDocumentStream.service !== ctx.service) { + return apiError(404, ErrCodeNotFound, `Document stream not found`) + } + const entity = await ctx.entityManager.registry.getEntity( + markdownDocumentStream.entityUrl + ) + if (!entity) { + return apiError(404, ErrCodeNotFound, `Entity not found`) + } + const permission = method === `GET` || method === `HEAD` ? `read` : `write` + if (await canAccessEntity(ctx, entity, permission, request as Request)) { + if ( + permission === `write` && + ctx.entityManager.isForkWorkLockedEntity(entity.url) + ) { + return apiError( + 409, + ErrCodeNotRunning, + `Entity subtree is being forked` + ) + } + return undefined + } + return apiError( + 401, + ErrCodeUnauthorized, + `Principal is not allowed to ${permission} ${entity.url}` + ) + } if (!sharedStateId) { // Durable Streams also hosts non-Agents utility streams. Entity streams, // attachment streams, and shared-state streams are guarded above; paths that diff --git a/packages/agents-server/src/routing/entities-router.ts b/packages/agents-server/src/routing/entities-router.ts index efa5c79afa..35fcdc8624 100644 --- a/packages/agents-server/src/routing/entities-router.ts +++ b/packages/agents-server/src/routing/entities-router.ts @@ -238,6 +238,32 @@ const setTagBodySchema = Type.Object({ value: Type.String(), }) +const markdownDocumentCreateBodySchema = Type.Object( + { + id: Type.Optional(Type.String()), + title: Type.String(), + content: Type.Optional(Type.String()), + meta: Type.Optional(Type.Record(Type.String(), Type.Unknown())), + }, + { additionalProperties: false } +) + +const markdownDocumentWriteBodySchema = Type.Object( + { + content: Type.String(), + }, + { additionalProperties: false } +) + +const markdownDocumentEditBodySchema = Type.Object( + { + oldString: Type.String(), + newString: Type.String(), + replaceAll: Type.Optional(Type.Boolean()), + }, + { additionalProperties: false } +) + const entitySignalSchema = Type.Union([ Type.Literal(`SIGINT`), Type.Literal(`SIGHUP`), @@ -296,6 +322,11 @@ type SendBody = Static type InboxMessageBody = Static type ForkBody = Static type SetTagBody = Static +type MarkdownDocumentCreateBody = Static< + typeof markdownDocumentCreateBodySchema +> +type MarkdownDocumentWriteBody = Static +type MarkdownDocumentEditBody = Static type SignalBody = Static type ScheduleBody = Static type EventSourceSubscriptionBody = Static< @@ -390,6 +421,33 @@ entitiesRouter.delete( withEntityPermission(`write`), deleteAttachment ) +entitiesRouter.post( + `/:type/:instanceId/documents`, + withExistingEntity, + withSchema(markdownDocumentCreateBodySchema), + withEntityPermission(`write`), + createMarkdownDocument +) +entitiesRouter.get( + `/:type/:instanceId/documents/:documentId`, + withExistingEntity, + withEntityPermission(`read`), + readMarkdownDocument +) +entitiesRouter.put( + `/:type/:instanceId/documents/:documentId`, + withExistingEntity, + withSchema(markdownDocumentWriteBodySchema), + withEntityPermission(`write`), + writeMarkdownDocument +) +entitiesRouter.patch( + `/:type/:instanceId/documents/:documentId`, + withExistingEntity, + withSchema(markdownDocumentEditBodySchema), + withEntityPermission(`write`), + editMarkdownDocument +) entitiesRouter.patch( `/:type/:instanceId/inbox/:messageKey`, withExistingEntity, @@ -1264,6 +1322,90 @@ async function deleteAttachment( return json(result) } +async function createMarkdownDocument( + request: AgentsRouteRequest, + ctx: TenantContext +): Promise { + const principalMutationError = rejectPrincipalEntityMutation( + request, + `given documents` + ) + if (principalMutationError) return principalMutationError + + const parsed = routeBody(request) + const { entityUrl } = requireExistingEntityRoute(request) + const result = await ctx.entityManager.createMarkdownDocument(entityUrl, { + id: parsed.id, + title: parsed.title, + content: parsed.content, + createdBy: ctx.principal.url, + meta: parsed.meta, + }) + return json(result, { status: 201 }) +} + +async function readMarkdownDocument( + request: AgentsRouteRequest, + ctx: TenantContext +): Promise { + const { entityUrl } = requireExistingEntityRoute(request) + const result = await ctx.entityManager.readMarkdownDocument( + entityUrl, + decodeURIComponent(request.params.documentId) + ) + return json(result, { + headers: { + 'content-type': `application/json; charset=utf-8`, + 'cache-control': `no-store`, + }, + }) +} + +async function writeMarkdownDocument( + request: AgentsRouteRequest, + ctx: TenantContext +): Promise { + const principalMutationError = rejectPrincipalEntityMutation( + request, + `given documents` + ) + if (principalMutationError) return principalMutationError + + const parsed = routeBody(request) + const { entityUrl } = requireExistingEntityRoute(request) + const result = await ctx.entityManager.writeMarkdownDocument( + entityUrl, + decodeURIComponent(request.params.documentId), + { content: parsed.content, updatedBy: ctx.principal.url } + ) + return json(result) +} + +async function editMarkdownDocument( + request: AgentsRouteRequest, + ctx: TenantContext +): Promise { + const principalMutationError = rejectPrincipalEntityMutation( + request, + `given documents` + ) + if (principalMutationError) return principalMutationError + + const parsed = routeBody(request) + const { entityUrl } = requireExistingEntityRoute(request) + const result = await ctx.entityManager.editMarkdownDocument( + entityUrl, + decodeURIComponent(request.params.documentId), + { + oldString: parsed.oldString, + newString: parsed.newString, + replaceAll: parsed.replaceAll, + updatedBy: ctx.principal.url, + } + ) + return json(result) +} + async function updateInboxMessage( request: AgentsRouteRequest, ctx: TenantContext diff --git a/packages/agents-server/src/stream-client.ts b/packages/agents-server/src/stream-client.ts index 96e92de279..b37561931a 100644 --- a/packages/agents-server/src/stream-client.ts +++ b/packages/agents-server/src/stream-client.ts @@ -310,6 +310,49 @@ export class StreamClient { }) } + async appendBytes( + path: string, + data: Uint8Array, + opts?: { + contentType?: string + producerId?: string + epoch?: number + seq?: number + } + ): Promise { + return await withSpan(`stream.appendBytes`, async (span) => { + span.setAttributes({ + [ATTR.STREAM_PATH]: path, + [ATTR.STREAM_OP]: `appendBytes`, + }) + const headers: Record = { + 'content-type': opts?.contentType ?? `application/octet-stream`, + } + if (opts?.producerId) headers[`Producer-Id`] = opts.producerId + if (opts?.epoch !== undefined) { + headers[`Producer-Epoch`] = String(opts.epoch) + } + if (opts?.seq !== undefined) { + headers[`Producer-Seq`] = String(opts.seq) + } + injectTraceHeaders(headers) + + const response = await fetch(this.streamUrl(path), { + method: `POST`, + headers: await this.requestHeaders(headers), + body: data, + }) + if (!response.ok) { + throw new Error( + `Stream append failed: ${response.status} ${await response.text()}` + ) + } + return { + offset: response.headers.get(`stream-next-offset`) ?? ``, + } + }) + } + async appendIdempotent( path: string, data: Uint8Array | string, diff --git a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts index 4cab89f72b..2896f9e355 100644 --- a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts +++ b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it, vi } from 'vitest' import { EntityManager } from '../src/entity-manager' import { SchemaValidator } from '../src/electric-agents/schema-validator' +import { + MARKDOWN_DOCUMENT_CONTENT_MIME, + MARKDOWN_DOCUMENT_TRANSPORT_MIME, +} from '../src/markdown-documents' const observedItemSchema = { type: `object`, @@ -94,6 +98,65 @@ function attachmentManifest(value: Record) { } } +function createMarkdownDocumentManager() { + const jsonStreams = new Map>>() + const binaryStreams = new Map>() + const streamClient = { + create: vi.fn(async (path: string) => { + if (binaryStreams.has(path)) { + const error = new Error(`Stream already exists`) as Error & { + status?: number + } + error.status = 409 + throw error + } + binaryStreams.set(path, []) + }), + appendBytes: vi.fn(async (path: string, data: Uint8Array) => { + binaryStreams.get(path)?.push(data) + }), + append: vi.fn(async (path: string, data: Uint8Array) => { + const event = JSON.parse(new TextDecoder().decode(data)) + const stream = jsonStreams.get(path) ?? [] + stream.push(event) + jsonStreams.set(path, stream) + }), + delete: vi.fn(async (path: string) => { + binaryStreams.delete(path) + }), + read: vi.fn(async (path: string) => ({ + messages: (binaryStreams.get(path) ?? []).map((data, index) => ({ + data, + offset: String(index), + })), + })), + readJson: vi.fn(async (path: string) => jsonStreams.get(path) ?? []), + } + + return { + manager: new EntityManager({ + registry: { + getEntity: vi.fn().mockResolvedValue({ + url: `/chat/session-1`, + status: `running`, + streams: { main: `/chat/session-1` }, + }), + getEntityType: vi.fn(), + replaceEntityManifestSource: vi.fn(), + replaceSharedStateLink: vi.fn(), + close: vi.fn(), + } as any, + streamClient: streamClient as any, + validator: new SchemaValidator(), + wakeRegistry: { + setTimeoutCallback: vi.fn(), + setDebounceCallback: vi.fn(), + } as any, + }), + streamClient, + } +} + describe(`ElectricAgentsManager.validateWriteEvent`, () => { it(`validates delete events against old_value instead of value`, async () => { const manager = createManager() @@ -140,6 +203,53 @@ describe(`ElectricAgentsManager.validateWriteEvent`, () => { }) }) +describe(`ElectricAgentsManager markdown documents`, () => { + it(`stores markdown as framed Yjs updates and exposes a manifest document entry`, async () => { + const { manager, streamClient } = createMarkdownDocumentManager() + + const created = await manager.createMarkdownDocument(`/chat/session-1`, { + id: `notes`, + title: `Session notes`, + content: `# Notes\n\nDraft`, + createdBy: `/principal/user:u1`, + }) + + expect(created.document).toMatchObject({ + key: `document:notes`, + kind: `document`, + id: `notes`, + docPath: `agents/chat/session-1/documents/notes`, + streamPath: `/v1/yjs/default/docs/agents/chat/session-1/documents/notes`, + mimeType: MARKDOWN_DOCUMENT_TRANSPORT_MIME, + contentMimeType: MARKDOWN_DOCUMENT_CONTENT_MIME, + title: `Session notes`, + createdBy: `/principal/user:u1`, + }) + expect(streamClient.create).toHaveBeenCalledWith( + `/yjs/default/docs/agents/chat/session-1/documents/notes/.updates`, + { contentType: `application/octet-stream` } + ) + + await expect( + manager.readMarkdownDocument(`/chat/session-1`, `notes`) + ).resolves.toMatchObject({ + document: expect.objectContaining({ id: `notes` }), + content: `# Notes\n\nDraft`, + }) + + await manager.editMarkdownDocument(`/chat/session-1`, `notes`, { + oldString: `Draft`, + newString: `Ready`, + }) + + await expect( + manager.readMarkdownDocument(`/chat/session-1`, `notes`) + ).resolves.toMatchObject({ + content: `# Notes\n\nReady`, + }) + }) +}) + describe(`ElectricAgentsManager attachments`, () => { it(`does not delete an existing stream when duplicate attachment creation conflicts`, async () => { const create = vi.fn().mockRejectedValue({ status: 409 }) diff --git a/packages/agents-server/test/electric-agents-routes.test.ts b/packages/agents-server/test/electric-agents-routes.test.ts index 6e136dd75f..e2fad7c12d 100644 --- a/packages/agents-server/test/electric-agents-routes.test.ts +++ b/packages/agents-server/test/electric-agents-routes.test.ts @@ -248,6 +248,94 @@ describe(`ElectricAgentsRoutes schedule endpoints`, () => { }) }) +describe(`ElectricAgentsRoutes markdown document endpoints`, () => { + it(`routes document create, read, write, and edit requests to the manager`, async () => { + const document = { + key: `document:notes`, + kind: `document`, + id: `notes`, + docPath: `agents/chat/test/documents/notes`, + streamPath: `/v1/yjs/test/docs/agents/chat/test/documents/notes`, + mimeType: `application/vnd.electric-agents.markdown-yjs`, + contentMimeType: `text/markdown`, + title: `Notes`, + createdAt: `2026-01-01T00:00:00.000Z`, + } + const manager = { + registry: { + getEntity: vi.fn().mockResolvedValue({ url: `/chat/test` }), + getEntityType: vi.fn(), + }, + createMarkdownDocument: vi + .fn() + .mockResolvedValue({ txid: `tx-create`, document }), + readMarkdownDocument: vi + .fn() + .mockResolvedValue({ document, content: `# Notes` }), + writeMarkdownDocument: vi + .fn() + .mockResolvedValue({ txid: `tx-write`, document, content: `# Ready` }), + editMarkdownDocument: vi + .fn() + .mockResolvedValue({ txid: `tx-edit`, document, content: `# Done` }), + } as any + + const createResponse = await routeResponse( + manager, + `POST`, + `/_electric/entities/chat/test/documents`, + { id: `notes`, title: `Notes`, content: `# Notes` } + ) + expect(createResponse.status).toBe(201) + expect(manager.createMarkdownDocument).toHaveBeenCalledWith(`/chat/test`, { + id: `notes`, + title: `Notes`, + content: `# Notes`, + createdBy: `/principal/system:dev-local`, + meta: undefined, + }) + + const readResponse = await routeResponse( + manager, + `GET`, + `/_electric/entities/chat/test/documents/notes` + ) + expect(await responseJson(readResponse)).toEqual({ + document, + content: `# Notes`, + }) + + await routeResponse( + manager, + `PUT`, + `/_electric/entities/chat/test/documents/notes`, + { content: `# Ready` } + ) + expect(manager.writeMarkdownDocument).toHaveBeenCalledWith( + `/chat/test`, + `notes`, + { content: `# Ready`, updatedBy: `/principal/system:dev-local` } + ) + + await routeResponse( + manager, + `PATCH`, + `/_electric/entities/chat/test/documents/notes`, + { oldString: `Ready`, newString: `Done`, replaceAll: true } + ) + expect(manager.editMarkdownDocument).toHaveBeenCalledWith( + `/chat/test`, + `notes`, + { + oldString: `Ready`, + newString: `Done`, + replaceAll: true, + updatedBy: `/principal/system:dev-local`, + } + ) + }) +}) + describe(`ElectricAgentsRoutes cron stream ensure endpoint`, () => { it(`rejects cron ensure requests without an expression in the schema layer`, async () => { const manager = { diff --git a/packages/agents/skills/markdown-docs.md b/packages/agents/skills/markdown-docs.md new file mode 100644 index 0000000000..70e1f1ed81 --- /dev/null +++ b/packages/agents/skills/markdown-docs.md @@ -0,0 +1,94 @@ +--- +description: Create and edit collaborative markdown documents in the app workspace +whenToUse: User wants a markdown doc, notes, plan, draft, report, or document they can open and edit in the app UI +keywords: + - markdown doc + - collaborative document + - notes + - draft + - report + - plan + - workspace editor + - manifest +user-invocable: true +max: 9000 +--- + +# Markdown Docs + +Use this skill when the user wants a document that appears in the Electric +Agents UI and can be opened, edited, and watched live. + +## Core Rule + +Collaborative markdown docs are not filesystem files. + +- Use `create_markdown_doc`, `read_markdown_doc`, `write_markdown_doc`, and + `edit_markdown_doc` for docs the user should open in the workspace UI. +- Use filesystem `write`/`edit` only when the user asks for an actual file path + in the workspace or repo, such as `docs/foo.md`, `README.md`, or + `/tmp/report.md`. + +## When To Create A Collaborative Doc + +Use `create_markdown_doc` when the user says things like: + +- "make a markdown doc" +- "create a doc" +- "write some notes" +- "draft a plan" +- "make a report I can edit" +- "add this to the manifest" +- "create a document I can open" +- "put this in a doc" + +If the user says "file", "repo", "workspace", or gives a path, ask one short +clarifying question if the destination is ambiguous. + +## Create Workflow + +1. Choose a concise title. +2. Use `create_markdown_doc`. +3. Include initial markdown content if the user supplied enough detail. +4. After creation, tell the user the document is available from this entity's + manifest or timeline and can be opened in the markdown editor. + +Example tool call: + +```json +{ + "title": "Launch Plan", + "content": "# Launch Plan\n\n## Goals\n\n- ...\n" +} +``` + +Do not also write a `.md` file unless the user explicitly asked for a filesystem +copy. + +## Edit Workflow + +For small edits: + +1. Use `read_markdown_doc` first. +2. Use `edit_markdown_doc` with an exact `old_string`. +3. If the target text appears multiple times, make `old_string` more specific or + set `replace_all` only when replacing every occurrence is clearly intended. + +For broad rewrites: + +1. Use `read_markdown_doc` first unless you just created or wrote the doc in the + same wake. +2. Use `write_markdown_doc` with the full replacement markdown. + +Both write and edit tool results include diffs. Use those diffs to summarize +what changed. + +## Response Style + +After creating a doc, keep the response short: + +- State the title. +- State that it is available in the manifest/timeline. +- Mention any useful next action, such as "open it to edit collaboratively". + +Do not paste the entire document back into chat unless the user asks. diff --git a/packages/agents/src/agents/horton.ts b/packages/agents/src/agents/horton.ts index c6a6b82b5a..f5ebec4d53 100644 --- a/packages/agents/src/agents/horton.ts +++ b/packages/agents/src/agents/horton.ts @@ -185,6 +185,7 @@ export function buildHortonSystemPrompt( opts: { hasDocsSupport?: boolean hasEventSourceTools?: boolean + hasMarkdownDocumentTools?: boolean hasSkills?: boolean docsUrl?: string modelProvider?: string @@ -197,9 +198,15 @@ export function buildHortonSystemPrompt( const eventSourceTools = opts.hasEventSourceTools ? `\n- list_event_sources: list external webhook/event feeds you can subscribe to, including available buckets and parameters\n- subscribe_event_source: subscribe yourself to one of those feeds or buckets so matching future events wake you\n- list_event_source_subscriptions: list your active event source subscriptions\n- unsubscribe_event_source: remove one of your event source subscriptions by id` : `` + const markdownDocumentTools = opts.hasMarkdownDocumentTools + ? `\n- create_markdown_doc: create a collaborative markdown document that appears in this entity's manifest and opens in the workspace editor\n- read_markdown_doc: read a collaborative markdown document\n- write_markdown_doc: replace a collaborative markdown document's full content\n- edit_markdown_doc: targeted string replacement in a collaborative markdown document` + : `` const skillsTools = opts.hasSkills ? `\n- use_skill: load a skill (knowledge, instructions, or a tutorial) into your context to help with the user's request\n- remove_skill: unload a skill from context when you're done with it` : `` + const markdownDocumentGuidance = opts.hasMarkdownDocumentTools + ? `\n# Collaborative Markdown Docs\n- If the user asks you to create a markdown doc, notes, draft, brief, plan, report, or any document they should open/edit in the app UI, use create_markdown_doc. Do not use filesystem write unless they ask for a file path or repo/workspace file.\n- For larger document workflows, load the markdown-docs skill first with use_skill, then use the markdown document tools.\n- After creating a collaborative doc, mention that it is available from this entity's manifest/timeline.` + : `` const docsGuidance = opts.hasDocsSupport ? `\n- For ANY question about Electric Agents or this framework, ALWAYS use search_electric_agents_docs FIRST. Do not use web_search or fetch_url for Electric Agents topics unless the docs search returns no useful results.\n- The search tool returns chunk content directly โ€” you do not need to read the source files.\n- Use repo read/bash tools only for non-doc files or when you need to inspect exact implementation code in the workspace.` : `` @@ -262,13 +269,13 @@ When a user opens with a greeting ("hi", "hello", "hey", etc.) or a broad statem - fetch_url: fetch and convert a URL to markdown - spawn_worker: dispatch a subagent for an isolated task - send: send a message to an Electric Agent/entity. To schedule future work for yourself, call send with self: true and afterMs. -${eventSourceTools}${docsTools}${skillsTools} +${eventSourceTools}${markdownDocumentTools}${docsTools}${skillsTools} # Working with files - Prefer edit over write when modifying existing files. - You must read a file before you can edit it. - Use absolute paths or paths relative to the current working directory. -${modelGuidance}${docsGuidance}${skillsGuidance}${onboardingGuidance}${docsUrlGuidance} +${markdownDocumentGuidance}${modelGuidance}${docsGuidance}${skillsGuidance}${onboardingGuidance}${docsUrlGuidance} # Risky actions Pause and confirm with the user before: @@ -517,6 +524,9 @@ function createAssistantHandler(options: { const hasEventSourceTools = tools.some( (tool) => getToolName(tool) === `list_event_sources` ) + const hasMarkdownDocumentTools = tools.some( + (tool) => getToolName(tool) === `create_markdown_doc` + ) const titlePromise = !ctx.tags.title ? (async () => { @@ -649,6 +659,7 @@ function createAssistantHandler(options: { modelProvider: modelConfig.provider, modelId: String(modelConfig.model), hasEventSourceTools, + hasMarkdownDocumentTools, }), ...modelConfig, // mcp.tools() inserts sentinel objects that the runtime's diff --git a/packages/agents/src/bootstrap.ts b/packages/agents/src/bootstrap.ts index 597597e66b..fd97a710d3 100644 --- a/packages/agents/src/bootstrap.ts +++ b/packages/agents/src/bootstrap.ts @@ -8,7 +8,10 @@ import { createEntityRegistry, createRuntimeHandler, } from '@electric-ax/agents-runtime' -import { createEventSourceTools } from '@electric-ax/agents-runtime/tools' +import { + createEventSourceTools, + createMarkdownDocumentTools, +} from '@electric-ax/agents-runtime/tools' import { chooseDefaultSandbox, isE2BAvailable, @@ -85,7 +88,10 @@ export function createBuiltinElectricTools( custom?: BuiltinElectricToolsFactory ): BuiltinElectricToolsFactory { return async (context) => { - const builtinTools = createEventSourceTools(context) + const builtinTools = [ + ...createEventSourceTools(context), + ...createMarkdownDocumentTools(context), + ] const customTools = custom ? await custom(context) : [] return dedupeToolsByName([...builtinTools, ...customTools]) } diff --git a/packages/agents/test/horton-system-prompt.test.ts b/packages/agents/test/horton-system-prompt.test.ts index d582b18962..014ca57527 100644 --- a/packages/agents/test/horton-system-prompt.test.ts +++ b/packages/agents/test/horton-system-prompt.test.ts @@ -39,6 +39,24 @@ describe(`buildHortonSystemPrompt`, () => { expect(prompt).not.toContain(`subscribe_event_source`) }) + it(`describes collaborative markdown docs when document tools are available`, () => { + const prompt = buildHortonSystemPrompt(`/tmp/test`, { + hasMarkdownDocumentTools: true, + hasSkills: true, + }) + + expect(prompt).toContain(`create_markdown_doc`) + expect(prompt).toContain(`Collaborative Markdown Docs`) + expect(prompt).toContain(`Do not use filesystem write`) + expect(prompt).toContain(`markdown-docs skill`) + }) + + it(`omits collaborative markdown docs when document tools are unavailable`, () => { + const prompt = buildHortonSystemPrompt(`/tmp/test`) + expect(prompt).not.toContain(`create_markdown_doc`) + expect(prompt).not.toContain(`Collaborative Markdown Docs`) + }) + it(`includes docs URL guidance alongside local docs support`, () => { const prompt = buildHortonSystemPrompt(`/tmp/test`, { hasDocsSupport: true, diff --git a/packages/agents/test/horton-tool-composition.test.ts b/packages/agents/test/horton-tool-composition.test.ts index accadb1e56..73f1af3d33 100644 --- a/packages/agents/test/horton-tool-composition.test.ts +++ b/packages/agents/test/horton-tool-composition.test.ts @@ -216,7 +216,7 @@ describe(`horton tool composition`, () => { ) }) - it(`adds event source tools through the built-in electric tool factory`, async () => { + it(`adds event source and markdown document tools through the built-in electric tool factory`, async () => { const tools = await createBuiltinElectricTools()( createElectricToolsContext() ) @@ -228,6 +228,10 @@ describe(`horton tool composition`, () => { `subscribe_event_source`, `list_event_source_subscriptions`, `unsubscribe_event_source`, + `create_markdown_doc`, + `read_markdown_doc`, + `write_markdown_doc`, + `edit_markdown_doc`, ]) ) expect( @@ -240,9 +244,12 @@ describe(`horton tool composition`, () => { tools.find((tool) => tool.name === `list_event_source_subscriptions`) ?.description ).not.toContain(`manifest-backed`) + expect( + tools.find((tool) => tool.name === `create_markdown_doc`)?.description + ).toContain(`not a filesystem file`) }) - it(`includes event source electric tools in Horton and describes them in the prompt`, async () => { + it(`includes electric tools in Horton and describes them in the prompt`, async () => { const electricTools = await createBuiltinElectricTools()( createElectricToolsContext() ) @@ -253,8 +260,12 @@ describe(`horton tool composition`, () => { expect(names).toContain(`list_event_sources`) expect(names).toContain(`subscribe_event_source`) + expect(names).toContain(`create_markdown_doc`) + expect(names).toContain(`edit_markdown_doc`) expect(cfg.systemPrompt).toContain(`list_event_sources`) expect(cfg.systemPrompt).toContain(`subscribe_event_source`) + expect(cfg.systemPrompt).toContain(`create_markdown_doc`) + expect(cfg.systemPrompt).toContain(`Collaborative Markdown Docs`) }) it(`includes the default built-in toolset`, async () => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2528e3137b..91f2e877d7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -340,7 +340,7 @@ importers: version: 0.2.17(@electric-sql/pglite@0.2.17)(react@18.3.1) '@electric-sql/pglite-repl': specifier: ^0.2.17 - version: 0.2.17(@babel/runtime@7.29.7)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.5.0)(@codemirror/theme-one-dark@6.1.2)(@electric-sql/pglite@0.2.17)(@lezer/common@1.2.3)(codemirror@6.0.1(@lezer/common@1.2.3)) + version: 0.2.17(@babel/runtime@7.29.7)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/theme-one-dark@6.1.2)(@electric-sql/pglite@0.2.17)(@lezer/common@1.5.2)(codemirror@6.0.1(@lezer/common@1.5.2)) '@electric-sql/pglite-sync': specifier: ^0.2.20 version: 0.2.20(@electric-sql/pglite@0.2.17) @@ -1952,6 +1952,9 @@ importers: itty-router: specifier: ^5.0.23 version: 5.0.23 + lib0: + specifier: ^0.2.99 + version: 0.2.99 lmdb: specifier: ^3.5.1 version: 3.5.4 @@ -1967,6 +1970,9 @@ importers: undici: specifier: ^7.24.7 version: 7.25.0 + yjs: + specifier: ^13.6.26 + version: 13.6.26 devDependencies: '@electric-ax/agents': specifier: workspace:* @@ -2035,12 +2041,24 @@ importers: '@base-ui/react': specifier: ^1.4.1 version: 1.4.1(@types/react@19.2.14)(date-fns@4.1.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + '@codemirror/lang-markdown': + specifier: ^6.5.0 + version: 6.5.0 + '@codemirror/state': + specifier: ^6.6.0 + version: 6.6.0 + '@codemirror/view': + specifier: ^6.43.0 + version: 6.43.0 '@durable-streams/client': specifier: ^0.2.6 version: 0.2.6 '@durable-streams/state': specifier: ^0.3.1 version: 0.3.1(@tanstack/db@0.6.7(typescript@5.9.3)) + '@durable-streams/y-durable-streams': + specifier: 0.2.7 + version: 0.2.7(lib0@0.2.99)(y-protocols@1.0.6(yjs@13.6.26))(yjs@13.6.26) '@electric-ax/agents-runtime': specifier: workspace:* version: link:../agents-runtime @@ -2068,12 +2086,18 @@ importers: '@tanstack/react-virtual': specifier: ^3.13.23 version: 3.13.24(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + codemirror: + specifier: ^6.0.1 + version: 6.0.1(@lezer/common@1.5.2) fractional-indexing: specifier: ^3.2.0 version: 3.2.0 katex: specifier: ^0.16.45 version: 0.16.45 + lib0: + specifier: ^0.2.99 + version: 0.2.99 lucide-react: specifier: ^0.561.0 version: 0.561.0(react@19.1.0) @@ -2113,6 +2137,15 @@ importers: streamdown: specifier: ^2.5.0 version: 2.5.0(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + y-codemirror.next: + specifier: 0.3.5 + version: 0.3.5(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(yjs@13.6.26) + y-protocols: + specifier: ^1.0.6 + version: 1.0.6(yjs@13.6.26) + yjs: + specifier: ^13.6.26 + version: 13.6.26 zod: specifier: ^3.25.76 version: 3.25.76 @@ -4463,9 +4496,18 @@ packages: '@codemirror/commands@6.7.1': resolution: {integrity: sha512-llTrboQYw5H4THfhN4U3qCnSZ1SOJ60ohhz+SzU0ADGtwlc533DtklQP0vSFaQuCPDn3BPpOd1GbbnUtwNjsrw==} + '@codemirror/lang-css@6.3.1': + resolution: {integrity: sha512-kr5fwBGiGtmz6l0LSJIbno9QrifNMUusivHbnA1H6Dmqy4HZFte3UAICix1VuKo0lMPKQr2rqB+0BkKi/S3Ejg==} + + '@codemirror/lang-html@6.4.11': + resolution: {integrity: sha512-9NsXp7Nwp891pQchI7gPdTwBuSuT3K65NGTHWHNJ55HjYcHLllr0rbIZNdOzas9ztc1EUVBlHou85FFZS4BNnw==} + '@codemirror/lang-javascript@6.2.2': resolution: {integrity: sha512-VGQfY+FCc285AhWuwjYxQyUQcYurWlxdKYT4bqwr3Twnd5wP5WSeu52t4tvvuWmljT4EmgEgZCqSieokhtY8hg==} + '@codemirror/lang-markdown@6.5.0': + resolution: {integrity: sha512-0K40bZ35jpHya6FriukbgaleaqzBLZfOh7HuzqbMxBXkbYMJDxfF39c23xOgxFezR+3G+tR2/Mup+Xk865OMvw==} + '@codemirror/lang-sql@6.8.0': resolution: {integrity: sha512-aGLmY4OwGqN3TdSx3h6QeA1NrvaYtF7kkoWR/+W7/JzB0gQtJ+VJxewlnE3+VImhA4WVlhmkJr109PefOOhjLg==} @@ -4493,9 +4535,6 @@ packages: '@codemirror/view@6.35.2': resolution: {integrity: sha512-u04R04XFCYCNaHoNRr37WUUAfnxKPwPdqV+370NiO6i85qB1J/qCD/WbbMJsyJfRWhXIJXAe2BG/oTzAggqv4A==} - '@codemirror/view@6.41.1': - resolution: {integrity: sha512-ToDnWKbBnke+ZLrP6vgTTDScGi5H37YYuZGniQaBzxMVdtCxMrslsmtnOvbPZk4RX9bvkQqnWR/WS/35tJA0qg==} - '@codemirror/view@6.43.0': resolution: {integrity: sha512-V7ZCLQO3Jus9hzh2jVCCPW3mO4IBMr43O37PqSUYautJSnnJF41YlgLw21x0fLJTYvJ+Vkm6Gp+qKGH9pltgXA==} @@ -4678,6 +4717,15 @@ packages: '@tanstack/db': optional: true + '@durable-streams/y-durable-streams@0.2.7': + resolution: {integrity: sha512-AxHQ0PkW4S4+vyPpyWCUy0IRFOp5c+SmprpmORairZEGV4xeF9EdMIhMm/awgqFgddkGdIhlswvuRDTXIZdv6g==} + engines: {node: '>=18.0.0'} + hasBin: true + peerDependencies: + lib0: ^0.2.0 + y-protocols: ^1.0.0 + yjs: ^13.0.0 + '@ecies/ciphers@0.2.5': resolution: {integrity: sha512-GalEZH4JgOMHYYcYmVqnFirFsjZHeoGMDt9IxEnM9F7GRUUyUksJ7Ou53L83WHJq3RWKD3AcBpo0iQh0oMpf8A==} engines: {bun: '>=1', deno: '>=2', node: '>=16'} @@ -6725,12 +6773,18 @@ packages: '@lezer/common@1.5.2': resolution: {integrity: sha512-sxQE460fPZyU3sdc8lafxiPwJHBzZRy/udNFynGQky1SePYBdhkBl1kOagA9uT3pxR8K09bOrmTUqA9wb/PjSQ==} + '@lezer/css@1.3.3': + resolution: {integrity: sha512-RzBo8r+/6QJeow7aPHIpGVIH59xTcJXp399820gZoMo9noQDRVpJLheIBUicYwKcsbOYoBRoLZlf2720dG/4Tg==} + '@lezer/highlight@1.2.1': resolution: {integrity: sha512-Z5duk4RN/3zuVO7Jq0pGLJ3qynpxUVsh7IbUbGj88+uV2ApSAn6kWg2au3iJb+0Zi7kKtqffIESgNcRXWZWmSA==} '@lezer/highlight@1.2.3': resolution: {integrity: sha512-qXdH7UqTvGfdVBINrgKhDsVTJTxactNNxLk7+UMwZhU13lMHaOBlJe9Vqp907ya56Y3+ed2tlqzys7jDkTmW0g==} + '@lezer/html@1.3.13': + resolution: {integrity: sha512-oI7n6NJml729m7pjm9lvLvmXbdoMoi2f+1pwSDJkl9d68zGr7a9Btz8NdHTGQZtW2DA25ybeuv/SyDb9D5tseg==} + '@lezer/javascript@1.4.21': resolution: {integrity: sha512-lL+1fcuxWYPURMM/oFZLEDm0XuLN128QPV+VuGtKpeaOGdcl9F2LYC3nh1S9LkPqx9M0mndZFdXCipNAZpzIkQ==} @@ -6740,6 +6794,9 @@ packages: '@lezer/lr@1.4.2': resolution: {integrity: sha512-pu0K1jCIdnQ12aWNaAVU5bzi7Bd1w54J3ECgANPmYLtQKP0HBj2cE/5coBD66MT10xbtIuUr7tg0Shbsvk0mDA==} + '@lezer/markdown@1.6.4': + resolution: {integrity: sha512-N0SxazMj4k65DBfaf1azqtMZd6u7MqluP84/NZnB/io8Td9aleFmAhz9hcbvSfsxT5tdYlJ5qgv5aMJGY4zEtA==} + '@lmdb/lmdb-darwin-arm64@3.5.4': resolution: {integrity: sha512-Kk4Kz3iyu1QiLsLZBS9Af1eSKUC8VR2T+/jyE2iAyuGw2VwK08pp5iTbZnXn6sWu0LogO/RFktMxOjiDA2sS3w==} cpu: [arm64] @@ -24728,68 +24785,89 @@ snapshots: '@chevrotain/utils@12.0.0': {} - '@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3)': + '@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.2.3)': dependencies: '@codemirror/language': 6.10.6 - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.35.2 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 '@lezer/common': 1.2.3 - '@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.5.2)': + '@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2)': dependencies: '@codemirror/language': 6.10.6 - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.35.2 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 '@lezer/common': 1.5.2 - '@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.41.1)(@lezer/common@1.2.3)': + '@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.2.3)': dependencies: - '@codemirror/language': 6.10.6 - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.41.1 + '@codemirror/language': 6.12.3 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 '@lezer/common': 1.2.3 - '@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)(@lezer/common@1.5.2)': + '@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2)': dependencies: '@codemirror/language': 6.12.3 '@codemirror/state': 6.6.0 - '@codemirror/view': 6.41.1 + '@codemirror/view': 6.43.0 '@lezer/common': 1.5.2 '@codemirror/commands@6.7.1': dependencies: - '@codemirror/language': 6.10.6 - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.35.2 - '@lezer/common': 1.2.3 + '@codemirror/language': 6.12.3 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 + '@lezer/common': 1.5.2 + + '@codemirror/lang-css@6.3.1(@codemirror/view@6.43.0)': + dependencies: + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2) + '@codemirror/language': 6.12.3 + '@codemirror/state': 6.6.0 + '@lezer/common': 1.5.2 + '@lezer/css': 1.3.3 + transitivePeerDependencies: + - '@codemirror/view' + + '@codemirror/lang-html@6.4.11': + dependencies: + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2) + '@codemirror/lang-css': 6.3.1(@codemirror/view@6.43.0) + '@codemirror/lang-javascript': 6.2.2 + '@codemirror/language': 6.12.3 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 + '@lezer/common': 1.5.2 + '@lezer/css': 1.3.3 + '@lezer/html': 1.3.13 '@codemirror/lang-javascript@6.2.2': dependencies: - '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3) + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.2.3) '@codemirror/language': 6.10.6 '@codemirror/lint': 6.8.4 - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.35.2 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 '@lezer/common': 1.2.3 '@lezer/javascript': 1.4.21 - '@codemirror/lang-sql@6.8.0(@codemirror/view@6.35.2)': + '@codemirror/lang-markdown@6.5.0': dependencies: - '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3) - '@codemirror/language': 6.10.6 - '@codemirror/state': 6.5.0 - '@lezer/common': 1.2.3 - '@lezer/highlight': 1.2.1 - '@lezer/lr': 1.4.2 - transitivePeerDependencies: - - '@codemirror/view' + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2) + '@codemirror/lang-html': 6.4.11 + '@codemirror/language': 6.12.3 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 + '@lezer/common': 1.5.2 + '@lezer/markdown': 1.6.4 - '@codemirror/lang-sql@6.8.0(@codemirror/view@6.41.1)': + '@codemirror/lang-sql@6.8.0(@codemirror/view@6.43.0)': dependencies: - '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.41.1)(@lezer/common@1.2.3) - '@codemirror/language': 6.10.6 - '@codemirror/state': 6.5.0 - '@lezer/common': 1.2.3 + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2) + '@codemirror/language': 6.12.3 + '@codemirror/state': 6.6.0 + '@lezer/common': 1.5.2 '@lezer/highlight': 1.2.1 '@lezer/lr': 1.4.2 transitivePeerDependencies: @@ -24797,17 +24875,17 @@ snapshots: '@codemirror/language@6.10.6': dependencies: - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.35.2 - '@lezer/common': 1.2.3 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 + '@lezer/common': 1.5.2 '@lezer/highlight': 1.2.1 '@lezer/lr': 1.4.2 - style-mod: 4.1.2 + style-mod: 4.1.3 '@codemirror/language@6.12.3': dependencies: '@codemirror/state': 6.6.0 - '@codemirror/view': 6.41.1 + '@codemirror/view': 6.43.0 '@lezer/common': 1.5.2 '@lezer/highlight': 1.2.3 '@lezer/lr': 1.4.10 @@ -24815,14 +24893,14 @@ snapshots: '@codemirror/lint@6.8.4': dependencies: - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.41.1 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 crelt: 1.0.6 '@codemirror/search@6.5.8': dependencies: - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.41.1 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 crelt: 1.0.6 '@codemirror/state@6.5.0': @@ -24841,16 +24919,9 @@ snapshots: '@lezer/highlight': 1.2.3 '@codemirror/view@6.35.2': - dependencies: - '@codemirror/state': 6.5.0 - style-mod: 4.1.2 - w3c-keyname: 2.2.8 - - '@codemirror/view@6.41.1': dependencies: '@codemirror/state': 6.6.0 - crelt: 1.0.6 - style-mod: 4.1.3 + style-mod: 4.1.2 w3c-keyname: 2.2.8 '@codemirror/view@6.43.0': @@ -25135,6 +25206,13 @@ snapshots: optionalDependencies: '@tanstack/db': 0.6.7(typescript@5.9.3) + '@durable-streams/y-durable-streams@0.2.7(lib0@0.2.99)(y-protocols@1.0.6(yjs@13.6.26))(yjs@13.6.26)': + dependencies: + '@durable-streams/client': 0.2.6 + lib0: 0.2.99 + y-protocols: 1.0.6(yjs@13.6.26) + yjs: 13.6.26 + '@ecies/ciphers@0.2.5(@noble/ciphers@1.3.0)': dependencies: '@noble/ciphers': 1.3.0 @@ -25185,18 +25263,18 @@ snapshots: '@electric-sql/pglite': 0.4.5 react: 19.2.5 - '@electric-sql/pglite-repl@0.2.17(@babel/runtime@7.29.7)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.5.0)(@codemirror/theme-one-dark@6.1.2)(@electric-sql/pglite@0.2.17)(@lezer/common@1.2.3)(codemirror@6.0.1(@lezer/common@1.2.3))': + '@electric-sql/pglite-repl@0.2.17(@babel/runtime@7.29.7)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/theme-one-dark@6.1.2)(@electric-sql/pglite@0.2.17)(@lezer/common@1.5.2)(codemirror@6.0.1(@lezer/common@1.5.2))': dependencies: - '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3) + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2) '@codemirror/commands': 6.7.1 - '@codemirror/lang-sql': 6.8.0(@codemirror/view@6.35.2) + '@codemirror/lang-sql': 6.8.0(@codemirror/view@6.43.0) '@codemirror/language': 6.10.6 - '@codemirror/view': 6.35.2 + '@codemirror/view': 6.43.0 '@electric-sql/pglite-react': 0.2.17(@electric-sql/pglite@0.2.17)(react@19.2.5) - '@uiw/codemirror-theme-github': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2) - '@uiw/codemirror-theme-xcode': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2) - '@uiw/codemirror-themes': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2) - '@uiw/react-codemirror': 4.23.6(@babel/runtime@7.29.7)(@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3))(@codemirror/language@6.10.6)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.5.0)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.35.2)(codemirror@6.0.1(@lezer/common@1.2.3))(react-dom@19.2.5(react@19.2.5))(react@19.2.5) + '@uiw/codemirror-theme-github': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) + '@uiw/codemirror-theme-xcode': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) + '@uiw/codemirror-themes': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) + '@uiw/react-codemirror': 4.23.6(@babel/runtime@7.29.7)(@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2))(@codemirror/language@6.10.6)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.43.0)(codemirror@6.0.1(@lezer/common@1.5.2))(react-dom@19.2.5(react@19.2.5))(react@19.2.5) psql-describe: 0.1.6 react: 19.2.5 react-dom: 19.2.5(react@19.2.5) @@ -25213,16 +25291,16 @@ snapshots: '@electric-sql/pglite-repl@0.3.5(@babel/runtime@7.29.7)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/theme-one-dark@6.1.2)(@electric-sql/pglite@0.4.5)(@lezer/common@1.5.2)(codemirror@6.0.1(@lezer/common@1.5.2))': dependencies: - '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)(@lezer/common@1.5.2) + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2) '@codemirror/commands': 6.7.1 - '@codemirror/lang-sql': 6.8.0(@codemirror/view@6.41.1) + '@codemirror/lang-sql': 6.8.0(@codemirror/view@6.43.0) '@codemirror/language': 6.12.3 - '@codemirror/view': 6.41.1 + '@codemirror/view': 6.43.0 '@electric-sql/pglite-react': 0.3.5(@electric-sql/pglite@0.4.5)(react@19.2.5) - '@uiw/codemirror-theme-github': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1) - '@uiw/codemirror-theme-xcode': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1) - '@uiw/codemirror-themes': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1) - '@uiw/react-codemirror': 4.23.5(@babel/runtime@7.29.7)(@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)(@lezer/common@1.5.2))(@codemirror/language@6.12.3)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.41.1)(codemirror@6.0.1(@lezer/common@1.5.2))(react-dom@19.2.5(react@19.2.5))(react@19.2.5) + '@uiw/codemirror-theme-github': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) + '@uiw/codemirror-theme-xcode': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) + '@uiw/codemirror-themes': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) + '@uiw/react-codemirror': 4.23.5(@babel/runtime@7.29.7)(@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2))(@codemirror/language@6.12.3)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.43.0)(codemirror@6.0.1(@lezer/common@1.5.2))(react-dom@19.2.5(react@19.2.5))(react@19.2.5) psql-describe: 0.1.6 react: 19.2.5 react-dom: 19.2.5(react@19.2.5) @@ -27325,6 +27403,12 @@ snapshots: '@lezer/common@1.5.2': {} + '@lezer/css@1.3.3': + dependencies: + '@lezer/common': 1.5.2 + '@lezer/highlight': 1.2.3 + '@lezer/lr': 1.4.10 + '@lezer/highlight@1.2.1': dependencies: '@lezer/common': 1.5.2 @@ -27333,9 +27417,15 @@ snapshots: dependencies: '@lezer/common': 1.5.2 + '@lezer/html@1.3.13': + dependencies: + '@lezer/common': 1.5.2 + '@lezer/highlight': 1.2.3 + '@lezer/lr': 1.4.10 + '@lezer/javascript@1.4.21': dependencies: - '@lezer/common': 1.2.3 + '@lezer/common': 1.5.2 '@lezer/highlight': 1.2.1 '@lezer/lr': 1.4.2 @@ -27347,6 +27437,11 @@ snapshots: dependencies: '@lezer/common': 1.5.2 + '@lezer/markdown@1.6.4': + dependencies: + '@lezer/common': 1.5.2 + '@lezer/highlight': 1.2.3 + '@lmdb/lmdb-darwin-arm64@3.5.4': optional: true @@ -27501,7 +27596,7 @@ snapshots: proxy-agent: 6.5.0 typebox: 1.1.34 undici: 7.25.0 - zod-to-json-schema: 3.25.2(zod@3.25.76) + zod-to-json-schema: 3.25.2(zod@4.3.6) transitivePeerDependencies: - '@modelcontextprotocol/sdk' - aws-crt @@ -32914,78 +33009,78 @@ snapshots: '@typescript-eslint/types': 8.46.0 eslint-visitor-keys: 4.2.1 - '@uiw/codemirror-extensions-basic-setup@4.23.5(@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)(@lezer/common@1.5.2))(@codemirror/commands@6.7.1)(@codemirror/language@6.12.3)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)': + '@uiw/codemirror-extensions-basic-setup@4.23.5(@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2))(@codemirror/commands@6.7.1)(@codemirror/language@6.12.3)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)': dependencies: - '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)(@lezer/common@1.5.2) + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2) '@codemirror/commands': 6.7.1 '@codemirror/language': 6.12.3 '@codemirror/lint': 6.8.4 '@codemirror/search': 6.5.8 '@codemirror/state': 6.6.0 - '@codemirror/view': 6.41.1 + '@codemirror/view': 6.43.0 - '@uiw/codemirror-extensions-basic-setup@4.23.6(@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3))(@codemirror/commands@6.7.1)(@codemirror/language@6.10.6)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)': + '@uiw/codemirror-extensions-basic-setup@4.23.6(@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2))(@codemirror/commands@6.7.1)(@codemirror/language@6.10.6)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)': dependencies: - '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3) + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2) '@codemirror/commands': 6.7.1 '@codemirror/language': 6.10.6 '@codemirror/lint': 6.8.4 '@codemirror/search': 6.5.8 - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.35.2 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 - '@uiw/codemirror-theme-github@4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)': + '@uiw/codemirror-theme-github@4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)': dependencies: - '@uiw/codemirror-themes': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1) + '@uiw/codemirror-themes': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) transitivePeerDependencies: - '@codemirror/language' - '@codemirror/state' - '@codemirror/view' - '@uiw/codemirror-theme-github@4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)': + '@uiw/codemirror-theme-github@4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)': dependencies: - '@uiw/codemirror-themes': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2) + '@uiw/codemirror-themes': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) transitivePeerDependencies: - '@codemirror/language' - '@codemirror/state' - '@codemirror/view' - '@uiw/codemirror-theme-xcode@4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)': + '@uiw/codemirror-theme-xcode@4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)': dependencies: - '@uiw/codemirror-themes': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1) + '@uiw/codemirror-themes': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) transitivePeerDependencies: - '@codemirror/language' - '@codemirror/state' - '@codemirror/view' - '@uiw/codemirror-theme-xcode@4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)': + '@uiw/codemirror-theme-xcode@4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)': dependencies: - '@uiw/codemirror-themes': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2) + '@uiw/codemirror-themes': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) transitivePeerDependencies: - '@codemirror/language' - '@codemirror/state' - '@codemirror/view' - '@uiw/codemirror-themes@4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)': + '@uiw/codemirror-themes@4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)': dependencies: '@codemirror/language': 6.12.3 '@codemirror/state': 6.6.0 - '@codemirror/view': 6.41.1 + '@codemirror/view': 6.43.0 - '@uiw/codemirror-themes@4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)': + '@uiw/codemirror-themes@4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)': dependencies: '@codemirror/language': 6.10.6 - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.35.2 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 - '@uiw/react-codemirror@4.23.5(@babel/runtime@7.29.7)(@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)(@lezer/common@1.5.2))(@codemirror/language@6.12.3)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.41.1)(codemirror@6.0.1(@lezer/common@1.5.2))(react-dom@19.2.5(react@19.2.5))(react@19.2.5)': + '@uiw/react-codemirror@4.23.5(@babel/runtime@7.29.7)(@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2))(@codemirror/language@6.12.3)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.43.0)(codemirror@6.0.1(@lezer/common@1.5.2))(react-dom@19.2.5(react@19.2.5))(react@19.2.5)': dependencies: '@babel/runtime': 7.29.7 '@codemirror/commands': 6.7.1 '@codemirror/state': 6.6.0 '@codemirror/theme-one-dark': 6.1.2 - '@codemirror/view': 6.41.1 - '@uiw/codemirror-extensions-basic-setup': 4.23.5(@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)(@lezer/common@1.5.2))(@codemirror/commands@6.7.1)(@codemirror/language@6.12.3)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1) + '@codemirror/view': 6.43.0 + '@uiw/codemirror-extensions-basic-setup': 4.23.5(@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2))(@codemirror/commands@6.7.1)(@codemirror/language@6.12.3)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) codemirror: 6.0.1(@lezer/common@1.5.2) react: 19.2.5 react-dom: 19.2.5(react@19.2.5) @@ -32995,15 +33090,15 @@ snapshots: - '@codemirror/lint' - '@codemirror/search' - '@uiw/react-codemirror@4.23.6(@babel/runtime@7.29.7)(@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3))(@codemirror/language@6.10.6)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.5.0)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.35.2)(codemirror@6.0.1(@lezer/common@1.2.3))(react-dom@19.2.5(react@19.2.5))(react@19.2.5)': + '@uiw/react-codemirror@4.23.6(@babel/runtime@7.29.7)(@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2))(@codemirror/language@6.10.6)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.43.0)(codemirror@6.0.1(@lezer/common@1.5.2))(react-dom@19.2.5(react@19.2.5))(react@19.2.5)': dependencies: '@babel/runtime': 7.29.7 '@codemirror/commands': 6.7.1 - '@codemirror/state': 6.5.0 + '@codemirror/state': 6.6.0 '@codemirror/theme-one-dark': 6.1.2 - '@codemirror/view': 6.35.2 - '@uiw/codemirror-extensions-basic-setup': 4.23.6(@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3))(@codemirror/commands@6.7.1)(@codemirror/language@6.10.6)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2) - codemirror: 6.0.1(@lezer/common@1.2.3) + '@codemirror/view': 6.43.0 + '@uiw/codemirror-extensions-basic-setup': 4.23.6(@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2))(@codemirror/commands@6.7.1)(@codemirror/language@6.10.6)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) + codemirror: 6.0.1(@lezer/common@1.5.2) react: 19.2.5 react-dom: 19.2.5(react@19.2.5) transitivePeerDependencies: @@ -33257,7 +33352,7 @@ snapshots: obug: 2.1.1 std-env: 4.1.0 tinyrainbow: 3.1.0 - vitest: 4.1.5(@opentelemetry/api@1.9.1)(@types/node@22.19.17)(@vitest/coverage-v8@4.1.5)(jsdom@29.1.0(@noble/hashes@2.0.1))(vite@7.3.2(@types/node@22.19.17)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.48.0)(tsx@4.20.3)(yaml@2.9.0)) + vitest: 4.1.5(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(@vitest/coverage-v8@4.1.5)(jsdom@29.1.0(@noble/hashes@2.0.1))(vite@7.1.7(@types/node@25.9.1)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.48.0)(tsx@4.20.3)(yaml@2.9.0)) '@vitest/expect@3.2.4': dependencies: @@ -34876,25 +34971,25 @@ snapshots: codemirror@6.0.1(@lezer/common@1.2.3): dependencies: - '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3) + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.2.3) '@codemirror/commands': 6.7.1 - '@codemirror/language': 6.10.6 + '@codemirror/language': 6.12.3 '@codemirror/lint': 6.8.4 '@codemirror/search': 6.5.8 - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.35.2 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 transitivePeerDependencies: - '@lezer/common' codemirror@6.0.1(@lezer/common@1.5.2): dependencies: - '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.5.2) + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2) '@codemirror/commands': 6.7.1 - '@codemirror/language': 6.10.6 + '@codemirror/language': 6.12.3 '@codemirror/lint': 6.8.4 '@codemirror/search': 6.5.8 - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.35.2 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 transitivePeerDependencies: - '@lezer/common' @@ -46729,6 +46824,13 @@ snapshots: lib0: 0.2.99 yjs: 13.6.26 + y-codemirror.next@0.3.5(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(yjs@13.6.26): + dependencies: + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 + lib0: 0.2.99 + yjs: 13.6.26 + y-indexeddb@9.0.12(yjs@13.6.26): dependencies: lib0: 0.2.99 From 28eae32d7079c4ff3f5b873501e06b5d1a5cb14f Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Sun, 7 Jun 2026 22:48:29 +0100 Subject: [PATCH 2/8] Refine collaborative markdown docs --- AGENTS_MARKDOWN_DOCS_PLAN.md | 51 +++- packages/agents-runtime/src/entity-schema.ts | 16 +- .../test/markdown-docs-tools.test.ts | 5 +- .../src/components/EntityTimeline.tsx | 4 + .../views/MarkdownDocumentView.test.ts | 34 +++ .../components/views/MarkdownDocumentView.tsx | 106 +++++-- .../lib/workspace/workspaceReducer.test.ts | 27 ++ packages/agents-server/package.json | 1 + packages/agents-server/src/entity-manager.ts | 283 +++++++++++++++++- .../agents-server/src/markdown-documents.ts | 17 ++ ...ic-agents-manager-write-validation.test.ts | 69 ++++- .../test/electric-agents-routes.test.ts | 136 ++++++++- .../test/electric-agents-status.test.ts | 40 ++- pnpm-lock.yaml | 17 +- 14 files changed, 751 insertions(+), 55 deletions(-) create mode 100644 packages/agents-server-ui/src/components/views/MarkdownDocumentView.test.ts diff --git a/AGENTS_MARKDOWN_DOCS_PLAN.md b/AGENTS_MARKDOWN_DOCS_PLAN.md index 1bb1965867..ffee9b0977 100644 --- a/AGENTS_MARKDOWN_DOCS_PLAN.md +++ b/AGENTS_MARKDOWN_DOCS_PLAN.md @@ -163,7 +163,8 @@ Tasks: Acceptance: - Entity state can contain a `manifest` event with `kind: 'document'`. -- Existing manifest consumers continue to parse older entries. +- Document manifest entries use the strict shape above; older draft document + manifest shapes are not supported. ### 2. Server Document API @@ -458,7 +459,7 @@ Tasks: ```ts workspace.helpers.openEntity(entityUrl, { viewId: 'markdown-doc', - viewParams: { docId: manifest.id }, + viewParams: { doc: manifest.id }, }) ``` @@ -489,7 +490,7 @@ registerView({ }) ``` -- Resolve `docId` from `viewParams`. +- Resolve `doc` from `viewParams`. - Find document manifest from entity DB. - Construct `Y.Doc`, `Awareness`, and `YjsProvider`. - Use `baseUrl` pointing at the agents server durable-stream proxy. @@ -670,7 +671,7 @@ Tasks: - Add document manifest row rendering. - Add open action using `viewId: 'markdown-doc'` and - `viewParams: { docId }`. + `viewParams: { doc }`. - Register the `markdown-doc` entity view. - Add CodeMirror markdown editor bound to `ydoc.getText('markdown')`. - Pass auth/principal headers to `YjsProvider`. @@ -712,7 +713,7 @@ Tasks: - user edits it. - agent edits it with exact replacement. - two windows/tiles see concurrent updates and presence. - - forked entity receives an independent document. +- forked entity receives an independent document. Suggested commands after `pnpm install` from repo root: @@ -727,3 +728,43 @@ pnpm --filter @electric-ax/agents-server-ui typecheck Streaming edits should be a later design/implementation after the single PR lands and the non-streaming collaborative document workflow is stable. + +## Current Implementation Status + +Implemented in `samwillis/agents-markdown-docs`: + +- Strict document manifest metadata: + - `provider: 'y-durable-streams'` + - `docId` + - `docPath` + - `streamPath` + - `transportMimeType: 'application/vnd.electric-agents.markdown-yjs'` + - `contentMimeType: 'text/markdown'` + - `yTextName: 'markdown'` +- Server-mediated create/read/write/edit document API backed by framed Yjs + updates. +- Runtime markdown document tools with file-like read-before-edit behavior and + diff details. +- Public Yjs document routes and private backing stream routes guarded by entity + permissions, including awareness streams. +- Fork handling for document update streams and remapped `docPath`, `docId`, + and `streamPath` manifest fields. +- CodeMirror markdown editor view backed by `YjsProvider` and `Y.Text`. +- Manifest and context-drawer open/split-right actions for document entries. +- User awareness in the editor and server-published agent awareness around + create/write/edit tools, including status and cursor/edit range. +- Focused runtime, server, and UI tests plus package typechecks for the touched + packages. + +Deferred: + +- Snapshot discovery/compaction implementation beyond the current MVP redirect + behavior. +- Token-by-token/streaming document edits. + +Manual verification note: + +- Automated tests cover the new server, runtime, fork, auth, and UI helper + behavior. A full desktop two-window/two-tile manual pass still needs a running + agents server plus desktop UI; the in-app browser check against the only + detected local UI port was blocked by browser policy after the tab crashed. diff --git a/packages/agents-runtime/src/entity-schema.ts b/packages/agents-runtime/src/entity-schema.ts index 159a27cc6a..2958b0f9e2 100644 --- a/packages/agents-runtime/src/entity-schema.ts +++ b/packages/agents-runtime/src/entity-schema.ts @@ -315,10 +315,13 @@ type ManifestDocumentEntryValue = { key?: string kind: `document` id: string + provider: `y-durable-streams` + docId: string docPath: string streamPath: string - mimeType: `application/vnd.electric-agents.markdown-yjs` + transportMimeType: `application/vnd.electric-agents.markdown-yjs` contentMimeType: `text/markdown` + yTextName: `markdown` title: string createdAt: string createdBy?: string @@ -798,10 +801,15 @@ function createManifestSchema(): Schema< ...timelineOrderField, kind: z.literal(`document`), id: z.string(), + provider: z.literal(`y-durable-streams`), + docId: z.string(), docPath: z.string(), streamPath: z.string(), - mimeType: z.literal(`application/vnd.electric-agents.markdown-yjs`), + transportMimeType: z.literal( + `application/vnd.electric-agents.markdown-yjs` + ), contentMimeType: z.literal(`text/markdown`), + yTextName: z.literal(`markdown`), title: z.string(), createdAt: z.string(), createdBy: z.string().optional(), @@ -944,8 +952,12 @@ export type Manifest = ManifestUnion & { createdBy?: string error?: string meta?: Record + provider?: `y-durable-streams` + docId?: string docPath?: string + transportMimeType?: `application/vnd.electric-agents.markdown-yjs` contentMimeType?: `text/markdown` + yTextName?: `markdown` title?: string updatedAt?: string name?: string diff --git a/packages/agents-runtime/test/markdown-docs-tools.test.ts b/packages/agents-runtime/test/markdown-docs-tools.test.ts index 37b1f3a19a..87150bdf1f 100644 --- a/packages/agents-runtime/test/markdown-docs-tools.test.ts +++ b/packages/agents-runtime/test/markdown-docs-tools.test.ts @@ -6,10 +6,13 @@ function createToolContext() { key: `document:notes`, kind: `document`, id: `notes`, + provider: `y-durable-streams`, + docId: `agents/chat/session/documents/notes`, docPath: `agents/chat/session/documents/notes`, streamPath: `/v1/yjs/default/docs/agents/chat/session/documents/notes`, - mimeType: `application/vnd.electric-agents.markdown-yjs`, + transportMimeType: `application/vnd.electric-agents.markdown-yjs`, contentMimeType: `text/markdown`, + yTextName: `markdown`, title: `Notes`, createdAt: `2026-06-07T00:00:00.000Z`, } as const diff --git a/packages/agents-server-ui/src/components/EntityTimeline.tsx b/packages/agents-server-ui/src/components/EntityTimeline.tsx index 4b5fa2ef52..0870b2f1b5 100644 --- a/packages/agents-server-ui/src/components/EntityTimeline.tsx +++ b/packages/agents-server-ui/src/components/EntityTimeline.tsx @@ -871,6 +871,10 @@ function manifestDetails( return [ { label: `Title`, value: manifest.title }, { label: `MIME`, value: manifest.contentMimeType }, + { label: `Transport`, value: manifest.transportMimeType }, + { label: `Provider`, value: manifest.provider }, + { label: `Y.Text`, value: manifest.yTextName }, + { label: `Doc ID`, value: manifest.docId }, { label: `Path`, value: manifest.docPath }, ] case `context`: diff --git a/packages/agents-server-ui/src/components/views/MarkdownDocumentView.test.ts b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.test.ts new file mode 100644 index 0000000000..66414026db --- /dev/null +++ b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'vitest' +import { markdownDocumentConnectionConfig } from './MarkdownDocumentView' +import type { ManifestDocumentEntry } from '@electric-ax/agents-runtime/client' + +describe(`markdownDocumentConnectionConfig`, () => { + it(`uses explicit provider doc metadata for editor connections`, () => { + const config = markdownDocumentConnectionConfig( + `http://localhost:4437/app`, + { + key: `document:notes`, + kind: `document`, + id: `notes`, + provider: `y-durable-streams`, + docId: `agents/chat/session/documents/notes`, + docPath: `agents/chat/session/documents/notes`, + streamPath: `/v1/yjs/default/docs/agents/chat/session/documents/notes`, + transportMimeType: `application/vnd.electric-agents.markdown-yjs`, + contentMimeType: `text/markdown`, + yTextName: `markdown`, + title: `Notes`, + createdAt: `2026-01-01T00:00:00.000Z`, + } as ManifestDocumentEntry + ) + + expect(config).toMatchObject({ + providerUrl: `http://localhost:4437/app/v1/yjs/default`, + docId: `agents/chat/session/documents/notes`, + yTextName: `markdown`, + }) + expect(config.docUrl.toString()).toBe( + `http://localhost:4437/app/v1/yjs/default/docs/agents/chat/session/documents/notes` + ) + }) +}) diff --git a/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx index 1eb8ccf936..d468f84e50 100644 --- a/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx +++ b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx @@ -5,7 +5,7 @@ import { EditorView, basicSetup } from 'codemirror' import { keymap } from '@codemirror/view' import { YjsProvider } from '@durable-streams/y-durable-streams' import { yCollab, yUndoManagerKeymap } from 'y-codemirror.next' -import { Awareness } from 'y-protocols/awareness' +import { Awareness, removeAwarenessStates } from 'y-protocols/awareness' import * as Y from 'yjs' import { useCurrentPrincipal } from '../../hooks/useCurrentPrincipal' import { getConfiguredServerHeaders, serverFetch } from '../../lib/auth-fetch' @@ -19,6 +19,13 @@ type DocumentResponse = { content: string } +type RemoteUser = { + name: string + status?: string + color?: string + expiresAt?: number +} + function entityApiUrl(baseUrl: string, entityUrl: string, suffix: string): URL { const url = new URL(baseUrl) url.pathname = `${url.pathname.replace(/\/+$/, ``)}/_electric/entities${entityUrl}${suffix}` @@ -50,6 +57,25 @@ function providerBaseUrl(baseUrl: string, streamPath: string): string { return url.toString().replace(/\/+$/, ``) } +export function markdownDocumentConnectionConfig( + baseUrl: string, + documentEntry: ManifestDocumentEntry +): { + providerUrl: string + docUrl: URL + docId: string + yTextName: string +} { + const providerUrl = providerBaseUrl(baseUrl, documentEntry.streamPath) + const docId = documentEntry.docId + return { + providerUrl, + docId, + yTextName: documentEntry.yTextName, + docUrl: new URL(`${providerUrl}/docs/${docId}`), + } +} + export function MarkdownDocumentView({ baseUrl, entityUrl, @@ -58,12 +84,13 @@ export function MarkdownDocumentView({ const documentId = viewParams?.doc ?? null const editorRef = useRef(null) const editorViewRef = useRef(null) + const remoteStateFirstSeenRef = useRef>(new Map()) const [documentEntry, setDocumentEntry] = useState(null) const [status, setStatus] = useState< `loading` | `connecting` | `connected` | `disconnected` | `error` >(`loading`) - const [remoteUsers, setRemoteUsers] = useState>([]) + const [remoteUsers, setRemoteUsers] = useState>([]) const { principal } = useCurrentPrincipal() useEffect(() => { @@ -111,18 +138,17 @@ export function MarkdownDocumentView({ colorLight: userColor.light, }) - const docUrl = new URL( - `${providerBaseUrl(baseUrl, documentEntry.streamPath)}/docs/${documentEntry.docPath}` - ) + const { providerUrl, docUrl, docId, yTextName } = + markdownDocumentConnectionConfig(baseUrl, documentEntry) const provider = new YjsProvider({ doc: ydoc, - baseUrl: providerBaseUrl(baseUrl, documentEntry.streamPath), - docId: documentEntry.docPath, + baseUrl: providerUrl, + docId, awareness, headers: getConfiguredServerHeaders(docUrl), liveMode: `sse`, }) - const ytext = ydoc.getText(`markdown`) + const ytext = ydoc.getText(yTextName) const state = EditorState.create({ doc: ytext.toString(), extensions: [ @@ -137,21 +163,65 @@ export function MarkdownDocumentView({ editorViewRef.current = view const updateRemoteUsers = (): void => { - const names: Array = [] + const users: Array = [] + const staleClients: Array = [] + const seenClients = new Set() + const now = Date.now() awareness.getStates().forEach((state, clientId) => { if (clientId === awareness.clientID) return - const user = (state as { user?: { name?: string } }).user - if (user?.name) names.push(user.name) + seenClients.add(clientId) + const user = ( + state as { + user?: { + name?: string + status?: string + color?: string + role?: string + expiresAt?: number + } + } + ).user + const firstSeen = + remoteStateFirstSeenRef.current.get(clientId) ?? Date.now() + remoteStateFirstSeenRef.current.set(clientId, firstSeen) + const isExpired = + typeof user?.expiresAt === `number` + ? user.expiresAt <= now + : user?.role === `agent` && + user.status === `editing` && + now - firstSeen > 5_000 + if (isExpired) { + staleClients.push(clientId) + return + } + if (user?.name) { + users.push({ + name: user.name, + status: user.status, + color: user.color, + expiresAt: user.expiresAt, + }) + } }) - setRemoteUsers(names) + for (const clientId of remoteStateFirstSeenRef.current.keys()) { + if (!seenClients.has(clientId)) { + remoteStateFirstSeenRef.current.delete(clientId) + } + } + if (staleClients.length > 0) { + removeAwarenessStates(awareness, staleClients, `stale-agent-presence`) + } + setRemoteUsers(users) } const statusHandler = (next: typeof status): void => setStatus(next) provider.on(`status`, statusHandler) awareness.on(`change`, updateRemoteUsers) + const stalePresenceInterval = window.setInterval(updateRemoteUsers, 1_000) provider.connect() setStatus(`connecting`) return () => { + window.clearInterval(stalePresenceInterval) provider.off(`status`, statusHandler) awareness.off(`change`, updateRemoteUsers) provider.destroy() @@ -174,15 +244,17 @@ export function MarkdownDocumentView({
{status} - {remoteUsers.slice(0, 3).map((name) => { - const color = colorFor(name) + {remoteUsers.slice(0, 3).map((user) => { + const color = user.color ?? colorFor(user.name).color return ( - + - {name} + + {user.status ? `${user.name} ยท ${user.status}` : user.name} + ) })} diff --git a/packages/agents-server-ui/src/lib/workspace/workspaceReducer.test.ts b/packages/agents-server-ui/src/lib/workspace/workspaceReducer.test.ts index 19b2a67a8b..c9a08b0418 100644 --- a/packages/agents-server-ui/src/lib/workspace/workspaceReducer.test.ts +++ b/packages/agents-server-ui/src/lib/workspace/workspaceReducer.test.ts @@ -67,6 +67,33 @@ describe(`workspaceReducer`, () => { expect(right.entityUrl).toBe(`/horton/bar`) }) + it(`opens a markdown document view in a right split`, () => { + const after1 = run(EMPTY_WORKSPACE, { + type: `open-tile`, + tile: { entityUrl: `/horton/foo`, viewId: `chat` }, + }) + const fooId = rootAsTile(after1).id + const ws = workspaceReducer(after1, { + type: `open-tile`, + tile: { + entityUrl: `/horton/foo`, + viewId: `markdown-doc`, + viewParams: { doc: `notes` }, + }, + target: { tileId: fooId, position: `split-right` }, + }) + + const split = rootAsSplit(ws) + expect(split.direction).toBe(`horizontal`) + const right = split.children[1].node as Tile + expect(right).toMatchObject({ + entityUrl: `/horton/foo`, + viewId: `markdown-doc`, + viewParams: { doc: `notes` }, + }) + expect(ws.activeTileId).toBe(right.id) + }) + it(`split-up places the new tile above the existing one`, () => { const after1 = run(EMPTY_WORKSPACE, { type: `open-tile`, diff --git a/packages/agents-server/package.json b/packages/agents-server/package.json index 48c7ebc68b..98889e1731 100644 --- a/packages/agents-server/package.json +++ b/packages/agents-server/package.json @@ -65,6 +65,7 @@ "pino-pretty": "^13.0.0", "postgres": "^3.4.0", "undici": "^7.24.7", + "y-protocols": "^1.0.6", "yjs": "^13.6.26" }, "devDependencies": { diff --git a/packages/agents-server/src/entity-manager.ts b/packages/agents-server/src/entity-manager.ts index adfc9cc655..727f32c590 100644 --- a/packages/agents-server/src/entity-manager.ts +++ b/packages/agents-server/src/entity-manager.ts @@ -1,5 +1,6 @@ import { createHash, randomUUID } from 'node:crypto' import fastq from 'fastq' +import { Awareness, encodeAwarenessUpdate } from 'y-protocols/awareness' import * as Y from 'yjs' import { COMPOSER_INPUT_MESSAGE_TYPE, @@ -57,9 +58,12 @@ import { DEFAULT_TENANT_ID } from './tenant.js' import { ATTR, withSpan } from './tracing.js' import { MARKDOWN_DOCUMENT_CONTENT_MIME, + MARKDOWN_DOCUMENT_PROVIDER, + MARKDOWN_DOCUMENT_TEXT_NAME, MARKDOWN_DOCUMENT_TRANSPORT_MIME, assertMarkdownDocumentMatchesEntity, frameYjsUpdate, + getMarkdownDocumentAwarenessStreamPath, getMarkdownDocumentDocPath, getMarkdownDocumentUpdateStreamPath, markdownText, @@ -182,6 +186,8 @@ export interface CreateMarkdownDocumentRequest { export interface UpdateMarkdownDocumentRequest { content: string updatedBy?: string + presenceBefore?: { anchor: number; head: number } + presenceAfter?: { anchor: number; head: number } } export interface EditMarkdownDocumentRequest { @@ -195,10 +201,13 @@ export type ManifestMarkdownDocumentEntry = { key: string kind: `document` id: string + provider: typeof MARKDOWN_DOCUMENT_PROVIDER + docId: string docPath: string streamPath: string - mimeType: typeof MARKDOWN_DOCUMENT_TRANSPORT_MIME + transportMimeType: typeof MARKDOWN_DOCUMENT_TRANSPORT_MIME contentMimeType: typeof MARKDOWN_DOCUMENT_CONTENT_MIME + yTextName: typeof MARKDOWN_DOCUMENT_TEXT_NAME title: string createdAt: string createdBy?: string @@ -275,6 +284,74 @@ function validateMarkdownDocumentId(id: string): void { } } +function principalDisplayName(principalUrl: string): string { + const raw = principalUrl.split(`/principal/`).at(-1) ?? principalUrl + let decoded = raw + try { + decoded = decodeURIComponent(raw) + } catch { + // Fall back to the raw key when the principal URL is not URI encoded. + } + const withoutPrefix = decoded.replace(/^(user|agent|entity|system):/, ``) + return withoutPrefix || decoded || principalUrl +} + +function principalRole(principalUrl: string): `agent` | `user` | `system` { + const raw = principalUrl.split(`/principal/`).at(-1) ?? principalUrl + let decoded = raw + try { + decoded = decodeURIComponent(raw) + } catch { + // Fall back to the raw key when the principal URL is not URI encoded. + } + if (decoded.startsWith(`user:`)) return `user` + if (decoded.startsWith(`system:`)) return `system` + return `agent` +} + +function principalColor(principalUrl: string): { + color: string + colorLight: string +} { + const colors = [ + [`#2563eb`, `#2563eb33`], + [`#059669`, `#05966933`], + [`#dc2626`, `#dc262633`], + [`#7c3aed`, `#7c3aed33`], + [`#c2410c`, `#c2410c33`], + [`#0f766e`, `#0f766e33`], + ] as const + let hash = 0 + for (let i = 0; i < principalUrl.length; i += 1) { + hash = (hash * 31 + principalUrl.charCodeAt(i)) >>> 0 + } + const [color, colorLight] = colors[hash % colors.length]! + return { color, colorLight } +} + +function markdownDocumentPresenceClientId( + docPath: string, + principalUrl: string +): number { + const digest = createHash(`sha256`) + .update(`${docPath}\0${principalUrl}`) + .digest() + const id = digest.readUInt32BE(0) + return id === 0 ? 1 : id +} + +function createMarkdownDocumentAwareness( + docPath: string, + principalUrl: string | undefined +): Awareness { + const awarenessDoc = new Y.Doc() + if (principalUrl) { + ;(awarenessDoc as { clientID: number }).clientID = + markdownDocumentPresenceClientId(docPath, principalUrl) + } + return new Awareness(awarenessDoc) +} + function getEntityAttachmentStreamPath( entityUrl: string, attachmentId: string @@ -2111,6 +2188,7 @@ export class EntityManager { continue } next.docPath = getMarkdownDocumentDocPath(forkUrl, next.id) + next.docId = next.docPath next.streamPath = getEntityMarkdownDocumentUrlPath( this.tenantId, forkUrl, @@ -2677,6 +2755,93 @@ export class EntityManager { ) } + private async publishMarkdownDocumentPresence( + docPath: string, + doc: Y.Doc, + awareness: Awareness, + principalUrl: string | undefined, + status: `editing`, + anchor: number, + head: number, + seq: number + ): Promise { + if (!principalUrl) return + const awarenessPath = getMarkdownDocumentAwarenessStreamPath( + this.tenantId, + docPath, + `default` + ) + const text = markdownText(doc) + const boundedAnchor = Math.max(0, Math.min(anchor, text.length)) + const boundedHead = Math.max(0, Math.min(head, text.length)) + const colors = principalColor(principalUrl) + const now = Date.now() + awareness.setLocalState({ + user: { + name: principalDisplayName(principalUrl), + principalUrl, + role: principalRole(principalUrl), + status, + updatedAt: now, + expiresAt: now + 5_000, + color: colors.color, + colorLight: colors.colorLight, + }, + cursor: { + anchor: Y.createRelativePositionFromTypeIndex(text, boundedAnchor), + head: Y.createRelativePositionFromTypeIndex(text, boundedHead), + }, + }) + await this.streamClient + .create(awarenessPath, { contentType: `application/octet-stream` }) + .catch((error) => { + if (!isStreamCreateConflict(error)) throw error + }) + await this.streamClient.appendBytes( + awarenessPath, + frameYjsUpdate(encodeAwarenessUpdate(awareness, [awareness.clientID])), + { + producerId: `agent-doc-presence-${docPath}`, + epoch: Date.now(), + seq, + } + ) + } + + private async clearMarkdownDocumentPresence( + docPath: string, + awareness: Awareness, + principalUrl: string | undefined, + seq: number + ): Promise { + if (!principalUrl) return + const awarenessPath = getMarkdownDocumentAwarenessStreamPath( + this.tenantId, + docPath, + `default` + ) + awareness.setLocalState(null) + await this.streamClient + .appendBytes( + awarenessPath, + frameYjsUpdate(encodeAwarenessUpdate(awareness, [awareness.clientID])), + { + producerId: `agent-doc-presence-${docPath}`, + epoch: Date.now(), + seq, + } + ) + .catch(() => undefined) + } + + private async bestEffortMarkdownDocumentPresence( + action: () => Promise + ): Promise { + await action().catch((error) => { + serverLog.warn(`[agent-server] markdown document presence failed:`, error) + }) + } + async createMarkdownDocument( entityUrl: string, req: CreateMarkdownDocumentRequest @@ -2710,14 +2875,17 @@ export class EntityManager { key: manifestMarkdownDocumentKey(id), kind: `document`, id, + provider: MARKDOWN_DOCUMENT_PROVIDER, + docId: docPath, docPath, streamPath: getEntityMarkdownDocumentUrlPath( this.tenantId, entityUrl, id ), - mimeType: MARKDOWN_DOCUMENT_TRANSPORT_MIME, + transportMimeType: MARKDOWN_DOCUMENT_TRANSPORT_MIME, contentMimeType: MARKDOWN_DOCUMENT_CONTENT_MIME, + yTextName: MARKDOWN_DOCUMENT_TEXT_NAME, title: req.title.trim() || `Untitled document`, createdAt: now, ...(req.createdBy ? { createdBy: req.createdBy } : {}), @@ -2730,9 +2898,26 @@ export class EntityManager { contentType: `application/octet-stream`, }) streamCreated = true - if (req.content) { + const content = req.content + if (content) { const doc = new Y.Doc() - const update = replaceMarkdownText(doc, req.content) + const awareness = createMarkdownDocumentAwareness( + docPath, + req.createdBy + ) + await this.bestEffortMarkdownDocumentPresence(() => + this.publishMarkdownDocumentPresence( + docPath, + doc, + awareness, + req.createdBy, + `editing`, + 0, + 0, + 0 + ) + ) + const update = replaceMarkdownText(doc, content) await this.streamClient.appendBytes( updateStreamPath, frameYjsUpdate(update), @@ -2742,6 +2927,26 @@ export class EntityManager { seq: 0, } ) + await this.bestEffortMarkdownDocumentPresence(() => + this.publishMarkdownDocumentPresence( + docPath, + doc, + awareness, + req.createdBy, + `editing`, + content.length, + content.length, + 1 + ) + ) + await this.bestEffortMarkdownDocumentPresence(() => + this.clearMarkdownDocumentPresence( + docPath, + awareness, + req.createdBy, + 2 + ) + ) } await this.writeManifestEntry( entityUrl, @@ -2834,16 +3039,55 @@ export class EntityManager { current.document.docPath ) const doc = await readMarkdownYDoc(this.streamClient, updateStreamPath) - const update = replaceMarkdownText(doc, req.content) - await this.streamClient.appendBytes( - updateStreamPath, - frameYjsUpdate(update), - { - producerId: `agent-doc-write-${id}`, - epoch: Date.now(), - seq: 0, - } + const awareness = createMarkdownDocumentAwareness( + current.document.docPath, + req.updatedBy + ) + await this.bestEffortMarkdownDocumentPresence(() => + this.publishMarkdownDocumentPresence( + current.document.docPath, + doc, + awareness, + req.updatedBy, + `editing`, + req.presenceBefore?.anchor ?? current.content.length, + req.presenceBefore?.head ?? current.content.length, + 0 + ) ) + try { + const update = replaceMarkdownText(doc, req.content) + await this.streamClient.appendBytes( + updateStreamPath, + frameYjsUpdate(update), + { + producerId: `agent-doc-write-${id}`, + epoch: Date.now(), + seq: 0, + } + ) + await this.bestEffortMarkdownDocumentPresence(() => + this.publishMarkdownDocumentPresence( + current.document.docPath, + doc, + awareness, + req.updatedBy, + `editing`, + req.presenceAfter?.anchor ?? req.content.length, + req.presenceAfter?.head ?? req.content.length, + 1 + ) + ) + } finally { + await this.bestEffortMarkdownDocumentPresence(() => + this.clearMarkdownDocumentPresence( + current.document.docPath, + awareness, + req.updatedBy, + 2 + ) + ) + } const txid = randomUUID() const nextDocument: ManifestMarkdownDocumentEntry = { ...current.document, @@ -2894,9 +3138,22 @@ export class EntityManager { const content = req.replaceAll ? current.content.split(req.oldString).join(req.newString) : current.content.replace(req.oldString, req.newString) + const index = current.content.indexOf(req.oldString) + const finalIndex = + req.replaceAll && req.newString.length > 0 + ? content.lastIndexOf(req.newString) + req.newString.length + : index + req.newString.length return await this.writeMarkdownDocument(entityUrl, id, { content, updatedBy: req.updatedBy, + presenceBefore: { + anchor: index, + head: index, + }, + presenceAfter: { + anchor: finalIndex, + head: finalIndex, + }, }) } diff --git a/packages/agents-server/src/markdown-documents.ts b/packages/agents-server/src/markdown-documents.ts index 838187de5c..8a0d63e907 100644 --- a/packages/agents-server/src/markdown-documents.ts +++ b/packages/agents-server/src/markdown-documents.ts @@ -1,5 +1,6 @@ import * as decoding from 'lib0/decoding' import * as encoding from 'lib0/encoding' +import { applyAwarenessUpdate } from 'y-protocols/awareness' import * as Y from 'yjs' import type { ElectricAgentsEntity } from './electric-agents-types.js' import type { StreamClient } from './stream-client.js' @@ -8,6 +9,7 @@ export const MARKDOWN_DOCUMENT_TRANSPORT_MIME = `application/vnd.electric-agents.markdown-yjs` as const export const MARKDOWN_DOCUMENT_CONTENT_MIME = `text/markdown` as const export const MARKDOWN_DOCUMENT_TEXT_NAME = `markdown` as const +export const MARKDOWN_DOCUMENT_PROVIDER = `y-durable-streams` as const export interface ParsedMarkdownDocumentPath { entityType: string @@ -117,6 +119,21 @@ export function applyFramedYjsUpdates(doc: Y.Doc, data: Uint8Array): void { } } +export function applyFramedAwarenessUpdates( + awareness: Parameters[0], + data: Uint8Array +): void { + if (data.length === 0) return + const decoder = decoding.createDecoder(data) + while (decoding.hasContent(decoder)) { + applyAwarenessUpdate( + awareness, + decoding.readVarUint8Array(decoder), + `server` + ) + } +} + export async function readMarkdownYDoc( streamClient: StreamClient, updateStreamPath: string diff --git a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts index 2896f9e355..5a2f41b084 100644 --- a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts +++ b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts @@ -1,9 +1,15 @@ import { describe, expect, it, vi } from 'vitest' +import { Awareness } from 'y-protocols/awareness' +import * as Y from 'yjs' import { EntityManager } from '../src/entity-manager' import { SchemaValidator } from '../src/electric-agents/schema-validator' import { MARKDOWN_DOCUMENT_CONTENT_MIME, + MARKDOWN_DOCUMENT_PROVIDER, + MARKDOWN_DOCUMENT_TEXT_NAME, MARKDOWN_DOCUMENT_TRANSPORT_MIME, + applyFramedAwarenessUpdates, + getMarkdownDocumentAwarenessStreamPath, } from '../src/markdown-documents' const observedItemSchema = { @@ -154,9 +160,32 @@ function createMarkdownDocumentManager() { } as any, }), streamClient, + binaryStreams, } } +function collectAwarenessStates(frames: Array) { + const doc = new Y.Doc() + const awareness = new Awareness(doc) + const states: Array<{ + user: Record + cursor?: { anchor?: unknown; head?: unknown } + }> = [] + for (const frame of frames) { + applyFramedAwarenessUpdates(awareness, frame) + for (const state of awareness.getStates().values()) { + const entry = state as { + user?: Record + cursor?: { anchor?: unknown; head?: unknown } + } + if (entry.user) { + states.push({ user: entry.user, cursor: entry.cursor }) + } + } + } + return states +} + describe(`ElectricAgentsManager.validateWriteEvent`, () => { it(`validates delete events against old_value instead of value`, async () => { const manager = createManager() @@ -205,25 +234,29 @@ describe(`ElectricAgentsManager.validateWriteEvent`, () => { describe(`ElectricAgentsManager markdown documents`, () => { it(`stores markdown as framed Yjs updates and exposes a manifest document entry`, async () => { - const { manager, streamClient } = createMarkdownDocumentManager() + const { manager, streamClient, binaryStreams } = + createMarkdownDocumentManager() const created = await manager.createMarkdownDocument(`/chat/session-1`, { id: `notes`, title: `Session notes`, content: `# Notes\n\nDraft`, - createdBy: `/principal/user:u1`, + createdBy: `/principal/agent:horton`, }) expect(created.document).toMatchObject({ key: `document:notes`, kind: `document`, id: `notes`, + provider: MARKDOWN_DOCUMENT_PROVIDER, + docId: `agents/chat/session-1/documents/notes`, docPath: `agents/chat/session-1/documents/notes`, streamPath: `/v1/yjs/default/docs/agents/chat/session-1/documents/notes`, - mimeType: MARKDOWN_DOCUMENT_TRANSPORT_MIME, + transportMimeType: MARKDOWN_DOCUMENT_TRANSPORT_MIME, contentMimeType: MARKDOWN_DOCUMENT_CONTENT_MIME, + yTextName: MARKDOWN_DOCUMENT_TEXT_NAME, title: `Session notes`, - createdBy: `/principal/user:u1`, + createdBy: `/principal/agent:horton`, }) expect(streamClient.create).toHaveBeenCalledWith( `/yjs/default/docs/agents/chat/session-1/documents/notes/.updates`, @@ -240,6 +273,7 @@ describe(`ElectricAgentsManager markdown documents`, () => { await manager.editMarkdownDocument(`/chat/session-1`, `notes`, { oldString: `Draft`, newString: `Ready`, + updatedBy: `/principal/agent:horton`, }) await expect( @@ -247,6 +281,33 @@ describe(`ElectricAgentsManager markdown documents`, () => { ).resolves.toMatchObject({ content: `# Notes\n\nReady`, }) + + const awarenessPath = getMarkdownDocumentAwarenessStreamPath( + `default`, + `agents/chat/session-1/documents/notes`, + `default` + ) + const states = collectAwarenessStates( + binaryStreams.get(awarenessPath) ?? [] + ) + expect(states.map((state) => state.user)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: `horton`, + role: `agent`, + status: `editing`, + principalUrl: `/principal/agent:horton`, + }), + ]) + ) + expect(states.every((state) => state.user.status === `editing`)).toBe(true) + expect( + states.every( + (state) => + JSON.stringify(state.cursor?.anchor) === + JSON.stringify(state.cursor?.head) + ) + ).toBe(true) }) }) diff --git a/packages/agents-server/test/electric-agents-routes.test.ts b/packages/agents-server/test/electric-agents-routes.test.ts index e2fad7c12d..a79111616a 100644 --- a/packages/agents-server/test/electric-agents-routes.test.ts +++ b/packages/agents-server/test/electric-agents-routes.test.ts @@ -30,7 +30,8 @@ async function routeResponse( key: `system:dev-local`, url: `/principal/system:dev-local`, }, - headers?: HeadersInit + headers?: HeadersInit, + ctxOverrides: Partial = {} ): Promise { const result = await globalRouter.fetch( createRequest(method, path, body, rawBody, headers), @@ -39,6 +40,7 @@ async function routeResponse( entityManager: manager, isShuttingDown: () => false, principal, + ...ctxOverrides, } as unknown as TenantContext ) expect(result).toBeInstanceOf(Response) @@ -254,10 +256,13 @@ describe(`ElectricAgentsRoutes markdown document endpoints`, () => { key: `document:notes`, kind: `document`, id: `notes`, + provider: `y-durable-streams`, + docId: `agents/chat/test/documents/notes`, docPath: `agents/chat/test/documents/notes`, streamPath: `/v1/yjs/test/docs/agents/chat/test/documents/notes`, - mimeType: `application/vnd.electric-agents.markdown-yjs`, + transportMimeType: `application/vnd.electric-agents.markdown-yjs`, contentMimeType: `text/markdown`, + yTextName: `markdown`, title: `Notes`, createdAt: `2026-01-01T00:00:00.000Z`, } @@ -334,6 +339,133 @@ describe(`ElectricAgentsRoutes markdown document endpoints`, () => { } ) }) + + it(`guards public Yjs document routes and forwards authorized document streams`, async () => { + const fetchSpy = vi + .spyOn(globalThis, `fetch`) + .mockResolvedValue(new Response(null, { status: 204 })) + const manager = { + registry: { + getEntity: vi.fn().mockResolvedValue({ + url: `/chat/test`, + created_by: `/principal/user:owner`, + }), + hasEntityPermission: vi.fn().mockResolvedValue(false), + pruneExpiredPermissionGrants: vi.fn(), + }, + isForkWorkLockedEntity: vi.fn().mockReturnValue(false), + } as any + + try { + const denied = await routeResponse( + manager, + `GET`, + `/v1/yjs/test/docs/agents/chat/test/documents/notes`, + undefined, + false, + { + kind: `user`, + id: `other`, + key: `user:other`, + url: `/principal/user:other`, + } + ) + expect(denied.status).toBe(401) + expect(fetchSpy).not.toHaveBeenCalled() + + const allowed = await routeResponse( + manager, + `GET`, + `/v1/yjs/test/docs/agents/chat/test/documents/notes?offset=-1`, + undefined, + false, + undefined, + undefined, + { + durableStreamsUrl: `http://durable.local`, + } + ) + expect(allowed.status).toBe(204) + expect(fetchSpy).toHaveBeenCalledOnce() + const [url, init] = fetchSpy.mock.calls[0]! + expect(String(url)).toContain( + `/yjs/test/docs/agents/chat/test/documents/notes/.updates?offset=-1` + ) + expect(init).toMatchObject({ method: `GET` }) + } finally { + fetchSpy.mockRestore() + } + }) + + it(`guards private Yjs document streams and forwards authorized awareness streams`, async () => { + const fetchSpy = vi + .spyOn(globalThis, `fetch`) + .mockResolvedValue(new Response(null, { status: 204 })) + const endRead = vi.fn() + const manager = { + registry: { + getEntity: vi.fn().mockResolvedValue({ + url: `/chat/test`, + created_by: `/principal/user:owner`, + }), + hasEntityPermission: vi.fn().mockResolvedValue(false), + pruneExpiredPermissionGrants: vi.fn(), + }, + isForkWorkLockedEntity: vi.fn().mockReturnValue(false), + } as any + + try { + const denied = await routeResponse( + manager, + `GET`, + `/yjs/test/docs/agents/chat/test/documents/notes/.awareness/default`, + undefined, + false, + { + kind: `user`, + id: `other`, + key: `user:other`, + url: `/principal/user:other`, + }, + undefined, + { + durableStreamsUrl: `http://durable.local`, + entityBridgeManager: { + beginClientRead: vi.fn().mockResolvedValue(endRead), + touchByStreamPath: vi.fn(), + } as any, + } + ) + expect(denied.status).toBe(401) + expect(fetchSpy).not.toHaveBeenCalled() + + const allowed = await routeResponse( + manager, + `GET`, + `/yjs/test/docs/agents/chat/test/documents/notes/.awareness/default`, + undefined, + false, + undefined, + undefined, + { + durableStreamsUrl: `http://durable.local`, + entityBridgeManager: { + beginClientRead: vi.fn().mockResolvedValue(endRead), + touchByStreamPath: vi.fn(), + } as any, + } + ) + expect(allowed.status).toBe(204) + expect(fetchSpy).toHaveBeenCalledOnce() + const [url, init] = fetchSpy.mock.calls[0]! + expect(String(url)).toContain( + `/yjs/test/docs/agents/chat/test/documents/notes/.awareness/default` + ) + expect(init).toMatchObject({ method: `GET` }) + } finally { + fetchSpy.mockRestore() + } + }) }) describe(`ElectricAgentsRoutes cron stream ensure endpoint`, () => { diff --git a/packages/agents-server/test/electric-agents-status.test.ts b/packages/agents-server/test/electric-agents-status.test.ts index 0d2b1643b0..79862c236d 100644 --- a/packages/agents-server/test/electric-agents-status.test.ts +++ b/packages/agents-server/test/electric-agents-status.test.ts @@ -446,6 +446,25 @@ describe(`ElectricAgentsManager.forkSubtree`, () => { }, }, }, + { + type: `manifest`, + key: `document:notes`, + headers: { operation: `insert` }, + value: { + key: `document:notes`, + kind: `document`, + id: `notes`, + provider: `y-durable-streams`, + docId: `agents/manager/root/documents/notes`, + docPath: `agents/manager/root/documents/notes`, + streamPath: `/v1/yjs/default/docs/agents/manager/root/documents/notes`, + transportMimeType: `application/vnd.electric-agents.markdown-yjs`, + contentMimeType: `text/markdown`, + yTextName: `markdown`, + title: `Notes`, + createdAt: `2026-01-01T00:00:00.000Z`, + }, + }, ] }), exists: vi.fn().mockResolvedValue(false), @@ -501,13 +520,19 @@ describe(`ElectricAgentsManager.forkSubtree`, () => { expect.stringMatching(/^\/_electric\/shared-state\/board-fork-/), `/_electric/shared-state/board` ) + expect(streamClient.fork).toHaveBeenCalledWith( + `/yjs/default/docs/agents/manager/root-copy/documents/notes/.updates`, + `/yjs/default/docs/agents/manager/root/documents/notes/.updates` + ) - const manifestInserts = appendedEvents.filter( + const manifestWrites = appendedEvents.filter( (event) => event.type === `manifest` && - (event.headers as Record).operation === `insert` + [`insert`, `update`].includes( + String((event.headers as Record).operation) + ) ) - expect(manifestInserts).toEqual( + expect(manifestWrites).toEqual( expect.arrayContaining([ expect.objectContaining({ value: expect.objectContaining({ @@ -521,6 +546,15 @@ describe(`ElectricAgentsManager.forkSubtree`, () => { id: expect.stringMatching(/^board-fork-/), }), }), + expect.objectContaining({ + value: expect.objectContaining({ + kind: `document`, + id: `notes`, + docId: `agents/manager/root-copy/documents/notes`, + docPath: `agents/manager/root-copy/documents/notes`, + streamPath: `/v1/yjs/default/docs/agents/manager/root-copy/documents/notes`, + }), + }), ]) ) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 91f2e877d7..1d364430db 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -793,7 +793,7 @@ importers: version: link:../../packages/react-hooks next: specifier: ^14.2.5 - version: 14.2.17(@babel/core@7.29.0)(@opentelemetry/api@1.9.1)(@playwright/test@1.52.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + version: 14.2.17(@opentelemetry/api@1.9.1)(@playwright/test@1.52.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) pg: specifier: ^8.12.0 version: 8.13.1 @@ -1970,6 +1970,9 @@ importers: undici: specifier: ^7.24.7 version: 7.25.0 + y-protocols: + specifier: ^1.0.6 + version: 1.0.6(yjs@13.6.26) yjs: specifier: ^13.6.26 version: 13.6.26 @@ -27586,7 +27589,7 @@ snapshots: '@mariozechner/pi-ai@0.70.2': dependencies: - '@anthropic-ai/sdk': 0.90.0(zod@4.4.3) + '@anthropic-ai/sdk': 0.90.0(zod@4.3.6) '@aws-sdk/client-bedrock-runtime': 3.1037.0 '@google/genai': 1.50.1(@modelcontextprotocol/sdk@1.29.0(zod@4.3.6)) '@mistralai/mistralai': 2.2.1 @@ -33352,7 +33355,7 @@ snapshots: obug: 2.1.1 std-env: 4.1.0 tinyrainbow: 3.1.0 - vitest: 4.1.5(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(@vitest/coverage-v8@4.1.5)(jsdom@29.1.0(@noble/hashes@2.0.1))(vite@7.1.7(@types/node@25.9.1)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.48.0)(tsx@4.20.3)(yaml@2.9.0)) + vitest: 4.1.5(@opentelemetry/api@1.9.1)(@types/node@22.19.17)(@vitest/coverage-v8@4.1.5)(jsdom@29.1.0(@noble/hashes@2.0.1))(vite@7.3.2(@types/node@22.19.17)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.48.0)(tsx@4.20.3)(yaml@2.9.0)) '@vitest/expect@3.2.4': dependencies: @@ -40927,7 +40930,7 @@ snapshots: netmask@2.1.1: {} - next@14.2.17(@babel/core@7.29.0)(@opentelemetry/api@1.9.1)(@playwright/test@1.52.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + next@14.2.17(@opentelemetry/api@1.9.1)(@playwright/test@1.52.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: '@next/env': 14.2.17 '@swc/helpers': 0.5.5 @@ -40937,7 +40940,7 @@ snapshots: postcss: 8.4.31 react: 18.3.1 react-dom: 18.3.1(react@18.3.1) - styled-jsx: 5.1.1(@babel/core@7.29.0)(react@18.3.1) + styled-jsx: 5.1.1(react@18.3.1) optionalDependencies: '@next/swc-darwin-arm64': 14.2.17 '@next/swc-darwin-x64': 14.2.17 @@ -44448,12 +44451,10 @@ snapshots: stylis: 4.3.2 tslib: 2.6.2 - styled-jsx@5.1.1(@babel/core@7.29.0)(react@18.3.1): + styled-jsx@5.1.1(react@18.3.1): dependencies: client-only: 0.0.1 react: 18.3.1 - optionalDependencies: - '@babel/core': 7.29.0 styleq@0.1.3: {} From 6e455e905f73ab308e1225d54015d965b05f2ec1 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Sun, 7 Jun 2026 23:05:15 +0100 Subject: [PATCH 3/8] Polish markdown document editor styling --- .../views/MarkdownDocumentView.module.css | 247 ++++++++++++++++-- .../components/views/MarkdownDocumentView.tsx | 62 ++++- 2 files changed, 281 insertions(+), 28 deletions(-) diff --git a/packages/agents-server-ui/src/components/views/MarkdownDocumentView.module.css b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.module.css index bbb7dacced..defc4a595b 100644 --- a/packages/agents-server-ui/src/components/views/MarkdownDocumentView.module.css +++ b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.module.css @@ -1,19 +1,30 @@ .root { + --markdown-doc-editor-bg: #fff; + display: flex; min-height: 0; height: 100%; flex-direction: column; - background: var(--color-panel, #fff); + background: var(--ds-bg); + color: var(--ds-text-1); + font-family: var(--ds-font-body); } .bar { display: flex; align-items: center; justify-content: space-between; - gap: 12px; - padding: 8px 12px; - border-bottom: 1px solid var(--color-border, #d7dce2); - background: var(--color-bg-subtle, #f7f8fa); + gap: var(--ds-space-3); + min-height: 36px; + box-sizing: border-box; + padding: 6px var(--ds-space-3); + border-top: 1px solid var(--ds-divider); + border-bottom: 1px solid var(--ds-divider); + background: var(--ds-surface); +} + +:global(:root[data-theme='dark']) .root { + --markdown-doc-editor-bg: var(--ds-bg); } .title { @@ -21,36 +32,223 @@ overflow: hidden; text-overflow: ellipsis; white-space: nowrap; - font-size: 13px; + color: var(--ds-text-1); + font-family: var(--ds-font-heading); + font-size: var(--ds-text-sm); + line-height: var(--ds-text-sm-lh); font-weight: 600; } .status { flex: 0 0 auto; - color: var(--color-text-muted, #67717f); - font-size: 12px; + color: var(--ds-text-3); + font-size: var(--ds-text-xs); + line-height: var(--ds-text-xs-lh); } -.editor { +.connectionStatus { + display: inline-flex; + align-items: center; + justify-content: center; + width: var(--ds-icon-sm); + height: var(--ds-icon-sm); + flex: 0 0 auto; + color: var(--ds-text-4); +} + +.connectionStatus[data-status='connected'] { + color: var(--ds-green-11); +} + +.connectionStatus[data-status='connecting'], +.connectionStatus[data-status='loading'] { + color: var(--ds-accent-11); +} + +.connectionStatus[data-status='disconnected'] { + color: var(--ds-text-4); +} + +.connectionStatus[data-status='error'] { + color: var(--ds-red-11); +} + +.editorScrollArea { min-height: 0; flex: 1; + overflow: hidden; + background: var(--markdown-doc-editor-bg); +} + +.editorViewport { + background: var(--markdown-doc-editor-bg); +} + +.editor { + min-height: 100%; + background: var(--markdown-doc-editor-bg); } .editor :global(.cm-editor) { - height: 100%; - font-size: 14px; + min-height: 100%; + background: var(--markdown-doc-editor-bg); + color: var(--ds-text-1); + font-family: var(--ds-font-mono); + font-size: 13px; + line-height: 1.6; +} + +.editor :global(.cm-editor.cm-focused) { + outline: none; } .editor :global(.cm-scroller) { - font-family: - ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, 'Liberation Mono', - 'Courier New', monospace; + font-family: var(--ds-font-mono); + background: var(--markdown-doc-editor-bg); + overflow: visible; +} + +.editor :global(.cm-content) { + min-height: 100%; + padding: 8px 0 36px; + caret-color: var(--ds-accent-11); +} + +.editor :global(.cm-line) { + box-sizing: border-box; + padding: 0 8px; +} + +.editor :global(.cm-cursor) { + border-left-color: var(--ds-accent-11); +} + +.editor :global(.cm-selectionBackground), +.editor :global(.cm-focused .cm-selectionBackground) { + background: var(--ds-accent-a4) !important; +} + +.editor :global(.cm-activeLine) { + background: var(--ds-gray-a2); +} + +.editor :global(.cm-gutters) { + background: var(--ds-bg-subtle); + color: var(--ds-text-4); + border-right: 1px solid var(--ds-divider); + font-family: var(--ds-font-mono); + font-size: var(--ds-text-sm); +} + +.editor :global(.cm-activeLineGutter) { + background: var(--ds-gray-a2); + color: var(--ds-text-2); +} + +.editor :global(.cm-lineNumbers .cm-gutterElement) { + min-width: 24px; + padding: 0 6px; +} + +.editor :global(.cm-foldGutter .cm-gutterElement) { + width: 10px; + padding: 0 2px; + color: var(--ds-text-4); +} + +.editor :global(.cm-matchingBracket), +.editor :global(.cm-nonmatchingBracket) { + background: var(--ds-accent-a3); + color: var(--ds-text-1); +} + +.editor :global(.cm-tooltip), +.editor :global(.cm-tooltip-autocomplete) { + overflow: hidden; + border: 1px solid var(--ds-overlay-border); + border-radius: var(--ds-radius-3); + background: var(--ds-surface-raised); + color: var(--ds-text-1); + box-shadow: var(--ds-overlay-shadow); + font-family: var(--ds-font-body); + font-size: var(--ds-text-sm); +} + +.editor :global(.cm-tooltip-autocomplete ul li[aria-selected]) { + background: var(--ds-bg-hover); + color: var(--ds-text-1); +} + +.editor :global(.cm-panels) { + border-color: var(--ds-divider); + background: var(--ds-surface); + color: var(--ds-text-1); + font-family: var(--ds-font-body); + font-size: var(--ds-text-sm); +} + +.editor :global(.cm-panels-top) { + border-bottom: 1px solid var(--ds-divider); +} + +.editor :global(.cm-panels-bottom) { + border-top: 1px solid var(--ds-divider); +} + +.editor :global(.cm-search) { + display: flex; + align-items: center; + gap: var(--ds-space-2); + padding: 6px var(--ds-space-3); +} + +.editor :global(.cm-search label) { + display: inline-flex; + align-items: center; + gap: 4px; + color: var(--ds-text-2); +} + +.editor :global(.cm-search input) { + min-height: 24px; + box-sizing: border-box; + border: 1px solid var(--ds-border-1); + border-radius: var(--ds-radius-2); + background: var(--ds-input-bg); + color: var(--ds-text-1); + font-family: var(--ds-font-body); + font-size: var(--ds-text-sm); + outline: none; + padding: 2px 8px; +} + +.editor :global(.cm-search input:focus) { + border-color: var(--ds-accent-9); + box-shadow: 0 0 0 1px var(--ds-accent-9); +} + +.editor :global(.cm-search button) { + min-height: 24px; + border: 1px solid transparent; + border-radius: var(--ds-radius-2); + background: transparent; + color: var(--ds-text-2); + cursor: pointer; + font-family: var(--ds-font-body); + font-size: var(--ds-text-sm); + font-weight: 500; + padding: 2px 8px; +} + +.editor :global(.cm-search button:hover) { + background: var(--ds-bg-hover); + color: var(--ds-text-1); } .presence { display: flex; align-items: center; - gap: 6px; + gap: var(--ds-space-2); min-width: 0; } @@ -59,16 +257,19 @@ height: 8px; flex: 0 0 auto; border-radius: 999px; + box-shadow: 0 0 0 1px var(--ds-surface); } .empty { - padding: 16px; - color: var(--color-text-muted, #67717f); - font-size: 13px; + padding: var(--ds-space-5); + color: var(--ds-text-3); + font-family: var(--ds-font-body); + font-size: var(--ds-text-sm); + line-height: var(--ds-text-sm-lh); } :global(.yRemoteSelection) { - opacity: 0.35; + opacity: 0.24; } :global(.yRemoteSelectionHead) { @@ -76,6 +277,7 @@ box-sizing: border-box; height: 1.2em; border-left: 2px solid; + opacity: 0.95; } :global(.yRemoteSelectionHead)::after { @@ -84,11 +286,12 @@ left: -2px; z-index: 10; padding: 1px 4px; - border-radius: 3px; + border-radius: var(--ds-radius-1); color: white; content: attr(data-user-name); - font-family: system-ui, sans-serif; - font-size: 10px; + font-family: var(--ds-font-body); + font-size: var(--ds-text-2xs); line-height: 1.2; white-space: nowrap; + box-shadow: var(--ds-shadow-1); } diff --git a/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx index d468f84e50..968e38a522 100644 --- a/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx +++ b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx @@ -4,21 +4,31 @@ import { EditorState } from '@codemirror/state' import { EditorView, basicSetup } from 'codemirror' import { keymap } from '@codemirror/view' import { YjsProvider } from '@durable-streams/y-durable-streams' +import { Plug, TriangleAlert, Unplug } from 'lucide-react' import { yCollab, yUndoManagerKeymap } from 'y-codemirror.next' import { Awareness, removeAwarenessStates } from 'y-protocols/awareness' import * as Y from 'yjs' import { useCurrentPrincipal } from '../../hooks/useCurrentPrincipal' import { getConfiguredServerHeaders, serverFetch } from '../../lib/auth-fetch' import { principalKeyFromInput } from '../../lib/principals' +import { Icon, ScrollArea } from '../../ui' import styles from './MarkdownDocumentView.module.css' import type { EntityViewProps } from '../../lib/workspace/viewRegistry' import type { ManifestDocumentEntry } from '@electric-ax/agents-runtime/client' +import type { LucideIcon } from 'lucide-react' type DocumentResponse = { document: ManifestDocumentEntry content: string } +type DocumentConnectionStatus = + | `loading` + | `connecting` + | `connected` + | `disconnected` + | `error` + type RemoteUser = { name: string status?: string @@ -57,6 +67,34 @@ function providerBaseUrl(baseUrl: string, streamPath: string): string { return url.toString().replace(/\/+$/, ``) } +function connectionStatusLabel(status: DocumentConnectionStatus): string { + switch (status) { + case `loading`: + return `Loading document` + case `connecting`: + return `Connecting` + case `connected`: + return `Connected` + case `disconnected`: + return `Disconnected` + case `error`: + return `Connection error` + } +} + +function connectionStatusIcon(status: DocumentConnectionStatus): LucideIcon { + switch (status) { + case `error`: + return TriangleAlert + case `disconnected`: + return Unplug + case `loading`: + case `connecting`: + case `connected`: + return Plug + } +} + export function markdownDocumentConnectionConfig( baseUrl: string, documentEntry: ManifestDocumentEntry @@ -87,9 +125,7 @@ export function MarkdownDocumentView({ const remoteStateFirstSeenRef = useRef>(new Map()) const [documentEntry, setDocumentEntry] = useState(null) - const [status, setStatus] = useState< - `loading` | `connecting` | `connected` | `disconnected` | `error` - >(`loading`) + const [status, setStatus] = useState(`loading`) const [remoteUsers, setRemoteUsers] = useState>([]) const { principal } = useCurrentPrincipal() @@ -213,7 +249,8 @@ export function MarkdownDocumentView({ } setRemoteUsers(users) } - const statusHandler = (next: typeof status): void => setStatus(next) + const statusHandler = (next: DocumentConnectionStatus): void => + setStatus(next) provider.on(`status`, statusHandler) awareness.on(`change`, updateRemoteUsers) const stalePresenceInterval = window.setInterval(updateRemoteUsers, 1_000) @@ -243,7 +280,14 @@ export function MarkdownDocumentView({ {documentEntry?.title ?? `Markdown document`}
- {status} + + + {remoteUsers.slice(0, 3).map((user) => { const color = user.color ?? colorFor(user.name).color return ( @@ -263,7 +307,13 @@ export function MarkdownDocumentView({ {status === `error` ? (
Document could not be opened.
) : ( -
+ +
+ )}
) From efdc8303e0768b42c8bdf72e495aac7e59740430 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Mon, 8 Jun 2026 07:42:27 +0100 Subject: [PATCH 4/8] Add collaborative markdown document tools --- .changeset/quiet-markdown-docs.md | 10 + packages/agents-runtime/package.json | 3 + packages/agents-runtime/src/create-handler.ts | 33 +- packages/agents-runtime/src/entity-schema.ts | 43 +- .../agents-runtime/src/entity-timeline.ts | 24 +- packages/agents-runtime/src/markdown-yjs.ts | 220 +++++++ .../agents-runtime/src/outbound-bridge.ts | 140 +++- packages/agents-runtime/src/pi-adapter.ts | 82 ++- packages/agents-runtime/src/process-wake.ts | 22 +- .../src/runtime-server-client.ts | 174 +++-- .../agents-runtime/src/tools/markdown-docs.ts | 611 +++++++++++++++++- packages/agents-runtime/src/types.ts | 48 +- .../test/markdown-docs-tools.test.ts | 212 +++++- .../components/views/MarkdownDocumentView.tsx | 1 - packages/agents-server/src/entity-manager.ts | 392 ----------- .../src/routing/entities-router.ts | 93 +-- ...ic-agents-manager-write-validation.test.ts | 80 +-- .../test/electric-agents-routes.test.ts | 50 +- packages/agents/skills/markdown-docs.md | 19 +- packages/agents/src/agents/horton.ts | 4 +- .../agents/test/horton-system-prompt.test.ts | 2 + .../test/horton-tool-composition.test.ts | 30 + pnpm-lock.yaml | 9 + 23 files changed, 1473 insertions(+), 829 deletions(-) create mode 100644 .changeset/quiet-markdown-docs.md create mode 100644 packages/agents-runtime/src/markdown-yjs.ts diff --git a/.changeset/quiet-markdown-docs.md b/.changeset/quiet-markdown-docs.md new file mode 100644 index 0000000000..6b4f6ee90a --- /dev/null +++ b/.changeset/quiet-markdown-docs.md @@ -0,0 +1,10 @@ +--- +"@electric-ax/agents": patch +"@electric-ax/agents-runtime": patch +"@electric-ax/agents-server": patch +"@electric-ax/agents-server-ui": patch +--- + +Add collaborative markdown document tools backed by Yjs durable streams. + +Horton can create, read, replace, edit, and stream inserts into markdown documents by mutating a wake-local Y.Doc and appending binary Yjs updates to the document stream. The server now keeps markdown document handling thin by creating document streams and serving manifest metadata while document content changes flow through the Yjs stream. diff --git a/packages/agents-runtime/package.json b/packages/agents-runtime/package.json index 893162ef26..8855b4f043 100644 --- a/packages/agents-runtime/package.json +++ b/packages/agents-runtime/package.json @@ -121,10 +121,13 @@ "cron-parser": "^5.5.0", "diff": "^9.0.0", "jsdom": "^28.1.0", + "lib0": "^0.2.99", "pino": "^10.3.1", "pino-pretty": "^13.0.0", "turndown": "^7.2.2", "turndown-plugin-gfm": "^1.0.2", + "y-protocols": "^1.0.6", + "yjs": "^13.6.26", "zod": "^4.3.6", "zod-to-json-schema": "^3.25.2" }, diff --git a/packages/agents-runtime/src/create-handler.ts b/packages/agents-runtime/src/create-handler.ts index a8d2a26d72..cbb21a56c1 100644 --- a/packages/agents-runtime/src/create-handler.ts +++ b/packages/agents-runtime/src/create-handler.ts @@ -21,6 +21,7 @@ import type { HeadersProvider, ManifestDocumentEntry, ProcessWakeConfig, + RuntimePrincipal, WakeNotification, WebhookNotification, } from './types' @@ -73,6 +74,7 @@ export interface RuntimeRouterConfig { createElectricTools?: (context: { entityUrl: string entityType: string + principal?: RuntimePrincipal args: Readonly> db: EntityStreamDBWithActions events: Array @@ -102,27 +104,20 @@ export interface RuntimeRouterConfig { createMarkdownDocument: (opts: { id?: string title: string - content?: string meta?: Record }) => Promise<{ txid: string; document: ManifestDocumentEntry }> - readMarkdownDocument: (opts: { - id: string - }) => Promise<{ document: ManifestDocumentEntry; content: string }> - writeMarkdownDocument: (opts: { id: string; content: string }) => Promise<{ - txid: string - document: ManifestDocumentEntry - content: string - }> - editMarkdownDocument: (opts: { - id: string - oldString: string - newString: string - replaceAll?: boolean - }) => Promise<{ - txid: string - document: ManifestDocumentEntry - content: string - }> + readMarkdownDocumentStream: ( + streamPath: string, + opts?: { offset?: string } + ) => Promise<{ bytes: Uint8Array; offset?: string }> + appendMarkdownDocumentUpdate: ( + streamPath: string, + update: Uint8Array + ) => Promise<{ offset?: string }> + appendMarkdownDocumentAwareness: ( + streamPath: string, + update: Uint8Array + ) => Promise<{ offset?: string }> }) => Array | Promise> /** * Optional observer for background wake failures. Return true to mark the diff --git a/packages/agents-runtime/src/entity-schema.ts b/packages/agents-runtime/src/entity-schema.ts index 2958b0f9e2..5c7e364e34 100644 --- a/packages/agents-runtime/src/entity-schema.ts +++ b/packages/agents-runtime/src/entity-schema.ts @@ -171,12 +171,28 @@ type ToolCallValue = { run_id?: string tool_call_id?: string tool_name: string - status: `started` | `args_complete` | `executing` | `completed` | `failed` + status: + | `started` + | `args_streaming` + | `args_complete` + | `executing` + | `completed` + | `failed` args?: unknown + args_preview?: unknown result?: unknown error?: string duration_ms?: number } +type ToolArgDeltaValue = { + key?: string + tool_call_key: string + tool_call_id?: string + run_id?: string + seq: number + delta: string + content_index?: number +} type ReasoningValue = { key?: string status: `streaming` | `completed` @@ -519,18 +535,33 @@ function createToolCallSchema(): Schema { tool_name: z.string(), status: z.enum([ `started`, + `args_streaming`, `args_complete`, `executing`, `completed`, `failed`, ]), args: z.unknown().optional(), + args_preview: z.unknown().optional(), result: z.unknown().optional(), error: z.string().optional(), duration_ms: z.number().int().optional(), }) } +function createToolArgDeltaSchema(): Schema { + return z.object({ + key: z.string().optional(), + ...timelineOrderField, + tool_call_key: z.string(), + tool_call_id: z.string().optional(), + run_id: z.string().optional(), + seq: z.number().int(), + delta: z.string(), + content_index: z.number().int().optional(), + }) +} + function createReasoningSchema(): Schema { return z.object({ key: z.string().optional(), @@ -887,6 +918,7 @@ export type Step = SequencedPersistedRow export type Text = SequencedPersistedRow export type TextDelta = SequencedPersistedRow export type ToolCall = SequencedPersistedRow +export type ToolArgDelta = SequencedPersistedRow export type Reasoning = SequencedPersistedRow export type ErrorEvent = SequencedPersistedRow export type MessageReceived = SequencedPersistedRow @@ -1011,6 +1043,8 @@ export const BUILT_IN_EVENT_SCHEMAS = { text_delta: createTextDeltaSchema() as unknown as BuiltInEntitySchema, tool_call: createToolCallSchema() as unknown as BuiltInEntitySchema, + tool_arg_delta: + createToolArgDeltaSchema() as unknown as BuiltInEntitySchema, reasoning: createReasoningSchema() as unknown as BuiltInEntitySchema, error: createErrorEventSchema() as unknown as BuiltInEntitySchema, @@ -1047,6 +1081,7 @@ type EntityCollectionsDefinition = { texts: CollectionDefinition textDeltas: CollectionDefinition toolCalls: CollectionDefinition + toolArgDeltas: CollectionDefinition reasoning: CollectionDefinition errors: CollectionDefinition inbox: CollectionDefinition @@ -1095,6 +1130,12 @@ export const builtInCollections: EntityCollectionsDefinition = { type: `tool_call`, primaryKey: `key`, }, + toolArgDeltas: { + schema: + BUILT_IN_EVENT_SCHEMAS.tool_arg_delta as StandardSchemaV1, + type: `tool_arg_delta`, + primaryKey: `key`, + }, reasoning: { schema: BUILT_IN_EVENT_SCHEMAS.reasoning as StandardSchemaV1, type: `reasoning`, diff --git a/packages/agents-runtime/src/entity-timeline.ts b/packages/agents-runtime/src/entity-timeline.ts index 0520982298..295ec24c73 100644 --- a/packages/agents-runtime/src/entity-timeline.ts +++ b/packages/agents-runtime/src/entity-timeline.ts @@ -38,7 +38,13 @@ export type EntityTimelineContentItem = toolCallId: string toolName: string args: Record - status: `started` | `args_complete` | `executing` | `completed` | `failed` + status: + | `started` + | `args_streaming` + | `args_complete` + | `executing` + | `completed` + | `failed` result?: string error?: string isError: boolean @@ -89,7 +95,13 @@ export interface IncludesToolCall { run_id: string order: TimelineOrder tool_name: string - status: `started` | `args_complete` | `executing` | `completed` | `failed` + status: + | `started` + | `args_streaming` + | `args_complete` + | `executing` + | `completed` + | `failed` args?: unknown result?: unknown error?: string @@ -202,7 +214,13 @@ export interface EntityTimelineToolCallItem { order: TimelineOrder tool_call_id?: string tool_name: string - status: `started` | `args_complete` | `executing` | `completed` | `failed` + status: + | `started` + | `args_streaming` + | `args_complete` + | `executing` + | `completed` + | `failed` args?: unknown result?: unknown error?: string diff --git a/packages/agents-runtime/src/markdown-yjs.ts b/packages/agents-runtime/src/markdown-yjs.ts new file mode 100644 index 0000000000..754cc41fb4 --- /dev/null +++ b/packages/agents-runtime/src/markdown-yjs.ts @@ -0,0 +1,220 @@ +import * as decoding from 'lib0/decoding' +import * as encoding from 'lib0/encoding' +import { Awareness, encodeAwarenessUpdate } from 'y-protocols/awareness' +import * as Y from 'yjs' + +export const MARKDOWN_DOCUMENT_TEXT_NAME = `markdown` as const + +export function frameYjsUpdate(update: Uint8Array): Uint8Array { + const encoder = encoding.createEncoder() + encoding.writeVarUint8Array(encoder, update) + return encoding.toUint8Array(encoder) +} + +export function applyFramedYjsUpdates(doc: Y.Doc, data: Uint8Array): void { + if (data.length === 0) return + const decoder = decoding.createDecoder(data) + while (decoding.hasContent(decoder)) { + Y.applyUpdate(doc, decoding.readVarUint8Array(decoder), `agent`) + } +} + +export function markdownText( + doc: Y.Doc, + name: string = MARKDOWN_DOCUMENT_TEXT_NAME +): Y.Text { + return doc.getText(name) +} + +export function createMarkdownYDoc(data: Uint8Array): Y.Doc { + const doc = new Y.Doc() + applyFramedYjsUpdates(doc, data) + return doc +} + +export function replaceMarkdownText( + doc: Y.Doc, + content: string, + textName: string = MARKDOWN_DOCUMENT_TEXT_NAME +): Uint8Array { + const before = Y.encodeStateVector(doc) + const text = markdownText(doc, textName) + doc.transact(() => { + text.delete(0, text.length) + if (content.length > 0) text.insert(0, content) + }, `agent`) + return Y.encodeStateAsUpdate(doc, before) +} + +export function editMarkdownText( + doc: Y.Doc, + oldString: string, + newString: string, + replaceAll: boolean | undefined, + textName: string = MARKDOWN_DOCUMENT_TEXT_NAME +): { + update: Uint8Array + content: string + replacements: number + cursorIndex?: number +} { + const text = markdownText(doc, textName) + const beforeContent = text.toString() + const matches = beforeContent.split(oldString).length - 1 + if (matches === 0 || (!replaceAll && matches > 1)) { + return { + update: new Uint8Array(), + content: beforeContent, + replacements: matches, + } + } + + const before = Y.encodeStateVector(doc) + let cursorIndex = 0 + if (replaceAll) { + let cursor = 0 + doc.transact(() => { + while (true) { + const index = text.toString().indexOf(oldString, cursor) + if (index < 0) break + text.delete(index, oldString.length) + text.insert(index, newString) + cursor = index + newString.length + cursorIndex = cursor + } + }, `agent`) + } else { + const index = beforeContent.indexOf(oldString) + doc.transact(() => { + text.delete(index, oldString.length) + text.insert(index, newString) + }, `agent`) + cursorIndex = index + newString.length + } + return { + update: Y.encodeStateAsUpdate(doc, before), + content: text.toString(), + replacements: matches, + cursorIndex, + } +} + +export function insertMarkdownText( + doc: Y.Doc, + content: string, + opts?: { + index?: number + position?: Y.RelativePosition + textName?: string + } +): { + update: Uint8Array + index: number + nextIndex: number + nextPosition: Y.RelativePosition +} { + const text = markdownText(doc, opts?.textName) + const absolute = opts?.position + ? Y.createAbsolutePositionFromRelativePosition(opts.position, doc) + : null + const index = + absolute && absolute.type === text + ? Math.max(0, Math.min(absolute.index, text.length)) + : Math.max(0, Math.min(opts?.index ?? text.length, text.length)) + const before = Y.encodeStateVector(doc) + if (content.length > 0) { + doc.transact(() => { + text.insert(index, content) + }, `agent`) + } + const nextIndex = index + content.length + return { + update: Y.encodeStateAsUpdate(doc, before), + index, + nextIndex, + nextPosition: Y.createRelativePositionFromTypeIndex(text, nextIndex), + } +} + +export function relativePositionAtMarkdownIndex( + doc: Y.Doc, + index: number, + textName: string = MARKDOWN_DOCUMENT_TEXT_NAME +): Y.RelativePosition { + const text = markdownText(doc, textName) + const boundedIndex = Math.max(0, Math.min(index, text.length)) + return Y.createRelativePositionFromTypeIndex(text, boundedIndex) +} + +export function markdownIndexFromRelativePosition( + doc: Y.Doc, + position: Y.RelativePosition, + textName: string = MARKDOWN_DOCUMENT_TEXT_NAME +): number | undefined { + const text = markdownText(doc, textName) + const absolute = Y.createAbsolutePositionFromRelativePosition(position, doc) + if (!absolute || absolute.type !== text) return undefined + return Math.max(0, Math.min(absolute.index, text.length)) +} + +export function encodeMarkdownAwarenessUpdate(opts: { + doc: Y.Doc + docPath: string + principalUrl: string + name: string + role: `agent` | `user` | `system` + status?: `editing` + anchor?: number + head?: number + color: string + colorLight: string + clear?: boolean + textName?: string +}): Uint8Array { + const awarenessDoc = new Y.Doc() + ;(awarenessDoc as { clientID: number }).clientID = + markdownDocumentPresenceClientId(opts.docPath, opts.principalUrl) + const awareness = new Awareness(awarenessDoc) + if (opts.clear) { + awareness.setLocalState(null) + } else { + const text = markdownText(opts.doc, opts.textName) + const anchor = Math.max( + 0, + Math.min(opts.anchor ?? text.length, text.length) + ) + const head = Math.max(0, Math.min(opts.head ?? anchor, text.length)) + const now = Date.now() + awareness.setLocalState({ + user: { + name: opts.name, + principalUrl: opts.principalUrl, + role: opts.role, + status: opts.status ?? `editing`, + updatedAt: now, + expiresAt: now + 5_000, + color: opts.color, + colorLight: opts.colorLight, + }, + cursor: { + anchor: Y.createRelativePositionFromTypeIndex(text, anchor), + head: Y.createRelativePositionFromTypeIndex(text, head), + }, + }) + } + return frameYjsUpdate(encodeAwarenessUpdate(awareness, [awareness.clientID])) +} + +function markdownDocumentPresenceClientId( + docPath: string, + principalUrl: string +): number { + let hash = 2166136261 + const input = `${docPath}\0${principalUrl}` + for (let i = 0; i < input.length; i += 1) { + hash ^= input.charCodeAt(i) + hash = Math.imul(hash, 16777619) + } + const id = hash >>> 0 + return id === 0 ? 1 : id +} diff --git a/packages/agents-runtime/src/outbound-bridge.ts b/packages/agents-runtime/src/outbound-bridge.ts index 2c81851df1..444a5c1e27 100644 --- a/packages/agents-runtime/src/outbound-bridge.ts +++ b/packages/agents-runtime/src/outbound-bridge.ts @@ -110,6 +110,18 @@ export interface OutboundBridge { onTextStart: () => void onTextDelta: (delta: string) => void onTextEnd: () => void + onToolCallArgsStart( + toolCallId: string, + name: string, + argsPreview?: unknown + ): void + onToolCallArgsDelta( + toolCallId: string, + name: string, + delta: string, + opts?: { contentIndex?: number; argsPreview?: unknown } + ): void + onToolCallArgsEnd(toolCallId: string, name: string, args: unknown): void onToolCallStart(toolCallId: string, name: string, args: unknown): void onToolCallStart(name: string, args: unknown): void onToolCallEnd( @@ -154,7 +166,7 @@ export function createOutboundBridge( let currentTextRunKey: string | null = null const toolCallsById = new Map< string, - { key: string; runKey: string; args: unknown } + { key: string; runKey: string; args: unknown; argSeq: number } >() const legacyToolCallIdsByName = new Map>() const requireActiveRun = (action: string): string => { @@ -165,6 +177,65 @@ export function createOutboundBridge( } return currentRunKey } + const ensureToolCall = ( + toolCallId: string, + name: string, + opts?: { + args?: unknown + argsPreview?: unknown + status?: `started` | `args_streaming` | `args_complete` | `executing` + } + ): { key: string; runKey: string; args: unknown; argSeq: number } => { + const runKey = requireActiveRun(`ensureToolCall`) + const existing = toolCallsById.get(toolCallId) + if (existing) { + if (opts && (`args` in opts || `argsPreview` in opts || opts.status)) { + const nextArgs = `args` in opts ? opts.args : existing.args + if (`args` in opts) existing.args = opts.args + writeEvent( + entityStateSchema.toolCalls.update({ + key: existing.key, + value: { + tool_call_id: toolCallId, + tool_name: name, + status: opts.status ?? `args_streaming`, + args: nextArgs, + ...(opts.argsPreview !== undefined && { + args_preview: opts.argsPreview, + }), + run_id: existing.runKey, + } as never, + }) as ChangeEvent + ) + } + return existing + } + const key = `tc-${counters.tc++}` + persistSeed() + const created = { + key, + runKey, + args: opts && `args` in opts ? opts.args : undefined, + argSeq: 0, + } + toolCallsById.set(toolCallId, created) + writeEvent( + entityStateSchema.toolCalls.insert({ + key, + value: { + tool_call_id: toolCallId, + tool_name: name, + status: opts?.status ?? `started`, + args: created.args, + ...(opts?.argsPreview !== undefined && { + args_preview: opts.argsPreview, + }), + run_id: runKey, + } as never, + }) as ChangeEvent + ) + return created + } return { onRunStart() { @@ -277,15 +348,58 @@ export function createOutboundBridge( ) }, + onToolCallArgsStart( + toolCallId: string, + name: string, + argsPreview?: unknown + ) { + ensureToolCall(toolCallId, name, { + status: `args_streaming`, + argsPreview, + }) + }, + + onToolCallArgsDelta( + toolCallId: string, + name: string, + delta: string, + opts?: { contentIndex?: number; argsPreview?: unknown } + ) { + const toolCall = ensureToolCall(toolCallId, name, { + status: `args_streaming`, + }) + const seq = toolCall.argSeq++ + writeEvent( + entityStateSchema.toolArgDeltas.insert({ + key: `${toolCall.key}:args-${seq}`, + value: { + tool_call_key: toolCall.key, + tool_call_id: toolCallId, + run_id: toolCall.runKey, + seq, + delta, + ...(opts?.contentIndex !== undefined && { + content_index: opts.contentIndex, + }), + } as never, + }) as ChangeEvent + ) + }, + + onToolCallArgsEnd(toolCallId: string, name: string, args: unknown) { + ensureToolCall(toolCallId, name, { + status: `args_complete`, + args, + }) + }, + onToolCallStart( toolCallIdOrName: string, nameOrArgs: string | unknown, maybeArgs?: unknown ) { - const runKey = requireActiveRun(`onToolCallStart`) - const key = `tc-${counters.tc++}` const legacyCall = maybeArgs === undefined - const toolCallId = legacyCall ? key : toolCallIdOrName + const toolCallId = legacyCall ? `tc-${counters.tc}` : toolCallIdOrName const name = legacyCall ? toolCallIdOrName : (nameOrArgs as string) const args = legacyCall ? nameOrArgs : maybeArgs if (legacyCall) { @@ -293,20 +407,10 @@ export function createOutboundBridge( ids.push(toolCallId) legacyToolCallIdsByName.set(name, ids) } - persistSeed() - toolCallsById.set(toolCallId, { key, runKey, args }) - writeEvent( - entityStateSchema.toolCalls.insert({ - key, - value: { - tool_call_id: toolCallId, - tool_name: name, - status: `started`, - args, - run_id: runKey, - } as never, - }) as ChangeEvent - ) + ensureToolCall(toolCallId, name, { + status: `executing`, + args, + }) }, onToolCallEnd( diff --git a/packages/agents-runtime/src/pi-adapter.ts b/packages/agents-runtime/src/pi-adapter.ts index 71c4d0f99d..02918e995e 100644 --- a/packages/agents-runtime/src/pi-adapter.ts +++ b/packages/agents-runtime/src/pi-adapter.ts @@ -17,7 +17,6 @@ import type { ChangeEvent } from '@durable-streams/state' import type { AgentEvent, AgentMessage, - AgentTool, StreamFn, } from '@mariozechner/pi-agent-core' import type { @@ -26,7 +25,12 @@ import type { Provider, SimpleStreamOptions, } from '@mariozechner/pi-ai' -import type { LLMContentBlock, LLMMessage, LLMMessageContent } from './types' +import type { + AgentTool, + LLMContentBlock, + LLMMessage, + LLMMessageContent, +} from './types' // ============================================================================ // Options @@ -284,7 +288,24 @@ export function createPiAgentAdapter( case `message_update`: { const assistantEvent = (event as Record) .assistantMessageEvent as - | { type: string; delta?: string } + | { + type: string + contentIndex?: number + delta?: string + toolCall?: { + id?: string + name?: string + arguments?: Record + } + partial?: { + content?: Array<{ + type?: string + id?: string + name?: string + arguments?: Record + }> + } + } | undefined if (assistantEvent?.type === `text_delta`) { if (!textStarted) { @@ -293,6 +314,61 @@ export function createPiAgentAdapter( } bridge.onTextDelta(assistantEvent.delta ?? ``) textDeltaCount++ + } else if ( + assistantEvent?.type === `toolcall_start` || + assistantEvent?.type === `toolcall_delta` || + assistantEvent?.type === `toolcall_end` + ) { + const contentIndex = assistantEvent.contentIndex + const partialToolCall = + typeof contentIndex === `number` + ? assistantEvent.partial?.content?.[contentIndex] + : undefined + const toolCall = assistantEvent.toolCall ?? partialToolCall + const toolCallId = toolCall?.id + const toolName = toolCall?.name + const argsPreview = toolCall?.arguments + if (toolCallId && toolName) { + if (assistantEvent.type === `toolcall_start`) { + bridge.onToolCallArgsStart( + toolCallId, + toolName, + argsPreview + ) + } else if (assistantEvent.type === `toolcall_delta`) { + const delta = assistantEvent.delta ?? `` + bridge.onToolCallArgsDelta(toolCallId, toolName, delta, { + contentIndex, + argsPreview, + }) + const tool = opts.tools.find( + (candidate) => candidate.name === toolName + ) + if (tool?.onArgsDelta) { + void Promise.resolve( + tool.onArgsDelta({ + toolCallId, + toolName, + contentIndex, + delta, + argsPreview, + }) + ).catch((error) => { + runtimeLog.warn( + logPrefix, + `streaming tool arg hook failed for ${toolName}:`, + error + ) + }) + } + } else { + bridge.onToolCallArgsEnd( + toolCallId, + toolName, + argsPreview + ) + } + } } else { runtimeLog.debug( logPrefix, diff --git a/packages/agents-runtime/src/process-wake.ts b/packages/agents-runtime/src/process-wake.ts index 8d1a224d81..564841f84c 100644 --- a/packages/agents-runtime/src/process-wake.ts +++ b/packages/agents-runtime/src/process-wake.ts @@ -1952,6 +1952,7 @@ export async function processWake( ? await config.createElectricTools({ entityUrl, entityType: typeName, + principal: notification.principal, args: entityArgs, db, events: currentWakeEvents, @@ -1986,21 +1987,12 @@ export async function processWake( entityUrl, ...opts, }), - readMarkdownDocument: (opts) => - serverClient.readMarkdownDocument({ - entityUrl, - ...opts, - }), - writeMarkdownDocument: (opts) => - serverClient.writeMarkdownDocument({ - entityUrl, - ...opts, - }), - editMarkdownDocument: (opts) => - serverClient.editMarkdownDocument({ - entityUrl, - ...opts, - }), + readMarkdownDocumentStream: (streamPath, opts) => + serverClient.readMarkdownDocumentStream(streamPath, opts), + appendMarkdownDocumentUpdate: (streamPath, update) => + serverClient.appendMarkdownDocumentUpdate(streamPath, update), + appendMarkdownDocumentAwareness: (streamPath, update) => + serverClient.appendMarkdownDocumentAwareness(streamPath, update), }) : [] diff --git a/packages/agents-runtime/src/runtime-server-client.ts b/packages/agents-runtime/src/runtime-server-client.ts index 20d6476056..44872ca2be 100644 --- a/packages/agents-runtime/src/runtime-server-client.ts +++ b/packages/agents-runtime/src/runtime-server-client.ts @@ -18,6 +18,13 @@ export type { EntitySignal } from './entity-schema' const ELECTRIC_PRINCIPAL_HEADER = `electric-principal` +function bytesBody(bytes: Uint8Array): ArrayBuffer { + return bytes.buffer.slice( + bytes.byteOffset, + bytes.byteOffset + bytes.byteLength + ) as ArrayBuffer +} + export interface RuntimeServerClientConfig { baseUrl: string fetch?: typeof globalThis.fetch @@ -129,33 +136,20 @@ export interface RuntimeServerClient { entityUrl: string id?: string title: string - content?: string meta?: Record }) => Promise<{ txid: string; document: ManifestDocumentEntry }> - readMarkdownDocument: (options: { - entityUrl: string - id: string - }) => Promise<{ document: ManifestDocumentEntry; content: string }> - writeMarkdownDocument: (options: { - entityUrl: string - id: string - content: string - }) => Promise<{ - txid: string - document: ManifestDocumentEntry - content: string - }> - editMarkdownDocument: (options: { - entityUrl: string - id: string - oldString: string - newString: string - replaceAll?: boolean - }) => Promise<{ - txid: string - document: ManifestDocumentEntry - content: string - }> + readMarkdownDocumentStream: ( + streamPath: string, + opts?: { offset?: string } + ) => Promise<{ bytes: Uint8Array; offset?: string }> + appendMarkdownDocumentUpdate: ( + streamPath: string, + update: Uint8Array + ) => Promise<{ offset?: string }> + appendMarkdownDocumentAwareness: ( + streamPath: string, + update: Uint8Array + ) => Promise<{ offset?: string }> spawnEntity: (options: SpawnEntityOptions) => Promise getEntity: (entityUrl: string) => Promise ensureSharedStateStream: ( @@ -433,19 +427,17 @@ export function createRuntimeServerClient( entityUrl, id, title, - content, meta, }: { entityUrl: string id?: string title: string - content?: string meta?: Record }): Promise<{ txid: string; document: ManifestDocumentEntry }> => { const response = await request(`${entityRpcPath(entityUrl)}/documents`, { method: `POST`, headers: { 'content-type': `application/json` }, - body: JSON.stringify({ id, title, content, meta }), + body: JSON.stringify({ id, title, meta }), }) if (!response.ok) { throw new Error( @@ -458,95 +450,73 @@ export function createRuntimeServerClient( } } - const readMarkdownDocument = async ({ - entityUrl, - id, - }: { - entityUrl: string - id: string - }): Promise<{ document: ManifestDocumentEntry; content: string }> => { - const response = await request( - `${entityRpcPath(entityUrl)}/documents/${encodeURIComponent(id)}`, - { method: `GET` } - ) + const readMarkdownDocumentStream = async ( + streamPath: string, + opts?: { offset?: string } + ): Promise<{ bytes: Uint8Array; offset?: string }> => { + const url = new URL(streamPath, `http://agent-runtime.local`) + if (opts?.offset !== undefined) { + url.searchParams.set(`offset`, opts.offset) + } + const path = `${url.pathname}${url.search}` + const response = await request(path, { method: `GET` }) if (!response.ok) { throw new Error( - `read markdown document ${id} on ${entityUrl} failed (${response.status}): ${await readErrorText(response)}` + `read markdown document stream ${path} failed (${response.status}): ${await readErrorText(response)}` ) } - return (await response.json()) as { - document: ManifestDocumentEntry - content: string + return { + bytes: new Uint8Array(await response.arrayBuffer()), + offset: response.headers.get(`stream-next-offset`) ?? undefined, } } - const writeMarkdownDocument = async ({ - entityUrl, - id, - content, - }: { - entityUrl: string - id: string - content: string - }): Promise<{ - txid: string - document: ManifestDocumentEntry - content: string - }> => { - const response = await request( - `${entityRpcPath(entityUrl)}/documents/${encodeURIComponent(id)}`, - { - method: `PUT`, - headers: { 'content-type': `application/json` }, - body: JSON.stringify({ content }), - } - ) + const appendMarkdownDocumentUpdate = async ( + streamPath: string, + update: Uint8Array + ): Promise<{ offset?: string }> => { + const response = await request(streamPath, { + method: `POST`, + headers: { 'content-type': `application/octet-stream` }, + body: bytesBody(update), + }) if (!response.ok) { throw new Error( - `write markdown document ${id} on ${entityUrl} failed (${response.status}): ${await readErrorText(response)}` + `append markdown document update ${streamPath} failed (${response.status}): ${await readErrorText(response)}` ) } - return (await response.json()) as { - txid: string - document: ManifestDocumentEntry - content: string + return { + offset: response.headers.get(`stream-next-offset`) ?? undefined, } } - const editMarkdownDocument = async ({ - entityUrl, - id, - oldString, - newString, - replaceAll, - }: { - entityUrl: string - id: string - oldString: string - newString: string - replaceAll?: boolean - }): Promise<{ - txid: string - document: ManifestDocumentEntry - content: string - }> => { - const response = await request( - `${entityRpcPath(entityUrl)}/documents/${encodeURIComponent(id)}`, - { - method: `PATCH`, - headers: { 'content-type': `application/json` }, - body: JSON.stringify({ oldString, newString, replaceAll }), - } - ) + const appendMarkdownDocumentAwareness = async ( + streamPath: string, + update: Uint8Array + ): Promise<{ offset?: string }> => { + const awarenessPath = `${streamPath}?awareness=default` + const append = () => + request(awarenessPath, { + method: `POST`, + headers: { 'content-type': `application/octet-stream` }, + body: bytesBody(update), + }) + let response = await append() + if (response.status === 404) { + response = await request(awarenessPath, { + method: `PUT`, + headers: { 'content-type': `application/octet-stream` }, + body: bytesBody(update), + }) + if (response.status === 409) response = await append() + } if (!response.ok) { throw new Error( - `edit markdown document ${id} on ${entityUrl} failed (${response.status}): ${await readErrorText(response)}` + `append markdown document awareness ${streamPath} failed (${response.status}): ${await readErrorText(response)}` ) } - return (await response.json()) as { - txid: string - document: ManifestDocumentEntry - content: string + return { + offset: response.headers.get(`stream-next-offset`) ?? undefined, } } @@ -952,9 +922,9 @@ export function createRuntimeServerClient( createAttachment, readAttachment, createMarkdownDocument, - readMarkdownDocument, - writeMarkdownDocument, - editMarkdownDocument, + readMarkdownDocumentStream, + appendMarkdownDocumentUpdate, + appendMarkdownDocumentAwareness, spawnEntity, getEntity, ensureSharedStateStream, diff --git a/packages/agents-runtime/src/tools/markdown-docs.ts b/packages/agents-runtime/src/tools/markdown-docs.ts index b2487ad54d..2d2739d312 100644 --- a/packages/agents-runtime/src/tools/markdown-docs.ts +++ b/packages/agents-runtime/src/tools/markdown-docs.ts @@ -1,6 +1,20 @@ import { createTwoFilesPatch } from 'diff' import { Type } from '@sinclair/typebox' +import { + applyFramedYjsUpdates, + createMarkdownYDoc, + editMarkdownText, + encodeMarkdownAwarenessUpdate, + frameYjsUpdate, + insertMarkdownText, + markdownIndexFromRelativePosition, + markdownText, + relativePositionAtMarkdownIndex, + replaceMarkdownText, +} from '../markdown-yjs' import type { AgentTool, ProcessWakeConfig } from '../types' +import type { ManifestDocumentEntry } from '../entity-schema' +import type * as Y from 'yjs' type ElectricToolContext = Parameters< NonNullable @@ -10,10 +24,310 @@ function docLabel(id: string): string { return `markdown-doc:${id}` } +type InsertMarkdownArgs = { + id: string + content: string + index?: number +} + +type SetCursorArgs = { + id: string + index?: number + before?: string + after?: string + occurrence?: number +} + +type InsertSession = { + id?: string + inserted: string + nextIndex?: number + nextPosition?: Y.RelativePosition + seq: number + streamed: boolean + pending: Promise + error?: unknown +} + +type MaterializedMarkdownDocument = { + document: ManifestDocumentEntry + doc: Y.Doc + textName: string + streamOffset?: string +} + +function isManifestDocumentEntry( + value: unknown +): value is ManifestDocumentEntry { + if (!value || typeof value !== `object`) return false + const entry = value as Partial + return ( + entry.kind === `document` && + typeof entry.id === `string` && + entry.provider === `y-durable-streams` && + typeof entry.docPath === `string` && + typeof entry.streamPath === `string` && + entry.transportMimeType === + `application/vnd.electric-agents.markdown-yjs` && + entry.contentMimeType === `text/markdown` && + entry.yTextName === `markdown` && + typeof entry.title === `string` + ) +} + +function asInsertArgs(value: unknown): Partial { + if (!value || typeof value !== `object`) return {} + const input = value as Record + return { + ...(typeof input.id === `string` && { id: input.id }), + ...(typeof input.content === `string` && { content: input.content }), + ...(typeof input.index === `number` && Number.isFinite(input.index) + ? { index: input.index } + : {}), + } +} + export function createMarkdownDocumentTools( context: ElectricToolContext ): Array { const readDocs = new Map() + const insertSessions = new Map() + const materializedDocs = new Map() + const cursorPositions = new Map() + + const findManifestDocument = ( + id: string + ): ManifestDocumentEntry | undefined => { + const manifests = context.db.collections.manifests?.toArray as + | Array + | undefined + return manifests?.find( + (entry): entry is ManifestDocumentEntry => + isManifestDocumentEntry(entry) && entry.id === id + ) + } + + const refreshDocument = async ( + id: string, + materialized: MaterializedMarkdownDocument + ): Promise => { + const result = await context.readMarkdownDocumentStream( + materialized.document.streamPath, + materialized.streamOffset + ? { offset: materialized.streamOffset } + : undefined + ) + applyFramedYjsUpdates(materialized.doc, result.bytes) + if (result.offset !== undefined) { + materialized.streamOffset = result.offset + } + readDocs.set(id, contentOf(materialized)) + } + + const materializeDocument = async ( + id: string + ): Promise => { + const cached = materializedDocs.get(id) + if (cached) { + await refreshDocument(id, cached) + return cached + } + const document = findManifestDocument(id) + if (!document) { + throw new Error( + `Markdown document ${JSON.stringify( + id + )} is not in this entity's manifest. Create it with create_markdown_doc first.` + ) + } + const result = await context.readMarkdownDocumentStream(document.streamPath) + const doc = createMarkdownYDoc(result.bytes) + const materialized = { + document, + doc, + textName: document.yTextName, + ...(result.offset !== undefined ? { streamOffset: result.offset } : {}), + } + materializedDocs.set(id, materialized) + readDocs.set(id, markdownText(doc, document.yTextName).toString()) + return materialized + } + + const contentOf = (materialized: MaterializedMarkdownDocument): string => + markdownText(materialized.doc, materialized.textName).toString() + + const cacheEmptyDocument = ( + document: ManifestDocumentEntry + ): MaterializedMarkdownDocument => { + const materialized = { + document, + doc: createMarkdownYDoc(new Uint8Array()), + textName: document.yTextName, + } + materializedDocs.set(document.id, materialized) + readDocs.set(document.id, ``) + return materialized + } + + const appendDocumentUpdate = async ( + id: string, + materialized: MaterializedMarkdownDocument, + update: Uint8Array + ): Promise => { + if (update.length === 0) return + try { + const result = await context.appendMarkdownDocumentUpdate( + materialized.document.streamPath, + frameYjsUpdate(update) + ) + if (result.offset !== undefined) { + materialized.streamOffset = result.offset + } + } catch (error) { + materializedDocs.delete(id) + throw error + } + readDocs.set(id, contentOf(materialized)) + } + + const appendPresence = async ( + materialized: MaterializedMarkdownDocument, + opts: { anchor?: number; head?: number; clear?: boolean } + ): Promise => { + const principalUrl = + context.principal?.url ?? + `/principal/entity:${encodeURIComponent(context.entityUrl)}` + await context + .appendMarkdownDocumentAwareness( + materialized.document.streamPath, + encodeMarkdownAwarenessUpdate({ + doc: materialized.doc, + docPath: materialized.document.docPath, + principalUrl, + name: principalDisplayName(principalUrl), + role: principalRole(principalUrl), + status: `editing`, + anchor: opts.anchor, + head: opts.head, + color: principalColor(principalUrl).color, + colorLight: principalColor(principalUrl).colorLight, + clear: opts.clear, + textName: materialized.textName, + }) + ) + .catch(() => undefined) + } + + const applyInsertChunk = async ( + id: string, + chunk: string, + session: InsertSession, + index?: number + ): Promise => { + const materialized = await materializeDocument(id) + const result = insertMarkdownText(materialized.doc, chunk, { + index: session.nextPosition + ? undefined + : (session.nextIndex ?? (index !== undefined ? index : undefined)), + position: + session.nextPosition ?? + (index === undefined ? cursorPositions.get(id) : undefined), + textName: materialized.textName, + }) + await appendPresence(materialized, { + anchor: result.index, + head: result.index, + }) + await appendDocumentUpdate(id, materialized, result.update) + await appendPresence(materialized, { + anchor: result.nextIndex, + head: result.nextIndex, + }) + session.nextIndex = result.nextIndex + session.nextPosition = result.nextPosition + cursorPositions.set(id, result.nextPosition) + session.streamed = true + } + + const setCursor = async ( + id: string, + index: number + ): Promise<{ materialized: MaterializedMarkdownDocument; index: number }> => { + const materialized = await materializeDocument(id) + const text = markdownText(materialized.doc, materialized.textName) + const boundedIndex = Math.max(0, Math.min(index, text.length)) + const position = relativePositionAtMarkdownIndex( + materialized.doc, + boundedIndex, + materialized.textName + ) + cursorPositions.set(id, position) + return { materialized, index: boundedIndex } + } + + const resolveCursorIndex = ( + content: string, + args: SetCursorArgs + ): { index?: number; error?: string } => { + const locatorCount = + (args.index !== undefined ? 1 : 0) + + (args.before !== undefined ? 1 : 0) + + (args.after !== undefined ? 1 : 0) + if (locatorCount > 1) { + return { error: `Pass only one of index, before, or after.` } + } + if (args.index !== undefined) return { index: args.index } + const needle = args.before ?? args.after + if (needle === undefined) return { index: content.length } + if (needle.length === 0) { + return { error: `before/after must not be empty.` } + } + const occurrence = Math.max(1, Math.floor(args.occurrence ?? 1)) + let from = 0 + let found = -1 + for (let count = 0; count < occurrence; count += 1) { + found = content.indexOf(needle, from) + if (found < 0) { + return { + error: `Could not find occurrence ${occurrence} of ${JSON.stringify( + needle + )}.`, + } + } + from = found + needle.length + } + return { index: args.after !== undefined ? found + needle.length : found } + } + + const enqueueInsert = ( + toolCallId: string, + action: (session: InsertSession) => Promise + ): void => { + const session = + insertSessions.get(toolCallId) ?? + ({ + inserted: ``, + seq: 0, + streamed: false, + pending: Promise.resolve(), + } satisfies InsertSession) + insertSessions.set(toolCallId, session) + session.pending = session.pending + .then(() => action(session)) + .catch((error) => { + session.error = error + }) + } + + const awaitInsertSession = async ( + toolCallId: string + ): Promise => { + const session = insertSessions.get(toolCallId) + if (!session) return undefined + await session.pending + if (session.error) throw session.error + return session + } return [ { @@ -40,9 +354,22 @@ export function createMarkdownDocumentTools( const result = await context.createMarkdownDocument({ id, title, - content, }) - readDocs.set(result.document.id, content ?? ``) + const materialized = cacheEmptyDocument(result.document) + if (content && content.length > 0) { + await appendPresence(materialized, { anchor: 0, head: 0 }) + const update = replaceMarkdownText( + materialized.doc, + content, + materialized.textName + ) + await appendDocumentUpdate(result.document.id, materialized, update) + await appendPresence(materialized, { + anchor: content.length, + head: content.length, + }) + await appendPresence(materialized, { clear: true }) + } return { content: [ { @@ -54,6 +381,159 @@ export function createMarkdownDocumentTools( } }, }, + { + name: `set_markdown_doc_cursor`, + label: `Set Markdown Doc Cursor`, + description: `Set the stateful insertion cursor for a collaborative markdown document. The cursor is stored as a Yjs relative position for this wake, so later insert_markdown_doc calls can stream at that position even if the document changes around it. Pass exactly one of index, before, or after; omit all three to place the cursor at the end.`, + parameters: Type.Object({ + id: Type.String({ description: `Document id.` }), + index: Type.Optional( + Type.Number({ + description: `Optional UTF-16 text offset for the cursor.`, + }) + ), + before: Type.Optional( + Type.String({ + description: `Place the cursor before this literal markdown text.`, + }) + ), + after: Type.Optional( + Type.String({ + description: `Place the cursor after this literal markdown text.`, + }) + ), + occurrence: Type.Optional( + Type.Number({ + description: `1-based occurrence for before/after matching. Defaults to 1.`, + }) + ), + }), + execute: async (_toolCallId, params) => { + const args = params as SetCursorArgs + const materialized = await materializeDocument(args.id) + const content = contentOf(materialized) + const resolved = resolveCursorIndex(content, args) + if (resolved.error || resolved.index === undefined) { + return { + content: [ + { + type: `text` as const, + text: `Error: ${resolved.error ?? `could not resolve cursor`}`, + }, + ], + details: { cursorSet: false }, + } + } + const result = await setCursor(args.id, resolved.index) + return { + content: [ + { + type: `text` as const, + text: `Set markdown document ${args.id} cursor at index ${result.index}`, + }, + ], + details: { + document: result.materialized.document, + cursorSet: true, + index: result.index, + }, + } + }, + }, + { + name: `insert_markdown_doc`, + label: `Insert Markdown Doc`, + description: `Insert markdown into a collaborative app document. When the model streams the content argument, the insertion is applied incrementally to the wake-local Yjs document and appended to the document stream so open editors can watch it appear. Put id and optional index before content in the tool arguments. If index is omitted, the current set_markdown_doc_cursor position is used; if no cursor is set, content is appended.`, + parameters: Type.Object({ + id: Type.String({ description: `Document id.` }), + index: Type.Optional( + Type.Number({ + description: `Optional UTF-16 text offset. Omit to append to the end of the current document.`, + }) + ), + content: Type.String({ description: `Markdown content to insert.` }), + }), + onArgsDelta: ({ toolCallId, argsPreview }) => { + const args = asInsertArgs(argsPreview) + if (!args.id || typeof args.content !== `string`) return + enqueueInsert(toolCallId, async (session) => { + session.id = args.id + if (session.nextIndex === undefined && args.index !== undefined) { + session.nextIndex = args.index + } + if (!args.content!.startsWith(session.inserted)) return + const chunk = args.content!.slice(session.inserted.length) + if (chunk.length === 0) return + session.inserted = args.content! + await applyInsertChunk(args.id!, chunk, session, args.index) + session.seq++ + }) + }, + execute: async (toolCallId, params) => { + const { id, content, index } = params as InsertMarkdownArgs + const session = await awaitInsertSession(toolCallId) + let inserted = session?.inserted ?? `` + let streamed = session?.streamed ?? false + let nextIndex = session?.nextIndex ?? index + + if (content !== inserted) { + if (inserted.length === 0 || content.startsWith(inserted)) { + const remaining = + inserted.length === 0 ? content : content.slice(inserted.length) + if (remaining.length > 0) { + const finalSession = + session ?? + ({ + inserted: ``, + seq: 0, + streamed: false, + pending: Promise.resolve(), + } satisfies InsertSession) + await applyInsertChunk(id, remaining, finalSession, nextIndex) + nextIndex = finalSession.nextIndex + inserted = content + streamed = streamed || remaining.length !== content.length + } + } else { + const materialized = materializedDocs.get(id) + if (materialized) { + await appendPresence(materialized, { clear: true }) + } + insertSessions.delete(toolCallId) + return { + content: [ + { + type: `text` as const, + text: `Error: streamed content diverged from final insert content; no final reconciliation was applied.`, + }, + ], + details: { inserted: inserted.length, expected: content.length }, + } + } + } + + const materialized = await materializeDocument(id) + await appendPresence(materialized, { clear: true }) + const finalContent = contentOf(materialized) + readDocs.set(id, finalContent) + insertSessions.delete(toolCallId) + return { + content: [ + { + type: `text` as const, + text: `Inserted ${content.length} characters into markdown document ${id}`, + }, + ], + details: { + document: materialized.document, + streamed, + insertedBytes: new TextEncoder().encode(content).length, + nextIndex, + }, + } + }, + executionMode: `sequential`, + }, { name: `read_markdown_doc`, label: `Read Markdown Doc`, @@ -63,18 +543,27 @@ export function createMarkdownDocumentTools( }), execute: async (_toolCallId, params) => { const { id } = params as { id: string } - const result = await context.readMarkdownDocument({ id }) - readDocs.set(id, result.content) + const materialized = await materializeDocument(id) + const content = contentOf(materialized) + const cursorIndex = cursorPositions.has(id) + ? markdownIndexFromRelativePosition( + materialized.doc, + cursorPositions.get(id)!, + materialized.textName + ) + : undefined + readDocs.set(id, content) return { content: [ { type: `text` as const, - text: result.content, + text: content, }, ], details: { - document: result.document, - bytes: new TextEncoder().encode(result.content).length, + document: materialized.document, + bytes: new TextEncoder().encode(content).length, + cursorIndex, }, } }, @@ -89,10 +578,20 @@ export function createMarkdownDocumentTools( }), execute: async (_toolCallId, params) => { const { id, content } = params as { id: string; content: string } - const before = - readDocs.get(id) ?? - (await context.readMarkdownDocument({ id })).content - const result = await context.writeMarkdownDocument({ id, content }) + const materialized = await materializeDocument(id) + const before = contentOf(materialized) + await appendPresence(materialized, { anchor: 0, head: 0 }) + const update = replaceMarkdownText( + materialized.doc, + content, + materialized.textName + ) + await appendDocumentUpdate(id, materialized, update) + await appendPresence(materialized, { + anchor: content.length, + head: content.length, + }) + await appendPresence(materialized, { clear: true }) readDocs.set(id, content) const diff = createTwoFilesPatch( docLabel(id), @@ -110,14 +609,15 @@ export function createMarkdownDocumentTools( text: `Wrote markdown document ${id}`, }, ], - details: { document: result.document, txid: result.txid, diff }, + details: { document: materialized.document, diff }, } }, + executionMode: `sequential`, }, { name: `edit_markdown_doc`, label: `Edit Markdown Doc`, - description: `Replace text in a collaborative app markdown document, not a filesystem file. The document must be read with read_markdown_doc earlier in this wake. By default old_string must occur exactly once; set replace_all to true to replace every occurrence.`, + description: `Replace text in a collaborative app markdown document by appending a Yjs update, not by writing a filesystem file. Read the document first when you need to inspect current content. By default old_string must occur exactly once; set replace_all to true to replace every occurrence.`, parameters: Type.Object({ id: Type.String({ description: `Document id.` }), old_string: Type.String({ @@ -135,18 +635,8 @@ export function createMarkdownDocumentTools( new_string: string replace_all?: boolean } - const before = readDocs.get(id) - if (before === undefined) { - return { - content: [ - { - type: `text` as const, - text: `Document ${id} has not been read in this wake; call read_markdown_doc first.`, - }, - ], - details: { replacements: 0 }, - } - } + const materialized = await materializeDocument(id) + const before = contentOf(materialized) const matches = before.split(old_string).length - 1 if (matches === 0) { @@ -169,12 +659,21 @@ export function createMarkdownDocumentTools( } } - const result = await context.editMarkdownDocument({ - id, - oldString: old_string, - newString: new_string, - replaceAll: replace_all, + const index = before.indexOf(old_string) + await appendPresence(materialized, { anchor: index, head: index }) + const result = editMarkdownText( + materialized.doc, + old_string, + new_string, + replace_all, + materialized.textName + ) + await appendDocumentUpdate(id, materialized, result.update) + await appendPresence(materialized, { + anchor: result.cursorIndex, + head: result.cursorIndex, }) + await appendPresence(materialized, { clear: true }) readDocs.set(id, result.content) const diff = createTwoFilesPatch( docLabel(id), @@ -194,9 +693,59 @@ export function createMarkdownDocumentTools( }`, }, ], - details: { replacements: matches, document: result.document, diff }, + details: { + replacements: matches, + document: materialized.document, + diff, + }, } }, + executionMode: `sequential`, }, ] } + +function principalDisplayName(principalUrl: string): string { + const raw = principalUrl.split(`/principal/`).at(-1) ?? principalUrl + let decoded = raw + try { + decoded = decodeURIComponent(raw) + } catch { + // Keep the raw value when the URL segment is not URI encoded. + } + const withoutPrefix = decoded.replace(/^(user|agent|entity|system):/, ``) + return withoutPrefix || decoded || principalUrl +} + +function principalRole(principalUrl: string): `agent` | `user` | `system` { + const raw = principalUrl.split(`/principal/`).at(-1) ?? principalUrl + let decoded = raw + try { + decoded = decodeURIComponent(raw) + } catch { + // Keep the raw value when the URL segment is not URI encoded. + } + if (decoded.startsWith(`user:`)) return `user` + if (decoded.startsWith(`system:`)) return `system` + return `agent` +} + +function principalColor(principalUrl: string): { + color: string + colorLight: string +} { + const colors = [ + [`#2563eb`, `#2563eb33`], + [`#059669`, `#05966933`], + [`#dc2626`, `#dc262633`], + [`#7c3aed`, `#7c3aed33`], + [`#c2410c`, `#c2410c33`], + [`#0f766e`, `#0f766e33`], + ] as const + let hash = 0 + for (let i = 0; i < principalUrl.length; i += 1) { + hash = (hash * 31 + principalUrl.charCodeAt(i)) >>> 0 + } + const [color, colorLight] = colors[hash % colors.length]! + return { color, colorLight } +} diff --git a/packages/agents-runtime/src/types.ts b/packages/agents-runtime/src/types.ts index e935c01909..0ab6636d17 100644 --- a/packages/agents-runtime/src/types.ts +++ b/packages/agents-runtime/src/types.ts @@ -387,6 +387,7 @@ export type TimelineItem = error: string | null status: | `started` + | `args_streaming` | `args_complete` | `executing` | `completed` @@ -717,6 +718,7 @@ export interface ProcessWakeConfig { createElectricTools?: (context: { entityUrl: string entityType: string + principal?: RuntimePrincipal args: Readonly> db: EntityStreamDBWithActions events: Array @@ -746,27 +748,20 @@ export interface ProcessWakeConfig { createMarkdownDocument: (opts: { id?: string title: string - content?: string meta?: Record }) => Promise<{ txid: string; document: ManifestDocumentEntry }> - readMarkdownDocument: (opts: { - id: string - }) => Promise<{ document: ManifestDocumentEntry; content: string }> - writeMarkdownDocument: (opts: { id: string; content: string }) => Promise<{ - txid: string - document: ManifestDocumentEntry - content: string - }> - editMarkdownDocument: (opts: { - id: string - oldString: string - newString: string - replaceAll?: boolean - }) => Promise<{ - txid: string - document: ManifestDocumentEntry - content: string - }> + readMarkdownDocumentStream: ( + streamPath: string, + opts?: { offset?: string } + ) => Promise<{ bytes: Uint8Array; offset?: string }> + appendMarkdownDocumentUpdate: ( + streamPath: string, + update: Uint8Array + ) => Promise<{ offset?: string }> + appendMarkdownDocumentAwareness: ( + streamPath: string, + update: Uint8Array + ) => Promise<{ offset?: string }> }) => Array | Promise> /** Optional shutdown signal to end idle waits during host teardown. */ shutdownSignal?: AbortSignal @@ -903,7 +898,20 @@ export type AgentRunResult = { usage: { tokens: number; duration: number } } -export type AgentTool = PiAgentTool +export interface ToolArgumentDeltaContext { + toolCallId: string + toolName: string + contentIndex?: number + delta: string + argsPreview?: unknown +} + +export type AgentTool = PiAgentTool & { + onArgsDelta?: ( + context: ToolArgumentDeltaContext, + signal?: AbortSignal + ) => Promise | void +} export type AgentModel = string | Model export interface AgentConfig { diff --git a/packages/agents-runtime/test/markdown-docs-tools.test.ts b/packages/agents-runtime/test/markdown-docs-tools.test.ts index 87150bdf1f..1e13061940 100644 --- a/packages/agents-runtime/test/markdown-docs-tools.test.ts +++ b/packages/agents-runtime/test/markdown-docs-tools.test.ts @@ -1,6 +1,36 @@ import { describe, expect, it, vi } from 'vitest' +import * as Y from 'yjs' +import { + createMarkdownYDoc, + frameYjsUpdate, + markdownText, +} from '../src/markdown-yjs' import { createMarkdownDocumentTools } from '../src/tools/markdown-docs' +function concatBytes(a: Uint8Array, b: Uint8Array): Uint8Array { + const next = new Uint8Array(a.length + b.length) + next.set(a, 0) + next.set(b, a.length) + return next +} + +function concatFrames(frames: Array): Uint8Array { + return frames.reduce( + (bytes, frame) => concatBytes(bytes, frame), + new Uint8Array() + ) +} + +function streamBytesFromContent(content: string): Uint8Array { + const doc = new Y.Doc() + markdownText(doc).insert(0, content) + return frameYjsUpdate(Y.encodeStateAsUpdate(doc)) +} + +function contentFromStream(streamBytes: Uint8Array): string { + return markdownText(createMarkdownYDoc(streamBytes)).toString() +} + function createToolContext() { const document = { key: `document:notes`, @@ -16,17 +46,18 @@ function createToolContext() { title: `Notes`, createdAt: `2026-06-07T00:00:00.000Z`, } as const - let content = `# Notes\n\nFirst line\n` + let streamFrames = [streamBytesFromContent(`# Notes\n\nFirst line\n`)] return { context: { entityUrl: `/chat/session`, entityType: `chat`, + principal: { url: `/principal/agent:horton`, kind: `agent` }, args: {}, - db: { collections: { manifests: { toArray: [] } } }, + db: { collections: { manifests: { toArray: [document] } } }, events: [], createMarkdownDocument: vi.fn( - async (opts: { id?: string; title: string; content?: string }) => { - content = opts.content ?? `` + async (opts: { id?: string; title: string }) => { + streamFrames = [] return { txid: `tx-create`, document: { @@ -37,25 +68,24 @@ function createToolContext() { } } ), - readMarkdownDocument: vi.fn(async () => ({ document, content })), - writeMarkdownDocument: vi.fn( - async (opts: { id: string; content: string }) => { - content = opts.content - return { txid: `tx-write`, document, content } + readMarkdownDocumentStream: vi.fn( + async (_streamPath: string, opts?: { offset?: string }) => { + const offset = + opts?.offset !== undefined ? Number.parseInt(opts.offset, 10) : 0 + const start = Number.isFinite(offset) && offset >= 0 ? offset : 0 + return { + bytes: concatFrames(streamFrames.slice(start)), + offset: String(streamFrames.length), + } } ), - editMarkdownDocument: vi.fn( - async (opts: { - oldString: string - newString: string - replaceAll?: boolean - }) => { - content = opts.replaceAll - ? content.split(opts.oldString).join(opts.newString) - : content.replace(opts.oldString, opts.newString) - return { txid: `tx-edit`, document, content } + appendMarkdownDocumentUpdate: vi.fn( + async (_streamPath: string, update: Uint8Array) => { + streamFrames.push(update) + return { offset: String(streamFrames.length) } } ), + appendMarkdownDocumentAwareness: vi.fn(async () => ({})), upsertCronSchedule: vi.fn(), upsertFutureSendSchedule: vi.fn(), deleteSchedule: vi.fn(), @@ -63,13 +93,41 @@ function createToolContext() { subscribeToEventSource: vi.fn(), unsubscribeFromEventSource: vi.fn(), } as any, - getContent: () => content, + getContent: () => contentFromStream(concatFrames(streamFrames)), + appendExternalText: (text: string) => { + const streamBytes = concatFrames(streamFrames) + const doc = createMarkdownYDoc(streamBytes) + const yText = markdownText(doc) + const before = Y.encodeStateVector(doc) + yText.insert(yText.length, text) + streamFrames.push(frameYjsUpdate(Y.encodeStateAsUpdate(doc, before))) + }, } } describe(`markdown document tools`, () => { - it(`requires read_markdown_doc before edit_markdown_doc`, async () => { - const { context } = createToolContext() + it(`creates the server document empty and appends initial content as a Yjs update`, async () => { + const { context, getContent } = createToolContext() + const create = createMarkdownDocumentTools(context).find( + (tool) => tool.name === `create_markdown_doc` + )! + + await create.execute(`tool-create`, { + id: `notes`, + title: `Notes`, + content: `# Created\n\nInitial content`, + }) + + expect(context.createMarkdownDocument).toHaveBeenCalledWith({ + id: `notes`, + title: `Notes`, + }) + expect(context.appendMarkdownDocumentUpdate).toHaveBeenCalledTimes(1) + expect(getContent()).toBe(`# Created\n\nInitial content`) + }) + + it(`materializes and edits markdown documents through Yjs stream updates`, async () => { + const { context, getContent } = createToolContext() const edit = createMarkdownDocumentTools(context).find( (tool) => tool.name === `edit_markdown_doc` )! @@ -80,12 +138,10 @@ describe(`markdown document tools`, () => { new_string: `Second`, }) - expect(context.editMarkdownDocument).not.toHaveBeenCalled() - expect(result.details).toMatchObject({ replacements: 0 }) - expect(result.content[0]).toMatchObject({ - type: `text`, - text: expect.stringContaining(`read_markdown_doc first`), - }) + expect(context.appendMarkdownDocumentUpdate).toHaveBeenCalledTimes(1) + expect(context.appendMarkdownDocumentAwareness).toHaveBeenCalled() + expect(getContent()).toContain(`Second line`) + expect(result.details).toMatchObject({ replacements: 1 }) }) it(`edits a read document and returns a diff`, async () => { @@ -101,14 +157,102 @@ describe(`markdown document tools`, () => { new_string: `Second line`, }) - expect(context.editMarkdownDocument).toHaveBeenCalledWith({ - id: `notes`, - oldString: `First line`, - newString: `Second line`, - replaceAll: undefined, - }) + expect(context.appendMarkdownDocumentUpdate).toHaveBeenCalledTimes(1) expect(getContent()).toContain(`Second line`) expect(result.details).toMatchObject({ replacements: 1 }) expect(String((result.details as any).diff)).toContain(`Second line`) }) + + it(`streams insert_markdown_doc content deltas before final execution`, async () => { + const { context, getContent } = createToolContext() + const insert = createMarkdownDocumentTools(context).find( + (tool) => tool.name === `insert_markdown_doc` + )! + + await insert.onArgsDelta?.({ + toolCallId: `tool-insert`, + toolName: `insert_markdown_doc`, + delta: `"Hello`, + argsPreview: { id: `notes`, content: `Hello` }, + }) + await insert.onArgsDelta?.({ + toolCallId: `tool-insert`, + toolName: `insert_markdown_doc`, + delta: ` world"`, + argsPreview: { id: `notes`, content: `Hello world` }, + }) + + const result = await insert.execute(`tool-insert`, { + id: `notes`, + content: `Hello world`, + }) + + expect(context.appendMarkdownDocumentUpdate).toHaveBeenCalledTimes(2) + expect(context.appendMarkdownDocumentAwareness).toHaveBeenCalled() + expect(getContent()).toContain(`Hello world`) + expect(result.details).toMatchObject({ streamed: true }) + }) + + it(`streams insert_markdown_doc at a saved Yjs-relative cursor`, async () => { + const { context, getContent } = createToolContext() + const tools = createMarkdownDocumentTools(context) + const setCursor = tools.find( + (tool) => tool.name === `set_markdown_doc_cursor` + )! + const insert = tools.find((tool) => tool.name === `insert_markdown_doc`)! + + const cursorResult = await setCursor.execute(`tool-cursor`, { + id: `notes`, + after: `# Notes\n`, + }) + expect(cursorResult.details).toMatchObject({ + cursorSet: true, + index: `# Notes\n`.length, + }) + + await insert.onArgsDelta?.({ + toolCallId: `tool-insert-cursor`, + toolName: `insert_markdown_doc`, + delta: `"Inserted`, + argsPreview: { id: `notes`, content: `Inserted` }, + }) + await insert.onArgsDelta?.({ + toolCallId: `tool-insert-cursor`, + toolName: `insert_markdown_doc`, + delta: ` text\n"`, + argsPreview: { id: `notes`, content: `Inserted text\n` }, + }) + + await insert.execute(`tool-insert-cursor`, { + id: `notes`, + content: `Inserted text\n`, + }) + + expect(getContent()).toBe(`# Notes\nInserted text\n\nFirst line\n`) + expect(context.appendMarkdownDocumentUpdate).toHaveBeenCalledTimes(2) + }) + + it(`refreshes a cached Yjs document from the stream before editing`, async () => { + const { context, getContent, appendExternalText } = createToolContext() + const tools = createMarkdownDocumentTools(context) + const read = tools.find((tool) => tool.name === `read_markdown_doc`)! + const edit = tools.find((tool) => tool.name === `edit_markdown_doc`)! + + await read.execute(`tool-read`, { id: `notes` }) + appendExternalText(`External line\n`) + + await edit.execute(`tool-edit`, { + id: `notes`, + old_string: `External line`, + new_string: `Refreshed line`, + }) + + expect(getContent()).toContain(`Refreshed line`) + expect(context.readMarkdownDocumentStream).toHaveBeenCalledTimes(2) + expect(context.readMarkdownDocumentStream).toHaveBeenNthCalledWith( + 2, + `/v1/yjs/default/docs/agents/chat/session/documents/notes`, + { offset: `1` } + ) + }) }) diff --git a/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx index 968e38a522..580fd33e3e 100644 --- a/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx +++ b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx @@ -19,7 +19,6 @@ import type { LucideIcon } from 'lucide-react' type DocumentResponse = { document: ManifestDocumentEntry - content: string } type DocumentConnectionStatus = diff --git a/packages/agents-server/src/entity-manager.ts b/packages/agents-server/src/entity-manager.ts index 727f32c590..0b48b3cf4d 100644 --- a/packages/agents-server/src/entity-manager.ts +++ b/packages/agents-server/src/entity-manager.ts @@ -1,7 +1,5 @@ import { createHash, randomUUID } from 'node:crypto' import fastq from 'fastq' -import { Awareness, encodeAwarenessUpdate } from 'y-protocols/awareness' -import * as Y from 'yjs' import { COMPOSER_INPUT_MESSAGE_TYPE, assertTags, @@ -62,13 +60,8 @@ import { MARKDOWN_DOCUMENT_TEXT_NAME, MARKDOWN_DOCUMENT_TRANSPORT_MIME, assertMarkdownDocumentMatchesEntity, - frameYjsUpdate, - getMarkdownDocumentAwarenessStreamPath, getMarkdownDocumentDocPath, getMarkdownDocumentUpdateStreamPath, - markdownText, - readMarkdownYDoc, - replaceMarkdownText, } from './markdown-documents.js' import type { queueAsPromised } from 'fastq' import type { SchedulerClient } from './scheduler.js' @@ -178,25 +171,10 @@ type ManifestAttachmentEntry = { export interface CreateMarkdownDocumentRequest { id?: string title: string - content?: string createdBy?: string meta?: Record } -export interface UpdateMarkdownDocumentRequest { - content: string - updatedBy?: string - presenceBefore?: { anchor: number; head: number } - presenceAfter?: { anchor: number; head: number } -} - -export interface EditMarkdownDocumentRequest { - oldString: string - newString: string - replaceAll?: boolean - updatedBy?: string -} - export type ManifestMarkdownDocumentEntry = { key: string kind: `document` @@ -284,74 +262,6 @@ function validateMarkdownDocumentId(id: string): void { } } -function principalDisplayName(principalUrl: string): string { - const raw = principalUrl.split(`/principal/`).at(-1) ?? principalUrl - let decoded = raw - try { - decoded = decodeURIComponent(raw) - } catch { - // Fall back to the raw key when the principal URL is not URI encoded. - } - const withoutPrefix = decoded.replace(/^(user|agent|entity|system):/, ``) - return withoutPrefix || decoded || principalUrl -} - -function principalRole(principalUrl: string): `agent` | `user` | `system` { - const raw = principalUrl.split(`/principal/`).at(-1) ?? principalUrl - let decoded = raw - try { - decoded = decodeURIComponent(raw) - } catch { - // Fall back to the raw key when the principal URL is not URI encoded. - } - if (decoded.startsWith(`user:`)) return `user` - if (decoded.startsWith(`system:`)) return `system` - return `agent` -} - -function principalColor(principalUrl: string): { - color: string - colorLight: string -} { - const colors = [ - [`#2563eb`, `#2563eb33`], - [`#059669`, `#05966933`], - [`#dc2626`, `#dc262633`], - [`#7c3aed`, `#7c3aed33`], - [`#c2410c`, `#c2410c33`], - [`#0f766e`, `#0f766e33`], - ] as const - let hash = 0 - for (let i = 0; i < principalUrl.length; i += 1) { - hash = (hash * 31 + principalUrl.charCodeAt(i)) >>> 0 - } - const [color, colorLight] = colors[hash % colors.length]! - return { color, colorLight } -} - -function markdownDocumentPresenceClientId( - docPath: string, - principalUrl: string -): number { - const digest = createHash(`sha256`) - .update(`${docPath}\0${principalUrl}`) - .digest() - const id = digest.readUInt32BE(0) - return id === 0 ? 1 : id -} - -function createMarkdownDocumentAwareness( - docPath: string, - principalUrl: string | undefined -): Awareness { - const awarenessDoc = new Y.Doc() - if (principalUrl) { - ;(awarenessDoc as { clientID: number }).clientID = - markdownDocumentPresenceClientId(docPath, principalUrl) - } - return new Awareness(awarenessDoc) -} - function getEntityAttachmentStreamPath( entityUrl: string, attachmentId: string @@ -2755,93 +2665,6 @@ export class EntityManager { ) } - private async publishMarkdownDocumentPresence( - docPath: string, - doc: Y.Doc, - awareness: Awareness, - principalUrl: string | undefined, - status: `editing`, - anchor: number, - head: number, - seq: number - ): Promise { - if (!principalUrl) return - const awarenessPath = getMarkdownDocumentAwarenessStreamPath( - this.tenantId, - docPath, - `default` - ) - const text = markdownText(doc) - const boundedAnchor = Math.max(0, Math.min(anchor, text.length)) - const boundedHead = Math.max(0, Math.min(head, text.length)) - const colors = principalColor(principalUrl) - const now = Date.now() - awareness.setLocalState({ - user: { - name: principalDisplayName(principalUrl), - principalUrl, - role: principalRole(principalUrl), - status, - updatedAt: now, - expiresAt: now + 5_000, - color: colors.color, - colorLight: colors.colorLight, - }, - cursor: { - anchor: Y.createRelativePositionFromTypeIndex(text, boundedAnchor), - head: Y.createRelativePositionFromTypeIndex(text, boundedHead), - }, - }) - await this.streamClient - .create(awarenessPath, { contentType: `application/octet-stream` }) - .catch((error) => { - if (!isStreamCreateConflict(error)) throw error - }) - await this.streamClient.appendBytes( - awarenessPath, - frameYjsUpdate(encodeAwarenessUpdate(awareness, [awareness.clientID])), - { - producerId: `agent-doc-presence-${docPath}`, - epoch: Date.now(), - seq, - } - ) - } - - private async clearMarkdownDocumentPresence( - docPath: string, - awareness: Awareness, - principalUrl: string | undefined, - seq: number - ): Promise { - if (!principalUrl) return - const awarenessPath = getMarkdownDocumentAwarenessStreamPath( - this.tenantId, - docPath, - `default` - ) - awareness.setLocalState(null) - await this.streamClient - .appendBytes( - awarenessPath, - frameYjsUpdate(encodeAwarenessUpdate(awareness, [awareness.clientID])), - { - producerId: `agent-doc-presence-${docPath}`, - epoch: Date.now(), - seq, - } - ) - .catch(() => undefined) - } - - private async bestEffortMarkdownDocumentPresence( - action: () => Promise - ): Promise { - await action().catch((error) => { - serverLog.warn(`[agent-server] markdown document presence failed:`, error) - }) - } - async createMarkdownDocument( entityUrl: string, req: CreateMarkdownDocumentRequest @@ -2898,56 +2721,6 @@ export class EntityManager { contentType: `application/octet-stream`, }) streamCreated = true - const content = req.content - if (content) { - const doc = new Y.Doc() - const awareness = createMarkdownDocumentAwareness( - docPath, - req.createdBy - ) - await this.bestEffortMarkdownDocumentPresence(() => - this.publishMarkdownDocumentPresence( - docPath, - doc, - awareness, - req.createdBy, - `editing`, - 0, - 0, - 0 - ) - ) - const update = replaceMarkdownText(doc, content) - await this.streamClient.appendBytes( - updateStreamPath, - frameYjsUpdate(update), - { - producerId: `agent-doc-create-${id}`, - epoch: 0, - seq: 0, - } - ) - await this.bestEffortMarkdownDocumentPresence(() => - this.publishMarkdownDocumentPresence( - docPath, - doc, - awareness, - req.createdBy, - `editing`, - content.length, - content.length, - 1 - ) - ) - await this.bestEffortMarkdownDocumentPresence(() => - this.clearMarkdownDocumentPresence( - docPath, - awareness, - req.createdBy, - 2 - ) - ) - } await this.writeManifestEntry( entityUrl, document.key, @@ -2992,171 +2765,6 @@ export class EntityManager { return manifest as unknown as ManifestMarkdownDocumentEntry } - async readMarkdownDocument( - entityUrl: string, - id: string - ): Promise<{ - document: ManifestMarkdownDocumentEntry - content: string - }> { - const document = await this.getMarkdownDocument(entityUrl, id) - if (!document) { - throw new ElectricAgentsError(ErrCodeNotFound, `Document not found`, 404) - } - const doc = await readMarkdownYDoc( - this.streamClient, - getMarkdownDocumentUpdateStreamPath(this.tenantId, document.docPath) - ) - return { document, content: markdownText(doc).toString() } - } - - async writeMarkdownDocument( - entityUrl: string, - id: string, - req: UpdateMarkdownDocumentRequest - ): Promise<{ - txid: string - document: ManifestMarkdownDocumentEntry - content: string - }> { - const entity = await this.registry.getEntity(entityUrl) - if (!entity) { - throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404) - } - if (rejectsNormalWrites(entity.status)) { - throw new ElectricAgentsError( - ErrCodeNotRunning, - `Entity is not accepting writes`, - 409 - ) - } - if (this.isForkWorkLockedEntity(entityUrl)) { - this.assertEntityNotForkWorkLocked(entityUrl) - } - const current = await this.readMarkdownDocument(entityUrl, id) - const updateStreamPath = getMarkdownDocumentUpdateStreamPath( - this.tenantId, - current.document.docPath - ) - const doc = await readMarkdownYDoc(this.streamClient, updateStreamPath) - const awareness = createMarkdownDocumentAwareness( - current.document.docPath, - req.updatedBy - ) - await this.bestEffortMarkdownDocumentPresence(() => - this.publishMarkdownDocumentPresence( - current.document.docPath, - doc, - awareness, - req.updatedBy, - `editing`, - req.presenceBefore?.anchor ?? current.content.length, - req.presenceBefore?.head ?? current.content.length, - 0 - ) - ) - try { - const update = replaceMarkdownText(doc, req.content) - await this.streamClient.appendBytes( - updateStreamPath, - frameYjsUpdate(update), - { - producerId: `agent-doc-write-${id}`, - epoch: Date.now(), - seq: 0, - } - ) - await this.bestEffortMarkdownDocumentPresence(() => - this.publishMarkdownDocumentPresence( - current.document.docPath, - doc, - awareness, - req.updatedBy, - `editing`, - req.presenceAfter?.anchor ?? req.content.length, - req.presenceAfter?.head ?? req.content.length, - 1 - ) - ) - } finally { - await this.bestEffortMarkdownDocumentPresence(() => - this.clearMarkdownDocumentPresence( - current.document.docPath, - awareness, - req.updatedBy, - 2 - ) - ) - } - const txid = randomUUID() - const nextDocument: ManifestMarkdownDocumentEntry = { - ...current.document, - updatedAt: new Date().toISOString(), - } - await this.writeManifestEntry( - entityUrl, - nextDocument.key, - `upsert`, - nextDocument as unknown as Record, - { txid } - ) - return { txid, document: nextDocument, content: req.content } - } - - async editMarkdownDocument( - entityUrl: string, - id: string, - req: EditMarkdownDocumentRequest - ): Promise<{ - txid: string - document: ManifestMarkdownDocumentEntry - content: string - }> { - if (req.oldString === ``) { - throw new ElectricAgentsError( - ErrCodeInvalidRequest, - `oldString must not be empty`, - 400 - ) - } - const current = await this.readMarkdownDocument(entityUrl, id) - const matches = current.content.split(req.oldString).length - 1 - if (matches === 0) { - throw new ElectricAgentsError( - ErrCodeInvalidRequest, - `oldString was not found in document`, - 400 - ) - } - if (!req.replaceAll && matches > 1) { - throw new ElectricAgentsError( - ErrCodeInvalidRequest, - `oldString appears multiple times; set replaceAll to replace all matches`, - 400 - ) - } - const content = req.replaceAll - ? current.content.split(req.oldString).join(req.newString) - : current.content.replace(req.oldString, req.newString) - const index = current.content.indexOf(req.oldString) - const finalIndex = - req.replaceAll && req.newString.length > 0 - ? content.lastIndexOf(req.newString) + req.newString.length - : index + req.newString.length - return await this.writeMarkdownDocument(entityUrl, id, { - content, - updatedBy: req.updatedBy, - presenceBefore: { - anchor: index, - head: index, - }, - presenceAfter: { - anchor: finalIndex, - head: finalIndex, - }, - }) - } - // ========================================================================== // Tag Updates // ========================================================================== diff --git a/packages/agents-server/src/routing/entities-router.ts b/packages/agents-server/src/routing/entities-router.ts index 35fcdc8624..fbf4dc93ca 100644 --- a/packages/agents-server/src/routing/entities-router.ts +++ b/packages/agents-server/src/routing/entities-router.ts @@ -242,28 +242,11 @@ const markdownDocumentCreateBodySchema = Type.Object( { id: Type.Optional(Type.String()), title: Type.String(), - content: Type.Optional(Type.String()), meta: Type.Optional(Type.Record(Type.String(), Type.Unknown())), }, { additionalProperties: false } ) -const markdownDocumentWriteBodySchema = Type.Object( - { - content: Type.String(), - }, - { additionalProperties: false } -) - -const markdownDocumentEditBodySchema = Type.Object( - { - oldString: Type.String(), - newString: Type.String(), - replaceAll: Type.Optional(Type.Boolean()), - }, - { additionalProperties: false } -) - const entitySignalSchema = Type.Union([ Type.Literal(`SIGINT`), Type.Literal(`SIGHUP`), @@ -325,8 +308,6 @@ type SetTagBody = Static type MarkdownDocumentCreateBody = Static< typeof markdownDocumentCreateBodySchema > -type MarkdownDocumentWriteBody = Static -type MarkdownDocumentEditBody = Static type SignalBody = Static type ScheduleBody = Static type EventSourceSubscriptionBody = Static< @@ -434,20 +415,6 @@ entitiesRouter.get( withEntityPermission(`read`), readMarkdownDocument ) -entitiesRouter.put( - `/:type/:instanceId/documents/:documentId`, - withExistingEntity, - withSchema(markdownDocumentWriteBodySchema), - withEntityPermission(`write`), - writeMarkdownDocument -) -entitiesRouter.patch( - `/:type/:instanceId/documents/:documentId`, - withExistingEntity, - withSchema(markdownDocumentEditBodySchema), - withEntityPermission(`write`), - editMarkdownDocument -) entitiesRouter.patch( `/:type/:instanceId/inbox/:messageKey`, withExistingEntity, @@ -1337,7 +1304,6 @@ async function createMarkdownDocument( const result = await ctx.entityManager.createMarkdownDocument(entityUrl, { id: parsed.id, title: parsed.title, - content: parsed.content, createdBy: ctx.principal.url, meta: parsed.meta, }) @@ -1349,61 +1315,22 @@ async function readMarkdownDocument( ctx: TenantContext ): Promise { const { entityUrl } = requireExistingEntityRoute(request) - const result = await ctx.entityManager.readMarkdownDocument( + const document = await ctx.entityManager.getMarkdownDocument( entityUrl, decodeURIComponent(request.params.documentId) ) - return json(result, { - headers: { - 'content-type': `application/json; charset=utf-8`, - 'cache-control': `no-store`, - }, - }) -} - -async function writeMarkdownDocument( - request: AgentsRouteRequest, - ctx: TenantContext -): Promise { - const principalMutationError = rejectPrincipalEntityMutation( - request, - `given documents` - ) - if (principalMutationError) return principalMutationError - - const parsed = routeBody(request) - const { entityUrl } = requireExistingEntityRoute(request) - const result = await ctx.entityManager.writeMarkdownDocument( - entityUrl, - decodeURIComponent(request.params.documentId), - { content: parsed.content, updatedBy: ctx.principal.url } - ) - return json(result) -} - -async function editMarkdownDocument( - request: AgentsRouteRequest, - ctx: TenantContext -): Promise { - const principalMutationError = rejectPrincipalEntityMutation( - request, - `given documents` - ) - if (principalMutationError) return principalMutationError - - const parsed = routeBody(request) - const { entityUrl } = requireExistingEntityRoute(request) - const result = await ctx.entityManager.editMarkdownDocument( - entityUrl, - decodeURIComponent(request.params.documentId), + if (!document) { + throw new ElectricAgentsError(ErrCodeNotFound, `Document not found`, 404) + } + return json( + { document }, { - oldString: parsed.oldString, - newString: parsed.newString, - replaceAll: parsed.replaceAll, - updatedBy: ctx.principal.url, + headers: { + 'content-type': `application/json; charset=utf-8`, + 'cache-control': `no-store`, + }, } ) - return json(result) } async function updateInboxMessage( diff --git a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts index 5a2f41b084..9532f3b829 100644 --- a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts +++ b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts @@ -1,6 +1,4 @@ import { describe, expect, it, vi } from 'vitest' -import { Awareness } from 'y-protocols/awareness' -import * as Y from 'yjs' import { EntityManager } from '../src/entity-manager' import { SchemaValidator } from '../src/electric-agents/schema-validator' import { @@ -8,8 +6,6 @@ import { MARKDOWN_DOCUMENT_PROVIDER, MARKDOWN_DOCUMENT_TEXT_NAME, MARKDOWN_DOCUMENT_TRANSPORT_MIME, - applyFramedAwarenessUpdates, - getMarkdownDocumentAwarenessStreamPath, } from '../src/markdown-documents' const observedItemSchema = { @@ -160,32 +156,9 @@ function createMarkdownDocumentManager() { } as any, }), streamClient, - binaryStreams, } } -function collectAwarenessStates(frames: Array) { - const doc = new Y.Doc() - const awareness = new Awareness(doc) - const states: Array<{ - user: Record - cursor?: { anchor?: unknown; head?: unknown } - }> = [] - for (const frame of frames) { - applyFramedAwarenessUpdates(awareness, frame) - for (const state of awareness.getStates().values()) { - const entry = state as { - user?: Record - cursor?: { anchor?: unknown; head?: unknown } - } - if (entry.user) { - states.push({ user: entry.user, cursor: entry.cursor }) - } - } - } - return states -} - describe(`ElectricAgentsManager.validateWriteEvent`, () => { it(`validates delete events against old_value instead of value`, async () => { const manager = createManager() @@ -233,14 +206,12 @@ describe(`ElectricAgentsManager.validateWriteEvent`, () => { }) describe(`ElectricAgentsManager markdown documents`, () => { - it(`stores markdown as framed Yjs updates and exposes a manifest document entry`, async () => { - const { manager, streamClient, binaryStreams } = - createMarkdownDocumentManager() + it(`creates an empty Yjs update stream and exposes a manifest document entry`, async () => { + const { manager, streamClient } = createMarkdownDocumentManager() const created = await manager.createMarkdownDocument(`/chat/session-1`, { id: `notes`, title: `Session notes`, - content: `# Notes\n\nDraft`, createdBy: `/principal/agent:horton`, }) @@ -262,52 +233,13 @@ describe(`ElectricAgentsManager markdown documents`, () => { `/yjs/default/docs/agents/chat/session-1/documents/notes/.updates`, { contentType: `application/octet-stream` } ) - + expect(streamClient.appendBytes).not.toHaveBeenCalled() await expect( - manager.readMarkdownDocument(`/chat/session-1`, `notes`) + manager.getMarkdownDocument(`/chat/session-1`, `notes`) ).resolves.toMatchObject({ - document: expect.objectContaining({ id: `notes` }), - content: `# Notes\n\nDraft`, - }) - - await manager.editMarkdownDocument(`/chat/session-1`, `notes`, { - oldString: `Draft`, - newString: `Ready`, - updatedBy: `/principal/agent:horton`, - }) - - await expect( - manager.readMarkdownDocument(`/chat/session-1`, `notes`) - ).resolves.toMatchObject({ - content: `# Notes\n\nReady`, + id: `notes`, + title: `Session notes`, }) - - const awarenessPath = getMarkdownDocumentAwarenessStreamPath( - `default`, - `agents/chat/session-1/documents/notes`, - `default` - ) - const states = collectAwarenessStates( - binaryStreams.get(awarenessPath) ?? [] - ) - expect(states.map((state) => state.user)).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - name: `horton`, - role: `agent`, - status: `editing`, - principalUrl: `/principal/agent:horton`, - }), - ]) - ) - expect(states.every((state) => state.user.status === `editing`)).toBe(true) - expect( - states.every( - (state) => - JSON.stringify(state.cursor?.anchor) === - JSON.stringify(state.cursor?.head) - ) - ).toBe(true) }) }) diff --git a/packages/agents-server/test/electric-agents-routes.test.ts b/packages/agents-server/test/electric-agents-routes.test.ts index a79111616a..61581648a4 100644 --- a/packages/agents-server/test/electric-agents-routes.test.ts +++ b/packages/agents-server/test/electric-agents-routes.test.ts @@ -251,7 +251,7 @@ describe(`ElectricAgentsRoutes schedule endpoints`, () => { }) describe(`ElectricAgentsRoutes markdown document endpoints`, () => { - it(`routes document create, read, write, and edit requests to the manager`, async () => { + it(`routes document create and metadata read requests to the manager`, async () => { const document = { key: `document:notes`, kind: `document`, @@ -274,28 +274,19 @@ describe(`ElectricAgentsRoutes markdown document endpoints`, () => { createMarkdownDocument: vi .fn() .mockResolvedValue({ txid: `tx-create`, document }), - readMarkdownDocument: vi - .fn() - .mockResolvedValue({ document, content: `# Notes` }), - writeMarkdownDocument: vi - .fn() - .mockResolvedValue({ txid: `tx-write`, document, content: `# Ready` }), - editMarkdownDocument: vi - .fn() - .mockResolvedValue({ txid: `tx-edit`, document, content: `# Done` }), + getMarkdownDocument: vi.fn().mockResolvedValue(document), } as any const createResponse = await routeResponse( manager, `POST`, `/_electric/entities/chat/test/documents`, - { id: `notes`, title: `Notes`, content: `# Notes` } + { id: `notes`, title: `Notes` } ) expect(createResponse.status).toBe(201) expect(manager.createMarkdownDocument).toHaveBeenCalledWith(`/chat/test`, { id: `notes`, title: `Notes`, - content: `# Notes`, createdBy: `/principal/system:dev-local`, meta: undefined, }) @@ -307,37 +298,36 @@ describe(`ElectricAgentsRoutes markdown document endpoints`, () => { ) expect(await responseJson(readResponse)).toEqual({ document, - content: `# Notes`, }) + expect(manager.getMarkdownDocument).toHaveBeenCalledWith( + `/chat/test`, + `notes` + ) + }) - await routeResponse( + it(`does not expose semantic markdown document write or edit endpoints`, async () => { + const manager = { + registry: { + getEntity: vi.fn().mockResolvedValue({ url: `/chat/test` }), + getEntityType: vi.fn(), + }, + } as any + + const putResponse = await routeResponse( manager, `PUT`, `/_electric/entities/chat/test/documents/notes`, { content: `# Ready` } ) - expect(manager.writeMarkdownDocument).toHaveBeenCalledWith( - `/chat/test`, - `notes`, - { content: `# Ready`, updatedBy: `/principal/system:dev-local` } - ) + expect(putResponse.status).toBe(404) - await routeResponse( + const patchResponse = await routeResponse( manager, `PATCH`, `/_electric/entities/chat/test/documents/notes`, { oldString: `Ready`, newString: `Done`, replaceAll: true } ) - expect(manager.editMarkdownDocument).toHaveBeenCalledWith( - `/chat/test`, - `notes`, - { - oldString: `Ready`, - newString: `Done`, - replaceAll: true, - updatedBy: `/principal/system:dev-local`, - } - ) + expect(patchResponse.status).toBe(404) }) it(`guards public Yjs document routes and forwards authorized document streams`, async () => { diff --git a/packages/agents/skills/markdown-docs.md b/packages/agents/skills/markdown-docs.md index 70e1f1ed81..f27e1d7e5e 100644 --- a/packages/agents/skills/markdown-docs.md +++ b/packages/agents/skills/markdown-docs.md @@ -23,7 +23,8 @@ Agents UI and can be opened, edited, and watched live. Collaborative markdown docs are not filesystem files. -- Use `create_markdown_doc`, `read_markdown_doc`, `write_markdown_doc`, and +- Use `create_markdown_doc`, `set_markdown_doc_cursor`, + `insert_markdown_doc`, `read_markdown_doc`, `write_markdown_doc`, and `edit_markdown_doc` for docs the user should open in the workspace UI. - Use filesystem `write`/`edit` only when the user asks for an actual file path in the workspace or repo, such as `docs/foo.md`, `README.md`, or @@ -74,12 +75,28 @@ For small edits: 3. If the target text appears multiple times, make `old_string` more specific or set `replace_all` only when replacing every occurrence is clearly intended. +The markdown tools materialize the collaborative Yjs document from its durable +stream during the wake. `write_markdown_doc`, `edit_markdown_doc`, and +`insert_markdown_doc` append binary Yjs updates to that stream; do not write +markdown documents to the local filesystem unless the user explicitly asks for a +filesystem file. + For broad rewrites: 1. Use `read_markdown_doc` first unless you just created or wrote the doc in the same wake. 2. Use `write_markdown_doc` with the full replacement markdown. +For adding new long content to an existing doc: + +1. Use `read_markdown_doc` if you need to inspect the target location. +2. Use `set_markdown_doc_cursor` with `index`, `before`, or `after` when the + insertion belongs at a specific location. +3. Use `insert_markdown_doc`. +4. Pass `id` and optional `index` before `content` in the tool arguments. If + `index` is omitted, the saved Yjs-relative cursor is used; if no cursor is + set, the content is appended to the current document. + Both write and edit tool results include diffs. Use those diffs to summarize what changed. diff --git a/packages/agents/src/agents/horton.ts b/packages/agents/src/agents/horton.ts index f5ebec4d53..1732686440 100644 --- a/packages/agents/src/agents/horton.ts +++ b/packages/agents/src/agents/horton.ts @@ -199,13 +199,13 @@ export function buildHortonSystemPrompt( ? `\n- list_event_sources: list external webhook/event feeds you can subscribe to, including available buckets and parameters\n- subscribe_event_source: subscribe yourself to one of those feeds or buckets so matching future events wake you\n- list_event_source_subscriptions: list your active event source subscriptions\n- unsubscribe_event_source: remove one of your event source subscriptions by id` : `` const markdownDocumentTools = opts.hasMarkdownDocumentTools - ? `\n- create_markdown_doc: create a collaborative markdown document that appears in this entity's manifest and opens in the workspace editor\n- read_markdown_doc: read a collaborative markdown document\n- write_markdown_doc: replace a collaborative markdown document's full content\n- edit_markdown_doc: targeted string replacement in a collaborative markdown document` + ? `\n- create_markdown_doc: create a collaborative markdown document that appears in this entity's manifest and opens in the workspace editor\n- set_markdown_doc_cursor: choose a stateful Yjs-relative insertion cursor for a collaborative markdown document\n- insert_markdown_doc: stream or insert markdown into an existing collaborative document so open editors can watch content appear\n- read_markdown_doc: read a collaborative markdown document\n- write_markdown_doc: replace a collaborative markdown document's full content\n- edit_markdown_doc: targeted string replacement in a collaborative markdown document` : `` const skillsTools = opts.hasSkills ? `\n- use_skill: load a skill (knowledge, instructions, or a tutorial) into your context to help with the user's request\n- remove_skill: unload a skill from context when you're done with it` : `` const markdownDocumentGuidance = opts.hasMarkdownDocumentTools - ? `\n# Collaborative Markdown Docs\n- If the user asks you to create a markdown doc, notes, draft, brief, plan, report, or any document they should open/edit in the app UI, use create_markdown_doc. Do not use filesystem write unless they ask for a file path or repo/workspace file.\n- For larger document workflows, load the markdown-docs skill first with use_skill, then use the markdown document tools.\n- After creating a collaborative doc, mention that it is available from this entity's manifest/timeline.` + ? `\n# Collaborative Markdown Docs\n- If the user asks you to create a markdown doc, notes, draft, brief, plan, report, or any document they should open/edit in the app UI, use create_markdown_doc. Do not use filesystem write unless they ask for a file path or repo/workspace file.\n- For larger document workflows, load the markdown-docs skill first with use_skill, then use the markdown document tools.\n- Use set_markdown_doc_cursor before insert_markdown_doc when inserting at a specific location; insert_markdown_doc can then stream content at that Yjs-relative cursor.\n- After creating a collaborative doc, mention that it is available from this entity's manifest/timeline.` : `` const docsGuidance = opts.hasDocsSupport ? `\n- For ANY question about Electric Agents or this framework, ALWAYS use search_electric_agents_docs FIRST. Do not use web_search or fetch_url for Electric Agents topics unless the docs search returns no useful results.\n- The search tool returns chunk content directly โ€” you do not need to read the source files.\n- Use repo read/bash tools only for non-doc files or when you need to inspect exact implementation code in the workspace.` diff --git a/packages/agents/test/horton-system-prompt.test.ts b/packages/agents/test/horton-system-prompt.test.ts index 014ca57527..360cbfeb18 100644 --- a/packages/agents/test/horton-system-prompt.test.ts +++ b/packages/agents/test/horton-system-prompt.test.ts @@ -46,6 +46,8 @@ describe(`buildHortonSystemPrompt`, () => { }) expect(prompt).toContain(`create_markdown_doc`) + expect(prompt).toContain(`set_markdown_doc_cursor`) + expect(prompt).toContain(`insert_markdown_doc`) expect(prompt).toContain(`Collaborative Markdown Docs`) expect(prompt).toContain(`Do not use filesystem write`) expect(prompt).toContain(`markdown-docs skill`) diff --git a/packages/agents/test/horton-tool-composition.test.ts b/packages/agents/test/horton-tool-composition.test.ts index 73f1af3d33..5b3e27b2b8 100644 --- a/packages/agents/test/horton-tool-composition.test.ts +++ b/packages/agents/test/horton-tool-composition.test.ts @@ -80,9 +80,24 @@ async function captureToolset(args: Record = {}) { } function createElectricToolsContext() { + const document = { + key: `document:notes`, + kind: `document`, + id: `notes`, + provider: `y-durable-streams`, + docId: `agents/horton/smoke/documents/notes`, + docPath: `agents/horton/smoke/documents/notes`, + streamPath: `/v1/yjs/default/docs/agents/horton/smoke/documents/notes`, + transportMimeType: `application/vnd.electric-agents.markdown-yjs`, + contentMimeType: `text/markdown`, + yTextName: `markdown`, + title: `Notes`, + createdAt: new Date(0).toISOString(), + } return { entityUrl: `/horton/smoke/main`, entityType: `horton`, + principal: { url: `/principal/agent:horton`, kind: `agent` }, args: {}, db: { collections: { manifests: { toArray: [] } }, @@ -111,6 +126,15 @@ function createElectricToolsContext() { }, })), unsubscribeFromEventSource: vi.fn(async () => ({ txid: `tx-unsubscribe` })), + createMarkdownDocument: vi.fn(async () => ({ + txid: `tx-create-doc`, + document, + })), + readMarkdownDocumentStream: vi.fn(async () => ({ + bytes: new Uint8Array(), + })), + appendMarkdownDocumentUpdate: vi.fn(async () => ({})), + appendMarkdownDocumentAwareness: vi.fn(async () => ({})), } as any } @@ -229,6 +253,8 @@ describe(`horton tool composition`, () => { `list_event_source_subscriptions`, `unsubscribe_event_source`, `create_markdown_doc`, + `set_markdown_doc_cursor`, + `insert_markdown_doc`, `read_markdown_doc`, `write_markdown_doc`, `edit_markdown_doc`, @@ -261,10 +287,14 @@ describe(`horton tool composition`, () => { expect(names).toContain(`list_event_sources`) expect(names).toContain(`subscribe_event_source`) expect(names).toContain(`create_markdown_doc`) + expect(names).toContain(`set_markdown_doc_cursor`) + expect(names).toContain(`insert_markdown_doc`) expect(names).toContain(`edit_markdown_doc`) expect(cfg.systemPrompt).toContain(`list_event_sources`) expect(cfg.systemPrompt).toContain(`subscribe_event_source`) expect(cfg.systemPrompt).toContain(`create_markdown_doc`) + expect(cfg.systemPrompt).toContain(`set_markdown_doc_cursor`) + expect(cfg.systemPrompt).toContain(`insert_markdown_doc`) expect(cfg.systemPrompt).toContain(`Collaborative Markdown Docs`) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1d364430db..50b15cbe03 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1849,6 +1849,9 @@ importers: jsdom: specifier: ^28.1.0 version: 28.1.0(@noble/hashes@2.0.1) + lib0: + specifier: ^0.2.99 + version: 0.2.99 pino: specifier: ^10.3.1 version: 10.3.1 @@ -1864,6 +1867,12 @@ importers: turndown-plugin-gfm: specifier: ^1.0.2 version: 1.0.2 + y-protocols: + specifier: ^1.0.6 + version: 1.0.6(yjs@13.6.26) + yjs: + specifier: ^13.6.26 + version: 13.6.26 zod: specifier: ^4.3.6 version: 4.3.6 From 271f7f50d8c3626041de78be1e9a5c2824647c7f Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Mon, 8 Jun 2026 20:34:21 +0100 Subject: [PATCH 5/8] Fix markdown insert presence cursor --- .../agents-runtime/src/tools/markdown-docs.ts | 4 - .../test/markdown-docs-tools.test.ts | 77 ++++++++++++++++++- 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/packages/agents-runtime/src/tools/markdown-docs.ts b/packages/agents-runtime/src/tools/markdown-docs.ts index 2d2739d312..985d1f4d97 100644 --- a/packages/agents-runtime/src/tools/markdown-docs.ts +++ b/packages/agents-runtime/src/tools/markdown-docs.ts @@ -234,10 +234,6 @@ export function createMarkdownDocumentTools( (index === undefined ? cursorPositions.get(id) : undefined), textName: materialized.textName, }) - await appendPresence(materialized, { - anchor: result.index, - head: result.index, - }) await appendDocumentUpdate(id, materialized, result.update) await appendPresence(materialized, { anchor: result.nextIndex, diff --git a/packages/agents-runtime/test/markdown-docs-tools.test.ts b/packages/agents-runtime/test/markdown-docs-tools.test.ts index 1e13061940..dd6c2a26e1 100644 --- a/packages/agents-runtime/test/markdown-docs-tools.test.ts +++ b/packages/agents-runtime/test/markdown-docs-tools.test.ts @@ -1,4 +1,6 @@ import { describe, expect, it, vi } from 'vitest' +import * as decoding from 'lib0/decoding' +import { Awareness, applyAwarenessUpdate } from 'y-protocols/awareness' import * as Y from 'yjs' import { createMarkdownYDoc, @@ -31,6 +33,49 @@ function contentFromStream(streamBytes: Uint8Array): string { return markdownText(createMarkdownYDoc(streamBytes)).toString() } +async function waitForCondition( + predicate: () => boolean, + message: string +): Promise { + for (let i = 0; i < 20; i += 1) { + if (predicate()) return + await new Promise((resolve) => setTimeout(resolve, 0)) + } + throw new Error(message) +} + +function applyFramedAwarenessUpdate( + awareness: Awareness, + data: Uint8Array +): void { + const decoder = decoding.createDecoder(data) + while (decoding.hasContent(decoder)) { + applyAwarenessUpdate(awareness, decoding.readVarUint8Array(decoder), `test`) + } +} + +function cursorHeadIndexFromAwarenessFrame( + doc: Y.Doc, + frame: Uint8Array +): number | undefined { + const awareness = new Awareness(new Y.Doc()) + applyFramedAwarenessUpdate(awareness, frame) + for (const state of awareness.getStates().values()) { + const cursor = ( + state as { + cursor?: { head?: Y.RelativePosition; anchor?: Y.RelativePosition } + } + ).cursor + if (!cursor?.head) continue + const absolute = Y.createAbsolutePositionFromRelativePosition( + cursor.head, + doc + ) + return absolute?.index + } + return undefined +} + function createToolContext() { const document = { key: `document:notes`, @@ -47,6 +92,7 @@ function createToolContext() { createdAt: `2026-06-07T00:00:00.000Z`, } as const let streamFrames = [streamBytesFromContent(`# Notes\n\nFirst line\n`)] + const awarenessFrames: Array = [] return { context: { entityUrl: `/chat/session`, @@ -85,7 +131,12 @@ function createToolContext() { return { offset: String(streamFrames.length) } } ), - appendMarkdownDocumentAwareness: vi.fn(async () => ({})), + appendMarkdownDocumentAwareness: vi.fn( + async (_streamPath: string, update: Uint8Array) => { + awarenessFrames.push(update) + return {} + } + ), upsertCronSchedule: vi.fn(), upsertFutureSendSchedule: vi.fn(), deleteSchedule: vi.fn(), @@ -94,6 +145,8 @@ function createToolContext() { unsubscribeFromEventSource: vi.fn(), } as any, getContent: () => contentFromStream(concatFrames(streamFrames)), + getDoc: () => createMarkdownYDoc(concatFrames(streamFrames)), + getAwarenessFrames: () => awarenessFrames, appendExternalText: (text: string) => { const streamBytes = concatFrames(streamFrames) const doc = createMarkdownYDoc(streamBytes) @@ -164,7 +217,8 @@ describe(`markdown document tools`, () => { }) it(`streams insert_markdown_doc content deltas before final execution`, async () => { - const { context, getContent } = createToolContext() + const { context, getContent, getDoc, getAwarenessFrames } = + createToolContext() const insert = createMarkdownDocumentTools(context).find( (tool) => tool.name === `insert_markdown_doc` )! @@ -175,12 +229,29 @@ describe(`markdown document tools`, () => { delta: `"Hello`, argsPreview: { id: `notes`, content: `Hello` }, }) + await waitForCondition( + () => context.appendMarkdownDocumentAwareness.mock.calls.length === 1, + `expected first streamed insert presence update` + ) + expect(context.appendMarkdownDocumentAwareness).toHaveBeenCalledTimes(1) + expect( + cursorHeadIndexFromAwarenessFrame(getDoc(), getAwarenessFrames().at(-1)!) + ).toBe(getContent().length) + await insert.onArgsDelta?.({ toolCallId: `tool-insert`, toolName: `insert_markdown_doc`, delta: ` world"`, argsPreview: { id: `notes`, content: `Hello world` }, }) + await waitForCondition( + () => context.appendMarkdownDocumentAwareness.mock.calls.length === 2, + `expected second streamed insert presence update` + ) + expect(context.appendMarkdownDocumentAwareness).toHaveBeenCalledTimes(2) + expect( + cursorHeadIndexFromAwarenessFrame(getDoc(), getAwarenessFrames().at(-1)!) + ).toBe(getContent().length) const result = await insert.execute(`tool-insert`, { id: `notes`, @@ -188,7 +259,7 @@ describe(`markdown document tools`, () => { }) expect(context.appendMarkdownDocumentUpdate).toHaveBeenCalledTimes(2) - expect(context.appendMarkdownDocumentAwareness).toHaveBeenCalled() + expect(context.appendMarkdownDocumentAwareness).toHaveBeenCalledTimes(3) expect(getContent()).toContain(`Hello world`) expect(result.details).toMatchObject({ streamed: true }) }) From 3922409a342c51a4a7de5ba12de70e3e7dd88c89 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Mon, 8 Jun 2026 20:41:55 +0100 Subject: [PATCH 6/8] Add streaming markdown replacement tool --- packages/agents-runtime/src/markdown-yjs.ts | 31 ++ .../agents-runtime/src/tools/markdown-docs.ts | 349 ++++++++++++++++++ .../test/markdown-docs-tools.test.ts | 82 ++++ packages/agents/skills/markdown-docs.md | 23 +- packages/agents/src/agents/horton.ts | 4 +- .../agents/test/horton-system-prompt.test.ts | 1 + .../test/horton-tool-composition.test.ts | 2 + 7 files changed, 484 insertions(+), 8 deletions(-) diff --git a/packages/agents-runtime/src/markdown-yjs.ts b/packages/agents-runtime/src/markdown-yjs.ts index 754cc41fb4..9b9b3a1bc6 100644 --- a/packages/agents-runtime/src/markdown-yjs.ts +++ b/packages/agents-runtime/src/markdown-yjs.ts @@ -136,6 +136,37 @@ export function insertMarkdownText( } } +export function deleteMarkdownTextRange( + doc: Y.Doc, + index: number, + length: number, + textName: string = MARKDOWN_DOCUMENT_TEXT_NAME +): { + update: Uint8Array + index: number + length: number + position: Y.RelativePosition +} { + const text = markdownText(doc, textName) + const boundedIndex = Math.max(0, Math.min(index, text.length)) + const boundedLength = Math.max( + 0, + Math.min(length, text.length - boundedIndex) + ) + const before = Y.encodeStateVector(doc) + if (boundedLength > 0) { + doc.transact(() => { + text.delete(boundedIndex, boundedLength) + }, `agent`) + } + return { + update: Y.encodeStateAsUpdate(doc, before), + index: boundedIndex, + length: boundedLength, + position: Y.createRelativePositionFromTypeIndex(text, boundedIndex), + } +} + export function relativePositionAtMarkdownIndex( doc: Y.Doc, index: number, diff --git a/packages/agents-runtime/src/tools/markdown-docs.ts b/packages/agents-runtime/src/tools/markdown-docs.ts index 985d1f4d97..7bb6e4dc06 100644 --- a/packages/agents-runtime/src/tools/markdown-docs.ts +++ b/packages/agents-runtime/src/tools/markdown-docs.ts @@ -3,6 +3,7 @@ import { Type } from '@sinclair/typebox' import { applyFramedYjsUpdates, createMarkdownYDoc, + deleteMarkdownTextRange, editMarkdownText, encodeMarkdownAwarenessUpdate, frameYjsUpdate, @@ -30,6 +31,15 @@ type InsertMarkdownArgs = { index?: number } +type ReplaceMarkdownArgs = { + id: string + content: string + old_string?: string + occurrence?: number + index?: number + length?: number +} + type SetCursorArgs = { id: string index?: number @@ -49,6 +59,14 @@ type InsertSession = { error?: unknown } +type ReplaceSession = InsertSession & { + prepared?: boolean + deleted?: string + deleteIndex?: number + deleteLength?: number + beforeContent?: string +} + type MaterializedMarkdownDocument = { document: ManifestDocumentEntry doc: Y.Doc @@ -87,11 +105,34 @@ function asInsertArgs(value: unknown): Partial { } } +function asReplaceArgs(value: unknown): Partial { + if (!value || typeof value !== `object`) return {} + const input = value as Record + return { + ...(typeof input.id === `string` && { id: input.id }), + ...(typeof input.content === `string` && { content: input.content }), + ...(typeof input.old_string === `string` && { + old_string: input.old_string, + }), + ...(typeof input.occurrence === `number` && + Number.isFinite(input.occurrence) + ? { occurrence: input.occurrence } + : {}), + ...(typeof input.index === `number` && Number.isFinite(input.index) + ? { index: input.index } + : {}), + ...(typeof input.length === `number` && Number.isFinite(input.length) + ? { length: input.length } + : {}), + } +} + export function createMarkdownDocumentTools( context: ElectricToolContext ): Array { const readDocs = new Map() const insertSessions = new Map() + const replaceSessions = new Map() const materializedDocs = new Map() const cursorPositions = new Map() @@ -295,6 +336,124 @@ export function createMarkdownDocumentTools( return { index: args.after !== undefined ? found + needle.length : found } } + const resolveReplaceRange = ( + content: string, + args: Omit + ): { index?: number; length?: number; deleted?: string; error?: string } => { + const hasOldString = args.old_string !== undefined + const hasRange = args.index !== undefined || args.length !== undefined + if (hasOldString && hasRange) { + return { error: `Pass either old_string or index/length, not both.` } + } + if (!hasOldString && !hasRange) { + return { error: `Pass old_string or index/length to choose a range.` } + } + if (hasOldString) { + const oldString = args.old_string! + if (oldString.length === 0) { + return { error: `old_string must not be empty.` } + } + const occurrence = + args.occurrence === undefined + ? undefined + : Math.max(1, Math.floor(args.occurrence)) + let from = 0 + let found = -1 + let count = 0 + while (true) { + const index = content.indexOf(oldString, from) + if (index < 0) break + count += 1 + if (occurrence === undefined || count === occurrence) { + found = index + if (occurrence !== undefined) break + } + from = index + oldString.length + } + if (found < 0) { + return { + error: + occurrence === undefined + ? `old_string not found.` + : `Could not find occurrence ${occurrence} of old_string.`, + } + } + if (occurrence === undefined && count > 1) { + return { + error: `found ${count} matches for old_string; pass occurrence to choose one or provide a more specific old_string.`, + } + } + return { + index: found, + length: oldString.length, + deleted: oldString, + } + } + + if (args.index === undefined || args.length === undefined) { + return { error: `Pass both index and length for explicit ranges.` } + } + const index = Math.max(0, Math.min(Math.floor(args.index), content.length)) + const length = Math.max( + 0, + Math.min(Math.floor(args.length), content.length - index) + ) + if (length === 0) { + return { error: `Replacement range length must be greater than zero.` } + } + return { + index, + length, + deleted: content.slice(index, index + length), + } + } + + const prepareReplaceSession = async ( + session: ReplaceSession, + args: Omit + ): Promise => { + const materialized = await materializeDocument(args.id) + if (session.prepared) return materialized + + const before = contentOf(materialized) + const range = resolveReplaceRange(before, args) + if ( + range.error || + range.index === undefined || + range.length === undefined + ) { + throw new Error(range.error ?? `Could not resolve replacement range.`) + } + + await appendPresence(materialized, { + anchor: range.index, + head: range.index + range.length, + }) + const deletion = deleteMarkdownTextRange( + materialized.doc, + range.index, + range.length, + materialized.textName + ) + await appendDocumentUpdate(args.id, materialized, deletion.update) + await appendPresence(materialized, { + anchor: deletion.index, + head: deletion.index, + }) + + session.id = args.id + session.nextIndex = deletion.index + session.nextPosition = deletion.position + session.prepared = true + session.deleted = + range.deleted ?? before.slice(range.index, range.index + range.length) + session.deleteIndex = deletion.index + session.deleteLength = deletion.length + session.beforeContent = before + cursorPositions.set(args.id, deletion.position) + return materialized + } + const enqueueInsert = ( toolCallId: string, action: (session: InsertSession) => Promise @@ -325,6 +484,36 @@ export function createMarkdownDocumentTools( return session } + const enqueueReplace = ( + toolCallId: string, + action: (session: ReplaceSession) => Promise + ): void => { + const session = + replaceSessions.get(toolCallId) ?? + ({ + inserted: ``, + seq: 0, + streamed: false, + pending: Promise.resolve(), + } satisfies ReplaceSession) + replaceSessions.set(toolCallId, session) + session.pending = session.pending + .then(() => action(session)) + .catch((error) => { + session.error = error + }) + } + + const awaitReplaceSession = async ( + toolCallId: string + ): Promise => { + const session = replaceSessions.get(toolCallId) + if (!session) return undefined + await session.pending + if (session.error) throw session.error + return session + } + return [ { name: `create_markdown_doc`, @@ -530,6 +719,166 @@ export function createMarkdownDocumentTools( }, executionMode: `sequential`, }, + { + name: `replace_markdown_doc_range`, + label: `Replace Markdown Doc Range`, + description: `Delete one range from a collaborative app markdown document, then stream or insert replacement markdown at that location. Use old_string for a unique literal match, old_string plus occurrence for repeated text, or index plus length for an explicit UTF-16 range. The agent cursor follows the end of streamed replacement text.`, + parameters: Type.Object({ + id: Type.String({ description: `Document id.` }), + old_string: Type.Optional( + Type.String({ + description: `Literal markdown text to replace. Must be unique unless occurrence is provided.`, + }) + ), + occurrence: Type.Optional( + Type.Number({ + description: `1-based occurrence to replace when old_string appears multiple times.`, + }) + ), + index: Type.Optional( + Type.Number({ + description: `Optional UTF-16 start offset for an explicit replacement range.`, + }) + ), + length: Type.Optional( + Type.Number({ + description: `UTF-16 length for an explicit replacement range. Required when index is used.`, + }) + ), + content: Type.String({ + description: `Replacement markdown content to stream into the deleted range.`, + }), + }), + onArgsDelta: ({ toolCallId, argsPreview }) => { + const args = asReplaceArgs(argsPreview) + if (!args.id || typeof args.content !== `string`) return + if ( + args.old_string === undefined && + (args.index === undefined || args.length === undefined) + ) { + return + } + enqueueReplace(toolCallId, async (session) => { + await prepareReplaceSession(session, { + id: args.id!, + old_string: args.old_string, + occurrence: args.occurrence, + index: args.index, + length: args.length, + }) + if (!args.content!.startsWith(session.inserted)) return + const chunk = args.content!.slice(session.inserted.length) + if (chunk.length === 0) return + session.inserted = args.content! + await applyInsertChunk(args.id!, chunk, session) + session.seq++ + }) + }, + execute: async (toolCallId, params) => { + const args = params as ReplaceMarkdownArgs + const session = + (await awaitReplaceSession(toolCallId)) ?? + ({ + inserted: ``, + seq: 0, + streamed: false, + pending: Promise.resolve(), + } satisfies ReplaceSession) + + try { + await prepareReplaceSession(session, { + id: args.id, + old_string: args.old_string, + occurrence: args.occurrence, + index: args.index, + length: args.length, + }) + } catch (error) { + replaceSessions.delete(toolCallId) + return { + content: [ + { + type: `text` as const, + text: `Error: ${error instanceof Error ? error.message : `could not prepare replacement`}`, + }, + ], + details: { replaced: false }, + } + } + + let inserted = session.inserted + let streamed = session.streamed + let nextIndex = session.nextIndex + if (args.content !== inserted) { + if (inserted.length === 0 || args.content.startsWith(inserted)) { + const remaining = + inserted.length === 0 + ? args.content + : args.content.slice(inserted.length) + if (remaining.length > 0) { + await applyInsertChunk(args.id, remaining, session) + nextIndex = session.nextIndex + inserted = args.content + streamed = streamed || remaining.length !== args.content.length + } + } else { + const materialized = materializedDocs.get(args.id) + if (materialized) { + await appendPresence(materialized, { clear: true }) + } + replaceSessions.delete(toolCallId) + return { + content: [ + { + type: `text` as const, + text: `Error: streamed replacement content diverged from final content; no final reconciliation was applied.`, + }, + ], + details: { + replaced: false, + inserted: inserted.length, + expected: args.content.length, + }, + } + } + } + + const materialized = await materializeDocument(args.id) + await appendPresence(materialized, { clear: true }) + const finalContent = contentOf(materialized) + readDocs.set(args.id, finalContent) + replaceSessions.delete(toolCallId) + const diff = createTwoFilesPatch( + docLabel(args.id), + docLabel(args.id), + session.beforeContent ?? finalContent, + finalContent, + undefined, + undefined, + { context: 3 } + ) + return { + content: [ + { + type: `text` as const, + text: `Replaced ${session.deleteLength ?? 0} characters in markdown document ${args.id}`, + }, + ], + details: { + document: materialized.document, + replaced: true, + deleted: session.deleted, + deleteIndex: session.deleteIndex, + deleteLength: session.deleteLength, + streamed, + insertedBytes: new TextEncoder().encode(args.content).length, + nextIndex, + diff, + }, + } + }, + executionMode: `sequential`, + }, { name: `read_markdown_doc`, label: `Read Markdown Doc`, diff --git a/packages/agents-runtime/test/markdown-docs-tools.test.ts b/packages/agents-runtime/test/markdown-docs-tools.test.ts index dd6c2a26e1..b3a5ab4425 100644 --- a/packages/agents-runtime/test/markdown-docs-tools.test.ts +++ b/packages/agents-runtime/test/markdown-docs-tools.test.ts @@ -303,6 +303,88 @@ describe(`markdown document tools`, () => { expect(context.appendMarkdownDocumentUpdate).toHaveBeenCalledTimes(2) }) + it(`replaces a markdown range with one delete update and one insert update`, async () => { + const { context, getContent } = createToolContext() + const replace = createMarkdownDocumentTools(context).find( + (tool) => tool.name === `replace_markdown_doc_range` + )! + + const result = await replace.execute(`tool-replace`, { + id: `notes`, + old_string: `First line`, + content: `Replacement line`, + }) + + expect(getContent()).toBe(`# Notes\n\nReplacement line\n`) + expect(context.appendMarkdownDocumentUpdate).toHaveBeenCalledTimes(2) + expect(context.appendMarkdownDocumentAwareness).toHaveBeenCalledTimes(4) + expect(result.details).toMatchObject({ + replaced: true, + deleted: `First line`, + streamed: false, + }) + }) + + it(`streams replace_markdown_doc_range replacement content at the deleted range`, async () => { + const { context, getContent, getDoc, getAwarenessFrames } = + createToolContext() + const replace = createMarkdownDocumentTools(context).find( + (tool) => tool.name === `replace_markdown_doc_range` + )! + + await replace.onArgsDelta?.({ + toolCallId: `tool-stream-replace`, + toolName: `replace_markdown_doc_range`, + delta: `"Replacement`, + argsPreview: { + id: `notes`, + old_string: `First line`, + content: `Replacement`, + }, + }) + await waitForCondition( + () => context.appendMarkdownDocumentAwareness.mock.calls.length === 3, + `expected replacement delete and first streamed insert presence updates` + ) + expect(getContent()).toBe(`# Notes\n\nReplacement\n`) + expect( + cursorHeadIndexFromAwarenessFrame(getDoc(), getAwarenessFrames().at(-1)!) + ).toBe(getContent().length - 1) + + await replace.onArgsDelta?.({ + toolCallId: `tool-stream-replace`, + toolName: `replace_markdown_doc_range`, + delta: ` line"`, + argsPreview: { + id: `notes`, + old_string: `First line`, + content: `Replacement line`, + }, + }) + await waitForCondition( + () => context.appendMarkdownDocumentAwareness.mock.calls.length === 4, + `expected second streamed replacement presence update` + ) + expect(getContent()).toBe(`# Notes\n\nReplacement line\n`) + expect( + cursorHeadIndexFromAwarenessFrame(getDoc(), getAwarenessFrames().at(-1)!) + ).toBe(getContent().length - 1) + + const result = await replace.execute(`tool-stream-replace`, { + id: `notes`, + old_string: `First line`, + content: `Replacement line`, + }) + + expect(context.appendMarkdownDocumentUpdate).toHaveBeenCalledTimes(3) + expect(context.appendMarkdownDocumentAwareness).toHaveBeenCalledTimes(5) + expect(result.details).toMatchObject({ + replaced: true, + streamed: true, + deleted: `First line`, + }) + }) + it(`refreshes a cached Yjs document from the stream before editing`, async () => { const { context, getContent, appendExternalText } = createToolContext() const tools = createMarkdownDocumentTools(context) diff --git a/packages/agents/skills/markdown-docs.md b/packages/agents/skills/markdown-docs.md index f27e1d7e5e..73a4a32f5e 100644 --- a/packages/agents/skills/markdown-docs.md +++ b/packages/agents/skills/markdown-docs.md @@ -24,8 +24,9 @@ Agents UI and can be opened, edited, and watched live. Collaborative markdown docs are not filesystem files. - Use `create_markdown_doc`, `set_markdown_doc_cursor`, - `insert_markdown_doc`, `read_markdown_doc`, `write_markdown_doc`, and - `edit_markdown_doc` for docs the user should open in the workspace UI. + `insert_markdown_doc`, `replace_markdown_doc_range`, `read_markdown_doc`, + `write_markdown_doc`, and `edit_markdown_doc` for docs the user should open + in the workspace UI. - Use filesystem `write`/`edit` only when the user asks for an actual file path in the workspace or repo, such as `docs/foo.md`, `README.md`, or `/tmp/report.md`. @@ -75,11 +76,21 @@ For small edits: 3. If the target text appears multiple times, make `old_string` more specific or set `replace_all` only when replacing every occurrence is clearly intended. +For replacing a section with new long content that should appear live: + +1. Use `read_markdown_doc` if you need to inspect or disambiguate the target. +2. Use `replace_markdown_doc_range` with a unique `old_string`, or with + `old_string` plus `occurrence` for repeated text. +3. For exact offsets, use `index` plus `length` instead of `old_string`. +4. Put the range selector before `content` in the tool arguments so the range is + deleted once and replacement content can stream into that Yjs-relative + position. + The markdown tools materialize the collaborative Yjs document from its durable -stream during the wake. `write_markdown_doc`, `edit_markdown_doc`, and -`insert_markdown_doc` append binary Yjs updates to that stream; do not write -markdown documents to the local filesystem unless the user explicitly asks for a -filesystem file. +stream during the wake. `write_markdown_doc`, `edit_markdown_doc`, +`insert_markdown_doc`, and `replace_markdown_doc_range` append binary Yjs +updates to that stream; do not write markdown documents to the local filesystem +unless the user explicitly asks for a filesystem file. For broad rewrites: diff --git a/packages/agents/src/agents/horton.ts b/packages/agents/src/agents/horton.ts index 1732686440..7cd285d039 100644 --- a/packages/agents/src/agents/horton.ts +++ b/packages/agents/src/agents/horton.ts @@ -199,13 +199,13 @@ export function buildHortonSystemPrompt( ? `\n- list_event_sources: list external webhook/event feeds you can subscribe to, including available buckets and parameters\n- subscribe_event_source: subscribe yourself to one of those feeds or buckets so matching future events wake you\n- list_event_source_subscriptions: list your active event source subscriptions\n- unsubscribe_event_source: remove one of your event source subscriptions by id` : `` const markdownDocumentTools = opts.hasMarkdownDocumentTools - ? `\n- create_markdown_doc: create a collaborative markdown document that appears in this entity's manifest and opens in the workspace editor\n- set_markdown_doc_cursor: choose a stateful Yjs-relative insertion cursor for a collaborative markdown document\n- insert_markdown_doc: stream or insert markdown into an existing collaborative document so open editors can watch content appear\n- read_markdown_doc: read a collaborative markdown document\n- write_markdown_doc: replace a collaborative markdown document's full content\n- edit_markdown_doc: targeted string replacement in a collaborative markdown document` + ? `\n- create_markdown_doc: create a collaborative markdown document that appears in this entity's manifest and opens in the workspace editor\n- set_markdown_doc_cursor: choose a stateful Yjs-relative insertion cursor for a collaborative markdown document\n- insert_markdown_doc: stream or insert markdown into an existing collaborative document so open editors can watch content appear\n- replace_markdown_doc_range: delete a range and stream replacement markdown into that location\n- read_markdown_doc: read a collaborative markdown document\n- write_markdown_doc: replace a collaborative markdown document's full content\n- edit_markdown_doc: targeted string replacement in a collaborative markdown document` : `` const skillsTools = opts.hasSkills ? `\n- use_skill: load a skill (knowledge, instructions, or a tutorial) into your context to help with the user's request\n- remove_skill: unload a skill from context when you're done with it` : `` const markdownDocumentGuidance = opts.hasMarkdownDocumentTools - ? `\n# Collaborative Markdown Docs\n- If the user asks you to create a markdown doc, notes, draft, brief, plan, report, or any document they should open/edit in the app UI, use create_markdown_doc. Do not use filesystem write unless they ask for a file path or repo/workspace file.\n- For larger document workflows, load the markdown-docs skill first with use_skill, then use the markdown document tools.\n- Use set_markdown_doc_cursor before insert_markdown_doc when inserting at a specific location; insert_markdown_doc can then stream content at that Yjs-relative cursor.\n- After creating a collaborative doc, mention that it is available from this entity's manifest/timeline.` + ? `\n# Collaborative Markdown Docs\n- If the user asks you to create a markdown doc, notes, draft, brief, plan, report, or any document they should open/edit in the app UI, use create_markdown_doc. Do not use filesystem write unless they ask for a file path or repo/workspace file.\n- For larger document workflows, load the markdown-docs skill first with use_skill, then use the markdown document tools.\n- Use set_markdown_doc_cursor before insert_markdown_doc when inserting at a specific location; insert_markdown_doc can then stream content at that Yjs-relative cursor.\n- Use replace_markdown_doc_range when replacing existing prose with new content that should visibly stream into the deleted range.\n- After creating a collaborative doc, mention that it is available from this entity's manifest/timeline.` : `` const docsGuidance = opts.hasDocsSupport ? `\n- For ANY question about Electric Agents or this framework, ALWAYS use search_electric_agents_docs FIRST. Do not use web_search or fetch_url for Electric Agents topics unless the docs search returns no useful results.\n- The search tool returns chunk content directly โ€” you do not need to read the source files.\n- Use repo read/bash tools only for non-doc files or when you need to inspect exact implementation code in the workspace.` diff --git a/packages/agents/test/horton-system-prompt.test.ts b/packages/agents/test/horton-system-prompt.test.ts index 360cbfeb18..e404093210 100644 --- a/packages/agents/test/horton-system-prompt.test.ts +++ b/packages/agents/test/horton-system-prompt.test.ts @@ -48,6 +48,7 @@ describe(`buildHortonSystemPrompt`, () => { expect(prompt).toContain(`create_markdown_doc`) expect(prompt).toContain(`set_markdown_doc_cursor`) expect(prompt).toContain(`insert_markdown_doc`) + expect(prompt).toContain(`replace_markdown_doc_range`) expect(prompt).toContain(`Collaborative Markdown Docs`) expect(prompt).toContain(`Do not use filesystem write`) expect(prompt).toContain(`markdown-docs skill`) diff --git a/packages/agents/test/horton-tool-composition.test.ts b/packages/agents/test/horton-tool-composition.test.ts index 5b3e27b2b8..6a6c8975c9 100644 --- a/packages/agents/test/horton-tool-composition.test.ts +++ b/packages/agents/test/horton-tool-composition.test.ts @@ -255,6 +255,7 @@ describe(`horton tool composition`, () => { `create_markdown_doc`, `set_markdown_doc_cursor`, `insert_markdown_doc`, + `replace_markdown_doc_range`, `read_markdown_doc`, `write_markdown_doc`, `edit_markdown_doc`, @@ -288,6 +289,7 @@ describe(`horton tool composition`, () => { expect(names).toContain(`subscribe_event_source`) expect(names).toContain(`create_markdown_doc`) expect(names).toContain(`set_markdown_doc_cursor`) + expect(names).toContain(`replace_markdown_doc_range`) expect(names).toContain(`insert_markdown_doc`) expect(names).toContain(`edit_markdown_doc`) expect(cfg.systemPrompt).toContain(`list_event_sources`) From 43041359c2d24339541be349577dfa7fea8d9667 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Mon, 8 Jun 2026 20:55:33 +0100 Subject: [PATCH 7/8] Fix markdown document agent presence --- .../views/MarkdownDocumentView.test.ts | 41 +++++++++- .../components/views/MarkdownDocumentView.tsx | 77 ++++++++++++++++++- packages/agents-server/src/entity-manager.ts | 16 ++++ .../src/routing/durable-streams-router.ts | 7 +- ...ic-agents-manager-write-validation.test.ts | 6 +- .../test/electric-agents-routes.test.ts | 42 ++++++++++ 6 files changed, 183 insertions(+), 6 deletions(-) diff --git a/packages/agents-server-ui/src/components/views/MarkdownDocumentView.test.ts b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.test.ts index 66414026db..7746958de6 100644 --- a/packages/agents-server-ui/src/components/views/MarkdownDocumentView.test.ts +++ b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.test.ts @@ -1,7 +1,23 @@ import { describe, expect, it } from 'vitest' -import { markdownDocumentConnectionConfig } from './MarkdownDocumentView' +import * as encoding from 'lib0/encoding' +import { + Awareness, + encodeAwarenessUpdate, + type Awareness as AwarenessType, +} from 'y-protocols/awareness' +import * as Y from 'yjs' +import { + applyMarkdownAwarenessFrames, + markdownDocumentConnectionConfig, +} from './MarkdownDocumentView' import type { ManifestDocumentEntry } from '@electric-ax/agents-runtime/client' +function frame(update: Uint8Array): Uint8Array { + const encoder = encoding.createEncoder() + encoding.writeVarUint8Array(encoder, update) + return encoding.toUint8Array(encoder) +} + describe(`markdownDocumentConnectionConfig`, () => { it(`uses explicit provider doc metadata for editor connections`, () => { const config = markdownDocumentConnectionConfig( @@ -32,3 +48,26 @@ describe(`markdownDocumentConnectionConfig`, () => { ) }) }) + +describe(`applyMarkdownAwarenessFrames`, () => { + it(`applies lib0-framed awareness updates`, () => { + const sourceDoc = new Y.Doc() + const source = new Awareness(sourceDoc) + source.setLocalState({ + user: { name: `horton`, role: `agent`, status: `editing` }, + cursor: { anchor: 4, head: 4 }, + }) + + const target = new Awareness(new Y.Doc()) as AwarenessType + applyMarkdownAwarenessFrames( + target, + frame(encodeAwarenessUpdate(source, [source.clientID])) + ) + + const remoteState = target.getStates().get(source.clientID) + expect(remoteState).toMatchObject({ + user: { name: `horton`, role: `agent`, status: `editing` }, + cursor: { anchor: 4, head: 4 }, + }) + }) +}) diff --git a/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx index 580fd33e3e..5705411c27 100644 --- a/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx +++ b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx @@ -6,7 +6,13 @@ import { keymap } from '@codemirror/view' import { YjsProvider } from '@durable-streams/y-durable-streams' import { Plug, TriangleAlert, Unplug } from 'lucide-react' import { yCollab, yUndoManagerKeymap } from 'y-codemirror.next' -import { Awareness, removeAwarenessStates } from 'y-protocols/awareness' +import * as decoding from 'lib0/decoding' +import { + Awareness, + applyAwarenessUpdate, + encodeAwarenessUpdate, + removeAwarenessStates, +} from 'y-protocols/awareness' import * as Y from 'yjs' import { useCurrentPrincipal } from '../../hooks/useCurrentPrincipal' import { getConfiguredServerHeaders, serverFetch } from '../../lib/auth-fetch' @@ -94,6 +100,68 @@ function connectionStatusIcon(status: DocumentConnectionStatus): LucideIcon { } } +export function applyMarkdownAwarenessFrames( + awareness: Awareness, + data: Uint8Array +): void { + if (data.length === 0) return + const decoder = decoding.createDecoder(data) + while (decoding.hasContent(decoder)) { + applyAwarenessUpdate( + awareness, + decoding.readVarUint8Array(decoder), + `server` + ) + } +} + +async function primeMarkdownAwareness( + awareness: Awareness, + docUrl: URL, + signal: AbortSignal +): Promise { + const awarenessUrl = new URL(docUrl) + awarenessUrl.searchParams.set(`awareness`, `default`) + awarenessUrl.searchParams.set(`offset`, `-1`) + const response = await serverFetch(awarenessUrl, { + method: `GET`, + headers: getConfiguredServerHeaders(awarenessUrl), + signal, + }) + if (signal.aborted) return + if (response.status === 404) return + if (!response.ok) return + const bytes = new Uint8Array(await response.arrayBuffer()) + if (signal.aborted) return + + const snapshot = new Awareness(new Y.Doc()) + applyMarkdownAwarenessFrames(snapshot, bytes) + const now = Date.now() + const activeAgents = Array.from(snapshot.getStates()) + .filter(([, state]) => { + const user = ( + state as { + user?: { role?: string; status?: string; expiresAt?: number } + } + ).user + return ( + user?.role === `agent` && + user.status === `editing` && + typeof user.expiresAt === `number` && + user.expiresAt > now + ) + }) + .map(([clientId]) => clientId) + if (activeAgents.length > 0) { + applyAwarenessUpdate( + awareness, + encodeAwarenessUpdate(snapshot, activeAgents), + `server` + ) + } + snapshot.destroy() +} + export function markdownDocumentConnectionConfig( baseUrl: string, documentEntry: ManifestDocumentEntry @@ -175,6 +243,12 @@ export function MarkdownDocumentView({ const { providerUrl, docUrl, docId, yTextName } = markdownDocumentConnectionConfig(baseUrl, documentEntry) + const awarenessPrimeController = new AbortController() + void primeMarkdownAwareness( + awareness, + docUrl, + awarenessPrimeController.signal + ).catch(() => undefined) const provider = new YjsProvider({ doc: ydoc, baseUrl: providerUrl, @@ -257,6 +331,7 @@ export function MarkdownDocumentView({ setStatus(`connecting`) return () => { + awarenessPrimeController.abort() window.clearInterval(stalePresenceInterval) provider.off(`status`, statusHandler) awareness.off(`change`, updateRemoteUsers) diff --git a/packages/agents-server/src/entity-manager.ts b/packages/agents-server/src/entity-manager.ts index 0b48b3cf4d..8d746ea316 100644 --- a/packages/agents-server/src/entity-manager.ts +++ b/packages/agents-server/src/entity-manager.ts @@ -61,6 +61,7 @@ import { MARKDOWN_DOCUMENT_TRANSPORT_MIME, assertMarkdownDocumentMatchesEntity, getMarkdownDocumentDocPath, + getMarkdownDocumentAwarenessStreamPath, getMarkdownDocumentUpdateStreamPath, } from './markdown-documents.js' import type { queueAsPromised } from 'fastq' @@ -2692,6 +2693,11 @@ export class EntityManager { this.tenantId, docPath ) + const awarenessStreamPath = getMarkdownDocumentAwarenessStreamPath( + this.tenantId, + docPath, + `default` + ) const now = new Date().toISOString() const txid = randomUUID() const document: ManifestMarkdownDocumentEntry = { @@ -2716,11 +2722,16 @@ export class EntityManager { } let streamCreated = false + let awarenessStreamCreated = false try { await this.streamClient.create(updateStreamPath, { contentType: `application/octet-stream`, }) streamCreated = true + await this.streamClient.create(awarenessStreamPath, { + contentType: `application/octet-stream`, + }) + awarenessStreamCreated = true await this.writeManifestEntry( entityUrl, document.key, @@ -2729,6 +2740,11 @@ export class EntityManager { { txid } ) } catch (error) { + if (awarenessStreamCreated) { + await this.streamClient + .delete(awarenessStreamPath) + .catch(() => undefined) + } if (streamCreated) { await this.streamClient.delete(updateStreamPath).catch(() => undefined) } diff --git a/packages/agents-server/src/routing/durable-streams-router.ts b/packages/agents-server/src/routing/durable-streams-router.ts index a30ac9b74f..ddc93b7ae0 100644 --- a/packages/agents-server/src/routing/durable-streams-router.ts +++ b/packages/agents-server/src/routing/durable-streams-router.ts @@ -701,7 +701,7 @@ async function yjsDocumentRoute( request, undefined, `stream`, - rewriteRequestUrlPath(request.url, streamPath) + rewriteYjsDocumentStreamUrl(request.url, streamPath) ) if (!isAwareness && (upstream.ok || upstream.status === 409)) { await ctx.streamClient @@ -724,7 +724,7 @@ async function yjsDocumentRoute( request, undefined, `stream`, - rewriteRequestUrlPath(request.url, streamPath) + rewriteYjsDocumentStreamUrl(request.url, streamPath) ) return responseFromUpstream(upstream) } @@ -755,9 +755,10 @@ async function proxyPassThrough( } } -function rewriteRequestUrlPath(requestUrl: string, path: string): string { +function rewriteYjsDocumentStreamUrl(requestUrl: string, path: string): string { const url = new URL(requestUrl) url.pathname = path + url.searchParams.delete(`awareness`) return url.toString() } diff --git a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts index 9532f3b829..15f2c316ba 100644 --- a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts +++ b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts @@ -206,7 +206,7 @@ describe(`ElectricAgentsManager.validateWriteEvent`, () => { }) describe(`ElectricAgentsManager markdown documents`, () => { - it(`creates an empty Yjs update stream and exposes a manifest document entry`, async () => { + it(`creates Yjs update and awareness streams and exposes a manifest document entry`, async () => { const { manager, streamClient } = createMarkdownDocumentManager() const created = await manager.createMarkdownDocument(`/chat/session-1`, { @@ -233,6 +233,10 @@ describe(`ElectricAgentsManager markdown documents`, () => { `/yjs/default/docs/agents/chat/session-1/documents/notes/.updates`, { contentType: `application/octet-stream` } ) + expect(streamClient.create).toHaveBeenCalledWith( + `/yjs/default/docs/agents/chat/session-1/documents/notes/.awareness/default`, + { contentType: `application/octet-stream` } + ) expect(streamClient.appendBytes).not.toHaveBeenCalled() await expect( manager.getMarkdownDocument(`/chat/session-1`, `notes`) diff --git a/packages/agents-server/test/electric-agents-routes.test.ts b/packages/agents-server/test/electric-agents-routes.test.ts index 61581648a4..767f92eb2f 100644 --- a/packages/agents-server/test/electric-agents-routes.test.ts +++ b/packages/agents-server/test/electric-agents-routes.test.ts @@ -387,6 +387,48 @@ describe(`ElectricAgentsRoutes markdown document endpoints`, () => { } }) + it(`forwards Yjs awareness document routes to the awareness stream`, async () => { + const fetchSpy = vi + .spyOn(globalThis, `fetch`) + .mockResolvedValue(new Response(null, { status: 204 })) + const manager = { + registry: { + getEntity: vi.fn().mockResolvedValue({ + url: `/chat/test`, + created_by: `/principal/user:owner`, + }), + hasEntityPermission: vi.fn().mockResolvedValue(false), + pruneExpiredPermissionGrants: vi.fn(), + }, + isForkWorkLockedEntity: vi.fn().mockReturnValue(false), + } as any + + try { + const allowed = await routeResponse( + manager, + `GET`, + `/v1/yjs/test/docs/agents/chat/test/documents/notes?awareness=default&offset=-1`, + undefined, + false, + undefined, + undefined, + { + durableStreamsUrl: `http://durable.local`, + } + ) + expect(allowed.status).toBe(204) + expect(fetchSpy).toHaveBeenCalledOnce() + const [url, init] = fetchSpy.mock.calls[0]! + expect(String(url)).toContain( + `/yjs/test/docs/agents/chat/test/documents/notes/.awareness/default?offset=-1` + ) + expect(String(url)).not.toContain(`awareness=`) + expect(init).toMatchObject({ method: `GET` }) + } finally { + fetchSpy.mockRestore() + } + }) + it(`guards private Yjs document streams and forwards authorized awareness streams`, async () => { const fetchSpy = vi .spyOn(globalThis, `fetch`) From 2df6d627cebeab2ad129841d188ecef09652690e Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Mon, 8 Jun 2026 21:10:59 +0100 Subject: [PATCH 8/8] Allow workers to edit shared markdown docs --- packages/agents-runtime/src/markdown-yjs.ts | 6 +- .../agents-runtime/src/tools/markdown-docs.ts | 23 +++- .../test/markdown-docs-tools.test.ts | 85 +++++++++++++- packages/agents/src/agents/horton.ts | 2 +- packages/agents/src/agents/worker.ts | 27 ++++- packages/agents/src/tools/spawn-worker.ts | 111 ++++++++++++++++-- .../agents/test/spawn-worker-tool.test.ts | 74 +++++++++++- 7 files changed, 308 insertions(+), 20 deletions(-) diff --git a/packages/agents-runtime/src/markdown-yjs.ts b/packages/agents-runtime/src/markdown-yjs.ts index 9b9b3a1bc6..c0471de654 100644 --- a/packages/agents-runtime/src/markdown-yjs.ts +++ b/packages/agents-runtime/src/markdown-yjs.ts @@ -192,6 +192,7 @@ export function encodeMarkdownAwarenessUpdate(opts: { doc: Y.Doc docPath: string principalUrl: string + clientKey?: string name: string role: `agent` | `user` | `system` status?: `editing` @@ -204,7 +205,10 @@ export function encodeMarkdownAwarenessUpdate(opts: { }): Uint8Array { const awarenessDoc = new Y.Doc() ;(awarenessDoc as { clientID: number }).clientID = - markdownDocumentPresenceClientId(opts.docPath, opts.principalUrl) + markdownDocumentPresenceClientId( + opts.docPath, + opts.clientKey ?? opts.principalUrl + ) const awareness = new Awareness(awarenessDoc) if (opts.clear) { awareness.setLocalState(null) diff --git a/packages/agents-runtime/src/tools/markdown-docs.ts b/packages/agents-runtime/src/tools/markdown-docs.ts index 7bb6e4dc06..683f5a25c4 100644 --- a/packages/agents-runtime/src/tools/markdown-docs.ts +++ b/packages/agents-runtime/src/tools/markdown-docs.ts @@ -74,6 +74,14 @@ type MaterializedMarkdownDocument = { streamOffset?: string } +function injectedMarkdownDocuments( + args: Readonly> +): Array { + const docs = args.markdownDocs + if (!Array.isArray(docs)) return [] + return docs.filter(isManifestDocumentEntry) +} + function isManifestDocumentEntry( value: unknown ): value is ManifestDocumentEntry { @@ -142,9 +150,15 @@ export function createMarkdownDocumentTools( const manifests = context.db.collections.manifests?.toArray as | Array | undefined - return manifests?.find( - (entry): entry is ManifestDocumentEntry => - isManifestDocumentEntry(entry) && entry.id === id + return ( + manifests?.find( + (entry): entry is ManifestDocumentEntry => + isManifestDocumentEntry(entry) && entry.id === id + ) ?? + injectedMarkdownDocuments(context.args).find( + (entry): entry is ManifestDocumentEntry => + isManifestDocumentEntry(entry) && entry.id === id + ) ) } @@ -178,7 +192,7 @@ export function createMarkdownDocumentTools( throw new Error( `Markdown document ${JSON.stringify( id - )} is not in this entity's manifest. Create it with create_markdown_doc first.` + )} is not in this entity's manifest or injected document refs. Create it with create_markdown_doc first or pass the document ref to this worker.` ) } const result = await context.readMarkdownDocumentStream(document.streamPath) @@ -245,6 +259,7 @@ export function createMarkdownDocumentTools( doc: materialized.doc, docPath: materialized.document.docPath, principalUrl, + clientKey: `${principalUrl}\0${context.entityUrl}`, name: principalDisplayName(principalUrl), role: principalRole(principalUrl), status: `editing`, diff --git a/packages/agents-runtime/test/markdown-docs-tools.test.ts b/packages/agents-runtime/test/markdown-docs-tools.test.ts index b3a5ab4425..147173c404 100644 --- a/packages/agents-runtime/test/markdown-docs-tools.test.ts +++ b/packages/agents-runtime/test/markdown-docs-tools.test.ts @@ -4,6 +4,7 @@ import { Awareness, applyAwarenessUpdate } from 'y-protocols/awareness' import * as Y from 'yjs' import { createMarkdownYDoc, + encodeMarkdownAwarenessUpdate, frameYjsUpdate, markdownText, } from '../src/markdown-yjs' @@ -76,7 +77,14 @@ function cursorHeadIndexFromAwarenessFrame( return undefined } -function createToolContext() { +function createToolContext( + opts: { + manifestDocuments?: Array + markdownDocs?: Array + entityUrl?: string + principalUrl?: string + } = {} +) { const document = { key: `document:notes`, kind: `document`, @@ -95,11 +103,20 @@ function createToolContext() { const awarenessFrames: Array = [] return { context: { - entityUrl: `/chat/session`, + entityUrl: opts.entityUrl ?? `/chat/session`, entityType: `chat`, - principal: { url: `/principal/agent:horton`, kind: `agent` }, - args: {}, - db: { collections: { manifests: { toArray: [document] } } }, + principal: { + url: opts.principalUrl ?? `/principal/agent:horton`, + kind: `agent`, + }, + args: { + ...(opts.markdownDocs ? { markdownDocs: opts.markdownDocs } : {}), + }, + db: { + collections: { + manifests: { toArray: opts.manifestDocuments ?? [document] }, + }, + }, events: [], createMarkdownDocument: vi.fn( async (opts: { id?: string; title: string }) => { @@ -155,10 +172,49 @@ function createToolContext() { yText.insert(yText.length, text) streamFrames.push(frameYjsUpdate(Y.encodeStateAsUpdate(doc, before))) }, + document, } } describe(`markdown document tools`, () => { + it(`uses the optional awareness client key to distinguish same-principal editors`, () => { + const doc = new Y.Doc() + markdownText(doc).insert(0, `hello`) + const awareness = new Awareness(new Y.Doc()) + + applyFramedAwarenessUpdate( + awareness, + encodeMarkdownAwarenessUpdate({ + doc, + docPath: `agents/chat/session/documents/notes`, + principalUrl: `/principal/agent:horton`, + clientKey: `/principal/agent:horton\0/chat/session`, + name: `horton`, + role: `agent`, + color: `#000000`, + colorLight: `#00000033`, + }) + ) + applyFramedAwarenessUpdate( + awareness, + encodeMarkdownAwarenessUpdate({ + doc, + docPath: `agents/chat/session/documents/notes`, + principalUrl: `/principal/agent:horton`, + clientKey: `/principal/agent:horton\0/worker/one`, + name: `worker`, + role: `agent`, + color: `#111111`, + colorLight: `#11111133`, + }) + ) + + const remoteStates = Array.from(awareness.getStates()).filter( + ([clientId]) => clientId !== awareness.clientID + ) + expect(remoteStates).toHaveLength(2) + }) + it(`creates the server document empty and appends initial content as a Yjs update`, async () => { const { context, getContent } = createToolContext() const create = createMarkdownDocumentTools(context).find( @@ -197,6 +253,25 @@ describe(`markdown document tools`, () => { expect(result.details).toMatchObject({ replacements: 1 }) }) + it(`reads injected markdown document refs without a local manifest entry`, async () => { + const base = createToolContext() + const { context } = createToolContext({ + manifestDocuments: [], + markdownDocs: [base.document], + entityUrl: `/worker/subagent`, + }) + const read = createMarkdownDocumentTools(context).find( + (tool) => tool.name === `read_markdown_doc` + )! + + const result = await read.execute(`tool-read-injected`, { id: `notes` }) + + expect(context.readMarkdownDocumentStream).toHaveBeenCalledWith( + base.document.streamPath + ) + expect((result.content[0] as { text: string }).text).toContain(`# Notes`) + }) + it(`edits a read document and returns a diff`, async () => { const { context, getContent } = createToolContext() const tools = createMarkdownDocumentTools(context) diff --git a/packages/agents/src/agents/horton.ts b/packages/agents/src/agents/horton.ts index 7cd285d039..b53f1acd11 100644 --- a/packages/agents/src/agents/horton.ts +++ b/packages/agents/src/agents/horton.ts @@ -205,7 +205,7 @@ export function buildHortonSystemPrompt( ? `\n- use_skill: load a skill (knowledge, instructions, or a tutorial) into your context to help with the user's request\n- remove_skill: unload a skill from context when you're done with it` : `` const markdownDocumentGuidance = opts.hasMarkdownDocumentTools - ? `\n# Collaborative Markdown Docs\n- If the user asks you to create a markdown doc, notes, draft, brief, plan, report, or any document they should open/edit in the app UI, use create_markdown_doc. Do not use filesystem write unless they ask for a file path or repo/workspace file.\n- For larger document workflows, load the markdown-docs skill first with use_skill, then use the markdown document tools.\n- Use set_markdown_doc_cursor before insert_markdown_doc when inserting at a specific location; insert_markdown_doc can then stream content at that Yjs-relative cursor.\n- Use replace_markdown_doc_range when replacing existing prose with new content that should visibly stream into the deleted range.\n- After creating a collaborative doc, mention that it is available from this entity's manifest/timeline.` + ? `\n# Collaborative Markdown Docs\n- If the user asks you to create a markdown doc, notes, draft, brief, plan, report, or any document they should open/edit in the app UI, use create_markdown_doc. Do not use filesystem write unless they ask for a file path or repo/workspace file.\n- For larger document workflows, load the markdown-docs skill first with use_skill, then use the markdown document tools.\n- Use set_markdown_doc_cursor before insert_markdown_doc when inserting at a specific location; insert_markdown_doc can then stream content at that Yjs-relative cursor.\n- Use replace_markdown_doc_range when replacing existing prose with new content that should visibly stream into the deleted range.\n- When spawning a worker to read or edit a collaborative markdown doc, include the doc id in spawn_worker's markdownDocIds and include the specific markdown document tools that worker needs.\n- After creating a collaborative doc, mention that it is available from this entity's manifest/timeline.` : `` const docsGuidance = opts.hasDocsSupport ? `\n- For ANY question about Electric Agents or this framework, ALWAYS use search_electric_agents_docs FIRST. Do not use web_search or fetch_url for Electric Agents topics unless the docs search returns no useful results.\n- The search tool returns chunk content directly โ€” you do not need to read the source files.\n- Use repo read/bash tools only for non-doc files or when you need to inspect exact implementation code in the workspace.` diff --git a/packages/agents/src/agents/worker.ts b/packages/agents/src/agents/worker.ts index da833feb01..12c90e8e63 100644 --- a/packages/agents/src/agents/worker.ts +++ b/packages/agents/src/agents/worker.ts @@ -10,7 +10,11 @@ import { createSendTool, } from '@electric-ax/agents-runtime/tools' import type { Sandbox } from '@electric-ax/agents-runtime/sandbox' -import { WORKER_TOOL_NAMES, createSpawnWorkerTool } from '../tools/spawn-worker' +import { + MARKDOWN_WORKER_TOOL_NAMES, + WORKER_TOOL_NAMES, + createSpawnWorkerTool, +} from '../tools/spawn-worker' import { REASONING_EFFORT_VALUES, resolveBuiltinModelConfig, @@ -48,6 +52,21 @@ function isRecord(value: unknown): value is Record { return value !== null && typeof value === `object` } +function isMarkdownWorkerToolName( + value: WorkerToolName +): value is (typeof MARKDOWN_WORKER_TOOL_NAMES)[number] { + return (MARKDOWN_WORKER_TOOL_NAMES as ReadonlyArray).includes(value) +} + +function electricTool( + ctx: HandlerContext, + name: string +): AgentTool | undefined { + return ctx.electricTools.find((tool) => tool.name === name) as + | AgentTool + | undefined +} + function parseWorkerArgs(value: Readonly>): WorkerArgs { if ( typeof value.systemPrompt !== `string` || @@ -156,6 +175,12 @@ function buildToolsForWorker( case `send`: out.push(createSendTool(ctx.send, { selfEntityUrl: ctx.entityUrl })) break + default: + if (isMarkdownWorkerToolName(name)) { + const tool = electricTool(ctx, name) + if (tool) out.push(tool) + } + break } } return out diff --git a/packages/agents/src/tools/spawn-worker.ts b/packages/agents/src/tools/spawn-worker.ts index 57fddd9d5a..b4117b4853 100644 --- a/packages/agents/src/tools/spawn-worker.ts +++ b/packages/agents/src/tools/spawn-worker.ts @@ -3,7 +3,20 @@ import { nanoid } from 'nanoid' import { serverLog } from '../log' import type { BuiltinAgentModelConfig } from '../model-catalog' import type { AgentTool } from '@mariozechner/pi-agent-core' -import type { HandlerContext } from '@electric-ax/agents-runtime' +import type { + HandlerContext, + ManifestDocumentEntry, +} from '@electric-ax/agents-runtime' + +export const MARKDOWN_WORKER_TOOL_NAMES = [ + `create_markdown_doc`, + `set_markdown_doc_cursor`, + `insert_markdown_doc`, + `replace_markdown_doc_range`, + `read_markdown_doc`, + `write_markdown_doc`, + `edit_markdown_doc`, +] as const export const WORKER_TOOL_NAMES = [ `bash`, @@ -14,10 +27,66 @@ export const WORKER_TOOL_NAMES = [ `fetch_url`, `spawn_worker`, `send`, + ...MARKDOWN_WORKER_TOOL_NAMES, ] as const export type WorkerToolName = (typeof WORKER_TOOL_NAMES)[number] +function isManifestDocumentEntry( + value: unknown +): value is ManifestDocumentEntry { + if (!value || typeof value !== `object`) return false + const entry = value as Partial + return ( + entry.kind === `document` && + typeof entry.id === `string` && + entry.provider === `y-durable-streams` && + typeof entry.docPath === `string` && + typeof entry.streamPath === `string` && + entry.transportMimeType === + `application/vnd.electric-agents.markdown-yjs` && + entry.contentMimeType === `text/markdown` && + entry.yTextName === `markdown` && + typeof entry.title === `string` + ) +} + +function manifestMarkdownDocuments( + ctx: HandlerContext +): Array { + const manifests = ctx.db.collections.manifests?.toArray as + | Array + | undefined + const injectedDocs = Array.isArray(ctx.args?.markdownDocs) + ? ctx.args.markdownDocs.filter(isManifestDocumentEntry) + : [] + return [ + ...(manifests?.filter(isManifestDocumentEntry) ?? []), + ...injectedDocs, + ] +} + +function selectedMarkdownDocuments( + ctx: HandlerContext, + ids: ReadonlyArray | undefined +): { documents: Array; missing: Array } { + if (!ids || ids.length === 0) return { documents: [], missing: [] } + const docsById = new Map( + manifestMarkdownDocuments(ctx).map((document) => [document.id, document]) + ) + const documents: Array = [] + const missing: Array = [] + for (const id of [...new Set(ids)]) { + const document = docsById.get(id) + if (document) { + documents.push(document) + } else { + missing.push(id) + } + } + return { documents, missing } +} + export function createSpawnWorkerTool( ctx: HandlerContext, modelConfig?: BuiltinAgentModelConfig @@ -39,13 +108,20 @@ export function createSpawnWorkerTool( initialMessage: Type.String({ description: `First user message sent to the worker. Be concrete: include file paths, line numbers, and the form of answer you want back. This is what kicks off its run โ€” without it the worker will idle. Describe the concrete task to perform and what form of message you want back.`, }), + markdownDocIds: Type.Optional( + Type.Array(Type.String(), { + description: `Optional collaborative markdown document ids from this entity's manifest to make available to the worker. Include the matching markdown tools in tools when the worker should read or edit them.`, + }) + ), }), execute: async (_toolCallId, params) => { - const { systemPrompt, tools, initialMessage } = params as { - systemPrompt: string - tools: Array - initialMessage: string - } + const { systemPrompt, tools, initialMessage, markdownDocIds } = + params as { + systemPrompt: string + tools: Array + initialMessage: string + markdownDocIds?: Array + } if (!Array.isArray(tools) || tools.length === 0) { return { content: [ @@ -68,6 +144,22 @@ export function createSpawnWorkerTool( details: { spawned: false }, } } + const { documents: markdownDocs, missing: missingMarkdownDocIds } = + selectedMarkdownDocuments(ctx, markdownDocIds) + if (missingMarkdownDocIds.length > 0) { + return { + content: [ + { + type: `text` as const, + text: `Error: markdown document ids not found in this entity's manifest: ${missingMarkdownDocIds.join(`, `)}.`, + }, + ], + details: { + spawned: false, + missingMarkdownDocIds, + }, + } + } const id = nanoid(10) const workerModelArgs = modelConfig @@ -83,7 +175,12 @@ export function createSpawnWorkerTool( const handle = await ctx.spawn( `worker`, id, - { systemPrompt, tools, ...workerModelArgs }, + { + systemPrompt, + tools, + ...(markdownDocs.length > 0 ? { markdownDocs } : {}), + ...workerModelArgs, + }, { initialMessage, wake: { on: `runFinished`, includeResponse: true }, diff --git a/packages/agents/test/spawn-worker-tool.test.ts b/packages/agents/test/spawn-worker-tool.test.ts index 58bfbfebbe..4e543115d8 100644 --- a/packages/agents/test/spawn-worker-tool.test.ts +++ b/packages/agents/test/spawn-worker-tool.test.ts @@ -1,5 +1,23 @@ import { describe, expect, it, vi } from 'vitest' -import { createSpawnWorkerTool } from '../src/tools/spawn-worker' +import { + WORKER_TOOL_NAMES, + createSpawnWorkerTool, +} from '../src/tools/spawn-worker' + +const manifestDocument = { + key: `document:notes`, + kind: `document`, + id: `notes`, + provider: `y-durable-streams`, + docId: `agents/chat/session/documents/notes`, + docPath: `agents/chat/session/documents/notes`, + streamPath: `/v1/yjs/default/docs/agents/chat/session/documents/notes`, + transportMimeType: `application/vnd.electric-agents.markdown-yjs`, + contentMimeType: `text/markdown`, + yTextName: `markdown`, + title: `Notes`, + createdAt: `2026-06-07T00:00:00.000Z`, +} as const describe(`spawn_worker tool`, () => { it(`spawns a worker entity with runFinished + includeResponse and forwards the initial message`, async () => { @@ -67,6 +85,60 @@ describe(`spawn_worker tool`, () => { }) }) + it(`passes selected collaborative markdown document refs to the spawned worker`, async () => { + const spawn = vi.fn(async (type, id) => ({ + entityUrl: `/${type}/${id}`, + writeToken: `tok`, + txid: 1, + })) + const ctx = { + spawn, + db: { collections: { manifests: { toArray: [manifestDocument] } } }, + } as any + const tool = createSpawnWorkerTool(ctx) + + await tool.execute(`call-doc`, { + systemPrompt: `Edit the shared doc.`, + tools: [`read_markdown_doc`, `insert_markdown_doc`], + initialMessage: `Read notes and append a summary.`, + markdownDocIds: [`notes`], + }) + + const [, , args] = spawn.mock.calls[0]! as Array + expect(args).toMatchObject({ + systemPrompt: `Edit the shared doc.`, + tools: [`read_markdown_doc`, `insert_markdown_doc`], + markdownDocs: [manifestDocument], + }) + }) + + it(`rejects unknown collaborative markdown document refs`, async () => { + const spawn = vi.fn() + const ctx = { + spawn, + db: { collections: { manifests: { toArray: [manifestDocument] } } }, + } as any + const tool = createSpawnWorkerTool(ctx) + + const result = await tool.execute(`call-doc-missing`, { + systemPrompt: `Edit the shared doc.`, + tools: [`read_markdown_doc`], + initialMessage: `Read notes.`, + markdownDocIds: [`missing`], + }) + + expect((result.content[0] as { text: string }).text).toMatch( + /not found in this entity's manifest/ + ) + expect(spawn).not.toHaveBeenCalled() + }) + + it(`allows workers to request collaborative markdown document tools`, () => { + expect(WORKER_TOOL_NAMES).toContain(`read_markdown_doc`) + expect(WORKER_TOOL_NAMES).toContain(`insert_markdown_doc`) + expect(WORKER_TOOL_NAMES).toContain(`replace_markdown_doc_range`) + }) + it(`rejects when tools is empty`, async () => { const spawn = vi.fn() const ctx = { spawn } as any