feat(collab-doc): server-authoritative Yjs seeding (#6008)

Server-authoritative Yjs seeding for collaborative file documents: the realtime relay
fetches a Yjs seed built from the file's markdown (via the shared TipTap engine) and
applies it once per room, replacing the client-seeder election/handshake entirely.

- DOM-free markdown<->Yjs conversion core (markdownToYDoc / yDocToMarkdown /
  applyMarkdownToYDoc) reusing the client markdown engine for parity by construction
- Internal x-api-key seed endpoint + realtime fetch; single attempt bounded under the
  client readiness deadline, guard-release for join-driven retry, read-only fallback on
  persistent failure
- Client readiness gate = synced && server seed flag; jsdom wired for the Next standalone
  build (serverExternalPackages + outputFileTracingIncludes)

Foundation only — copilot-into-doc + markdown projection + durable persistence are Stage C.

Author: waleedlatif1

apps/realtime/src/handlers/file-doc-seed.ts

@@ -0,0 +1,45 @@
+import { env, getBaseUrl } from '@/env'
+
+/**
+ * Bound the seed fetch so a slow/unreachable app never wedges a room's first join. Kept comfortably
+ * BELOW the client's readiness deadline (`READINESS_DEADLINE_MS`, 12s, in `file-doc-provider.ts`), so
+ * a single attempt either lands or fails within the window the client is willing to wait — otherwise
+ * the client gives up first and a late success can't reach it. Generous for a real seed: collab is
+ * gated client-side to ≤256 KB documents (`isRoundTripSafe`), which convert markdown→Yjs in well
+ * under a second; the 5 MB server cap is only a backstop and is never reached for a collab file.
+ */
+const SEED_FETCH_TIMEOUT_MS = 8_000
+
+/**
+ * Ask the app to build a server-authoritative seed (markdown → Yjs) for a file's collaborative
+ * document. Only the app can — it owns the editor extensions + blob access — so the relay delegates
+ * over the internal endpoint.
+ *
+ * Returns the Yjs update to apply, or `null` for a genuinely empty/missing file (an empty document
+ * is correct). THROWS on a transport failure (non-2xx / network / timeout) so the caller can tell a
+ * real empty from a failure it should be allowed to retry.
+ */
+export async function fetchFileDocSeed(
+  workspaceId: string,
+  fileId: string
+): Promise<Uint8Array | null> {
+  const response = await fetch(`${getBaseUrl()}/api/internal/file-doc/seed`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET },
+    body: JSON.stringify({ workspaceId, fileId }),
+    signal: AbortSignal.timeout(SEED_FETCH_TIMEOUT_MS),
+  })
+  if (!response.ok) {
+    throw new Error(`Seed fetch failed for file ${fileId}: ${response.status}`)
+  }
+  const body = (await response.json()) as { update?: unknown }
+  const update = body?.update
+  // A well-formed response is `{ update: base64-string | null }`. Anything else is a contract
+  // violation, not a "genuinely empty file" — throw so the caller retries rather than silently
+  // treating a malformed body as empty and stranding the room unseeded.
+  if (update === null) return null
+  if (typeof update !== 'string') {
+    throw new Error(`Seed fetch for file ${fileId} returned a malformed body`)
+  }
+  return new Uint8Array(Buffer.from(update, 'base64'))
+}

apps/realtime/src/handlers/file-doc.test.ts

@@ -36,30 +36,13 @@ import * as awarenessProtocol from 'y-protocols/awareness'
 import * as syncProtocol from 'y-protocols/sync'
 import * as Y from 'yjs'
 import { resolveAvatarUrl } from '@/handlers/avatar'
+import { fetchFileDocSeed } from '@/handlers/file-doc-seed'
 import { resolveRoomJoinAuth } from '@/handlers/room-join-auth'
 import type { AuthenticatedSocket } from '@/middleware/auth'
 import type { IRoomManager } from '@/rooms'
 
 const logger = createLogger('FileDocHandlers')
 
-/**
- * How long an elected seeder has to import the initial content before the server
- * re-elects another client. Seeding is a local, synchronous editor write, so this
- * only trips when a seeder never completes it (a bug, or a client withholding the
- * seed) — it keeps the document from staying empty for the rest of the room.
- */
-const SEED_DEADLINE_MS = 10_000
-
-/**
- * Max times the room re-offers seeding to its owners after every one of them has
- * missed a deadline. Without this, a sole client whose seed *fetch fails* (not just
- * slow) is added to `triedSeeders`, re-election finds no un-tried owner, and the
- * document is left permanently empty until that client reloads. Each exhausted round
- * clears `triedSeeders` and re-offers, giving a transient failure a bounded number of
- * retries (~MAX × deadline) before the room genuinely gives up.
- */
-const MAX_SEED_ROUNDS = 3
-
 /** A socket's presence ownership within a room. */
 interface FileDocOwner {
   /**
@@ -78,23 +61,15 @@ interface FileDocOwner {
 }
 
 interface FileDocRoom {
-  /** The `workspace_files.id` this room edits (for seed-request payloads). */
+  /** The `workspace_files.id` this room edits. */
   fileId: string
   doc: Y.Doc
   awareness: awarenessProtocol.Awareness
   /** socketId → its presence ownership. */
   owners: Map<string, FileDocOwner>
-  /** The socket currently elected to seed initial content, or `null`. */
-  seederSocketId: string | null
-  /** Deadline timer for the current seeder to complete the seed, or `null`. */
-  seedTimer: ReturnType<typeof setTimeout> | null
-  /** Sockets that were elected but failed to seed within the deadline; skipped
-   * on re-election so a single stuck/withholding client can't block the room. */
-  triedSeeders: Set<string>
-  /** Count of full re-offer rounds spent after every owner missed a deadline;
-   * bounds the retry loop (see {@link MAX_SEED_ROUNDS}) so a permanently-broken
-   * seed doesn't reset forever. */
-  seedRounds: number
+  /** True once the server-side seed fetch has started, so concurrent joins don't each fetch.
+   * Reset on a fetch FAILURE so a later join can retry (a genuinely empty file stays empty). */
+  serverSeedStarted: boolean
 }
 
 /** Live documents keyed by Socket.IO room name. Module-global: one Y.Doc per file. */
@@ -183,73 +158,51 @@ function awarenessUpdateClientIds(update: Uint8Array): number[] {
   return ids
 }
 
-function clearSeedTimer(room: FileDocRoom) {
-  if (room.seedTimer !== null) {
-    clearTimeout(room.seedTimer)
-    room.seedTimer = null
-  }
-}
-
 /**
- * Drop a room's document + awareness (and its seed timer) once it has no owners,
- * so an idle file holds no memory. A later joiner re-creates and re-seeds it.
+ * Drop a room's document + awareness once it has no owners, so an idle file holds no memory.
+ * A later joiner re-creates and re-seeds it from the file's current markdown.
  */
 function destroyRoomIfIdle(name: string) {
   const room = fileDocRooms.get(name)
   if (!room || room.owners.size > 0) return
-  clearSeedTimer(room)
   room.awareness.destroy()
   room.doc.destroy()
   fileDocRooms.delete(name)
 }
 
 /**
- * Elect a client to seed an unseeded document and ask it to import the stored
- * markdown, if one is needed and not already assigned. Called after a join (to
- * elect the newcomer), after the elected seeder leaves (to hand the role to a
- * remaining client), and after a seed deadline lapses (to pass over a client that
- * never seeded). Skips clients that already failed to seed, and arms a deadline so
- * a stuck or withholding seeder can't leave the document empty for the whole room.
- * A no-op once the document is seeded, a seeder is already assigned, or no
- * un-tried client remains.
+ * Seed a room's document server-side, once, on the first join: ask the app to build the seed (the
+ * file's current markdown → Yjs, through the exact editor engine) and apply it, which relays the
+ * content to every connected client via `doc.on('update')`. No client is elected to import content.
+ *
+ * `isDocSeeded` is the sufficient guard: content only ever reaches the doc alongside the seed flag
+ * (this seed, or a client's offline fallback), so an unseeded doc is genuinely empty and safe to seed.
+ * A genuinely empty/missing file returns `null` (a read error throws instead), so still set the flag —
+ * an empty doc must reach readiness, not wait forever. After the fetch, re-check the room is still
+ * live and unseeded (an owner may have left, or a client seeded it, while the fetch was in flight).
+ *
+ * Recovery on failure is deliberately simple — no in-room retry loop: a single attempt bounded by a
+ * timeout shorter than the client's readiness deadline, then release the guard. A transient failure
+ * is re-attempted by the next join/reconnect; a persistent one lets the connected client's readiness
+ * deadline lapse into its read-only fallback. (An in-room backoff retry can outlast that client
+ * deadline, so it would keep trying a doc the client has already given up on — worse, not better.)
  */
-function electSeederIfNeeded(io: Server, room: FileDocRoom) {
-  if (room.seederSocketId !== null || isDocSeeded(room.doc)) return
-
-  let elected: string | null = null
-  for (const socketId of room.owners.keys()) {
-    if (!room.triedSeeders.has(socketId)) {
-      elected = socketId
-      break
-    }
+async function ensureServerSeed(
+  name: string,
+  room: FileDocRoom,
+  workspaceId: string
+): Promise<void> {
+  if (room.serverSeedStarted || isDocSeeded(room.doc)) return
+  room.serverSeedStarted = true
+  try {
+    const update = await fetchFileDocSeed(workspaceId, room.fileId)
+    if (fileDocRooms.get(name) !== room || isDocSeeded(room.doc)) return
+    if (update) Y.applyUpdate(room.doc, update)
+    else room.doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true)
+  } catch (error) {
+    logger.warn(`Server seed failed for file ${room.fileId} (workspace ${workspaceId})`, error)
+    room.serverSeedStarted = false
   }
-  if (elected === null) return
-
-  room.seederSocketId = elected
-  io.to(elected).emit(FILE_DOC_EVENTS.SEED_REQUEST, { fileId: room.fileId })
-
-  clearSeedTimer(room)
-  room.seedTimer = setTimeout(() => {
-    room.seedTimer = null
-    // Only act if this election is still the pending one and it never seeded.
-    if (room.seederSocketId !== elected || isDocSeeded(room.doc)) return
-    room.triedSeeders.add(elected)
-    room.seederSocketId = null
-    electSeederIfNeeded(io, room)
-    // If that left no seeder (every current owner has now been tried) while the doc is still
-    // unseeded and owners remain, re-offer the whole room a bounded number of times — otherwise a
-    // sole client whose seed fetch failed leaves the document permanently empty.
-    if (
-      room.seederSocketId === null &&
-      !isDocSeeded(room.doc) &&
-      room.owners.size > 0 &&
-      room.seedRounds < MAX_SEED_ROUNDS
-    ) {
-      room.seedRounds += 1
-      room.triedSeeders.clear()
-      electSeederIfNeeded(io, room)
-    }
-  }, SEED_DEADLINE_MS)
 }
 
 /**
@@ -272,16 +225,11 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom {
     doc,
     awareness,
     owners: new Map(),
-    seederSocketId: null,
-    seedTimer: null,
-    triedSeeders: new Set(),
-    seedRounds: 0,
+    serverSeedStarted: false,
   }
   fileDocRooms.set(name, room)
 
   doc.on('update', (update: Uint8Array, origin: unknown) => {
-    // Once the document is seeded, the seed deadline is moot — cancel it.
-    if (room.seedTimer !== null && isDocSeeded(doc)) clearSeedTimer(room)
     const encoder = encoding.createEncoder()
     encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC)
     syncProtocol.writeUpdate(encoder, update)
@@ -370,8 +318,8 @@ function handleMessage(socket: AuthenticatedSocket, data: unknown) {
 
 /**
  * Remove a socket from its file-doc room: clear its awareness state (so its caret
- * disappears for everyone else), hand off the seeder role if it left before
- * seeding, and drop the room's document when the last collaborator leaves.
+ * disappears for everyone else) and drop the room's document when the last
+ * collaborator leaves.
  * Exported for the disconnect handler; safe to call for a socket in no room.
  */
 export function cleanupFileDocForSocket(socketId: string, io: Server, endOfLife = false): void {
@@ -401,13 +349,6 @@ export function cleanupFileDocForSocket(socketId: string, io: Server, endOfLife
     broadcastFileDocPresence(io, name, room)
   }
 
-  // Hand off the seeder role: if the elected seeder left before it seeded, elect
-  // a remaining client so the document doesn't stay permanently empty.
-  if (room.seederSocketId === socketId) {
-    room.seederSocketId = null
-    electSeederIfNeeded(io, room)
-  }
-
   destroyRoomIfIdle(name)
 }
 
@@ -519,10 +460,6 @@ export function setupWorkspaceFileDocHandlers(
         awarenessProtocol.removeAwarenessStates(entry.awareness, [owner.clientId], null)
         socketToRoomName.delete(otherSid)
         io.in(otherSid).socketsLeave(name)
-        // If the evicted socket held the seeder role, release it so the election at the end of
-        // this join re-elects (electSeederIfNeeded no-ops while seederSocketId is set) — otherwise
-        // an unseeded document would stay empty until the seed deadline expires.
-        if (entry.seederSocketId === otherSid) entry.seederSocketId = null
       }
 
       // Only now that the rebind is guaranteed to succeed, leave a previously-joined document if
@@ -568,7 +505,9 @@ export function setupWorkspaceFileDocHandlers(
         socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(awarenessEncoder))
       }
 
-      electSeederIfNeeded(io, entry)
+      // Seed the document server-side (once). Fire-and-forget: the join completes immediately and
+      // the seed relays to this socket via `doc.on('update')` the moment it lands.
+      if (authorized.workspaceId) void ensureServerSeed(name, entry, authorized.workspaceId)
 
       logger.info(`User ${userId} joined file-doc room ${fileId}`)
     } catch (error) {

apps/realtime/src/handlers/file-doc.ts

@@ -0,0 +1,70 @@
+/**
+ * @vitest-environment node
+ */
+import { createMockRequest } from '@sim/testing'
+import { NextResponse } from 'next/server'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockCheckInternalApiKey, mockBuildFileDocSeed } = vi.hoisted(() => ({
+  mockCheckInternalApiKey: vi.fn(),
+  mockBuildFileDocSeed: vi.fn(),
+}))
+
+vi.mock('@/lib/copilot/request/http', () => ({
+  checkInternalApiKey: mockCheckInternalApiKey,
+  createUnauthorizedResponse: () => NextResponse.json({ error: 'Unauthorized' }, { status: 401 }),
+}))
+
+vi.mock('@/lib/collab-doc/seed', () => ({
+  buildFileDocSeed: mockBuildFileDocSeed,
+}))
+
+import { POST } from './route'
+
+function seedRequest(body: unknown) {
+  return createMockRequest('POST', body, { 'x-api-key': 'internal' })
+}
+
+describe('POST /api/internal/file-doc/seed', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockCheckInternalApiKey.mockReturnValue({ success: true })
+  })
+
+  // Regression guard for the auth-helper choice: the realtime relay authenticates with
+  // `x-api-key: INTERNAL_API_SECRET`, so this route MUST gate on `checkInternalApiKey`. Wiring the
+  // Bearer-JWT-only `checkInternalAuth` (which forbids `x-api-key`) 401s every real seed fetch.
+  it('401s when the internal api key is rejected, without building a seed', async () => {
+    mockCheckInternalApiKey.mockReturnValue({ success: false })
+    const res = await POST(seedRequest({ workspaceId: 'ws-1', fileId: 'file-1' }))
+    expect(res.status).toBe(401)
+    expect(mockBuildFileDocSeed).not.toHaveBeenCalled()
+  })
+
+  it('returns the seed as base64 for an authorized request', async () => {
+    mockBuildFileDocSeed.mockResolvedValue({ update: new Uint8Array([1, 2, 3, 4]) })
+    const res = await POST(seedRequest({ workspaceId: 'ws-1', fileId: 'file-1' }))
+    expect(res.status).toBe(200)
+    expect((await res.json()).update).toBe(Buffer.from([1, 2, 3, 4]).toString('base64'))
+    expect(mockBuildFileDocSeed).toHaveBeenCalledWith('ws-1', 'file-1')
+  })
+
+  it('returns update:null for a genuinely absent file', async () => {
+    mockBuildFileDocSeed.mockResolvedValue(null)
+    const res = await POST(seedRequest({ workspaceId: 'ws-1', fileId: 'missing' }))
+    expect(res.status).toBe(200)
+    expect((await res.json()).update).toBeNull()
+  })
+
+  it('400s on a body missing required fields (contract validation, after auth)', async () => {
+    const res = await POST(seedRequest({ workspaceId: 'ws-1' }))
+    expect(res.status).toBe(400)
+    expect(mockBuildFileDocSeed).not.toHaveBeenCalled()
+  })
+
+  it('500s when the seed build throws (a read error the relay should retry)', async () => {
+    mockBuildFileDocSeed.mockRejectedValue(new Error('db down'))
+    const res = await POST(seedRequest({ workspaceId: 'ws-1', fileId: 'file-1' }))
+    expect(res.status).toBe(500)
+  })
+})

apps/sim/app/api/internal/file-doc/seed/route.test.ts

@@ -0,0 +1,39 @@
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import type { NextRequest } from 'next/server'
+import { NextResponse } from 'next/server'
+import { buildFileDocSeedContract } from '@/lib/api/contracts/file-doc'
+import { parseRequest } from '@/lib/api/server'
+import { buildFileDocSeed } from '@/lib/collab-doc/seed'
+import { checkInternalApiKey, createUnauthorizedResponse } from '@/lib/copilot/request/http'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+
+const logger = createLogger('FileDocSeedAPI')
+
+/**
+ * POST /api/internal/file-doc/seed — build a server-authoritative collaborative-document seed
+ * (markdown → Yjs) for the realtime relay to apply on room creation. Internal only: gated on the
+ * shared `x-api-key: INTERNAL_API_SECRET` secret, matching the header the realtime relay sends
+ * (`apps/realtime/src/handlers/file-doc-seed.ts`) and the realtime server's own inbound validator.
+ */
+export const POST = withRouteHandler(async (request: NextRequest) => {
+  const auth = checkInternalApiKey(request)
+  if (!auth.success) return createUnauthorizedResponse()
+
+  const parsed = await parseRequest(buildFileDocSeedContract, request, {})
+  if (!parsed.success) return parsed.response
+  const { workspaceId, fileId } = parsed.data.body
+
+  try {
+    const seed = await buildFileDocSeed(workspaceId, fileId)
+    return NextResponse.json({
+      update: seed ? Buffer.from(seed.update).toString('base64') : null,
+    })
+  } catch (error) {
+    logger.error('Failed to build file-doc seed', { workspaceId, fileId, error })
+    return NextResponse.json(
+      { error: getErrorMessage(error, 'Failed to build seed') },
+      { status: 500 }
+    )
+  }
+})