feat: paginate cross-host history sessions

docs/OBSERVABILITY.md

@@ -225,6 +225,27 @@ secondary tab row directly beneath the main Observability tab; the Left/Right, H
 also move between these two views when the tab row has focus. Their canonical hashes are
 `#observability/live` and `#observability/history`.
 
+### History loading, pagination, and coverage
+
+History is loaded newest-first in bounded pages. The browser requests an initial page of up to
+100 sessions, automatically asks for the next opaque continuation token as the list approaches
+its sentinel, and keeps a visible **Load older sessions** fallback when automatic loading is not
+available. The project browser uses the server's full project metadata, so a page containing only
+the newest sessions does not make a project's total look smaller than it is.
+
+The server keeps the history scan separate from the live tailer. It discovers eligible Claude and
+Codex transcript files by file modification time, materializes a stable short-lived snapshot, and
+then pages that snapshot. Each paginated response reports `pagination.total`, `hasMore`, and an
+opaque `nextPageToken`; `coverage` reports the per-host candidate/returned file counts, the file
+limit, the scan time, and whether discovery was complete. An incomplete scan is disclosed in the
+History view rather than presented as an authoritative empty or complete result.
+
+This was added compatibly. `GET /api/live/history` without `limit`, `pageToken`, or `projectKey`
+continues to return the pre-pagination snapshot shape. Clients that understand pagination opt in
+with those query parameters and receive the same snapshot fields plus additive `pagination` and
+`coverage` fields. Continuation tokens are short-lived and scoped to their project/window snapshot;
+an expired or malformed token returns `400` so a client can restart from the first page.
+
 ### Live and Review playback
 
 An active session opens in **Live** mode and follows new topology and transcript
@@ -394,6 +415,7 @@ unbounded content snapshot.
 | Symptom | Explanation |
 |---------|-------------|
 | No sessions | No supported metadata was found within the bounded newest-first discovery set |
+| History says the scan is incomplete | The per-host discovery bound was reached; older files may not be represented, so widen the configured source or rerun after reducing the corpus |
 | Live shows 0 projects | No retained root currently satisfies the Live predicate; switch to History for past sessions |
 | History shows 0 projects | No retained non-live root exists; current work, if any, remains in Live |
 | Green or moving content appears in History | This violates the Observability contract; refresh, then report it as a presentation defect if it remains |

src/lib/dashboard-server.mjs

@@ -961,9 +961,20 @@ export function startDashboard({
           return;
         }
         const sinceMs = windowToSinceMs(query.get('window'));
-        sendJson(res, 200, publicLivePayload(await service.historySnapshot({ sinceMs })));
-      } catch {
-        sendJson(res, 503, { error: 'live telemetry unavailable' });
+        const pageRequested = query.has('limit') || query.has('pageToken') || query.has('projectKey');
+        const payload = pageRequested && typeof service.historyPage === 'function'
+          ? await service.historyPage({
+            sinceMs,
+            projectKey: query.get('projectKey') || null,
+            pageToken: query.get('pageToken') || null,
+            limit: clampInt(query.get('limit'), 100, 1, 250),
+          })
+          : await service.historySnapshot({ sinceMs });
+        sendJson(res, 200, publicLivePayload(payload));
+      } catch (error) {
+        sendJson(res, error?.code === 'INVALID_HISTORY_PAGE_TOKEN' ? 400 : 503,
+          { error: error?.code === 'INVALID_HISTORY_PAGE_TOKEN'
+            ? 'invalid history page token' : 'live telemetry unavailable' });
       } finally {
         scheduleLiveIdle();
       }

src/lib/dashboard/live/styles.mjs

@@ -28,4 +28,5 @@ export const LIVE_CSS = `
 .live-scope-bar{display:flex;justify-content:center;margin:0 0 10px}.live-scope-tabs{max-width:min(360px,100%)}.live-scope-tabs .seg-btn{min-width:92px;justify-content:center}.live-workspace{height:calc(100vh - 205px);min-height:500px}@media(max-width:900px){.live-scope-bar{margin-bottom:8px}.live-workspace{height:auto;min-height:0}}
 .live-panel{margin:0 -12px;padding-bottom:15px}.live-toolbar{min-height:76px;align-items:flex-start;margin-bottom:18px}.live-view-summary{display:block;margin-top:4px;color:var(--ink-dim);font-size:8px}.live-selection dd{margin:0}.live-detail-branch{display:inline-flex;align-items:center;gap:5px;min-width:0}.live-detail-branch .live-branch-icon{width:11px;height:11px}.live-playback{position:static;z-index:1;left:auto;right:auto;bottom:auto;flex:0 0 47px;min-height:47px;height:auto;display:grid;grid-template-columns:auto minmax(120px,1fr) auto auto auto auto;align-items:center;gap:8px;border:0;border-top:1px solid color-mix(in srgb,var(--line) 85%,transparent);border-radius:0;padding:5px 9px;background:color-mix(in srgb,var(--panel) 94%,transparent);box-shadow:none;backdrop-filter:none}.live-playback[hidden]{display:none}.live-playback[data-mode=live]{display:flex;justify-content:flex-end;min-height:39px;flex-basis:39px}.live-playback[data-mode=live] #live-playback-toggle,.live-playback[data-mode=live] #live-playback-range,.live-playback[data-mode=live] #live-playback-time,.live-playback[data-mode=live] #live-playback-speed{display:none}.live-playback[data-mode=review]{border:0;border-top:1px solid color-mix(in srgb,var(--warn) 42%,var(--line))}@media(max-width:900px){.live-panel{margin:0;padding-bottom:18px}.live-playback{position:static;left:auto;right:auto;bottom:auto}}@media(max-width:760px){.live-playback{grid-template-columns:auto minmax(90px,1fr) auto}.live-playback output{display:none}.live-playback select,.live-playback #live-resume-live{grid-row:auto}}
 @media(prefers-reduced-motion:reduce){.live-workspace,.live-transcript-toggle svg{transition:none}.live-edge,.live-node .node-status,.live-node .node-aura,.live-tool-card,.live-tool-halo,.live-session-identity i,.live-project::before{animation:none!important}.live-flow-dot{display:none}}
+.live-history-more{display:flex;align-items:center;justify-content:space-between;gap:8px;margin:4px 0;padding:9px 8px;border:1px solid color-mix(in srgb,var(--line) 78%,transparent);border-radius:9px;color:var(--ink-dim);font-size:8px;text-align:center}.live-history-more span{min-width:0}.live-history-more button{padding:5px 8px;font-size:8px;white-space:nowrap}
 `;

src/lib/live/live-sessions-service.mjs

@@ -1,4 +1,5 @@
 import path from 'node:path';
+import { randomUUID } from 'node:crypto';
 import { claudeDir, codexDir, observabilityWorkspacePath } from '../paths.mjs';
 import { readCodexState as defaultReadCodexState } from '../codex-state.mjs';
 import { adaptClaudeRecord } from './claude-adapter.mjs';
@@ -16,16 +17,20 @@ import { canonicalSessionKey, resolveProjectIdentity } from './project-label.mjs
 import { workspaceFromSource } from './git-workspace.mjs';
 import { WorkspaceSnapshotStore } from './workspace-store.mjs';
 import {
-  bootstrapRecords, codexTranscriptId, discoverJsonl,
+  bootstrapRecords, codexTranscriptId, discoverJsonl, discoverJsonlDetailed,
 } from './native-transcript-discovery.mjs';
 
 // historySnapshot() is a one-shot on-demand scan, not a continuously-tailed
 // live feed, so it can afford limits well above the live path's maxFiles(256)
-// /maxSessions(100) defaults — those stay small purely to keep the always-on
-// tailer set cheap. discoverJsonl's own hard 4096-file safety cap is the real
-// backstop for the "all time" window on a machine with a large corpus.
-const HISTORY_MAX_FILES = 2048;
-const HISTORY_MAX_SESSIONS = 1000;
+// /maxSessions(100) defaults. History pages are sliced only after this scan;
+// the live projection's smaller bound must never decide which host's history
+// survives. The detailed discovery result makes any safety boundary visible.
+const HISTORY_MAX_FILES = 8192;
+const HISTORY_MAX_SESSIONS = HISTORY_MAX_FILES * 2;
+const HISTORY_DEFAULT_PAGE_SIZE = 100;
+const HISTORY_MAX_PAGE_SIZE = 250;
+const HISTORY_PAGE_TTL_MS = 60_000;
+const HISTORY_PAGE_CACHE_SIZE = 8;
 
 /**
  * Coordinates bounded transcript tailers into one privacy-safe live projection.
@@ -46,6 +51,7 @@ export class LiveSessionsService {
   #lastRuntimeScan = 0;
   #runtimeSurvey = null;
   #workspaceStore = null;
+  #historyPages = new Map();
 
   constructor(options = {}) {
     const roots = options.roots ?? {};
@@ -107,6 +113,7 @@ export class LiveSessionsService {
     this.#tailers.clear();
     this.#contexts.clear();
     this.#runtimeBindings.clear();
+    this.#historyPages.clear();
     this.#started = false;
   }
 
@@ -279,14 +286,73 @@ export class LiveSessionsService {
    * renders identically whichever scope produced it.
    * @param {{ sinceMs?: number|null }} [options] sinceMs is an epoch-ms
    *   cutoff on file mtime; omit/null scans "all time".
-   * @returns {ReturnType<typeof serializeLiveProjection>}
+   * @returns {ReturnType<typeof serializeLiveProjection> & {coverage: object}}
    */
   historySnapshot({ sinceMs = null } = {}) {
-    const claude = discoverJsonl(this.#options.roots.claude, {
+    return this.#scanHistory({ sinceMs });
+  }
+
+  /**
+   * Return one stable, project-scoped page from a retained history snapshot.
+   * The snapshot is cached briefly so scrolling does not rescan or reorder the
+   * same history set between requests. pageToken is intentionally opaque to
+   * callers and contains no filesystem identity.
+   * @param {{ sinceMs?: number|null, projectKey?: string|null,
+   *   limit?: number, pageToken?: string|null }} [options]
+   */
+  historyPage({ sinceMs = null, projectKey = null, limit = HISTORY_DEFAULT_PAGE_SIZE,
+    pageToken = null } = {}) {
+    const pageSize = Math.min(HISTORY_MAX_PAGE_SIZE, Math.max(1,
+      Number.parseInt(String(limit), 10) || HISTORY_DEFAULT_PAGE_SIZE));
+    let entry;
+    let offset = 0;
+    if (pageToken) {
+      const token = decodeHistoryPageToken(pageToken);
+      entry = this.#historyPages.get(token.snapshotId);
+      if (!entry || Date.now() - entry.createdAt > HISTORY_PAGE_TTL_MS
+        || entry.projectKey !== (projectKey ?? null)) {
+        throw invalidHistoryPageToken();
+      }
+      offset = Number.isInteger(token.offset) && token.offset >= 0
+        && token.offset <= entry.sessions.length ? token.offset : -1;
+      if (offset < 0) throw invalidHistoryPageToken();
+    } else {
+      const snapshot = this.#scanHistory({ sinceMs });
+      const sessions = snapshot.sessions
+        .filter((session) => !projectKey || session.projectKey === projectKey)
+        .sort(compareHistorySessions);
+      entry = { snapshot, sessions, projectKey: projectKey ?? null, sinceMs: sinceMs ?? null,
+        snapshotId: randomUUID(), createdAt: Date.now() };
+      this.#historyPages.set(entry.snapshotId, entry);
+      while (this.#historyPages.size > HISTORY_PAGE_CACHE_SIZE) {
+        this.#historyPages.delete(this.#historyPages.keys().next().value);
+      }
+    }
+
+    const sessions = entry.sessions.slice(offset, offset + pageSize);
+    const nextOffset = offset + sessions.length;
+    const hasMore = nextOffset < entry.sessions.length;
+    return {
+      ...entry.snapshot,
+      sessions,
+      pagination: {
+        pageSize, offset, returned: sessions.length,
+        total: entry.sessions.length,
+        totalExact: entry.snapshot.coverage.complete,
+        hasMore,
+        nextPageToken: hasMore
+          ? encodeHistoryPageToken({ snapshotId: entry.snapshotId, offset: nextOffset }) : null,
+      },
+    };
+  }
+
+  #scanHistory({ sinceMs = null } = {}) {
+    const claude = discoverJsonlDetailed(this.#options.roots.claude, {
       maxDepth: 3, maxFiles: HISTORY_MAX_FILES, sinceMs, accept: () => true,
     });
-    const codex = discoverJsonl(this.#options.roots.codex, {
-      maxDepth: 4, maxFiles: HISTORY_MAX_FILES, sinceMs, accept: (name) => name.startsWith('rollout-'),
+    const codex = discoverJsonlDetailed(this.#options.roots.codex, {
+      maxDepth: 4, maxFiles: HISTORY_MAX_FILES, sinceMs,
+      accept: (name) => name.startsWith('rollout-'),
     });
     let projection = emptyLiveProjection();
     const ingest = (file, adapter, context) => {
@@ -298,10 +364,10 @@ export class LiveSessionsService {
         }
       }
     };
-    for (const file of claude) {
+    for (const file of claude.files) {
       ingest(file, 'claude', { adapter: 'claude', sessionId: path.basename(file, '.jsonl'), project: 'unknown' });
     }
-    for (const file of codex) {
+    for (const file of codex.files) {
       ingest(file, 'codex', { adapter: 'codex', sessionId: codexTranscriptId(file), meta: {} });
     }
     // A one-shot scan never observes the process ending, so the reducer's
@@ -316,7 +382,19 @@ export class LiveSessionsService {
     projection = sweepLiveProjection(projection, {
       now: this.#options.now(), quiescentMs: 0, expiryMs: 0, pendingExpiryMs: 0,
     });
-    return serializeLiveProjection(projection);
+    const snapshot = serializeLiveProjection(projection);
+    return {
+      ...snapshot,
+      coverage: {
+        complete: !claude.truncated && !codex.truncated,
+        timeBasis: 'file-mtime',
+        scannedAt: this.#options.now(),
+        sources: {
+          claude: historyDiscoveryCoverage(claude),
+          codex: historyDiscoveryCoverage(codex),
+        },
+      },
+    };
   }
 
   /** Configuration reads are per-project, so memoize by session cwd. */
@@ -552,3 +630,39 @@ export class LiveSessionsService {
     });
   }
 }
+
+function compareHistorySessions(left, right) {
+  return Date.parse(right.updatedAt ?? 0) - Date.parse(left.updatedAt ?? 0)
+    || String(left.host ?? '').localeCompare(String(right.host ?? ''))
+    || String(left.id ?? '').localeCompare(String(right.id ?? ''));
+}
+
+function historyDiscoveryCoverage(discovery) {
+  return {
+    candidateFiles: discovery.candidateCount,
+    returnedFiles: discovery.returnedCount,
+    fileLimit: HISTORY_MAX_FILES,
+    truncated: discovery.truncated,
+  };
+}
+
+function encodeHistoryPageToken(value) {
+  return Buffer.from(JSON.stringify(value), 'utf8').toString('base64url');
+}
+
+function decodeHistoryPageToken(value) {
+  try {
+    const parsed = JSON.parse(Buffer.from(String(value), 'base64url').toString('utf8'));
+    if (!parsed || typeof parsed.snapshotId !== 'string'
+      || !Number.isInteger(parsed.offset) || parsed.offset < 0) throw new Error('invalid');
+    return parsed;
+  } catch {
+    throw invalidHistoryPageToken();
+  }
+}
+
+function invalidHistoryPageToken() {
+  return Object.assign(new Error('invalid history page token'), {
+    code: 'INVALID_HISTORY_PAGE_TOKEN',
+  });
+}

src/lib/live/native-transcript-discovery.mjs

@@ -5,6 +5,8 @@ const safeEntries = (dir) => {
   try { return fs.readdirSync(dir, { withFileTypes: true }); } catch { return []; }
 };
 
+const DISCOVERY_HARD_LIMIT = 16_384;
+
 /**
  * @param {string} root
  * @param {{ maxDepth: number, maxFiles: number, accept: (name: string) => boolean,
@@ -14,11 +16,27 @@ const safeEntries = (dir) => {
  *   existing caller (the live tailer, project-discovery) relies on.
  */
 export function discoverJsonl(root, { maxDepth, maxFiles, accept, sinceMs = null }) {
+  return discoverJsonlDetailed(root, { maxDepth, maxFiles, accept, sinceMs }).files;
+}
+
+/**
+ * Discover files while retaining enough bounded-source evidence for callers to
+ * say whether the returned set is complete. The legacy discoverJsonl() wrapper
+ * deliberately keeps returning only paths for existing tailer callers.
+ */
+export function discoverJsonlDetailed(root, { maxDepth, maxFiles, accept, sinceMs = null }) {
   const found = [];
+  let truncated = false;
   const visit = (dir, depth) => {
-    if (depth > maxDepth || found.length >= 4096) return;
+    if (depth > maxDepth || found.length >= DISCOVERY_HARD_LIMIT) {
+      if (found.length >= DISCOVERY_HARD_LIMIT) truncated = true;
+      return;
+    }
     for (const entry of safeEntries(dir)) {
-      if (found.length >= 4096) break;
+      if (found.length >= DISCOVERY_HARD_LIMIT) {
+        truncated = true;
+        break;
+      }
       const file = path.join(dir, entry.name);
       if (entry.isDirectory()) visit(file, depth + 1);
       else if (entry.isFile() && entry.name.endsWith('.jsonl') && accept(entry.name)) {
@@ -30,8 +48,14 @@ export function discoverJsonl(root, { maxDepth, maxFiles, accept, sinceMs = null
     }
   };
   visit(root, 0);
-  return found.sort((a, b) => b.mtimeMs - a.mtimeMs)
-    .slice(0, maxFiles).map((entry) => entry.file);
+  const ordered = found.sort((a, b) => b.mtimeMs - a.mtimeMs);
+  const files = ordered.slice(0, maxFiles).map((entry) => entry.file);
+  return {
+    files,
+    candidateCount: ordered.length,
+    returnedCount: files.length,
+    truncated: truncated || ordered.length > files.length,
+  };
 }
 
 /**

tests/dashboard.test.cjs

@@ -1380,6 +1380,41 @@ async function main() {
       assert(historyCalls[2].sinceMs === null, '?window=all must scan without a cutoff');
     });
 
+    await test('GET /api/live/history adds pagination metadata without changing the legacy shape', async () => {
+      const pageCalls = [];
+      live.historyPage = (opts) => {
+        pageCalls.push(opts);
+        return {
+          schemaVersion: 2, cursor: null,
+          sessions: [{ id: 'h1', project: 'agentic-kit', projectKey: 'project:test', nodes: [], edges: [] }],
+          projects: [{ id: 'project:test', label: 'agentic-kit', sessionCount: 2 }],
+          pagination: {
+            pageSize: opts.limit, offset: 0, returned: 1, total: 2,
+            totalExact: true, hasMore: true, nextPageToken: 'opaque-page-token',
+          },
+          coverage: { complete: true, timeBasis: 'file-mtime' },
+        };
+      };
+      const r = await get(liveSrv.url + 'api/live/history?window=1y&limit=1&projectKey=project%3Atest', liveSrv.token);
+      assert(r.status === 200, 'expected 200, got ' + r.status);
+      const body = JSON.parse(r.body);
+      assert(body.pagination.nextPageToken === 'opaque-page-token', 'pagination token must pass through');
+      assert(body.coverage.complete === true, 'coverage must pass through');
+      assert(pageCalls[0].limit === 1, 'limit must pass through');
+      assert(pageCalls[0].projectKey === 'project:test', 'project key must pass through');
+      assert(!r.body.includes('/Users/private'), 'pagination must preserve the same privacy scrubber');
+
+      const historyPage = live.historyPage;
+      live.historyPage = undefined;
+      try {
+        const legacy = await get(liveSrv.url + 'api/live/history?window=1y&limit=1', liveSrv.token);
+        assert(legacy.status === 200, 'new callers must fall back to a pre-pagination service');
+        assert(!JSON.parse(legacy.body).pagination, 'legacy fallback must preserve the old response shape');
+      } finally {
+        live.historyPage = historyPage;
+      }
+    });
+
     await test('GET /api/live/events with no token → 401 before any subscribe', async () => {
       const before = liveCalls.subscribe;
       const r = await getRaw(liveSrv.port, '/api/live/events');
@@ -1806,7 +1841,7 @@ async function main() {
   // is the suite where it matters most — the traversal-guard and credential-
   // leak tests live here and were the reviewer's cited example of a block
   // that could silently vanish with the old harness never noticing.
-  const EXPECTED = 72;
+  const EXPECTED = 73;
   if (passed + failed !== EXPECTED) {
     console.error(`\nPLAN MISMATCH: expected ${EXPECTED} tests, ran ${passed + failed}`);
     process.exit(1);