Feat/improve e2e

controlplane/src/dashboards/handler.ts

@@ -5,13 +5,14 @@
  * Dashboard API Handlers
  */
 
-// REVISION: dashboards-v6-link-sync-with-rbac
+// REVISION: dashboards-v8-teardown-without-snapshot
 
 import type { Env, Dashboard, DashboardItem, DashboardEdge } from '../types';
 import { syncItemToLinked, syncEdgeToLinked } from '../links/handler';
 import { populateFromTemplate } from '../templates/handler';
 import type { EnvWithDriveCache } from '../storage/drive-cache';
 import { FlyMachinesClient } from '../sandbox/fly-machines';
+import { SandboxClient } from '../sandbox/client';
 import { preProvisionDashboardSandbox } from '../sessions/handler';
 
 function generateId(): string {
@@ -282,21 +283,38 @@ export async function deleteDashbоard(
     return Response.json({ error: 'E79304: Not found or not owner' }, { status: 404 });
   }
 
-  // Stop any active sessions first so their PTYs are killed in the sandbox.
-  // CASCADE-deleting the session rows would otherwise orphan live processes
-  // (shells, and agent children like node/chromium) in the VM. Mirrors deleteItem.
+  // Mark active sessions stopped. Deliberately NOT via stopSession().
+  //
+  // stopSession is the "user closed a terminal, keep the dashboard usable" path:
+  // when it stops the last session it captures a workspace snapshot into R2 for
+  // recovery, and deletes that session's PTY. Both are wrong when the dashboard
+  // itself is going away — the snapshot is written moments before its dashboard
+  // ceases to exist and nothing ever collects it, so every deletion leaked an
+  // R2 object.
+  //
+  // It was also called in a sequential loop, one round trip per terminal, each
+  // able to block for the full 15s sandbox timeout. That made deletion slowest
+  // in exactly the case where it matters most — a wedged sandbox, which is the
+  // state a user is trying to clean up. The single bounded deleteSession below
+  // replaces all of it: destroying the sandbox session tears down every PTY it
+  // owns, so the per-PTY deletes were redundant anyway.
   try {
-    const activeSessions = await env.DB.prepare(`
-      SELECT id FROM sessions WHERE dashboard_id = ? AND status IN ('creating', 'active')
-    `).bind(dashboardId).all<{ id: string }>();
-    if (activeSessions.results.length > 0) {
-      const { stоpSessiоn } = await import('../sessions/handler');
-      for (const session of activeSessions.results) {
-        await stоpSessiоn(env as EnvWithDriveCache, session.id, userId);
-      }
-    }
+    const now = new Date().toISOString();
+    await env.DB.prepare(`
+      UPDATE sessions SET status = 'stopped', stopped_at = ?
+      WHERE dashboard_id = ? AND status IN ('creating', 'active')
+    `).bind(now, dashboardId).run();
   } catch {
-    // Best-effort — don't block dashboard deletion if session cleanup fails.
+    // Best-effort — don't block dashboard deletion if the status update fails.
+  }
+
+  // Drop the dashboard's R2 objects (workspace snapshot, mirror manifests).
+  // Nothing cascades these; without this they outlive the dashboard forever.
+  try {
+    const { purgeDashbоardStorage } = await import('../sessions/handler');
+    await purgeDashbоardStorage(env as EnvWithDriveCache, dashboardId);
+  } catch (e) {
+    console.error(`[deleteDashboard] Storage purge failed for ${dashboardId}: ${e}`);
   }
 
   // Delete dependent records that don't have ON DELETE CASCADE
@@ -332,12 +350,55 @@ export async function deleteDashbоard(
     .bind(dashboardId)
     .run();
 
+  // Read the sandbox row once: both the session release below and the Fly
+  // teardown after it need it.
+  const sandboxRow = await env.DB.prepare(`
+    SELECT sandbox_session_id, sandbox_machine_id, fly_volume_id FROM dashboard_sandboxes WHERE dashboard_id = ?
+  `).bind(dashboardId).first<{
+    sandbox_session_id: string;
+    sandbox_machine_id: string;
+    fly_volume_id: string;
+  }>();
+
+  // Release the sandbox session — UNCONDITIONALLY, not just on Fly.
+  //
+  // A sandbox session owns real resources inside the VM: its PTYs and a full
+  // Chromium/Xvfb/x11vnc stack, several hundred MB each. Session.Close() is
+  // reachable only via DELETE /sessions/:id, and this was previously called
+  // only from the Fly-gated block below plus a create-race dedup. On any
+  // deployment sharing one sandbox (local dev, desktop, self-hosted), deleting
+  // a dashboard therefore freed its D1 rows and nothing else, so every
+  // dashboard ever created leaked a browser stack until the VM exhausted its
+  // memory — at which point the Go server was starved badly enough that even
+  // GET /health timed out, and every subsequent session create failed with
+  // "fetch error: The operation was aborted".
+  //
+  // Destroying the Fly machine (below) implicitly reclaims everything, so this
+  // is redundant there and merely tidy; it is load-bearing everywhere else.
+  //
+  // Best-effort: an unreachable or already-wedged sandbox must not block
+  // deleting the dashboard, or the user cannot clean up the very state that is
+  // wedging it.
+  if (sandboxRow?.sandbox_session_id && env.SANDBOX_URL) {
+    try {
+      const sandbox = new SandboxClient(env.SANDBOX_URL, env.SANDBOX_INTERNAL_TOKEN);
+      await sandbox.deleteSession(
+        sandboxRow.sandbox_session_id,
+        sandboxRow.sandbox_machine_id || undefined
+      );
+      console.log(
+        `[deleteDashboard] Released sandbox session ${sandboxRow.sandbox_session_id} for dashboard ${dashboardId}`
+      );
+    } catch (e) {
+      console.error(
+        `[deleteDashboard] Failed to release sandbox session ${sandboxRow.sandbox_session_id}: ${e}`
+      );
+    }
+  }
+
   // Destroy Fly machine and volume for this dashboard (best-effort)
   if (env.FLY_API_TOKEN && env.FLY_APP_NAME) {
     try {
-      const sandboxRow = await env.DB.prepare(`
-        SELECT sandbox_machine_id, fly_volume_id FROM dashboard_sandboxes WHERE dashboard_id = ?
-      `).bind(dashboardId).first<{ sandbox_machine_id: string; fly_volume_id: string }>();
       if (sandboxRow?.sandbox_machine_id) {
         const fly = new FlyMachinesClient(env.FLY_APP_NAME, env.FLY_API_TOKEN);
         try {

controlplane/src/index.ts

@@ -1180,6 +1180,38 @@ async function handleRequest(request: Request, env: EnvWithBindings, ctx: Pick<E
     return analytics.getAdminMetrics(env, auth.user!.email, request);
   }
 
+  // POST /admin/storage/orphan-sweep - reclaim R2 objects whose dashboard is gone.
+  //
+  // Backfill for dashboards deleted before purgeDashboardStorage swept by
+  // prefix; those removed only manifests and orphaned every cached file.
+  // DRY RUN unless the body carries { "apply": true } — the blast radius is a
+  // whole bucket, so deletion must be asked for explicitly. Optional
+  // { "limit": n } processes a batch at a time.
+  if (
+    segments[0] === 'admin' &&
+    segments[1] === 'storage' &&
+    segments[2] === 'orphan-sweep' &&
+    segments.length === 3 &&
+    method === 'POST'
+  ) {
+    const authError = requireAuth(auth);
+    if (authError) return authError;
+    if (!isAdminEmail(env, auth.user!.email)) {
+      return Response.json({ error: 'E79810: Admin access required' }, { status: 403 });
+    }
+    // A PAT must not be able to mass-delete stored content.
+    const patError = rejectPatAuth(auth);
+    if (patError) return patError;
+
+    const body = await request.json().catch(() => ({})) as { apply?: boolean; limit?: number };
+    const { sweepOrphanedStorage } = await import('./storage/orphan-sweep');
+    const result = await sweepOrphanedStorage(ensureDriveCache(env), {
+      apply: body.apply === true,
+      limit: typeof body.limit === 'number' ? body.limit : undefined,
+    });
+    return Response.json(result);
+  }
+
   // POST /auth/logout - clear session cookie
   if (segments[0] === 'auth' && segments[1] === 'logout' && segments.length === 2 && method === 'POST') {
     return authLogout.logout(request, env);

controlplane/src/sessions/handler.ts

@@ -926,6 +926,108 @@ function workspaceSnapshotKey(dashboardId: string): string {
   return `workspace/${dashboardId}/snapshot.json`;
 }
 
+/** Mirror providers with a known key namespace; see discoverMirrorProviders. */
+const MIRROR_PROVIDERS = ['github', 'box', 'onedrive'] as const;
+
+/** Bounds the delete loop at 1M objects rather than spinning forever. */
+const MAX_PURGE_ROUNDS = 1000;
+
+/**
+ * Delete every object under an R2 prefix.
+ *
+ * list() caps at 1000 keys, so anything with more cached files than that needs
+ * repeating — otherwise the tail is silently left behind, the same class of bug
+ * as deleting only manifests.
+ *
+ * Deliberately re-lists from the start each round instead of paging with a
+ * cursor: we delete the very keys we just listed, and a cursor that encodes a
+ * position rather than a key skips survivors once earlier keys disappear. (An
+ * earlier cursor version of this passed by hand and failed the 2500-object
+ * test.) Every round removes everything it saw, so this always terminates.
+ */
+async function purgeR2Prefix(bucket: R2Bucket, prefix: string): Promise<number> {
+  let deleted = 0;
+
+  for (let round = 0; round < MAX_PURGE_ROUNDS; round++) {
+    const listed = await bucket.list({ prefix, limit: 1000 });
+    const keys = listed.objects.map((object) => object.key);
+    if (keys.length === 0) {
+      return deleted;
+    }
+    await bucket.delete(keys);
+    deleted += keys.length;
+  }
+
+  console.error(
+    `[purgeDashboardStorage] Hit the ${MAX_PURGE_ROUNDS}-round cap for ${prefix}; objects may remain`
+  );
+  return deleted;
+}
+
+/**
+ * Mirror keys are namespaced by provider, so purging a dashboard needs the set
+ * of providers. Discovered from the bucket rather than trusted from a constant:
+ * a provider added later would otherwise keep leaking files with nothing to
+ * signal it. Falls back to (and always includes) the known list, so this still
+ * works if delimited listing returns nothing.
+ */
+async function discоverMirrorPrоviders(bucket: R2Bucket): Promise<string[]> {
+  const providers = new Set<string>(MIRROR_PROVIDERS);
+  try {
+    const listed = await bucket.list({ prefix: 'mirror/', delimiter: '/' });
+    for (const prefix of listed.delimitedPrefixes ?? []) {
+      // "mirror/<provider>/" -> "<provider>"
+      const provider = prefix.slice('mirror/'.length).replace(/\/$/, '');
+      if (provider) providers.add(provider);
+    }
+  } catch {
+    // Delimited listing unsupported or failed — the known list still applies.
+  }
+  return [...providers];
+}
+
+/**
+ * Delete every R2 object belonging to a dashboard.
+ *
+ * These live outside D1, so nothing cascades them: deleting a dashboard used to
+ * leave its cached content in the bucket forever.
+ *
+ * Purges by PREFIX rather than by known key. Each family stores per-file objects
+ * alongside its manifest —
+ *   workspace/<id>/snapshot.json
+ *   drive/<id>/manifest.json          + drive/<id>/files/<fileId>
+ *   mirror/<provider>/<id>/manifest.json + mirror/<provider>/<id>/files/<fileId>
+ * — so deleting just the manifests orphaned every uploaded and mirrored file,
+ * indefinitely, and left the dashboard's content unreferenced but billable.
+ * Sweeping the prefix also means a new key shape under it is covered
+ * automatically, instead of quietly leaking until someone notices.
+ *
+ * Callers: the dev clear-workspace endpoint, and deleteDashboard.
+ */
+export async function purgeDashbоardStorage(
+  env: EnvWithDriveCache,
+  dashboardId: string
+): Promise<void> {
+  const bucket = env.DRIVE_CACHE;
+  const prefixes = [`workspace/${dashboardId}/`, `drive/${dashboardId}/`];
+  for (const provider of await discоverMirrorPrоviders(bucket)) {
+    prefixes.push(`mirror/${provider}/${dashboardId}/`);
+  }
+
+  let total = 0;
+  for (const prefix of prefixes) {
+    try {
+      total += await purgeR2Prefix(bucket, prefix);
+    } catch (e) {
+      // Best-effort per prefix: one failure must not strand the others.
+      console.error(`[purgeDashboardStorage] Failed to purge ${prefix}: ${e}`);
+    }
+  }
+  if (total > 0) {
+    console.log(`[purgeDashboardStorage] Deleted ${total} object(s) for dashboard ${dashboardId}`);
+  }
+}
+
 export async function clearWorkspaceDev(
   request: Request,
   env: EnvWithDriveCache,
@@ -952,11 +1054,7 @@ export async function clearWorkspaceDev(
     return Response.json({ error: 'E79792: Not found or no access' }, { status: 404 });
   }
 
-  await env.DRIVE_CACHE.delete(workspaceSnapshotKey(data.dashboardId));
-  await env.DRIVE_CACHE.delete(driveManifestKey(data.dashboardId));
-  for (const provider of ['github', 'box', 'onedrive']) {
-    await env.DRIVE_CACHE.delete(mirrorManifestKey(provider, data.dashboardId));
-  }
+  await purgeDashbоardStorage(env, data.dashboardId);
 
   const now = new Date().toISOString();
   await env.DB.prepare(`