fix(setup,compose): bundle Redis, fix socket reconnect, and harden the setup wizard

apps/sim/lib/core/security/csp-socket-fallback.test.ts

@@ -0,0 +1,46 @@
+/**
+ * @vitest-environment node
+ *
+ * The bundled docker-compose stack runs NODE_ENV=production, serves the app from
+ * localhost, and leaves NEXT_PUBLIC_SOCKET_URL unset. getSocketUrl() falls back
+ * to localhost:3002 for a localhost page regardless of NODE_ENV, so the CSP has
+ * to permit that origin or the browser blocks the handshake and Socket.IO
+ * retries forever. Its own file because vi.mock is hoisted per-module and the
+ * sibling suite needs NEXT_PUBLIC_SOCKET_URL set.
+ */
+import { createEnvMock } from '@sim/testing'
+import { describe, expect, it, vi } from 'vitest'
+
+vi.mock('@/lib/core/config/env', () =>
+  createEnvMock({
+    NEXT_PUBLIC_APP_URL: 'http://localhost:3000',
+    NEXT_PUBLIC_SOCKET_URL: undefined,
+  })
+)
+
+vi.mock('@/lib/core/config/env-flags', () => ({
+  isDev: false,
+  isHosted: false,
+  isReactGrabEnabled: false,
+}))
+
+import { generateRuntimeCSP } from './csp'
+
+describe('generateRuntimeCSP — socket fallback on a localhost origin', () => {
+  it('permits the default socket origin when NEXT_PUBLIC_SOCKET_URL is unset', () => {
+    const csp = generateRuntimeCSP()
+
+    expect(csp).toContain('http://localhost:3002')
+    expect(csp).toContain('ws://localhost:3002')
+  })
+
+  it('keeps the socket sources inside connect-src', () => {
+    const connectSrc = generateRuntimeCSP()
+      .split(';')
+      .map((directive) => directive.trim())
+      .find((directive) => directive.startsWith('connect-src'))
+
+    expect(connectSrc).toBeDefined()
+    expect(connectSrc).toContain('ws://localhost:3002')
+  })
+})

apps/sim/lib/core/security/csp.ts

@@ -16,6 +16,23 @@ function toWebSocketUrl(httpUrl: string): string {
   return httpUrl.replace('http://', 'ws://').replace('https://', 'wss://')
 }
 
+/**
+ * Kept in sync with LOCALHOST_HOSTNAMES in ../utils/urls by hand: this module is
+ * loaded by next.config.ts before `@/` aliases resolve, so it cannot import from
+ * there (see the note above).
+ */
+const LOCALHOST_HOSTNAMES = new Set(['localhost', '127.0.0.1', '[::1]', '::1'])
+
+/** Mirrors getSocketUrl's localhost check — those origins fall back to DEFAULT_SOCKET_URL. */
+function isLocalhostUrl(url: string): boolean {
+  if (!url) return false
+  try {
+    return LOCALHOST_HOSTNAMES.has(new URL(url).hostname)
+  } catch {
+    return false
+  }
+}
+
 function getHostnameFromUrl(url: string | undefined): string[] {
   if (!url) return []
   try {
@@ -208,7 +225,15 @@ export function buildCSPString(directives: CSPDirectives): string {
 export function generateRuntimeCSP(): string {
   const appUrl = getEnv('NEXT_PUBLIC_APP_URL') || ''
 
-  const socketUrl = getEnv('NEXT_PUBLIC_SOCKET_URL') || (isDev ? DEFAULT_SOCKET_URL : '')
+  // Must permit whatever getSocketUrl() actually connects to, or the browser
+  // blocks the handshake and Socket.IO retries forever. That helper falls back
+  // to DEFAULT_SOCKET_URL whenever the page is served from localhost — which
+  // includes a production build (docker compose sets NODE_ENV=production), so
+  // keying this on isDev alone left the bundled stack with a CSP that forbade
+  // its own realtime port. A non-localhost origin still resolves to the page
+  // origin, which appUrl already covers, so nothing is loosened there.
+  const socketUrl =
+    getEnv('NEXT_PUBLIC_SOCKET_URL') || (isDev || isLocalhostUrl(appUrl) ? DEFAULT_SOCKET_URL : '')
   const socketWsUrl = socketUrl ? toWebSocketUrl(socketUrl) : ''
   const ollamaUrl = getEnv('OLLAMA_URL') || (isDev ? DEFAULT_OLLAMA_URL : '')
 

docker-compose.local.yml

@@ -21,15 +21,17 @@ services:
       - ENCRYPTION_KEY=${ENCRYPTION_KEY:-dev-encryption-key-at-least-32-chars}
       - API_ENCRYPTION_KEY=${API_ENCRYPTION_KEY:-}
       - INTERNAL_API_SECRET=${INTERNAL_API_SECRET:-dev-internal-api-secret-min-32-chars}
-      - REDIS_URL=${REDIS_URL:-}
-      - COPILOT_API_KEY=${COPILOT_API_KEY}
-      - SIM_AGENT_API_URL=${SIM_AGENT_API_URL}
+      - REDIS_URL=${REDIS_URL:-redis://redis:6379}
+      - COPILOT_API_KEY=${COPILOT_API_KEY:-}
+      - SIM_AGENT_API_URL=${SIM_AGENT_API_URL:-}
       - OLLAMA_URL=${OLLAMA_URL:-http://localhost:11434}
       - SOCKET_SERVER_URL=${SOCKET_SERVER_URL:-http://realtime:3002}
       - NEXT_PUBLIC_SOCKET_URL=${NEXT_PUBLIC_SOCKET_URL:-}
     depends_on:
       db:
         condition: service_healthy
+      redis:
+        condition: service_healthy
       migrations:
         condition: service_completed_successfully
       realtime:
@@ -55,10 +57,12 @@ services:
       - BETTER_AUTH_URL=${BETTER_AUTH_URL:-http://localhost:3000}
       - BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET:-dev-secret-at-least-32-characters-long}
       - INTERNAL_API_SECRET=${INTERNAL_API_SECRET:-dev-internal-api-secret-min-32-chars}
-      - REDIS_URL=${REDIS_URL:-}
+      - REDIS_URL=${REDIS_URL:-redis://redis:6379}
     depends_on:
       db:
         condition: service_healthy
+      redis:
+        condition: service_healthy
     restart: unless-stopped
     ports:
       - '3002:3002'
@@ -86,6 +90,20 @@ services:
     command: ['bun', 'run', 'db:migrate']
     restart: 'no'
 
+  # Backs pub/sub (live Chat task-status and table events) and the shared caches.
+  # The app falls back to PostgreSQL for storage when REDIS_URL is unset, but the
+  # pub/sub channels have no fallback — without Redis, live status never streams.
+  # Not published to the host: only the app and realtime containers need it, and
+  # binding 6379 would collide with a Redis already running locally.
+  redis:
+    image: redis:7-alpine
+    restart: always
+    healthcheck:
+      test: ['CMD', 'redis-cli', 'ping']
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
   db:
     image: pgvector/pgvector:pg17
     restart: always

docker-compose.prod.yml

@@ -30,20 +30,23 @@ services:
       - ENCRYPTION_KEY=${ENCRYPTION_KEY}
       - API_ENCRYPTION_KEY=${API_ENCRYPTION_KEY:-}
       - INTERNAL_API_SECRET=${INTERNAL_API_SECRET}
-      - REDIS_URL=${REDIS_URL:-}
-      - COPILOT_API_KEY=${COPILOT_API_KEY}
-      - SIM_AGENT_API_URL=${SIM_AGENT_API_URL}
+      - REDIS_URL=${REDIS_URL:-redis://redis:6379}
+      - COPILOT_API_KEY=${COPILOT_API_KEY:-}
+      - SIM_AGENT_API_URL=${SIM_AGENT_API_URL:-}
       - OLLAMA_URL=${OLLAMA_URL:-http://localhost:11434}
       - SOCKET_SERVER_URL=${SOCKET_SERVER_URL:-http://realtime:3002}
-      # NEXT_PUBLIC_SOCKET_URL is read by the browser. Leaving it unset lets the
-      # client default to the page's own origin (assumes the reverse proxy routes
-      # /socket.io). Set it explicitly only when the realtime service is on a
-      # different host:port from the app (e.g. wss://socket.example.com).
+      # NEXT_PUBLIC_SOCKET_URL is read by the browser. Leave it unset for this
+      # stack: the client already falls back to localhost:3002 for a localhost
+      # page, and a proxied deployment needs the page-origin fallback that an
+      # explicit value would suppress. Set it only when realtime is on a
+      # different host:port (e.g. wss://socket.example.com).
       - NEXT_PUBLIC_SOCKET_URL=${NEXT_PUBLIC_SOCKET_URL:-}
       - ADMISSION_GATE_MAX_INFLIGHT=${ADMISSION_GATE_MAX_INFLIGHT:-500}
     depends_on:
       db:
         condition: service_healthy
+      redis:
+        condition: service_healthy
       migrations:
         condition: service_completed_successfully
       realtime:
@@ -74,10 +77,12 @@ services:
       - BETTER_AUTH_URL=${BETTER_AUTH_URL:-http://localhost:3000}
       - BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET}
       - INTERNAL_API_SECRET=${INTERNAL_API_SECRET}
-      - REDIS_URL=${REDIS_URL:-}
+      - REDIS_URL=${REDIS_URL:-redis://redis:6379}
     depends_on:
       db:
         condition: service_healthy
+      redis:
+        condition: service_healthy
     healthcheck:
       test: ['CMD', 'curl', '-fsS', 'http://127.0.0.1:3002/health']
       interval: 90s
@@ -96,6 +101,20 @@ services:
     command: ['bun', 'run', 'db:migrate']
     restart: 'no'
 
+  # Backs pub/sub (live Chat task-status and table events) and the shared caches.
+  # The app falls back to PostgreSQL for storage when REDIS_URL is unset, but the
+  # pub/sub channels have no fallback — without Redis, live status never streams.
+  # Not published to the host: only the app and realtime containers need it, and
+  # binding 6379 would collide with a Redis already running locally.
+  redis:
+    image: redis:7-alpine
+    restart: unless-stopped
+    healthcheck:
+      test: ['CMD', 'redis-cli', 'ping']
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
   db:
     image: pgvector/pgvector:pg17
     restart: unless-stopped

scripts/setup/db.ts

@@ -9,6 +9,18 @@ import { glyph, theme } from './theme.ts'
 
 const DEFAULT_DSN = 'postgresql://postgres:postgres@localhost:5432/simstudio'
 
+/** Postgres' wire message when the password is wrong — a live server, not a dead one. */
+const AUTH_FAILURE = /password authentication failed/i
+
+/**
+ * Percent-encodes the password so characters that are structural in a URL
+ * (`@`, `:`, `/`, `#`, `?`) can't re-parse the DSN into a different host — which
+ * would fail a password that is actually correct.
+ */
+function buildDsn(password: string, hostPort: string | number): string {
+  return `postgresql://postgres:${encodeURIComponent(password)}@localhost:${hostPort}/simstudio`
+}
+
 export function docker(args: string[]): void {
   const result = spawnSync('docker', args, { encoding: 'utf8' })
   if (result.status !== 0) {
@@ -63,7 +75,9 @@ function inspectManagedContainer(): ManagedContainer | null {
 
   return {
     running: running === 'true',
-    dsn: `postgresql://postgres:${password}@localhost:${hostPort}/simstudio`,
+    // Read back from the container env verbatim, so it may be a password the
+    // user supplied for an existing volume — encode it like any other.
+    dsn: buildDsn(password, hostPort),
   }
 }
 
@@ -122,6 +136,81 @@ async function promptExternalDsn(): Promise<string> {
   }
 }
 
+const DB_VOLUME = 'sim-postgres-data'
+
+/** True once initdb has run in the volume — PG_VERSION only exists after bootstrap. */
+function volumeInitialized(): boolean {
+  if (spawnSync('docker', ['volume', 'inspect', DB_VOLUME], { stdio: 'ignore' }).status !== 0) {
+    return false
+  }
+  // Read the marker from inside the volume; the image is already local, so this
+  // costs nothing extra and beats assuming "volume exists" means "bootstrapped"
+  // (a failed first run leaves an empty volume behind).
+  return (
+    spawnSync(
+      'docker',
+      [
+        'run',
+        '--rm',
+        '-v',
+        `${DB_VOLUME}:/pgdata`,
+        '--entrypoint',
+        'test',
+        'pgvector/pgvector:pg17',
+        '-f',
+        '/pgdata/PG_VERSION',
+      ],
+      { stdio: 'ignore' }
+    ).status === 0
+  )
+}
+
+/**
+ * The volume already holds a cluster whose password we cannot read back. Either
+ * the user supplies it, or the data goes — silently generating a new password
+ * would produce a container that never authenticates.
+ */
+async function resolveExistingVolume(): Promise<string> {
+  p.log.warn(
+    `The ${DB_VOLUME} volume already contains a database, but its password is not recoverable — Postgres ignores POSTGRES_PASSWORD on an existing data directory.`
+  )
+  const choice = await p.select({
+    message: 'How should the wizard proceed?',
+    options: [
+      {
+        value: 'password',
+        label: 'Keep the data — I have its password',
+        hint: 'from a previous .env, or your notes',
+      },
+      {
+        value: 'wipe',
+        label: 'Delete the old data and start fresh',
+        hint: `removes the ${DB_VOLUME} volume — this cannot be undone`,
+      },
+    ],
+    initialValue: 'password',
+  })
+  if (choice === 'password') {
+    return p.password({
+      message: `Password for the existing ${DB_VOLUME} database`,
+      validate: (value) => (value ? undefined : 'required'),
+    })
+  }
+  const sure = await p.confirm({
+    message: theme.error(`Permanently delete the ${DB_VOLUME} volume and all its data?`),
+    initialValue: false,
+  })
+  if (!sure) {
+    throw new SetupError('kept the existing database volume, so setup cannot continue.', [
+      're-run and supply the password, or remove it yourself:',
+      theme.command(`docker volume rm ${DB_VOLUME}`),
+    ])
+  }
+  docker(['volume', 'rm', DB_VOLUME])
+  p.log.step(`Removed ${DB_VOLUME}`)
+  return generateSecret().slice(0, 24)
+}
+
 /**
  * Provisions the managed container, reconciling with one that already exists
  * rather than colliding on the name. Recreating is always an explicit choice —
@@ -142,15 +231,27 @@ async function startManagedContainer(detection: Detection): Promise<string> {
       throw new SetupError(`the existing ${DB_CONTAINER} container is not usable.`, [
         `inspect: ${theme.command(`docker logs ${DB_CONTAINER}`)}`,
         `remove it: ${theme.command(`docker rm -f ${DB_CONTAINER}`)}`,
-        `start clean: ${theme.command('docker volume rm sim-postgres-data')} drops its data too`,
+        `start clean: ${theme.command(`docker volume rm ${DB_VOLUME}`)} drops its data too`,
       ])
     }
     docker(['rm', '-f', DB_CONTAINER])
   }
 
-  const password = generateSecret().slice(0, 24)
   const hostPort = detection.postgresPortOpen ? 5433 : 5432
-  const dsn = `postgresql://postgres:${password}@localhost:${hostPort}/simstudio`
+  // POSTGRES_PASSWORD only applies when initdb runs on an empty data directory.
+  // The volume outlives the container (sim down keeps it, so does `docker rm`),
+  // so once the container is gone the password it was created with is
+  // unrecoverable — inspectManagedContainer reads it from the container, not the
+  // volume. Running with a freshly generated password against an initialized
+  // volume starts a healthy Postgres that rejects every connection with
+  // "password authentication failed", which surfaces as a bogus "container did
+  // not become healthy". Ask instead of guessing.
+  const password = volumeInitialized()
+    ? await resolveExistingVolume()
+    : generateSecret().slice(0, 24)
+  // A user-supplied password can contain @ : / # — raw interpolation would
+  // re-parse the DSN into a different host and fail a password that is correct.
+  const dsn = buildDsn(password, hostPort)
   docker([
     'run',
     '-d',
@@ -159,7 +260,7 @@ async function startManagedContainer(detection: Detection): Promise<string> {
     '--label',
     'managed-by=sim-setup',
     '-v',
-    'sim-postgres-data:/var/lib/postgresql/data',
+    `${DB_VOLUME}:/var/lib/postgresql/data`,
     '-e',
     `POSTGRES_PASSWORD=${password}`,
     '-e',
@@ -170,9 +271,31 @@ async function startManagedContainer(detection: Detection): Promise<string> {
   ])
   const spin = p.spinner()
   spin.start(`Starting ${DB_CONTAINER} container on :${hostPort}…`)
-  const healthy = await waitFor(async () => (await pgProbe(dsn)).ok, 45_000, 1500)
+  let lastError = ''
+  const healthy = await waitFor(
+    async () => {
+      const probe = await pgProbe(dsn)
+      if (!probe.ok) lastError = probe.error ?? ''
+      return probe.ok
+    },
+    45_000,
+    1500
+  )
   if (!healthy) {
     spin.stop(`${glyph.fail} container did not become healthy`)
+    // Postgres running and refusing the password is a different failure from
+    // Postgres never starting, and it is the likely one on the keep-the-volume
+    // path. Reporting it as "did not become healthy" is the exact confusion
+    // this whole change set exists to remove.
+    if (AUTH_FAILURE.test(lastError)) {
+      throw new SetupError(
+        `Postgres started, but rejected that password for the existing ${DB_VOLUME} volume.`,
+        [
+          're-run and enter the password the volume was created with',
+          `or discard the old data: ${theme.command(`docker rm -f ${DB_CONTAINER} && docker volume rm ${DB_VOLUME}`)}`,
+        ]
+      )
+    }
     const logs = spawnSync('docker', ['logs', '--tail', '20', DB_CONTAINER], { encoding: 'utf8' })
     throw new SetupError(
       `the Postgres container failed to start. Last logs:\n${logs.stdout}${logs.stderr}`,