feat(compose,setup): bundle Redis, always configure it, fix lifecycle detection

TheodoreSpeaks · TheodoreSpeaks · commit dcd57332b913 · 2026-07-25T12:36:43.000-07:00
Compose shipped no redis service at all — REDIS_URL was ${REDIS_URL:-} in both app and realtime, so every self-hosted stack ran without it. Storage silently falls back to PostgreSQL, but the pub/sub channels (live Chat task-status, table events) have no fallback, so live updates never arrived.

- compose (prod + local): add a redis:7-alpine service with a healthcheck, default REDIS_URL to redis://redis:6379, and make app/realtime depend on it being healthy. Not published to the host — only the containers need it, and binding 6379 would collide with a local Redis. An external REDIS_URL in root .env still overrides. Deliberately not written into root .env: doctor pings REDIS_URL from the host, and a compose-internal hostname would fail that probe the same way DATABASE_URL would.
- dev mode: configure Redis in quick too. Quick uses a new non-interactive ensureRedis (adopt whatever answers, else start the managed container, warn only if Docker is unavailable); custom keeps the ladder, with corrected copy — the old prompt claimed Redis was only for multi-replica.
- lifecycle: detect compose stacks via 'docker compose ls' instead of probing '-f &lt;file&gt; ps' in the working directory. Compose derives the project name from the directory it was started in, so the old probe found a stack only when run from the checkout that launched it (a globally linked sim never could) and listed the same stack once per candidate file. compose ls reports the real project and its config file, so one stack yields one install from anywhere; non-Sim projects are filtered by compose filename. Every compose op now runs in that stack's directory.
- lifecycle: distinguish 'Docker unreachable' from 'nothing installed'. With the daemon down, status reported containers as 'absent' and suggested re-running setup; it now says Docker is down and marks state unknown.
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
@@ -21,7 +21,7 @@ services:
       - ENCRYPTION_KEY=${ENCRYPTION_KEY:-dev-encryption-key-at-least-32-chars}
       - API_ENCRYPTION_KEY=${API_ENCRYPTION_KEY:-}
       - INTERNAL_API_SECRET=${INTERNAL_API_SECRET:-dev-internal-api-secret-min-32-chars}
-      - REDIS_URL=${REDIS_URL:-}
+      - REDIS_URL=${REDIS_URL:-redis://redis:6379}
       - COPILOT_API_KEY=${COPILOT_API_KEY}
       - SIM_AGENT_API_URL=${SIM_AGENT_API_URL}
       - OLLAMA_URL=${OLLAMA_URL:-http://localhost:11434}
@@ -30,6 +30,8 @@ services:
     depends_on:
       db:
         condition: service_healthy
+      redis:
+        condition: service_healthy
       migrations:
         condition: service_completed_successfully
       realtime:
@@ -55,10 +57,12 @@ services:
       - BETTER_AUTH_URL=${BETTER_AUTH_URL:-http://localhost:3000}
       - BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET:-dev-secret-at-least-32-characters-long}
       - INTERNAL_API_SECRET=${INTERNAL_API_SECRET:-dev-internal-api-secret-min-32-chars}
-      - REDIS_URL=${REDIS_URL:-}
+      - REDIS_URL=${REDIS_URL:-redis://redis:6379}
     depends_on:
       db:
         condition: service_healthy
+      redis:
+        condition: service_healthy
     restart: unless-stopped
     ports:
       - '3002:3002'
@@ -86,6 +90,20 @@ services:
     command: ['bun', 'run', 'db:migrate']
     restart: 'no'
 
+  # Backs pub/sub (live Chat task-status and table events) and the shared caches.
+  # The app falls back to PostgreSQL for storage when REDIS_URL is unset, but the
+  # pub/sub channels have no fallback — without Redis, live status never streams.
+  # Not published to the host: only the app and realtime containers need it, and
+  # binding 6379 would collide with a Redis already running locally.
+  redis:
+    image: redis:7-alpine
+    restart: always
+    healthcheck:
+      test: ['CMD', 'redis-cli', 'ping']
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
   db:
     image: pgvector/pgvector:pg17
     restart: always
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
@@ -30,7 +30,7 @@ services:
       - ENCRYPTION_KEY=${ENCRYPTION_KEY}
       - API_ENCRYPTION_KEY=${API_ENCRYPTION_KEY:-}
       - INTERNAL_API_SECRET=${INTERNAL_API_SECRET}
-      - REDIS_URL=${REDIS_URL:-}
+      - REDIS_URL=${REDIS_URL:-redis://redis:6379}
       - COPILOT_API_KEY=${COPILOT_API_KEY}
       - SIM_AGENT_API_URL=${SIM_AGENT_API_URL}
       - OLLAMA_URL=${OLLAMA_URL:-http://localhost:11434}
@@ -44,6 +44,8 @@ services:
     depends_on:
       db:
         condition: service_healthy
+      redis:
+        condition: service_healthy
       migrations:
         condition: service_completed_successfully
       realtime:
@@ -74,10 +76,12 @@ services:
       - BETTER_AUTH_URL=${BETTER_AUTH_URL:-http://localhost:3000}
       - BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET}
       - INTERNAL_API_SECRET=${INTERNAL_API_SECRET}
-      - REDIS_URL=${REDIS_URL:-}
+      - REDIS_URL=${REDIS_URL:-redis://redis:6379}
     depends_on:
       db:
         condition: service_healthy
+      redis:
+        condition: service_healthy
     healthcheck:
       test: ['CMD', 'curl', '-fsS', 'http://127.0.0.1:3002/health']
       interval: 90s
@@ -96,6 +100,20 @@ services:
     command: ['bun', 'run', 'db:migrate']
     restart: 'no'
 
+  # Backs pub/sub (live Chat task-status and table events) and the shared caches.
+  # The app falls back to PostgreSQL for storage when REDIS_URL is unset, but the
+  # pub/sub channels have no fallback — without Redis, live status never streams.
+  # Not published to the host: only the app and realtime containers need it, and
+  # binding 6379 would collide with a Redis already running locally.
+  redis:
+    image: redis:7-alpine
+    restart: unless-stopped
+    healthcheck:
+      test: ['CMD', 'redis-cli', 'ping']
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
   db:
     image: pgvector/pgvector:pg17
     restart: unless-stopped
diff --git a/scripts/setup/lifecycle.ts b/scripts/setup/lifecycle.ts
@@ -1,4 +1,5 @@
 import { spawnSync } from 'node:child_process'
+import path from 'node:path'
 import { DB_CONTAINER, type Detection, REDIS_CONTAINER, runDetection } from './detect.ts'
 import { archiveEnvFile, ROOT } from './env-files.ts'
 import { SetupError } from './errors.ts'
@@ -38,28 +39,37 @@ function shq(value: string): string {
   return `'${value.replace(/'/g, `'\\''`)}'`
 }
 
+/** Is the Docker daemon reachable? Distinguishes "nothing installed" from "can't see". */
+function dockerReachable(): boolean {
+  return spawnSync('docker', ['info'], { stdio: 'ignore' }).status === 0
+}
+
 /** Non-throwing docker probe; returns trimmed stdout or null on any failure. */
-function dockerText(args: string[]): string | null {
-  const result = spawnSync('docker', args, { cwd: ROOT, encoding: 'utf8' })
+function dockerText(args: string[], cwd: string = ROOT): string | null {
+  const result = spawnSync('docker', args, { cwd, encoding: 'utf8' })
   return result.status === 0 ? result.stdout.trim() : null
 }
 
 /** Docker command whose output the user should see (up, logs); returns exit code. */
-function dockerInherit(args: string[]): number {
-  return spawnSync('docker', args, { cwd: ROOT, stdio: 'inherit' }).status ?? 1
+function dockerInherit(args: string[], cwd: string = ROOT): number {
+  return spawnSync('docker', args, { cwd, stdio: 'inherit' }).status ?? 1
 }
 
 /** Docker command that must succeed; throws a SetupError with stderr on failure. */
-function dockerRun(args: string[], failMessage: string): void {
-  const result = spawnSync('docker', args, { cwd: ROOT, encoding: 'utf8' })
+function dockerRun(args: string[], failMessage: string, cwd: string = ROOT): void {
+  const result = spawnSync('docker', args, { cwd, encoding: 'utf8' })
   if (result.status !== 0) {
     throw new SetupError(`${failMessage}: ${result.stderr.trim() || result.stdout.trim()}`)
   }
 }
 
 interface ComposeInstall {
   kind: 'compose'
+  /** Absolute path to the compose file Docker recorded for the project. */
   file: string
+  /** Directory the stack was brought up from — every compose op runs here. */
+  dir: string
+  project: string
 }
 interface DevInstall {
   kind: 'dev'
@@ -74,15 +84,48 @@ interface K8sInstall {
 }
 type Install = ComposeInstall | DevInstall | K8sInstall
 
+interface ComposeProject {
+  Name: string
+  Status: string
+  ConfigFiles: string
+}
+
 /**
- * A compose project brought up from ROOT reuses the same project name at `ps`,
- * so probing each candidate compose file recovers exactly which one owns
- * containers — no need to guess the project name or persist the choice.
+ * Ask Docker which compose projects exist rather than guessing from the working
+ * directory. Compose derives a project name from the directory it was started
+ * in, so probing `-f <file> ps` only ever finds a stack when you happen to stand
+ * in the checkout that launched it — a globally linked `sim` would never see one
+ * — and it reports the same stack once per candidate file, since both files map
+ * to the same directory-derived project. `compose ls` records the real project
+ * and the exact config file, so one running stack yields exactly one install
+ * wherever it was started from. Projects whose compose file isn't one of ours
+ * (a devcontainer, an unrelated app) are filtered out by filename.
  */
 function composeInstalls(): ComposeInstall[] {
-  return COMPOSE_FILES.filter((file) => dockerText(['compose', '-f', file, 'ps', '-aq'])).map(
-    (file) => ({ kind: 'compose', file })
-  )
+  const raw = dockerText(['compose', 'ls', '-a', '--format', 'json'])
+  if (!raw) return []
+  let projects: ComposeProject[]
+  try {
+    projects = JSON.parse(raw)
+  } catch {
+    return []
+  }
+  const installs: ComposeInstall[] = []
+  for (const project of projects) {
+    // ConfigFiles is a comma-separated list when a stack was started with -f more than once.
+    const file = (project.ConfigFiles ?? '')
+      .split(',')
+      .map((entry) => entry.trim())
+      .find((entry) => (COMPOSE_FILES as readonly string[]).includes(path.basename(entry)))
+    if (!file) continue
+    installs.push({
+      kind: 'compose',
+      file,
+      dir: path.dirname(file),
+      project: project.Name,
+    })
+  }
+  return installs
 }
 
 /** Dev mode owns the split env files and, usually, the managed Postgres/Redis. */
@@ -124,7 +167,8 @@ function detectInstalls(detection: Detection): Install[] {
 }
 
 function describeInstall(install: Install): string {
-  if (install.kind === 'compose') return `Docker Compose (${install.file})`
+  if (install.kind === 'compose')
+    return `Docker Compose (project ${install.project} in ${install.dir})`
   if (install.kind === 'dev') return 'Local dev (managed Postgres/Redis)'
   // Naming a non-local cluster is the guard against acting on the wrong one after
   // an ambient context switch — every destructive confirm renders this string.
@@ -165,7 +209,7 @@ function start(install: Install): void {
   if (install.kind === 'compose') {
     const spin = p.spinner()
     spin.start('Starting containers…')
-    dockerRun(['compose', '-f', install.file, 'up', '-d'], 'docker compose up failed')
+    dockerRun(['compose', '-f', install.file, 'up', '-d'], 'docker compose up failed', install.dir)
     spin.stop('Containers up')
     p.note(
       [`open ${APP_URL}`, 'follow logs:  sim logs', 'stop:         sim stop'].join('\n'),
@@ -190,7 +234,7 @@ function stop(install: Install): void {
   if (install.kind === 'compose') {
     const spin = p.spinner()
     spin.start('Stopping containers…')
-    dockerRun(['compose', '-f', install.file, 'stop'], 'docker compose stop failed')
+    dockerRun(['compose', '-f', install.file, 'stop'], 'docker compose stop failed', install.dir)
     spin.stop('Containers stopped (data kept)')
     p.note(['start again:  sim start', 'remove:       sim down'].join('\n'), 'Stopped')
     return
@@ -220,7 +264,11 @@ function restart(install: Install): void {
   if (install.kind === 'compose') {
     const spin = p.spinner()
     spin.start('Restarting containers…')
-    dockerRun(['compose', '-f', install.file, 'restart'], 'docker compose restart failed')
+    dockerRun(
+      ['compose', '-f', install.file, 'restart'],
+      'docker compose restart failed',
+      install.dir
+    )
     spin.stop('Containers restarted')
     p.note(`open ${APP_URL}`, 'Running')
     return
@@ -237,7 +285,7 @@ function restart(install: Install): void {
 
 function showLogs(install: Install): void {
   if (install.kind === 'compose') {
-    dockerInherit(['compose', '-f', install.file, 'logs', '-f', '--tail', '100'])
+    dockerInherit(['compose', '-f', install.file, 'logs', '-f', '--tail', '100'], install.dir)
     return
   }
   if (install.kind === 'dev') {
@@ -268,7 +316,7 @@ async function down(install: Install): Promise<void> {
     return
   }
   if (install.kind === 'compose') {
-    dockerRun(['compose', '-f', install.file, 'down'], 'docker compose down failed')
+    dockerRun(['compose', '-f', install.file, 'down'], 'docker compose down failed', install.dir)
     p.log.step('Containers removed (volumes kept)')
     return
   }
@@ -311,7 +359,11 @@ async function reset(install: Install | null): Promise<void> {
     if (backup) p.log.step(`Archived ${backup}`)
   }
   if (install?.kind === 'compose') {
-    dockerRun(['compose', '-f', install.file, 'down', '-v'], 'docker compose down -v failed')
+    dockerRun(
+      ['compose', '-f', install.file, 'down', '-v'],
+      'docker compose down -v failed',
+      install.dir
+    )
     p.log.step('Containers and volumes removed')
   } else if (install?.kind === 'dev') {
     const names = managedNames(install)
@@ -343,14 +395,29 @@ async function reset(install: Install | null): Promise<void> {
 async function status(): Promise<void> {
   const detection = await runDetection()
   const installs = detectInstalls(detection)
+  const docker = dockerReachable()
   console.log(`\n${theme.heading('◆ Sim status')}\n`)
+  // Every container probe goes through Docker, so when the daemon is down the
+  // honest answer is "unknown", not "absent" — and a compose stack is invisible
+  // entirely. Saying "no install detected" there sends the user to re-run setup
+  // for what is really a stopped Docker Desktop.
+  if (!docker) {
+    console.log(
+      ` ${glyph.warn} Docker is not reachable — container and Compose state below is unknown.`
+    )
+    console.log(`   ${theme.muted('start Docker Desktop (or OrbStack), then re-run this.')}\n`)
+  }
   if (installs.length === 0) {
-    console.log(` ${glyph.warn} No Sim install detected — run ${theme.command('sim setup')}.`)
+    console.log(
+      docker
+        ? ` ${glyph.warn} No Sim install detected — run ${theme.command('sim setup')}.`
+        : ` ${glyph.warn} No install detected, but that may just be Docker being down.`
+    )
     return
   }
   for (const install of installs) console.log(` ${glyph.pass} ${describeInstall(install)}`)
   const containerState = (state: { state: 'running' | 'stopped' } | null) =>
-    state ? state.state : 'absent'
+    docker ? (state ? state.state : 'absent') : 'unknown (docker down)'
   console.log()
   console.log(` postgres (${DB_CONTAINER}):  ${containerState(detection.dbContainer)}`)
   console.log(` redis (${REDIS_CONTAINER}):     ${containerState(detection.redisContainer)}`)
diff --git a/scripts/setup/modes/dev.ts b/scripts/setup/modes/dev.ts
@@ -7,7 +7,7 @@ import { ROOT, readEnvFile, writeEnvValues } from '../env-files.ts'
 import { SetupError } from '../errors.ts'
 import { pgProbe } from '../probes.ts'
 import * as p from '../prompter.ts'
-import { resolveRedis } from '../redis.ts'
+import { ensureRedis, resolveRedis } from '../redis.ts'
 import {
   collectSecrets,
   promptCopilotKey,
@@ -62,8 +62,8 @@ async function runMigrations(dsn: string): Promise<void> {
 async function promptRedis(detection: Detection, existing?: string): Promise<string | null> {
   const wants = await p.confirm({
     message:
-      'Configure Redis? (only needed for multi-replica — single instance runs fine without it)',
-    initialValue: Boolean(existing),
+      'Configure Redis? (powers live Chat status and table events; storage falls back to Postgres)',
+    initialValue: true,
   })
   if (!wants) return null
   return resolveRedis(detection, existing)
@@ -121,12 +121,20 @@ export async function runDevMode(
   if (copilotKey) values.COPILOT_API_KEY = copilotKey
   Object.assign(values, await promptLlmKeys(detection, !quick))
 
+  // Redis is set up in every mode, quick included. Storage falls back to
+  // PostgreSQL without it, but the pub/sub channels (live Chat task-status,
+  // table events) have no fallback — skipping it silently produces an install
+  // where live updates never arrive. Quick configures it with no questions at
+  // all; custom keeps the opt-out and the where-should-it-live ladder.
+  const redisUrl = quick
+    ? await ensureRedis(detection, simAfter.vars.get('REDIS_URL'))
+    : await promptRedis(detection, simAfter.vars.get('REDIS_URL'))
+  if (redisUrl) {
+    values.REDIS_URL = redisUrl
+    writeEnvValues('realtime', { REDIS_URL: redisUrl })
+  }
+
   if (!quick) {
-    const redisUrl = await promptRedis(detection, simAfter.vars.get('REDIS_URL'))
-    if (redisUrl) {
-      values.REDIS_URL = redisUrl
-      writeEnvValues('realtime', { REDIS_URL: redisUrl })
-    }
     const trigger = await promptTrigger()
     if (trigger) Object.assign(values, trigger)
     const storage = await promptStorage(simAfter.vars, false)
diff --git a/scripts/setup/redis.ts b/scripts/setup/redis.ts
@@ -79,6 +79,37 @@ async function promptRedisUrl(existing?: string): Promise<string> {
   return url
 }
 
+/**
+ * Non-interactive Redis for quick mode: adopt whatever already answers, else
+ * start the managed container. Quick's contract is sensible defaults with no
+ * questions, and Redis is not optional enough to skip — pub/sub (live Chat
+ * status, table events) has no PostgreSQL fallback. Returns null only when
+ * nothing answers and Docker is unavailable, so the caller can warn instead of
+ * failing the whole setup.
+ */
+export async function ensureRedis(detection: Detection, existing?: string): Promise<string | null> {
+  if (existing && (await redisPing(existing)).ok) return existing
+  if (detection.redisPortOpen && (await redisPing(LOCAL_URL)).ok) {
+    p.log.step(`Using the Redis already on :6379`)
+    return LOCAL_URL
+  }
+  if (detection.redisContainer?.managed) {
+    if (detection.redisContainer.state === 'stopped') docker(['start', REDIS_CONTAINER])
+    const managedUrl = managedRedisUrl()
+    if (managedUrl && (await waitFor(async () => (await redisPing(managedUrl)).ok, 15_000, 500))) {
+      p.log.step(`Reusing ${REDIS_CONTAINER}`)
+      return managedUrl
+    }
+  }
+  if (!(await ensureDocker(false))) {
+    p.log.warn(
+      'Docker is unavailable, so Redis was not configured — live Chat status and table events will not stream.'
+    )
+    return null
+  }
+  return startManagedRedis(detection)
+}
+
 /**
  * Redis ladder, mirroring the Postgres one: reuse what's running (with
  * consent), restart/start a wizard-managed container, or take a URL.
