From 3b7938977d027932bff0ce3196d2ea7a678ff29f Mon Sep 17 00:00:00 2001
From: Brian Love <brian@liveloveapp.com>
Date: Fri, 18 Sep 2026 11:27:21 -0700
Subject: [PATCH] fix(lifecycle): enrichment reconciliation backs off and gives
 up

A job that fails before a Dawn research attempt is submitted carries no
research_attempt deadline, so the only terminal condition never applied and the
job deferred every fifteen seconds forever. Four production jobs reached ~2,400
attempts. Reconciliation now backs off exponentially to a fifteen minute
ceiling and fails for manual review after a bounded number of attempts.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 .../src/enrichment/dawn-jobs.spec.ts          | 41 +++++++++++++++++++
 apps/lifecycle/src/enrichment/dawn-jobs.ts    | 17 +++++++-
 2 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/apps/lifecycle/src/enrichment/dawn-jobs.spec.ts b/apps/lifecycle/src/enrichment/dawn-jobs.spec.ts
index 9541306eb..64350a4b3 100644
--- a/apps/lifecycle/src/enrichment/dawn-jobs.spec.ts
+++ b/apps/lifecycle/src/enrichment/dawn-jobs.spec.ts
@@ -521,4 +521,45 @@ describe('Dawn Growth job orchestration', () => {
     expect(deps.recordCleanupAbsence).toHaveBeenCalled();
     expect(deps.finishCleanup).not.toHaveBeenCalled();
   });
+
+  // Production defect: every runaway job had no research_attempt in its payload,
+  // so the expiresAt deadline never applied and the job deferred every fifteen
+  // seconds indefinitely (four jobs reached ~2,400 attempts over six days).
+  const reconciling = (attempts: number) => {
+    const f = fixture();
+    f.deps.readDomain.mockRejectedValue(new Error('offline'));
+    return {
+      ...f,
+      run: () => f.handlers.enrich(db, { ...job, payload: {}, attempts }, {}),
+    };
+  };
+  const deferredDelay = (deps: { defer: ReturnType<typeof vi.fn> }) =>
+    (
+      deps.defer.mock.calls.at(-1)?.[1] as { availableAt: Date }
+    ).availableAt.getTime() - now.getTime();
+  it('backs reconciliation off exponentially when the payload carries no attempt deadline', async () => {
+    const first = reconciling(0);
+    expect(await first.run()).toBe('deferred');
+    expect(first.deps.fail).not.toHaveBeenCalled();
+    const second = reconciling(3);
+    expect(await second.run()).toBe('deferred');
+    const ceiling = reconciling(9);
+    expect(await ceiling.run()).toBe('deferred');
+    const a = deferredDelay(first.deps),
+      b = deferredDelay(second.deps),
+      c = deferredDelay(ceiling.deps);
+    expect(a).toBe(15000);
+    expect(b).toBeGreaterThan(a);
+    expect(c).toBeGreaterThan(b);
+    expect(c).toBeLessThanOrEqual(15 * 60000);
+  });
+  it('fails reconciliation for manual review once the attempt cap is reached', async () => {
+    const capped = reconciling(20);
+    expect(await capped.run()).toBe('failed');
+    expect(capped.deps.defer).not.toHaveBeenCalled();
+    expect(capped.deps.fail).toHaveBeenCalledWith(
+      db,
+      expect.objectContaining({ errorCode: 'dawn_reconciliation_exhausted' })
+    );
+  });
 });
diff --git a/apps/lifecycle/src/enrichment/dawn-jobs.ts b/apps/lifecycle/src/enrichment/dawn-jobs.ts
index 95d4e02e7..2cd9da27e 100644
--- a/apps/lifecycle/src/enrichment/dawn-jobs.ts
+++ b/apps/lifecycle/src/enrichment/dawn-jobs.ts
@@ -42,6 +42,12 @@ import { createTraceTransport } from '../../../growth-research/src/production/tr
 
 const TERMINAL = new Set(['success', 'error', 'interrupted', 'timeout']);
 const RECOVERY_GRACE_MS = 5 * 60000;
+// The research_attempt deadline below only exists once an attempt was submitted,
+// so a job that keeps failing before submission carries no deadline at all. These
+// bounds are the payload-independent terminal condition for that case.
+const MAX_RECONCILIATION_ATTEMPTS = 20;
+const RECONCILIATION_BASE_DELAY_MS = 15_000;
+const RECONCILIATION_MAX_DELAY_MS = 15 * 60_000;
 const CLEANUP_HORIZON_MS = 7 * 86400000;
 const UUID =
   /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
@@ -170,10 +176,19 @@ export function createDawnJobHandlers(
         input.now.getTime() >= Date.parse(expiresAt) + RECOVERY_GRACE_MS
       )
         return fail('dawn_recovery_deadline');
+      const attempts = job.attempts ?? 0;
+      if (attempts >= MAX_RECONCILIATION_ATTEMPTS)
+        return fail('dawn_reconciliation_exhausted');
       await d.defer(db, {
         ...input,
         errorCode,
-        availableAt: new Date(input.now.getTime() + 15000),
+        availableAt: new Date(
+          input.now.getTime() +
+            Math.min(
+              RECONCILIATION_MAX_DELAY_MS,
+              RECONCILIATION_BASE_DELAY_MS * 2 ** Math.min(attempts, 10)
+            )
+        ),
       });
       return 'deferred' as const;
     };