fix(publish-content): retry transient HANA pool-timeout 500s (#2286)

package.json

@@ -302,13 +302,25 @@
           "kind": "hana",
           "model": [
             "db/hana"
-          ]
+          ],
+          "pool": {
+            "acquireTimeoutMillis": 5000,
+            "min": 0,
+            "max": 100,
+            "fifo": true
+          }
         },
         "[production]": {
           "kind": "hana",
           "model": [
             "db/hana"
-          ]
+          ],
+          "pool": {
+            "acquireTimeoutMillis": 5000,
+            "min": 0,
+            "max": 100,
+            "fifo": true
+          }
         }
       },
       "audit-log": {

scripts/__tests__/publish-client.test.ts

@@ -2,6 +2,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest';
 import {
   beginSession, appendBatch, commitSession, abortSession, fetchRemoteHashes, renderConceptsPhase
 } from '../lib/publish-client.js';
+import { withRetry } from '../lib/publish-retry.js';
 
 const baseUrl = 'http://localhost:4004';
 const apiKey  = 'test-key';
@@ -36,6 +37,43 @@ describe('publish-client', () => {
     ).rejects.toMatchObject({ status: 409 });
   });
 
+  // #2286 — begin under withRetry rides out a transient HANA pool-acquire 500
+  // ("Pool resource could not be acquired within 1s") instead of aborting the
+  // whole prod rebuild. This is the exact composition main() now uses.
+  it('beginSession under withRetry recovers from a transient pool-timeout 500', async () => {
+    const fetchMock = vi.fn()
+      .mockResolvedValueOnce({
+        ok: false, status: 500,
+        text: () => Promise.resolve('{"error":"Pool resource could not be acquired within 1s"}'),
+      })
+      .mockResolvedValueOnce({
+        ok: true, status: 201,
+        json: () => Promise.resolve({ sessionId: 'abc', version: 8, expiresAt: '2026-09-14T00:00:00Z' }),
+      });
+    vi.stubGlobal('fetch', fetchMock);
+    const out = await withRetry(
+      () => beginSession({ baseUrl, apiKey, trigger: 't', hugoVersion: 'v1', expectedSlugCount: 1 }),
+      { attempts: 5, backoffMs: [0] }
+    );
+    expect(out).toMatchObject({ sessionId: 'abc', version: 8 });
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+  });
+
+  // A non-transient status (e.g. 409 lock held) must NOT be retried.
+  it('beginSession under withRetry does not retry a permanent 409', async () => {
+    const fetchMock = vi.fn().mockResolvedValue({
+      ok: false, status: 409, text: () => Promise.resolve('lock held'),
+    });
+    vi.stubGlobal('fetch', fetchMock);
+    await expect(
+      withRetry(
+        () => beginSession({ baseUrl, apiKey, trigger: 't', hugoVersion: 'v1', expectedSlugCount: 1 }),
+        { attempts: 5, backoffMs: [0] }
+      )
+    ).rejects.toMatchObject({ status: 409 });
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+  });
+
   it('appendBatch posts files/metadata/bodyTexts and returns server result', async () => {
     const fetchMock = vi.fn().mockResolvedValue({
       ok: true, status: 202,

scripts/publish-content.ts

@@ -1106,7 +1106,20 @@ async function main() {
   let remoteHashes: Record<string, string> = {};
   if (mode !== 'force') {
     log(`Fetching remote hashes from ${opts.baseUrl}/content/hashes...`);
-    try { remoteHashes = await fetchRemoteHashes({ baseUrl: opts.baseUrl, apiKey: opts.apiKey }); }
+    // #2286 — retry transient 500s (HANA pool-acquire timeout) so a momentary
+    // load spike during delta detection doesn't fatally abort the rebuild
+    // before a session is even opened. 503 already returns {} inside the client.
+    try {
+      remoteHashes = await withRetry(
+        () => fetchRemoteHashes({ baseUrl: opts.baseUrl, apiKey: opts.apiKey }),
+        {
+          attempts: 4, backoffMs: [1000, 3000, 6000], jitterRatio: 0.2,
+          onAttemptFail: (attempt, err, willRetry) => {
+            console.error(`[publish-content] /content/hashes fetch failed (attempt ${attempt}/4): ${formatErrorChain(err)}${willRetry ? ' — retrying' : ''}`);
+          },
+        }
+      );
+    }
     catch (err) {
       console.error(`Cannot reach ${opts.baseUrl}/content/hashes: ${formatErrorChain(err)}`);
       process.exit(1);
@@ -1201,11 +1214,25 @@ async function main() {
   // small and the server happily accepts them mixed with regular slugs).
   const sidecarKeys = await collectSidecars(opts.hugoDir, payload, log, channel);
 
-  const begin = await beginSession({
-    baseUrl: opts.baseUrl, apiKey: opts.apiKey,
-    trigger: opts.trigger, hugoVersion: opts.hugoVersion, expectedSlugCount: targetSlugs.length,
-    initiator: opts.initiator,
-  });
+  // #2286 — begin was the ONE publish HTTP call in the happy path with neither
+  // withRetry nor a try/catch, so a transient HANA pool-acquire 500 ("Pool
+  // resource could not be acquired within 1s") aborted the whole rebuild via
+  // the top-level Fatal catch. Wrap it like commit: 500/502/503/504 are already
+  // classified transient in publish-retry.ts; jitter de-syncs the three DevRel
+  // projects so a briefly-saturated pool isn't re-hit in lockstep.
+  const begin = await withRetry(
+    () => beginSession({
+      baseUrl: opts.baseUrl, apiKey: opts.apiKey,
+      trigger: opts.trigger, hugoVersion: opts.hugoVersion, expectedSlugCount: targetSlugs.length,
+      initiator: opts.initiator,
+    }),
+    {
+      attempts: 5, backoffMs: [2000, 5000, 10000, 20000], jitterRatio: 0.2,
+      onAttemptFail: (attempt, err, willRetry) => {
+        console.error(`[publish-content] begin failed (attempt ${attempt}/5): ${formatErrorChain(err)}${willRetry ? ' — retrying' : ''}`);
+      },
+    }
+  );
   log(`Session ${begin.sessionId} version ${begin.version} (expires ${begin.expiresAt})`);
 
   const allKeys = [...targetSlugs, ...sidecarKeys];