diff --git a/package.json b/package.json index 67c352164..5a568ae27 100644 --- a/package.json +++ b/package.json @@ -302,13 +302,25 @@ "kind": "hana", "model": [ "db/hana" - ] + ], + "pool": { + "acquireTimeoutMillis": 5000, + "min": 0, + "max": 100, + "fifo": true + } }, "[production]": { "kind": "hana", "model": [ "db/hana" - ] + ], + "pool": { + "acquireTimeoutMillis": 5000, + "min": 0, + "max": 100, + "fifo": true + } } }, "audit-log": { diff --git a/scripts/__tests__/publish-client.test.ts b/scripts/__tests__/publish-client.test.ts index 7ad180922..08518b791 100644 --- a/scripts/__tests__/publish-client.test.ts +++ b/scripts/__tests__/publish-client.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { beginSession, appendBatch, commitSession, abortSession, fetchRemoteHashes, renderConceptsPhase } from '../lib/publish-client.js'; +import { withRetry } from '../lib/publish-retry.js'; const baseUrl = 'http://localhost:4004'; const apiKey = 'test-key'; @@ -36,6 +37,43 @@ describe('publish-client', () => { ).rejects.toMatchObject({ status: 409 }); }); + // #2286 — begin under withRetry rides out a transient HANA pool-acquire 500 + // ("Pool resource could not be acquired within 1s") instead of aborting the + // whole prod rebuild. This is the exact composition main() now uses. + it('beginSession under withRetry recovers from a transient pool-timeout 500', async () => { + const fetchMock = vi.fn() + .mockResolvedValueOnce({ + ok: false, status: 500, + text: () => Promise.resolve('{"error":"Pool resource could not be acquired within 1s"}'), + }) + .mockResolvedValueOnce({ + ok: true, status: 201, + json: () => Promise.resolve({ sessionId: 'abc', version: 8, expiresAt: '2026-09-14T00:00:00Z' }), + }); + vi.stubGlobal('fetch', fetchMock); + const out = await withRetry( + () => beginSession({ baseUrl, apiKey, trigger: 't', hugoVersion: 'v1', expectedSlugCount: 1 }), + { attempts: 5, backoffMs: [0] } + ); + expect(out).toMatchObject({ sessionId: 'abc', version: 8 }); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + // A non-transient status (e.g. 409 lock held) must NOT be retried. + it('beginSession under withRetry does not retry a permanent 409', async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: false, status: 409, text: () => Promise.resolve('lock held'), + }); + vi.stubGlobal('fetch', fetchMock); + await expect( + withRetry( + () => beginSession({ baseUrl, apiKey, trigger: 't', hugoVersion: 'v1', expectedSlugCount: 1 }), + { attempts: 5, backoffMs: [0] } + ) + ).rejects.toMatchObject({ status: 409 }); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + it('appendBatch posts files/metadata/bodyTexts and returns server result', async () => { const fetchMock = vi.fn().mockResolvedValue({ ok: true, status: 202, diff --git a/scripts/publish-content.ts b/scripts/publish-content.ts index f08fbfa45..ce2c44945 100644 --- a/scripts/publish-content.ts +++ b/scripts/publish-content.ts @@ -1106,7 +1106,20 @@ async function main() { let remoteHashes: Record = {}; if (mode !== 'force') { log(`Fetching remote hashes from ${opts.baseUrl}/content/hashes...`); - try { remoteHashes = await fetchRemoteHashes({ baseUrl: opts.baseUrl, apiKey: opts.apiKey }); } + // #2286 — retry transient 500s (HANA pool-acquire timeout) so a momentary + // load spike during delta detection doesn't fatally abort the rebuild + // before a session is even opened. 503 already returns {} inside the client. + try { + remoteHashes = await withRetry( + () => fetchRemoteHashes({ baseUrl: opts.baseUrl, apiKey: opts.apiKey }), + { + attempts: 4, backoffMs: [1000, 3000, 6000], jitterRatio: 0.2, + onAttemptFail: (attempt, err, willRetry) => { + console.error(`[publish-content] /content/hashes fetch failed (attempt ${attempt}/4): ${formatErrorChain(err)}${willRetry ? ' — retrying' : ''}`); + }, + } + ); + } catch (err) { console.error(`Cannot reach ${opts.baseUrl}/content/hashes: ${formatErrorChain(err)}`); process.exit(1); @@ -1201,11 +1214,25 @@ async function main() { // small and the server happily accepts them mixed with regular slugs). const sidecarKeys = await collectSidecars(opts.hugoDir, payload, log, channel); - const begin = await beginSession({ - baseUrl: opts.baseUrl, apiKey: opts.apiKey, - trigger: opts.trigger, hugoVersion: opts.hugoVersion, expectedSlugCount: targetSlugs.length, - initiator: opts.initiator, - }); + // #2286 — begin was the ONE publish HTTP call in the happy path with neither + // withRetry nor a try/catch, so a transient HANA pool-acquire 500 ("Pool + // resource could not be acquired within 1s") aborted the whole rebuild via + // the top-level Fatal catch. Wrap it like commit: 500/502/503/504 are already + // classified transient in publish-retry.ts; jitter de-syncs the three DevRel + // projects so a briefly-saturated pool isn't re-hit in lockstep. + const begin = await withRetry( + () => beginSession({ + baseUrl: opts.baseUrl, apiKey: opts.apiKey, + trigger: opts.trigger, hugoVersion: opts.hugoVersion, expectedSlugCount: targetSlugs.length, + initiator: opts.initiator, + }), + { + attempts: 5, backoffMs: [2000, 5000, 10000, 20000], jitterRatio: 0.2, + onAttemptFail: (attempt, err, willRetry) => { + console.error(`[publish-content] begin failed (attempt ${attempt}/5): ${formatErrorChain(err)}${willRetry ? ' — retrying' : ''}`); + }, + } + ); log(`Session ${begin.sessionId} version ${begin.version} (expires ${begin.expiresAt})`); const allKeys = [...targetSlugs, ...sidecarKeys];