From 4cd3b1ccd409550ec03ee19483d557a4262ee138 Mon Sep 17 00:00:00 2001 From: chasprowebdev Date: Wed, 27 May 2026 14:15:41 -0400 Subject: [PATCH] fix(api): prevent proxy timeout on large-org ZIP exports --- .../evidence-export.controller.spec.ts | 3 ++ .../evidence-export.controller.ts | 8 +++ .../evidence-export.service.spec.ts | 8 ++- .../evidence-export.service.ts | 50 +++++++++++++++++++ 4 files changed, 68 insertions(+), 1 deletion(-) diff --git a/apps/api/src/tasks/evidence-export/evidence-export.controller.spec.ts b/apps/api/src/tasks/evidence-export/evidence-export.controller.spec.ts index e9f29c13e1..5c6f651ff5 100644 --- a/apps/api/src/tasks/evidence-export/evidence-export.controller.spec.ts +++ b/apps/api/src/tasks/evidence-export/evidence-export.controller.spec.ts @@ -53,6 +53,7 @@ function makeFakeResponse() { const emitter = new EventEmitter(); const res = Object.assign(emitter, { setHeader: jest.fn(), + flushHeaders: jest.fn(), status: jest.fn(function (this: unknown) { return res; }), @@ -135,6 +136,7 @@ describe('EvidenceExportController', () => { 'Content-Disposition', `attachment; filename="acme_mytask_evidence_2026-04-22.zip"`, ); + expect(res.flushHeaders).toHaveBeenCalledTimes(1); expect(archive.pipe).toHaveBeenCalledWith(res); }); @@ -273,6 +275,7 @@ describe('AuditorEvidenceExportController', () => { 'Content-Disposition', `attachment; filename="acme_all-evidence_2026-04-22.zip"`, ); + expect(res.flushHeaders).toHaveBeenCalledTimes(1); expect(archive.pipe).toHaveBeenCalledWith(res); }); }); diff --git a/apps/api/src/tasks/evidence-export/evidence-export.controller.ts b/apps/api/src/tasks/evidence-export/evidence-export.controller.ts index 3a5cd924d6..fd516e93e9 100644 --- a/apps/api/src/tasks/evidence-export/evidence-export.controller.ts +++ b/apps/api/src/tasks/evidence-export/evidence-export.controller.ts @@ -188,6 +188,11 @@ export class EvidenceExportController { 'Content-Disposition', `attachment; filename="${filename}"`, ); + // Push the response status line + headers to the wire immediately so + // upstream proxies (Cloudflare, ALB, etc.) don't apply their idle-timeout + // while we assemble the first archive entry — a TTFB > ~60s on a large + // org otherwise surfaces in the browser as `TypeError: Failed to fetch`. + res.flushHeaders(); pipeArchiveToResponse({ archive, @@ -261,6 +266,9 @@ export class AuditorEvidenceExportController { 'Content-Disposition', `attachment; filename="${filename}"`, ); + // See note on the task variant above — flush early so a slow first task + // doesn't blow past the proxy idle timeout for large orgs. + res.flushHeaders(); pipeArchiveToResponse({ archive, diff --git a/apps/api/src/tasks/evidence-export/evidence-export.service.spec.ts b/apps/api/src/tasks/evidence-export/evidence-export.service.spec.ts index ab6de156d8..568552dbdd 100644 --- a/apps/api/src/tasks/evidence-export/evidence-export.service.spec.ts +++ b/apps/api/src/tasks/evidence-export/evidence-export.service.spec.ts @@ -180,10 +180,15 @@ describe('EvidenceExportService — streaming ZIPs', () => { await mock.finalized; const paths = mock.appendCalls.map((c) => c.options.name); + // EXPORT_INFO.txt is appended first to flush a ZIP byte through proxies + // before the slow per-task data load runs. expect(paths[0]).toBe( - 'acme-corp_soc-2-access-review_evidence/00-summary.pdf', + 'acme-corp_soc-2-access-review_evidence/EXPORT_INFO.txt', ); expect(paths[1]).toBe( + 'acme-corp_soc-2-access-review_evidence/00-summary.pdf', + ); + expect(paths[2]).toBe( 'acme-corp_soc-2-access-review_evidence/01-attachments/contract.pdf', ); @@ -580,6 +585,7 @@ describe('EvidenceExportService — streaming ZIPs', () => { const paths = mock.appendCalls.map((c) => c.options.name); expect(paths).toEqual([ + 'acme-corp_soc-2-access-review_evidence/EXPORT_INFO.txt', 'acme-corp_soc-2-access-review_evidence/00-summary.pdf', ]); expect(s3Client!.send).not.toHaveBeenCalled(); diff --git a/apps/api/src/tasks/evidence-export/evidence-export.service.ts b/apps/api/src/tasks/evidence-export/evidence-export.service.ts index cc77678d57..20a6d955bc 100644 --- a/apps/api/src/tasks/evidence-export/evidence-export.service.ts +++ b/apps/api/src/tasks/evidence-export/evidence-export.service.ts @@ -33,6 +33,30 @@ const safeStringify = configureStringify({ deterministic: false, }); +function buildExportInfo( + info: + | { kind: 'task'; taskId: string } + | { + kind: 'organization'; + organizationName: string; + organizationId: string; + taskCount: number; + }, +): string { + const lines = [ + 'Evidence export', + `Started at: ${new Date().toISOString()}`, + ]; + if (info.kind === 'task') { + lines.push(`Task ID: ${info.taskId}`); + } else { + lines.push(`Organization: ${info.organizationName}`); + lines.push(`Organization ID: ${info.organizationId}`); + lines.push(`Tasks included: ${info.taskCount}`); + } + return lines.join('\n') + '\n'; +} + @Injectable() export class EvidenceExportService { private readonly logger = new Logger(EvidenceExportService.name); @@ -196,6 +220,17 @@ export class EvidenceExportService { }): Promise { const { archive, organizationId, taskId, folderName, options } = params; + // Force the archiver to emit a real ZIP byte immediately, before the + // per-task data load runs. Combined with res.flushHeaders() upstream this + // keeps the response visibly alive through any proxy idle timer. + archive.append( + Buffer.from( + buildExportInfo({ kind: 'task', taskId }), + 'utf-8', + ), + { name: `${folderName}/EXPORT_INFO.txt` }, + ); + const [headers, attachments] = await Promise.all([ getAutomationHeaders({ organizationId, taskId }), getTaskAttachments(organizationId, taskId), @@ -337,6 +372,21 @@ export class EvidenceExportService { options, } = params; + // Push the first ZIP byte out immediately so proxies see a live stream + // before the slow per-task loop begins. See populateTaskArchive note. + archive.append( + Buffer.from( + buildExportInfo({ + kind: 'organization', + organizationName, + organizationId, + taskCount: taskIds.length, + }), + 'utf-8', + ), + { name: `${orgFolder}/EXPORT_INFO.txt` }, + ); + const manifestEntries: Array<{ id: string; title: string;