[dev] [Marfuen] mariano/fix-evidence-export-oom-streaming

apps/api/src/tasks/evidence-export/evidence-data-loader.ts

@@ -197,22 +197,38 @@ export async function loadFullAutomation({
   taskId: string;
   header: NormalizedAutomation;
 }): Promise<NormalizedAutomation> {
+  const runs: NormalizedEvidenceRun[] = [];
+  for await (const batch of streamAutomationRuns({ taskId, header })) {
+    runs.push(...batch);
+  }
+  return { ...header, runs };
+}
+
+// Yields runs in batches so callers can process incrementally without
+// accumulating all runs in memory. Each batch is GC-eligible after processing.
+export async function* streamAutomationRuns({
+  taskId,
+  header,
+}: {
+  taskId: string;
+  header: NormalizedAutomation;
+}): AsyncGenerator<NormalizedEvidenceRun[]> {
   if (header.type === 'app_automation' && header.checkId) {
-    return loadAppAutomationRuns(taskId, header);
+    yield* streamAppRuns(taskId, header.checkId);
+  } else {
+    yield* streamCustomRuns(taskId, header.id);
   }
-  return loadCustomAutomationRuns(taskId, header);
 }
 
-async function loadAppAutomationRuns(
+async function* streamAppRuns(
   taskId: string,
-  header: NormalizedAutomation,
-): Promise<NormalizedAutomation> {
-  const runs: NormalizedEvidenceRun[] = [];
+  checkId: string,
+): AsyncGenerator<NormalizedEvidenceRun[]> {
   let cursor: { id: string } | undefined;
 
   for (;;) {
     const batch = await db.integrationCheckRun.findMany({
-      where: { taskId, checkId: header.checkId },
+      where: { taskId, checkId },
       include: {
         results: true,
         connection: { include: { provider: true } },
@@ -224,28 +240,25 @@ async function loadAppAutomationRuns(
 
     if (batch.length === 0) break;
 
-    for (const run of batch) {
-      runs.push(normalizeAppAutomationRun(toAppAutomationRun(run)));
-    }
+    yield batch.map((run) =>
+      normalizeAppAutomationRun(toAppAutomationRun(run)),
+    );
 
     if (batch.length < RUN_BATCH_SIZE) break;
     cursor = { id: batch[batch.length - 1].id };
   }
-
-  return { ...header, runs };
 }
 
-async function loadCustomAutomationRuns(
+async function* streamCustomRuns(
   taskId: string,
-  header: NormalizedAutomation,
-): Promise<NormalizedAutomation> {
-  const runs: NormalizedEvidenceRun[] = [];
+  automationId: string,
+): AsyncGenerator<NormalizedEvidenceRun[]> {
   let cursor: { id: string } | undefined;
 
   for (;;) {
     const batch = await db.evidenceAutomationRun.findMany({
       where: {
-        evidenceAutomation: { id: header.id, taskId },
+        evidenceAutomation: { id: automationId, taskId },
         version: { not: null },
       },
       include: {
@@ -258,15 +271,13 @@ async function loadCustomAutomationRuns(
 
     if (batch.length === 0) break;
 
-    for (const run of batch) {
-      runs.push(normalizeCustomAutomationRun(toCustomAutomationRun(run)));
-    }
+    yield batch.map((run) =>
+      normalizeCustomAutomationRun(toCustomAutomationRun(run)),
+    );
 
     if (batch.length < RUN_BATCH_SIZE) break;
     cursor = { id: batch[batch.length - 1].id };
   }
-
-  return { ...header, runs };
 }
 
 // Prisma result → normalizer interface mappers (single source of truth).

apps/api/src/tasks/evidence-export/evidence-export.controller.spec.ts

@@ -1,5 +1,10 @@
 // Mocks must be declared before any SUT import so guards' transitive deps
 // (Prisma, better-auth) don't instantiate in Jest.
+const mockTrigger = jest.fn();
+jest.mock('@trigger.dev/sdk', () => ({
+  tasks: { trigger: mockTrigger },
+}));
+
 jest.mock('@db', () => ({
   ...jest.requireActual('@prisma/client'),
   db: {},
@@ -223,18 +228,15 @@ describe('EvidenceExportController', () => {
 
 describe('AuditorEvidenceExportController', () => {
   let controller: AuditorEvidenceExportController;
-  let service: jest.Mocked<
-    Pick<EvidenceExportService, 'streamOrganizationEvidenceZip'>
-  >;
 
   beforeEach(async () => {
-    service = {
-      streamOrganizationEvidenceZip: jest.fn(),
-    };
+    mockTrigger.mockReset().mockResolvedValue({
+      id: 'run_123',
+      publicAccessToken: 'tok_abc',
+    });
 
     const moduleRef = await Test.createTestingModule({
       controllers: [AuditorEvidenceExportController],
-      providers: [{ provide: EvidenceExportService, useValue: service }],
     })
       .overrideGuard(HybridAuthGuard)
       .useValue({ canActivate: () => true })
@@ -245,34 +247,16 @@ describe('AuditorEvidenceExportController', () => {
     controller = moduleRef.get(AuditorEvidenceExportController);
   });
 
-  it('pipes the org-wide archive to response with correct headers', async () => {
-    const archive = makeFakeArchive();
-    service.streamOrganizationEvidenceZip.mockResolvedValue({
-      archive: archive as unknown as import('archiver').Archiver,
-      filename: 'acme_all-evidence_2026-04-22.zip',
-    });
-    const req = makeFakeRequest();
-    const res = makeFakeResponse();
+  it('triggers a background task and returns runId + token', async () => {
+    const result = await controller.exportAllEvidence('org_1', 'true');
 
-    await controller.exportAllEvidence(
-      'org_1',
-      'true',
-      req as unknown as import('express').Request,
-      res as unknown as import('express').Response,
-    );
-
-    expect(service.streamOrganizationEvidenceZip).toHaveBeenCalledWith(
-      'org_1',
-      { includeRawJson: true },
-    );
-    expect(res.setHeader).toHaveBeenCalledWith(
-      'Content-Type',
-      'application/zip',
+    expect(mockTrigger).toHaveBeenCalledWith(
+      'export-organization-evidence',
+      { organizationId: 'org_1', includeJson: true },
     );
-    expect(res.setHeader).toHaveBeenCalledWith(
-      'Content-Disposition',
-      `attachment; filename="acme_all-evidence_2026-04-22.zip"`,
-    );
-    expect(archive.pipe).toHaveBeenCalledWith(res);
+    expect(result).toEqual({
+      runId: 'run_123',
+      publicAccessToken: 'tok_abc',
+    });
   });
 });

apps/api/src/tasks/evidence-export/evidence-export.controller.ts

@@ -1,6 +1,7 @@
 import {
   Controller,
   Get,
+  Post,
   Param,
   Query,
   Req,
@@ -16,6 +17,7 @@ import {
   ApiSecurity,
   ApiTags,
 } from '@nestjs/swagger';
+import { tasks } from '@trigger.dev/sdk';
 import type { Request, Response } from 'express';
 import type { Archiver } from 'archiver';
 import { AuditRead } from '../../audit/skip-audit-log.decorator';
@@ -200,7 +202,8 @@ export class EvidenceExportController {
 }
 
 /**
- * Auditor-only controller for bulk evidence export
+ * Auditor-only controller for bulk evidence export.
+ * The heavy work runs in a Trigger.dev background task to avoid OOM in the API.
  */
 @ApiTags('Evidence Export (Auditor)')
 @Controller({ path: 'evidence-export', version: '1' })
@@ -209,18 +212,13 @@ export class EvidenceExportController {
 export class AuditorEvidenceExportController {
   private readonly logger = new Logger(AuditorEvidenceExportController.name);
 
-  constructor(private readonly evidenceExportService: EvidenceExportService) {}
-
-  /**
-   * Export all evidence for the organization (auditor only)
-   */
-  @Get('all')
+  @Post('all')
   @RequirePermission('evidence', 'read')
   @AuditRead()
   @ApiOperation({
-    summary: 'Export all organization evidence as ZIP (Auditor only)',
+    summary: 'Trigger bulk evidence export (Auditor only)',
     description:
-      'Generate and download a ZIP file containing all automation evidence across all tasks. Only accessible by auditors.',
+      'Starts a background job that generates a ZIP of all evidence. Returns a run ID for progress tracking.',
   })
   @ApiQuery({
     name: 'includeJson',
@@ -229,46 +227,27 @@ export class AuditorEvidenceExportController {
     required: false,
   })
   @ApiResponse({
-    status: 200,
-    description: 'ZIP file generated successfully',
-    content: {
-      'application/zip': {},
-    },
-  })
-  @ApiResponse({
-    status: 403,
-    description: 'Access denied - Auditor role required',
+    status: 201,
+    description: 'Export job started',
   })
   async exportAllEvidence(
     @OrganizationId() organizationId: string,
     @Query('includeJson') includeJson: string,
-    @Req() req: Request,
-    @Res() res: Response,
   ) {
-    this.logger.log('Auditor exporting all evidence', {
+    this.logger.log('Auditor triggering bulk evidence export', {
       organizationId,
       includeJson: includeJson === 'true',
     });
 
-    const { archive, filename } =
-      await this.evidenceExportService.streamOrganizationEvidenceZip(
-        organizationId,
-        { includeRawJson: includeJson === 'true' },
-      );
-
-    res.setHeader('Content-Type', 'application/zip');
-    res.setHeader(
-      'Content-Disposition',
-      `attachment; filename="${filename}"`,
-    );
-
-    pipeArchiveToResponse({
-      archive,
-      req,
-      res,
-      logger: this.logger,
-      tag: `org ${organizationId}`,
+    const handle = await tasks.trigger('export-organization-evidence', {
+      organizationId,
+      includeJson: includeJson === 'true',
     });
+
+    return {
+      runId: handle.id,
+      publicAccessToken: handle.publicAccessToken,
+    };
   }
 }
 

apps/api/src/tasks/evidence-export/evidence-export.service.spec.ts

@@ -87,6 +87,18 @@ jest.mock('@db', () => ({
 jest.mock('./evidence-pdf-generator', () => ({
   generateTaskSummaryPDF: jest.fn(() => Buffer.from('SUMMARY-PDF')),
   generateAutomationPDF: jest.fn(() => Buffer.from('AUTOMATION-PDF')),
+  generateAutomationPDFFromStream: jest.fn(
+    async (
+      _header: unknown,
+      _context: unknown,
+      runBatches: AsyncIterable<unknown>,
+    ) => {
+      for await (const _batch of runBatches) {
+        /* drain so underlying DB queries execute */
+      }
+      return Buffer.from('AUTOMATION-PDF');
+    },
+  ),
   sanitizeFilename: (name: string) =>
     name
       .toLowerCase()