fix(automation): region-aware run-history compaction keeps loop containers + early failures (#3234)

.changeset/region-aware-history-compaction.md

@@ -0,0 +1,23 @@
+---
+'@objectstack/service-automation': minor
+---
+
+fix(automation): region-aware run-history compaction keeps loop containers + early failures (#3234)
+
+`compactStepsForHistory` bounded a terminal run's persisted step log to the last
+`MAX_PERSISTED_HISTORY_STEPS` entries with a plain tail-slice. With the ADR-0031
+structured-region step logs (#1505) a single `loop` can emit
+`iterations × body-steps` entries, so the tail-slice dropped the
+`loop`/`parallel`/`try_catch` **container** step (it precedes all its body steps)
+and every early iteration — leaving `getRun`/`listRuns` (after a process restart
+or ring-buffer eviction) with body steps the Runs surface could no longer nest,
+and silently hiding an early failure.
+
+Compaction is now region-aware (new exported `compactStepLogForHistory`): over
+budget it keeps the run's structural backbone — every top-level step (including
+the region container steps) and every failure, each pulled in with its ancestor
+container chain — plus the most recent body steps, order-preserving and
+hard-capped at `max` so `steps_json` stays bounded (#2585). Every retained body
+step keeps its enclosing container(s), so the compacted log never contains an
+orphan and the observability surface's per-iteration / per-region nesting still
+reconstructs.

content/docs/automation/flows.mdx

@@ -445,10 +445,12 @@ GET /api/v1/automation/{flow}/runs/{runId}/screen  # re-fetch a paused screen
 ```
 
 Each run's `steps[]` records every executed node — including loop iterations,
-parallel branch bodies, and try/catch region steps — and the Studio flow
-designer surfaces the same data in its **Runs** side panel. Run history is an
-in-memory ring buffer (the durable rows in `sys_automation_run` hold only
-*live pauses*), so history starts fresh on each boot.
+parallel branch bodies, and try/catch region steps — which the Studio flow
+designer surfaces, nested by iteration / branch / handler, in its **Runs** side
+panel. Recent runs are held in an in-memory ring buffer; terminal runs
+(completed / failed) are also mirrored to `sys_automation_run` as durable
+history with a bounded step log, so `listRuns` / `getRun` still report a run's
+status, steps, and failure reason after a restart or ring-buffer eviction.
 
 ## Edges
 

packages/services/service-automation/src/engine.ts

@@ -352,6 +352,88 @@ export interface StepLogEntry {
     regionKind?: string;
 }
 
+/**
+ * Compact a run's step log for durable history (#2585, #3234).
+ *
+ * Under {@link MAX_PERSISTED_HISTORY_STEPS} the log is persisted whole (only
+ * `error.stack` is dropped — the code/message pair is the designer-facing "why";
+ * stacks bloat rows without aiding the Runs surface).
+ *
+ * Over budget — a long `loop` alone can emit `iterations × body-steps` entries —
+ * a plain tail-slice would drop the `loop`/`parallel`/`try_catch` **container**
+ * step (it precedes all its body steps) and every early iteration, leaving the
+ * Runs surface with body steps it can no longer nest and, worse, silently hiding
+ * an early failure. Instead select a bounded, order-preserving subset that keeps
+ * the run's structural backbone:
+ *
+ *  1. Every **top-level** step (`parentNodeId === undefined`) — `start`/`end`,
+ *     main-graph nodes, and the region container steps. Bounded by the flow's
+ *     static node count, not by loop iterations.
+ *  2. Every **failure**, wherever it occurred — the reason the run is worth
+ *     keeping — each pulled in with its ancestor container chain for context.
+ *  3. The most **recent** body steps (the tail shows what the run was doing when
+ *     it ended), each also pulled in with its ancestor chain.
+ *
+ * Every retained body step therefore keeps its enclosing container(s), so the
+ * compacted log never contains an orphan and the observability surface's
+ * per-iteration / per-region nesting still reconstructs; the result is
+ * hard-capped at `max` so `steps_json` stays bounded (#2585).
+ */
+export function compactStepLogForHistory(
+    steps: StepLogEntry[],
+    max: number = MAX_PERSISTED_HISTORY_STEPS,
+): StepLogEntry[] {
+    const strip = (s: StepLogEntry): StepLogEntry =>
+        s.error?.stack ? { ...s, error: { code: s.error.code, message: s.error.message } } : s;
+
+    if (steps.length <= max) return steps.map(strip);
+
+    // Nearest preceding container-instance index for each step (its parent), or
+    // -1 when top-level / its container is not in the log. The flat log is
+    // pre-order, so a step's container is the closest earlier step whose nodeId
+    // equals this step's parentNodeId (the same instance `buildStepTree` nests
+    // under). O(n) via a running last-seen-index map.
+    const parentIdx = new Array<number>(steps.length).fill(-1);
+    const lastSeen = new Map<string, number>();
+    for (let i = 0; i < steps.length; i++) {
+        const pid = steps[i].parentNodeId;
+        if (pid !== undefined) parentIdx[i] = lastSeen.get(pid) ?? -1;
+        lastSeen.set(steps[i].nodeId, i);
+    }
+    // Indices of `i`'s ancestor chain (i first) not already selected in `into`.
+    const missingChain = (i: number, into: Set<number>): number[] => {
+        const chain: number[] = [];
+        for (let k = i; k >= 0 && !into.has(k); k = parentIdx[k]) chain.push(k);
+        return chain;
+    };
+
+    const keep = new Set<number>();
+    // (1) + (2): structural backbone + every failure, each with its container chain.
+    for (let i = 0; i < steps.length; i++) {
+        if (steps[i].parentNodeId === undefined || steps[i].status === 'failure') {
+            for (const k of missingChain(i, keep)) keep.add(k);
+        }
+    }
+
+    const emit = (): StepLogEntry[] => {
+        let idx = [...keep].sort((a, b) => a - b);
+        // The backbone alone can exceed the cap on a very large flow — keep the
+        // most recent `max` selected steps so the row stays bounded.
+        if (idx.length > max) idx = idx.slice(idx.length - max);
+        return idx.map((i) => strip(steps[i]));
+    };
+    if (keep.size >= max) return emit();
+
+    // (3): fill the remaining budget with the most recent body steps, each with
+    // its ancestor chain so no retained body step is left orphaned.
+    for (let i = steps.length - 1; i >= 0 && keep.size < max; i--) {
+        if (keep.has(i)) continue;
+        const chain = missingChain(i, keep);
+        if (keep.size + chain.length <= max) for (const k of chain) keep.add(k);
+    }
+    return emit();
+}
+
 /**
  * Internal execution log entry — compatible with ExecutionLog from spec.
  */
@@ -2049,16 +2131,15 @@ export class AutomationEngine implements IAutomationService {
     }
 
     /**
-     * Compact a run's step log for durable history: keep the newest
-     * {@link MAX_PERSISTED_HISTORY_STEPS} steps (the tail carries the failure)
-     * and drop `error.stack` (the code/message pair is the designer-facing
-     * "why"; stacks bloat rows without aiding the Runs surface). Bounds the
-     * `steps_json` column so history rows stay cheap under retention (#2585).
+     * Compact a run's step log for durable history. Delegates to the region-aware
+     * {@link compactStepLogForHistory} (#3234): under {@link MAX_PERSISTED_HISTORY_STEPS}
+     * the log is kept whole (stacks stripped); over budget it keeps the run's
+     * structural backbone (top-level + container steps + every failure) plus the
+     * most recent body steps, so a long loop's container survives and the Runs
+     * surface can still nest what it retains.
      */
     private compactStepsForHistory(steps: StepLogEntry[]): StepLogEntry[] {
-        return steps.slice(-MAX_PERSISTED_HISTORY_STEPS).map((s) =>
-            s.error?.stack ? { ...s, error: { code: s.error.code, message: s.error.message } } : s,
-        );
+        return compactStepLogForHistory(steps, MAX_PERSISTED_HISTORY_STEPS);
     }
 
     /**

packages/services/service-automation/src/run-history.test.ts

@@ -7,7 +7,8 @@
 // never break the run that produced it.
 
 import { describe, it, expect } from 'vitest';
-import { AutomationEngine } from './engine.js';
+import { AutomationEngine, compactStepLogForHistory, MAX_PERSISTED_HISTORY_STEPS } from './engine.js';
+import type { StepLogEntry } from './engine.js';
 import { InMemorySuspendedRunStore } from './suspended-run-store.js';
 import type { AutomationContext } from '@objectstack/spec/contracts';
 
@@ -150,3 +151,108 @@ describe('automation run history (durable observability)', () => {
         await flush();
     });
 });
+
+// ── #3234: region-aware durable-history compaction ──────────────────────────
+
+const AT = '2026-07-18T00:00:00.000Z';
+
+/** A run log for a top-level `loop` over `iterations` items (one body step each,
+ *  optionally failing one). Mirrors the engine's folded region log (#1479):
+ *  container step top-level, body steps tagged with `parentNodeId`/`iteration`. */
+function loopRunLog(iterations: number, opts: { failAt?: number } = {}): StepLogEntry[] {
+    const steps: StepLogEntry[] = [
+        { nodeId: 'start', nodeType: 'start', status: 'success', startedAt: AT },
+        { nodeId: 'each', nodeType: 'loop', status: 'success', startedAt: AT },
+    ];
+    for (let i = 0; i < iterations; i++) {
+        const fail = opts.failAt === i;
+        steps.push({
+            nodeId: 'body', nodeType: 'http', status: fail ? 'failure' : 'success', startedAt: AT,
+            parentNodeId: 'each', iteration: i, regionKind: 'loop-body',
+            ...(fail ? { error: { code: 'E_HTTP', message: 'boom', stack: 'at foo\nat bar' } } : {}),
+        });
+    }
+    steps.push({ nodeId: 'end', nodeType: 'end', status: 'success', startedAt: AT });
+    return steps;
+}
+
+/** Every retained body step has an earlier retained step whose nodeId is its
+ *  parentNodeId — i.e. the compacted log has no orphan, so the observability
+ *  surface can still nest what survived. */
+function hasNoOrphans(steps: StepLogEntry[]): boolean {
+    for (let i = 0; i < steps.length; i++) {
+        const pid = steps[i].parentNodeId;
+        if (pid === undefined) continue;
+        let found = false;
+        for (let j = i - 1; j >= 0; j--) if (steps[j].nodeId === pid) { found = true; break; }
+        if (!found) return false;
+    }
+    return true;
+}
+
+describe('compactStepLogForHistory (#3234 region-aware history compaction)', () => {
+    it('keeps a small log whole and strips error stacks', () => {
+        const log = loopRunLog(3, { failAt: 1 });
+        const out = compactStepLogForHistory(log, MAX_PERSISTED_HISTORY_STEPS);
+        expect(out).toHaveLength(log.length);
+        const failed = out.find((s) => s.status === 'failure');
+        expect(failed?.error?.message).toBe('boom');
+        expect(failed?.error?.stack).toBeUndefined();
+    });
+
+    it('caps an over-budget loop at `max`', () => {
+        const out = compactStepLogForHistory(loopRunLog(500), 200);
+        expect(out.length).toBeLessThanOrEqual(200);
+        expect(out.length).toBeGreaterThan(0);
+    });
+
+    it('retains the loop CONTAINER + top-level backbone so body steps never orphan (the fix)', () => {
+        // Pre-fix, a plain tail-slice dropped `start` + the `each` container
+        // (they precede 500 body steps), leaving the Runs surface unable to nest.
+        const out = compactStepLogForHistory(loopRunLog(500), 200);
+        expect(out.some((s) => s.nodeId === 'each' && s.nodeType === 'loop')).toBe(true);
+        expect(out.some((s) => s.nodeId === 'start')).toBe(true);
+        expect(out.some((s) => s.nodeId === 'end')).toBe(true);
+        expect(hasNoOrphans(out)).toBe(true);
+    });
+
+    it('keeps the most recent iterations (the tail)', () => {
+        const out = compactStepLogForHistory(loopRunLog(500), 200);
+        const iters = out.filter((s) => s.regionKind === 'loop-body').map((s) => s.iteration!);
+        expect(Math.max(...iters)).toBe(499);
+    });
+
+    it('keeps an EARLY failure even though it is not in the tail', () => {
+        // A plain tail-slice would silently drop a failure at iteration 2 of 500.
+        const out = compactStepLogForHistory(loopRunLog(500, { failAt: 2 }), 200);
+        const failed = out.find((s) => s.status === 'failure');
+        expect(failed?.iteration).toBe(2);
+        expect(failed?.error?.stack).toBeUndefined();
+        expect(hasNoOrphans(out)).toBe(true);
+    });
+
+    it('preserves original execution order', () => {
+        const out = compactStepLogForHistory(loopRunLog(500, { failAt: 2 }), 200);
+        const iters = out.filter((s) => s.regionKind === 'loop-body').map((s) => s.iteration!);
+        expect(iters).toEqual([...iters].sort((a, b) => a - b));
+    });
+
+    it('keeps a nested inner container for its retained body steps (no orphan)', () => {
+        // outer loop (top-level) → inner loop (per outer iteration) → body steps.
+        const steps: StepLogEntry[] = [
+            { nodeId: 'start', nodeType: 'start', status: 'success', startedAt: AT },
+            { nodeId: 'outer', nodeType: 'loop', status: 'success', startedAt: AT },
+        ];
+        for (let o = 0; o < 60; o++) {
+            steps.push({ nodeId: 'inner', nodeType: 'loop', status: 'success', startedAt: AT, parentNodeId: 'outer', iteration: o, regionKind: 'loop-body' });
+            for (let n = 0; n < 5; n++) {
+                steps.push({ nodeId: 'body', nodeType: 'http', status: 'success', startedAt: AT, parentNodeId: 'inner', iteration: n, regionKind: 'loop-body' });
+            }
+        }
+        steps.push({ nodeId: 'end', nodeType: 'end', status: 'success', startedAt: AT });
+        const out = compactStepLogForHistory(steps, 200);
+        expect(out.length).toBeLessThanOrEqual(200);
+        expect(hasNoOrphans(out)).toBe(true);
+        expect(out.some((s) => s.nodeId === 'outer')).toBe(true);
+    });
+});