From 279a77f5124da1ed320f383b1f39f1e892928d55 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Fri, 21 Aug 2026 13:40:31 -0400 Subject: [PATCH 1/3] Report the prerender server's own heap The prerender server is long-lived and holds memory from renders it has already finished, but nothing it emits describes its own heap. The only place that memory has been visible is the crash trace V8 prints on the way out, which means the state leading up to a crash has to be reconstructed afterwards from how long each task survived. Emit heap used, total, limit, rss and external on a fixed interval, and serve the same numbers from the health endpoint. The limit is worth carrying explicitly: V8 sizes its default old-space ceiling from visible memory rather than from the task's allocation, so what a running process actually took cannot be derived from the ECS memory setting. The line goes out before the queue snapshot's quiet-path return, and separately from it. The heap keeps growing while the pool sits idle, so gating it on current load would hide the growth that has no queue behind it to explain it, and keeping it off the snapshot line leaves existing greps of that line alone. The dashboard charts the highest instance rather than a breakdown: these logs carry no per-task label, and the instance nearest the limit is the one worth watching. Co-Authored-By: Claude Opus 5 (1M context) --- .../service-prerender-server.json | 128 +++++++++++++++++- .../realm-server/prerender/heap-telemetry.ts | 54 ++++++++ .../realm-server/prerender/prerender-app.ts | 13 +- .../realm-server/prerender/prerenderer.ts | 12 ++ 4 files changed, 203 insertions(+), 4 deletions(-) create mode 100644 packages/realm-server/prerender/heap-telemetry.ts diff --git a/packages/observability/grafanactl/resources/dashboards/boxel-status/service-prerender-server.json b/packages/observability/grafanactl/resources/dashboards/boxel-status/service-prerender-server.json index 693c125713a..e561ac13401 100644 --- a/packages/observability/grafanactl/resources/dashboards/boxel-status/service-prerender-server.json +++ b/packages/observability/grafanactl/resources/dashboards/boxel-status/service-prerender-server.json @@ -24,7 +24,7 @@ } ] }, - "description": "Per-service deep-dive for the Prerender Server ECS service. Currently stub-only — shows filtered logs from Loki. CloudWatch CPU / memory / running-task-count panels are TBD and will land when ECS cluster naming is standardized in observability config.", + "description": "Per-service deep-dive for the Prerender Server ECS service. Request rate and Node heap come from the service's own log lines via Loki, alongside filtered logs. CloudWatch CPU and running-task-count panels are TBD and will land when ECS cluster naming is standardized in observability config.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, @@ -199,6 +199,130 @@ "title": "Request rate (total / 4xx / 5xx)", "type": "timeseries" }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "description": "The prerender server's own Node heap, from the `prerender-heap` log line each instance emits on a fixed interval. Charted as the highest value across instances in each window, because Loki carries no per-task label here and the instance closest to the limit is the one that matters.\n\n`limit` is the effective `--max-old-space-size`. V8 derives its default from visible memory rather than from the ECS memory setting, so this is the only place the value a running task actually took is readable. Reaching it is fatal: the process dies with `Reached heap limit`, ECS replaces the task, and the surviving instances inherit its realm affinities and serve renders from cold tabs.\n\n`used` climbing steadily between task restarts \u2014 rather than sawtoothing under load and settling when idle \u2014 means memory is being retained, not merely in use. `external` covers strings and buffers held outside the JS heap, where response serialisation holds memory that `used` does not account for.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "MB", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decmbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "max_over_time({service=\"prerender\"} |~ \"prerender-heap\" | regexp \"heapUsedMB=(?P[0-9]+)\" | unwrap heapUsedMB [$__interval])", + "legendFormat": "used", + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "max_over_time({service=\"prerender\"} |~ \"prerender-heap\" | regexp \"heapLimitMB=(?P[0-9]+)\" | unwrap heapLimitMB [$__interval])", + "legendFormat": "limit", + "queryType": "range", + "refId": "B" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "max_over_time({service=\"prerender\"} |~ \"prerender-heap\" | regexp \"rssMB=(?P[0-9]+)\" | unwrap rssMB [$__interval])", + "legendFormat": "rss", + "queryType": "range", + "refId": "C" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "max_over_time({service=\"prerender\"} |~ \"prerender-heap\" | regexp \"externalMB=(?P[0-9]+)\" | unwrap externalMB [$__interval])", + "legendFormat": "external", + "queryType": "range", + "refId": "D" + } + ], + "title": "Node heap (highest instance)", + "type": "timeseries" + }, { "datasource": { "type": "loki", @@ -209,7 +333,7 @@ "h": 16, "w": 24, "x": 0, - "y": 8 + "y": 16 }, "id": 1, "options": { diff --git a/packages/realm-server/prerender/heap-telemetry.ts b/packages/realm-server/prerender/heap-telemetry.ts new file mode 100644 index 00000000000..889e2965950 --- /dev/null +++ b/packages/realm-server/prerender/heap-telemetry.ts @@ -0,0 +1,54 @@ +import { getHeapStatistics } from 'node:v8'; + +// The prerender server's own Node heap, as distinct from the browser-side +// JS heap the render paths already report (`jsHeapUsedMB` on a paused +// stack capture). A prerender server is long-lived and holds memory from +// work it has already finished, so these are the numbers that say whether +// a task is heading for `FATAL ERROR: Reached heap limit` — a crash that +// takes the task's warm tabs with it and leaves the surviving instances +// serving cold ones. +export interface HeapTelemetry { + heapUsedMB: number; + heapTotalMB: number; + heapLimitMB: number; + rssMB: number; + externalMB: number; +} + +function mb(bytes: number): number { + return Math.round(bytes / 1024 / 1024); +} + +export function heapTelemetry(): HeapTelemetry { + let heap = getHeapStatistics(); + let mem = process.memoryUsage(); + return { + heapUsedMB: mb(heap.used_heap_size), + heapTotalMB: mb(heap.total_heap_size), + // V8 sizes its default old-space limit from visible memory rather + // than from the task's allocation, so this is not derivable from the + // ECS memory setting — reporting it makes the effective + // `--max-old-space-size` readable from a running task instead of + // inferred from a task definition plus a Node version. + heapLimitMB: mb(heap.heap_size_limit), + rssMB: mb(mem.rss), + // Strings and buffers held outside the JS heap. Worth its own field + // rather than folding into rss: heap exhaustion here has surfaced + // during response serialisation, and memory retained that way shows + // up here rather than in `heapUsedMB`. + externalMB: mb(mem.external), + }; +} + +// Rendered as `key=value` pairs to match the surrounding prerender log +// lines, so the same greps work and a Loki query can pull any single +// field out with one `regexp` stage. +export function formatHeapTelemetry(telemetry: HeapTelemetry): string { + return ( + `heapUsedMB=${telemetry.heapUsedMB} ` + + `heapTotalMB=${telemetry.heapTotalMB} ` + + `heapLimitMB=${telemetry.heapLimitMB} ` + + `rssMB=${telemetry.rssMB} ` + + `externalMB=${telemetry.externalMB}` + ); +} diff --git a/packages/realm-server/prerender/prerender-app.ts b/packages/realm-server/prerender/prerender-app.ts index 263c99dc205..99e7a1cfdce 100644 --- a/packages/realm-server/prerender/prerender-app.ts +++ b/packages/realm-server/prerender/prerender-app.ts @@ -22,6 +22,7 @@ import { import { Prerenderer } from './index.ts'; import type { Timings } from './render-runner.ts'; import { resolvePrerenderManagerURL } from './config.ts'; +import { heapTelemetry } from './heap-telemetry.ts'; import { PRERENDER_HOST_SHELL_HASH_HEADER, PRERENDER_JOB_ID_HEADER, @@ -122,11 +123,19 @@ export function buildPrerenderApp(options: { PRERENDER_SERVER_STATUS_DRAINING, ); ctxt.set('Content-Type', 'application/json'); - ctxt.body = JSON.stringify({ ready: false, draining: true }); + ctxt.body = JSON.stringify({ + ready: false, + draining: true, + memory: heapTelemetry(), + }); return; } ctxt.set('Content-Type', 'application/json'); - ctxt.body = JSON.stringify({ ready: true }); + // `memory` makes a single task's heap readable on demand — including + // `heapLimitMB`, which is the only way to confirm from outside what + // `--max-old-space-size` a running process actually took. The Docker + // HEALTHCHECK discards this body, so the extra fields cost it nothing. + ctxt.body = JSON.stringify({ ready: true, memory: heapTelemetry() }); ctxt.status = 200; }); diff --git a/packages/realm-server/prerender/prerenderer.ts b/packages/realm-server/prerender/prerenderer.ts index 6227c9ece53..55bac83349d 100644 --- a/packages/realm-server/prerender/prerenderer.ts +++ b/packages/realm-server/prerender/prerenderer.ts @@ -21,6 +21,7 @@ import { toAffinityKey } from './affinity.ts'; import { PrerenderCancelledError, throwIfAborted } from './prerender-cancel.ts'; import { AffinityActivityTracker } from './affinity-activity.ts'; import { AsyncSemaphore } from './async-semaphore.ts'; +import { formatHeapTelemetry, heapTelemetry } from './heap-telemetry.ts'; import { type BatchOwner, computeBatchClearCacheGate, @@ -989,6 +990,17 @@ export class Prerenderer { return; } this.#queueSnapshotInterval = setInterval(() => { + // Emitted before the quiet-path return below, and on its own line + // rather than appended to the snapshot. The heap holds memory from + // work already finished and keeps growing with the pool idle, so + // gating this on current load would hide exactly the growth that + // has no queue behind it to explain it. Kept separate from the + // snapshot line so existing greps of that line are unaffected. + try { + log.info('prerender-heap %s', formatHeapTelemetry(heapTelemetry())); + } catch (e) { + log.warn('heap telemetry log failed:', e); + } try { let snap = this.#pagePool.getQueueDepthSnapshot(); if (snap.affinities.length === 0 && snap.totalPending === 0) { From 26de2ef5a591455213e5b53a4cf56f0202284799 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Fri, 21 Aug 2026 13:58:21 -0400 Subject: [PATCH 2/3] Assert the liveness payload's heap fields The liveness test compared the whole response body against `{ ready: true }`, so reporting heap alongside it fails that assertion outright. Assert `ready` on its own and check the heap block beside it. The field names are pinned deliberately: they are read by name from outside this codebase, so a rename would leave the heap dashboard's panels empty without failing anywhere. Values vary per run, so beyond the shape the only claim worth making is that the heap in use is positive and within the limit. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/prerender-server-test.ts | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/realm-server/tests/prerender-server-test.ts b/packages/realm-server/tests/prerender-server-test.ts index b6de336b071..8da7b28295c 100644 --- a/packages/realm-server/tests/prerender-server-test.ts +++ b/packages/realm-server/tests/prerender-server-test.ts @@ -130,7 +130,28 @@ module(basename(import.meta.filename), function () { test('liveness', async function (assert) { let res = await request.get('/').set('Accept', 'application/json'); assert.strictEqual(res.status, 200, 'HTTP 200'); - assert.deepEqual(res.body, { ready: true }, 'ready payload'); + assert.true(res.body.ready, 'ready payload'); + + // The field names are pinned because they are read by name outside + // this codebase: the heap dashboard extracts each one out of the + // `prerender-heap` log line with a regex, and a rename here would + // leave those panels silently empty rather than failing anywhere. + // The values themselves vary per run, so assert only the shape and + // the one relationship that always has to hold. + let memory = res.body.memory; + assert.deepEqual( + Object.keys(memory).sort(), + ['externalMB', 'heapLimitMB', 'heapTotalMB', 'heapUsedMB', 'rssMB'], + 'memory reports the expected fields', + ); + for (let [field, value] of Object.entries(memory)) { + assert.strictEqual(typeof value, 'number', `${field} is a number`); + } + assert.true(memory.heapUsedMB > 0, 'heap in use is positive'); + assert.true( + memory.heapUsedMB <= memory.heapLimitMB, + 'heap in use is within the limit', + ); }); test('it handles prerender request', async function (assert) { From 2cb3fc3a945667a1e6a6337576794dfe70d37369 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Fri, 21 Aug 2026 14:53:47 -0400 Subject: [PATCH 3/3] Floor the heap panel's step at the reporting cadence Each instance reports its heap every 30 seconds and the instances are not aligned with one another, so over a half-hour window Grafana picks a step of a few seconds and most windows contain no sample at all. The series draws as isolated spikes separated by gaps, and the legend's mean is pulled towards whichever instances happened to land inside a window rather than describing the fleet. Pin the panel's min step to a minute, which is comfortably clear of the reporting interval even with the instances unaligned, and span nulls so a missed window bridges rather than breaking the line. Co-Authored-By: Claude Opus 5 (1M context) --- .../dashboards/boxel-status/service-prerender-server.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/observability/grafanactl/resources/dashboards/boxel-status/service-prerender-server.json b/packages/observability/grafanactl/resources/dashboards/boxel-status/service-prerender-server.json index e561ac13401..c1738659d47 100644 --- a/packages/observability/grafanactl/resources/dashboards/boxel-status/service-prerender-server.json +++ b/packages/observability/grafanactl/resources/dashboards/boxel-status/service-prerender-server.json @@ -204,7 +204,7 @@ "type": "loki", "uid": "loki" }, - "description": "The prerender server's own Node heap, from the `prerender-heap` log line each instance emits on a fixed interval. Charted as the highest value across instances in each window, because Loki carries no per-task label here and the instance closest to the limit is the one that matters.\n\n`limit` is the effective `--max-old-space-size`. V8 derives its default from visible memory rather than from the ECS memory setting, so this is the only place the value a running task actually took is readable. Reaching it is fatal: the process dies with `Reached heap limit`, ECS replaces the task, and the surviving instances inherit its realm affinities and serve renders from cold tabs.\n\n`used` climbing steadily between task restarts \u2014 rather than sawtoothing under load and settling when idle \u2014 means memory is being retained, not merely in use. `external` covers strings and buffers held outside the JS heap, where response serialisation holds memory that `used` does not account for.", + "description": "The prerender server's own Node heap, from the `prerender-heap` log line each instance emits on a fixed interval. Charted as the highest value across instances in each window, because Loki carries no per-task label here and the instance closest to the limit is the one that matters.\n\n`limit` is the effective `--max-old-space-size`. V8 derives its default from visible memory rather than from the ECS memory setting, so this is the only place the value a running task actually took is readable. Reaching it is fatal: the process dies with `Reached heap limit`, ECS replaces the task, and the surviving instances inherit its realm affinities and serve renders from cold tabs.\n\n`used` climbing steadily between task restarts \u2014 rather than sawtoothing under load and settling when idle \u2014 means memory is being retained, not merely in use. `external` covers strings and buffers held outside the JS heap, where response serialisation holds memory that `used` does not account for.\n\nMin step is pinned to 1m because each instance reports every 30s: a finer step would leave most windows with no sample in them, which draws as isolated spikes and drags the mean down towards the instances that happened to land in a window.", "fieldConfig": { "defaults": { "color": { @@ -229,7 +229,7 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": false, + "spanNulls": true, "stacking": { "group": "A", "mode": "none" @@ -321,7 +321,8 @@ } ], "title": "Node heap (highest instance)", - "type": "timeseries" + "type": "timeseries", + "interval": "1m" }, { "datasource": {