From de05c0092d370352a1710575b00ab165e030b91d Mon Sep 17 00:00:00 2001 From: luwanglin Date: Fri, 21 Aug 2026 22:54:37 +0800 Subject: [PATCH 1/3] feat(health): make health monitor thresholds configurable via env vars Every threshold in thresholds.ts was hardcoded, so operators could not tune alert sensitivity for their hardware without patching source. Read overrides from AGENTMEMORY_HEALTH_* environment variables (defaults < env < caller-supplied config), validate them, and ignore missing/invalid values so a typo can never break health evaluation. Document the variables in .env.example. Uses the same variable names proposed in #226, adding input validation and precedence rules. Signed-off-by: luwanglin --- .env.example | 9 ++++ src/health/thresholds.ts | 41 +++++++++++++- test/health-thresholds-env.test.ts | 86 ++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 test/health-thresholds-env.test.ts diff --git a/.env.example b/.env.example index 9d346ea19..4521ae6b3 100644 --- a/.env.example +++ b/.env.example @@ -103,6 +103,15 @@ # SUMMARIZE_CHUNK_SIZE=400 # When mem::summarize sees a session larger than this, it chunks observations and map-reduces (chunk-summarize → reduce-merge) to stay within the LLM's context window. Default 400 ≈ 50k tokens per chunk at ~110 tok/obs. Native sessions are capped by MAX_OBS_PER_SESSION; chunking primarily matters for bulk-imported jsonl sessions, which bypass that cap. # SUMMARIZE_CHUNK_CONCURRENCY=6 # Parallel chunk LLM calls during chunked summarize. Default 6 fits ~100-chunk sessions under iii's 180s function-invocation timeout at typical ~8s/call. High-throughput providers (Novita, DeepInfra, DeepSeek) commonly allow 100+ concurrent — bump this for very large imported sessions. +# Health monitor thresholds (/agentmemory/health). Defaults shown; unset to keep them. +# AGENTMEMORY_HEALTH_EVENTLOOP_WARN_MS=100 # Event loop lag → degraded +# AGENTMEMORY_HEALTH_EVENTLOOP_CRITICAL_MS=500 # Event loop lag → critical +# AGENTMEMORY_HEALTH_CPU_WARN_PCT=80 # Process CPU % → degraded +# AGENTMEMORY_HEALTH_CPU_CRITICAL_PCT=90 # Process CPU % → critical +# AGENTMEMORY_HEALTH_MEM_WARN_PCT=80 # Heap-used ratio % → degraded (RSS floor applies) +# AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT=95 # Heap-used ratio % → critical (RSS floor applies) +# AGENTMEMORY_HEALTH_MEM_RSS_FLOOR_MB=512 # Min RSS (MiB) before memory alerts fire + # ----------------------------------------------------------------------------- # 5. Behaviour flags # ----------------------------------------------------------------------------- diff --git a/src/health/thresholds.ts b/src/health/thresholds.ts index 7279afad1..c83712c4f 100644 --- a/src/health/thresholds.ts +++ b/src/health/thresholds.ts @@ -20,11 +20,50 @@ const DEFAULTS: ThresholdConfig = { memoryRssFloorBytes: 512 * 1024 * 1024, }; +/** + * Environment variable overrides for every threshold. Percent values are + * plain numbers (e.g. "90"); the RSS floor is expressed in MiB. + */ +const ENV_VARS: Record = { + eventLoopLagWarnMs: "AGENTMEMORY_HEALTH_EVENTLOOP_WARN_MS", + eventLoopLagCriticalMs: "AGENTMEMORY_HEALTH_EVENTLOOP_CRITICAL_MS", + cpuWarnPercent: "AGENTMEMORY_HEALTH_CPU_WARN_PCT", + cpuCriticalPercent: "AGENTMEMORY_HEALTH_CPU_CRITICAL_PCT", + memoryWarnPercent: "AGENTMEMORY_HEALTH_MEM_WARN_PCT", + memoryCriticalPercent: "AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT", + memoryRssFloorBytes: "AGENTMEMORY_HEALTH_MEM_RSS_FLOOR_MB", +}; + +/** Parse a positive finite number, ignoring missing/invalid values. */ +function parseThreshold(raw: string | undefined): number | undefined { + if (raw === undefined || raw.trim() === "") return undefined; + const value = Number(raw); + return Number.isFinite(value) && value > 0 ? value : undefined; +} + +/** + * Threshold overrides from the environment. Invalid values are ignored so a + * typo can never disable or break health evaluation. + */ +export function thresholdOverridesFromEnv( + env: NodeJS.ProcessEnv = process.env, +): Partial { + const overrides: Partial = {}; + for (const key of Object.keys(ENV_VARS) as (keyof ThresholdConfig)[]) { + const parsed = parseThreshold(env[ENV_VARS[key]]); + if (parsed === undefined) continue; + overrides[key] = + key === "memoryRssFloorBytes" ? parsed * 1024 * 1024 : parsed; + } + return overrides; +} + export function evaluateHealth( snapshot: HealthSnapshot, config: Partial = {}, ): { status: "healthy" | "degraded" | "critical"; alerts: string[]; notes: string[] } { - const cfg = { ...DEFAULTS, ...config }; + // Precedence: defaults < environment < caller-supplied config. + const cfg = { ...DEFAULTS, ...thresholdOverridesFromEnv(), ...config }; const alerts: string[] = []; const notes: string[] = []; let critical = false; diff --git a/test/health-thresholds-env.test.ts b/test/health-thresholds-env.test.ts new file mode 100644 index 000000000..1ae5b924f --- /dev/null +++ b/test/health-thresholds-env.test.ts @@ -0,0 +1,86 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + evaluateHealth, + thresholdOverridesFromEnv, +} from "../src/health/thresholds.js"; +import type { HealthSnapshot } from "../src/types.js"; + +function snap(over: Partial = {}): HealthSnapshot { + return { + connectionState: "connected", + workers: [], + memory: { heapUsed: 0, heapTotal: 1, rss: 0, external: 0 }, + cpu: { userMicros: 0, systemMicros: 0, percent: 0 }, + eventLoopLagMs: 0, + uptimeSeconds: 1, + kvConnectivity: { status: "ok", latencyMs: 1 }, + status: "healthy", + alerts: [], + ...over, + }; +} + +describe("thresholdOverridesFromEnv", () => { + it("returns no overrides when no env vars are set", () => { + expect(thresholdOverridesFromEnv({})).toEqual({}); + }); + + it("parses every supported variable", () => { + const overrides = thresholdOverridesFromEnv({ + AGENTMEMORY_HEALTH_EVENTLOOP_WARN_MS: "200", + AGENTMEMORY_HEALTH_EVENTLOOP_CRITICAL_MS: "900", + AGENTMEMORY_HEALTH_CPU_WARN_PCT: "70", + AGENTMEMORY_HEALTH_CPU_CRITICAL_PCT: "85", + AGENTMEMORY_HEALTH_MEM_WARN_PCT: "75", + AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT: "92", + AGENTMEMORY_HEALTH_MEM_RSS_FLOOR_MB: "1024", + }); + expect(overrides).toEqual({ + eventLoopLagWarnMs: 200, + eventLoopLagCriticalMs: 900, + cpuWarnPercent: 70, + cpuCriticalPercent: 85, + memoryWarnPercent: 75, + memoryCriticalPercent: 92, + memoryRssFloorBytes: 1024 * 1024 * 1024, + }); + }); + + it("ignores missing, empty, non-numeric, and non-positive values", () => { + const overrides = thresholdOverridesFromEnv({ + AGENTMEMORY_HEALTH_CPU_WARN_PCT: "", + AGENTMEMORY_HEALTH_CPU_CRITICAL_PCT: "not-a-number", + AGENTMEMORY_HEALTH_MEM_WARN_PCT: "0", + AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT: "-5", + }); + expect(overrides).toEqual({}); + }); +}); + +describe("evaluateHealth env threshold overrides", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("applies env thresholds when evaluating", () => { + vi.stubEnv("AGENTMEMORY_HEALTH_CPU_CRITICAL_PCT", "50"); + const s = snap({ cpu: { userMicros: 0, systemMicros: 0, percent: 60 } }); + const { status, alerts } = evaluateHealth(s); + expect(status).toBe("critical"); + expect(alerts.some((a) => a.startsWith("cpu_critical_"))).toBe(true); + }); + + it("caller-supplied config wins over env", () => { + vi.stubEnv("AGENTMEMORY_HEALTH_CPU_CRITICAL_PCT", "50"); + const s = snap({ cpu: { userMicros: 0, systemMicros: 0, percent: 60 } }); + const { status } = evaluateHealth(s, { cpuCriticalPercent: 90 }); + expect(status).toBe("healthy"); + }); + + it("invalid env values fall back to defaults", () => { + vi.stubEnv("AGENTMEMORY_HEALTH_CPU_CRITICAL_PCT", "bogus"); + const s = snap({ cpu: { userMicros: 0, systemMicros: 0, percent: 95 } }); + const { status } = evaluateHealth(s); + expect(status).toBe("critical"); + }); +}); From abb496dba8f9b473b4bace95b8fa7f0075037864 Mon Sep 17 00:00:00 2001 From: luwanglin Date: Fri, 21 Aug 2026 23:57:35 +0800 Subject: [PATCH 2/3] fix(health): reject RSS floor env values that overflow to Infinity A finite MiB value like 1e308 overflows the MiB-to-bytes conversion and stores Infinity as memoryRssFloorBytes, silently disabling every memory alert. Skip the override when the converted value is not finite and cover it with a regression test. Also drop explanatory comments per repo guidelines and declare the standard iii-sdk mock in the env test. Signed-off-by: luwanglin --- src/health/thresholds.ts | 19 +++++++------------ test/health-thresholds-env.test.ts | 10 ++++++++++ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/health/thresholds.ts b/src/health/thresholds.ts index c83712c4f..4cc503c5d 100644 --- a/src/health/thresholds.ts +++ b/src/health/thresholds.ts @@ -20,10 +20,6 @@ const DEFAULTS: ThresholdConfig = { memoryRssFloorBytes: 512 * 1024 * 1024, }; -/** - * Environment variable overrides for every threshold. Percent values are - * plain numbers (e.g. "90"); the RSS floor is expressed in MiB. - */ const ENV_VARS: Record = { eventLoopLagWarnMs: "AGENTMEMORY_HEALTH_EVENTLOOP_WARN_MS", eventLoopLagCriticalMs: "AGENTMEMORY_HEALTH_EVENTLOOP_CRITICAL_MS", @@ -34,17 +30,12 @@ const ENV_VARS: Record = { memoryRssFloorBytes: "AGENTMEMORY_HEALTH_MEM_RSS_FLOOR_MB", }; -/** Parse a positive finite number, ignoring missing/invalid values. */ function parseThreshold(raw: string | undefined): number | undefined { if (raw === undefined || raw.trim() === "") return undefined; const value = Number(raw); return Number.isFinite(value) && value > 0 ? value : undefined; } -/** - * Threshold overrides from the environment. Invalid values are ignored so a - * typo can never disable or break health evaluation. - */ export function thresholdOverridesFromEnv( env: NodeJS.ProcessEnv = process.env, ): Partial { @@ -52,8 +43,13 @@ export function thresholdOverridesFromEnv( for (const key of Object.keys(ENV_VARS) as (keyof ThresholdConfig)[]) { const parsed = parseThreshold(env[ENV_VARS[key]]); if (parsed === undefined) continue; - overrides[key] = - key === "memoryRssFloorBytes" ? parsed * 1024 * 1024 : parsed; + if (key === "memoryRssFloorBytes") { + const bytes = parsed * 1024 * 1024; + if (!Number.isFinite(bytes)) continue; + overrides[key] = bytes; + } else { + overrides[key] = parsed; + } } return overrides; } @@ -62,7 +58,6 @@ export function evaluateHealth( snapshot: HealthSnapshot, config: Partial = {}, ): { status: "healthy" | "degraded" | "critical"; alerts: string[]; notes: string[] } { - // Precedence: defaults < environment < caller-supplied config. const cfg = { ...DEFAULTS, ...thresholdOverridesFromEnv(), ...config }; const alerts: string[] = []; const notes: string[] = []; diff --git a/test/health-thresholds-env.test.ts b/test/health-thresholds-env.test.ts index 1ae5b924f..b49e01657 100644 --- a/test/health-thresholds-env.test.ts +++ b/test/health-thresholds-env.test.ts @@ -1,4 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("iii-sdk", () => ({})); + import { evaluateHealth, thresholdOverridesFromEnv, @@ -55,6 +58,13 @@ describe("thresholdOverridesFromEnv", () => { }); expect(overrides).toEqual({}); }); + + it("rejects RSS floor values that overflow the MiB-to-bytes conversion", () => { + const overrides = thresholdOverridesFromEnv({ + AGENTMEMORY_HEALTH_MEM_RSS_FLOOR_MB: "1e308", + }); + expect(overrides).toEqual({}); + }); }); describe("evaluateHealth env threshold overrides", () => { From 0fa9f4303bb1d070e6e4e46392117d564a478ec7 Mon Sep 17 00:00:00 2001 From: luwanglin Date: Sat, 22 Aug 2026 00:00:23 +0800 Subject: [PATCH 3/3] test(health): fill in iii-sdk mock surface per review Signed-off-by: luwanglin --- test/health-thresholds-env.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/health-thresholds-env.test.ts b/test/health-thresholds-env.test.ts index b49e01657..0ef8e4c54 100644 --- a/test/health-thresholds-env.test.ts +++ b/test/health-thresholds-env.test.ts @@ -1,6 +1,9 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -vi.mock("iii-sdk", () => ({})); +vi.mock("iii-sdk", () => ({ + sdk: { trigger: vi.fn() }, + kv: { get: vi.fn(), set: vi.fn(), list: vi.fn() }, +})); import { evaluateHealth,