diff --git a/.env.example b/.env.example index 9d346ea19..4521ae6b3 100644 --- a/.env.example +++ b/.env.example @@ -103,6 +103,15 @@ # SUMMARIZE_CHUNK_SIZE=400 # When mem::summarize sees a session larger than this, it chunks observations and map-reduces (chunk-summarize → reduce-merge) to stay within the LLM's context window. Default 400 ≈ 50k tokens per chunk at ~110 tok/obs. Native sessions are capped by MAX_OBS_PER_SESSION; chunking primarily matters for bulk-imported jsonl sessions, which bypass that cap. # SUMMARIZE_CHUNK_CONCURRENCY=6 # Parallel chunk LLM calls during chunked summarize. Default 6 fits ~100-chunk sessions under iii's 180s function-invocation timeout at typical ~8s/call. High-throughput providers (Novita, DeepInfra, DeepSeek) commonly allow 100+ concurrent — bump this for very large imported sessions. +# Health monitor thresholds (/agentmemory/health). Defaults shown; unset to keep them. +# AGENTMEMORY_HEALTH_EVENTLOOP_WARN_MS=100 # Event loop lag → degraded +# AGENTMEMORY_HEALTH_EVENTLOOP_CRITICAL_MS=500 # Event loop lag → critical +# AGENTMEMORY_HEALTH_CPU_WARN_PCT=80 # Process CPU % → degraded +# AGENTMEMORY_HEALTH_CPU_CRITICAL_PCT=90 # Process CPU % → critical +# AGENTMEMORY_HEALTH_MEM_WARN_PCT=80 # Heap-used ratio % → degraded (RSS floor applies) +# AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT=95 # Heap-used ratio % → critical (RSS floor applies) +# AGENTMEMORY_HEALTH_MEM_RSS_FLOOR_MB=512 # Min RSS (MiB) before memory alerts fire + # ----------------------------------------------------------------------------- # 5. Behaviour flags # ----------------------------------------------------------------------------- diff --git a/src/health/thresholds.ts b/src/health/thresholds.ts index 7279afad1..4cc503c5d 100644 --- a/src/health/thresholds.ts +++ b/src/health/thresholds.ts @@ -20,11 +20,45 @@ const DEFAULTS: ThresholdConfig = { memoryRssFloorBytes: 512 * 1024 * 1024, }; +const ENV_VARS: Record = { + eventLoopLagWarnMs: "AGENTMEMORY_HEALTH_EVENTLOOP_WARN_MS", + eventLoopLagCriticalMs: "AGENTMEMORY_HEALTH_EVENTLOOP_CRITICAL_MS", + cpuWarnPercent: "AGENTMEMORY_HEALTH_CPU_WARN_PCT", + cpuCriticalPercent: "AGENTMEMORY_HEALTH_CPU_CRITICAL_PCT", + memoryWarnPercent: "AGENTMEMORY_HEALTH_MEM_WARN_PCT", + memoryCriticalPercent: "AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT", + memoryRssFloorBytes: "AGENTMEMORY_HEALTH_MEM_RSS_FLOOR_MB", +}; + +function parseThreshold(raw: string | undefined): number | undefined { + if (raw === undefined || raw.trim() === "") return undefined; + const value = Number(raw); + return Number.isFinite(value) && value > 0 ? value : undefined; +} + +export function thresholdOverridesFromEnv( + env: NodeJS.ProcessEnv = process.env, +): Partial { + const overrides: Partial = {}; + for (const key of Object.keys(ENV_VARS) as (keyof ThresholdConfig)[]) { + const parsed = parseThreshold(env[ENV_VARS[key]]); + if (parsed === undefined) continue; + if (key === "memoryRssFloorBytes") { + const bytes = parsed * 1024 * 1024; + if (!Number.isFinite(bytes)) continue; + overrides[key] = bytes; + } else { + overrides[key] = parsed; + } + } + return overrides; +} + export function evaluateHealth( snapshot: HealthSnapshot, config: Partial = {}, ): { status: "healthy" | "degraded" | "critical"; alerts: string[]; notes: string[] } { - const cfg = { ...DEFAULTS, ...config }; + const cfg = { ...DEFAULTS, ...thresholdOverridesFromEnv(), ...config }; const alerts: string[] = []; const notes: string[] = []; let critical = false; diff --git a/test/health-thresholds-env.test.ts b/test/health-thresholds-env.test.ts new file mode 100644 index 000000000..0ef8e4c54 --- /dev/null +++ b/test/health-thresholds-env.test.ts @@ -0,0 +1,99 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("iii-sdk", () => ({ + sdk: { trigger: vi.fn() }, + kv: { get: vi.fn(), set: vi.fn(), list: vi.fn() }, +})); + +import { + evaluateHealth, + thresholdOverridesFromEnv, +} from "../src/health/thresholds.js"; +import type { HealthSnapshot } from "../src/types.js"; + +function snap(over: Partial = {}): HealthSnapshot { + return { + connectionState: "connected", + workers: [], + memory: { heapUsed: 0, heapTotal: 1, rss: 0, external: 0 }, + cpu: { userMicros: 0, systemMicros: 0, percent: 0 }, + eventLoopLagMs: 0, + uptimeSeconds: 1, + kvConnectivity: { status: "ok", latencyMs: 1 }, + status: "healthy", + alerts: [], + ...over, + }; +} + +describe("thresholdOverridesFromEnv", () => { + it("returns no overrides when no env vars are set", () => { + expect(thresholdOverridesFromEnv({})).toEqual({}); + }); + + it("parses every supported variable", () => { + const overrides = thresholdOverridesFromEnv({ + AGENTMEMORY_HEALTH_EVENTLOOP_WARN_MS: "200", + AGENTMEMORY_HEALTH_EVENTLOOP_CRITICAL_MS: "900", + AGENTMEMORY_HEALTH_CPU_WARN_PCT: "70", + AGENTMEMORY_HEALTH_CPU_CRITICAL_PCT: "85", + AGENTMEMORY_HEALTH_MEM_WARN_PCT: "75", + AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT: "92", + AGENTMEMORY_HEALTH_MEM_RSS_FLOOR_MB: "1024", + }); + expect(overrides).toEqual({ + eventLoopLagWarnMs: 200, + eventLoopLagCriticalMs: 900, + cpuWarnPercent: 70, + cpuCriticalPercent: 85, + memoryWarnPercent: 75, + memoryCriticalPercent: 92, + memoryRssFloorBytes: 1024 * 1024 * 1024, + }); + }); + + it("ignores missing, empty, non-numeric, and non-positive values", () => { + const overrides = thresholdOverridesFromEnv({ + AGENTMEMORY_HEALTH_CPU_WARN_PCT: "", + AGENTMEMORY_HEALTH_CPU_CRITICAL_PCT: "not-a-number", + AGENTMEMORY_HEALTH_MEM_WARN_PCT: "0", + AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT: "-5", + }); + expect(overrides).toEqual({}); + }); + + it("rejects RSS floor values that overflow the MiB-to-bytes conversion", () => { + const overrides = thresholdOverridesFromEnv({ + AGENTMEMORY_HEALTH_MEM_RSS_FLOOR_MB: "1e308", + }); + expect(overrides).toEqual({}); + }); +}); + +describe("evaluateHealth env threshold overrides", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("applies env thresholds when evaluating", () => { + vi.stubEnv("AGENTMEMORY_HEALTH_CPU_CRITICAL_PCT", "50"); + const s = snap({ cpu: { userMicros: 0, systemMicros: 0, percent: 60 } }); + const { status, alerts } = evaluateHealth(s); + expect(status).toBe("critical"); + expect(alerts.some((a) => a.startsWith("cpu_critical_"))).toBe(true); + }); + + it("caller-supplied config wins over env", () => { + vi.stubEnv("AGENTMEMORY_HEALTH_CPU_CRITICAL_PCT", "50"); + const s = snap({ cpu: { userMicros: 0, systemMicros: 0, percent: 60 } }); + const { status } = evaluateHealth(s, { cpuCriticalPercent: 90 }); + expect(status).toBe("healthy"); + }); + + it("invalid env values fall back to defaults", () => { + vi.stubEnv("AGENTMEMORY_HEALTH_CPU_CRITICAL_PCT", "bogus"); + const s = snap({ cpu: { userMicros: 0, systemMicros: 0, percent: 95 } }); + const { status } = evaluateHealth(s); + expect(status).toBe("critical"); + }); +});