From 58ce1ff90bbe69d511c8f533d31c9af09dbb526a Mon Sep 17 00:00:00 2001 From: Austin Kurpuis Date: Thu, 13 Aug 2026 17:28:12 -0700 Subject: [PATCH 1/2] Label container-tool launch failures instead of letting them escape uncaught MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The container-Tool (ToolRun/Job) branch of runTool awaited launch() and the job result with no catch, unlike the agent-backed branch right above it. A failure before the tool ever ran -- e.g. the k8s API call in ToolRunLauncher.launch() failing -- propagated uncaught past every handler in the graph and reached the SSE layer as server.ts's generic `❌ ${err.message}`, with no indication of which tool or that launch (not the tool itself) was what failed. This is what a user saw as a bare "❌ fetch failed" trying to extract a recipe: recipe-scraper never got a chance to run. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01B5MkAovb1gWz4G34wSXaVC --- .../src/agent/graph.test.ts | 19 +++++++++++++++++++ apps/agent-orchestrator/src/agent/graph.ts | 15 +++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/apps/agent-orchestrator/src/agent/graph.test.ts b/apps/agent-orchestrator/src/agent/graph.test.ts index 4f825da..8d509f5 100644 --- a/apps/agent-orchestrator/src/agent/graph.test.ts +++ b/apps/agent-orchestrator/src/agent/graph.test.ts @@ -417,6 +417,25 @@ describe("buildAgentGraph", () => { expect(final.error).toMatch(/tool failed \(extraction\)/); }); + it("surfaces a launch failure (e.g. the k8s API call itself failing) as a labeled graph error instead of an uncaught, contextless exception", async () => { + // Regression test: `containerToolLauncher.launch()` throwing (rather than + // resolving and the Job later reporting a `failed` Event) used to escape + // every catch in the graph and reach the SSE layer as a bare + // `err.message` -- e.g. "fetch failed" from the Kubernetes client, with + // no indication which tool was being launched or that launch itself (not + // the tool) was what failed. + const deps = baseDeps({ + containerToolLauncher: { + launch: vi.fn().mockRejectedValue(new TypeError("fetch failed")), + } as unknown as ContainerToolLauncher, + }); + const graph = buildAgentGraph(deps); + + const final = await graph.invoke({ request: "do a thing", authToken: "tok" }); + + expect(final.error).toMatch(/tool recipe-scraper failed to launch: fetch failed/); + }); + it("runs a LocalTool via the executor sidecar instead of launching a Job (ADR 0014)", async () => { const localTool: ToolDescriptor = { id: "http-get-node", diff --git a/apps/agent-orchestrator/src/agent/graph.ts b/apps/agent-orchestrator/src/agent/graph.ts index 3f1a625..c1a2a50 100644 --- a/apps/agent-orchestrator/src/agent/graph.ts +++ b/apps/agent-orchestrator/src/agent/graph.ts @@ -1982,6 +1982,21 @@ export function buildAgentGraph(deps: AgentGraphDeps) { } event = await awaitResult; + } catch (err) { + // Unlike the agent-backed branch above, nothing here has yet + // produced a structured `failed` Event -- `launch()` can throw + // before the Job/ToolRun even exists (e.g. the k8s API call itself + // failing), and `awaitResult` can reject for reasons outside the + // tool's own control. Both used to propagate uncaught out of this + // node, past every catch in the graph, and surface at the SSE layer + // as a bare `err.message` -- e.g. "fetch failed" from the + // Kubernetes client with no indication which tool, which call, or + // that a ToolRun was never even created. Catching and labeling here + // keeps that context. + return { + jobId, + error: `tool ${tool.id} failed to launch: ${err instanceof Error ? err.message : String(err)}`, + }; } finally { unsubscribeProgress(); } From 6074b5e5619bf4a19d035e024aeb5529e1253f16 Mon Sep 17 00:00:00 2001 From: Austin Kurpuis Date: Thu, 13 Aug 2026 20:48:45 -0700 Subject: [PATCH 2/2] Stop routing production turns to a Temporal engine that was never deployed charts/agent-controller/values.yaml defaults agent-orchestrator.config.agentEngine to "temporal" and temporal-engine.enabled to true (docs/adr/0036's rollout), and values-production.yaml never overrode either back off. This production deployment has no Temporal cluster at all, and the temporal-engine subchart's gateway/worker images have no registry prefix (containerd resolves them against Docker Hub, which has never had them) -- confirmed live via `kubectl describe pod`: both Deployments have sat in ImagePullBackOff for 6+ days. Every turn was still being forwarded to that nonexistent gateway over HTTP, and the resulting dead-connection error is what silently arrives at end users as a bare "fetch failed" -- this is the actual cause of the "recipe skill broke" report, not the container-tool-launch gap fixed in the prior commit (that one's still a real bug, just not this incident). Overrides agentEngine/temporalEngineUrl back to "" and disables the temporal-engine subchart, restoring every turn to the in-process LangGraph loop this environment has always actually run on. Verified with `helm template` against both values files: AGENT_ENGINE/AGENT_TEMPORAL_ENGINE_URL no longer render on the orchestrator Deployment and no temporal-engine Deployment/Service/RBAC objects render at all. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01B5MkAovb1gWz4G34wSXaVC --- .../agent-controller/values-production.yaml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/charts/agent-controller/values-production.yaml b/charts/agent-controller/values-production.yaml index 4179549..c5cf754 100644 --- a/charts/agent-controller/values-production.yaml +++ b/charts/agent-controller/values-production.yaml @@ -81,6 +81,16 @@ agent-orchestrator: gatewayUrl: "http://agent-controller-integration-gateway:8090" config: + # This cluster has no Temporal deployment at all (see temporal-engine.enabled + # below) -- clear the umbrella chart's own default of "temporal" + # (charts/agent-controller/values.yaml, docs/adr/0036's rollout default) so + # every turn runs the in-process LangGraph loop instead of being forwarded + # to a gateway Service backed by pods that were never buildable here. + # Helm deep-merges this `config` map key-by-key against that default, so + # omitting this key (rather than setting it) would silently inherit + # "temporal" -- which is exactly what had been happening. + agentEngine: "" + temporalEngineUrl: "" # oidc against Pocket ID (kubernetes/manifests/services/pocket-id in # imaustink/homelab), not Google -- the 2026-07-19 outage was caused by # Google id_tokens: neither caller could present one (Open WebUI has no @@ -194,6 +204,18 @@ agent-orchestrator: redis: enabled: true +# No Temporal cluster exists in this environment (docs/adr/0036 "Assumes a +# reachable Temporal cluster; no server is bundled") -- override the umbrella +# chart's own default of enabled: true (charts/agent-controller/values.yaml) +# so this release stops deploying the gateway/worker Deployments here. Without +# this they run permanently in ImagePullBackOff (their image names have no +# registry prefix, so containerd resolves them against Docker Hub, which has +# never had them) while agent-orchestrator.config.agentEngine above still +# forwarded every turn to the (nonexistent) gateway -- the underlying cause of +# turns failing with a bare "fetch failed". +temporal-engine: + enabled: false + core-controller: enabled: true fullnameOverride: core-controller