diff --git a/src/index.ts b/src/index.ts index 444f57f..73fca71 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,11 +2,7 @@ import { z } from 'zod'; import * as http from 'http'; import * as https from 'https'; import type { RoutingDecision as CanonicalRoutingDecision } from '@bodanglin/verdict-contracts'; -import { - adaptRoutingDecision, - createFallbackRoutingDecision, - extractRuntimeId, -} from './adapters/contract-to-middleware.js'; +import { adaptRoutingDecision } from './adapters/contract-to-middleware'; const UNSAFE_OBJECT_KEYS = new Set(['__proto__', 'prototype', 'constructor']); @@ -353,6 +349,8 @@ export interface GatewayConfig { apiKey?: string; providerConnIds?: Record; transportAdapter?: TransportAdapterConfig; + decisionEndpoint?: string; + decisionTimeoutMs?: number; } export type TransportAdapterKind = 'openai-compatible' | 'omniroute-documented'; @@ -391,6 +389,8 @@ export class LlmGateNode { private apiKey: string; private providerConnIds: Record; private transportAdapter: NormalizedTransportAdapter; + private decisionEndpoint: string | null; + private decisionTimeoutMs: number; private autoDetectorRan = false; private usageCache: Record = {}; @@ -413,6 +413,9 @@ export class LlmGateNode { config.apiKey || process.env.OMNIROUTE_API_KEY || process.env.OPENAI_API_KEY || ''; this.providerConnIds = config.providerConnIds || {}; this.transportAdapter = this.normalizeTransportAdapter(config.transportAdapter); + this.decisionEndpoint = + config.decisionEndpoint || process.env.VERDICT_CORE_DECISION_ENDPOINT || null; + this.decisionTimeoutMs = config.decisionTimeoutMs ?? 2000; this.autoDetectDependencies(); } @@ -730,6 +733,40 @@ export class LlmGateNode { stat.score = successRate + latencyBonus; } + private isCanonicalRoutingDecision(value: unknown): value is CanonicalRoutingDecision { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const decision = value as Record; + if (!decision.selected_route || typeof decision.selected_route !== 'object') return false; + if (decision.schema_version !== undefined && decision.schema_version !== '1') return false; + if (decision.exclusions !== undefined && !Array.isArray(decision.exclusions)) return false; + if (decision.fallback_plan !== undefined && !Array.isArray(decision.fallback_plan)) + return false; + return true; + } + + private async fetchCoreDecision(body: unknown): Promise { + if (!this.decisionEndpoint) return null; + + const response = await fetch(this.decisionEndpoint, { + method: 'POST', + headers: { 'Content-Type': 'application/json', ...this.buildAdapterHeaders() }, + body: JSON.stringify(body ?? {}), + signal: AbortSignal.timeout(this.decisionTimeoutMs), + }); + + if (!response.ok) return null; + + const payload = await response.json(); + if (!this.isCanonicalRoutingDecision(payload)) return null; + + const canonical = payload as CanonicalRoutingDecision; + const route = canonical.selected_route as Record; + if (route.availability === 'unavailable' || route.availability === 'denied') return null; + if (route.decision === 'denied' || route.decision === 'unavailable') return null; + + return adaptRoutingDecision(canonical); + } + /** * Intercepts preliminary evaluations to log heuristic latency. * @returns Express Request Handler. @@ -738,6 +775,15 @@ export class LlmGateNode { return async (req: any, res: any, next: any) => { const start = Date.now(); try { + const coreDecision = await this.fetchCoreDecision(req.body); + if (this.decisionEndpoint && !coreDecision) { + return res.status(503).json({ error: 'Routing decision unavailable or denied.' }); + } + if (coreDecision) { + req.llmRouter = { decision: { ...coreDecision, latencyMs: Date.now() - start } }; + return next(); + } + const body = req.body || {}; const prompt = JSON.stringify(body); const targetTier = this.evaluateTier(prompt); @@ -751,7 +797,10 @@ export class LlmGateNode { }, }; next(); - } catch (err) { + } catch (_err) { + if (this.decisionEndpoint) { + return res.status(503).json({ error: 'Routing decision unavailable or denied.' }); + } req.llmRouter = { decision: { model: this.primaryModel, diff --git a/tests/router.test.ts b/tests/router.test.ts index 1468ee1..f7c8621 100644 --- a/tests/router.test.ts +++ b/tests/router.test.ts @@ -704,6 +704,90 @@ describe('LlmGateNode', () => { ).toBe(true); }); + it('uses a configured core canonical routing decision', async () => { + const coreDecision = { + selected_route: { + runtime_id: 'openai/gpt-4o-mini', + availability: 'healthy', + decision: 'routed', + latency_ms: 12, + }, + policy_floor: 'standard', + explanation: 'core approved', + schema_version: '1', + }; + jest + .spyOn(globalThis, 'fetch') + .mockResolvedValue(new Response(JSON.stringify(coreDecision), { status: 200 })); + const app = express(); + const gateway = new LlmGateNode({ decisionEndpoint: 'http://core/decide' }); + + app.use(express.json()); + app.post( + '/v1/chat/completions', + gateway.middleware(), + (req: Request & { llmRouter?: unknown }, res: ExpressResponse) => { + res.status(200).json({ llmRouter: req.llmRouter }); + } + ); + + const response = await request(app) + .post('/v1/chat/completions') + .send(validRequest) + .expect(200); + + expect(globalThis.fetch).toHaveBeenCalledWith('http://core/decide', expect.any(Object)); + expect(response.body.llmRouter.decision).toMatchObject({ + model: 'gpt-4o-mini', + provider: 'openai', + tier: 2, + reason: 'core approved', + }); + expect( + MiddlewareRoutingDecisionSchema.safeParse(response.body.llmRouter.decision).success + ).toBe(true); + }); + + it.each([ + ['unavailable endpoint', Promise.resolve(new Response('nope', { status: 503 }))], + [ + 'denied decision', + Promise.resolve( + new Response( + JSON.stringify({ + selected_route: { runtime_id: 'openai/gpt-4o-mini', decision: 'denied' }, + schema_version: '1', + }), + { status: 200 } + ) + ), + ], + ['malformed decision', Promise.resolve(new Response(JSON.stringify({ nope: true })))], + ['network failure', () => Promise.reject(new Error('down'))], + ])('fails closed with 503 when core decision is %s', async (_name, fetchResult) => { + jest + .spyOn(globalThis, 'fetch') + .mockImplementation(() => + typeof fetchResult === 'function' + ? (fetchResult() as Promise) + : (fetchResult as Promise) + ); + const app = express(); + const gateway = new LlmGateNode({ decisionEndpoint: 'http://core/decide' }); + + app.use(express.json()); + app.post('/v1/chat/completions', gateway.middleware(), (_req, res) => { + res.status(200).json({ reached: true }); + }); + + const response = await request(app) + .post('/v1/chat/completions') + .send(validRequest) + .expect(503); + + expect(response.body).toEqual({ error: 'Routing decision unavailable or denied.' }); + }); + it('returns a JSON parse error before middleware execution for malformed JSON', async () => { const response = await request(createApp()) .post('/v1/chat/completions')