feat(models): add external safety benchmark metadata

apps/web/src/app/api/cron/sync-enkrypt/route.test.ts

@@ -0,0 +1,170 @@
+jest.mock('@/lib/config.server', () => ({
+  get CRON_SECRET() {
+    return mockCronSecret;
+  },
+}));
+
+jest.mock('@kilocode/worker-utils/scheduled-job-observability', () => ({
+  createScheduledJobRun: jest.fn(options => ({
+    runId: 'run-id',
+    startedAt: 0,
+    ...options,
+  })),
+  buildScheduledJobSuccessEvent: jest.fn((_run, fields) => ({ outcome: 'succeeded', ...fields })),
+  buildScheduledJobFailureEvent: jest.fn(({ metadata }) => ({
+    outcome: 'failed',
+    exception_name: 'Error',
+    ...metadata,
+  })),
+  emitScheduledJobEvent: jest.fn(),
+}));
+
+jest.mock('@/lib/model-stats/sync-enkrypt', () => ({ syncEnkryptBenchmarks: jest.fn() }));
+jest.mock('@sentry/nextjs', () => ({ captureException: jest.fn() }));
+
+import { NextRequest } from 'next/server';
+import { captureException } from '@sentry/nextjs';
+import {
+  buildScheduledJobFailureEvent,
+  buildScheduledJobSuccessEvent,
+  createScheduledJobRun,
+  emitScheduledJobEvent,
+} from '@kilocode/worker-utils/scheduled-job-observability';
+import { syncEnkryptBenchmarks } from '@/lib/model-stats/sync-enkrypt';
+import { GET, maxDuration } from './route';
+
+let mockCronSecret: string | undefined = 'cron-secret';
+const mockSyncEnkryptBenchmarks = jest.mocked(syncEnkryptBenchmarks);
+
+function request(authorization?: string) {
+  return new NextRequest('http://localhost/api/cron/sync-enkrypt', {
+    headers: authorization === undefined ? undefined : { authorization },
+  });
+}
+
+describe('GET /api/cron/sync-enkrypt', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+    mockSyncEnkryptBenchmarks.mockReset();
+    mockCronSecret = 'cron-secret';
+  });
+
+  it.each([
+    undefined,
+    '',
+    'Bearer wrong-secret',
+    'bearer cron-secret',
+    'cron-secret',
+    'Bearer cron-secret-extra',
+    'Bearer  cron-secret',
+  ])('rejects authorization %p before starting any work', async authorization => {
+    const response = await GET(request(authorization));
+
+    expect(response.status).toBe(401);
+    expect(await response.json()).toEqual({ error: 'Unauthorized' });
+    expect(mockSyncEnkryptBenchmarks).not.toHaveBeenCalled();
+    expect(createScheduledJobRun).not.toHaveBeenCalled();
+    expect(emitScheduledJobEvent).not.toHaveBeenCalled();
+    expect(captureException).not.toHaveBeenCalled();
+  });
+
+  it.each([undefined, ''])('rejects an unconfigured secret %p', async secret => {
+    mockCronSecret = secret;
+
+    const response = await GET(request(`Bearer ${secret}`));
+
+    expect(response.status).toBe(401);
+    expect(await response.json()).toEqual({ error: 'Unauthorized' });
+    expect(mockSyncEnkryptBenchmarks).not.toHaveBeenCalled();
+    expect(createScheduledJobRun).not.toHaveBeenCalled();
+    expect(emitScheduledJobEvent).not.toHaveBeenCalled();
+    expect(captureException).not.toHaveBeenCalled();
+  });
+
+  it('returns only summary counters and emits one success event', async () => {
+    const counters = {
+      fetchedCount: 5,
+      matchedCount: 2,
+      unmatchedCount: 2,
+      ambiguousCount: 1,
+      updatedCount: 2,
+    };
+    const result = {
+      ...counters,
+      unmatchedModelNames: ['unmatched-model-a', 'unmatched-model-b'],
+      upstreamMetadata: 'not-for-output',
+    };
+    mockSyncEnkryptBenchmarks.mockResolvedValue(result);
+
+    const response = await GET(request('Bearer cron-secret'));
+
+    expect(response.status).toBe(200);
+    expect(await response.json()).toEqual({ success: true, ...counters });
+    expect(maxDuration).toBe(120);
+    expect(mockSyncEnkryptBenchmarks).toHaveBeenCalledTimes(1);
+    expect(mockSyncEnkryptBenchmarks).toHaveBeenCalledWith();
+    expect(createScheduledJobRun).toHaveBeenCalledTimes(1);
+    expect(createScheduledJobRun).toHaveBeenCalledWith({
+      jobName: 'web.sync_enkrypt',
+      environment: process.env.VERCEL_ENV ?? process.env.NODE_ENV,
+    });
+    expect(buildScheduledJobSuccessEvent).toHaveBeenCalledWith(
+      expect.objectContaining({ runId: 'run-id', jobName: 'web.sync_enkrypt' }),
+      {
+        fetched_count: 5,
+        matched_count: 2,
+        unmatched_count: 2,
+        ambiguous_count: 1,
+        updated_count: 2,
+      }
+    );
+    expect(emitScheduledJobEvent).toHaveBeenCalledTimes(1);
+    expect(emitScheduledJobEvent).toHaveBeenCalledWith({
+      outcome: 'succeeded',
+      fetched_count: 5,
+      matched_count: 2,
+      unmatched_count: 2,
+      ambiguous_count: 1,
+      updated_count: 2,
+    });
+    expect(buildScheduledJobFailureEvent).not.toHaveBeenCalled();
+    expect(captureException).not.toHaveBeenCalled();
+  });
+
+  it.each([
+    new Error('sensitive upstream response'),
+    { body: 'sensitive upstream response', headers: { authorization: 'sensitive header' } },
+    new Error('ENKRYPT_API_KEY is not configured'),
+  ])('returns a generic failure and emits safe observability for %p', async failure => {
+    mockSyncEnkryptBenchmarks.mockRejectedValue(failure);
+
+    const response = await GET(request('Bearer cron-secret'));
+    const safeError = new Error('Failed to sync Enkrypt benchmarks');
+
+    expect(response.status).toBe(500);
+    expect(await response.json()).toEqual({
+      success: false,
+      error: 'Failed to sync Enkrypt benchmarks',
+    });
+    expect(mockSyncEnkryptBenchmarks).toHaveBeenCalledTimes(1);
+    expect(captureException).toHaveBeenCalledTimes(1);
+    expect(captureException).toHaveBeenCalledWith(safeError, {
+      tags: { endpoint: 'cron/sync-enkrypt' },
+    });
+    expect(jest.mocked(captureException).mock.calls[0]?.[0]).not.toBe(failure);
+    expect(buildScheduledJobFailureEvent).toHaveBeenCalledWith({
+      context: expect.objectContaining({ runId: 'run-id', jobName: 'web.sync_enkrypt' }),
+      jobName: 'web.sync_enkrypt',
+      environment: process.env.VERCEL_ENV ?? process.env.NODE_ENV,
+      error: safeError,
+      metadata: { sync_failure_count: 1 },
+    });
+    expect(emitScheduledJobEvent).toHaveBeenCalledTimes(1);
+    expect(emitScheduledJobEvent).toHaveBeenCalledWith({
+      outcome: 'failed',
+      exception_name: 'Error',
+      sync_failure_count: 1,
+    });
+    expect(buildScheduledJobSuccessEvent).not.toHaveBeenCalled();
+  });
+});

apps/web/src/app/api/cron/sync-enkrypt/route.ts

@@ -0,0 +1,68 @@
+import { NextResponse } from 'next/server';
+import type { NextRequest } from 'next/server';
+import { captureException } from '@sentry/nextjs';
+import {
+  buildScheduledJobFailureEvent,
+  buildScheduledJobSuccessEvent,
+  createScheduledJobRun,
+  emitScheduledJobEvent,
+} from '@kilocode/worker-utils/scheduled-job-observability';
+import { CRON_SECRET } from '@/lib/config.server';
+import { syncEnkryptBenchmarks } from '@/lib/model-stats/sync-enkrypt';
+
+export const maxDuration = 120;
+
+export async function GET(request: NextRequest) {
+  const authHeader = request.headers.get('authorization');
+  if (!CRON_SECRET || authHeader !== `Bearer ${CRON_SECRET}`) {
+    return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
+  }
+
+  const run = createScheduledJobRun({
+    jobName: 'web.sync_enkrypt',
+    environment: process.env.VERCEL_ENV ?? process.env.NODE_ENV,
+  });
+
+  try {
+    const { fetchedCount, matchedCount, unmatchedCount, ambiguousCount, updatedCount } =
+      await syncEnkryptBenchmarks();
+
+    emitScheduledJobEvent(
+      buildScheduledJobSuccessEvent(run, {
+        fetched_count: fetchedCount,
+        matched_count: matchedCount,
+        unmatched_count: unmatchedCount,
+        ambiguous_count: ambiguousCount,
+        updated_count: updatedCount,
+      })
+    );
+
+    return NextResponse.json({
+      success: true,
+      fetchedCount,
+      matchedCount,
+      unmatchedCount,
+      ambiguousCount,
+      updatedCount,
+    });
+  } catch {
+    const error = new Error('Failed to sync Enkrypt benchmarks');
+    captureException(error, {
+      tags: { endpoint: 'cron/sync-enkrypt' },
+    });
+    emitScheduledJobEvent(
+      buildScheduledJobFailureEvent({
+        context: run,
+        jobName: run.jobName,
+        environment: run.environment,
+        error,
+        metadata: { sync_failure_count: 1 },
+      })
+    );
+
+    return NextResponse.json(
+      { success: false, error: 'Failed to sync Enkrypt benchmarks' },
+      { status: 500 }
+    );
+  }
+}

apps/web/src/lib/ai-gateway/providers/openrouter/index.ts

@@ -24,6 +24,7 @@ import { getOpenRouterModelsMetadataFromDatabase } from '@/lib/ai-gateway/provid
 import { getPreferredProviderOrder } from '@/lib/ai-gateway/providers/apply-provider-specific-logic';
 import { normalizeInferenceProviderId } from '@/lib/ai-gateway/providers/openrouter/inference-provider-id';
 import { getTerminalBenchSummaries, terminalBenchFor } from '@/lib/model-stats/terminal-bench';
+import { enkryptFor, getEnkryptBenchmarks } from '@/lib/model-stats/enkrypt';
 import { isFreeNemotronModel, NVIDIA_TRIAL_TOS } from '@/lib/ai-gateway/providers/nvidia';
 import { applyCustomPricingToModel } from '@/lib/ai-gateway/custom-pricing';
 import { addMonths } from 'date-fns';
@@ -114,7 +115,10 @@ async function enhancedModelList(models: OpenRouterModel[]) {
   const autoModels = buildAutoModels();
   const endpointsMetadata = await getOpenRouterModelsMetadataFromDatabase();
   const hasEndpointsMetadata = Object.keys(endpointsMetadata).length > 0;
-  const summaries = await getTerminalBenchSummaries();
+  const [summaries, enkryptBenchmarks] = await Promise.all([
+    getTerminalBenchSummaries(),
+    getEnkryptBenchmarks(),
+  ]);
   const enhancedModels = await Promise.all(
     models
       .filter(
@@ -150,6 +154,10 @@ async function enhancedModelList(models: OpenRouterModel[]) {
           .filter(m => m.status === 'public')
           .map(model => convertFromKiloExclusiveModel(model))
       )
+      .map(model => {
+        const enkrypt = enkryptFor(enkryptBenchmarks, model.id);
+        return { ...model, ...(enkrypt && { enkrypt }) };
+      })
       .concat(autoModels)
       .map(applyCustomPricingToModel)
       .map(async (model: OpenRouterModel) => {

apps/web/src/lib/config.server.ts

@@ -163,6 +163,7 @@ export const BYOK_ENCRYPTION_KEY = requireEnv(
 
 // Artificial Analysis API
 export const ARTIFICIAL_ANALYSIS_API_KEY = getEnvVariable('ARTIFICIAL_ANALYSIS_API_KEY');
+export const ENKRYPT_API_KEY = getEnvVariable('ENKRYPT_API_KEY');
 
 // Cron jobs
 export const CRON_SECRET = getEnvVariable('CRON_SECRET');