From 9a3f40eaa0a119ac87ebe9aa5a2fa7ffb8040714 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Fri, 17 Jul 2026 00:05:51 -0700 Subject: [PATCH 1/2] fix(observability): restore historical continuity in the AI Usage dashboard #5522's hard-cutover rename of the gittensory_ metric prefix to loopover_ left this dashboard's 9 loopover_ai_* queries with no fallback to their pre-rebrand names, so every panel here only ever showed data recorded after that cutover -- confirmed live, both metric names have real historical series in Prometheus going back well past the cutover. Applies the same (loopover_x or gittensory_x) union fix already shipped for grafana/dashboards/gittensory.json (#6779/#6787), including that fix's own lesson: a label matcher like {provider="codex"} must bind to each side of the union individually, never to the closing paren of the union as a whole -- verified all 9 rewritten queries directly against live Prometheus before deploying. Deployed to edge-nl-01 and restarted Grafana to force a clean reprovision. --- grafana/dashboards/ai-usage.json | 18 ++++---- ...elfhost-grafana-ai-usage-dashboard.test.ts | 43 ++++++++++++++++--- 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/grafana/dashboards/ai-usage.json b/grafana/dashboards/ai-usage.json index 18f44c35ef..9f0964f921 100644 --- a/grafana/dashboards/ai-usage.json +++ b/grafana/dashboards/ai-usage.json @@ -291,7 +291,7 @@ "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "unit": "currencyUSD", "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 } } }, "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi" } }, - "targets": [{ "refId": "A", "expr": "sum by (provider) (loopover_ai_cost_usd_total) or vector(0)", "legendFormat": "{{provider}}" }] + "targets": [{ "refId": "A", "expr": "sum by (provider) ((loopover_ai_cost_usd_total or gittensory_ai_cost_usd_total)) or vector(0)", "legendFormat": "{{provider}}" }] }, { "id": 16, @@ -304,7 +304,7 @@ "targets": [ { "refId": "A", - "expr": "sum by (provider) ((rate(loopover_ai_input_tokens_total[5m]) + rate(loopover_ai_output_tokens_total[5m])) * 60)", + "expr": "sum by (provider) (((rate(loopover_ai_input_tokens_total[5m]) or rate(gittensory_ai_input_tokens_total[5m])) + (rate(loopover_ai_output_tokens_total[5m]) or rate(gittensory_ai_output_tokens_total[5m]))) * 60)", "legendFormat": "{{provider}}" } ] @@ -318,8 +318,8 @@ "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 } } }, "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi" } }, "targets": [ - { "refId": "A", "expr": "sum by (model, effort) (increase(loopover_ai_requests_total[1h]))", "legendFormat": "{{model}} · {{effort}}" }, - { "refId": "B", "expr": "sum by (primary, fallback) (increase(loopover_ai_review_model_fallback_total[1h]))", "legendFormat": "fallback {{primary}}→{{fallback}}" } + { "refId": "A", "expr": "sum by (model, effort) ((increase(loopover_ai_requests_total[1h]) or increase(gittensory_ai_requests_total[1h])))", "legendFormat": "{{model}} · {{effort}}" }, + { "refId": "B", "expr": "sum by (primary, fallback) ((increase(loopover_ai_review_model_fallback_total[1h]) or increase(gittensory_ai_review_model_fallback_total[1h])))", "legendFormat": "fallback {{primary}}→{{fallback}}" } ] }, { @@ -331,8 +331,8 @@ "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 } } }, "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi" } }, "targets": [ - { "refId": "A", "expr": "sum by (provider, kind) (loopover_ai_input_tokens_total)", "legendFormat": "{{provider}} {{kind}} in" }, - { "refId": "B", "expr": "sum by (provider, kind) (loopover_ai_output_tokens_total)", "legendFormat": "{{provider}} {{kind}} out" } + { "refId": "A", "expr": "sum by (provider, kind) ((loopover_ai_input_tokens_total or gittensory_ai_input_tokens_total))", "legendFormat": "{{provider}} {{kind}} in" }, + { "refId": "B", "expr": "sum by (provider, kind) ((loopover_ai_output_tokens_total or gittensory_ai_output_tokens_total))", "legendFormat": "{{provider}} {{kind}} out" } ] }, { @@ -347,7 +347,7 @@ "targets": [ { "refId": "A", - "expr": "sum by (model, effort) (increase(loopover_ai_requests_total{provider=\"codex\"}[$__rate_interval]))", + "expr": "sum by (model, effort) ((increase(loopover_ai_requests_total{provider=\"codex\"}[$__rate_interval]) or increase(gittensory_ai_requests_total{provider=\"codex\"}[$__rate_interval])))", "legendFormat": "{{model}} / {{effort}}" } ] @@ -361,8 +361,8 @@ "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "fillOpacity": 70, "lineWidth": 1, "stacking": { "mode": "normal" } } } }, "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ - { "refId": "A", "expr": "sum by (kind) (increase(loopover_ai_input_tokens_total{provider=\"codex\"}[$__rate_interval]))", "legendFormat": "input {{kind}}" }, - { "refId": "B", "expr": "sum by (kind) (increase(loopover_ai_output_tokens_total{provider=\"codex\"}[$__rate_interval]))", "legendFormat": "output {{kind}}" } + { "refId": "A", "expr": "sum by (kind) ((increase(loopover_ai_input_tokens_total{provider=\"codex\"}[$__rate_interval]) or increase(gittensory_ai_input_tokens_total{provider=\"codex\"}[$__rate_interval])))", "legendFormat": "input {{kind}}" }, + { "refId": "B", "expr": "sum by (kind) ((increase(loopover_ai_output_tokens_total{provider=\"codex\"}[$__rate_interval]) or increase(gittensory_ai_output_tokens_total{provider=\"codex\"}[$__rate_interval])))", "legendFormat": "output {{kind}}" } ] }, { diff --git a/test/unit/selfhost-grafana-ai-usage-dashboard.test.ts b/test/unit/selfhost-grafana-ai-usage-dashboard.test.ts index 91449ab8f6..25ad7899aa 100644 --- a/test/unit/selfhost-grafana-ai-usage-dashboard.test.ts +++ b/test/unit/selfhost-grafana-ai-usage-dashboard.test.ts @@ -190,16 +190,49 @@ describe("Loopover - AI usage dashboard (Phase B2 consolidation)", () => { it("carries over the exact Prometheus expressions from the removed dashboards, byte-for-byte (no copy-paste drift)", () => { const targets = readDashboard().panels.flatMap((panel) => panel.targets ?? []); // From gittensory.json's removed "AI Usage & Cost" row. - expect(targets.some((t) => t.expr === "sum by (provider) (loopover_ai_cost_usd_total) or vector(0)")).toBe(true); - expect(targets.some((t) => t.expr === "sum by (provider) ((rate(loopover_ai_input_tokens_total[5m]) + rate(loopover_ai_output_tokens_total[5m])) * 60)")).toBe(true); - expect(targets.some((t) => t.expr === "sum by (model, effort) (increase(loopover_ai_requests_total[1h]))")).toBe(true); - expect(targets.some((t) => t.expr === "sum by (primary, fallback) (increase(loopover_ai_review_model_fallback_total[1h]))")).toBe(true); + expect(targets.some((t) => t.expr === "sum by (provider) ((loopover_ai_cost_usd_total or gittensory_ai_cost_usd_total)) or vector(0)")).toBe(true); + expect(targets.some((t) => t.expr === "sum by (provider) (((rate(loopover_ai_input_tokens_total[5m]) or rate(gittensory_ai_input_tokens_total[5m])) + (rate(loopover_ai_output_tokens_total[5m]) or rate(gittensory_ai_output_tokens_total[5m]))) * 60)")).toBe(true); + expect(targets.some((t) => t.expr === "sum by (model, effort) ((increase(loopover_ai_requests_total[1h]) or increase(gittensory_ai_requests_total[1h])))")).toBe(true); + expect(targets.some((t) => t.expr === "sum by (primary, fallback) ((increase(loopover_ai_review_model_fallback_total[1h]) or increase(gittensory_ai_review_model_fallback_total[1h])))")).toBe(true); // From codex-usage.json. - expect(targets.some((t) => t.expr === "sum by (model, effort) (increase(loopover_ai_requests_total{provider=\"codex\"}[$__rate_interval]))")).toBe(true); + expect(targets.some((t) => t.expr === "sum by (model, effort) ((increase(loopover_ai_requests_total{provider=\"codex\"}[$__rate_interval]) or increase(gittensory_ai_requests_total{provider=\"codex\"}[$__rate_interval])))")).toBe(true); // From claude-usage.json's OTEL section (uses $claudeModel, not $model, to stay independent of the durable-log filters). expect(targets.some((t) => t.expr === "sum(last_over_time(claude_code_cost_usage_USD_total{model=~\"$claudeModel\"}[$__range]))")).toBe(true); }); + // REGRESSION: #5522 hard-cutover renamed this dashboard's loopover_ai_* queries from their pre-rebrand + // gittensory_ai_* names with no historical fallback, so every panel here only ever showed data recorded + // after that cutover -- confirmed live (both metric names have real historical series in Prometheus). + // Mirrors the (loopover_x or gittensory_x) union fix applied to grafana/dashboards/gittensory.json in + // #6779/#6787, including that fix's own lesson: a label matcher like {provider="codex"} must bind to each + // side of the union individually, never to the closing paren of the union as a whole. + it("unions every loopover_ai_* query with its pre-rebrand gittensory_ai_* counterpart for historical continuity (#5522 follow-up)", () => { + const targets = readDashboard().panels.flatMap((panel) => panel.targets ?? []); + + for (const target of targets) { + if (!target.expr?.includes("loopover_ai_")) continue; + expect(target.expr, `missing historical union: ${target.expr}`).toContain("gittensory_ai_"); + expect(target.expr, `invalid PromQL -- label matcher applied after a closing paren: ${target.expr}`).not.toMatch(/\)\s*\{/); + } + + expect(targets.some((t) => t.expr === 'sum by (provider, kind) ((loopover_ai_input_tokens_total or gittensory_ai_input_tokens_total))')).toBe(true); + expect(targets.some((t) => t.expr === 'sum by (provider, kind) ((loopover_ai_output_tokens_total or gittensory_ai_output_tokens_total))')).toBe(true); + expect( + targets.some( + (t) => + t.expr === + 'sum by (kind) ((increase(loopover_ai_input_tokens_total{provider="codex"}[$__rate_interval]) or increase(gittensory_ai_input_tokens_total{provider="codex"}[$__rate_interval])))', + ), + ).toBe(true); + expect( + targets.some( + (t) => + t.expr === + 'sum by (kind) ((increase(loopover_ai_output_tokens_total{provider="codex"}[$__rate_interval]) or increase(gittensory_ai_output_tokens_total{provider="codex"}[$__rate_interval])))', + ), + ).toBe(true); + }); + it("keeps the Claude OTEL section on its own $claudeModel variable, never the durable log's $provider/$feature/$model", () => { const dashboard = readDashboard(); const otelRowIndex = dashboard.panels.findIndex((p) => p.title?.includes("Claude Code native OTEL")); From acb5597c813750c5a8a20893cc69b030830cf6b8 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Fri, 17 Jul 2026 00:09:17 -0700 Subject: [PATCH 2/2] fix(observability): restore historical continuity in the GPU Metrics dashboard Same #5522 hard-cutover gap as ai-usage.json: the 3 AI-provider request metrics (rate/latency/error-rate panels) had no fallback to their pre-rebrand gittensory_ names, truncating history to the post-cutover window. All 3 metrics confirmed to have real historical series in Prometheus under the old name before applying the union fix. No existing test file covered this dashboard at all -- added one scoped to this fix rather than backfilling full dashboard coverage. Deployed to edge-nl-01 and restarted Grafana to force a clean reprovision. --- grafana/dashboards/gpu-metrics.json | 10 +-- ...host-grafana-gpu-metrics-dashboard.test.ts | 73 +++++++++++++++++++ 2 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 test/unit/selfhost-grafana-gpu-metrics-dashboard.test.ts diff --git a/grafana/dashboards/gpu-metrics.json b/grafana/dashboards/gpu-metrics.json index 7629ebeb3f..9d2867fa79 100644 --- a/grafana/dashboards/gpu-metrics.json +++ b/grafana/dashboards/gpu-metrics.json @@ -95,7 +95,7 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (provider, request_kind) (rate(loopover_ai_provider_request_duration_seconds_count[5m]))", + "expr": "sum by (provider, request_kind) ((rate(loopover_ai_provider_request_duration_seconds_count[5m]) or rate(gittensory_ai_provider_request_duration_seconds_count[5m])))", "legendFormat": "{{provider}} / {{request_kind}}", "refId": "A" } @@ -110,9 +110,9 @@ "title": "AI Request Latency (p50 / p95 / p99)", "type": "timeseries", "targets": [ - { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(loopover_ai_provider_request_duration_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A" }, - { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(loopover_ai_provider_request_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B" }, - { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(loopover_ai_provider_request_duration_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C" } + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.50, sum by (le) ((rate(loopover_ai_provider_request_duration_seconds_bucket[5m]) or rate(gittensory_ai_provider_request_duration_seconds_bucket[5m]))))", "legendFormat": "p50", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, sum by (le) ((rate(loopover_ai_provider_request_duration_seconds_bucket[5m]) or rate(gittensory_ai_provider_request_duration_seconds_bucket[5m]))))", "legendFormat": "p95", "refId": "B" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, sum by (le) ((rate(loopover_ai_provider_request_duration_seconds_bucket[5m]) or rate(gittensory_ai_provider_request_duration_seconds_bucket[5m]))))", "legendFormat": "p99", "refId": "C" } ] }, { @@ -127,7 +127,7 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (provider, request_kind) (rate(loopover_ai_provider_request_errors_total[5m]))", + "expr": "sum by (provider, request_kind) ((rate(loopover_ai_provider_request_errors_total[5m]) or rate(gittensory_ai_provider_request_errors_total[5m])))", "legendFormat": "{{provider}} / {{request_kind}}", "refId": "A" } diff --git a/test/unit/selfhost-grafana-gpu-metrics-dashboard.test.ts b/test/unit/selfhost-grafana-gpu-metrics-dashboard.test.ts new file mode 100644 index 0000000000..e6ccefddc3 --- /dev/null +++ b/test/unit/selfhost-grafana-gpu-metrics-dashboard.test.ts @@ -0,0 +1,73 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +type DashboardTarget = { expr?: string }; +type DashboardPanel = { title?: string; targets?: DashboardTarget[] }; +type Dashboard = { panels: DashboardPanel[] }; + +const dashboardPath = join(process.cwd(), "grafana/dashboards/gpu-metrics.json"); + +function readDashboard(): Dashboard { + return JSON.parse(readFileSync(dashboardPath, "utf8")) as Dashboard; +} + +describe("LoopOver GPU Metrics Grafana dashboard", () => { + // REGRESSION: #5522 hard-cutover renamed this dashboard's 3 loopover_ai_provider_* queries from their + // pre-rebrand gittensory_ai_provider_* names with no historical fallback, so every panel here only ever + // showed data recorded after that cutover -- confirmed live, both metric names have real historical series + // in Prometheus going back well past the cutover. Mirrors the (loopover_x or gittensory_x) union fix + // already shipped for grafana/dashboards/gittensory.json (#6779/#6787) and ai-usage.json (#5522 follow-up). + it("unions every loopover_ai_provider_* query with its pre-rebrand gittensory_ai_provider_* counterpart for historical continuity", () => { + const targets = readDashboard().panels.flatMap((panel) => panel.targets ?? []); + + for (const target of targets) { + if (!target.expr?.includes("loopover_ai_provider")) continue; + expect(target.expr, `missing historical union: ${target.expr}`).toContain("gittensory_ai_provider"); + expect(target.expr, `invalid PromQL -- label matcher applied after a closing paren: ${target.expr}`).not.toMatch(/\)\s*\{/); + } + + expect( + targets.some( + (t) => + t.expr === + "sum by (provider, request_kind) ((rate(loopover_ai_provider_request_duration_seconds_count[5m]) or rate(gittensory_ai_provider_request_duration_seconds_count[5m])))", + ), + ).toBe(true); + expect( + targets.some( + (t) => + t.expr === + "histogram_quantile(0.50, sum by (le) ((rate(loopover_ai_provider_request_duration_seconds_bucket[5m]) or rate(gittensory_ai_provider_request_duration_seconds_bucket[5m]))))", + ), + ).toBe(true); + expect( + targets.some( + (t) => + t.expr === + "histogram_quantile(0.95, sum by (le) ((rate(loopover_ai_provider_request_duration_seconds_bucket[5m]) or rate(gittensory_ai_provider_request_duration_seconds_bucket[5m]))))", + ), + ).toBe(true); + expect( + targets.some( + (t) => + t.expr === + "histogram_quantile(0.99, sum by (le) ((rate(loopover_ai_provider_request_duration_seconds_bucket[5m]) or rate(gittensory_ai_provider_request_duration_seconds_bucket[5m]))))", + ), + ).toBe(true); + expect( + targets.some( + (t) => + t.expr === + "sum by (provider, request_kind) ((rate(loopover_ai_provider_request_errors_total[5m]) or rate(gittensory_ai_provider_request_errors_total[5m])))", + ), + ).toBe(true); + }); + + it("declares a stable title and uid", () => { + const dashboard = readDashboard() as Dashboard & { title?: string; uid?: string }; + + expect(dashboard.title).toBe("LoopOver — GPU Metrics"); + expect(dashboard.uid).toBe("loopover-gpu"); + }); +});