feat(webapp,run-engine): queue metrics and health dashboard

.server-changes/queue-metrics-dashboard.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: feature
+---
+
+Queue metrics and health on the Queues page: per-queue depth, throughput, concurrency, throttling, and scheduling-delay charts, plus a per-queue detail view, live queue stats and a backlog chart on the task detail page, and a waiting-in-queue explainer in the run inspector for runs that have not started yet. Off by default; enabled per organization.

apps/webapp/app/components/primitives/UsageSparkline.tsx

@@ -27,6 +27,8 @@ export type UsageSparklineProps = {
   color?: string;
   /** Unit shown in the tooltip (e.g. calls, tokens). */
   unitLabel?: UnitLabel;
+  /** Trailing scalar shown after the chart. Defaults to the sum of buckets (override for gauges, e.g. peak). */
+  total?: number;
   /** Format the trailing total. Defaults to `toLocaleString`. */
   formatTotal?: (total: number) => string;
   /** Class for the trailing total label. */
@@ -44,14 +46,16 @@ export function UsageSparkline({
   bucketIntervalMs,
   color = "#3B82F6",
   unitLabel = { singular: "call", plural: "calls" },
+  total: totalOverride,
   formatTotal,
   totalClassName = "text-blue-400",
 }: UsageSparklineProps) {
-  if (!data || data.every((v) => v === 0)) {
+  const hasTotalOverride = totalOverride !== undefined;
+  if (!data || data.length === 0 || (data.every((v) => v === 0) && !hasTotalOverride)) {
     return <span className="text-text-dimmed">–</span>;
   }
 
-  const total = data.reduce((a, b) => a + b, 0);
+  const total = totalOverride ?? data.reduce((a, b) => a + b, 0);
   const max = Math.max(...data);
 
   // Map each bucket to a dated point so the tooltip can show the window it

apps/webapp/app/components/primitives/charts/Chart.tsx

@@ -216,7 +216,7 @@ const ChartTooltipContent = React.forwardRef<
                       )}
                       <div
                         className={cn(
-                          "flex flex-1 justify-between leading-none",
+                          "flex flex-1 justify-between gap-3 leading-none",
                           nestLabel ? "items-end" : "items-center"
                         )}
                       >

apps/webapp/app/components/primitives/charts/ChartLine.tsx

@@ -4,6 +4,7 @@ import {
   CartesianGrid,
   Line,
   LineChart,
+  ReferenceLine,
   XAxis,
   YAxis,
   type XAxisProps,
@@ -48,12 +49,38 @@ export type ChartLineRendererProps = {
   tooltipLabelFormatter?: (label: string, payload: any[]) => string;
   /** Optional formatter for numeric tooltip values (e.g. bytes, duration) */
   tooltipValueFormatter?: (value: number) => string;
+  /** Draw a dot at each data point. Defaults to true; turn off for dense/compact charts. */
+  showDots?: boolean;
+  /** Horizontal reference lines (e.g. limits); the y-domain extends to include them. */
+  referenceLines?: Array<{ y: number; label?: string; color?: string }>;
   /** Width injected by ResponsiveContainer */
   width?: number;
   /** Height injected by ResponsiveContainer */
   height?: number;
 };
 
+/** Reference-line label: right-aligned just below the line (recharts injects viewBox). */
+function ReferenceLineLabel({
+  viewBox,
+  value,
+}: {
+  viewBox?: { x: number; y: number; width: number };
+  value: string;
+}) {
+  if (!viewBox) return null;
+  return (
+    <text
+      x={viewBox.x + viewBox.width - 4}
+      y={viewBox.y + 12}
+      textAnchor="end"
+      fill="#878C99"
+      fontSize={10}
+    >
+      {value}
+    </text>
+  );
+}
+
 /**
  * Line chart renderer for the compound component system.
  * Must be used within a Chart.Root.
@@ -73,6 +100,8 @@ export function ChartLineRenderer({
   stacked = false,
   tooltipLabelFormatter,
   tooltipValueFormatter,
+  showDots = true,
+  referenceLines,
   width,
   height,
 }: ChartLineRendererProps) {
@@ -176,6 +205,17 @@ export function ChartLineRenderer({
           labelFormatter={tooltipLabelFormatter}
         />
         {/* Note: Legend is now rendered by ChartRoot outside the chart container */}
+        {referenceLines?.map((line) => (
+          <ReferenceLine
+            key={`ref-${line.y}-${line.label ?? ""}`}
+            y={line.y}
+            stroke={line.color ?? "#4D525B"}
+            strokeDasharray="4 4"
+            strokeWidth={1}
+            ifOverflow="extendDomain"
+            label={line.label ? <ReferenceLineLabel value={line.label} /> : undefined}
+          />
+        ))}
         {visibleSeries.map((key) => (
           <Area
             key={key}
@@ -222,14 +262,25 @@ export function ChartLineRenderer({
         labelFormatter={tooltipLabelFormatter}
       />
       {/* Note: Legend is now rendered by ChartRoot outside the chart container */}
+      {referenceLines?.map((line) => (
+        <ReferenceLine
+          key={`ref-${line.y}-${line.label ?? ""}`}
+          y={line.y}
+          stroke={line.color ?? "#4D525B"}
+          strokeDasharray="4 4"
+          strokeWidth={1}
+          ifOverflow="extendDomain"
+          label={line.label ? <ReferenceLineLabel value={line.label} /> : undefined}
+        />
+      ))}
       {visibleSeries.map((key) => (
         <Line
           key={key}
           dataKey={key}
           type={lineType}
           stroke={config[key]?.color}
           strokeWidth={1}
-          dot={{ r: 1.5, fill: config[key]?.color, strokeWidth: 0 }}
+          dot={showDots ? { r: 1.5, fill: config[key]?.color, strokeWidth: 0 } : false}
           activeDot={{ r: 4 }}
           isAnimationActive={false}
         />

apps/webapp/app/components/query/QueryEditor.tsx

@@ -72,7 +72,7 @@ import type { action as titleAction } from "~/routes/resources.orgs.$organizatio
 import type { QueryScope } from "~/services/queryService.server";
 import { downloadFile, rowsToCSV, rowsToJSON } from "~/utils/dataExport";
 import { organizationBillingPath } from "~/utils/pathBuilder";
-import { querySchemas } from "~/v3/querySchemas";
+import { visibleQuerySchemas } from "~/v3/querySchemas";
 
 /** Convert a Date or ISO string to ISO string format */
 function toISOString(value: Date | string): string {
@@ -245,7 +245,7 @@ const QueryEditorForm = forwardRef<
       <TSQLEditor
         defaultValue={query}
         onChange={setQuery}
-        schema={querySchemas}
+        schema={visibleQuerySchemas}
         linterEnabled={true}
         showCopyButton={true}
         showClearButton={true}

apps/webapp/app/components/queues/QueueMetricCards.tsx

@@ -0,0 +1,227 @@
+import { useMemo } from "react";
+import { buildActivityTimeAxis } from "~/components/primitives/charts/activityTimeAxis";
+import {
+  Chart,
+  type ChartConfig,
+  type ChartState,
+} from "~/components/primitives/charts/ChartCompound";
+import { ChartCard } from "~/components/primitives/charts/ChartCard";
+import {
+  useMetricResourceQuery,
+  type MetricResourceTimeRange,
+} from "~/hooks/useMetricResourceQuery";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import { useSearchParams } from "~/hooks/useSearchParam";
+import { cn } from "~/utils/cn";
+
+// Shared building blocks for queue-metric UI (queue detail page, task detail page,
+// run inspector). All CH-derived data is fetched client-side through useQueueMetric
+// so pages render instantly; loaders only supply live counts and identifiers.
+
+export const QUEUE_METRIC_COLORS = {
+  running: "#6366F1",
+  limit: "#4D525B",
+  queued: "#A78BFA",
+  p50: "#22D3EE",
+  p95: "#F59E0B",
+  p99: "#EF4444",
+  throttled: "#F59E0B",
+  ckKeys: "#34D399",
+  ckWait: "#F59E0B",
+};
+
+export const QUEUE_METRICS_DEFAULT_PERIOD = "1d";
+
+export type QueueMetricIds = {
+  organizationId: string;
+  projectId: string;
+  environmentId: string;
+};
+
+export type QueueMetricTimeRange = MetricResourceTimeRange;
+
+export function useQueueMetric(
+  query: string,
+  opts: {
+    ids: QueueMetricIds;
+    timeRange: QueueMetricTimeRange;
+    queueName: string;
+    fillGaps?: boolean;
+    /** Match the host page's TimeFilter default (e.g. "7d" on task detail). */
+    defaultPeriod?: string;
+  }
+) {
+  return useMetricResourceQuery(query, {
+    ...opts.ids,
+    timeRange: opts.timeRange,
+    defaultPeriod: opts.defaultPeriod ?? QUEUE_METRICS_DEFAULT_PERIOD,
+    queues: [opts.queueName],
+    fillGaps: opts.fillGaps,
+  });
+}
+
+export function toNumber(value: number | string | null | undefined): number {
+  const n = typeof value === "number" ? value : Number(value);
+  return Number.isFinite(n) ? n : 0;
+}
+
+export function clickhouseTimeToMs(value: unknown): number {
+  const s = String(value).replace(" ", "T");
+  return Date.parse(s.endsWith("Z") ? s : `${s}Z`);
+}
+
+export function formatWaitMs(ms: number): string {
+  if (ms < 1000) return `${Math.round(ms)}ms`;
+  if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`;
+  if (ms < 3_600_000) return `${(ms / 60_000).toFixed(1)}m`;
+  return `${(ms / 3_600_000).toFixed(1)}h`;
+}
+
+export type QueueMetricSeriesConfig = { key: string; label: string; color: string };
+
+export function QueueMetricChartCard({
+  title,
+  query,
+  series,
+  ids,
+  timeRange,
+  queueName,
+  valueFormat,
+  fillGaps,
+  defaultPeriod,
+  className,
+}: {
+  title: string;
+  query: string;
+  series: QueueMetricSeriesConfig[];
+  ids: QueueMetricIds;
+  timeRange: QueueMetricTimeRange;
+  queueName: string;
+  valueFormat?: (value: number) => string;
+  fillGaps?: boolean;
+  defaultPeriod?: string;
+  className?: string;
+}) {
+  const { rows, showLoading, failed } = useQueueMetric(query, {
+    ids,
+    timeRange,
+    queueName,
+    fillGaps,
+    defaultPeriod,
+  });
+
+  const data = useMemo(() => {
+    return rows
+      .map((r) => {
+        const point: { bucket: number } & Record<string, number> = {
+          bucket: clickhouseTimeToMs(r.t),
+        };
+        for (const s of series) point[s.key] = toNumber(r[s.key]);
+        return point;
+      })
+      .filter((p) => Number.isFinite(p.bucket));
+  }, [rows, series]);
+
+  const chartConfig = useMemo(() => {
+    const cfg: ChartConfig = {};
+    for (const s of series) cfg[s.key] = { label: s.label, color: s.color };
+    return cfg;
+  }, [series]);
+
+  const { tickFormatter, tooltipLabelFormatter } = useMemo(
+    () => buildActivityTimeAxis(data),
+    [data]
+  );
+
+  const state: ChartState = showLoading ? "loading" : failed ? "invalid" : undefined;
+
+  return (
+    <div className={className ?? "h-64"}>
+      <ChartCard title={title}>
+        <Chart.Root
+          config={chartConfig}
+          data={data}
+          dataKey="bucket"
+          series={series.map((s) => s.key)}
+          state={state}
+          fillContainer
+        >
+          <Chart.Line
+            lineType="monotone"
+            xAxisProps={{ tickFormatter }}
+            yAxisProps={valueFormat ? { tickFormatter: (v: number) => valueFormat(v) } : undefined}
+            tooltipLabelFormatter={tooltipLabelFormatter}
+            tooltipValueFormatter={valueFormat}
+          />
+        </Chart.Root>
+      </ChartCard>
+    </div>
+  );
+}
+
+export type QueueLiveCounts = { queued: number; running: number };
+
+// Compact two-line stat block for task detail sidebars: live counts from the loader,
+// then delay p95 + peak backlog over the page's TimeFilter range.
+export function QueueSidebarStats({
+  live,
+  ids,
+  queueName,
+  defaultPeriod,
+}: {
+  live: QueueLiveCounts;
+  ids: QueueMetricIds;
+  queueName: string;
+  defaultPeriod?: string;
+}) {
+  const { value } = useSearchParams();
+  const timeRange: QueueMetricTimeRange = {
+    period: value("period") ?? null,
+    from: value("from") ?? null,
+    to: value("to") ?? null,
+  };
+
+  const { rows, showLoading } = useQueueMetric(
+    `SELECT max(max_queued) AS peak_queued,\n  round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS worst_p95\nFROM queue_metrics`,
+    { ids, timeRange, queueName, defaultPeriod }
+  );
+  const row = rows[0];
+  const worstP95 = row ? toNumber(row.worst_p95) : 0;
+  const peakQueued = row ? toNumber(row.peak_queued) : 0;
+
+  return (
+    <>
+      <Paragraph variant="extra-small" className="tabular-nums text-text-dimmed">
+        Queued now {live.queued.toLocaleString()} · Running now {live.running.toLocaleString()}
+      </Paragraph>
+      <Paragraph variant="extra-small" className="tabular-nums text-text-dimmed">
+        {showLoading || !row
+          ? "…"
+          : `Delay p95 ${worstP95 > 0 ? formatWaitMs(worstP95) : "–"} · Peak backlog ${peakQueued.toLocaleString()}`}
+      </Paragraph>
+    </>
+  );
+}
+
+export function QueueMetricStat({
+  label,
+  value,
+  className,
+  loading,
+}: {
+  label: string;
+  value: string;
+  className?: string;
+  loading?: boolean;
+}) {
+  return (
+    <div className="rounded-sm border border-grid-dimmed bg-background-bright px-3 py-2">
+      <div className="text-xs text-text-dimmed">{label}</div>
+      {loading ? (
+        <div className="mt-1 h-6 w-12 animate-pulse rounded bg-grid-bright/50" />
+      ) : (
+        <div className={cn("text-2xl tabular-nums text-text-bright", className)}>{value}</div>
+      )}
+    </div>
+  );
+}