fix(dashboard,charts): send widget query options to the server, order funnel stages by the pipeline (framework#3588)

.changeset/dataset-widget-query-options.md

@@ -0,0 +1,37 @@
+---
+"@object-ui/core": patch
+"@object-ui/plugin-charts": patch
+"@object-ui/plugin-dashboard": patch
+---
+
+fix(dashboard,charts): send widget `dateGranularity`/`sortBy`/`limit` to the query, and give funnels a real stage order (framework#3588)
+
+`DatasetWidget` never read `widget.options`. Four keys an author writes there
+change the query the server compiles, so a widget declaring
+`options: { dateGranularity: 'month' }` grouped by the raw timestamp and drew
+one bar per record, and `sortBy`/`sortOrder` produced no ordering at all.
+
+- `DatasetWidget` lowers `options.dateGranularity`, `options.sortBy` +
+  `options.sortOrder`, and `options.limit` into the `DatasetSelection` it posts.
+  A `sortBy` naming something the widget does not project is dropped rather than
+  sent, so a stale sort key left by an edit degrades to "unordered" instead of
+  failing the widget against the server's stricter validation. These keys also
+  join the refetch signature, so editing one in the designer refetches instead
+  of re-rendering the previous grid.
+- Funnel stages follow a **declared order**. `AdvancedChartImpl` sorted funnel
+  segments by value descending, unconditionally — which overrode any server
+  ordering and rendered a sales pipeline as a tidy narrowing shape whatever the
+  stages' real sequence, hiding the very anomaly (a bulge at Proposal) the chart
+  exists to show. It now honours a `categoryOrder`, which `DatasetWidget`
+  derives from the dimension field's picklist option order — the pipeline order
+  an author already declared on the object — or from an explicit
+  `options.stageOrder`. With no declared order the value-descending default is
+  unchanged, and a category missing from the order is kept (after the declared
+  ones), never dropped.
+- New `@object-ui/core` helpers `buildCategoryOrder` / `buildCategoryRank`,
+  keyed by both the stored value and the display label like the existing
+  `buildOptionColorMap`, so ordering works whether or not the server resolved
+  the dimension's labels.
+
+Requires the framework-side fix in objectstack#3588 for the selection keys to
+take effect server-side.

packages/core/src/utils/__tests__/category-order.test.ts

@@ -0,0 +1,78 @@
+/**
+ * ObjectUI
+ * Copyright (c) 2024-present ObjectStack Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/**
+ * framework#3588 — a picklist's declared option order is the domain order, and
+ * ordered-sequence charts (funnel/pyramid) must render along it rather than
+ * alphabetically or by value.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { buildCategoryOrder, buildCategoryRank } from '../chart-series';
+
+const STAGES = [
+  { value: 'qualification', label: 'Qualification' },
+  { value: 'needs_analysis', label: 'Needs Analysis' },
+  { value: 'proposal', label: 'Proposal' },
+  { value: 'negotiation', label: 'Negotiation' },
+];
+
+describe('buildCategoryOrder', () => {
+  it('keeps the declared order and keys by BOTH value and label', () => {
+    expect(buildCategoryOrder(STAGES)).toEqual([
+      'qualification', 'Qualification',
+      'needs_analysis', 'Needs Analysis',
+      'proposal', 'Proposal',
+      'negotiation', 'Negotiation',
+    ]);
+  });
+
+  it('accepts bare string options', () => {
+    expect(buildCategoryOrder(['a', 'b'])).toEqual(['a', 'b']);
+  });
+
+  it('does not duplicate a label identical to its value', () => {
+    expect(buildCategoryOrder([{ value: 'open', label: 'open' }])).toEqual(['open']);
+  });
+
+  it('returns null for a field with no options, so callers keep their default', () => {
+    expect(buildCategoryOrder(undefined)).toBeNull();
+    expect(buildCategoryOrder([])).toBeNull();
+    expect(buildCategoryOrder('nope')).toBeNull();
+  });
+});
+
+describe('buildCategoryRank', () => {
+  it('ranks value and label of the same option identically enough to sort together', () => {
+    const rank = buildCategoryRank(buildCategoryOrder(STAGES))!;
+    // A row keyed by the stored value and one keyed by the display label both
+    // sort into the same slot relative to the other stages.
+    expect(rank.get('qualification')).toBeLessThan(rank.get('Needs Analysis')!);
+    expect(rank.get('Proposal')).toBeLessThan(rank.get('negotiation')!);
+  });
+
+  it('sorts the real pipeline out of alphabetical order', () => {
+    const rank = buildCategoryRank(buildCategoryOrder(STAGES))!;
+    // Alphabetical is the order analytics returns; the declared order is not it.
+    const alphabetical = ['Needs Analysis', 'Negotiation', 'Proposal', 'Qualification'];
+    const sorted = [...alphabetical].sort(
+      (a, b) => (rank.get(a) ?? Infinity) - (rank.get(b) ?? Infinity),
+    );
+    expect(sorted).toEqual(['Qualification', 'Needs Analysis', 'Proposal', 'Negotiation']);
+  });
+
+  it('first occurrence wins for a repeated key', () => {
+    const rank = buildCategoryRank(['a', 'b', 'a'])!;
+    expect(rank.get('a')).toBe(0);
+  });
+
+  it('returns null for an empty/absent order', () => {
+    expect(buildCategoryRank(null)).toBeNull();
+    expect(buildCategoryRank([])).toBeNull();
+  });
+});

packages/core/src/utils/chart-series.ts

@@ -197,6 +197,58 @@ export function buildOptionColorMap(options: unknown): Record<string, string> |
   return Object.keys(map).length > 0 ? map : null;
 }
 
+/**
+ * Build the DECLARED category order from a select field's `options` — the
+ * sequence the author wrote them in on the object (framework#3588).
+ *
+ * A picklist's option order is already the domain order: a sales `stage` field
+ * lists Qualification → Needs Analysis → Proposal → Negotiation because that IS
+ * the pipeline. Analytics groups by that field and returns buckets in whatever
+ * order the GROUP BY produced (usually alphabetical), which for an
+ * ordered-sequence chart — a funnel above all — draws a shape that reads as a
+ * pipeline but isn't one.
+ *
+ * Emits BOTH the stored `value` and its display `label` per option, adjacent
+ * and in declared order, for the same reason {@link buildOptionColorMap} keys
+ * by both: a chart row's category may carry either, depending on whether the
+ * server resolved the dimension's labels. Rank is "index of first match", so
+ * either key ranks the option identically.
+ *
+ * Returns `null` for a field with no options, so callers keep their existing
+ * ordering (a funnel falls back to descending by value).
+ */
+export function buildCategoryOrder(options: unknown): string[] | null {
+  if (!Array.isArray(options) || options.length === 0) return null;
+  const keys: string[] = [];
+  for (const opt of options) {
+    if (typeof opt === 'string' || typeof opt === 'number' || typeof opt === 'boolean') {
+      keys.push(String(opt));
+      continue;
+    }
+    if (opt && typeof opt === 'object') {
+      const o = opt as { value?: unknown; label?: unknown };
+      if (o.value != null) keys.push(String(o.value));
+      if (o.label != null && String(o.label) !== String(o.value)) keys.push(String(o.label));
+    }
+  }
+  return keys.length > 0 ? keys : null;
+}
+
+/**
+ * Rank map for {@link buildCategoryOrder} keys — `category → position`, first
+ * occurrence wins. Categories absent from the declared order get no entry; the
+ * caller decides where those sort (see `AdvancedChartImpl`, which keeps them
+ * after the declared ones rather than dropping them).
+ */
+export function buildCategoryRank(order: string[] | null | undefined): Map<string, number> | null {
+  if (!Array.isArray(order) || order.length === 0) return null;
+  const rank = new Map<string, number>();
+  order.forEach((key, i) => {
+    if (!rank.has(key)) rank.set(key, i);
+  });
+  return rank.size > 0 ? rank : null;
+}
+
 /**
  * Build a `{ value → label }` map from a select/enum field's `options`, for
  * resolving a grouped dimension's stored value to its display label (fed to

packages/plugin-charts/src/AdvancedChartImpl.funnelOrder.test.tsx

@@ -0,0 +1,141 @@
+/**
+ * ObjectUI
+ * Copyright (c) 2024-present ObjectStack Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/**
+ * framework#3588 — a funnel's shape asserts a sequence.
+ *
+ * The renderer used to sort funnel segments by value descending,
+ * unconditionally — so a sales pipeline rendered as a tidy narrowing shape
+ * regardless of the stages' real order, and any server-side ordering was
+ * silently overridden. With a declared `categoryOrder` the pipeline order wins;
+ * without one the value-descending default is preserved.
+ */
+
+import React from 'react';
+import { describe, it, expect, vi, afterEach } from 'vitest';
+import { render, cleanup } from '@testing-library/react';
+
+// Recharts' ResponsiveContainer measures its parent via ResizeObserver, which
+// reports 0×0 under the headless DOM — so the funnel never paints. Replace it
+// with a fixed-size passthrough so segments actually render.
+vi.mock('recharts', async () => {
+  const actual = await vi.importActual<any>('recharts');
+  return {
+    ...actual,
+    ResponsiveContainer: ({ children }: any) =>
+      React.cloneElement(children, { width: 320, height: 320 }),
+  };
+});
+
+import AdvancedChartImpl from './AdvancedChartImpl';
+
+afterEach(cleanup);
+
+/**
+ * Rows as analytics returns them: alphabetical, and deliberately NOT
+ * monotonically narrowing — Proposal (200) is the largest, so a value sort and
+ * the declared pipeline order produce visibly different funnels.
+ */
+const PIPELINE = [
+  { stage: 'Needs Analysis', total_amount: 120 },
+  { stage: 'Negotiation', total_amount: 90 },
+  { stage: 'Proposal', total_amount: 200 },
+  { stage: 'Qualification', total_amount: 150 },
+];
+
+/** As `buildCategoryOrder` emits it: value and label adjacent, in declared order. */
+const DECLARED = [
+  'qualification', 'Qualification',
+  'needs_analysis', 'Needs Analysis',
+  'proposal', 'Proposal',
+  'negotiation', 'Negotiation',
+];
+
+/**
+ * The segment labels in the order Recharts drew them. `<Funnel>` renders one
+ * `<text>` label per trapezoid in source order, which is exactly the ordering
+ * under test.
+ */
+function drawnOrder(container: HTMLElement): string[] {
+  return Array.from(container.querySelectorAll('text'))
+    .map((t) => t.textContent?.trim() ?? '')
+    .filter(Boolean);
+}
+
+function renderFunnel(extra: Record<string, unknown>) {
+  return render(
+    <AdvancedChartImpl
+      chartType="funnel"
+      data={PIPELINE}
+      xAxisKey="stage"
+      series={[{ dataKey: 'total_amount' }]}
+      config={{ total_amount: { label: 'Amount' } }}
+      isAnimationActive={false}
+      {...extra}
+    />,
+  );
+}
+
+describe('funnel stage order (#3588)', () => {
+  it('draws every stage (guards the assertions below against an empty render)', () => {
+    const { container } = renderFunnel({ categoryOrder: DECLARED });
+    expect(container.querySelectorAll('.recharts-funnel-trapezoid')).toHaveLength(4);
+    expect(drawnOrder(container)).toHaveLength(4);
+  });
+
+  it('draws in the DECLARED pipeline order, not by value', () => {
+    const { container } = renderFunnel({ categoryOrder: DECLARED });
+    expect(drawnOrder(container)).toEqual([
+      'Qualification', 'Needs Analysis', 'Proposal', 'Negotiation',
+    ]);
+  });
+
+  it('keeps the value-descending default when no order is declared', () => {
+    const { container } = renderFunnel({});
+    expect(drawnOrder(container)).toEqual([
+      'Proposal', 'Qualification', 'Needs Analysis', 'Negotiation',
+    ]);
+  });
+
+  it('keeps a category missing from the declared order, after the declared ones', () => {
+    const { container } = render(
+      <AdvancedChartImpl
+        chartType="funnel"
+        data={[...PIPELINE, { stage: 'Closed Won', total_amount: 10 }]}
+        xAxisKey="stage"
+        series={[{ dataKey: 'total_amount' }]}
+        config={{ total_amount: { label: 'Amount' } }}
+        isAnimationActive={false}
+        categoryOrder={DECLARED}
+      />,
+    );
+    // Never dropped — an unlisted stage still renders, and lands last.
+    expect(drawnOrder(container)).toEqual([
+      'Qualification', 'Needs Analysis', 'Proposal', 'Negotiation', 'Closed Won',
+    ]);
+  });
+
+  it('orders by the stored VALUE too, for rows the server left unlabeled', () => {
+    const { container } = render(
+      <AdvancedChartImpl
+        chartType="funnel"
+        data={[
+          { stage: 'negotiation', total_amount: 90 },
+          { stage: 'qualification', total_amount: 150 },
+          { stage: 'proposal', total_amount: 200 },
+        ]}
+        xAxisKey="stage"
+        series={[{ dataKey: 'total_amount' }]}
+        config={{ total_amount: { label: 'Amount' } }}
+        isAnimationActive={false}
+        categoryOrder={DECLARED}
+      />,
+    );
+    expect(drawnOrder(container)).toEqual(['qualification', 'proposal', 'negotiation']);
+  });
+});

packages/plugin-charts/src/AdvancedChartImpl.tsx

@@ -44,6 +44,7 @@ import {
   ChartConfig
 } from './ChartContainerImpl';
 import { mapScatterClick, mapTreemapClick, mapSankeyClick } from './chartDrillEvents';
+import { buildCategoryRank } from '@object-ui/core';
 
 // Default color fallback for chart series
 const DEFAULT_CHART_COLOR = 'hsl(var(--primary))';
@@ -109,6 +110,20 @@ export interface AdvancedChartImplProps {
    * categories absent here fall back to the positional palette, then the theme.
    */
   categoryColors?: Record<string, string>;
+  /**
+   * Declared category order for ordered-sequence charts (funnel / pyramid),
+   * keyed like {@link categoryColors} by the category VALUE or its display
+   * LABEL, listed in domain order.
+   *
+   * A funnel's shape asserts a sequence, so without this the renderer can only
+   * guess at one and sorts by value descending. That is right for a generic
+   * "biggest first" funnel and wrong for a sales pipeline, where the stages
+   * have a declared order (Qualification → Needs Analysis → Proposal →
+   * Negotiation) that a healthy pipeline happens to narrow along — and an
+   * unhealthy one does not. When supplied, this order wins; categories not
+   * listed keep their incoming relative order after the listed ones.
+   */
+  categoryOrder?: string[];
   /**
    * Optional drill-down click handler. Fires when a chart segment is clicked
    * with `{ category, series, value }`. Wired for bar/horizontal-bar/line/
@@ -160,6 +175,7 @@ export default function AdvancedChartImpl({
   className = '',
   colors,
   categoryColors,
+  categoryOrder,
   onChartClick,
   isAnimationActive,
 }: AdvancedChartImplProps) {
@@ -410,15 +426,32 @@ export default function AdvancedChartImpl({
     const funnelClickProps = handleFunnelClick
       ? { onClick: handleFunnelClick, style: { cursor: 'pointer' as const } }
       : {};
-    // Recharts <Funnel> draws segments in source order. For a visually
-    // correct funnel (largest at top, narrowing down) we sort descending
-    // by the numeric value of `dataKey` so authors don't have to pre-sort
-    // their dashboard data.
-    const funnelData = [...data].sort((a, b) => {
-      const av = Number(a?.[dataKey] ?? 0);
-      const bv = Number(b?.[dataKey] ?? 0);
-      return bv - av;
-    });
+    // Recharts <Funnel> draws segments in source order, so this decides the
+    // sequence the funnel asserts.
+    //
+    // With a DECLARED order (framework#3588) — a stage picklist's own option
+    // order, or an explicit `stageOrder` — that order wins: a sales funnel must
+    // read Qualification → Needs Analysis → Proposal → Negotiation whether or
+    // not each stage happens to hold more value than the next. Sorting such a
+    // pipeline by value would manufacture a tidy narrowing shape and hide the
+    // very anomaly (a bulge at Proposal) the chart exists to reveal. Categories
+    // missing from the declared order keep their incoming relative order,
+    // after the declared ones — never dropped.
+    //
+    // Without one, keep the long-standing default: descending by value, so a
+    // generic funnel still narrows downward without authors pre-sorting.
+    const categoryRank = buildCategoryRank(categoryOrder);
+    const funnelData = categoryRank
+      ? [...data].sort((a, b) => {
+          const ar = categoryRank.get(String(a?.[xAxisKey] ?? '')) ?? Number.MAX_SAFE_INTEGER;
+          const br = categoryRank.get(String(b?.[xAxisKey] ?? '')) ?? Number.MAX_SAFE_INTEGER;
+          return ar - br;
+        })
+      : [...data].sort((a, b) => {
+          const av = Number(a?.[dataKey] ?? 0);
+          const bv = Number(b?.[dataKey] ?? 0);
+          return bv - av;
+        });
     return (
       <ChartContainer config={config} className={className} {...containerProps}>
         <FunnelChart>