diff --git a/.changeset/calendar-day-upper-bound-memory-mongodb.md b/.changeset/calendar-day-upper-bound-memory-mongodb.md new file mode 100644 index 0000000000..fc26b2093c --- /dev/null +++ b/.changeset/calendar-day-upper-bound-memory-mongodb.md @@ -0,0 +1,30 @@ +--- +"@objectstack/driver-memory": patch +"@objectstack/driver-mongodb": patch +--- + +fix(driver-memory,driver-mongodb): a bare-day upper bound covers the whole day (#4042) + +The non-SQL half of #3777's calendar-day rule. Both drivers compiled a bare +`YYYY-MM-DD` `$lte` (and a `between` max) as-is, so on timestamp values the +window cut off at the final day's midnight — the dashboard date-range filter's +default configuration (`created_at`, 7 of 13 presets ending "today") lost the +current day, exactly as it did on SQL before #3777 was fixed. + +Both drivers now compile a bare-day upper bound half-open, sharing +`nextUtcCalendarDay` from `@objectstack/core`: + +- `driver-memory`: the Mongo-style and array `where` spellings in the mingo + lowering (`$lte`/`<=` → `$lt` next day; `$between`/`between` max the same), + the analytics cube-filter `lte`, and the analytics `dateRange` window — which + now also matches BOTH stored forms of a timestamp (ISO strings and `Date` + objects) instead of only `Date`s, since mingo compares cross-type as + never-equal. +- `driver-mongodb`: the `translateFilter` lowering, all three spellings + (`$lte`, `$between`, array `<=`/`lte`). + +Unchanged on purpose, matching the #3777 semantics table: full-ISO/`Date` +comparands keep instant semantics, and `$gte`/`$gt`/`$lt` keep their midnight +anchoring. Known remaining gap (tracked separately): values stored as BSON +`Date` (mongodb) or JS `Date` (memory `find()`) never match *string* comparands +of any operator — a storage-form problem, not a bound-semantics one. diff --git a/packages/plugins/driver-memory/src/memory-analytics.ts b/packages/plugins/driver-memory/src/memory-analytics.ts index d87451b7d3..b76ef47bad 100644 --- a/packages/plugins/driver-memory/src/memory-analytics.ts +++ b/packages/plugins/driver-memory/src/memory-analytics.ts @@ -3,7 +3,7 @@ import type { IAnalyticsService, AnalyticsResult, CubeMeta } from '@objectstack/spec/contracts'; import type { Cube, AnalyticsQuery } from '@objectstack/spec/data'; import type { InMemoryDriver } from './memory-driver.js'; -import { Logger, createLogger } from '@objectstack/core'; +import { Logger, createLogger, nextUtcCalendarDay } from '@objectstack/core'; /** * Configuration for MemoryAnalyticsService @@ -91,6 +91,13 @@ export class MemoryAnalyticsService implements IAnalyticsService { matchStage[fieldPath] = { $in: coerced }; } else if (mongoOp === '$nin') { matchStage[fieldPath] = { $nin: coerced }; + } else if (mongoOp === '$lte') { + // A bare-day `lte` bound means "through that whole day" (#4042; + // the SQL twin is #3777): compile half-open so timestamp values on + // the final day stay in. Order-equivalent to `$lte` for plain + // `YYYY-MM-DD` values. + const nextDay = nextUtcCalendarDay(coerced[0]); + matchStage[fieldPath] = nextDay != null ? { $lt: nextDay } : { $lte: coerced[0] }; } else { matchStage[fieldPath] = { [mongoOp]: coerced[0] }; } @@ -108,17 +115,39 @@ export class MemoryAnalyticsService implements IAnalyticsService { for (const timeDim of query.timeDimensions) { const fieldPath = this.resolveFieldPath(cube, timeDim.dimension); if (timeDim.dateRange) { - const range = Array.isArray(timeDim.dateRange) - ? timeDim.dateRange + const range = Array.isArray(timeDim.dateRange) + ? timeDim.dateRange : this.parseDateRangeString(timeDim.dateRange); - + if (range.length === 2) { + // The window matches BOTH stored forms of a datetime value — the + // in-memory table holds whatever the writer produced: `Date` + // objects from direct JS callers AND ISO strings (the driver's own + // `created_at` default, every REST/JSON write). Mingo compares + // cross-type as never-equal, so a single-form bound silently + // empties the other half — the same disease driver-sql's + // mixed-storage CASE repair cures, expressed as the `$or` a + // schemaless store allows. + // + // Both spellings are half-open on a bare-day end (#4042; the SQL + // twin is #3777): a `$lte`-at-midnight upper bound dropped the + // final day's rows for `Date` values and the string spelling + // inherits `<= day`'s whole-day intent via `< nextDay`. + const start = String(range[0]); + const end = String(range[1]); + const nextDay = nextUtcCalendarDay(end); + const stringBounds = nextDay != null + ? { $gte: start, $lt: nextDay } + : { $gte: start, $lte: end }; + const dateBounds = nextDay != null + ? { $gte: new Date(start), $lt: new Date(`${nextDay}T00:00:00.000Z`) } + : { $gte: new Date(start), $lte: new Date(end) }; pipeline.push({ $match: { - [fieldPath]: { - $gte: new Date(range[0]), - $lte: new Date(range[1]) - } + $or: [ + { [fieldPath]: stringBounds }, + { [fieldPath]: dateBounds }, + ], } }); } diff --git a/packages/plugins/driver-memory/src/memory-driver-calendar-day-upper-bound.test.ts b/packages/plugins/driver-memory/src/memory-driver-calendar-day-upper-bound.test.ts new file mode 100644 index 0000000000..6a19a275c4 --- /dev/null +++ b/packages/plugins/driver-memory/src/memory-driver-calendar-day-upper-bound.test.ts @@ -0,0 +1,149 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * Calendar-day upper bounds in the in-memory driver (#4042) — the + * driver-memory half of #3777's rule: a bare `YYYY-MM-DD` used as an upper + * bound (`$lte`, `<=`, a `between` max, a dateRange end) means "through that + * whole day" and compiles half-open (`$lt` next day). + * + * Rows store ISO-string timestamps — the driver's OWN storage form (its + * `created_at` default is `new Date().toISOString()`, and every REST/JSON + * write arrives as a string). Mingo compares strings lexicographically, so + * pre-fix `$lte: '2026-07-28'` cut the window at the midnight prefix and the + * final day's rows vanished, exactly like the SQL drivers. (`Date`-object + * rows are a separate storage-form problem — mingo compares cross-type as + * never-equal for EVERY operator, `$gte` included — covered for the analytics + * window below and tracked separately for `find()`.) + */ + +import { describe, it, expect, beforeEach } from 'vitest'; +import { InMemoryDriver } from './memory-driver.js'; +import { MemoryAnalyticsService } from './memory-analytics.js'; +import type { Cube } from '@objectstack/spec/data'; + +const ids = (rows: any[]) => rows.map((r: any) => r.id).sort(); + +describe('InMemoryDriver — bare-day $lte covers the whole day (#4042)', () => { + let driver: InMemoryDriver; + + beforeEach(async () => { + driver = new InMemoryDriver({ + initialData: { + task: [ + { id: 't_midnight', title: 't_midnight', created_at: '2026-07-28T00:00:00.000Z' }, + { id: 't_morning', title: 't_morning', created_at: '2026-07-28T09:15:00.000Z' }, + { id: 't_evening', title: 't_evening', created_at: '2026-07-28T21:40:00.000Z' }, + { id: 't_yesterday', title: 't_yesterday', created_at: '2026-07-27T14:00:00.000Z' }, + { id: 't_old', title: 't_old', created_at: '2026-04-19T10:00:00.000Z' }, + ], + }, + }); + await driver.connect(); + }); + + it('keeps the whole final day — the dashboard default-config window', async () => { + const found = await driver.find('task', { + where: { created_at: { $gte: '2026-04-29', $lte: '2026-07-28' } }, + } as any); + // Pre-fix: only [t_midnight, t_yesterday] — the 09:15 / 21:40 rows fell + // past the midnight-anchored string prefix. + expect(ids(found)).toEqual(['t_evening', 't_midnight', 't_morning', 't_yesterday']); + }); + + it('a full-ISO $lte keeps instant semantics — only the bare day is widened', async () => { + const found = await driver.find('task', { + where: { created_at: { $lte: '2026-07-28T12:00:00.000Z' } }, + } as any); + expect(ids(found)).toEqual(['t_midnight', 't_morning', 't_old', 't_yesterday']); + }); + + it('$gte / $gt / $lt keep their midnight anchoring', async () => { + const gte = await driver.find('task', { where: { created_at: { $gte: '2026-07-28' } } } as any); + expect(ids(gte)).toEqual(['t_evening', 't_midnight', 't_morning']); + + const gt = await driver.find('task', { where: { created_at: { $gt: '2026-07-28' } } } as any); + expect(ids(gt)).toEqual(['t_evening', 't_midnight', 't_morning']); // string '…T00:00' > '2026-07-28' + + const lt = await driver.find('task', { where: { created_at: { $lt: '2026-07-28' } } } as any); + expect(ids(lt)).toEqual(['t_old', 't_yesterday']); + }); + + it('$between with a bare-day max covers the whole final day', async () => { + const found = await driver.find('task', { + where: { created_at: { $between: ['2026-04-29', '2026-07-28'] } }, + } as any); + expect(ids(found)).toEqual(['t_evening', 't_midnight', 't_morning', 't_yesterday']); + }); + + it('stays correct inside an $or branch', async () => { + const found = await driver.find('task', { + where: { + $or: [ + { created_at: { $gte: '2026-07-28', $lte: '2026-07-28' } }, // "today" preset + { title: 't_old' }, + ], + }, + } as any); + expect(ids(found)).toEqual(['t_evening', 't_midnight', 't_morning', 't_old']); + }); + + it('applies to the array (`[field, op, value]`) where spelling too', async () => { + const lte = await driver.find('task', { + where: [['created_at', '>=', '2026-04-29'], 'and', ['created_at', '<=', '2026-07-28']], + } as any); + expect(ids(lte)).toEqual(['t_evening', 't_midnight', 't_morning', 't_yesterday']); + + const between = await driver.find('task', { + where: [['created_at', 'between', ['2026-04-29', '2026-07-28']]], + } as any); + expect(ids(between)).toEqual(['t_evening', 't_midnight', 't_morning', 't_yesterday']); + }); +}); + +describe('MemoryAnalyticsService — dateRange window (#4042)', () => { + const CUBE: Cube = { + name: 'tasks', + title: 'Tasks', + sql: 'task', + measures: { + count: { name: 'count', label: 'Count', type: 'count', sql: 'id' }, + }, + dimensions: { + created_at: { name: 'created_at', label: 'Created', type: 'time', sql: 'created_at' }, + }, + } as unknown as Cube; + + it('keeps the final day for BOTH stored forms — ISO strings and Date objects', async () => { + // One column, two writer forms: the driver's own ISO-string default and a + // direct JS `Date`. The window must keep the final day's rows of each — + // the $or-of-both-spellings that stands in for driver-sql's mixed-storage + // CASE repair. + const driver = new InMemoryDriver({ + initialData: { + task: [ + { id: 's_final_day', created_at: '2026-07-28T21:40:00.000Z' }, + { id: 's_in_range', created_at: '2026-07-10T08:00:00.000Z' }, + { id: 's_next_day', created_at: '2026-07-29T00:00:00.000Z' }, + { id: 'd_final_day', created_at: new Date('2026-07-28T09:15:00Z') }, + { id: 'd_in_range', created_at: new Date('2026-07-10T12:00:00Z') }, + { id: 'd_next_day', created_at: new Date('2026-07-29T00:00:00Z') }, + { id: 'd_old', created_at: new Date('2026-04-19T10:00:00Z') }, + ], + }, + }); + await driver.connect(); + const service = new MemoryAnalyticsService({ driver, cubes: [CUBE] }); + + const result = await service.query({ + cube: 'tasks', + measures: ['count'], + timeDimensions: [ + { dimension: 'created_at', dateRange: ['2026-04-29', '2026-07-28'] }, + ], + } as any); + + // 4 rows inside the window (2 per storage form, both final-day rows kept); + // the two next-day-midnight rows and the pre-window Date row are out. + expect(result.rows[0]?.count).toBe(4); + }); +}); diff --git a/packages/plugins/driver-memory/src/memory-driver.ts b/packages/plugins/driver-memory/src/memory-driver.ts index 5c836fb6ec..8a6e7d750f 100644 --- a/packages/plugins/driver-memory/src/memory-driver.ts +++ b/packages/plugins/driver-memory/src/memory-driver.ts @@ -3,7 +3,7 @@ import type { QueryAST, QueryInput, DriverOptions } from '@objectstack/spec/data'; import { canonicalAstOperator } from '@objectstack/spec/data'; import type { IDataDriver } from '@objectstack/spec/contracts'; -import { Logger, createLogger } from '@objectstack/core'; +import { Logger, createLogger, nextUtcCalendarDay } from '@objectstack/core'; import { Query, Aggregator } from 'mingo'; import { getValueByPath } from './memory-matcher.js'; @@ -777,8 +777,15 @@ export class InMemoryDriver implements IDataDriver { return { [field]: { $gte: value } }; case '<': return { [field]: { $lt: value } }; - case '<=': - return { [field]: { $lte: value } }; + case '<=': { + // A bare-day upper bound means "through that whole day" (#4042, the + // driver-sql twin is #3777): `<= 2026-07-28` on an ISO-timestamp value + // compiles half-open (`< 2026-07-29`), which is also order-equivalent + // to `<=` for plain `YYYY-MM-DD` date values — so no field-type lookup + // is needed, exactly the argument the preview evaluator uses. + const nextDay = nextUtcCalendarDay(value); + return { [field]: nextDay != null ? { $lt: nextDay } : { $lte: value } }; + } case 'in': return { [field]: { $in: value } }; case 'nin': case 'not_in': case 'notin': case 'not in': @@ -806,7 +813,13 @@ export class InMemoryDriver implements IDataDriver { return { [field]: { $ne: null } }; case 'between': if (Array.isArray(value) && value.length === 2) { - return { [field]: { $gte: value[0], $lte: value[1] } }; + // Bare-day max → half-open, inheriting `<=`'s whole-day rule (#4042). + const nextDay = nextUtcCalendarDay(value[1]); + return { + [field]: nextDay != null + ? { $gte: value[0], $lt: nextDay } + : { $gte: value[0], $lte: value[1] }, + }; } throw new Error( `[driver-memory] "between" on field "${field}" needs a two-element array, got ` + @@ -916,9 +929,21 @@ export class InMemoryDriver implements IDataDriver { case '$between': if (Array.isArray(val) && val.length === 2) { result.$gte = val[0]; - result.$lte = val[1]; + // Bare-day max → half-open, inheriting `$lte`'s whole-day rule (#4042). + const betweenNextDay = nextUtcCalendarDay(val[1]); + if (betweenNextDay != null) result.$lt = betweenNextDay; + else result.$lte = val[1]; } break; + case '$lte': { + // A bare-day upper bound means "through that whole day" (#4042; the + // driver-sql twin is #3777). Order-equivalent to `<=` for plain + // `YYYY-MM-DD` values, so it applies without a field-type lookup. + const nextDay = nextUtcCalendarDay(val); + if (nextDay != null) result.$lt = nextDay; + else result.$lte = val; + break; + } case '$null': // $null: true → field is null, $null: false → field is not null // Use $eq/$ne null for Mingo compatibility diff --git a/packages/plugins/driver-mongodb/src/mongodb-filter.test.ts b/packages/plugins/driver-mongodb/src/mongodb-filter.test.ts index a04db278ba..f728511b32 100644 --- a/packages/plugins/driver-mongodb/src/mongodb-filter.test.ts +++ b/packages/plugins/driver-mongodb/src/mongodb-filter.test.ts @@ -42,6 +42,44 @@ describe('MongoDB Filter Translator', () => { }); }); + describe('calendar-day upper bounds (#4042; the SQL twin is #3777)', () => { + it('a bare-day $lte compiles half-open — through the whole day', () => { + expect(translateFilter({ created_at: { $lte: '2026-07-28' } })).toEqual({ + created_at: { $lt: '2026-07-29' }, + }); + }); + + it('a full-ISO $lte keeps instant semantics — only the bare day is widened', () => { + expect(translateFilter({ created_at: { $lte: '2026-07-28T12:00:00.000Z' } })).toEqual({ + created_at: { $lte: '2026-07-28T12:00:00.000Z' }, + }); + }); + + it('bare-day $gte / $gt / $lt keep their midnight anchoring', () => { + expect(translateFilter({ created_at: { $gte: '2026-07-28' } })).toEqual({ + created_at: { $gte: '2026-07-28' }, + }); + expect(translateFilter({ created_at: { $lt: '2026-07-28' } })).toEqual({ + created_at: { $lt: '2026-07-28' }, + }); + }); + + it('$between with a bare-day max decomposes half-open, rolling the month', () => { + expect(translateFilter({ created_at: { $between: ['2026-04-29', '2026-07-31'] } })).toEqual({ + created_at: { $gte: '2026-04-29', $lt: '2026-08-01' }, + }); + }); + + it('array-style `<=` takes the same rule', () => { + expect(translateFilter([['created_at', '<=', '2026-07-28']])).toEqual({ + created_at: { $lt: '2026-07-29' }, + }); + expect(translateFilter([['created_at', '<=', '2026-07-28T12:00:00.000Z']])).toEqual({ + created_at: { $lte: '2026-07-28T12:00:00.000Z' }, + }); + }); + }); + describe('string operators', () => { it('translates $contains to $regex', () => { const result = translateFilter({ name: { $contains: 'test' } }); diff --git a/packages/plugins/driver-mongodb/src/mongodb-filter.ts b/packages/plugins/driver-mongodb/src/mongodb-filter.ts index d6a736fcb7..9453288857 100644 --- a/packages/plugins/driver-mongodb/src/mongodb-filter.ts +++ b/packages/plugins/driver-mongodb/src/mongodb-filter.ts @@ -17,6 +17,7 @@ */ import type { Filter } from 'mongodb'; +import { nextUtcCalendarDay } from '@objectstack/core'; /** * Translate an ObjectStack `where` clause into a MongoDB filter document. @@ -120,13 +121,26 @@ function translateFieldOperators(ops: Record): Record): Record