diff --git a/.changeset/datetime-storage-form-memory-mongodb.md b/.changeset/datetime-storage-form-memory-mongodb.md new file mode 100644 index 0000000000..ade5b1466c --- /dev/null +++ b/.changeset/datetime-storage-form-memory-mongodb.md @@ -0,0 +1,40 @@ +--- +"@objectstack/driver-memory": patch +"@objectstack/driver-mongodb": patch +--- + +fix(driver-memory,driver-mongodb): `Field.datetime` has one storage form per driver (#4047) + +The non-SQL counterpart of ADR-0053 D-B (#3912). Both drivers let the writer +decide a datetime value's runtime type, and both compare across types by type +bracket rather than by value — so a string comparand never matched a `Date` +value, in either direction, for **every** operator including `$gte`. + +A datetime column genuinely held both forms: the drivers' own +`created_at`/`updated_at` defaults bind a `Date` (mongo) or an ISO string +(memory), while REST/JSON writes, relative-date tokens and `initialData` +fixtures supply the other. A dashboard date window therefore answered with +whichever half happened to match the comparand's type — on MongoDB, where +`created_at` is a BSON `Date` and dashboard bounds are strings, that meant +**no rows at all**, which is worse than the final-day loss #3777 fixed. + +Each driver now has one canonical form, applied on write and to every filter +comparand: + +| Driver | `datetime` | `date` | +|---|---|---| +| `driver-mongodb` | BSON `Date` — the dialect's native instant, its `timestamptz` | `YYYY-MM-DD` text | +| `driver-memory` | canonical UTC ISO text (sorts chronologically under the string comparison mingo performs; survives JSON persistence) | `YYYY-MM-DD` text | + +Both learn their temporal fields from `syncSchema`, so an object that was never +declared is left exactly as written — the drivers do not guess types from +values. `driver-memory` additionally converges rows already in the table when +the schema arrives, which catches `initialData` fixtures and anything a +persistence adapter restored (the in-memory analogue of +`backfillCanonicalDatetimes`, and idempotent like it). + +`Field.date` deliberately stays timezone-naive text on both — converting it to +an instant would invent a midnight and re-couple it to a zone. The +calendar-day bound semantics from #3777/#4042 are unchanged and now compose +with the converged storage: the whole-day rewrite runs on the calendar string +first, and only the resulting bound is converted to the storage form. diff --git a/docs/adr/0053-date-and-datetime-semantics.md b/docs/adr/0053-date-and-datetime-semantics.md index d11a5475af..20248eeccc 100644 --- a/docs/adr/0053-date-and-datetime-semantics.md +++ b/docs/adr/0053-date-and-datetime-semantics.md @@ -717,3 +717,73 @@ row-result coverage for the `$lte`/`$between` upper-bound cells now lives in storage, dialect physical forms, boundary rollovers) and the strategy/preview suites; the full matrix program (relative-token × live-driver × timezone) remains open under D-A3. + +--- + +## Addendum (2026-07-30) — the non-SQL drivers get one storage form too (#4047) + +> **Status:** landed. Extends the 2026-07-29 addendum (D-B, `Field.datetime` +> has ONE storage form per SQL dialect) to `driver-memory` and +> `driver-mongodb`, which had the same defect for the same reason and one worse +> symptom. + +### D-E1 — Type-bracket comparison makes a mixed column WORSE than it is on SQL + +MongoDB compares across BSON types by type bracket, and mingo copies that rule +for JS types: a string comparand never matches a `Date` value, in either +direction, **for every operator** — `$gte` included, not just the upper bound +#3777 was about. SQLite's affinity rules at least left one half reachable; here +a window silently answers with whichever half matches the comparand's type. + +And both columns really were mixed. Each driver's own `created_at`/`updated_at` +default writes one form (`new Date()` on mongo, `new Date().toISOString()` in +memory) while REST/JSON writes, relative-date tokens and `initialData` fixtures +supply the other. On MongoDB the dashboard's default window — string bounds +against a BSON-`Date` `created_at` — therefore returned **nothing at all**. + +### D-E2 — One canon per driver, chosen by what the store natively has + +| Driver | `datetime` | Why | +|---|---|---| +| `driver-mongodb` | BSON `Date` | the dialect's native instant: indexable, range-queryable, timezone-unambiguous. The `timestamptz` of this driver. | +| `driver-memory` | canonical UTC ISO text | no native instant type exists; ISO-8601 UTC sorts chronologically under the plain string comparison mingo performs, and it is the wire form, so it survives JSON persistence unchanged. | + +`Field.date` stays timezone-naive `YYYY-MM-DD` text on both, for the Phase 1 +reason: an instant would invent a midnight and re-couple the value to a zone. + +Both drivers learn their temporal fields from `syncSchema` — the only place a +driver is handed an object definition — into the direct equivalent of +`SqlDriver.datetimeFields`/`dateFields`. An **undeclared** object is left +exactly as written: these drivers are schemaless stores, and guessing a type +from a value would coerce data the platform never claimed to own. + +### D-E3 — Existing rows, and how this composes with the bound semantics + +`driver-memory` converges the rows already in a table when the schema arrives, +which is what catches `initialData` fixtures and persistence-adapter restores +(both land before any schema is declared). It is the in-memory analogue of +`backfillCanonicalDatetimes` and idempotent like it. MongoDB gets no automatic +backfill — rewriting a real collection is a migration, not a boot step — so a +pre-existing collection keeps whatever it holds until migrated; new and updated +documents converge from the first write. + +Ordering is load-bearing where D-D meets D-E: the calendar-day upper-bound +rewrite (#3777/#4042) is a *calendar* operation and runs on the bare-day +STRING first; only the resulting bound is converted to the storage form. +Converting first would hand `nextUtcCalendarDay` a `Date`, which it correctly +refuses to widen — and the whole-day window would silently narrow back to an +instant. + +### D-E4 — Consequences for D-A3 + +The conformance matrix's **driver** axis now has non-SQL cells worth asserting, +and its **storage-form** axis gains `mixed-writer-form` for them. Row-result +cover lives in `mongodb-datetime-storage.test.ts` (against a real MongoDB via +`mongodb-memory-server`) and `memory-datetime-storage.test.ts`. Still open +under D-A3: the relative-token × live-driver × timezone combinations. + +One evaluator remains unaligned on both the D-D and D-E rules: +`@objectstack/formula`'s `matchesFilterCondition` (the RLS write-side `check` +path), which cannot depend on `@objectstack/core` for the shared primitive. +Deciding its home — a private copy, as `formula` already keeps for `today()`, +or lowering the utility into `spec`/`types` — is the remaining piece. diff --git a/packages/plugins/driver-memory/src/memory-datetime-storage.test.ts b/packages/plugins/driver-memory/src/memory-datetime-storage.test.ts new file mode 100644 index 0000000000..2d6d868ecb --- /dev/null +++ b/packages/plugins/driver-memory/src/memory-datetime-storage.test.ts @@ -0,0 +1,183 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * `Field.datetime` has ONE storage form in the in-memory driver — canonical + * UTC ISO text (#4047), the memory twin of ADR-0053 D-B (#3912). + * + * mingo compares across JS types the way MongoDB compares across BSON types: + * a string comparand never matches a `Date` value, in either direction, for + * EVERY operator — `$gte` included. And a datetime column really did hold + * both: the driver's own `created_at`/`updated_at` defaults write + * `new Date().toISOString()` (string) while `initialData` fixtures and direct + * SDK callers hand it `Date` objects. So a date window answered with whichever + * half matched the comparand's type and silently dropped the other. + * + * Canonical ISO TEXT is the right canon here — unlike mongo, this store has no + * native instant type, and ISO-8601 UTC text sorts chronologically under plain + * string comparison, which is exactly what mingo does. It is also the wire + * form, so a value round-trips through JSON persistence unchanged. + */ + +import { describe, it, expect, beforeEach } from 'vitest'; +import { InMemoryDriver } from './memory-driver.js'; + +const ids = (rows: any[]) => rows.map((r: any) => r.id).sort(); + +/** The object declaration is what teaches the driver which fields are temporal. */ +const TASK_SCHEMA = { + name: 'task', + fields: { + title: { type: 'string' }, + created_at: { type: 'datetime' }, + due_at: { type: 'datetime' }, + created_on: { type: 'date' }, + }, +}; + +describe('InMemoryDriver Field.datetime storage (#4047)', () => { + let driver: InMemoryDriver; + + beforeEach(async () => { + driver = new InMemoryDriver({}); + await driver.connect(); + await driver.syncSchema('task', TASK_SCHEMA); + }); + + /** Both writer forms of the same instant, in one column. */ + async function seedMixed(): Promise { + const rows: Array<[string, unknown]> = [ + ['s_morning', '2026-07-28T09:15:00.000Z'], + ['s_evening', '2026-07-28T21:40:00.000Z'], + ['d_midnight', new Date('2026-07-28T00:00:00Z')], + ['d_yesterday', new Date('2026-07-27T14:00:00Z')], + ['d_next_day', new Date('2026-07-29T00:00:00Z')], + ['s_old', '2026-04-19T10:00:00.000Z'], + ]; + for (const [id, created_at] of rows) { + await driver.create('task', { id, title: id, created_at }); + } + } + + it('stores every writer form as canonical UTC ISO text', async () => { + await seedMixed(); + const raw = await driver.find('task', {} as any); + for (const row of raw) { + expect(typeof (row as any).created_at, `${(row as any).id} stored form`).toBe('string'); + expect((row as any).created_at).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/); + } + const midnight = raw.find((r: any) => r.id === 'd_midnight') as any; + expect(midnight.created_at).toBe('2026-07-28T00:00:00.000Z'); + }); + + it('a date window reaches rows written in BOTH forms', async () => { + await seedMixed(); + const found = await driver.find('task', { + where: { created_at: { $gte: '2026-04-29', $lte: '2026-07-28' } }, + } as any); + expect(ids(found)).toEqual(['d_midnight', 'd_yesterday', 's_evening', 's_morning']); + }); + + it('a Date-object comparand reaches the same rows', async () => { + await seedMixed(); + const found = await driver.find('task', { + where: { + created_at: { + $gte: new Date('2026-04-29T00:00:00Z'), + $lt: new Date('2026-07-29T00:00:00Z'), + }, + }, + } as any); + expect(ids(found)).toEqual(['d_midnight', 'd_yesterday', 's_evening', 's_morning']); + }); + + it('keeps #3777/#4042 bound semantics on top of the converged storage', async () => { + await seedMixed(); + const gte = await driver.find('task', { where: { created_at: { $gte: '2026-07-28' } } } as any); + expect(ids(gte)).toEqual(['d_midnight', 'd_next_day', 's_evening', 's_morning']); + + const lt = await driver.find('task', { where: { created_at: { $lt: '2026-07-28' } } } as any); + expect(ids(lt)).toEqual(['d_yesterday', 's_old']); + + const instant = await driver.find('task', { + where: { created_at: { $lte: '2026-07-28T12:00:00.000Z' } }, + } as any); + expect(ids(instant)).toEqual(['d_midnight', 'd_yesterday', 's_morning', 's_old']); + }); + + it('$between and the array spelling take the same coercion', async () => { + await seedMixed(); + const between = await driver.find('task', { + where: { created_at: { $between: ['2026-04-29', '2026-07-28'] } }, + } as any); + expect(ids(between)).toEqual(['d_midnight', 'd_yesterday', 's_evening', 's_morning']); + + const array = await driver.find('task', { + where: [['created_at', '>=', '2026-04-29'], 'and', ['created_at', '<=', '2026-07-28']], + } as any); + expect(ids(array)).toEqual(['d_midnight', 'd_yesterday', 's_evening', 's_morning']); + }); + + it('normalises rows supplied as initialData, not just ones written through create()', async () => { + // A seeded fixture is a WRITE the driver never saw — but it is the shape + // most tests and demos use, so leaving it un-normalised would keep the + // mixed column alive on exactly the path people look at first. + const seeded = new InMemoryDriver({ + initialData: { + task: [ + { id: 'i_date', created_at: new Date('2026-07-28T09:15:00Z') }, + { id: 'i_iso', created_at: '2026-07-28T21:40:00.000Z' }, + ], + }, + }); + await seeded.connect(); + await seeded.syncSchema('task', TASK_SCHEMA); + + const found = await seeded.find('task', { + where: { created_at: { $gte: '2026-07-28', $lte: '2026-07-28' } }, + } as any); + expect(ids(found)).toEqual(['i_date', 'i_iso']); + }); + + it('update() puts the new value in the same form', async () => { + // `created_at` is deliberately immutable in this driver (it re-preserves + // the original on every update), so the mutable datetime field is the + // honest subject for an update-path assertion. + await driver.create('task', { id: 'u1', due_at: '2026-04-19T10:00:00.000Z' }); + await driver.update('task', 'u1', { due_at: new Date('2026-07-28T09:15:00Z') }); + const row: any = (await driver.find('task', { where: { id: 'u1' } } as any))[0]; + expect(row.due_at).toBe('2026-07-28T09:15:00.000Z'); + + // …and the converged value is reachable by a date window, which is the + // point of converging it. + const found = await driver.find('task', { + where: { due_at: { $gte: '2026-07-28', $lte: '2026-07-28' } }, + } as any); + expect(ids(found)).toEqual(['u1']); + }); + + it('a Field.date column stays calendar-day text — not widened into an instant', async () => { + for (const [id, created_on] of [ + ['on_early', '2026-04-19'], + ['on_mid', '2026-07-28'], + ['on_late', '2026-07-29'], + // A Date written to a `date` field collapses to its UTC calendar day. + ['on_obj', new Date('2026-07-28T21:40:00Z')], + ] as const) { + await driver.create('task', { id, created_on }); + } + const all = await driver.find('task', {} as any); + for (const row of all) expect(typeof (row as any).created_on).toBe('string'); + expect((all.find((r: any) => r.id === 'on_obj') as any).created_on).toBe('2026-07-28'); + + const found = await driver.find('task', { + where: { created_on: { $gte: '2026-04-29', $lte: '2026-07-28' } }, + } as any); + expect(ids(found)).toEqual(['on_mid', 'on_obj']); + }); + + it('an undeclared object is left alone (no schema → no coercion)', async () => { + await driver.create('freeform', { id: 'f1', when: new Date('2026-07-28T09:15:00Z') }); + const row: any = (await driver.find('freeform', {} as any))[0]; + expect(row.when).toBeInstanceOf(Date); + }); +}); diff --git a/packages/plugins/driver-memory/src/memory-driver.ts b/packages/plugins/driver-memory/src/memory-driver.ts index 8a6e7d750f..f59f26cf78 100644 --- a/packages/plugins/driver-memory/src/memory-driver.ts +++ b/packages/plugins/driver-memory/src/memory-driver.ts @@ -6,6 +6,11 @@ import type { IDataDriver } from '@objectstack/spec/contracts'; import { Logger, createLogger, nextUtcCalendarDay } from '@objectstack/core'; import { Query, Aggregator } from 'mingo'; import { getValueByPath } from './memory-matcher.js'; +import { + coerceTemporalValue, + indexTemporalFields, + type TemporalFieldKind, +} from './memory-temporal.js'; /** * Persistence adapter interface. @@ -87,6 +92,15 @@ export class InMemoryDriver implements IDataDriver { private config: InMemoryDriverConfig; private logger: Logger; private idCounters: Map = new Map(); + + /** + * Declared temporal fields per object, populated by {@link syncSchema} — + * this driver's equivalent of `SqlDriver.datetimeFields`/`dateFields`, and + * the only thing that lets write and filter agree on a storage form (#4047). + * An object absent from this map was never declared, so nothing is coerced + * for it: the driver does not guess types from values. + */ + private temporalFields: Map> = new Map(); private transactions: Map = new Map(); private persistenceAdapter: PersistenceAdapterInterface | null = null; @@ -265,7 +279,7 @@ export class InMemoryDriver implements IDataDriver { // 1. Filter using Mingo if (query.where) { - const mongoQuery = this.convertToMongoQuery(query.where); + const mongoQuery = this.convertToMongoQuery(query.where, object); if (mongoQuery && Object.keys(mongoQuery).length > 0) { const mingoQuery = new Query(mongoQuery); results = mingoQuery.find(results).all(); @@ -335,12 +349,12 @@ export class InMemoryDriver implements IDataDriver { const table = this.getTable(object); - const newRecord = { + const newRecord = this.toStorageForms(object, { id: data.id || this.generateId(object), ...data, created_at: data.created_at || new Date().toISOString(), updated_at: data.updated_at || new Date().toISOString(), - }; + }); table.push(newRecord); this.markDirty(); @@ -362,13 +376,13 @@ export class InMemoryDriver implements IDataDriver { return null; } - const updatedRecord = { + const updatedRecord = this.toStorageForms(object, { ...table[index], ...data, id: table[index].id, // Preserve original ID created_at: table[index].created_at, // Preserve created_at updated_at: new Date().toISOString(), - }; + }); table[index] = updatedRecord; this.markDirty(); @@ -420,7 +434,7 @@ export class InMemoryDriver implements IDataDriver { async count(object: string, query?: QueryAST, options?: DriverOptions) { let records = this.getTable(object); if (query?.where) { - const mongoQuery = this.convertToMongoQuery(query.where); + const mongoQuery = this.convertToMongoQuery(query.where, object); if (mongoQuery && Object.keys(mongoQuery).length > 0) { const mingoQuery = new Query(mongoQuery); records = mingoQuery.find(records).all(); @@ -449,7 +463,7 @@ export class InMemoryDriver implements IDataDriver { let targetRecords = table; if (query && query.where) { - const mongoQuery = this.convertToMongoQuery(query.where); + const mongoQuery = this.convertToMongoQuery(query.where, object); if (mongoQuery && Object.keys(mongoQuery).length > 0) { const mingoQuery = new Query(mongoQuery); targetRecords = mingoQuery.find(targetRecords).all(); @@ -461,11 +475,11 @@ export class InMemoryDriver implements IDataDriver { for (const record of targetRecords) { const index = table.findIndex(r => r.id === record.id); if (index !== -1) { - const updated = { + const updated = this.toStorageForms(object, { ...table[index], ...data, updated_at: new Date().toISOString() - }; + }); table[index] = updated; } } @@ -482,7 +496,7 @@ export class InMemoryDriver implements IDataDriver { const initialLength = table.length; if (query && query.where) { - const mongoQuery = this.convertToMongoQuery(query.where); + const mongoQuery = this.convertToMongoQuery(query.where, object); if (mongoQuery && Object.keys(mongoQuery).length > 0) { const mingoQuery = new Query(mongoQuery); const matched = mingoQuery.find(table).all(); @@ -588,7 +602,7 @@ export class InMemoryDriver implements IDataDriver { async distinct(object: string, field: string, query?: QueryInput): Promise { let records = this.getTable(object); if (query?.where) { - const mongoQuery = this.convertToMongoQuery(query.where); + const mongoQuery = this.convertToMongoQuery(query.where, object); if (mongoQuery && Object.keys(mongoQuery).length > 0) { const mingoQuery = new Query(mongoQuery); records = mingoQuery.find(records).all(); @@ -643,7 +657,7 @@ export class InMemoryDriver implements IDataDriver { }); let results = this.getTable(object).map((r) => ({ ...r })); if (query.where) { - const mongoQuery = this.convertToMongoQuery(query.where); + const mongoQuery = this.convertToMongoQuery(query.where, object); if (mongoQuery && Object.keys(mongoQuery).length > 0) { results = new Query(mongoQuery).find(results).all() as Record[]; } @@ -674,16 +688,16 @@ export class InMemoryDriver implements IDataDriver { * 3. Legacy Array Format: [['field', 'op', value], 'and', ['field2', 'op', value2]] * 4. MongoDB Format: { field: value } or { field: { $eq: value } } (passthrough) */ - private convertToMongoQuery(filters?: any): Record { + private convertToMongoQuery(filters?: any, object?: string): Record { if (!filters) return {}; // AST node format (ObjectQL QueryAST) if (!Array.isArray(filters) && typeof filters === 'object') { if (filters.type === 'comparison') { - return this.convertConditionToMongo(filters.field, filters.operator, filters.value) || {}; + return this.convertConditionToMongo(filters.field, filters.operator, filters.value, object) || {}; } if (filters.type === 'logical') { - const conditions = filters.conditions?.map((c: any) => this.convertToMongoQuery(c)) || []; + const conditions = filters.conditions?.map((c: any) => this.convertToMongoQuery(c, object)) || []; if (conditions.length === 0) return {}; if (conditions.length === 1) return conditions[0]; const op = filters.operator === 'or' ? '$or' : '$and'; @@ -691,7 +705,7 @@ export class InMemoryDriver implements IDataDriver { } // MongoDB/FilterCondition format: { field: value } or { field: { $op: value } } // Translate non-standard operators ($contains, $notContains, etc.) to Mingo-compatible format - return this.normalizeFilterCondition(filters); + return this.normalizeFilterCondition(filters, object); } // Legacy array format @@ -730,7 +744,7 @@ export class InMemoryDriver implements IDataDriver { // `convertConditionToMongo` now throws rather than returning null for an // operator it cannot express, so a dropped condition can no longer // silently widen the result set. - const cond = this.convertConditionToMongo(field, operator, value); + const cond = this.convertConditionToMongo(field, operator, value, object); if (cond) logicGroups[logicGroups.length - 1].conditions.push(cond); } else { throw new Error( @@ -760,7 +774,8 @@ export class InMemoryDriver implements IDataDriver { /** * Convert a single ObjectQL condition to MongoDB operator format. */ - private convertConditionToMongo(field: string, operator: string, value: any): Record | null { + private convertConditionToMongo(field: string, operator: string, value: any, object?: string): Record | null { + const store = (v: any) => this.toStorageForm(object, field, v); // Fold every accepted spelling of one comparison onto a single infix form, // so this switch has one case per comparison rather than one per spelling — // `VALID_AST_OPERATORS` accepts `>`, `gt`, `greater_than`, `greaterthan` and @@ -768,15 +783,15 @@ export class InMemoryDriver implements IDataDriver { // driver and driver-sql accept different vocabularies. #3948. switch (canonicalAstOperator(operator)) { case '=': case '==': - return { [field]: value }; + return { [field]: store(value) }; case '!=': case '<>': - return { [field]: { $ne: value } }; + return { [field]: { $ne: store(value) } }; case '>': - return { [field]: { $gt: value } }; + return { [field]: { $gt: store(value) } }; case '>=': - return { [field]: { $gte: value } }; + return { [field]: { $gte: store(value) } }; case '<': - return { [field]: { $lt: value } }; + return { [field]: { $lt: store(value) } }; case '<=': { // A bare-day upper bound means "through that whole day" (#4042, the // driver-sql twin is #3777): `<= 2026-07-28` on an ISO-timestamp value @@ -784,12 +799,12 @@ export class InMemoryDriver implements IDataDriver { // to `<=` for plain `YYYY-MM-DD` date values — so no field-type lookup // is needed, exactly the argument the preview evaluator uses. const nextDay = nextUtcCalendarDay(value); - return { [field]: nextDay != null ? { $lt: nextDay } : { $lte: value } }; + return { [field]: nextDay != null ? { $lt: store(nextDay) } : { $lte: store(value) } }; } case 'in': - return { [field]: { $in: value } }; + return { [field]: { $in: store(value) } }; case 'nin': case 'not_in': case 'notin': case 'not in': - return { [field]: { $nin: value } }; + return { [field]: { $nin: store(value) } }; case 'contains': case 'like': case 'ilike': return { [field]: { $regex: new RegExp(this.escapeRegex(value), 'i') } }; case 'notcontains': case 'not_contains': @@ -817,8 +832,8 @@ export class InMemoryDriver implements IDataDriver { const nextDay = nextUtcCalendarDay(value[1]); return { [field]: nextDay != null - ? { $gte: value[0], $lt: nextDay } - : { $gte: value[0], $lte: value[1] }, + ? { $gte: store(value[0]), $lt: store(nextDay) } + : { $gte: store(value[0]), $lte: store(value[1]) }, }; } throw new Error( @@ -842,7 +857,7 @@ export class InMemoryDriver implements IDataDriver { * operators ($contains, $notContains, $startsWith, $endsWith, $between, $null) * to Mingo-compatible equivalents ($regex, $gte/$lte, null checks). */ - private normalizeFilterCondition(filter: Record): Record { + private normalizeFilterCondition(filter: Record, object?: string): Record { const result: Record = {}; const extraAndConditions: Record[] = []; @@ -851,13 +866,13 @@ export class InMemoryDriver implements IDataDriver { // Recurse into logical operators if (key === '$and' || key === '$or') { result[key] = Array.isArray(value) - ? value.map((child: any) => this.normalizeFilterCondition(child)) + ? value.map((child: any) => this.normalizeFilterCondition(child, object)) : value; continue; } if (key === '$not') { result[key] = value && typeof value === 'object' - ? this.normalizeFilterCondition(value) + ? this.normalizeFilterCondition(value, object) : value; continue; } @@ -868,7 +883,7 @@ export class InMemoryDriver implements IDataDriver { } // Field-level: value may be primitive (implicit eq) or operator object if (value && typeof value === 'object' && !Array.isArray(value) && !(value instanceof Date) && !(value instanceof RegExp)) { - const normalized = this.normalizeFieldOperators(value); + const normalized = this.normalizeFieldOperators(value, this.temporalKind(object, key)); // Handle multiple regex conditions on the same field (e.g. $startsWith + $endsWith) if (normalized._multiRegex) { const regexConditions: Record[] = normalized._multiRegex; @@ -881,7 +896,8 @@ export class InMemoryDriver implements IDataDriver { result[key] = normalized; } } else { - result[key] = value; + // Implicit equality — still a comparand, so it takes the storage form. + result[key] = this.toStorageForm(object, key, value); } } @@ -907,7 +923,8 @@ export class InMemoryDriver implements IDataDriver { * When multiple regex-producing operators appear on the same field * (e.g. $startsWith + $endsWith), they are combined via $and. */ - private normalizeFieldOperators(ops: Record): Record { + private normalizeFieldOperators(ops: Record, kind?: TemporalFieldKind): Record { + const store = (v: any) => coerceTemporalValue(v, kind); const result: Record = {}; const regexConditions: Record[] = []; @@ -928,11 +945,11 @@ export class InMemoryDriver implements IDataDriver { break; case '$between': if (Array.isArray(val) && val.length === 2) { - result.$gte = val[0]; + result.$gte = store(val[0]); // Bare-day max → half-open, inheriting `$lte`'s whole-day rule (#4042). const betweenNextDay = nextUtcCalendarDay(val[1]); - if (betweenNextDay != null) result.$lt = betweenNextDay; - else result.$lte = val[1]; + if (betweenNextDay != null) result.$lt = store(betweenNextDay); + else result.$lte = store(val[1]); } break; case '$lte': { @@ -940,8 +957,8 @@ export class InMemoryDriver implements IDataDriver { // driver-sql twin is #3777). Order-equivalent to `<=` for plain // `YYYY-MM-DD` values, so it applies without a field-type lookup. const nextDay = nextUtcCalendarDay(val); - if (nextDay != null) result.$lt = nextDay; - else result.$lte = val; + if (nextDay != null) result.$lt = store(nextDay); + else result.$lte = store(val); break; } case '$null': @@ -953,6 +970,12 @@ export class InMemoryDriver implements IDataDriver { result.$ne = null; } break; + // Value comparisons take the field's storage form (#4047); the null / + // existence predicates above are value-independent and must not. + case '$eq': case '$ne': case '$gt': case '$gte': case '$lt': + case '$in': case '$nin': + result[op] = store(val); + break; default: result[op] = val; break; @@ -1099,6 +1122,22 @@ export class InMemoryDriver implements IDataDriver { this.db[object] = []; this.logger.info('Created in-memory table', { object }); } + // Learn the object's temporal fields, then converge the rows ALREADY in the + // table (#4047). Both halves matter: the map is what the write and filter + // paths consult from here on, and the retroactive pass is what catches rows + // this driver never wrote — `initialData` fixtures and anything a + // persistence adapter restored, both of which land before any schema is + // declared. It is the in-memory analogue of `backfillCanonicalDatetimes` + // (ADR-0053 D-B3) and, like it, is idempotent. + const kinds = indexTemporalFields(schema?.fields); + this.temporalFields.set(object, kinds); + if (kinds.size > 0) { + const table = this.db[object]; + for (let i = 0; i < table.length; i++) { + const converged = this.toStorageForms(object, table[i]); + if (converged !== table[i]) table[i] = converged; + } + } } async dropTable(object: string, options?: DriverOptions) { @@ -1113,6 +1152,39 @@ export class InMemoryDriver implements IDataDriver { // Helpers // =================================== + // ── Temporal storage form (#4047) ───────────────────────────────────────── + + /** The declared temporal kind of `field` on `object`, if any. */ + private temporalKind(object: string | undefined, field: string): TemporalFieldKind | undefined { + if (!object) return undefined; + return this.temporalFields.get(object)?.get(field); + } + + /** Put one filter comparand into the storage form of its field. */ + private toStorageForm(object: string | undefined, field: string, value: any): any { + return coerceTemporalValue(value, this.temporalKind(object, field)); + } + + /** + * Put every declared temporal field of a record into its storage form — the + * write half of the convention the filter path reads against. Returns the + * input unchanged (same reference) when nothing needed converting, so the + * common non-temporal case allocates nothing. + */ + private toStorageForms>(object: string, record: T): T { + const kinds = this.temporalFields.get(object); + if (!kinds || kinds.size === 0) return record; + let out: Record | undefined; + for (const [field, kind] of kinds) { + if (!(field in record)) continue; + const coerced = coerceTemporalValue(record[field], kind); + if (coerced === record[field]) continue; + out ??= { ...record }; + out[field] = coerced; + } + return (out as T) ?? record; + } + /** * Apply manual sorting (Mingo sort has CJS build issues). */ diff --git a/packages/plugins/driver-memory/src/memory-temporal.ts b/packages/plugins/driver-memory/src/memory-temporal.ts new file mode 100644 index 0000000000..7aecd8b1ab --- /dev/null +++ b/packages/plugins/driver-memory/src/memory-temporal.ts @@ -0,0 +1,119 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * One storage form per temporal field type in the in-memory driver (#4047) — + * the memory counterpart of ADR-0053 D-B, which gave `Field.datetime` a single + * UTC storage form on every SQL dialect (#3912). + * + * # Why this has to exist + * + * mingo compares across JS types the way MongoDB compares across BSON types: a + * string comparand never matches a `Date` value, in either direction, for every + * operator including `$gte`. And a datetime column genuinely held both forms — + * the driver's own `created_at`/`updated_at` defaults write + * `new Date().toISOString()`, while `initialData` fixtures and direct SDK + * callers hand it `Date` objects. A date window therefore answered with + * whichever half matched the comparand's type, silently dropping the other. + * + * # The canon + * + * | Field type | Stored as | Why | + * |---|---|---| + * | `datetime` | canonical UTC ISO text (`…T…Z`, ms precision) | this store has no native instant type; ISO-8601 UTC sorts chronologically under the plain string comparison mingo performs, and it is the wire form, so it survives JSON persistence unchanged. | + * | `date` | `YYYY-MM-DD` text | timezone-naive by ADR-0053 Phase 1 — an instant would re-couple it to a zone. | + * + * The rule is applied on write ({@link toStorageForms}) and to filter + * comparands, which is the pairing that keeps the two sides from disagreeing. + */ + +/** + * The canonical UTC ISO text form of a `Field.datetime` value. + * + * Total by design: an input this cannot interpret is returned unchanged rather + * than becoming `Invalid Date`, so junk keeps failing the comparison instead of + * silently becoming a wrong instant. A bare `YYYY-MM-DD` means **midnight + * UTC**, stated explicitly so it is never re-read in the host's local zone; the + * whole-day reading of a bare day used as an UPPER bound is the separate, + * operator-sensitive concern `nextUtcCalendarDay` handles (#3777/#4042) before + * this runs. + * + * Mirrors `SqlDriver`'s `canonicalUtcDatetime` — same inputs, same outputs — so + * a fixture that moves between the two backends compares identically. + */ +export function storageDatetimeValue(value: unknown): unknown { + if (value == null) return value; + if (value instanceof Date) { + return Number.isNaN(value.getTime()) ? value : value.toISOString(); + } + if (typeof value === 'number') { + if (!Number.isFinite(value)) return value; + const d = new Date(value); + return Number.isNaN(d.getTime()) ? value : d.toISOString(); + } + if (typeof value !== 'string') return value; + const s = value.trim(); + if (s === '') return value; + // A bare integer (in either form) is epoch milliseconds. + if (/^-?\d+$/.test(s)) { + const d = new Date(Number(s)); + return Number.isNaN(d.getTime()) ? value : d.toISOString(); + } + const iso = /^\d{4}-\d{2}-\d{2}$/.test(s) + ? `${s}T00:00:00.000Z` + // Zone-naive `YYYY-MM-DD[ T]HH:MM[:SS[.fff]]` → its wall clock IS UTC. + : /^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}(:\d{2}(\.\d+)?)?$/.test(s) + ? `${s.replace(' ', 'T')}Z` + : s; + const ms = Date.parse(iso); + return Number.isFinite(ms) ? new Date(ms).toISOString() : value; +} + +/** + * Collapse a `Field.date` value to timezone-naive `YYYY-MM-DD` — a `Date` to + * its UTC calendar day, a string to its leading date. Mirrors + * `SqlDriver.toDateOnly`. + */ +export function storageDateValue(value: unknown): unknown { + if (value == null) return value; + if (value instanceof Date) { + if (Number.isNaN(value.getTime())) return value; + const y = value.getUTCFullYear(); + const m = String(value.getUTCMonth() + 1).padStart(2, '0'); + const d = String(value.getUTCDate()).padStart(2, '0'); + return `${y}-${m}-${d}`; + } + if (typeof value === 'string') { + const trimmed = value.trim(); + if (/^\d{4}-\d{2}-\d{2}/.test(trimmed)) return trimmed.slice(0, 10); + } + return value; +} + +/** Which temporal rule a declared field takes, if any. */ +export type TemporalFieldKind = 'datetime' | 'date'; + +/** + * Put a value into the storage form of a field of `kind`. `undefined` kind — + * a non-temporal field, or an object that was never declared — passes through: + * the driver does not guess types from values. + */ +export function coerceTemporalValue(value: unknown, kind: TemporalFieldKind | undefined): unknown { + if (kind === undefined) return value; + if (Array.isArray(value)) return value.map((v) => coerceTemporalValue(v, kind)); + return kind === 'datetime' ? storageDatetimeValue(value) : storageDateValue(value); +} + +/** + * Index the declared temporal fields of one object. Called from `syncSchema` — + * the only place this driver is handed an object definition. + */ +export function indexTemporalFields( + fields: Record | undefined, +): Map { + const out = new Map(); + for (const [name, def] of Object.entries(fields ?? {})) { + if (def?.type === 'datetime') out.set(name, 'datetime'); + else if (def?.type === 'date') out.set(name, 'date'); + } + return out; +} diff --git a/packages/plugins/driver-mongodb/src/mongodb-aggregation.ts b/packages/plugins/driver-mongodb/src/mongodb-aggregation.ts index 99d70f2f37..813d7404ea 100644 --- a/packages/plugins/driver-mongodb/src/mongodb-aggregation.ts +++ b/packages/plugins/driver-mongodb/src/mongodb-aggregation.ts @@ -9,6 +9,7 @@ import type { Document } from 'mongodb'; import { translateFilter } from './mongodb-filter.js'; +import type { TemporalFieldKindResolver } from './mongodb-temporal.js'; /** * Aggregation function descriptor from QueryAST. @@ -38,12 +39,19 @@ export function buildAggregationPipeline(opts: { orderBy?: Array<{ field: string; order?: string }>; limit?: number; offset?: number; + /** + * Declared temporal kinds of the aggregated object, so a `$match` comparand + * lands in the field's storage form (#4047) — the aggregate path has to agree + * with `find()` about that, or the same window answers differently depending + * on which one the caller took. + */ + temporalKind?: TemporalFieldKindResolver; }): Document[] { const pipeline: Document[] = []; // $match stage if (opts.where) { - const matchFilter = translateFilter(opts.where); + const matchFilter = translateFilter(opts.where, opts.temporalKind); if (Object.keys(matchFilter).length > 0) { pipeline.push({ $match: matchFilter }); } diff --git a/packages/plugins/driver-mongodb/src/mongodb-datetime-storage.test.ts b/packages/plugins/driver-mongodb/src/mongodb-datetime-storage.test.ts new file mode 100644 index 0000000000..2d1e9484f2 --- /dev/null +++ b/packages/plugins/driver-mongodb/src/mongodb-datetime-storage.test.ts @@ -0,0 +1,179 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * `Field.datetime` has ONE storage form on MongoDB — BSON `Date` (#4047). + * + * The mongo twin of ADR-0053 D-B (#3912): before this, a datetime column held + * whichever form the writer produced. The driver's own `created_at`/`updated_at` + * defaults bind `new Date()` → BSON Date, while every REST/JSON write and every + * relative-date token arrives as an ISO **string** → BSON String. MongoDB + * compares across BSON types by *type bracket*, never by value, so a string + * comparand matched no Date row and vice versa — for EVERY operator, `$gte` + * included. The dashboard's default date window (`created_at`, a system + * datetime) therefore returned **nothing at all** on mongo, which is worse than + * #3777's "loses the final day". + * + * BSON Date is the right canon here for the same reason `timestamptz` is on + * Postgres: it is the dialect's native instant type, indexable and + * range-queryable. So the write path converges values to it and the filter path + * coerces comparands to it — the two halves that must never disagree. + */ + +import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import { MongoDBDriver } from './mongodb-driver.js'; + +let sharedMongod: MongoMemoryServer | undefined; +try { + sharedMongod = await MongoMemoryServer.create({ instance: { launchTimeout: 60_000 } }); +} catch (err) { + console.warn( + '[driver-mongodb] Skipping datetime-storage suite — mongodb-memory-server could not start: ' + + `${(err as Error)?.message ?? String(err)}`, + ); +} + +const ids = (rows: any[]) => rows.map((r: any) => r.id).sort(); + +describe.skipIf(!sharedMongod)('MongoDB Field.datetime storage (#4047)', () => { + const mongod = sharedMongod as MongoMemoryServer; + let driver: MongoDBDriver; + + beforeAll(async () => { + driver = new MongoDBDriver({ url: mongod.getUri(), database: 'dt_storage_db' }); + await driver.connect(); + }, 90_000); + + afterAll(async () => { + if (driver) await driver.disconnect(); + if (mongod) await mongod.stop(); + }); + + beforeEach(async () => { + const db = driver.getDb(); + for (const col of await db.listCollections().toArray()) { + await db.dropCollection(col.name); + } + // Declaring the object is what teaches the driver which fields are temporal + // — the mongo analogue of `initObjects` populating `datetimeFields`. + await driver.syncSchema('task', { + name: 'task', + fields: { + title: { type: 'string' }, + created_at: { type: 'datetime' }, + created_on: { type: 'date' }, + }, + }); + }); + + /** Insert the two writer forms of the SAME instant, plus neighbours. */ + async function seedMixed(): Promise { + const rows: Array<[string, unknown]> = [ + // ISO string — a REST/JSON write, the form that used to be unreachable + // from a Date comparand. + ['s_morning', '2026-07-28T09:15:00.000Z'], + ['s_evening', '2026-07-28T21:40:00.000Z'], + // JS Date — a direct SDK call / the driver's own timestamp defaults. + ['d_midnight', new Date('2026-07-28T00:00:00Z')], + ['d_yesterday', new Date('2026-07-27T14:00:00Z')], + // Out of window on both sides. + ['d_next_day', new Date('2026-07-29T00:00:00Z')], + ['s_old', '2026-04-19T10:00:00.000Z'], + ]; + for (const [id, created_at] of rows) { + await driver.create('task', { id, title: id, created_at }); + } + } + + it('stores every writer form as one BSON Date', async () => { + await seedMixed(); + const raw = await driver.getDb().collection('task').find({}).toArray(); + for (const doc of raw) { + expect(doc.created_at, `${doc.id} stored form`).toBeInstanceOf(Date); + } + // …and the instant survives the conversion, rather than being re-parsed + // in the server's local zone. + const morning = raw.find((d: any) => d.id === 's_morning')!; + expect((morning.created_at as Date).toISOString()).toBe('2026-07-28T09:15:00.000Z'); + }); + + it('a string date window reaches rows written in BOTH forms', async () => { + await seedMixed(); + // The dashboard's shape: bare-day bounds as ISO strings, against a + // system-injected datetime field. Pre-fix this returned [] on mongo. + const found = await driver.find('task', { + where: { created_at: { $gte: '2026-04-29', $lte: '2026-07-28' } }, + } as any); + expect(ids(found)).toEqual(['d_midnight', 'd_yesterday', 's_evening', 's_morning']); + }); + + it('a Date-object comparand reaches the same rows', async () => { + await seedMixed(); + const found = await driver.find('task', { + where: { + created_at: { + $gte: new Date('2026-04-29T00:00:00Z'), + $lt: new Date('2026-07-29T00:00:00Z'), + }, + }, + } as any); + expect(ids(found)).toEqual(['d_midnight', 'd_yesterday', 's_evening', 's_morning']); + }); + + it('keeps #3777/#4042 bound semantics on top of the converged storage', async () => { + await seedMixed(); + // Bare-day upper bound = the whole day (half-open), lower bound = midnight. + const gte = await driver.find('task', { where: { created_at: { $gte: '2026-07-28' } } } as any); + expect(ids(gte)).toEqual(['d_midnight', 'd_next_day', 's_evening', 's_morning']); + + const lt = await driver.find('task', { where: { created_at: { $lt: '2026-07-28' } } } as any); + expect(ids(lt)).toEqual(['d_yesterday', 's_old']); + + // A full-ISO upper bound keeps instant semantics. + const instant = await driver.find('task', { + where: { created_at: { $lte: '2026-07-28T12:00:00.000Z' } }, + } as any); + expect(ids(instant)).toEqual(['d_midnight', 'd_yesterday', 's_morning', 's_old']); + }); + + it('$between and the array spelling take the same coercion', async () => { + await seedMixed(); + const between = await driver.find('task', { + where: { created_at: { $between: ['2026-04-29', '2026-07-28'] } }, + } as any); + expect(ids(between)).toEqual(['d_midnight', 'd_yesterday', 's_evening', 's_morning']); + + const array = await driver.find('task', { + where: [['created_at', '>=', '2026-04-29'], 'and', ['created_at', '<=', '2026-07-28']], + } as any); + expect(ids(array)).toEqual(['d_midnight', 'd_yesterday', 's_evening', 's_morning']); + }); + + it('a Field.date column stays calendar-day TEXT — not widened into an instant', async () => { + // `date` is timezone-naive by ADR-0053 Phase 1; converting it to a BSON + // Date would invent a midnight instant and re-introduce the tz coupling + // the calendar-day form exists to avoid. + for (const [id, created_on] of [ + ['on_early', '2026-04-19'], + ['on_mid', '2026-07-28'], + ['on_late', '2026-07-29'], + ] as const) { + await driver.create('task', { id, title: id, created_on }); + } + const raw = await driver.getDb().collection('task').find({}).toArray(); + for (const doc of raw) expect(typeof doc.created_on).toBe('string'); + + const found = await driver.find('task', { + where: { created_on: { $gte: '2026-04-29', $lte: '2026-07-28' } }, + } as any); + expect(ids(found)).toEqual(['on_mid']); + }); + + it('an undeclared object is left alone (no schema → no coercion)', async () => { + // Nothing declared this collection, so the driver has no field kinds for + // it and must not guess: values pass through exactly as written. + await driver.create('freeform', { id: 'f1', when: '2026-07-28T09:15:00.000Z' }); + const raw = await driver.getDb().collection('freeform').find({}).toArray(); + expect(typeof raw[0].when).toBe('string'); + }); +}); diff --git a/packages/plugins/driver-mongodb/src/mongodb-driver.ts b/packages/plugins/driver-mongodb/src/mongodb-driver.ts index 65918eb552..f95b5bdd42 100644 --- a/packages/plugins/driver-mongodb/src/mongodb-driver.ts +++ b/packages/plugins/driver-mongodb/src/mongodb-driver.ts @@ -22,6 +22,12 @@ import { } from 'mongodb'; import { nanoid } from 'nanoid'; import { translateFilter } from './mongodb-filter.js'; +import { + coerceTemporalValue, + indexTemporalFields, + type TemporalFieldKind, + type TemporalFieldKindResolver, +} from './mongodb-temporal.js'; import { buildAggregationPipeline, postProcessAggregation, @@ -127,6 +133,15 @@ export class MongoDBDriver implements IDataDriver { private db!: Db; private config: MongoDBDriverConfig; + /** + * Declared temporal fields per object, populated by {@link syncSchema} — + * this driver's equivalent of `SqlDriver.datetimeFields`/`dateFields`, and + * the only thing that lets write and filter agree on a storage form (#4047). + * An object absent from this map was never declared, so nothing is coerced + * for it: the driver does not guess types from values. + */ + private temporalFields = new Map>(); + constructor(config: MongoDBDriverConfig) { // Refuse to even EXIST in a multi-tenant deployment (#3724). The check is // repeated in `connect()`; construction just fails earliest, before a host @@ -201,7 +216,7 @@ export class MongoDBDriver implements IDataDriver { const collection = this.getCollection(object); const session = this.getSession(options); - const filter = translateFilter(query.where); + const filter = translateFilter(query.where, this.temporalKindFor(object)); const findOptions: FindOptions = { session }; // Field projection @@ -242,7 +257,7 @@ export class MongoDBDriver implements IDataDriver { const collection = this.getCollection(object); const session = this.getSession(options); - const filter = translateFilter(query.where); + const filter = translateFilter(query.where, this.temporalKindFor(object)); const result = await collection.findOne(filter, { session, projection: { _id: 0 }, @@ -259,7 +274,7 @@ export class MongoDBDriver implements IDataDriver { const collection = this.getCollection(object); const session = this.getSession(options); - const filter = translateFilter(query.where); + const filter = translateFilter(query.where, this.temporalKindFor(object)); const findOptions: FindOptions = { session, projection: { _id: 0 }, @@ -290,7 +305,7 @@ export class MongoDBDriver implements IDataDriver { const session = this.getSession(options); const { _id, ...rest } = data; - const toInsert: Record = { ...rest }; + const toInsert: Record = { ...this.toStorageForms(object, rest) }; // Assign ID if (toInsert.id === undefined) { @@ -313,7 +328,8 @@ export class MongoDBDriver implements IDataDriver { const collection = this.getCollection(object); const session = this.getSession(options); - const { _id, id: dataId, ...updateData } = data; + const { _id, id: dataId, ...rawUpdate } = data; + const updateData: Record = { ...this.toStorageForms(object, rawUpdate) }; updateData.updated_at = new Date(); await collection.updateOne( @@ -382,7 +398,7 @@ export class MongoDBDriver implements IDataDriver { const collection = this.getCollection(object); const session = this.getSession(options); - const filter = query?.where ? translateFilter(query.where) : {}; + const filter = query?.where ? translateFilter(query.where, this.temporalKindFor(object)) : {}; return await collection.countDocuments(filter, { session }); } @@ -397,7 +413,7 @@ export class MongoDBDriver implements IDataDriver { const now = new Date(); const docs = dataArray.map((data) => { const { _id, ...rest } = data; - const doc: Record = { ...rest }; + const doc: Record = { ...this.toStorageForms(object, rest) }; if (doc.id === undefined) doc.id = nanoid(DEFAULT_ID_LENGTH); if (doc.created_at === undefined) doc.created_at = now; if (doc.updated_at === undefined) doc.updated_at = now; @@ -416,7 +432,8 @@ export class MongoDBDriver implements IDataDriver { const now = new Date(); const bulkOps = updates.map(({ id, data }) => { - const { _id, id: dataId, ...updateData } = data; + const { _id, id: dataId, ...rawUpdate } = data; + const updateData: Record = { ...this.toStorageForms(object, rawUpdate) }; updateData.updated_at = now; return { updateOne: { @@ -452,8 +469,9 @@ export class MongoDBDriver implements IDataDriver { const collection = this.getCollection(object); const session = this.getSession(options); - const filter = translateFilter(query.where); - const { _id, id, ...updateData } = data; + const filter = translateFilter(query.where, this.temporalKindFor(object)); + const { _id, id, ...rawUpdate } = data; + const updateData: Record = { ...this.toStorageForms(object, rawUpdate) }; updateData.updated_at = new Date(); const result = await collection.updateMany( @@ -469,7 +487,7 @@ export class MongoDBDriver implements IDataDriver { const collection = this.getCollection(object); const session = this.getSession(options); - const filter = translateFilter(query.where); + const filter = translateFilter(query.where, this.temporalKindFor(object)); const result = await collection.deleteMany(filter, { session }); return result.deletedCount; } @@ -491,6 +509,7 @@ export class MongoDBDriver implements IDataDriver { orderBy: query.orderBy as Array<{ field: string; order?: string }>, limit: query.limit, offset: query.offset, + temporalKind: this.temporalKindFor(object), }); const results = await collection.aggregate(pipeline, { session }).toArray(); @@ -529,6 +548,9 @@ export class MongoDBDriver implements IDataDriver { assertObjectsNotTenantScoped([{ object, schema }]); const objectDef = schema as { name: string; fields?: Record }; + // Learn which fields are temporal BEFORE any write can land, so the write + // path and the filter path share one storage convention (#4047). + this.temporalFields.set(object, indexTemporalFields(objectDef.fields)); await syncCollectionSchema(this.db, object, objectDef); } @@ -552,7 +574,7 @@ export class MongoDBDriver implements IDataDriver { async explain(object: string, query: QueryAST, _options?: DriverOptions): Promise { const collection = this.getCollection(object); - const filter = translateFilter(query.where); + const filter = translateFilter(query.where, this.temporalKindFor(object)); const explanation = await collection.find(filter).explain('executionStats'); return explanation; } @@ -585,6 +607,43 @@ export class MongoDBDriver implements IDataDriver { return field; } + // ── Temporal storage form (#4047) ───────────────────────────────────────── + + /** + * The declared-temporal-kind lookup for one object, handed to + * {@link translateFilter} so comparands land in the field's storage form. + * `undefined` for an undeclared object — nothing to coerce against. + */ + private temporalKindFor(object: string): TemporalFieldKindResolver | undefined { + const kinds = this.temporalFields.get(object); + if (!kinds || kinds.size === 0) return undefined; + return (field: string) => kinds.get(field); + } + + /** + * Put every declared temporal field of a document into its storage form — + * the write half of the convention {@link translateFilter} reads against. + * Applied by `create`/`update`/`bulkCreate`/`bulkUpdate`/`updateMany`, so no + * write path can leave a value in a form the filter path cannot reach. + * + * The driver's own `created_at`/`updated_at` stamps already bind a `Date`; + * this is what converts an ISO string from a REST/JSON write into the same + * BSON Date rather than leaving the collection holding both. + */ + private toStorageForms(object: string, data: Record): Record { + const kinds = this.temporalFields.get(object); + if (!kinds || kinds.size === 0) return data; + let out: Record | undefined; + for (const [field, kind] of kinds) { + if (!(field in data)) continue; + const coerced = coerceTemporalValue(data[field], kind); + if (coerced === data[field]) continue; + out ??= { ...data }; + out[field] = coerced; + } + return out ?? data; + } + private extractDatabaseName(url: string): string { // Extract database name from mongodb:// or mongodb+srv:// connection string // Format: mongodb://[user:pass@]host[:port]/database[?options] diff --git a/packages/plugins/driver-mongodb/src/mongodb-filter.ts b/packages/plugins/driver-mongodb/src/mongodb-filter.ts index 9453288857..297a9fcb59 100644 --- a/packages/plugins/driver-mongodb/src/mongodb-filter.ts +++ b/packages/plugins/driver-mongodb/src/mongodb-filter.ts @@ -18,6 +18,7 @@ import type { Filter } from 'mongodb'; import { nextUtcCalendarDay } from '@objectstack/core'; +import { coerceTemporalValue, type TemporalFieldKindResolver } from './mongodb-temporal.js'; /** * Translate an ObjectStack `where` clause into a MongoDB filter document. @@ -26,24 +27,36 @@ import { nextUtcCalendarDay } from '@objectstack/core'; * 1. A FilterCondition object (MongoDB-style with `$` operators) * 2. A legacy array-style filter `[[field, op, value], 'or', [field, op, value]]` * 3. A plain key-value object for implicit equality + * + * `temporalKind` resolves the declared temporal type of a field so comparands + * land in the column's storage form (#4047) — a `Field.datetime` comparand + * becomes a BSON `Date`, because MongoDB compares across BSON types by type + * bracket and a string comparand matches no Date row at all. Omitting it keeps + * the pure shape translation, which is what the standalone export is for. */ -export function translateFilter(where: unknown): Filter { +export function translateFilter( + where: unknown, + temporalKind?: TemporalFieldKindResolver, +): Filter { if (!where) return {}; // Legacy array-style filters if (Array.isArray(where)) { - return translateArrayFilter(where); + return translateArrayFilter(where, temporalKind); } if (typeof where !== 'object') return {}; - return translateCondition(where as Record); + return translateCondition(where as Record, temporalKind); } /** * Translate a FilterCondition object to a MongoDB filter. */ -function translateCondition(condition: Record): Filter { +function translateCondition( + condition: Record, + temporalKind?: TemporalFieldKindResolver, +): Filter { const mongoFilter: Record = {}; const andClauses: Filter[] = []; @@ -52,7 +65,7 @@ function translateCondition(condition: Record): Filter { case '$and': if (Array.isArray(value)) { andClauses.push({ - $and: value.map((sub) => translateCondition(sub as Record)), + $and: value.map((sub) => translateCondition(sub as Record, temporalKind)), }); } break; @@ -60,14 +73,14 @@ function translateCondition(condition: Record): Filter { case '$or': if (Array.isArray(value)) { andClauses.push({ - $or: value.map((sub) => translateCondition(sub as Record)), + $or: value.map((sub) => translateCondition(sub as Record, temporalKind)), }); } break; case '$not': if (value && typeof value === 'object') { - const inner = translateCondition(value as Record); + const inner = translateCondition(value as Record, temporalKind); // MongoDB $not applies per-field; for top-level negation use $nor andClauses.push({ $nor: [inner] }); } @@ -77,19 +90,19 @@ function translateCondition(condition: Record): Filter { // Skip query-level keys that are not filter conditions if (['limit', 'offset', 'fields', 'orderBy'].includes(key)) continue; - if (value !== null && typeof value === 'object' && !Array.isArray(value)) { + if (value !== null && typeof value === 'object' && !Array.isArray(value) && !(value instanceof Date)) { // Check if this is an operator object (has $ keys) const objValue = value as Record; const hasOps = Object.keys(objValue).some((k) => k.startsWith('$')); if (hasOps) { - mongoFilter[key] = translateFieldOperators(objValue); + mongoFilter[key] = translateFieldOperators(objValue, temporalKind?.(key)); } else { // Nested object — treat as exact match mongoFilter[key] = value; } } else { // Implicit equality - mongoFilter[key] = value; + mongoFilter[key] = coerceTemporalValue(value, temporalKind?.(key)); } } } @@ -109,9 +122,20 @@ function translateCondition(condition: Record): Filter { /** * Translate ObjectStack field-level operators into MongoDB operators. + * + * `kind` is the declared temporal type of the field these operators apply to, + * so each comparand lands in the field's storage form (#4047). Order matters + * and is load-bearing: the calendar-day upper-bound rewrite (#3777/#4042) runs + * on the STRING first — it is a calendar operation — and only the resulting + * bound is converted to the storage form. Converting first would leave + * `nextUtcCalendarDay` a `Date` it correctly refuses to widen. */ -function translateFieldOperators(ops: Record): Record { +function translateFieldOperators( + ops: Record, + kind?: 'datetime' | 'date', +): Record { const result: Record = {}; + const store = (v: unknown) => coerceTemporalValue(v, kind); for (const [op, value] of Object.entries(ops)) { switch (op) { @@ -123,6 +147,11 @@ function translateFieldOperators(ops: Record): Record): Record): Record): Record { +function translateArrayFilter( + filters: unknown[], + temporalKind?: TemporalFieldKindResolver, +): Filter { if (filters.length === 0) return {}; // Check if this is a single comparison tuple: [field, op, value] @@ -216,7 +245,7 @@ function translateArrayFilter(filters: unknown[]): Filter { const isOperator = ['=', '!=', '<>', '>', '>=', '<', '<=', 'in', 'nin', 'eq', 'ne', 'gt', 'gte', 'lt', 'lte', 'contains', 'like'].includes(possibleOp) || possibleOp.startsWith('$'); if (isOperator) { - return translateComparison(filters[0], possibleOp, filters[2]); + return translateComparison(filters[0], possibleOp, filters[2], temporalKind); } } @@ -241,8 +270,8 @@ function translateArrayFilter(filters: unknown[]): Filter { !Array.isArray(item[2]); const translated = isTuple - ? translateComparison(item[0], item[1], item[2]) - : translateArrayFilter(item); + ? translateComparison(item[0], item[1], item[2], temporalKind) + : translateArrayFilter(item, temporalKind); groups.push({ logic: nextLogic, filter: translated }); nextLogic = 'and'; @@ -280,36 +309,47 @@ function translateArrayFilter(filters: unknown[]): Filter { /** * Translate a single comparison `[field, operator, value]` tuple. */ -function translateComparison(field: string, op: string, value: unknown): Filter { +function translateComparison( + field: string, + op: string, + value: unknown, + temporalKind?: TemporalFieldKindResolver, +): Filter { const mappedField = mapFieldName(field); + // Resolve against the MAPPED name: `createdAt` is an alias of the declared + // `created_at`, and the field kinds are indexed under declared names. + const store = (v: unknown) => coerceTemporalValue(v, temporalKind?.(mappedField)); switch (op) { case '=': case 'eq': - return { [mappedField]: value }; + return { [mappedField]: store(value) }; case '!=': case '<>': case 'ne': - return { [mappedField]: { $ne: value } }; + return { [mappedField]: { $ne: store(value) } }; case '>': case 'gt': - return { [mappedField]: { $gt: value } }; + return { [mappedField]: { $gt: store(value) } }; case '>=': case 'gte': - return { [mappedField]: { $gte: value } }; + return { [mappedField]: { $gte: store(value) } }; case '<': case 'lt': - return { [mappedField]: { $lt: value } }; + return { [mappedField]: { $lt: store(value) } }; case '<=': case 'lte': { // Bare-day upper bound → half-open, `$lte`'s whole-day rule (#4042). + // Calendar first, storage form second — see translateFieldOperators. const nextDay = nextUtcCalendarDay(value); - return { [mappedField]: nextDay != null ? { $lt: nextDay } : { $lte: value } }; + return { + [mappedField]: nextDay != null ? { $lt: store(nextDay) } : { $lte: store(value) }, + }; } case 'in': - return { [mappedField]: { $in: value as unknown[] } }; + return { [mappedField]: { $in: store(value) as unknown[] } }; case 'nin': - return { [mappedField]: { $nin: value as unknown[] } }; + return { [mappedField]: { $nin: store(value) as unknown[] } }; case 'contains': case 'like': return { [mappedField]: { $regex: escapeRegex(String(value)), $options: 'i' } }; diff --git a/packages/plugins/driver-mongodb/src/mongodb-temporal.ts b/packages/plugins/driver-mongodb/src/mongodb-temporal.ts new file mode 100644 index 0000000000..bb5a42b989 --- /dev/null +++ b/packages/plugins/driver-mongodb/src/mongodb-temporal.ts @@ -0,0 +1,134 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * One storage form per temporal field type on MongoDB (#4047) — the mongo + * counterpart of ADR-0053 D-B, which gave `Field.datetime` a single UTC storage + * form on every SQL dialect (#3912). + * + * # Why this has to exist + * + * MongoDB compares across BSON types by **type bracket**, not by value: a + * `String` comparand never matches a `Date` field, in either direction, for + * every operator including `$gte`. So a collection whose datetime column held + * both forms — and every collection did, because the driver's own + * `created_at`/`updated_at` defaults bind `new Date()` while every REST/JSON + * write and every relative-date token arrives as an ISO string — answered a + * date window with whichever half happened to match the comparand's type, and + * frequently with nothing at all. That is strictly worse than #3777's "loses + * the final day": the dashboard's default window returned empty. + * + * # The canon + * + * | Field type | Stored as | Why | + * |---|---|---| + * | `datetime` | BSON `Date` | the dialect's native instant — indexable, range-queryable, timezone-unambiguous. The `timestamptz` of this driver. | + * | `date` | `YYYY-MM-DD` text | timezone-naive by ADR-0053 Phase 1. A BSON Date would invent a midnight instant and re-couple it to a zone. | + * | `time` | `HH:MM:SS[.fff]` text | same reasoning (#3994). | + * + * Write and filter must apply the SAME rule, or the halves disagree again — + * which is why {@link storageDatetimeValue} is the one function both call. + */ + +/** + * The BSON storage form of a `Field.datetime` value: a `Date` at the instant + * the input denotes. + * + * Total by design — an input this cannot interpret is returned unchanged rather + * than turned into `Invalid Date`, so unparseable junk keeps failing the + * comparison instead of silently becoming a wrong instant (the same totality + * `canonicalUtcDatetime` keeps on the SQL side). + * + * A bare `YYYY-MM-DD` means **midnight UTC**, stated explicitly so it cannot be + * re-read as midnight in the server's local zone. The whole-day reading of a + * bare day used as an UPPER bound is a separate, operator-sensitive concern + * handled before this by `nextUtcCalendarDay` (#3777/#4042) — this function + * only converts FORM. + */ +export function storageDatetimeValue(value: unknown): unknown { + if (value == null) return value; + if (value instanceof Date) return value; // already the storage form + if (typeof value === 'number') { + if (!Number.isFinite(value)) return value; + const d = new Date(value); + return Number.isNaN(d.getTime()) ? value : d; + } + if (typeof value !== 'string') return value; + const s = value.trim(); + if (s === '') return value; + // A bare integer (in either form) is epoch milliseconds. + if (/^-?\d+$/.test(s)) { + const d = new Date(Number(s)); + return Number.isNaN(d.getTime()) ? value : d; + } + const iso = /^\d{4}-\d{2}-\d{2}$/.test(s) + ? `${s}T00:00:00.000Z` + // Zone-naive `YYYY-MM-DD[ T]HH:MM[:SS[.fff]]` → its wall clock IS UTC, the + // same rule the SQL drivers apply to `CURRENT_TIMESTAMP`-written rows. + : /^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}(:\d{2}(\.\d+)?)?$/.test(s) + ? `${s.replace(' ', 'T')}Z` + : s; + const ms = Date.parse(iso); + return Number.isFinite(ms) ? new Date(ms) : value; +} + +/** + * Collapse a `Field.date` value to timezone-naive `YYYY-MM-DD` text — a `Date` + * to its UTC calendar day, a string to its leading date. Mirrors + * `SqlDriver.toDateOnly` so both backends agree on what a date *is*. + */ +export function storageDateValue(value: unknown): unknown { + if (value == null) return value; + if (value instanceof Date) { + if (Number.isNaN(value.getTime())) return value; + const y = value.getUTCFullYear(); + const m = String(value.getUTCMonth() + 1).padStart(2, '0'); + const d = String(value.getUTCDate()).padStart(2, '0'); + return `${y}-${m}-${d}`; + } + if (typeof value === 'string') { + const trimmed = value.trim(); + if (/^\d{4}-\d{2}-\d{2}/.test(trimmed)) return trimmed.slice(0, 10); + } + return value; +} + +/** Which temporal rule a declared field takes, if any. */ +export type TemporalFieldKind = 'datetime' | 'date'; + +/** + * Resolves the temporal kind of `field` on the object being queried, or + * `undefined` when the object was never declared (an ad-hoc collection) or the + * field is not temporal. An absent resolver means "coerce nothing" — the + * pre-#4047 behaviour, which is what keeps `translateFilter` usable as the + * standalone shape translator it is exported as. + */ +export type TemporalFieldKindResolver = (field: string) => TemporalFieldKind | undefined; + +/** + * Put a value into the storage form of `field`. Non-temporal fields and + * undeclared objects pass through untouched. + */ +export function coerceTemporalValue( + value: unknown, + kind: TemporalFieldKind | undefined, +): unknown { + if (kind === undefined) return value; + if (Array.isArray(value)) return value.map((v) => coerceTemporalValue(v, kind)); + return kind === 'datetime' ? storageDatetimeValue(value) : storageDateValue(value); +} + +/** + * Index the declared temporal fields of one object, for + * {@link TemporalFieldKindResolver}. Called from `syncSchema`, the only place a + * driver is handed an object definition. + */ +export function indexTemporalFields( + fields: Record | undefined, +): Map { + const out = new Map(); + for (const [name, def] of Object.entries(fields ?? {})) { + if (def?.type === 'datetime') out.set(name, 'datetime'); + else if (def?.type === 'date') out.set(name, 'date'); + } + return out; +}