From 6ce413a6ffdec4a4855ddb04142484964a43910c Mon Sep 17 00:00:00 2001 From: Afzal Ahmad Date: Mon, 16 Nov 2026 16:14:53 +0100 Subject: [PATCH] import: unmapped schema fields are null in every row (JSE-21) applyMapping (and so importDataset) now gives every schema field a column and a key in every row: a field without a column is null, with an entry in columns (title and letter '', type 'empty', its field). Rows always have the schema's shape, so validators such as Zod's .nullable() accept them; before, the key was missing. A required field without a column still gives one MAPPING_MISSING issue and no per-row REQUIRED issue. --- .changeset/unmapped-fields-null.md | 5 ++++ docs/api.md | 7 +++-- .../src/internal/import/mapping.ts | 15 ++++++---- .../test/import/mapping.test.ts | 9 +++--- .../test/import/pipeline.test.ts | 28 +++++++++++++++++++ 5 files changed, 52 insertions(+), 12 deletions(-) create mode 100644 .changeset/unmapped-fields-null.md diff --git a/.changeset/unmapped-fields-null.md b/.changeset/unmapped-fields-null.md new file mode 100644 index 0000000..b151bf2 --- /dev/null +++ b/.changeset/unmapped-fields-null.md @@ -0,0 +1,5 @@ +--- +"javascript-excel": minor +--- + +`importDataset` and `applyMapping`: a schema field without a column is now `null` in every row, and `columns` lists every schema field. An unmapped field's column has an empty title and letter, `type: 'empty'`, and its `field`. Rows always have the schema's shape, so validators such as Zod's `.nullable()` accept them; before this change the key was missing. A required field without a column still gives one `MAPPING_MISSING` issue, and is `null` in the rows. diff --git a/docs/api.md b/docs/api.md index 34f8029..d0043d0 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1077,7 +1077,7 @@ export interface ImportDatasetOptions { } export interface ImportDatasetResult { - columns: ImportedColumn[]; // with a schema: one per mapped field, in schema order + columns: ImportedColumn[]; // with a schema: one per schema field, in schema order (unmapped: letter '', type 'empty') data: Array>; // null-prototype objects rowNumbers: number[]; // spreadsheet row (1-based) of each data entry issues: ImportIssue[]; @@ -1222,8 +1222,9 @@ export interface MappedDataset { columns: ImportedColumn[]; data: Array dataset.rowNumbers?.[i] ?? i + 2); const issues: ImportIssue[] = []; const columns: ImportedColumn[] = []; - const plan: Array<{ out: string; from: string }> = []; + // from null: a field without a column. It still gets a column and a null in + // every row, so rows always have the schema's shape (JSE-21). + const plan: Array<{ out: string; from: string | null }> = []; // Every field name is reserved, mapped or not, so a kept column never poses as a field. const used = new Set(fields.map(([name]) => name)); for (const [name, spec] of fields) { @@ -127,6 +129,8 @@ export function applyMapping(dataset: MappableDataset, mapping: ColumnMapping, s if (spec.required) { issues.push({ row: 0, field: name, code: "MAPPING_MISSING", message: `No column for the required field "${name}".`, severity: "error" }); } + columns.push({ key: name, title: "", letter: "", type: "empty", field: name }); + plan.push({ out: name, from: null }); continue; } const c = byKey.get(from)!; @@ -134,7 +138,7 @@ export function applyMapping(dataset: MappableDataset, mapping: ColumnMapping, s plan.push({ out: name, from }); } if (schema.unmapped === "keep") { - const mapped = new Set(plan.map((p) => p.from)); + const mapped = new Set(plan.map((p) => p.from).filter((f): f is string => f !== null)); for (const c of dataset.columns) { if (mapped.has(c.key)) continue; let key = c.key; @@ -144,14 +148,15 @@ export function applyMapping(dataset: MappableDataset, mapping: ColumnMapping, s plan.push({ out: key, from: c.key }); } } - const required = fields.filter(([name, spec]) => spec.required && plan.some((p) => p.out === name)).map(([name]) => name); + // Only mapped required fields are checked per row; an unmapped one already has MAPPING_MISSING. + const required = fields.filter(([name, spec]) => spec.required && plan.some((p) => p.out === name && p.from !== null)).map(([name]) => name); const data = dataset.data.map((row, i) => { const out = Object.create(null) as Record; - for (const { out: k, from } of plan) out[k] = row[from] ?? null; + for (const { out: k, from } of plan) out[k] = from === null ? null : (row[from] ?? null); for (const name of required) { const v = out[name]; if (v === null || v === undefined || (typeof v === "string" && v.trim() === "")) { - const letter = byKey.get(plan.find((p) => p.out === name)!.from)?.letter; + const letter = byKey.get(plan.find((p) => p.out === name)!.from!)?.letter; issues.push({ row: rowNumbers[i]!, field: name, diff --git a/packages/javascript-excel/test/import/mapping.test.ts b/packages/javascript-excel/test/import/mapping.test.ts index e99812f..87eca66 100644 --- a/packages/javascript-excel/test/import/mapping.test.ts +++ b/packages/javascript-excel/test/import/mapping.test.ts @@ -38,8 +38,9 @@ describe("applyMapping", () => { it("renames to fields and reports REQUIRED and MAPPING_MISSING", () => { const schema: ImportSchema = { fields: { email: { required: true }, name: {}, phone: { required: true } } }; const r = applyMapping(ds, { email: "E-mail", name: "Name", phone: null }, schema); - expect(r.columns.map((c) => [c.key, c.title, c.letter])).toEqual([["email", "E-mail", "A"], ["name", "Name", "B"]]); - expect(r.data).toEqual([Object.assign(Object.create(null), { email: "a@x", name: "Ada" }), Object.assign(Object.create(null), { email: null, name: "Grace" })]); + // Every schema field has a column and a key in every row; an unmapped one is null (JSE-21). + expect(r.columns.map((c) => [c.key, c.title, c.letter, c.type, c.field])).toEqual([["email", "E-mail", "A", "mixed", "email"], ["name", "Name", "B", "mixed", "name"], ["phone", "", "", "empty", "phone"]]); + expect(r.data).toEqual([Object.assign(Object.create(null), { email: "a@x", name: "Ada", phone: null }), Object.assign(Object.create(null), { email: null, name: "Grace", phone: null })]); expect(r.rowNumbers).toEqual([3, 4]); expect(r.issues).toEqual([ { row: 0, field: "phone", code: "MAPPING_MISSING", message: 'No column for the required field "phone".', severity: "error" }, @@ -49,8 +50,8 @@ describe("applyMapping", () => { it("keeps unmapped columns, suffixing keys that collide with fields", () => { const r = applyMapping({ ...ds, columns: cols("E-mail", "name", "email2") , data: [{ "E-mail": "a", name: "N", email2: 1 }] }, { email: "E-mail" }, { fields: { email: {}, name: { aliases: ["zzz"] } }, unmapped: "keep" }); - expect(r.columns.map((c) => c.key)).toEqual(["email", "name_2", "email2"]); - expect(r.data[0]).toEqual(Object.assign(Object.create(null), { email: "a", name_2: "N", email2: 1 })); + expect(r.columns.map((c) => c.key)).toEqual(["email", "name", "name_2", "email2"]); + expect(r.data[0]).toEqual(Object.assign(Object.create(null), { email: "a", name: null, name_2: "N", email2: 1 })); }); it("a header named __proto__ is just a key", () => { diff --git a/packages/javascript-excel/test/import/pipeline.test.ts b/packages/javascript-excel/test/import/pipeline.test.ts index d1a96eb..c45c22a 100644 --- a/packages/javascript-excel/test/import/pipeline.test.ts +++ b/packages/javascript-excel/test/import/pipeline.test.ts @@ -103,6 +103,34 @@ describe("types", () => { }); }); +describe("unmapped schema fields (JSE-21)", () => { + const schema = { fields: { email: { aliases: ["E-mail"], required: true }, name: {}, joined: { type: "date" as const }, amount: { type: "number" as const } } }; + + it("an optional field without a column is null in every row and has a column", async () => { + const r = await importDataset("E-mail,Name\na@x,Ada\nb@x,Grace", { schema }); + expect(r.data).toEqual([ + Object.assign(Object.create(null), { email: "a@x", name: "Ada", joined: null, amount: null }), + Object.assign(Object.create(null), { email: "b@x", name: "Grace", joined: null, amount: null }), + ]); + expect(r.columns.map((c) => [c.key, c.letter, c.type, c.field])).toEqual([["email", "A", "string", "email"], ["name", "B", "string", "name"], ["joined", "", "empty", "joined"], ["amount", "", "empty", "amount"]]); + expect(r.meta.mapping).toEqual({ email: "E-mail", name: "Name", joined: null, amount: null }); + expect(r.issues).toEqual([]); + }); + + it("a required field without a column: MAPPING_MISSING once, null in the rows, no REQUIRED per row", async () => { + const r = await importDataset("Name\nAda\nGrace", { schema }); + expect(r.issues.map((i) => [i.code, i.field, i.row])).toEqual([["MAPPING_MISSING", "email", 0]]); + expect(r.data.map((d) => d["email"])).toEqual([null, null]); + expect(Object.keys(r.data[0]!)).toEqual(["email", "name", "joined", "amount"]); + }); + + it("every row has the same keys, in schema order", async () => { + const r = await importDataset("Amount,E-mail\n5,a@x\n,b@x", { schema }); + for (const row of r.data) expect(Object.keys(row)).toEqual(["email", "name", "joined", "amount"]); + expect(r.data.map((d) => d["amount"])).toEqual([5, null]); + }); +}); + describe("cells", () => { it("error cells become null with ERROR_CELL, or are kept", async () => { const xml = `v#N/A2`;