Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/unmapped-fields-null.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"javascript-excel": minor
---

`importDataset` and `applyMapping`: a schema field without a column is now `null` in every row, and `columns` lists every schema field. An unmapped field's column has an empty title and letter, `type: 'empty'`, and its `field`. Rows always have the schema's shape, so validators such as Zod's `.nullable()` accept them; before this change the key was missing. A required field without a column still gives one `MAPPING_MISSING` issue, and is `null` in the rows.
7 changes: 4 additions & 3 deletions docs/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -1077,7 +1077,7 @@ export interface ImportDatasetOptions {
}

export interface ImportDatasetResult {
columns: ImportedColumn[]; // with a schema: one per mapped field, in schema order
columns: ImportedColumn[]; // with a schema: one per schema field, in schema order (unmapped: letter '', type 'empty')
data: Array<Record<string, unknown>>; // null-prototype objects
rowNumbers: number[]; // spreadsheet row (1-based) of each data entry
issues: ImportIssue[];
Expand Down Expand Up @@ -1222,8 +1222,9 @@ export interface MappedDataset { columns: ImportedColumn[]; data: Array<Record<s
- `missing` lists required fields without a column.

`applyMapping(dataset, mapping, schema)` returns `{ columns, data, rowNumbers, issues }`:
- Rows are keyed by field names, in schema order. With `unmapped: 'keep'`, other columns follow under their own keys, suffixed `_2`, `_3` if a key equals a field name.
- A required field without a column gives one `MAPPING_MISSING` issue (row 0, severity `error`). The rows are still returned, so a UI can fix the mapping and apply it again without reading the file.
- Rows are keyed by field names, in schema order, and **every row has every schema field**: a field without a column is `null` in each row (JSE-21), so rows always have the schema's shape and a validator such as Zod's `.nullable()` accepts them. `columns` has one entry per schema field; an unmapped field's entry has `title: ''`, `letter: ''`, `type: 'empty'` and its `field`.
- With `unmapped: 'keep'`, the file's other columns follow under their own keys, suffixed `_2`, `_3` if a key equals a field name.
- A required field without a column gives one `MAPPING_MISSING` issue (row 0, severity `error`) and is `null` in the rows; there is no per-row `REQUIRED` issue for it. The rows are still returned, so a UI can fix the mapping and apply it again without reading the file.
- A required field that is empty in a row gives a `REQUIRED` issue (severity `error`) with the row and cell.

Validation of rows (functions, Standard Schema, async checks) is react-data-import's. javascript-excel reserves its error code, `VALIDATION_FAILED`.
Expand Down
15 changes: 10 additions & 5 deletions packages/javascript-excel/src/internal/import/mapping.ts
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,9 @@ export function applyMapping(dataset: MappableDataset, mapping: ColumnMapping, s
const rowNumbers = dataset.data.map((_, i) => dataset.rowNumbers?.[i] ?? i + 2);
const issues: ImportIssue[] = [];
const columns: ImportedColumn[] = [];
const plan: Array<{ out: string; from: string }> = [];
// from null: a field without a column. It still gets a column and a null in
// every row, so rows always have the schema's shape (JSE-21).
const plan: Array<{ out: string; from: string | null }> = [];
// Every field name is reserved, mapped or not, so a kept column never poses as a field.
const used = new Set<string>(fields.map(([name]) => name));
for (const [name, spec] of fields) {
Expand All @@ -127,14 +129,16 @@ export function applyMapping(dataset: MappableDataset, mapping: ColumnMapping, s
if (spec.required) {
issues.push({ row: 0, field: name, code: "MAPPING_MISSING", message: `No column for the required field "${name}".`, severity: "error" });
}
columns.push({ key: name, title: "", letter: "", type: "empty", field: name });
plan.push({ out: name, from: null });
continue;
}
const c = byKey.get(from)!;
columns.push({ key: name, title: c.title, letter: c.letter ?? "", type: c.type ?? "mixed", field: name });
plan.push({ out: name, from });
}
if (schema.unmapped === "keep") {
const mapped = new Set(plan.map((p) => p.from));
const mapped = new Set(plan.map((p) => p.from).filter((f): f is string => f !== null));
for (const c of dataset.columns) {
if (mapped.has(c.key)) continue;
let key = c.key;
Expand All @@ -144,14 +148,15 @@ export function applyMapping(dataset: MappableDataset, mapping: ColumnMapping, s
plan.push({ out: key, from: c.key });
}
}
const required = fields.filter(([name, spec]) => spec.required && plan.some((p) => p.out === name)).map(([name]) => name);
// Only mapped required fields are checked per row; an unmapped one already has MAPPING_MISSING.
const required = fields.filter(([name, spec]) => spec.required && plan.some((p) => p.out === name && p.from !== null)).map(([name]) => name);
const data = dataset.data.map((row, i) => {
const out = Object.create(null) as Record<string, unknown>;
for (const { out: k, from } of plan) out[k] = row[from] ?? null;
for (const { out: k, from } of plan) out[k] = from === null ? null : (row[from] ?? null);
for (const name of required) {
const v = out[name];
if (v === null || v === undefined || (typeof v === "string" && v.trim() === "")) {
const letter = byKey.get(plan.find((p) => p.out === name)!.from)?.letter;
const letter = byKey.get(plan.find((p) => p.out === name)!.from!)?.letter;
issues.push({
row: rowNumbers[i]!,
field: name,
Expand Down
9 changes: 5 additions & 4 deletions packages/javascript-excel/test/import/mapping.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,9 @@ describe("applyMapping", () => {
it("renames to fields and reports REQUIRED and MAPPING_MISSING", () => {
const schema: ImportSchema = { fields: { email: { required: true }, name: {}, phone: { required: true } } };
const r = applyMapping(ds, { email: "E-mail", name: "Name", phone: null }, schema);
expect(r.columns.map((c) => [c.key, c.title, c.letter])).toEqual([["email", "E-mail", "A"], ["name", "Name", "B"]]);
expect(r.data).toEqual([Object.assign(Object.create(null), { email: "a@x", name: "Ada" }), Object.assign(Object.create(null), { email: null, name: "Grace" })]);
// Every schema field has a column and a key in every row; an unmapped one is null (JSE-21).
expect(r.columns.map((c) => [c.key, c.title, c.letter, c.type, c.field])).toEqual([["email", "E-mail", "A", "mixed", "email"], ["name", "Name", "B", "mixed", "name"], ["phone", "", "", "empty", "phone"]]);
expect(r.data).toEqual([Object.assign(Object.create(null), { email: "a@x", name: "Ada", phone: null }), Object.assign(Object.create(null), { email: null, name: "Grace", phone: null })]);
expect(r.rowNumbers).toEqual([3, 4]);
expect(r.issues).toEqual([
{ row: 0, field: "phone", code: "MAPPING_MISSING", message: 'No column for the required field "phone".', severity: "error" },
Expand All @@ -49,8 +50,8 @@ describe("applyMapping", () => {

it("keeps unmapped columns, suffixing keys that collide with fields", () => {
const r = applyMapping({ ...ds, columns: cols("E-mail", "name", "email2") , data: [{ "E-mail": "a", name: "N", email2: 1 }] }, { email: "E-mail" }, { fields: { email: {}, name: { aliases: ["zzz"] } }, unmapped: "keep" });
expect(r.columns.map((c) => c.key)).toEqual(["email", "name_2", "email2"]);
expect(r.data[0]).toEqual(Object.assign(Object.create(null), { email: "a", name_2: "N", email2: 1 }));
expect(r.columns.map((c) => c.key)).toEqual(["email", "name", "name_2", "email2"]);
expect(r.data[0]).toEqual(Object.assign(Object.create(null), { email: "a", name: null, name_2: "N", email2: 1 }));
});

it("a header named __proto__ is just a key", () => {
Expand Down
28 changes: 28 additions & 0 deletions packages/javascript-excel/test/import/pipeline.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,34 @@ describe("types", () => {
});
});

describe("unmapped schema fields (JSE-21)", () => {
const schema = { fields: { email: { aliases: ["E-mail"], required: true }, name: {}, joined: { type: "date" as const }, amount: { type: "number" as const } } };

it("an optional field without a column is null in every row and has a column", async () => {
const r = await importDataset("E-mail,Name\na@x,Ada\nb@x,Grace", { schema });
expect(r.data).toEqual([
Object.assign(Object.create(null), { email: "a@x", name: "Ada", joined: null, amount: null }),
Object.assign(Object.create(null), { email: "b@x", name: "Grace", joined: null, amount: null }),
]);
expect(r.columns.map((c) => [c.key, c.letter, c.type, c.field])).toEqual([["email", "A", "string", "email"], ["name", "B", "string", "name"], ["joined", "", "empty", "joined"], ["amount", "", "empty", "amount"]]);
expect(r.meta.mapping).toEqual({ email: "E-mail", name: "Name", joined: null, amount: null });
expect(r.issues).toEqual([]);
});

it("a required field without a column: MAPPING_MISSING once, null in the rows, no REQUIRED per row", async () => {
const r = await importDataset("Name\nAda\nGrace", { schema });
expect(r.issues.map((i) => [i.code, i.field, i.row])).toEqual([["MAPPING_MISSING", "email", 0]]);
expect(r.data.map((d) => d["email"])).toEqual([null, null]);
expect(Object.keys(r.data[0]!)).toEqual(["email", "name", "joined", "amount"]);
});

it("every row has the same keys, in schema order", async () => {
const r = await importDataset("Amount,E-mail\n5,a@x\n,b@x", { schema });
for (const row of r.data) expect(Object.keys(row)).toEqual(["email", "name", "joined", "amount"]);
expect(r.data.map((d) => d["amount"])).toEqual([5, null]);
});
});

describe("cells", () => {
it("error cells become null with ERROR_CELL, or are kept", async () => {
const xml = `<sheetData><row r="1"><c r="A1" t="inlineStr"><is><t>v</t></is></c></row><row r="2"><c r="A2" t="e"><v>#N/A</v></c></row><row r="3"><c r="A3"><v>2</v></c></row></sheetData>`;
Expand Down
Loading