diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aded997..fe198b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -304,3 +304,40 @@ jobs: # with an `export =` in it. The package job above is what checks # the types, through the resolution a user actually gets. - run: npm run test:deno + + # The shared corpus, which is the same cases the engine runs against + # itself and every other client runs against theirs. A job of its own + # because it needs a second checkout, and on one platform because what + # it asks about is this client's value mapping rather than anything the + # operating system decides. + corpus: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: 24 + - uses: Swatinem/rust-cache@v2 + # The cases are versioned with the engine, so the revision comes + # out of the pin this client already builds against rather than + # being written down a second time and drifting away from it. + - id: pin + run: | + rev=$(sed -n 's/^zudb = .*rev = "\([0-9a-f]*\)".*/\1/p' Cargo.toml) + test -n "$rev" + echo "rev=$rev" >> "$GITHUB_OUTPUT" + - uses: actions/checkout@v7 + with: + repository: tamnd/zu + ref: ${{ steps.pin.outputs.rev }} + path: engine + - run: npm ci + - run: npm run build:debug + # The runner first, because its summary is the line a person reads + # when this job goes red, and the suite second, because it is the + # one that knows which cases this client is allowed to leave + # unanswered. + - run: node conformance/cli.mjs engine/conformance/cases + - run: node --test "test/conformance*.test.mjs" + env: + ZU_CASES: engine/conformance/cases diff --git a/Cargo.lock b/Cargo.lock index 75690d9..4197b1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -413,9 +413,9 @@ dependencies = [ [[package]] name = "foldhash" -version = "0.1.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "form_urlencoded" @@ -566,9 +566,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.5" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "foldhash", ] @@ -578,14 +578,17 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] [[package]] name = "hashlink" -version = "0.10.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +checksum = "32069d97bb81e38fa67eab65e3393bf804bb85969f2bc06bf13f64aef5aba248" dependencies = [ - "hashbrown 0.15.5", + "hashbrown 0.17.1", ] [[package]] @@ -845,9 +848,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libsqlite3-sys" -version = "0.35.0" +version = "0.38.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" dependencies = [ "cc", "pkg-config", @@ -1155,11 +1158,21 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror", +] + [[package]] name = "rusqlite" -version = "0.37.0" +version = "0.40.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" dependencies = [ "bitflags", "fallible-iterator", @@ -1167,6 +1180,7 @@ dependencies = [ "hashlink", "libsqlite3-sys", "smallvec", + "sqlite-wasm-rs", ] [[package]] @@ -1192,9 +1206,9 @@ checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "ruzstd" -version = "0.8.3" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7c1c839d570d835527c9a5e4db7cb2198683a988cb9d7293fc8674e6bd58fc8" +checksum = "a252f5e20f038fe7b4ea53e073e65398d652c864cc162fc77c56c2f13717b888" dependencies = [ "twox-hash", ] @@ -1279,6 +1293,18 @@ version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -1729,7 +1755,7 @@ dependencies = [ [[package]] name = "zu" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=526ac583359f6112bca11bdae3fb43cc6fd76b2e#526ac583359f6112bca11bdae3fb43cc6fd76b2e" dependencies = [ "zu-common", "zu-encoding", @@ -1745,7 +1771,7 @@ dependencies = [ [[package]] name = "zu-arrow" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=526ac583359f6112bca11bdae3fb43cc6fd76b2e#526ac583359f6112bca11bdae3fb43cc6fd76b2e" dependencies = [ "arrow", "zu-common", @@ -1755,7 +1781,7 @@ dependencies = [ [[package]] name = "zu-common" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=526ac583359f6112bca11bdae3fb43cc6fd76b2e#526ac583359f6112bca11bdae3fb43cc6fd76b2e" dependencies = [ "thiserror", ] @@ -1763,7 +1789,7 @@ dependencies = [ [[package]] name = "zu-encoding" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=526ac583359f6112bca11bdae3fb43cc6fd76b2e#526ac583359f6112bca11bdae3fb43cc6fd76b2e" dependencies = [ "ruzstd", "zu-common", @@ -1772,7 +1798,7 @@ dependencies = [ [[package]] name = "zu-exec" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=526ac583359f6112bca11bdae3fb43cc6fd76b2e#526ac583359f6112bca11bdae3fb43cc6fd76b2e" dependencies = [ "zu-common", "zu-query", @@ -1782,7 +1808,7 @@ dependencies = [ [[package]] name = "zu-query" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=526ac583359f6112bca11bdae3fb43cc6fd76b2e#526ac583359f6112bca11bdae3fb43cc6fd76b2e" dependencies = [ "crossbeam-deque", "zu-common", @@ -1793,7 +1819,7 @@ dependencies = [ [[package]] name = "zu-s3" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=526ac583359f6112bca11bdae3fb43cc6fd76b2e#526ac583359f6112bca11bdae3fb43cc6fd76b2e" dependencies = [ "crc32c", "object_store", @@ -1804,7 +1830,7 @@ dependencies = [ [[package]] name = "zu-sqlite" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=526ac583359f6112bca11bdae3fb43cc6fd76b2e#526ac583359f6112bca11bdae3fb43cc6fd76b2e" dependencies = [ "rusqlite", "zu-common", @@ -1814,7 +1840,7 @@ dependencies = [ [[package]] name = "zu-storage" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=526ac583359f6112bca11bdae3fb43cc6fd76b2e#526ac583359f6112bca11bdae3fb43cc6fd76b2e" dependencies = [ "zu-common", "zu-encoding", @@ -1823,7 +1849,7 @@ dependencies = [ [[package]] name = "zu-vector" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=526ac583359f6112bca11bdae3fb43cc6fd76b2e#526ac583359f6112bca11bdae3fb43cc6fd76b2e" dependencies = [ "zu-common", ] @@ -1831,7 +1857,7 @@ dependencies = [ [[package]] name = "zu-zu1" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=526ac583359f6112bca11bdae3fb43cc6fd76b2e#526ac583359f6112bca11bdae3fb43cc6fd76b2e" dependencies = [ "crc32c", "loom", diff --git a/Cargo.toml b/Cargo.toml index b1a3191..b8205e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "zudb-node" version = "0.0.1" edition = "2024" -rust-version = "1.97" +rust-version = "1.98" license = "Apache-2.0" repository = "https://github.com/tamnd/zu-node" authors = ["Tam Nguyen "] @@ -18,14 +18,14 @@ crate-type = ["cdylib"] # with (ADR 0002), so a revision is the honest way to say which one. # A local checkout is used instead with a `paths` override in # `.cargo/config.toml`, which is untracked on purpose. -zudb = { package = "zu", git = "https://github.com/tamnd/zu", rev = "0698a4eccd31670f0a875b6d097d7753440a51b3" } -zu-common = { git = "https://github.com/tamnd/zu", rev = "0698a4eccd31670f0a875b6d097d7753440a51b3" } +zudb = { package = "zu", git = "https://github.com/tamnd/zu", rev = "526ac583359f6112bca11bdae3fb43cc6fd76b2e" } +zu-common = { git = "https://github.com/tamnd/zu", rev = "526ac583359f6112bca11bdae3fb43cc6fd76b2e" } # The one translation from a result into Arrow, which lives in the engine # tree so that every client agrees about what a column becomes. `ipc` is # the only feature this client turns on: the C Data Interface hands over # a pointer and nothing in a JavaScript runtime can read one, so the way # out here is the bytes of an IPC stream. -zu-arrow = { git = "https://github.com/tamnd/zu", rev = "0698a4eccd31670f0a875b6d097d7753440a51b3", features = ["ipc"] } +zu-arrow = { git = "https://github.com/tamnd/zu", rev = "526ac583359f6112bca11bdae3fb43cc6fd76b2e", features = ["ipc"] } # N-API by way of napi-rs (ADR 0002). `napi9` is the version of N-API # this addon declares it needs, which is what makes one binary work # across Node 24, Node 26, Electron and Bun without a rebuild: the diff --git a/README.md b/README.md index 2d08d58..f437e7d 100644 --- a/README.md +++ b/README.md @@ -75,9 +75,9 @@ The switches come across, including `bigIntMode` and `temporal`, because a pool ## What works today -`connect`, `query`, `exec`, `stream`, `close`, `dispose` and `await using`. `duplicate`, for a second connection made from the first. Named parameters both ways, including lists, records and nesting. Every scalar the engine has, plus nodes, edges and paths with their tables named rather than numbered, and `ZuDate`, `ZuTime`, `ZuTimestamp` and `ZuDuration`, with `{ temporal: true }` and `toTemporal()` for the runtimes that have `Temporal`. Read-only connections, databases in memory, memory and thread limits. `bigIntMode`, per statement or per connection. An `AbortSignal` on any statement, and `rowsRead` and `progress` for watching the one running now. The full error surface above, and `isZuError` to recognize it. Streaming, as an async iterable, as batches and as a Web Stream. Transactions, with `inTransaction` on the connection. An appender, for loading rows a batch at a time, and `load` for building a whole database out of columns and an edge list. Registered frames, so an Arrow table or an object of typed arrays is something a statement can match on without the rows being copied. `columnar`, for a result read down its columns as the buffers themselves rather than across its rows as objects, and `arrow`, for the same result as an Arrow IPC stream that any Arrow reader takes. Prepared statements, compiled at the line that asked and run as often as wanted, and `explain` and `profile`, as a tree a program walks and as the listing a person reads. Both module formats, typed separately. +`connect`, `query`, `exec`, `stream`, `close`, `dispose` and `await using`. `duplicate`, for a second connection made from the first. Named parameters both ways, including lists, records and nesting. Every scalar the engine has, including BYTES as a `Uint8Array` both ways, plus nodes, edges and paths with their tables named rather than numbered, and `ZuDate`, `ZuTime`, `ZuTimestamp` and `ZuDuration`, with `{ temporal: true }` and `toTemporal()` for the runtimes that have `Temporal`. Read-only connections, databases in memory, memory and thread limits. `bigIntMode`, per statement or per connection. An `AbortSignal` on any statement, and `rowsRead` and `progress` for watching the one running now. The full error surface above, and `isZuError` to recognize it. Streaming, as an async iterable, as batches and as a Web Stream. Transactions, with `inTransaction` on the connection. An appender, for loading rows a batch at a time, and `load` for building a whole database out of columns and an edge list. Registered frames, so an Arrow table or an object of typed arrays is something a statement can match on without the rows being copied. `columnar`, for a result read down its columns as the buffers themselves rather than across its rows as objects, and `arrow`, for the same result as an Arrow IPC stream that any Arrow reader takes. Prepared statements, compiled at the line that asked and run as often as wanted, and `explain` and `profile`, as a tree a program walks and as the listing a person reads. Both module formats, typed separately. -Build it with `npm run build`, and run the suite with `npm test`. Nothing is published yet, so `npm i zudb` is not a thing you can type at anybody's terminal, but everything it will do is built and installed on every run of the release workflow. +Build it with `npm run build`, and run the suite with `npm test`. The shared cross-client corpus runs through `npm run corpus`, and `conformance/README.md` says what it checks and the one case this client cannot answer. Nothing is published yet, so `npm i zudb` is not a thing you can type at anybody's terminal, but everything it will do is built and installed on every run of the release workflow. ## Stopping a statement @@ -261,7 +261,7 @@ three columns, rows 608.7 ms 609 ns/row Walking what came back costs the same either way, at about 14 ns a row for a sum over the buffer and the same over the rows, which is worth saying because it is where the win is not. V8 reads a property of a small object about as fast as an element of a typed array. What it cannot do is make a million of those objects for nothing, and it is not the only thing not happening on the columnar line: the engine's sink fills those buffers as the statement runs, so a columnar read is a scan and a move, where a row read is a scan, a row per row and then an object per row on top of it. That is the whole of the twenty to ninety times above, and it is why the two lines of a pair are further apart the more columns there are. -Every column says what it is, and reading one is a switch on `type` rather than a series of tests for what is there. `values` carries everything of a fixed width: a `BigInt64Array` of integers, nanoseconds or months, a `Float64Array` of floats, an `Int32Array` of days, and for booleans a `Uint8Array` of one bit a row, least significant bit first. A string column has `data`, the bytes of every string end to end, and `offsets`, one more than there are rows, so row `i` is `data.subarray(offsets[i], offsets[i + 1])`. `validity` is one bit a row again, set meaning the row has a value, and it is null when nothing in the column is, so the common case costs a reader nothing to skip. `unit` says whether a cell counts days, nanoseconds or months, and `zone` is the minutes east of UTC a column of zoned times was written with. +Every column says what it is, and reading one is a switch on `type` rather than a series of tests for what is there. `values` carries everything of a fixed width: a `BigInt64Array` of integers, nanoseconds or months, a `Float64Array` of floats, an `Int32Array` of days, and for booleans a `Uint8Array` of one bit a row, least significant bit first. A string column has `data`, the bytes of every string end to end, and `offsets`, one more than there are rows, so row `i` is `data.subarray(offsets[i], offsets[i + 1])`. A `bytes` column arrives in those same two buffers and is a separate type rather than a flavour of `string`, because the octets under it are not text and a reader that decoded them as text would be handed something it cannot decode. `validity` is one bit a row again, set meaning the row has a value, and it is null when nothing in the column is, so the common case costs a reader nothing to skip. `unit` says whether a cell counts days, nanoseconds or months, and `zone` is the minutes east of UTC a column of zoned times was written with. That layout is Arrow's, which is the point of it. `apache-arrow` wraps a buffer of this shape without copying it, so a table is eleven lines and no dependency of this package: @@ -290,6 +290,8 @@ The same memory is on both sides of that: `table.getChild("age").data[0].values` Two things are not buffers, and both are named by the type rather than found out by looking. A column of nodes, rels, paths, lists or records has no fixed width cell, so it arrives as `items`, holding the same JavaScript values `query` would have made. A column of nothing but nulls has a length and nothing else, because there is nothing to put in a buffer. A column that mixes two types is refused, naming the column and the row that did it, since a columnar result holds one type per column and a column that quietly became strings is worse than one that would not build. +A temporal column is sometimes `value` too, for now. The engine's columnar sink fills a buffer for an integer, a real, a boolean and a string and has no arm for days, nanoseconds or months, so a date, a time or a duration reaching a projection any other way comes over as the values themselves. Which of the two a statement gets is the plan's business and not the caller's: the same property is cells in one statement and values in another, and adding an unrelated column to the projection is enough to move it. `type` says what arrived rather than what the statement declared, so such a column reads `value` with its `ZuDate` and `ZuDuration` objects in `items`, and `unit` and `zone` are null because there are no cells for them to describe. That is the point of switching on `type` rather than on the type the statement declared: a column saying `date` with nothing in `values` would be a switch that lands nowhere. + A statement that matched nothing still comes back with its columns, each one the type the plan declared and each one empty: a string column of no rows has its one starting offset, an integer column of no rows has a buffer of no elements. So a table built from an empty answer has the schema the same statement would have had with rows in it, and a loop that concatenates a page at a time does not have to hold the first page that had anything in it as a special case. `bigIntMode` says nothing here. A columnar read has one physical layout per type and an INT64 column is 64 bit cells however a caller would rather read one, which is the difference between a buffer and a value. The mode still decides what is inside `items`, where this client is making objects anyway. @@ -437,8 +439,8 @@ A date, a time, a timestamp and a duration come back as `ZuDate`, `ZuTime`, `ZuT ```ts await using conn = await connect("social.zu1", { temporal: true }); -const rows = await conn.query<{ on: Temporal.PlainDate }>(`MATCH (d:day) RETURN d.on AS on`); -rows[0].on.dayOfWeek; // 1, which no count of days was ever going to tell you +const rows = await conn.query<{ began: Temporal.PlainDate }>(`MATCH (d:dated) RETURN d.began AS began`); +rows[0].began.dayOfWeek; // 1, which no count of days was ever going to tell you ``` | zu | as a class | with `{ temporal: true }` | @@ -455,8 +457,8 @@ The exception is the one value zu holds that `Temporal` has no type for. A time The option is settled when the connection is opened and cannot be named per statement, unlike `bigIntMode`. Both spellings are exact and neither loses anything, so which one a program wants is a property of the program and not of the query, and a result whose classes changed halfway through a codebase is a result nobody can write a function against. A program that wants one value converted rather than all of them calls `toTemporal()`, which is on all four classes and needs no option anywhere: ```ts -const rows = await conn.query<{ on: ZuDate }>(`MATCH (d:day) RETURN d.on AS on`); -const plain = rows[0].on.toTemporal(); // Temporal.PlainDate +const rows = await conn.query<{ began: ZuDate }>(`MATCH (d:dated) RETURN d.began AS began`); +const plain = rows[0].began.toTemporal(); // Temporal.PlainDate ``` Going the other way needs no opt-in at all. A `Temporal` value passed as a parameter binds as the zu value it is, on every connection, whether or not that connection asked for `Temporal` on the way out, because recognizing one costs a property read and refusing one would be a rule nobody could guess. `PlainDate`, `PlainTime`, `PlainDateTime`, `ZonedDateTime`, `Instant` and `Duration` all bind. A `PlainYearMonth`, a `PlainMonthDay` or a value on a calendar that is not `iso8601` is refused by name, since zu holds neither, and so is a `Duration` counting both months and days, because no number of days is a month and a value holding both would have to invent an answer for one month after 31 January. diff --git a/bench/query.mjs b/bench/query.mjs index 91bbf42..2f40262 100644 --- a/bench/query.mjs +++ b/bench/query.mjs @@ -35,12 +35,12 @@ const conn = await connect(join(dir, 'bench.zu1')) // The declaring insert is written with literals, because that is what // tells the engine what each column holds. -await conn.exec("INSERT (p:person {id: 0, name: 'n0', on: DATE '2024-01-01'})") +await conn.exec("INSERT (p:person {id: 0, name: 'n0', began: DATE '2024-01-01'})") for (let start = 1; start < ROWS; start += BATCH) { const end = Math.min(start + BATCH, ROWS) const parts = [] for (let ix = start; ix < end; ix++) { - parts.push(`(p${ix}:person {id: ${ix}, name: 'n${ix}', on: DATE '2024-01-01'})`) + parts.push(`(p${ix}:person {id: ${ix}, name: 'n${ix}', began: DATE '2024-01-01'})`) } await conn.exec(`INSERT ${parts.join(', ')}`) } @@ -104,7 +104,7 @@ const cases = [ // holding one integer. The number to read the next one against. name: 'scan, one DATE column', per: 'row', - run: () => conn.query('MATCH (p:person) RETURN p.on AS on'), + run: () => conn.query('MATCH (p:person) RETURN p.began AS began'), }, ...(temporal ? [ @@ -115,7 +115,7 @@ const cases = [ // whatever the runtime's own class costs to construct. name: 'scan, one DATE as Temporal', per: 'row', - run: () => temporal.query('MATCH (p:person) RETURN p.on AS on'), + run: () => temporal.query('MATCH (p:person) RETURN p.began AS began'), }, ] : []), diff --git a/binding.d.cts b/binding.d.cts index a79cbbc..c1a25c0 100644 --- a/binding.d.cts +++ b/binding.d.cts @@ -79,6 +79,10 @@ export type ZuTemporalValue = typeof globalThis extends { * default and `Temporal` values on a connection opened with * `{ temporal: true }`. A time with an offset is the exception in both * directions: `Temporal` has no type for one, so it stays a `ZuTime`. + * + * BYTES is a `Uint8Array` and not a string. The bytes are octets and + * need not be text at all, so decoding them is the caller's call to + * make rather than this client's to make for them. */ export type ZuValue = | null @@ -94,6 +98,7 @@ export type ZuValue = | ZuTimestamp | ZuDuration | ZuTemporalValue + | Uint8Array | ZuValue[] | { [field: string]: ZuValue } @@ -102,11 +107,18 @@ export type ZuValue = * * Wider than what comes out, because a `number` that is whole binds as * INT64 and `undefined` binds as null, which is what makes an optional - * field of a plain object pass straight through. A `Temporal` value - * binds as the zu value it is on every connection, whether or not the - * connection asked for `Temporal` on the way out, because recognizing - * one costs a property read and refusing one would be a rule nobody - * could guess. + * field of a plain object pass straight through. Negative zero is the + * exception and binds as FLOAT64: no INT64 is negative zero, so binding + * it as one throws away the sign the caller went out of their way to + * write. A `Temporal` value binds as the zu value it is on every + * connection, whether or not the connection asked for `Temporal` on the + * way out, because recognizing one costs a property read and refusing + * one would be a rule nobody could guess. + * + * A `Uint8Array` binds as BYTES, and it is the only typed array that + * binds at all: an `Int32Array` is a buffer somebody meant to load + * rather than a value a statement holds, so it is refused instead of + * being read as the empty object it has no properties to be. */ export type ZuParam = | null @@ -120,6 +132,7 @@ export type ZuParam = | ZuTimestamp | ZuDuration | ZuTemporalValue + | Uint8Array | ZuParam[] | { [field: string]: ZuParam } @@ -155,10 +168,11 @@ export type ZuAppendValue = * One value of a registered frame's column, when the column is written * as a plain array. * - * The same values an appender takes, without the bytes: a column of - * BYTES is a column no statement can read back yet, so registering one - * would be naming data the caller cannot get at. There is no `null` - * either, for the reason there is none in a row of an appender. + * The same values an appender takes, without the bytes: a frame column + * is a run of values the engine reads where it lies, and byte strings + * are not a run of anything, so a BYTES column is refused rather than + * copied into a shape it does not have. There is no `null` either, for + * the reason there is none in a row of an appender. */ export type ZuFrameValue = | boolean @@ -299,6 +313,20 @@ export interface ZuNotice { * `zone`. `value` is the fallback for what no fixed width cell covers, * which is nodes, rels, paths, lists and records, and `null` is a * column that held nothing else. + * + * `bytes` arrives in the two buffers `string` arrives in and is not + * one: the bytes are octets and a reader that decoded them as text + * would be handed something it cannot decode, which is why the two + * have separate names for one layout. + * + * This is what arrived rather than what the statement declared, and the + * two differ for the temporal types today. The engine's columnar sink + * has no buffer for days, nanoseconds or months, so a date, a time or a + * duration sometimes comes over as the values themselves, and which of + * the two a statement gets is the plan's business rather than the + * caller's. Such a column is `value` with its `ZuDate` and `ZuDuration` + * objects in `items`, not `date` with an empty `values`, so that a + * switch on `type` always lands on a field that holds something. */ export type ZuColumnType = | 'null' @@ -306,6 +334,7 @@ export type ZuColumnType = | 'int' | 'float' | 'string' + | 'bytes' | 'date' | 'time' | 'datetime' @@ -319,8 +348,8 @@ export type ZuColumnType = * not apply, so reading one is a switch on `type` rather than a series * of tests for what is there. Which field carries the values follows * from the type: `values` for everything of a fixed width, `data` and - * `offsets` for strings, `items` for what no buffer covers, and none of - * them for a column of nulls. + * `offsets` for strings and byte strings, `items` for what no buffer + * covers, and none of them for a column of nulls. * * The buffers are the engine's own, handed over rather than copied, and * they are laid out the way Arrow lays them out: values end to end, a @@ -339,12 +368,16 @@ export interface ZuColumn { * booleans, least significant bit first. */ readonly values: BigInt64Array | Float64Array | Int32Array | Uint8Array | null - /** The bytes of every string end to end, for a string column. */ + /** + * The bytes of every value end to end, for a `string` or a `bytes` + * column. + */ readonly data: Uint8Array | null /** - * `length + 1` offsets into `data`, for a string column. Narrow until - * the bytes pass what a 32 bit offset addresses, which is the - * difference Arrow calls Utf8 against LargeUtf8. + * `length + 1` offsets into `data`, for a `string` or a `bytes` + * column. Narrow until the bytes pass what a 32 bit offset + * addresses, which is the difference Arrow calls Utf8 against + * LargeUtf8. */ readonly offsets: Int32Array | BigInt64Array | null /** The values themselves, for a column of type `value`. */ @@ -357,7 +390,11 @@ export interface ZuColumn { readonly validity: Uint8Array | null /** How many rows are null, which is zero when `validity` is null. */ readonly nulls: number - /** What one cell counts: `days`, `nanos` or `months`. */ + /** + * What one cell counts: `days`, `nanos` or `months`. Null where there + * are no cells to count, which includes a temporal column that + * arrived as a `value` column. + */ readonly unit: 'days' | 'nanos' | 'months' | null /** Minutes east of UTC, for a column of zoned times or datetimes. */ readonly zone: number | null diff --git a/conformance/README.md b/conformance/README.md new file mode 100644 index 0000000..5cdf342 --- /dev/null +++ b/conformance/README.md @@ -0,0 +1,71 @@ +# The shared corpus, run through this client + +The corpus is one set of hand written YAML files in the engine's repository, under `conformance/cases`. Every client reads the same files and runs the same statements, so a value that survives one binding and not another is a diff rather than an argument. This directory is the Node end of that: a reader for the subset of YAML the cases are written in, a decoder for the value encoding, a reader for the Arrow schema an export carries, and a runner that reports what happened in the form the reference runner reports it. + +It is a development tool and not part of the package. `package.json` lists the files that ship and `conformance` is not among them, so it is there when the repository is checked out and absent when `zudb` is installed. Nothing under the published entry points imports it. + +## Running it + +The cases live in the engine's repository, pinned in `Cargo.toml` to the same revision this client builds against: + +``` +git clone https://github.com/tamnd/zu /tmp/zu +git -C /tmp/zu checkout 526ac583359f6112bca11bdae3fb43cc6fd76b2e +node conformance/cli.mjs /tmp/zu/conformance/cases +``` + +which prints every case that did not pass and then one line saying what the run came to: + +``` +1412 cases, 1411 passed, 0 failed, 1 unsupported +``` + +`npm run corpus -- /tmp/zu/conformance/cases` is the same thing through the script. + +`--strict` turns an unsupported case into a failed run, `--quiet` prints the summary alone, and `--work DIR` keeps the databases the cases were run against instead of removing them, which is what to reach for when a failure wants opening. + +The exit code is 0 when nothing failed, and 1 when something did or when the corpus will not read. One and not two for a corpus that will not read, because the reference runner exits one and a report compared line for line is worth less if the two runners disagree about what the run came to. + +CI runs the same thing. The `corpus` job reads the revision out of `Cargo.toml`, checks the engine out at it, and runs both the report and the tests below against the cases it finds, so a client that drifts from the corpus fails on the pull request that drifted rather than on the release that shipped. + +## What it is checking + +Four things, and the last two are the ones this client had to work for. + +A client can decode a value and pass every case, because the case and the answer both went through the same decoder. So the reader here is another implementation of the corpus format rather than a consumer of one: it refuses what `crates/zu-corpus/src/yaml.rs` refuses, with the same words and the same line numbers, and the tests in `test/conformance-reader.test.mjs` and `test/conformance-cases.test.mjs` are that file's own tables ported case for case. A reader that grew a hole would pass its own tests and fail those. + +The value encoding is the same again. An INT64 written bare is refused, a value wider than the type it claims is refused, a float is exact or it is not a float, and a temporal is written the way the engine prints it. `conformance/temporal.mjs` spells the four accepted forms out rather than calling `Date.parse`, which reads `2024-1-1` and `Jan 1 2024` and is allowed by its own specification to read anything else it likes. What a report prints is the encoding's own spelling, so a failure can be pasted back into a case. + +The comparison is not `===`. `NaN` matches `NaN`, `-0` does not match `0`, and a `bigint` never matches a `number`, which is the rule that keeps an INT64 and a FLOAT64 apart: this client puts the first in a `bigint` and the second in a `number`, so a case wanting `1` and a client giving `1n` is a diff and not a rounding. + +And the Arrow half is read out of the bytes. This client hands out Arrow IPC rather than a C Data Interface stream, so there is no `ArrowSchema` to walk and no `format` string to read off one. `arrow.mjs` parses the schema flatbuffer at the head of the buffer, with a small vtable reader, and turns each field back into the format string a case is written in. A flatbuffer omits any field holding the schema's declared default, which is the trap: `Int.is_signed` defaults to false, so an unsigned column is written as a width and nothing else, and a reader that defaults it the other way reports every UINT64 as `l` and says nothing. That one cost five cases before it was found, and `test/conformance-arrow.test.mjs` now pins it against bytes the addon really produced rather than against a buffer written by hand, which would only prove the reader agrees with whoever wrote the buffer. + +## What this client cannot answer + +One case, `result/the-same-name-twice`: + +``` +RETURN 1 AS a, 2 AS a +``` + +The engine answers it with two columns, both named `a`, which is a legal result and one this client's row mapping has nowhere to put: `query` hands back an array of plain objects keyed by column name, and an object has one `a`. The second column would silently replace the first. + +So the runner reports it unsupported rather than failed, by name and with the reason in the line. Nothing went wrong: the engine answered correctly, and the value mapping this client documents is where the column went. A columnar read of the same statement gives both columns, because a columnar read is an array and an array keeps its order, and the corpus asserts on rows. + +Unsupported is a third outcome and not a softer kind of failure. A case the corpus wrote on purpose to catch a mapping cost is a case a client should be able to say it cannot answer, out loud and with the reason, rather than passing it on a value invented for the occasion or failing it as though the engine had got something wrong. + +## The one check that is weaker here than in the engine + +A case naming a condition writes its GQLSTATUS, and the reference reader checks that code against the table the standard defines, which lives in `zu-common`. This reader checks the shape, five characters of digits and capitals, because the table is not something the client has. A code of the right shape that no standard defines is caught by the reference runner and not by this one. The Python reader and the C reader take the same position. + +## The files + +`reader.mjs` is the YAML subset: block mappings, block sequences, scalars, and a refusal with a line number for everything else. There is no YAML dependency, which would read these files and a good deal more besides, and would hand back `9223372036854775807` as a float on the way. + +`values.mjs` is the `{type, value}` encoding, both directions, and the comparison. + +`temporal.mjs` is the four temporal spellings and the duration one, read and printed, with nanoseconds as a `bigint` throughout for the reason they are one everywhere else in this client: nanoseconds from the epoch passed 2^53 in 1970. + +`cases.mjs` is what a case is, `arrow.mjs` is the schema reader and the schema comparison, and `runner.mjs` runs them, one database per case with a fresh copy of the suite's load, so a case that leaked a table into the next one would be a failure that moves when the file is reordered. + +The tests are in `test/conformance-*.test.mjs`, eighty five of them, and they run with the rest of the suite under `npm test`. diff --git a/conformance/arrow.mjs b/conformance/arrow.mjs new file mode 100644 index 0000000..2668b9c --- /dev/null +++ b/conformance/arrow.mjs @@ -0,0 +1,498 @@ +/* What a result looks like on the way out through Arrow. + * + * A client that reads rows one at a time and a client that exports a + * million of them to a dataframe are the same client, and only one of + * those paths is covered by a case that asserts values. The other one + * has its own contract: a column of dates is a Date32 and not a string + * of digits, a year-month duration is a month-day-nano interval because + * that is the interval every reader implements, a node is a struct of + * the name of its table and the row it is, and a time with an offset is + * refused rather than quietly moved to UTC. None of that shows up in a + * row a case compares. + * + * So a case may say what the export gives as well as what the rows are, + * and the runner checks both against one statement. What it checks is + * the schema, field by field and into the nested types, and how many + * rows came back through the stream. The schema is spelled in the C Data + * Interface's own format strings, "l" for an int64 and "+s" for a + * struct, because that is the one spelling every language sees the same. + * + * Where this file differs from the other runners is where the client + * does. Go and Python are handed a C Data Interface stream and read the + * format strings straight off it. This package hands out Arrow IPC + * bytes, one buffer holding a schema message and then the batches, and + * there is no pointer to read a format string from. So the schema + * message is parsed here, out of the flatbuffer, and turned back into + * the format strings the case is written in. + * + * That is a mapping rather than a reading, and a mapping is an opinion, + * which is worth being uncomfortable about. It is written the way the C + * Data Interface specification writes it, one arm per Arrow type and + * nothing inferred, and it is short enough to read in full. The + * alternative was to make `apache-arrow` a dependency of the corpus, + * which pulls a tree into a checkout whose whole job is to have nothing + * in it, and which does not hand out format strings either, so it would + * have been the same mapping written against a second set of names. + * + * Values are not read back here. A consumer that decoded every array by + * hand in each of nine languages would be nine new decoders under test, + * which is more of our own code and not more of the contract; the rows + * the case already asserts are the same values by another road. + */ + +import { quote, refuse } from './reader.mjs' + +/* How a report names the whole result, which is the place the columns of + * an export are in. + */ +export const THE_RESULT = 'the result' + +/* The stream saying no, with what it said. It is a type of its own so + * that a refusal on the way out is told apart from a schema that does + * not match, which is what a case writing `refused` turns on. + */ +export class ArrowError extends Error { + constructor(message) { + super(message) + this.name = 'ArrowError' + } +} + +/* Reads the `arrow:` of a case. + * + * A field is a name, the format string, and the fields under it when it + * is a struct or a list. A list has exactly one field under it, which + * Arrow names "item", and a case writes that out rather than leaving it + * implied: a client that named it "element" would export something no + * reader lines up with what another client wrote. + */ +export function parseExport(node) { + const text = node.str() + if (text !== null) { + if (text === 'refused') return { refused: true, fields: [] } + throw refuse( + `line ${node.line}: \`arrow:\` is the columns the export gives, or \`refused\` for a result ` + + `Arrow has no type for, and this is ${quote(text)}`, + ) + } + return { refused: false, fields: exportFields(node) } +} + +function exportFields(node) { + const items = node.seq() + if (items === null) { + throw refuse(`line ${node.line}: \`arrow:\` is a sequence of fields, and this is ${node.what()}`) + } + return items.map((item) => exportField(item)) +} + +function exportField(node) { + const at = node.line + if (!node.map()) { + throw refuse( + `line ${at}: an Arrow field is a mapping of \`name\` and \`format\`, and this is ${node.what()}`, + ) + } + const unknown = node.unknown('name', 'format', 'children') + if (unknown.length > 0) throw refuse(`line ${at}: an Arrow field has no key ${quote(unknown[0])}`) + const text = (key) => { + const value = node.get(key) + const spelled = value === null ? null : value.str() + if (spelled === null) throw refuse(`line ${at}: an Arrow field has a \`${key}:\``) + return spelled + } + const name = text('name') + const format = text('format') + if (format === '') throw refuse(`line ${at}: an empty format string is not a type Arrow has`) + const under = node.get('children') + const children = under === null ? [] : exportFields(under) + // A nested format is the one thing about a format string this reader + // knows, and it is worth knowing here: a case that wrote the fields of + // a struct under a "u" would be asserting something the export cannot + // produce, and finding that out at load time says so with a line + // number rather than as a failure in a report. + const nested = format[0] === '+' + if (nested && children.length === 0) { + throw refuse( + `line ${at}: ${quote(format)} is a nested type and the fields under it are part of it`, + ) + } + if (!nested && children.length > 0) { + throw refuse(`line ${at}: ${quote(format)} holds no fields, so nothing goes under it`) + } + return { name, format, children } +} + +/* The columns a result gives through Arrow, how many rows its batches + * hold, and how many rows it says it holds. + * + * The last two are separate on purpose. The buffer carries a row count + * beside the bytes, and the bytes carry their own in the batches, and a + * client where those two disagree has a bug that no case asserting + * values would ever see. + */ +export function exported(arrow) { + const bytes = arrow.ipc + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + let fields = null + let rows = 0 + for (const message of messages(view)) { + const kind = message.header.uint8(HEADER_TYPE, MESSAGE_NONE) + const header = message.header.table(HEADER) + if (header === null) continue + if (kind === MESSAGE_SCHEMA) { + if (fields !== null) throw new ArrowError('the stream carries two schemas') + fields = schemaFields(header) + continue + } + if (kind === MESSAGE_RECORD_BATCH) { + if (fields === null) throw new ArrowError('a batch arrived before the schema') + rows += Number(header.int64(BATCH_LENGTH, 0n)) + } + // A dictionary batch is skipped rather than refused. Nothing this + // engine exports is dictionary encoded today, and a runner that fell + // over on one would be asserting that, which is not the contract. + } + if (fields === null) throw new ArrowError('the stream carries no schema') + return { fields, rows, declared: arrow.rows } +} + +/* What the export gave that the case did not want, or the empty string + * when the two agree. + * + * The comparison walks the schema and the case's fields together and + * stops at the first difference, for the reason the row comparison does: + * the first is nearly always the cause of the rest. + */ +export function schemaSays(got, want) { + return fieldsUnder('', got, want) +} + +/* The fields under one place, where the place is the dotted path of the + * field they are under and the empty one is the result itself. + */ +function fieldsUnder(prefix, got, want) { + const place = prefix === '' ? THE_RESULT : quote(prefix) + if (got.length !== want.length) { + return `arrow gives ${got.length} fields in ${place} where the case wants ${want.length}` + } + for (let i = 0; i < got.length; i++) { + if (got[i].name !== want[i].name) { + return ( + `arrow field ${i + 1} in ${place} is named ${quote(got[i].name)} where the case wants ` + + quote(want[i].name) + ) + } + // The path is the case's own names joined with dots, which is how a + // field inside a path inside a column is pointed at without printing + // the whole schema at somebody. + const path = prefix === '' ? want[i].name : `${prefix}.${want[i].name}` + if (got[i].format !== want[i].format) { + return ( + `arrow field ${quote(path)} is ${quote(got[i].format)} where the case wants ` + + quote(want[i].format) + ) + } + const why = fieldsUnder(path, got[i].children, want[i].children) + if (why !== '') return why + } + return '' +} + +/* The IPC framing. + * + * A message is a continuation marker, the length of its metadata, the + * metadata itself, and then a body of the length the metadata declares. + * The stream ends at a length of zero or at the end of the buffer, and a + * writer old enough to predate the marker leaves it out, which is why + * the length is looked at before it is trusted. + */ +function* messages(view) { + let at = 0 + while (at + 4 <= view.byteLength) { + let length = view.getInt32(at, true) + at += 4 + if (length === CONTINUATION) { + if (at + 4 > view.byteLength) return + length = view.getInt32(at, true) + at += 4 + } + if (length <= 0) return + if (at + length > view.byteLength) { + throw new ArrowError(`a message declares ${length} bytes of metadata past the end of the stream`) + } + const header = root(view, at) + at += length + const body = Number(header.int64(BODY_LENGTH, 0n)) + if (body < 0 || at + body > view.byteLength) { + throw new ArrowError(`a message declares a body of ${body} bytes past the end of the stream`) + } + yield { header } + // A body is padded out to eight, and the padding is not counted in + // the length the message declares. + at += body + ((8 - (body % 8)) % 8) + } +} + +const CONTINUATION = -1 + +// Message +const HEADER_TYPE = 1 +const HEADER = 2 +const BODY_LENGTH = 3 + +// MessageHeader +const MESSAGE_NONE = 0 +const MESSAGE_SCHEMA = 1 +const MESSAGE_RECORD_BATCH = 3 + +// Schema +const SCHEMA_FIELDS = 1 + +// RecordBatch +const BATCH_LENGTH = 0 + +// Field +const FIELD_NAME = 0 +const FIELD_TYPE_TYPE = 2 +const FIELD_TYPE = 3 +const FIELD_CHILDREN = 5 + +function schemaFields(schema) { + const out = [] + const count = schema.vectorLength(SCHEMA_FIELDS) + for (let i = 0; i < count; i++) out.push(walked(schema.element(SCHEMA_FIELDS, i))) + return out +} + +/* One field of an exported schema, and everything under it. */ +function walked(field) { + const children = [] + const count = field.vectorLength(FIELD_CHILDREN) + for (let i = 0; i < count; i++) children.push(walked(field.element(FIELD_CHILDREN, i))) + return { name: field.string(FIELD_NAME) ?? '', format: formatOf(field), children } +} + +/* The Type union, in the order the schema flatbuffer numbers it. */ +const NULL = 1 +const INT = 2 +const FLOATING_POINT = 3 +const BINARY = 4 +const UTF8 = 5 +const BOOL = 6 +const DECIMAL = 7 +const DATE = 8 +const TIME = 9 +const TIMESTAMP = 10 +const INTERVAL = 11 +const LIST = 12 +const STRUCT = 13 +const FIXED_SIZE_BINARY = 15 +const FIXED_SIZE_LIST = 16 +const MAP = 17 +const DURATION = 18 +const LARGE_BINARY = 19 +const LARGE_UTF8 = 20 +const LARGE_LIST = 21 + +/* The format string for a field's type, spelled the way the C Data + * Interface spells it. + * + * One arm per Arrow type and nothing inferred. A type this does not know + * is a refusal naming the number, because a schema this file cannot read + * is a report that would otherwise say the export was wrong when what + * happened is that the runner did not keep up. + */ +function formatOf(field) { + const which = field.uint8(FIELD_TYPE_TYPE, 0) + const type = field.table(FIELD_TYPE) + const say = (message) => new ArrowError(`${message} in field ${quote(field.string(FIELD_NAME) ?? '')}`) + switch (which) { + case NULL: + return 'n' + case BOOL: + return 'b' + case BINARY: + return 'z' + case LARGE_BINARY: + return 'Z' + case UTF8: + return 'u' + case LARGE_UTF8: + return 'U' + case STRUCT: + return '+s' + case LIST: + return '+l' + case LARGE_LIST: + return '+L' + case MAP: + return '+m' + case INT: { + const width = type === null ? 0 : type.int32(0, 0) + // Unsigned is the default in the schema, and a flatbuffer leaves + // out a field that holds its default, so an absent `is_signed` is + // an unsigned integer and not a missing answer. Defaulting the + // other way reads every UINT64 as an INT64 and reports `l` for a + // field the case wrote `L`. + const signed = type === null ? false : type.bool(1, false) + const spelled = { 8: 'c', 16: 's', 32: 'i', 64: 'l' }[width] + if (spelled === undefined) throw say(`an integer of ${width} bits`) + return signed ? spelled : spelled.toUpperCase() + } + case FLOATING_POINT: { + // Half is the default, and an export of a half float leaves the + // field out rather than writing a zero into it. + const precision = type === null ? 0 : type.int16(0, 0) + const spelled = ['e', 'f', 'g'][precision] + if (spelled === undefined) throw say(`a float of precision ${precision}`) + return spelled + } + case DECIMAL: { + const scale = type === null ? 0 : type.int32(1, 0) + const precision = type === null ? 0 : type.int32(0, 0) + const bits = type === null ? 128 : type.int32(2, 128) + return bits === 128 ? `d:${precision},${scale}` : `d:${precision},${scale},${bits}` + } + case DATE: { + const unit = type === null ? 1 : type.int16(0, 1) + if (unit === 0) return 'tdD' + if (unit === 1) return 'tdm' + throw say(`a date in unit ${unit}`) + } + case TIME: { + // Milliseconds here, unlike a timestamp, which the schema gives no + // default at all and so defaults to seconds. + const unit = type === null ? 1 : type.int16(0, 1) + const spelled = ['tts', 'ttm', 'ttu', 'ttn'][unit] + if (spelled === undefined) throw say(`a time in unit ${unit}`) + return spelled + } + case TIMESTAMP: { + const unit = type === null ? 0 : type.int16(0, 0) + const spelled = ['tss', 'tsm', 'tsu', 'tsn'][unit] + if (spelled === undefined) throw say(`a timestamp in unit ${unit}`) + // The zone is part of the type rather than beside it, and an + // absent one is a timestamp with no zone rather than one in UTC. + return `${spelled}:${type === null ? '' : (type.string(1) ?? '')}` + } + case DURATION: { + // Milliseconds, the way a time is and a timestamp is not. + const unit = type === null ? 1 : type.int16(0, 1) + const spelled = ['tDs', 'tDm', 'tDu', 'tDn'][unit] + if (spelled === undefined) throw say(`a duration in unit ${unit}`) + return spelled + } + case INTERVAL: { + const unit = type === null ? 0 : type.int16(0, 0) + const spelled = ['tiM', 'tiD', 'tin'][unit] + if (spelled === undefined) throw say(`an interval in unit ${unit}`) + return spelled + } + case FIXED_SIZE_BINARY: + return `w:${type === null ? 0 : type.int32(0, 0)}` + case FIXED_SIZE_LIST: + return `+w:${type === null ? 0 : type.int32(0, 0)}` + default: + throw say(`arrow type ${which}`) + } +} + +/* Enough of a flatbuffer reader to walk a schema message. + * + * A table is a signed offset back to its vtable and then its fields. The + * vtable is its own size, the table's size, and then one unsigned short + * per field holding where that field is inside the table, where a zero + * or a slot past the end of the vtable means the field is not there and + * the default stands. Everything is little endian, an offset to another + * object is unsigned and relative to where the offset itself is, a + * string is a length and then its bytes, and a vector is a length and + * then its elements. + * + * That is the whole format. It is here rather than generated because + * generated flatbuffer code for the Arrow schema is several thousand + * lines to read four numbers out of a header, and because a corpus + * checkout should pull nothing. + */ +class Table { + constructor(view, at) { + this.view = view + this.at = at + this.vtable = at - view.getInt32(at, true) + } + + /* Where a field is inside the table, or zero when it is not there. */ + slot(index) { + const size = this.view.getUint16(this.vtable, true) + const at = 4 + index * 2 + if (at >= size) return 0 + return this.view.getUint16(this.vtable + at, true) + } + + uint8(index, fallback) { + const at = this.slot(index) + return at === 0 ? fallback : this.view.getUint8(this.at + at) + } + + bool(index, fallback) { + const at = this.slot(index) + return at === 0 ? fallback : this.view.getUint8(this.at + at) !== 0 + } + + int16(index, fallback) { + const at = this.slot(index) + return at === 0 ? fallback : this.view.getInt16(this.at + at, true) + } + + int32(index, fallback) { + const at = this.slot(index) + return at === 0 ? fallback : this.view.getInt32(this.at + at, true) + } + + int64(index, fallback) { + const at = this.slot(index) + return at === 0 ? fallback : this.view.getBigInt64(this.at + at, true) + } + + /* Another table, at the far end of an offset. */ + table(index) { + const at = this.slot(index) + if (at === 0) return null + const from = this.at + at + return new Table(this.view, from + this.view.getUint32(from, true)) + } + + string(index) { + const at = this.slot(index) + if (at === 0) return null + let from = this.at + at + from += this.view.getUint32(from, true) + const length = this.view.getUint32(from, true) + const bytes = new Uint8Array( + this.view.buffer, + this.view.byteOffset + from + 4, + length, + ) + return new TextDecoder('utf-8', { fatal: true }).decode(bytes) + } + + vectorLength(index) { + const at = this.slot(index) + if (at === 0) return 0 + const from = this.at + at + return this.view.getUint32(from + this.view.getUint32(from, true), true) + } + + /* One element of a vector of tables. */ + element(index, i) { + const at = this.slot(index) + const from = this.at + at + const start = from + this.view.getUint32(from, true) + 4 + i * 4 + return new Table(this.view, start + this.view.getUint32(start, true)) + } +} + +/* The table a buffer opens with, which is where the root offset points. */ +function root(view, at) { + return new Table(view, at + view.getUint32(at, true)) +} diff --git a/conformance/cases.mjs b/conformance/cases.mjs new file mode 100644 index 0000000..445a74d --- /dev/null +++ b/conformance/cases.mjs @@ -0,0 +1,470 @@ +/* What a case is, and how a file of them is read. + * + * A case is a statement and what running it must produce. That is + * deliberately the whole of it. Every client in every language can run a + * statement and look at the rows that come back, so a corpus written in + * those terms is one every client can run, and a corpus written in terms + * of a client's own API would be nine corpora. + * + * The expectation is either rows or a condition. A case expecting a + * condition names the GQLSTATUS code, not the message, because the code + * is the contract and the message is prose that will improve. + * + * A statement may take parameters, which is the other direction the same + * values travel: a case with `params:` writes a value in the encoding, + * hands it to this client's own binding call, and asserts what came + * back. A client that decodes a date correctly and encodes it a day + * early passes every case that has no parameters in it. + * + * A case may say which connection each of its statements runs on, which + * is how a case about a transaction is written: a transaction is only + * observable from outside it, so a case that has to say what a commit + * means needs a second connection to say it to. A case that says nothing + * runs everything on one connection called main, which is every case but + * a handful. + */ + +import { readdir, readFile } from 'node:fs/promises' +import { basename, join } from 'node:path' + +import { parseExport } from './arrow.mjs' +import { parse, quote, refuse } from './reader.mjs' +import { decode, form, payload, typed } from './values.mjs' + +/* The schema version a file declares. It exists so that a corpus + * unpacked from an old release tells a new runner what it is instead of + * failing in the middle. + */ +export const SCHEMA = 4 + +/* The connection a statement runs on when the case does not name one. */ +export const MAIN = 'main' + +const SUITE_KEYS = ['schema', 'suite', 'doc', 'load', 'cases'] +const CASE_KEYS = [ + 'name', + 'doc', + 'setup', + 'on', + 'params', + 'query', + 'columns', + 'rows', + 'raises', + 'arrow', +] +const LOAD_KEYS = ['nodes', 'edges', 'count', 'columns', 'pairs'] + +/* Every suite in a directory, in the order a sorted listing gives, which + * is the order the reference runner walks them in. + */ +export async function readDir(directory) { + let names + try { + names = await readdir(directory) + } catch (err) { + throw refuse(`${directory}: ${err.message}`) + } + // Sorted, because a directory listing is the filesystem's order and a + // report that is diffed against another runner's has to walk them the + // same way. + const paths = names + .filter((name) => name.endsWith('.yaml')) + .sort() + .map((name) => join(directory, name)) + const suites = [] + for (const path of paths) { + let text + try { + text = await readFile(path, 'utf8') + } catch (err) { + throw refuse(`${path}: ${err.message}`) + } + let suite + try { + suite = read(text) + } catch (err) { + throw refuse(`${path}: ${err.message}`) + } + const stem = basename(path, '.yaml') + if (suite.name !== stem) { + throw refuse( + `${path}: the suite calls itself ${quote(suite.name)} and the file calls it ${quote(stem)}`, + ) + } + suites.push(suite) + } + if (suites.length === 0) throw refuse(`${directory}: no case files`) + return suites +} + +/* A suite, or the first thing in the file that is not one. */ +export function read(text) { + const doc = parse(text) + const unknown = doc.unknown(...SUITE_KEYS) + if (unknown.length > 0) { + throw refuse(`line ${doc.line}: a suite has no key ${quote(unknown[0])}`) + } + const schemaNode = doc.get('schema') + const schema = schemaNode === null ? null : schemaNode.str() + if (schema === null) throw refuse('the file does not open with `schema:`') + if (!/^[0-9]+$/.test(schema)) throw refuse(`${quote(schema)} is not a schema version`) + const version = Number(schema) + if (version !== SCHEMA) { + throw refuse(`this is schema ${version} and the runner reads schema ${SCHEMA}`) + } + + const name = field(doc, 'suite') + const docText = field(doc, 'doc') + const loadNode = doc.get('load') + const load = loadNode === null ? null : readLoad(loadNode) + + const casesNode = doc.get('cases') + if (casesNode === null) throw refuse('a suite with no `cases:`') + const items = casesNode.seq() + if (items === null) throw refuse('`cases:` is a sequence') + if (items.length === 0) throw refuse('a suite with no cases in it') + const cases = [] + // Names are what a report cites and what a binding's skip list names, + // so two cases sharing one is a report that says less than it looks + // like it does. + const seen = new Set() + for (const item of items) { + const one = readCase(item) + if (seen.has(one.name)) throw refuse(`two cases are called ${quote(one.name)}`) + seen.add(one.name) + cases.push(one) + } + return { name, doc: docText, load, cases } +} + +function field(node, key) { + const value = node.get(key) + if (value === null) throw refuse(`line ${node.line}: no \`${key}:\``) + const text = value.str() + if (text === null) throw refuse(`line ${node.line}: \`${key}:\` is one line of text`) + return text +} + +function readCase(node) { + const at = node.line + if (!node.map()) throw refuse(`line ${at}: a case is a mapping, and this is ${node.what()}`) + const unknown = node.unknown(...CASE_KEYS) + if (unknown.length > 0) throw refuse(`line ${at}: a case has no key ${quote(unknown[0])}`) + + const name = field(node, 'name') + if (!dashedWords(name)) { + throw refuse( + `line ${at}: ${quote(name)} is a case name, which is lower case words joined by dashes`, + ) + } + const doc = field(node, 'doc') + const query = field(node, 'query') + + const setup = [] + const setupNode = node.get('setup') + if (setupNode !== null) { + const items = setupNode.seq() + if (items === null) throw refuse(`line ${at}: \`setup:\` is a sequence of statements`) + for (const item of items) setup.push(readStep(item)) + } + + const onNode = node.get('on') + const on = onNode === null ? MAIN : connectionName(onNode) + const params = readParams(node) + const arrowNode = node.get('arrow') + const arrow = arrowNode === null ? null : parseExport(arrowNode) + + const one = { + name, + doc, + query, + line: at, + setup, + on, + params, + arrow, + hasColumns: false, + columns: [], + rows: [], + raises: '', + } + + const raisesNode = node.get('raises') + const columnsNode = node.get('columns') + if (raisesNode !== null && columnsNode !== null) { + throw refuse( + `line ${at}: a case that raises has no rows, and one that returns rows does not raise`, + ) + } + if (raisesNode !== null) { + const code = raisesNode.str() + if (code === null) throw refuse(`line ${at}: \`raises:\` is a GQLSTATUS code`) + if (!gqlstatusShaped(code)) { + throw refuse( + `line ${raisesNode.line}: ${quote(code)} is not the shape of a GQLSTATUS, which is five ` + + 'characters of digits and capitals', + ) + } + one.raises = code + return one + } + if (columnsNode === null) { + throw refuse( + `line ${at}: a case says what it produces, with \`columns:\` and \`rows:\` or with \`raises:\``, + ) + } + // Empty counts, because FINISH is a query that answers no columns at + // all, which is not the same as a query whose columns held no rows, + // and the corpus writes it as a `columns:` with nothing under it. + const names = columnsNode.seqOrEmpty() + if (names === null) throw refuse(`line ${at}: \`columns:\` is a sequence of names`) + const columns = names.map((item) => { + const text = item.str() + if (text === null) throw refuse(`line ${item.line}: a column name is one word`) + return text + }) + const rows = readRows(node) + for (const row of rows) { + if (row.length !== columns.length) { + throw refuse(`line ${at}: a row of ${row.length} against ${columns.length} columns`) + } + } + one.hasColumns = true + one.columns = columns + one.rows = rows + return one +} + +/* One setup statement, which is a line of its own or a line and the + * connection it runs on. + */ +function readStep(node) { + const text = node.str() + if (text !== null) return { on: MAIN, query: text } + if (!node.map()) { + throw refuse( + `line ${node.line}: a setup statement is one line, or \`on:\` and \`query:\`, and this is ` + + node.what(), + ) + } + const unknown = node.unknown('on', 'query') + if (unknown.length > 0) { + throw refuse(`line ${node.line}: a setup statement has no key ${quote(unknown[0])}`) + } + const onNode = node.get('on') + if (onNode === null) { + throw refuse( + `line ${node.line}: a setup statement written as a mapping names the connection it runs on`, + ) + } + return { on: connectionName(onNode), query: field(node, 'query') } +} + +/* The name of a connection, spelled the way a case name is, because a + * report cites it and a name a reader has to guess at is a report that + * says less than it looks like it does. + */ +function connectionName(node) { + const name = node.str() + if (name === null) throw refuse(`line ${node.line}: \`on:\` is the name of a connection`) + if (!dashedWords(name)) { + throw refuse( + `line ${node.line}: ${quote(name)} is a connection name, which is lower case words joined ` + + 'by dashes', + ) + } + return name +} + +/* The parameters a case binds, which is the value encoding with a name + * beside it. + * + * A name is what the statement spells after the $, so it is checked + * against what a statement may spell: a case whose name is "n one" is + * one no client can bind. + */ +function readParams(node) { + const paramsNode = node.get('params') + if (paramsNode === null) return [] + const items = paramsNode.seq() + if (items === null) throw refuse(`line ${paramsNode.line}: \`params:\` is a sequence`) + const out = [] + for (const item of items) { + const at = item.line + if (!item.map()) { + throw refuse( + `line ${at}: a parameter is a mapping of \`name\`, \`type\` and \`value\`, and this is ` + + item.what(), + ) + } + const unknown = item.unknown('name', 'type', 'value') + if (unknown.length > 0) throw refuse(`line ${at}: a parameter has no key ${quote(unknown[0])}`) + const name = field(item, 'name') + if (!wordOrUnderscore(name)) { + throw refuse( + `line ${at}: ${quote(name)} is a parameter name, which is what a statement writes after ` + + 'the `$`', + ) + } + if (out.some((held) => held.name === name)) { + throw refuse(`line ${at}: two parameters are called ${quote(name)}`) + } + out.push({ name, value: typed(item) }) + } + return out +} + +function readRows(node) { + const rowsNode = node.get('rows') + if (rowsNode === null) { + // A statement that returns no rows is a case worth having, and + // writing it as an absent `rows:` would make it the same shape as + // one somebody forgot to finish. + throw refuse( + `line ${node.line}: \`columns:\` with no \`rows:\`. A case expecting nothing back writes ` + + '`rows:` with an empty sequence under it.', + ) + } + const items = rowsNode.seqOrEmpty() + if (items === null) throw refuse(`line ${rowsNode.line}: \`rows:\` is a sequence of rows`) + return items.map((item) => { + const unknown = item.unknown('values') + if (unknown.length > 0) throw refuse(`line ${item.line}: a row has no key ${quote(unknown[0])}`) + const cellsNode = item.get('values') + if (cellsNode === null) { + throw refuse(`line ${item.line}: a row is a \`values:\` and the values under it`) + } + const cells = cellsNode.seqOrEmpty() + if (cells === null) { + throw refuse(`line ${cellsNode.line}: \`values:\` is a sequence of values`) + } + return cells.map((cell) => decode(cell)) + }) +} + +/* One node table, its columns, and the edges between its rows. + * + * Everything else in the corpus is an expression, and an expression says + * what a value means on the way out and nothing about how it got in. A + * load is the other half, and every runner puts it in through its own + * bulk load path, which for this client is the package's `load`. + */ +function readLoad(node) { + const at = node.line + if (!node.map()) throw refuse(`line ${at}: a load is a mapping, and this is ${node.what()}`) + const unknown = node.unknown(...LOAD_KEYS) + if (unknown.length > 0) throw refuse(`line ${at}: a load has no key ${quote(unknown[0])}`) + const nodes = tableName(node, 'nodes') + const edges = tableName(node, 'edges') + const countNode = node.get('count') + const countText = countNode === null ? null : countNode.str() + if (countText === null) { + throw refuse(`line ${at}: a load says how many rows it has, with \`count:\``) + } + if (!/^[0-9]+$/.test(countText)) throw refuse(`line ${at}: \`count:\` is a number of rows`) + const count = Number(countText) + if (count === 0) { + throw refuse(`line ${at}: a load of no rows is a load nothing can be read back from`) + } + + const columnsNode = node.get('columns') + if (columnsNode === null) throw refuse(`line ${at}: a load has \`columns:\``) + const items = columnsNode.seq() + if (items === null) throw refuse(`line ${at}: \`columns:\` is a sequence`) + const columns = [] + const seen = new Set() + for (const item of items) { + const column = readColumn(item, count) + if (seen.has(column.name)) throw refuse(`line ${at}: two columns are called ${quote(column.name)}`) + seen.add(column.name) + columns.push(column) + } + if (columns.length === 0) throw refuse(`line ${at}: a load with no columns holds no values`) + + const pairs = [] + const pairsNode = node.get('pairs') + if (pairsNode !== null) { + const edgeItems = pairsNode.seqOrEmpty() + if (edgeItems === null) throw refuse(`line ${at}: \`pairs:\` is a sequence of edges`) + for (const item of edgeItems) pairs.push(readEdge(item, count)) + } + return { nodes, edges, count, columns, pairs } +} + +function tableName(node, key) { + const text = field(node, key) + if (!wordOrUnderscore(text)) { + throw refuse(`line ${node.line}: ${quote(text)} is not a table name`) + } + return text +} + +/* One column of a load: a name, the type every value in it has, and the + * values in row order. + */ +function readColumn(node, count) { + const at = node.line + const unknown = node.unknown('name', 'type', 'values') + if (unknown.length > 0) throw refuse(`line ${at}: a column has no key ${quote(unknown[0])}`) + const name = tableName(node, 'name') + const type = field(node, 'type') + if (!form(type).known) { + throw refuse(`line ${at}: ${type} is not a type this encoding knows`) + } + const valuesNode = node.get('values') + const items = valuesNode === null ? null : valuesNode.seq() + if (items === null) throw refuse(`line ${at}: a column holds \`values:\` in row order`) + if (items.length !== count) { + throw refuse( + `line ${at}: column ${quote(name)} holds ${items.length} values against the ${count} rows ` + + 'the load declares', + ) + } + return { name, type, values: items.map((item) => payload(type, item)) } +} + +function readEdge(node, count) { + const at = node.line + const unknown = node.unknown('from', 'to') + if (unknown.length > 0) throw refuse(`line ${at}: an edge has no key ${quote(unknown[0])}`) + const ends = [] + for (const key of ['from', 'to']) { + const value = node.get(key) + const text = value === null ? null : value.str() + if (text === null) throw refuse(`line ${at}: an edge has a \`${key}:\` row number`) + if (!/^-?[0-9]+$/.test(text)) throw refuse(`line ${at}: \`${key}:\` is a row number`) + const end = Number(text) + if (end < 0 || end >= count) { + throw refuse( + `line ${at}: \`${key}: ${end}\` against a table of ${count} rows, which are numbered 0 ` + + `to ${count - 1}`, + ) + } + ends.push(end) + } + return ends +} + +/* Whether text is lower case ASCII words joined by dashes, which is how + * a case and a connection are named. + */ +function dashedWords(text) { + return text !== '' && /^[a-z0-9-]+$/.test(text) +} + +/* Whether text is ASCII letters, digits and underscores, which is what a + * statement may write after a $ and what a table may be called. + */ +function wordOrUnderscore(text) { + return text !== '' && /^[A-Za-z0-9_]+$/.test(text) +} + +/* Whether a code is the shape of a GQLSTATUS, which is five characters + * of digits and capitals. The shape and not the list: a corpus that had + * to be told about every code the standard defines would be one nobody + * could add a case to. + */ +function gqlstatusShaped(code) { + return /^[0-9A-Z]{5}$/.test(code) +} diff --git a/conformance/cli.mjs b/conformance/cli.mjs new file mode 100644 index 0000000..c9030e9 --- /dev/null +++ b/conformance/cli.mjs @@ -0,0 +1,92 @@ +#!/usr/bin/env node +/* Runs the shared cross-client corpus against this client and prints + * what happened. + * + * node conformance/cli.mjs ../zu/conformance/cases + * + * The report is the reference runner's, line for line, so a disagreement + * between two clients is a diff and not a reading exercise. It exits + * zero when nothing failed and one when something did or when the corpus + * will not read, which is also the reference runner's rule. + */ + +import { mkdtemp, mkdir, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { parseArgs } from 'node:util' + +import { readDir } from './cases.mjs' +import { FAILED, PASSED, UNSUPPORTED, count, line, run, summary } from './runner.mjs' + +const usage = [ + 'usage: corpus [flags] ', + 'run the shared corpus cases against this client', + ' --strict an unsupported case fails the run, which is what a release branch wants', + ' --quiet print the summary and nothing else', + ' --work= a directory to make the case databases under, kept rather than removed', +].join('\n') + +process.exitCode = await main() + +async function main() { + let options + let positionals + try { + ;({ values: options, positionals } = parseArgs({ + options: { + strict: { type: 'boolean', default: false }, + quiet: { type: 'boolean', default: false }, + work: { type: 'string', default: '' }, + }, + allowPositionals: true, + })) + } catch (err) { + console.error(err.message) + console.error(usage) + return 2 + } + if (positionals.length !== 1) { + console.error(usage) + return 2 + } + + let suites + try { + suites = await readDir(positionals[0]) + } catch (err) { + // One rather than two, because the reference runner exits one for a + // corpus it cannot read and a report that is compared line for line + // is worth less if the two disagree about what the run came to. + console.error('zu corpus:', err.message) + return 1 + } + + let directory = options.work + let temporary = '' + try { + if (directory === '') { + // Removed when the run ends, and each case removes its own as it + // finishes, so what is left in here at the end is the databases of + // the cases that failed. A run with --work keeps them. + temporary = await mkdtemp(join(tmpdir(), 'zu-corpus-')) + directory = temporary + } else { + await mkdir(directory, { recursive: true }) + } + } catch (err) { + console.error('zu corpus:', err.message) + return 1 + } + + try { + const ran = await run(suites, directory, (one) => { + if (!options.quiet && one.outcome !== PASSED) console.log(line(one)) + }) + console.log(summary(ran)) + if (count(ran, FAILED) > 0) return 1 + if (options.strict && count(ran, UNSUPPORTED) > 0) return 1 + return 0 + } finally { + if (temporary !== '') await rm(temporary, { recursive: true, force: true }) + } +} diff --git a/conformance/reader.mjs b/conformance/reader.mjs new file mode 100644 index 0000000..4c60ad2 --- /dev/null +++ b/conformance/reader.mjs @@ -0,0 +1,459 @@ +/* The subset of YAML the corpus is written in. + * + * YAML is a large language and the corpus needs a small corner of it: + * block mappings, block sequences, and scalars. Everything else is + * refused with a line number. The files are hand written and are read + * by people in nine repositories who did not write them, so a construct + * a reader quietly reinterpreted would be a case that says one thing to + * a reviewer and another to the runner. + * + * So: two space indentation and no tabs, `- ` with exactly one space, + * plain, single quoted and double quoted scalars on one line, and + * comments. No flow collections, no block scalars, no anchors, no + * aliases, no tags, no document markers, no multi document streams. + * + * This is the fifth implementation of that subset, after + * crates/zu-corpus/src/yaml.rs in the engine, conformance/c/yaml.c + * beside it, conformance/reader.py in zu-python and corpus/reader.go in + * zu-go. There is a YAML package on npm that would read these files, + * and would read a good deal more besides: it would take a flow + * sequence, a block scalar and an anchor, none of which a case may use. + * What the corpus needs is a reader that refuses, and the cheapest way + * to have one is to write it. + * + * Whether a scalar was quoted survives parsing, because the value + * encoding turns on it. An INT64 written bare is a number some reader + * in some language will round, and refusing it is the whole point of + * the encoding. + */ + +/* A file the corpus will not read, with the line it gave up on. + * + * A class of its own rather than a plain `Error` so that the command + * can tell a corpus it cannot read from a case that did not pass, which + * are two different exits. + */ +export class CorpusError extends Error { + constructor(message) { + super(message) + this.name = 'CorpusError' + } +} + +/* Every refusal in this file and the three beside it. + * + * The message is the whole of it: a reader that also carried a stack + * would be printing the shape of this package at somebody trying to fix + * a case. + */ +export function refuse(message) { + return new CorpusError(message) +} + +/* A string the way Rust's `{:?}` writes one. + * + * Every refusal in the corpus is written in five languages and diffed + * across them, so a value quoted one way here and another way there + * would be a difference in the report that is not a difference in the + * answer. `JSON.stringify` escapes every code point it thinks is + * unprintable and Rust does not, so the quoting is written out rather + * than borrowed. + */ +export function quote(text) { + let out = '"' + for (const c of text) { + if (c === '"' || c === '\\') out += '\\' + c + else if (c === '\n') out += '\\n' + else if (c === '\r') out += '\\r' + else if (c === '\t') out += '\\t' + else out += c + } + return out + '"' +} + +/* The four shapes a node is. + * + * `empty` is a key with nothing under it: it is a node rather than an + * error because a case that expects no rows back writes `rows:` and + * stops, and that is a real expectation which needs a spelling. Every + * accessor says no to it, so a `name:` left blank is still caught by + * whoever wanted a name. + */ +export const SCALAR = 'scalar' +export const SEQ = 'seq' +export const MAP = 'map' +export const EMPTY = 'empty' + +/* One node of a document, with the line it started on. */ +export class Node { + constructor(kind, line, { text = '', quoted = false, items = [], pairs = [] } = {}) { + this.kind = kind + this.line = line + this.text = text + this.quoted = quoted + this.items = items + this.pairs = pairs + } + + /* What kind of node this is, for an error that has to say what it + * found instead of what it wanted. + */ + what() { + if (this.kind === SCALAR) return 'a scalar' + if (this.kind === SEQ) return 'a sequence' + if (this.kind === MAP) return 'a mapping' + return 'nothing' + } + + /* The text of a scalar and whether it was written in quotes, or null + * for anything that is not a scalar. + */ + scalar() { + if (this.kind !== SCALAR) return null + return { text: this.text, quoted: this.quoted } + } + + /* The text of a scalar, for a caller the quoting does not concern, + * and null for anything else. + */ + str() { + return this.kind === SCALAR ? this.text : null + } + + /* The items of a sequence, or null. */ + seq() { + return this.kind === SEQ ? this.items : null + } + + /* A sequence, counting a key with nothing under it as the empty one. + * + * Only a caller for whom empty is a meaningful answer should reach + * for this. The rest want `seq`, so that a list somebody left + * unfinished is refused rather than read as none. + */ + seqOrEmpty() { + if (this.kind === EMPTY) return [] + return this.seq() + } + + /* The entries of a mapping, in the order they were written. */ + map() { + return this.kind === MAP ? this.pairs : null + } + + /* The value under one key, or null when this is not a mapping or the + * key is not in it. + */ + get(key) { + if (this.kind !== MAP) return null + for (const pair of this.pairs) if (pair.key === key) return pair.value + return null + } + + /* The keys that are not in `known`, so a caller can refuse a typo + * rather than drop the field on the floor. + */ + unknown(...known) { + if (this.kind !== MAP) return [] + return this.pairs.filter((p) => !known.includes(p.key)).map((p) => p.key) + } +} + +/* A document, or the first thing in it this reader will not read. */ +export function parse(text) { + const lines = lex(text) + if (lines.length === 0) throw refuse('the file has nothing in it') + if (lines[0].indent !== 0) throw refuse(`line ${lines[0].no}: the first line is indented`) + const at = { lines, i: 0 } + const node = parseNode(at, 0) + if (at.i < lines.length) { + throw refuse(`line ${lines[at.i].no}: this belongs to nothing above it`) + } + return node +} + +/* Lines, with blanks and comments dropped and every `- ` split into the + * item it opens and the content that followed it on the same line. + * + * Splitting here rather than in the parser is what lets `- name: x` and + * a `name: x` on its own line be the same shape by the time anything + * looks at them. + */ +function lex(text) { + const out = [] + const raws = text.split('\n') + for (let n = 0; n < raws.length; n++) { + const raw = raws[n] + const no = n + 1 + const tab = raw.indexOf('\t') + if (tab >= 0) { + throw refuse(`line ${no}: a tab at column ${tab + 1}, and indentation here is spaces`) + } + const content = trimRight(stripComment(raw)) + const rest0 = content.replace(/^ +/, '') + const indent = content.length - rest0.length + if (rest0 === '') continue + if (rest0 === '---' || rest0 === '...') { + throw refuse( + `line ${no}: ${quote(rest0)} opens or closes a document, and a file here holds one`, + ) + } + if (indent % 2 !== 0) { + throw refuse( + `line ${no}: indented ${indent}, and indentation here goes two spaces at a time`, + ) + } + + if (rest0 !== '-' && !rest0.startsWith('- ')) { + out.push({ indent, dash: false, text: rest0, no }) + continue + } + let rest = rest0.slice(1) + if (rest.startsWith(' ')) { + throw refuse( + `line ${no}: a \`- \` takes exactly one space, so that what follows it lines up with ` + + 'the lines under it', + ) + } + rest = rest.replace(/^ +/, '') + if (rest.startsWith('- ')) { + throw refuse( + `line ${no}: a sequence opening straight into another one, which nothing here needs`, + ) + } + out.push({ indent, dash: true, text: '', no }) + if (rest !== '') out.push({ indent: indent + 2, dash: false, text: rest, no }) + } + return out +} + +/* The trailing whitespace YAML ignores, which is not the set + * `String.trimEnd` takes off: a non-breaking space is whitespace to + * JavaScript and is content here. + */ +function trimRight(text) { + let end = text.length + while (end > 0 && ' \r\v\f'.includes(text[end - 1])) end-- + return text.slice(0, end) +} + +/* Everything from an unquoted ` #` on, dropped. + * + * Three rules keep this from eating content. A `#` starts a comment + * only with whitespace before it, because one inside a word is part of + * the word. A quote opens a quoted run only with whitespace before it, + * because a quote inside a word is part of the word too, which is what + * lets a `doc:` say "it's" without opening a run that never closes. And + * a quote that opens nothing that closes was not a run at all, which is + * what lets a `query:` hold `cast(' 42 ' AS INT64)`. + */ +function stripComment(text) { + for (let i = 0; i < text.length; i++) { + const c = text[i] + const opens = i === 0 || space(text[i - 1]) + if (c === '#' && opens) return text.slice(0, i) + if ((c === '"' || c === "'") && opens) { + const end = closingQuote(text.slice(i + 1), c) + if (end >= 0) i += 1 + end + } + } + return text +} + +/* Whether a character is one of the ones that can stand before a + * comment or a quote. + */ +function space(c) { + return c === ' ' || c === '\t' || c === '\r' || c === '\n' || c === '\v' || c === '\f' +} + +/* The offset of the quote that closes a run whose opening quote has + * already been passed, and -1 when the line ends first. + * + * The two styles hide a quote differently: a double quoted run escapes + * with a backslash, and a single quoted run doubles the quote, which is + * the only escape it has. + */ +function closingQuote(rest, mark) { + for (let i = 0; i < rest.length; i++) { + if (rest[i] === '\\' && mark === '"') { + i++ + continue + } + if (rest[i] === mark) { + if (mark === "'" && i + 1 < rest.length && rest[i + 1] === "'") { + i++ + continue + } + return i + } + } + return -1 +} + +/* The node that starts where the cursor is and is indented `indent`, + * leaving the cursor on the first line that is not part of it. + */ +function parseNode(at, indent) { + const here = at.lines[at.i] + if (here.dash) return parseSeq(at, indent) + // A mapping key is a bare word and a `:`. Anything else at this + // position is a scalar standing on its own, which is what the items + // of a sequence of scalars are. + if (splitKey(here.text)) return parseMap(at, indent) + at.i++ + return parseScalar(here.text, here.no) +} + +function parseSeq(at, indent) { + const start = at.lines[at.i].no + const items = [] + for (;;) { + const here = at.lines[at.i] + if (!here || !here.dash || here.indent !== indent) break + const opened = here.no + at.i++ + const next = at.lines[at.i] + if (next && next.indent === indent + 2) { + items.push(parseNode(at, indent + 2)) + } else if (next && next.indent > indent) { + throw refuse( + `line ${next.no}: indented ${next.indent}, where an item of the sequence on line ` + + `${opened} is indented ${indent + 2}`, + ) + } else { + throw refuse(`line ${opened}: a \`-\` with nothing after it`) + } + } + return new Node(SEQ, start, { items }) +} + +function parseMap(at, indent) { + const start = at.lines[at.i].no + const pairs = [] + for (;;) { + const here = at.lines[at.i] + if (!here || here.dash || here.indent !== indent) break + const split = splitKey(here.text) + if (!split) break + const { key, rest } = split + const opened = here.no + at.i++ + + let value + const next = at.lines[at.i] + if (rest !== '') { + value = parseScalar(rest, opened) + } else if (next && next.indent === indent + 2) { + value = parseNode(at, indent + 2) + } else if (next && next.indent > indent) { + throw refuse( + `line ${next.no}: indented ${next.indent}, where what is under \`${key}:\` on line ` + + `${opened} is indented ${indent + 2}`, + ) + } else { + value = new Node(EMPTY, opened) + } + if (pairs.some((p) => p.key === key)) { + throw refuse(`line ${opened}: ${key} is set twice in one mapping`) + } + pairs.push({ key, value }) + } + return new Node(MAP, start, { pairs }) +} + +/* The key and the rest of the line, when the line opens a mapping + * entry, and null when it does not. + * + * A key is a bare word, and the `:` after it ends the line or has a + * space after it, so that a plain scalar holding a colon is still a + * scalar. + */ +function splitKey(text) { + let key + let rest + const cut = text.indexOf(': ') + if (cut >= 0) { + key = text.slice(0, cut) + rest = text.slice(cut + 2).replace(/^ +/, '') + } else if (text.endsWith(':')) { + key = text.slice(0, -1) + rest = '' + } else { + return null + } + if (key === '') return null + for (const c of key) { + const bare = + c === '_' || c === '-' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') + if (!bare) return null + } + return { key, rest } +} + +function parseScalar(text, at) { + for (const mark of ['"', "'"]) { + if (text.length === 0 || text[0] !== mark) continue + const body = text.slice(1) + // The closing quote is found by scanning rather than by taking the + // last one on the line, so that `"a" and "b"` is refused instead of + // read as one scalar with quotes in the middle. + const end = closingQuote(body, mark) + if (end < 0) { + throw refuse(`line ${at}: a ${mark} that opens and does not close on its line`) + } + if (end + 1 !== body.length) { + throw refuse(`line ${at}: ${quote(body.slice(end + 1))} after the scalar ends`) + } + const inner = body.slice(0, end) + if (mark === "'") { + // A single quoted run has one escape, the doubled quote, and a + // backslash in it is a backslash. + return new Node(SCALAR, at, { text: inner.replaceAll("''", "'"), quoted: true }) + } + return new Node(SCALAR, at, { text: unescape(inner, at), quoted: true }) + } + if (text.length > 0 && '[]{}&*!|>%@`'.includes(text[0])) { + throw refuse( + `line ${at}: a plain scalar opening with '${text[0]}', which is a construct this reader ` + + 'does not read', + ) + } + return new Node(SCALAR, at, { text }) +} + +/* The escapes the corpus uses, which is a subset of YAML's. + * + * The ones that name a code point by its digits are not here, because + * the corpus writes those as the character itself and a case that wants + * the digits is testing the engine's own escapes inside a query rather + * than the file's. + */ +const escapes = { + '"': '"', + '\\': '\\', + n: '\n', + r: '\r', + t: '\t', + 0: '\0', + b: '\b', + f: '\f', +} + +function unescape(body, at) { + let out = '' + for (let i = 0; i < body.length; i++) { + if (body[i] !== '\\') { + out += body[i] + continue + } + if (i + 1 >= body.length) throw refuse(`line ${at}: a scalar ending in a backslash`) + const next = body[i + 1] + if (!Object.hasOwn(escapes, next)) { + throw refuse(`line ${at}: \\${next} is not an escape`) + } + out += escapes[next] + i++ + } + return out +} diff --git a/conformance/runner.mjs b/conformance/runner.mjs new file mode 100644 index 0000000..54b1648 --- /dev/null +++ b/conformance/runner.mjs @@ -0,0 +1,442 @@ +/* Running the corpus through this client, and saying what happened in + * the form the other runners are compared against. + * + * Each case gets a database of its own. Cases in a suite are written as + * if nothing came before them, and the cheapest way to keep that true is + * to make it true: a case that leaked a table into the next one would be + * a failure that moves when the file is reordered, which is the worst + * kind to be handed. + * + * An outcome is one of three things and not two. Passed and failed are + * obvious. Unsupported is the third, and it exists because the corpus is + * versioned with the engine and shipped to clients that will not all + * implement the same subset at the same time: a client that cannot yet + * parse a statement should say so, and a report should be able to tell + * that apart from an answer that came back wrong. + * + * What this prints is what the Rust runner prints, line for line, so + * that a disagreement between two clients is a diff and not a reading + * exercise. + */ + +import { rm } from 'node:fs/promises' +import { join } from 'node:path' + +import { connect, load } from 'zudb' + +import { exported, schemaSays } from './arrow.mjs' +import { MAIN } from './cases.mjs' +import { CorpusError, quote, refuse } from './reader.mjs' +import { cell, isInteger, same, show } from './values.mjs' + +/* A result this client's own value mapping has nowhere to put. + * + * Not a defect and not a failure. A case that asks for something the + * mapping cannot express is a case this client is honestly unable to + * answer, and saying so by name is worth more than either passing it on + * a value made up for the occasion or failing it as though the engine + * had got something wrong. The corpus writes those cases on purpose, so + * a client that hits one is being told what its mapping costs. + */ +export class Beyond extends Error { + constructor(message) { + super(message) + this.name = 'Beyond' + } +} + +/* The three outcomes. */ +export const PASSED = 'passed' +export const FAILED = 'failed' +export const UNSUPPORTED = 'unsupported' + +/* How a report spells an outcome, which is the reference runner's + * spelling and not a word of it different. + */ +export function mark(outcome) { + if (outcome === PASSED) return 'ok' + if (outcome === FAILED) return 'FAILED' + return 'unsupported' +} + +/* The line a report prints for one case. */ +export function line(ran) { + const head = `${ran.suite}/${ran.case} line ${ran.line} ${mark(ran.outcome)}` + return ran.detail === '' ? head : `${head}: ${ran.detail}` +} + +/* How many cases came to one outcome. */ +export function count(ran, outcome) { + return ran.filter((one) => one.outcome === outcome).length +} + +/* One line saying what the run came to, which is what a CI log keeps and + * what two runs are compared by. + */ +export function summary(ran) { + return ( + `${ran.length} cases, ${count(ran, PASSED)} passed, ${count(ran, FAILED)} failed, ` + + `${count(ran, UNSUPPORTED)} unsupported` + ) +} + +/* Runs every case of every suite, in the order they were written. + * + * `directory` is one the runner may make databases under. Each case gets + * its own file in it, named after the case, so that a failure leaves + * something to open. + * + * `onRan` is called with each case as it finishes, which is what makes a + * run of fourteen hundred cases print as it goes rather than at the end. + */ +export async function run(suites, directory, onRan) { + const ran = [] + for (const suite of suites) { + for (const one of suite.cases) { + const outcome = await runCase(suite, one, directory) + // A failure leaves its database behind, which is the one thing + // somebody reading the report will want to open. Everything else + // goes as it finishes, because a corpus of fourteen hundred cases + // is fourteen hundred files and holding them all until the run + // ends is gigabytes of a disk that has other work to do. The Rust + // and Go runners do the same. + if (outcome.outcome !== FAILED) { + const path = casePath(directory, suite.name, one.name) + await rm(path, { force: true }) + // The WAL sidecar goes with it. A database is and its log + // is .wal, and a log left beside a name the next run creates + // again is a log that run would adopt. + await rm(`${path}.wal`, { force: true }) + } + ran.push(outcome) + if (onRan) onRan(outcome) + } + } + return ran +} + +function casePath(directory, suite, name) { + return join(directory, `${suite}-${name}.zu`) +} + +async function runCase(suite, one, directory) { + const ran = (outcome, detail) => ({ + suite: suite.name, + case: one.name, + line: one.line, + outcome, + detail, + }) + const path = casePath(directory, suite.name, one.name) + + // The load goes in before the connection opens, because it is bulk + // load and bulk load is the path that builds the file rather than one + // that goes through a statement. Every case of the suite gets its own + // copy of it for the same reason every case gets its own database. + // + // A load makes the file, so the two halves of this are the two ways a + // database comes into being in this client and a case has exactly one + // of them. + if (suite.load !== null) { + try { + await applyLoad(suite.load, path) + } catch (err) { + return ran(FAILED, `the suite's load: ${errorText(err)}`) + } + } + + let main + try { + main = await connect(path) + } catch (err) { + return ran(FAILED, `opening ${path}: ${errorText(err)}`) + } + const open = [{ name: MAIN, conn: main }] + try { + return await withConnections(suite, one, open, ran) + } finally { + // In reverse, so that the connection the case was opened with is the + // last one to go, which is the order the ones after it were made + // from it in. + for (let i = open.length - 1; i >= 0; i--) open[i].conn.close() + } +} + +async function withConnections(suite, one, open, ran) { + for (const [i, step] of one.setup.entries()) { + let on + try { + on = await connection(open, step.on) + } catch (err) { + return ran(FAILED, `connecting as ${quote(step.on)}: ${errorText(err)}`) + } + try { + await on.exec(step.query) + } catch (err) { + // A setup that fails is not a result about the statement under + // test, so it is never a pass and never a quiet skip. + if (unsupported(err)) return ran(UNSUPPORTED, `setup ${i + 1}: ${errorText(err)}`) + return ran(FAILED, `setup ${i + 1} failed: ${errorText(err)}`) + } + } + + let on + try { + on = await connection(open, one.on) + } catch (err) { + return ran(FAILED, `connecting as ${quote(one.on)}: ${errorText(err)}`) + } + + const params = bound(one) + let rows + try { + rows = await on.query(one.query, params) + } catch (err) { + if (one.raises !== '') { + const code = statusCode(err) + if (code === '') { + return ran( + FAILED, + `failed with no GQLSTATUS where the case wants ${one.raises}: ${errorText(err)}`, + ) + } + if (code === one.raises) return ran(PASSED, '') + return ran(FAILED, `raised ${code} where the case wants ${one.raises}: ${errorText(err)}`) + } + if (unsupported(err)) return ran(UNSUPPORTED, errorText(err)) + return ran(FAILED, errorText(err)) + } + + if (one.raises !== '') return ran(FAILED, `returned rows where the case wants ${one.raises}`) + + let got + try { + got = readAll(rows) + } catch (err) { + if (err instanceof Beyond) return ran(UNSUPPORTED, err.message) + return ran(FAILED, errorText(err)) + } + const detail = compare(one.columns, one.rows, rows.columns, got) + if (detail !== '') return ran(FAILED, detail) + const arrow = await exportSays(one.arrow, on, one, params, got.length) + if (arrow !== '') return ran(FAILED, arrow) + return ran(PASSED, '') +} + +/* The connection a case named, made if this is the first mention of it. + * + * A new one is a duplicate of the case's own rather than a second open + * of the file, which is what a pool does: the two share the write side, + * so each sees what the other has committed. Opening the path twice + * would be two databases that happen to be the same file, which is a + * different thing and not what a case about a transaction means. + */ +async function connection(open, name) { + for (const had of open) { + if (had.name === name) return had.conn + } + const made = await open[0].conn.duplicate() + open.push({ name, conn: made }) + return made +} + +/* The parameters of a case, in the shape this client's own call takes, + * which is an object of name to value. + */ +function bound(one) { + if (one.params.length === 0) return null + const out = {} + for (const param of one.params) out[param.name] = param.value + return out +} + +/* Every row of a result, in the corpus's own shape. + * + * A row comes back as an object keyed by column name, so the positions a + * case is written in come from the result's own `columns`. Two columns + * of one name are one property, so a result holding them is one this + * client cannot read positionally at all, and the case is reported + * unsupported with the reason named rather than compared against a row + * this made up. + */ +function readAll(rows) { + const columns = rows.columns + if (new Set(columns).size !== columns.length) { + throw new Beyond( + 'the result has two columns of one name, and a row comes back keyed by name in this client', + ) + } + // Into the corpus's own shape here rather than at the comparison, + // because the engine's edge carries a field the corpus does not write + // and a path is a pair of lists where a case writes one walk. + return rows.map((row) => columns.map((name) => cell(row[name]))) +} + +/* Puts the suite's load in through this client's own bulk load path, + * which is the strongest form of the corpus question: the value crosses + * the boundary twice and by two different mechanisms. + */ +async function applyLoad(loadable, path) { + const columns = {} + for (const column of loadable.columns) columns[column.name] = loadColumn(column) + const options = { + nodes: loadable.nodes, + rels: loadable.edges, + rows: loadable.count, + columns, + } + if (loadable.pairs.length > 0) { + // Flat, which is the spelling that costs nothing: two elements an + // edge, read where they lie. + const edges = new Uint32Array(loadable.pairs.length * 2) + for (const [i, pair] of loadable.pairs.entries()) { + edges[i * 2] = pair[0] + edges[i * 2 + 1] = pair[1] + } + options.edges = edges + } + await load(path, options) +} + +/* One column of a load, in the shape the load call takes. + * + * The type is pinned by what the column declared rather than left to the + * values, because a plain array is read by its first value and a FLOAT64 + * column whose first value is a whole 1.0 is the number 1, which would + * settle the column as INT64 and put every value in it somewhere it does + * not belong. A typed array says the width outright, so the two types + * that have one get one. + */ +function loadColumn(column) { + if (isInteger(column.type)) return BigInt64Array.from(column.values) + if (column.type === 'FLOAT32' || column.type === 'FLOAT64') { + return Float64Array.from(column.values) + } + switch (column.type) { + case 'STRING': + case 'BOOL': + case 'DATE': + case 'LOCALTIME': + case 'ZONEDTIME': + case 'LOCALDATETIME': + case 'ZONEDDATETIME': + case 'DURATION': + // A class or a runtime value, which a column of this client holds + // as itself. The first value settles nothing here because every + // one of these is exactly one type. + return column.values + } + throw refuse(`a load column of ${column.type}, which this client has no column type for`) +} + +/* What the export gave that the case did not want, or the empty string + * when the case says nothing about it and when the two agree. + * + * A result Arrow has no type for is a refusal from the export rather + * than a condition from the statement, so a case saying `refused` is the + * case where the export failing is the right answer. + * + * This runs the statement a second time, where the Go and Python runners + * export the result they already read. Those clients are handed a C Data + * Interface stream off a result; this one is handed a buffer of Arrow + * IPC bytes by a call of its own, and there is no way to ask an already + * read result for them. Every case that asserts an export is a read, so + * running it twice says the same thing twice. + */ +async function exportSays(want, on, one, params, rows) { + if (want === null) return '' + let arrow + try { + arrow = await on.arrow(one.query, params) + } catch (err) { + if (want.refused) return '' + return `arrow refused the result: ${errorText(err)}` + } + if (want.refused) return 'arrow exported the result where the case wants a refusal' + let got + try { + got = exported(arrow) + } catch (err) { + return `arrow gave a stream this runner could not read: ${errorText(err)}` + } + const detail = schemaSays(got.fields, want.fields) + if (detail !== '') return detail + if (got.rows !== rows) { + return `arrow gives ${got.rows} rows where the case wants ${rows}` + } + if (got.declared !== got.rows) { + return `arrow says it holds ${got.declared} rows and its batches hold ${got.rows}` + } + return '' +} + +/* Whether a condition means the engine does not implement the statement + * rather than that the statement is wrong. + * + * The two GQL classes that say so are 42, syntax error or access rule + * violation, and 0A, feature not supported. A case landing on either is + * a case ahead of the engine, which the corpus allows on purpose: the + * cases are the contract and the engine catches up to them. + */ +function unsupported(err) { + const code = statusCode(err) + return code.startsWith('42') || code.startsWith('0A') +} + +/* The GQLSTATUS a failure carries, or the empty string for one that + * carries none. + */ +function statusCode(err) { + return typeof err?.code === 'string' ? err.code : '' +} + +/* What the engine said, which is what the Rust runner prints for the + * same failure. + * + * The message a condition carries already opens with its own code, which + * is why nothing is added here. + */ +function errorText(err) { + if (err instanceof CorpusError) return err.message + if (err instanceof Error) return err.message + return String(err) +} + +/* What differs between what a case wants and what came back, or the + * empty string if nothing does. + * + * It reports the first difference rather than all of them, because the + * first is nearly always the cause of the rest, and a report that prints + * a hundred rows is one nobody reads to the end. The order the checks + * run in is the reference runner's, so that two runners looking at the + * same wrong answer say the same thing about it. + */ +function compare(wantColumns, wantRows, gotColumns, gotRows) { + if (!sameNames(wantColumns, gotColumns)) { + return `columns ${names(gotColumns)} where the case wants ${names(wantColumns)}` + } + for (let i = 0; i < wantRows.length && i < gotRows.length; i++) { + const want = wantRows[i] + const got = gotRows[i] + for (let j = 0; j < want.length && j < got.length; j++) { + if (same(want[j], got[j])) continue + const name = j < wantColumns.length ? wantColumns[j] : '?' + return ( + `row ${i + 1} column ${name} is ${show(got[j])} where the case wants ${show(want[j])}` + ) + } + } + if (wantRows.length !== gotRows.length) { + return `${gotRows.length} rows where the case wants ${wantRows.length}` + } + return '' +} + +function sameNames(want, got) { + return want.length === got.length && want.every((name, i) => name === got[i]) +} + +/* A list of column names the way Rust's {:?} writes one. */ +function names(columns) { + return `[${columns.map((name) => `"${name}"`).join(', ')}]` +} diff --git a/conformance/temporal.mjs b/conformance/temporal.mjs new file mode 100644 index 0000000..83e1884 --- /dev/null +++ b/conformance/temporal.mjs @@ -0,0 +1,430 @@ +/* The temporal half of the encoding, written out rather than handed to + * `Date.parse` or to a library. + * + * A corpus reader is a second opinion about the text, and a second + * opinion that calls the same code the client calls is not one. + * `Date.parse` also takes a great deal this encoding does not, and is + * allowed by its own specification to take anything else it likes: it + * reads `2024-1-1`, it reads `Jan 1 2024`, and what it does with a + * date-only string is a rule that has changed twice. Writing the four + * spellings out is thirty lines and it says exactly what is accepted. + * + * The spellings are the ones the engine prints, which is the extended + * ISO 8601 form and nothing else: 2024-01-01 for a date, 12:34:56 with + * an optional fraction of one to nine digits for a time, the two joined + * with a T for a datetime, and Z or +07:00 for an offset. A basic-form + * 20240101 is refused, because a case that writes one is a case the + * other runners would read differently or not at all. + * + * A count of nanoseconds is a `bigint` here for the reason it is one + * everywhere else in this client: nanoseconds from the epoch pass 2^53 + * in 1970, so a `number` holding one is a value that is quietly not the + * value that went in. A count of days is a `number`, which is what + * `ZuDate` holds, and a day count stays exact for eight million years + * either side of the epoch. + * + * The failure spelling is `undefined` rather than an exception, because + * every caller of these is asking a question about text rather than + * running a step that can fail, and the one refusal message belongs at + * the place that knows which type was declared. + */ + +import { ZuDate, ZuDuration, ZuTime, ZuTimestamp } from 'zudb' + +export const NANOS_PER_SECOND = 1000000000n +export const NANOS_PER_MINUTE = 60n * NANOS_PER_SECOND +export const NANOS_PER_HOUR = 60n * NANOS_PER_MINUTE +export const NANOS_PER_DAY = 24n * NANOS_PER_HOUR + +const MILLIS_PER_DAY = 86400000 + +/* The widest count of nanoseconds or months the engine holds, which is + * a signed 64 bit integer. A duration past it is refused here rather + * than handed to a constructor that would throw, since what a case + * wrote is either a value or it is not and that is the same answer + * either way. + */ +const MIN_I64 = -9223372036854775808n +const MAX_I64 = 9223372036854775807n + +/* A date, as the count of days from 1970-01-01 that the client holds one + * as. + */ +export function parseDate(text) { + const days = dateDays(text) + return days === undefined ? undefined : new ZuDate(days) +} + +/* A time of day with no offset on the end. */ +export function parseLocalTime(text) { + const nanos = clockNanos(text) + return nanos === undefined ? undefined : new ZuTime(nanos, null) +} + +/* A time of day with an offset, which it carries as written: the count + * of nanoseconds is midnight in the offset's own day rather than + * midnight UTC, which is what this client's `ZuTime` holds and what + * makes 12:00:00+07:00 and 05:00:00Z two values rather than one. + */ +export function parseZonedTime(text) { + const zone = splitOffset(text) + if (zone === undefined) return undefined + const nanos = clockNanos(zone.rest) + return nanos === undefined ? undefined : new ZuTime(nanos, zone.offset) +} + +/* A date and a time with no offset, as the count of nanoseconds from + * 1970-01-01T00:00:00 read with no zone at all. + */ +export function parseLocalDateTime(text) { + const nanos = stampNanos(text) + return nanos === undefined ? undefined : new ZuTimestamp(nanos, null) +} + +/* An instant and the offset it was written with. + * + * The client holds the instant in UTC and the offset beside it, so the + * wall clock that was written is moved back by the offset to get there. + * Two texts an hour apart in zones an hour apart are the same instant + * and hold the same count, which is the point of keeping it that way. + */ +export function parseZonedDateTime(text) { + const zone = splitOffset(text) + if (zone === undefined) return undefined + const nanos = stampNanos(zone.rest) + if (nanos === undefined) return undefined + return new ZuTimestamp(nanos - BigInt(zone.offset) * NANOS_PER_MINUTE, zone.offset) +} + +/* YYYY-MM-DD as a count of days from the epoch. + * + * The date is built and read back rather than checked field by field, + * because that is the calendar answering the question about February + * rather than this file having an opinion about it. Built through + * `setUTCFullYear` rather than through `Date.UTC`, which reads a year + * under a hundred as that year plus nineteen hundred and would turn + * 0024-01-01 into a date in the twentieth century. + */ +export function dateDays(text) { + if (text.length !== 10 || text[4] !== '-' || text[7] !== '-') return undefined + const year = number(text.slice(0, 4)) + const month = number(text.slice(5, 7)) + const day = number(text.slice(8, 10)) + if (year === undefined || month === undefined || day === undefined) return undefined + const when = new Date(0) + when.setUTCFullYear(Number(year), Number(month) - 1, Number(day)) + if (Number.isNaN(when.getTime())) return undefined + // A date the calendar does not have comes back as the one it rolled + // over into, so 2023-02-30 reads back as March and is refused here. + if ( + BigInt(when.getUTCFullYear()) !== year || + BigInt(when.getUTCMonth() + 1) !== month || + BigInt(when.getUTCDate()) !== day + ) { + return undefined + } + return when.getTime() / MILLIS_PER_DAY +} + +/* HH:MM:SS, with a fraction of one to nine digits when there is one, as + * nanoseconds since midnight. + */ +export function clockNanos(text) { + const dot = text.indexOf('.') + const head = dot < 0 ? text : text.slice(0, dot) + const frac = dot < 0 ? null : text.slice(dot + 1) + if (head.length !== 8 || head[2] !== ':' || head[5] !== ':') return undefined + const hours = number(head.slice(0, 2)) + const minutes = number(head.slice(3, 5)) + const seconds = number(head.slice(6, 8)) + if (hours === undefined || minutes === undefined || seconds === undefined) return undefined + // No leap second, because the engine has no value for one: a time is + // nanoseconds since midnight and 23:59:60 is a second the count does + // not have. + if (hours > 23n || minutes > 59n || seconds > 59n) return undefined + const nanos = hours * NANOS_PER_HOUR + minutes * NANOS_PER_MINUTE + seconds * NANOS_PER_SECOND + if (frac === null) return nanos + // A point with nothing after it is not a fraction, and ten digits is + // finer than the engine counts, so neither is read as the number it + // resembles. + if (frac === '' || frac.length > 9) return undefined + const part = number(frac) + if (part === undefined) return undefined + return nanos + part * 10n ** BigInt(9 - frac.length) +} + +/* A date and a time joined with a T, as nanoseconds from + * 1970-01-01T00:00:00. + */ +export function stampNanos(text) { + const cut = text.indexOf('T') + if (cut < 0) return undefined + const days = dateDays(text.slice(0, cut)) + if (days === undefined) return undefined + const nanos = clockNanos(text.slice(cut + 1)) + if (nanos === undefined) return undefined + return BigInt(days) * NANOS_PER_DAY + nanos +} + +/* Takes the offset off the end of a zoned value and gives back what came + * before it, with the offset in minutes east of UTC. + * + * Zero is written Z rather than +00:00, which is what the engine prints, + * and both are read here because a case may assert either. Which one it + * was is not kept, since it is not part of the value: the engine holds + * an offset in minutes and prints zero as Z whichever way it went in. + */ +export function splitOffset(text) { + if (text.endsWith('Z')) return { rest: text.slice(0, -1), offset: 0 } + if (text.length < 7) return undefined + const zone = text.slice(-6) + const mark = zone[0] + if (mark !== '+' && mark !== '-') return undefined + if (zone[3] !== ':') return undefined + const hours = number(zone.slice(1, 3)) + const minutes = number(zone.slice(4, 6)) + if (hours === undefined || minutes === undefined || minutes > 59n) return undefined + const total = Number(hours) * 60 + Number(minutes) + // The standard's own limit, which is wider than any zone in use and is + // here so that a typo lands as a refusal rather than as a date a day + // away from the one that was meant. + if (total > 18 * 60) return undefined + return { rest: text.slice(0, -6), offset: mark === '-' ? -total : total } +} + +/* A run of ASCII digits as the number it spells, and `undefined` for + * anything else. + * + * Not `BigInt`, which takes a sign, a `0x` and whitespace around the + * whole of it, none of which belongs inside a temporal field. + */ +function number(text) { + if (text === '') return undefined + for (const c of text) if (c < '0' || c > '9') return undefined + return BigInt(text) +} + +/* An ISO 8601 duration, in the two kinds the engine keeps apart. + * + * A duration is months or it is nanoseconds and never both, because a + * month is not a number of days: adding one to a date is a different + * operation from adding thirty of them, and a type that held both would + * have to say which happens first. The client has a kind for each, so + * which one this is is part of what the case asserts. + * + * The text says which. A duration whose fields are years and months is + * the month kind and everything else is the nanosecond kind, and a + * duration with a field of each is refused rather than guessed at. That + * leaves one text the fields decide and the numbers cannot, which is a + * duration of nothing: P0M is no months and PT0S is no nanoseconds, and + * they are two values here where the Python runner has to call them one. + */ +export function parseDuration(text) { + let rest = text + const negative = rest.startsWith('-') + if (negative || rest.startsWith('+')) rest = rest.slice(1) + if (!rest.startsWith('P')) return undefined + rest = rest.slice(1) + const cut = rest.indexOf('T') + const dated = cut >= 0 + const day = dated ? rest.slice(0, cut) : rest + const clock = dated ? rest.slice(cut + 1) : '' + // A P with nothing under it is not a duration, and neither is a T with + // nothing after it. + if (day === '' && clock === '') return undefined + if (dated && clock === '') return undefined + + let months = 0n + let nanos = 0n + let sawMonths = !dated + for (const field of pieces(day)) { + if (!field.ok) return undefined + switch (field.unit) { + case 'Y': + months += field.whole * 12n + break + case 'M': + months += field.whole + break + case 'W': + nanos += field.whole * 7n * NANOS_PER_DAY + sawMonths = false + break + case 'D': + nanos += field.whole * NANOS_PER_DAY + sawMonths = false + break + default: + return undefined + } + // A fraction of a year, a month, a week or a day is a length that + // depends on which one it lands on, so it is refused here rather + // than turned into a number of nanoseconds that is right for some of + // them. + if (field.frac !== 0n) return undefined + } + for (const field of pieces(clock)) { + if (!field.ok) return undefined + switch (field.unit) { + case 'H': + nanos += field.whole * NANOS_PER_HOUR + break + case 'M': + nanos += field.whole * NANOS_PER_MINUTE + break + case 'S': + nanos += field.whole * NANOS_PER_SECOND + field.frac + break + default: + return undefined + } + if (field.frac !== 0n && field.unit !== 'S') return undefined + } + if (months !== 0n && nanos !== 0n) return undefined + if (negative) { + months = -months + nanos = -nanos + } + if (months < MIN_I64 || months > MAX_I64 || nanos < MIN_I64 || nanos > MAX_I64) return undefined + if (months !== 0n || (nanos === 0n && sawMonths)) return ZuDuration.ofMonths(months) + return ZuDuration.ofNanos(nanos) +} + +/* Half a duration split into its fields, each one number and the letter + * after it. + * + * A half that does not split gives back one field that is not ok, so + * that the caller refuses the text at the same place it refuses a unit + * it does not know. + */ +function pieces(text) { + const out = [] + let start = 0 + for (let i = 0; i < text.length; i++) { + const c = text[i] + if ((c >= '0' && c <= '9') || c === '.') continue + const one = { whole: 0n, frac: 0n, unit: c, ok: true } + const run = text.slice(start, i) + const dot = run.indexOf('.') + const head = dot < 0 ? run : run.slice(0, dot) + const frac = dot < 0 ? null : run.slice(dot + 1) + const whole = number(head) + if (whole === undefined) one.ok = false + else one.whole = whole + if (frac !== null) { + const part = frac === '' || frac.length > 9 ? undefined : number(frac) + if (part === undefined) one.ok = false + else one.frac = part * 10n ** BigInt(9 - frac.length) + } + out.push(one) + start = i + 1 + } + // Digits with no unit after them, which is the one thing left over + // that a caller has to hear about. + if (start !== text.length) out.push({ whole: 0n, frac: 0n, unit: '', ok: false }) + return out +} + +/* A date the way the engine prints one, from the count of days this + * client holds it as. + */ +export function showDate(days) { + const when = new Date(days * MILLIS_PER_DAY) + return ( + pad(when.getUTCFullYear(), 4) + + '-' + + pad(when.getUTCMonth() + 1, 2) + + '-' + + pad(when.getUTCDate(), 2) + ) +} + +/* A count of nanoseconds since midnight the way the engine prints one, + * which is seconds always and a fraction of nine digits when there is + * one. + * + * Nine and not the shortest that reads back: the report this goes into + * is diffed against the one the reference runner writes, and that one + * writes nine. + */ +export function showClock(nanos) { + const hours = nanos / NANOS_PER_HOUR + const minutes = (nanos % NANOS_PER_HOUR) / NANOS_PER_MINUTE + const seconds = (nanos % NANOS_PER_MINUTE) / NANOS_PER_SECOND + const frac = nanos % NANOS_PER_SECOND + const out = `${pad(hours, 2)}:${pad(minutes, 2)}:${pad(seconds, 2)}` + return frac === 0n ? out : `${out}.${pad(frac, 9)}` +} + +/* A count of nanoseconds from the epoch as a date and a time joined with + * a T. + */ +export function showStamp(nanos) { + const days = floorDiv(nanos, NANOS_PER_DAY) + return `${showDate(Number(days))}T${showClock(nanos - days * NANOS_PER_DAY)}` +} + +/* An offset in minutes east of UTC, which is Z at zero rather than + * +00:00. + */ +export function showOffset(offset) { + if (offset === 0) return 'Z' + const sign = offset < 0 ? '-' : '+' + const size = Math.abs(offset) + return `${sign}${pad(Math.floor(size / 60), 2)}:${pad(size % 60, 2)}` +} + +/* A month duration as the text that parses back to it: a field that is + * zero is left out, and a duration with nothing left in it is P0M, + * because P on its own is not a value. + */ +export function showMonths(count) { + const sign = count < 0n ? '-' : '' + const size = count < 0n ? -count : count + const years = size / 12n + const months = size % 12n + let out = `${sign}P` + if (years !== 0n) out += `${years}Y` + if (months !== 0n || years === 0n) out += `${months}M` + return out +} + +/* A nanosecond duration as the text that parses back to it, under the + * same rule, with PT0S for the one that is empty. + */ +export function showNanos(count) { + const sign = count < 0n ? '-' : '' + const size = count < 0n ? -count : count + const days = size / NANOS_PER_DAY + const rest = size % NANOS_PER_DAY + let out = `${sign}P` + if (days !== 0n) out += `${days}D` + if (rest === 0n && days !== 0n) return out + out += 'T' + const hours = rest / NANOS_PER_HOUR + const minutes = (rest % NANOS_PER_HOUR) / NANOS_PER_MINUTE + const seconds = (rest % NANOS_PER_MINUTE) / NANOS_PER_SECOND + const frac = rest % NANOS_PER_SECOND + if (hours !== 0n) out += `${hours}H` + if (minutes !== 0n) out += `${minutes}M` + if (seconds !== 0n || frac !== 0n || (hours === 0n && minutes === 0n)) { + out += `${seconds}` + if (frac !== 0n) out += `.${pad(frac, 9)}` + out += 'S' + } + return out +} + +/* A number in at least width digits, zeroes in front of it. */ +function pad(n, width) { + return `${n}`.padStart(width, '0') +} + +/* Division rounding towards minus infinity rather than towards zero, + * which is what turns an instant before the epoch into the day it is on + * rather than the day after it. + */ +function floorDiv(a, b) { + const q = a / b + return a % b !== 0n && a < 0n !== b < 0n ? q - 1n : q +} diff --git a/conformance/values.mjs b/conformance/values.mjs new file mode 100644 index 0000000..d3221e6 --- /dev/null +++ b/conformance/values.mjs @@ -0,0 +1,718 @@ +/* The {type, value} encoding a case writes its values in. + * + * Every value in the corpus is a mapping with a `type` naming the GQL + * type and a `value` holding the payload. The type is written down + * rather than inferred because the corpus is read by nine languages and + * inference is where they differ: a bare 1 is an integer in YAML, and + * which integer it becomes is a decision each host language makes on + * its own. + * + * The payload is a YAML scalar where a YAML scalar is exact, and a + * string where it is not. An integer wider than 53 bits is a string, + * because most YAML readers hand a number to a double, and JavaScript + * is the reader that makes the case for the rule: `1` in a `.yaml` file + * read by any library on npm is a `number`, and a `number` stops naming + * one integer at 2^53. A float is a string, for that reason and for + * NaN, inf and -0.0. A temporal value is a string, because YAML has no + * type that keeps an offset. + * + * NODE, EDGE and PATH are the values a graph has and a table does not. + * A node is `person#1`, the table it is a row of and which row of it. An + * edge is `knows#0->1`, its table and the two rows it runs between. A + * path is a sequence, like a list, holding a node and then an edge and a + * node for each hop. + * + * Refusing the wrong form is half the point, and refusing it here is + * what makes this a second reader of the corpus rather than a consumer + * of it. + * + * What a decoded value becomes is what this client hands back for the + * same value, because that is what a comparison has to be against. So an + * integer is a `bigint` whatever width the case declared: the declared + * width is dropped in the process, which is a fact about this engine + * rather than about the corpus, since its own value is one signed 64 bit + * integer either way. + * + * One thing this client does not need and the Python one does. There, a + * temporal written finer than a microsecond is a value the host language + * cannot hold, and a case carrying one is reported unsupported rather + * than run. Here a time is a `bigint` count of nanoseconds, which is the + * same resolution the engine keeps, so every temporal in the corpus is a + * value this client holds exactly. + */ + +import { ZuDate, ZuDuration, ZuNode, ZuRel, ZuTime, ZuTimestamp } from 'zudb' + +import { quote, refuse } from './reader.mjs' +import { + parseDate, + parseDuration, + parseLocalDateTime, + parseLocalTime, + parseZonedDateTime, + parseZonedTime, + showClock, + showDate, + showMonths, + showNanos, + showOffset, + showStamp, + NANOS_PER_MINUTE, +} from './temporal.mjs' + +/* A node as a case names it: the node table it is a row of and which + * row. + * + * A shape of the corpus's own rather than `ZuNode`, for the reason + * `EdgeAt` is not `ZuRel` and for one that is smaller here than it is in + * the other runners: a `ZuNode` holds its fields behind getters on the + * prototype, so two of them are never `===` and comparing a pair means + * reading them out anyway. + */ +export class NodeAt { + constructor(table, offset) { + /* The node table's name. */ + this.table = table + /* The row's number within that table, counted from zero in the + * order the load wrote it. + */ + this.offset = offset + } +} + +/* An edge as a case names it: the rel table it is in and the two rows it + * runs between. + * + * Not `ZuRel`, which carries a fourth field the corpus does not write. + * That field is `ord`, where the edge's properties sit, which is its + * place in the order the table was loaded in, and that is a number the + * loader chose rather than one the case did. A pair may run more than + * once, and a case that has to tell two parallel edges apart asserts a + * property of them instead. + */ +export class EdgeAt { + constructor(table, src, dst) { + /* The rel table's name. */ + this.table = table + /* The row numbers the edge runs from and to, within the node table + * the load names. + */ + this.src = src + this.dst = dst + } +} + +/* A path as a case writes it: nodes and edges alternating, a node at + * each end. + * + * Not the `{ nodes, rels }` this client hands back, for the reason + * `EdgeAt` is not `ZuRel`, and for one more: what a case compares is the + * walk, and two walks that cross the same edges are the same walk + * whichever copy of a parallel edge the engine happened to hand back. + */ +export class Walk { + constructor(elements) { + /* The walk in order, a `NodeAt` at each end and an `EdgeAt` between + * every two of them. + */ + this.elements = elements + } +} + +/* Whether a type's payload is written as a quoted string. False is a + * type a YAML scalar carries without loss, true is one it does not. + */ +const QUOTED = new Map([ + ['NULL', false], + ['BOOL', false], + ['INT8', false], + ['INT16', false], + ['INT32', false], + ['INT64', true], + ['UINT8', false], + ['UINT16', false], + ['UINT32', false], + ['UINT64', true], + ['FLOAT32', true], + ['FLOAT64', true], + ['STRING', false], + // A byte string is written in quotes because its hexits are digits as + // often as not: a bare 0041 is a number with a leading zero in one + // reader and the string it looks like in another, and neither of them + // is the two octets the case meant. + ['BYTES', true], + ['DATE', true], + ['LOCALTIME', true], + ['ZONEDTIME', true], + ['LOCALDATETIME', true], + ['ZONEDDATETIME', true], + ['DURATION', true], + ['LIST', false], + // A node and an edge are written in quotes because what a case spells + // is a name and two numbers with punctuation between them, which is + // text in every reader and a number in none. + ['NODE', true], + ['EDGE', true], + // A path is a sequence, like a list, because that is what it is: the + // nodes and edges of a walk, in the order they were walked. + ['PATH', false], +]) + +/* The types the encoding reserves a name for and the engine has no + * runtime value for yet, kept apart from an outright typo so that the + * error says which of the two it is. + */ +const RESERVED = ['DECIMAL'] + +/* The range each integer width holds, so that a case writing a value its + * own type cannot carry is refused rather than stored wider than it + * says. UINT64 stops at the signed maximum because the engine's integer + * is signed and 64 bits wide, and wrapping the top half into a negative + * would be a case that passes while meaning the opposite of what it + * says. + */ +const BOUNDS = new Map([ + ['INT8', [-128n, 127n]], + ['INT16', [-32768n, 32767n]], + ['INT32', [-2147483648n, 2147483647n]], + ['INT64', [-9223372036854775808n, 9223372036854775807n]], + ['UINT8', [0n, 255n]], + ['UINT16', [0n, 65535n]], + ['UINT32', [0n, 4294967295n]], + ['UINT64', [0n, 9223372036854775807n]], +]) + +/* Whether a type is written quoted, and whether it is a type at all. */ +export function form(type) { + return { quoted: QUOTED.get(type) === true, known: QUOTED.has(type) } +} + +/* Whether a type is one of the integer widths, which is what a load + * column of them turns on. + */ +export function isInteger(type) { + return BOUNDS.has(type) +} + +function unknownType(type) { + if (RESERVED.includes(type)) { + return `${type} is a type the encoding reserves and the engine has no value for` + } + return `${type} is not a type this encoding knows` +} + +/* The value a {type, value} mapping describes. */ +export function decode(node) { + if (!node.map()) { + throw refuse( + `line ${node.line}: a value is a mapping of \`type\` and \`value\`, and this is ${node.what()}`, + ) + } + const unknown = node.unknown('type', 'value') + if (unknown.length > 0) { + throw refuse(`line ${node.line}: a value has no key ${quote(unknown[0])}`) + } + return typed(node) +} + +/* The type and value of a mapping that carries more than those two, + * which is a parameter: it is a value with a name, and the name belongs + * to the case rather than to the encoding. + */ +export function typed(node) { + const at = node.line + const typeNode = node.get('type') + if (typeNode === null) throw refuse(`line ${at}: a value with no \`type\``) + const type = typeNode.str() + if (type === null) throw refuse(`line ${at}: a \`type\` that is not a name`) + + // Checked here as well as in `payload`, because a value whose type is + // not a type and which also has no `value` under it should be told + // about the type first: that is the mistake, and the missing payload + // is a consequence of it. + if (!form(type).known) throw refuse(`line ${at}: ${unknownType(type)}`) + + if (type === 'NULL') { + if (node.get('value') !== null) throw refuse(`line ${at}: NULL carries no \`value\``) + return null + } + const value = node.get('value') + if (value === null) throw refuse(`line ${at}: a ${type} with no \`value\``) + return payload(type, value) +} + +/* The value a payload spells under a type that has already been read. + * + * A row of a case names its type beside every value. A column of a load + * names it once at the top and every value under it is a bare payload, + * which is the same encoding with the type factored out, so it is the + * same function reading it. + */ +export function payload(type, value) { + const { quoted, known } = form(type) + if (!known) throw refuse(`line ${value.line}: ${unknownType(type)}`) + + if (type === 'LIST' || type === 'PATH') { + // The empty list is a value worth a case and needs a spelling, which + // is a `value:` with nothing under it. + const items = value.seqOrEmpty() + if (items === null) { + throw refuse( + `line ${value.line}: a ${type} holds a sequence of values, and this is ${value.what()}`, + ) + } + const decoded = items.map((item) => decode(item)) + return type === 'LIST' ? decoded : walk(decoded, value.line) + } + + const spelled = value.scalar() + if (spelled === null) { + throw refuse(`line ${value.line}: a ${type} holds one scalar, and this is ${value.what()}`) + } + const { text, quoted: wasQuoted } = spelled + const at = value.line + // The one rule the whole encoding exists for, checked before the text + // is looked at, because a value that parses is exactly the case where + // a silent misread would survive review. + if (quoted && !wasQuoted) { + // A node and an edge are quoted for a different reason from the + // numbers, so they are told a different reason. Both reasons are the + // same rule: a payload is quoted where a bare one would read as + // something else in some reader of this file. + if (type === 'NODE' || type === 'EDGE') { + throw refuse( + `line ${at}: ${type} is written in quotes, because ${text} is a name and two numbers ` + + 'and no reader has a scalar for that', + ) + } + throw refuse( + `line ${at}: ${type} is written in quotes, because a bare ${text} is a number and some ` + + 'reader of this file will round it', + ) + } + if (!quoted && wasQuoted && type !== 'STRING') { + throw refuse( + `line ${at}: ${type} is written without quotes, so that a reader cannot take it for a string`, + ) + } + + let out + if (type === 'NODE') out = nodeAt(text) + else if (type === 'EDGE') out = edgeAt(text) + else out = scalar(type, text) + if (out === undefined) throw refuse(`line ${at}: ${quote(text)} is not a ${type}`) + return out +} + +/* The nodes and edges of a walk, or what is wrong with the sequence + * somebody wrote. + * + * A path alternates and ends at both ends with a node, so a sequence + * that does not is a case that could never pass. Refusing it here rather + * than at the comparison is the difference between a message naming the + * line and a report saying the row differs. + */ +function walk(items, at) { + if (items.length % 2 === 0) { + throw refuse( + `line ${at}: a PATH is a node, then an edge and a node for each hop, so it holds an odd ` + + `number of values and this holds ${items.length}`, + ) + } + for (let i = 0; i < items.length; i++) { + const wantNode = i % 2 === 0 + let ok + let was + if (items[i] instanceof NodeAt) { + ok = wantNode + was = 'a NODE' + } else if (items[i] instanceof EdgeAt) { + ok = !wantNode + was = 'an EDGE' + } else { + ok = false + was = 'neither a NODE nor an EDGE' + } + if (!ok) { + const wanted = wantNode ? 'a NODE' : 'an EDGE' + throw refuse( + `line ${at}: a PATH alternates, so value ${i + 1} is ${was} where it should be ${wanted}`, + ) + } + } + return new Walk(items) +} + +/* A node, written as its table and the offset of its row: person#1. + * + * Split from the right, so that a table whose name holds a `#` is still + * readable. + */ +function nodeAt(text) { + const hash = text.lastIndexOf('#') + if (hash <= 0) return undefined + const offset = text.slice(hash + 1) + if (!digits(offset)) return undefined + return new NodeAt(text.slice(0, hash), BigInt(offset)) +} + +/* An edge, written as its table and the rows it runs between: + * knows#0->1. + */ +function edgeAt(text) { + const hash = text.lastIndexOf('#') + if (hash <= 0) return undefined + const ends = text.slice(hash + 1) + const arrow = ends.indexOf('->') + if (arrow < 0) return undefined + const src = ends.slice(0, arrow) + const dst = ends.slice(arrow + 2) + if (!digits(src) || !digits(dst)) return undefined + return new EdgeAt(text.slice(0, hash), BigInt(src), BigInt(dst)) +} + +/* Whether text is one or more ASCII digits and nothing else, which is + * what a row number in a node or an edge is. `BigInt` takes a leading + * sign, a leading `0x` and whitespace around the whole of it, and none + * of those is a row number anybody meant to write. + */ +function digits(text) { + if (text === '') return false + for (const c of text) if (c < '0' || c > '9') return false + return true +} + +/* The corpus's own shape for a value that came back from a statement. + * + * Everything a table holds is spelled the same on both sides and comes + * through untouched. A graph value is not: an edge carries a field the + * corpus does not write and a path arrives as two lists rather than as + * the walk it is, so they are put into the shapes above before anything + * is compared, which is what the Rust runner's `from_engine` does for + * the same reason. + */ +export function cell(value) { + if (value instanceof ZuNode) return new NodeAt(value.table, value.offset) + if (value instanceof ZuRel) return new EdgeAt(value.table, value.src, value.dst) + if (Array.isArray(value)) return value.map(cell) + if (isPath(value)) { + const elements = [] + for (let i = 0; i < value.nodes.length; i++) { + if (i > 0) elements.push(cell(value.rels[i - 1])) + elements.push(cell(value.nodes[i])) + } + return new Walk(elements) + } + if (isRecord(value)) { + const out = {} + for (const [name, item] of Object.entries(value)) out[name] = cell(item) + return out + } + return value +} + +/* Whether a value is the `{ nodes, rels }` this client hands a path back + * as. + * + * Structural, because a path has no class here: it is two arrays kept + * apart so that a caller asking about one of them does not have to write + * the stride by hand. The test is tight enough that a record cannot pass + * it by accident. A walk holds at least one node and one fewer edge than + * it holds nodes, so a record with an empty `nodes` and an empty `rels` + * in it is a record, and one holding anything but nodes and edges under + * those names is too. + */ +function isPath(value) { + if (value === null || typeof value !== 'object') return false + const keys = Object.keys(value) + if (keys.length !== 2 || !keys.includes('nodes') || !keys.includes('rels')) return false + if (!Array.isArray(value.nodes) || !Array.isArray(value.rels)) return false + if (value.nodes.length === 0 || value.rels.length !== value.nodes.length - 1) return false + return ( + value.nodes.every((one) => one instanceof ZuNode) && + value.rels.every((one) => one instanceof ZuRel) + ) +} + +/* Whether a value is a record, which is a plain object and not one of + * the classes this client hands values back as. + */ +function isRecord(value) { + if (value === null || typeof value !== 'object') return false + const proto = Object.getPrototypeOf(value) + return proto === Object.prototype || proto === null +} + +/* The value a type's text spells, and `undefined` for text that spells + * none. Undefined rather than null, because null is the value NULL + * spells and a decoder that used it for failure could not tell the two + * apart. + */ +function scalar(type, text) { + switch (type) { + case 'BOOL': + if (text === 'true') return true + if (text === 'false') return false + return undefined + case 'STRING': + return text + case 'BYTES': + return fromHexits(text) + case 'FLOAT32': + case 'FLOAT64': { + const f = readFloat(text) + if (f === undefined) return undefined + return type === 'FLOAT32' ? Math.fround(f) : f + } + case 'DATE': + return parseDate(text) + case 'LOCALTIME': + return parseLocalTime(text) + case 'ZONEDTIME': + return parseZonedTime(text) + case 'LOCALDATETIME': + return parseLocalDateTime(text) + case 'ZONEDDATETIME': + return parseZonedDateTime(text) + case 'DURATION': + return parseDuration(text) + } + const bound = BOUNDS.get(type) + if (bound === undefined) return undefined + let n + try { + n = BigInt(text) + } catch { + return undefined + } + // Written back out and compared, so that a leading plus, a leading + // zero, an underscore between digits and a `0x` in front are all + // refused rather than read as the number they resemble. `BigInt('')` + // is zero, which this catches along with the rest. + if (String(n) !== text) return undefined + if (n < bound[0] || n > bound[1]) return undefined + return n +} + +/* A float, including the three spellings YAML has no opinion about. + * + * They are spelled the way Rust prints them, because that is what the + * reference runner writes into a failure report and what a case is + * pasted from. + */ +function readFloat(text) { + if (text === 'NaN') return Number.NaN + if (text === 'inf') return Number.POSITIVE_INFINITY + if (text === '-inf') return Number.NEGATIVE_INFINITY + // A float is exact here, so `1` is not a FLOAT64 and neither is + // `1e400`. The first is an integer somebody meant to write as `1.0` + // and the second is `inf` under another name. + if (!/[.eE]/.test(text)) return undefined + // `Number` takes `Infinity`, `0x10`, `1_0` and the empty string, none + // of which the corpus writes and all of which would be a case that + // reads differently in the other three runners. + if (!/^[0-9.eE+-]+$/.test(text)) return undefined + const f = Number(text) + if (!Number.isFinite(f)) return undefined + return f +} + +/* Whether two values are the same value. + * + * Not `===`, for three reasons. A float: NaN is not equal to itself and + * a case asserting NaN has to pass, and 0.0 equals -0.0 and a case + * asserting -0.0 has to fail on 0.0, because the sign of zero is exactly + * the sort of thing that survives one binding and not another. The + * shapes that hold other values, which are compared by walking them. And + * every temporal value, which is a class holding its count behind a + * getter, so two of them are never `===` however equal they are. + * + * The type is part of the answer everywhere: a `bigint` is not the + * `number` beside it, and a year-month duration is not the day-time one + * that counts the same nothing. + */ +export function same(want, got) { + if (typeof want === 'number') { + if (typeof got !== 'number') return false + if (Number.isNaN(want) && Number.isNaN(got)) return true + return Object.is(want, got) + } + if (Array.isArray(want)) return Array.isArray(got) && sameAll(want, got) + if (Array.isArray(got)) return false + if (want instanceof Walk) return got instanceof Walk && sameAll(want.elements, got.elements) + if (want instanceof NodeAt) { + return got instanceof NodeAt && want.table === got.table && want.offset === got.offset + } + if (want instanceof EdgeAt) { + return ( + got instanceof EdgeAt && + want.table === got.table && + want.src === got.src && + want.dst === got.dst + ) + } + if (want instanceof ZuDate) return got instanceof ZuDate && want.days === got.days + if (want instanceof ZuTime) { + return got instanceof ZuTime && want.nanos === got.nanos && sameOffset(want, got) + } + if (want instanceof ZuTimestamp) { + return got instanceof ZuTimestamp && want.nanos === got.nanos && sameOffset(want, got) + } + if (want instanceof ZuDuration) { + return ( + got instanceof ZuDuration && + want.kind === got.kind && + want.months === got.months && + want.nanos === got.nanos + ) + } + if (want instanceof Uint8Array) { + if (!(got instanceof Uint8Array) || want.length !== got.length) return false + for (let i = 0; i < want.length; i++) if (want[i] !== got[i]) return false + return true + } + if (got instanceof Uint8Array) return false + if (isRecord(want)) { + if (!isRecord(got)) return false + const names = Object.keys(want) + if (names.length !== Object.keys(got).length) return false + return names.every((name) => name in got && same(want[name], got[name])) + } + if (isRecord(got)) return false + return want === got +} + +/* Whether two temporal values carry the same offset. + * + * A local one has no offset and the class spells that absence as `null`. + * `undefined` is taken for the same thing, because the constructor takes + * either and a case comparing what it built against what the engine gave + * back should not turn on which of the two words was used for nothing. + */ +function sameOffset(want, got) { + return (want.offset ?? null) === (got.offset ?? null) +} + +function sameAll(want, got) { + return want.length === got.length && want.every((one, i) => same(one, got[i])) +} + +/* How a value reads in a failure report, in the encoding's own spelling + * so that it can be pasted into a case, and line for line what the Rust + * runner prints so that two reports can be diffed. + */ +export function show(value) { + if (value === null || value === undefined) return 'NULL' + if (typeof value === 'boolean') return value ? 'BOOL true' : 'BOOL false' + if (typeof value === 'bigint') return `INT64 "${value}"` + if (typeof value === 'number') return `FLOAT64 "${showFloat(value)}"` + if (typeof value === 'string') return `STRING ${quote(value)}` + if (value instanceof Uint8Array) return `BYTES "${hexits(value)}"` + if (value instanceof ZuDate) return `DATE "${showDate(value.days)}"` + if (value instanceof ZuTime) { + if ((value.offset ?? null) === null) return `LOCALTIME "${showClock(value.nanos)}"` + return `ZONEDTIME "${showClock(value.nanos)}${showOffset(value.offset)}"` + } + if (value instanceof ZuTimestamp) { + if ((value.offset ?? null) === null) return `LOCALDATETIME "${showStamp(value.nanos)}"` + const wall = value.nanos + BigInt(value.offset) * NANOS_PER_MINUTE + return `ZONEDDATETIME "${showStamp(wall)}${showOffset(value.offset)}"` + } + if (value instanceof ZuDuration) { + if (value.kind === 'yearMonth') return `DURATION "${showMonths(value.months)}"` + return `DURATION "${showNanos(value.nanos)}"` + } + if (Array.isArray(value)) return `LIST [${value.map(show).join(', ')}]` + if (value instanceof Walk) return `PATH [${value.elements.map(show).join(', ')}]` + if (value instanceof NodeAt) return `NODE "${value.table}#${value.offset}"` + if (value instanceof EdgeAt) return `EDGE "${value.table}#${value.src}->${value.dst}"` + if (isRecord(value)) { + // Sorted, so that a report of one reads the same twice. A plain + // object keeps the order its keys were added in, which is the order + // the engine wrote them, and a failure that reorders its own fields + // between an engine and a case is a failure nobody can diff. + const fields = Object.keys(value) + .sort() + .map((name) => `${name}: ${show(value[name])}`) + return `RECORD {${fields.join(', ')}}` + } + // A `ZuNode`, a `ZuRel` or a path reaching here is a value the runner + // did not put through `cell`. It prints as itself, under a name that + // is not a type, so that a report carrying one cannot be mistaken for + // a case that could be pasted back into the corpus. + const name = value?.constructor?.name ?? typeof value + return `(${name}) ${JSON.stringify(value, (_, one) => (typeof one === 'bigint' ? `${one}` : one))}` +} + +/* A float the way Rust's {:?} writes one, which is the shortest text + * that reads back as the same double and always carries a point or an + * exponent. + * + * Written out rather than taken from `toString`, which switches to an + * exponent at a different place, writes the exponent with a sign, and + * prints a whole number with no point at all. Each of those is a report + * that differs from the reference one without the answer differing, + * which is the thing this whole file exists to avoid. + */ +export function showFloat(f) { + if (Number.isNaN(f)) return 'NaN' + if (f === Number.POSITIVE_INFINITY) return 'inf' + if (f === Number.NEGATIVE_INFINITY) return '-inf' + const negative = Object.is(f, -0) || f < 0 + const sign = negative ? '-' : '' + const g = negative ? -f : f + // The shortest digits that read back as this double, and where the + // point goes in them. `toExponential` with no argument is what says + // shortest, and it always writes the digits and the exponent apart + // whatever the size of the number. + const spelled = g.toExponential() + const at = spelled.indexOf('e') + const run = spelled.slice(0, at).replace('.', '') + const exp = Number(spelled.slice(at + 1)) + const point = exp + 1 + if (point <= -4 || point > 16) { + const out = run.length > 1 ? `${run[0]}.${run.slice(1)}` : run + return `${sign}${out}e${exp}` + } + if (point <= 0) return `${sign}0.${'0'.repeat(-point)}${run}` + if (point >= run.length) return `${sign}${run}${'0'.repeat(point - run.length)}.0` + return `${sign}${run.slice(0, point)}.${run.slice(point)}` +} + +/* A byte string the way the engine writes one: two hexits to a byte, + * upper case, no quotes and no X. + * + * Upper case because the standard writes the literal that way, and a + * reader comparing two of these is comparing text, so one case is one + * answer. + */ +export function hexits(raw) { + let out = '' + for (const b of raw) out += b.toString(16).toUpperCase().padStart(2, '0') + return out +} + +/* The bytes a run of hexits names, and `undefined` for anything that is + * not a run of hexits or that names half a byte. + * + * Space is allowed anywhere and dropped, which is what the standard's + * production allows and what lets a long literal be written in groups. + * Either case reads, because a value that went in as 00ab and came back + * as 00AB is the same value. + */ +function fromHexits(text) { + const nibbles = [] + for (const c of text) { + if (' \t\n\r\v\f'.includes(c)) continue + // `parseInt` with a radix of sixteen takes exactly the sixteen + // hexits and nothing else, in either case, which is the whole of the + // test here. + const n = Number.parseInt(c, 16) + if (Number.isNaN(n)) return undefined + nibbles.push(n) + } + if (nibbles.length % 2 !== 0) return undefined + const out = new Uint8Array(nibbles.length / 2) + for (let i = 0; i < nibbles.length; i += 2) out[i / 2] = (nibbles[i] << 4) | nibbles[i + 1] + return out +} diff --git a/etc/zudb.api.md b/etc/zudb.api.md index 30ff55c..4ff949a 100644 --- a/etc/zudb.api.md +++ b/etc/zudb.api.md @@ -172,6 +172,7 @@ export type ZuColumnType = | 'int' | 'float' | 'string' +| 'bytes' | 'date' | 'time' | 'datetime' @@ -324,6 +325,7 @@ export type ZuParam = | ZuTimestamp | ZuDuration | ZuTemporalValue +| Uint8Array | ZuParam[] | { [field: string]: ZuParam } @@ -533,6 +535,7 @@ export type ZuValue = | ZuTimestamp | ZuDuration | ZuTemporalValue +| Uint8Array | ZuValue[] | { [field: string]: ZuValue } diff --git a/package.json b/package.json index 057af1e..15f0d95 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,7 @@ "test:temporal": "node --harmony-temporal --test \"test/*.test.mjs\"", "test:bun": "bun test --timeout 60000 test/", "test:deno": "deno test --no-check --allow-read --allow-write --allow-env --allow-ffi \"test/*.test.mjs\"", + "corpus": "node conformance/cli.mjs", "check:types": "tsc --noEmit --project test/types/tsconfig.json", "check:package": "attw --pack .", "check:api": "api-extractor run --verbose", diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 45d0ca1..76cc5f5 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -3,5 +3,5 @@ # a different build of the same code. Held to the version in # tamnd/zu's toolchains.toml. [toolchain] -channel = "1.97.1" +channel = "1.98.0" components = ["rustfmt", "clippy"] diff --git a/src/columns.rs b/src/columns.rs index 514c59f..117c7a2 100644 --- a/src/columns.rs +++ b/src/columns.rs @@ -36,7 +36,10 @@ //! JavaScript values `query` would have made, and a column of nothing //! but nulls has a length and nothing else, because there is nothing to //! put in a buffer. Both are named by the column's `type` rather than -//! found out by looking. +//! found out by looking, and so is the third thing that is sometimes +//! not a buffer: the engine's sink has no arm for days, nanoseconds or +//! months, so a temporal column reaching a projection by the wrong path +//! comes over as values too (tamnd/zu#690). //! //! `bigIntMode` says nothing here. A columnar read has one physical //! layout per type and an INT64 column is 64 bit cells whatever a @@ -125,10 +128,30 @@ fn taken(columns: Columns<'_>, gqlstatus: &'static str, notices: Vec "value", + _ => kind(&column.ty), + }, + unit: match column.data { + ColumnData::Complex(_) => None, + _ => unit(&column.ty), + }, zone: match column.ty { - ColumnType::ZonedTime { offset } | ColumnType::ZonedDatetime { offset } => { + ColumnType::ZonedTime { offset } | ColumnType::ZonedDatetime { offset } + if !matches!(column.data, ColumnData::Complex(_)) => + { Some(offset as i32) } _ => None, @@ -141,7 +164,13 @@ fn taken(columns: Columns<'_>, gqlstatus: &'static str, notices: Vec Held::Days(values), ColumnData::Nanos(values) => Held::Nanos(values), ColumnData::Months(values) => Held::Months(values), - ColumnData::Str(column) => Held::Str { + // A byte string column keeps the two buffers a string + // column keeps, so it arrives the same way and it is + // `type` that tells the two apart. The bytes under a + // BYTES column are octets and decoding them as text is + // the caller's mistake to avoid, which is why the type + // says which it is. + ColumnData::Str(column) | ColumnData::Bytes(column) => Held::Str { bytes: column.bytes, offsets: column.offsets, }, @@ -177,6 +206,7 @@ fn kind(ty: &ColumnType) -> &'static str { ColumnType::Int => "int", ColumnType::Float => "float", ColumnType::Str => "string", + ColumnType::Bytes => "bytes", ColumnType::Date => "date", ColumnType::LocalTime | ColumnType::ZonedTime { .. } => "time", ColumnType::LocalDatetime | ColumnType::ZonedDatetime { .. } => "datetime", diff --git a/src/frame.rs b/src/frame.rs index 2fe4a1b..6d79b94 100644 --- a/src/frame.rs +++ b/src/frame.rs @@ -253,16 +253,25 @@ impl Bytes { fn extend(&mut self, raw: &[u8]) { match self { Bytes::Eight(v) => v.extend( - raw.chunks_exact(8) - .map(|word| u64::from_ne_bytes(word.try_into().expect("eight bytes"))), + raw.as_chunks::<8>() + .0 + .iter() + .copied() + .map(u64::from_ne_bytes), ), Bytes::Four(v) => v.extend( - raw.chunks_exact(4) - .map(|word| u32::from_ne_bytes(word.try_into().expect("four bytes"))), + raw.as_chunks::<4>() + .0 + .iter() + .copied() + .map(u32::from_ne_bytes), ), Bytes::Two(v) => v.extend( - raw.chunks_exact(2) - .map(|word| u16::from_ne_bytes(word.try_into().expect("two bytes"))), + raw.as_chunks::<2>() + .0 + .iter() + .copied() + .map(u16::from_ne_bytes), ), Bytes::One(v) => v.extend_from_slice(raw), Bytes::Offsets(_) => {} diff --git a/src/load.rs b/src/load.rs index 7b49ab4..920eb46 100644 --- a/src/load.rs +++ b/src/load.rs @@ -496,7 +496,7 @@ fn pairs(env: &Env, options: &Object<'_>, rows: u64) -> Result> ))); } let mut pairs = Vec::with_capacity(flat.len() / 2); - for (at, edge) in flat.chunks_exact(2).enumerate() { + for (at, edge) in flat.as_chunks::<2>().0.iter().enumerate() { pairs.push((within(at, edge[0])?, within(at, edge[1])?)); } return Ok(pairs); diff --git a/src/value.rs b/src/value.rs index 77a6c86..b784ebe 100644 --- a/src/value.rs +++ b/src/value.rs @@ -568,6 +568,11 @@ pub fn to_js<'env>( Value::Int(n) => int(env, column, *n, shape.spelling.ints), Value::Float(f) => (*f).into_unknown(env), Value::Str(s) => s.as_str().into_unknown(env), + // GV35, a byte string, which is octets and not text. It comes + // back as a `Uint8Array` rather than a string because the + // bytes need not be UTF-8 at all, and a client that decoded + // them would refuse half the values the type exists for. + Value::Bytes(bytes) => Uint8Array::new(bytes.clone()).into_unknown(env), Value::Node { table, offset } => node(*table, *offset, &shape.names) .into_instance(env)? .into_unknown(env), @@ -796,9 +801,17 @@ fn nested(env: &Env, name: &str, value: Unknown<'_>, depth: usize) -> Result { let n = f64::from_unknown(value)?; - if n.fract() == 0.0 && n.abs() <= 9_007_199_254_740_992.0 { + let negative_zero = n == 0.0 && n.is_sign_negative(); + if n.fract() == 0.0 && n.abs() <= 9_007_199_254_740_992.0 && !negative_zero { Ok(Value::Int(n as i64)) } else { Ok(Value::Float(n)) @@ -816,6 +829,26 @@ fn from_object(env: &Env, name: &str, value: Unknown<'_>, depth: usize) -> Resul if let Some(temporal) = temporal_from(env, &value)? { return Ok(Value::Temporal(temporal)); } + // A `Uint8Array` binds as GV35, a byte string, which is the one + // type whose values are octets rather than text. This goes before + // the array and before the record, because a typed array is + // neither: it answers no to `is_array` and has no own enumerable + // properties, so a plain object read would bind it as `{}` and + // compare against nothing. + if value.is_typedarray()? { + return match TypedArray::from_unknown(value)?.typed_array_type { + TypedArrayType::Uint8 | TypedArrayType::Uint8Clamped => { + Ok(Value::Bytes(Uint8Array::from_unknown(value)?.to_vec())) + } + _ => Err(Error::new( + Status::InvalidArg, + format!( + "parameter {name} is a typed array of some other width, and the only one a \ + statement holds is a Uint8Array, which binds as BYTES" + ), + )), + }; + } let object = Object::from_unknown(value)?; if object.is_array()? { let len = object.get_array_length()?; diff --git a/test/appender.test.mjs b/test/appender.test.mjs index 1aa0832..96d8f9f 100644 --- a/test/appender.test.mjs +++ b/test/appender.test.mjs @@ -313,7 +313,7 @@ test('something that is not an array of rows is refused', async (t) => { test('every column type the engine stores takes a value', async (t) => { const { conn } = await fresh(t) await conn.exec( - "INSERT (e:event {id: 1, on: DATE '2024-01-02', at: LOCAL DATETIME '2024-01-02T03:04:05', " + + "INSERT (e:event {id: 1, began: DATE '2024-01-02', moment: LOCAL DATETIME '2024-01-02T03:04:05', " + "took: DURATION 'PT1H', hot: true, ratio: 1.5})", ) @@ -329,12 +329,12 @@ test('every column type the engine stores takes a value', async (t) => { assert.equal(await rows.close(), 1) const found = await conn.query( - 'MATCH (e:event) RETURN e.id AS id, e.on AS on, e.at AS at, e.took AS took, e.hot AS hot, ' + + 'MATCH (e:event) RETURN e.id AS id, e.began AS began, e.moment AS moment, e.took AS took, e.hot AS hot, ' + 'e.ratio AS ratio', ) assert.equal(found.length, 2) - assert.equal(found[1].on.days, 20_000) - assert.equal(found[1].at.nanos, 1_700_000_000_000_000_000n) + assert.equal(found[1].began.days, 20_000) + assert.equal(found[1].moment.nanos, 1_700_000_000_000_000_000n) assert.equal(found[1].took.nanos, 7_200_000_000_000n) assert.equal(found[1].hot, false) assert.equal(found[1].ratio, 2.5) @@ -342,7 +342,7 @@ test('every column type the engine stores takes a value', async (t) => { test('a timestamp with an offset does not go in a column of local ones', async (t) => { const { conn } = await fresh(t) - await conn.exec("INSERT (e:event {id: 1, at: LOCAL DATETIME '2024-01-02T03:04:05'})") + await conn.exec("INSERT (e:event {id: 1, moment: LOCAL DATETIME '2024-01-02T03:04:05'})") const rows = await conn.appender('event') assert.throws( diff --git a/test/arrow.test.mjs b/test/arrow.test.mjs index f6ae62f..f6fe86f 100644 --- a/test/arrow.test.mjs +++ b/test/arrow.test.mjs @@ -15,7 +15,7 @@ import test from 'node:test' import { tableFromIPC } from 'apache-arrow' import { connect, load } from 'zudb' -import { fresh, isZuError, twoPeople } from './helper.mjs' +import { fresh, isZuError, tickRate, twoPeople } from './helper.mjs' // The table the bytes hold, which is the whole of what a caller writes. function read(answer) { @@ -97,18 +97,18 @@ test('floats and booleans are the Arrow types they are', async (t) => { test('a temporal column is the Arrow type that counts the same thing', async (t) => { const { conn } = await fresh(t) await conn.exec( - "INSERT (e:event {id: 1, on: DATE '2024-01-01', at: LOCAL DATETIME '2024-01-02T03:04:05', " + + "INSERT (e:event {id: 1, began: DATE '2024-01-01', moment: LOCAL DATETIME '2024-01-02T03:04:05', " + "took: DURATION 'PT1H'})", ) const table = read( - await conn.arrow('MATCH (e:event) RETURN e.on AS on, e.at AS at, e.took AS took'), + await conn.arrow('MATCH (e:event) RETURN e.began AS began, e.moment AS moment, e.took AS took'), ) // A date is days, a datetime is nanoseconds, and a day-time duration // is a duration rather than an interval, which is the difference // between a length of time and a calendar step. - assert.equal(String(column(table, 'on').type), 'Date32') - assert.equal(String(column(table, 'at').type), 'Timestamp') + assert.equal(String(column(table, 'began').type), 'Date32') + assert.equal(String(column(table, 'moment').type), 'Timestamp') assert.equal(String(column(table, 'took').type), 'Duration') }) @@ -324,26 +324,35 @@ test('the bytes are handed over rather than shared, so two reads are two buffers test('a million rows are one stream and the loop stays free while it is written', async (t) => { const { conn } = await fresh(t) - await conn.exec('INSERT (n:number {id: 1, at: 1})') + await conn.exec('INSERT (n:numbers {id: 1, moment: 1})') const rows = 1_000_000 - const appender = await conn.appender('number') + const appender = await conn.appender('numbers') for (let at = 2; at <= rows; at += 1) appender.appendRow([BigInt(at), BigInt(at)]) await appender.close() + const idle = await tickRate() let ticks = 0 const timer = setInterval(() => (ticks += 1), 1) const at = performance.now() - const answer = await conn.arrow('MATCH (n:number) RETURN n.at AS at') + const answer = await conn.arrow('MATCH (n:numbers) RETURN n.moment AS moment') const took = performance.now() - at clearInterval(timer) assert.equal(answer.rows, rows) const table = read(answer) assert.equal(table.numRows, rows) - assert.equal(column(table, 'at').get(rows - 1), BigInt(rows)) + assert.equal(column(table, 'moment').get(rows - 1), BigInt(rows)) // The whole write is on the threadpool, so the timer kept firing - // throughout it rather than queueing behind it. The bar is a tick - // every ten milliseconds of the read and not a fixed count, because a - // blocked loop fires none however long the read takes. - assert.ok(ticks > took / 10, `the event loop ticked ${ticks} times in ${took.toFixed(0)} ms`) + // throughout it rather than queueing behind it. The bar is a share of + // what this loop manages with nothing to do rather than a rate of its + // own: a blocked loop fires no timer at all however long the read + // takes, and how fast a free one fires is the platform's business. + // A fifth, which is far enough below idle to survive a loaded machine + // and still a hundred times what a loop parked in the addon would + // give. + assert.ok( + ticks / took > idle / 5, + `the event loop ticked ${ticks} times in ${took.toFixed(0)} ms, ` + + `against ${(idle * took).toFixed(0)} with nothing to do`, + ) }) diff --git a/test/columnar.test.mjs b/test/columnar.test.mjs index 2181961..36fade0 100644 --- a/test/columnar.test.mjs +++ b/test/columnar.test.mjs @@ -24,7 +24,7 @@ import { Vector, } from 'apache-arrow' -import { fresh, isZuError, twoPeople } from './helper.mjs' +import { fresh, isZuError, tickRate, twoPeople } from './helper.mjs' // The columns by name, since a test asks about one of them and the // order they were projected in is asserted where it is the question. @@ -131,29 +131,71 @@ test('a column of booleans is one bit a row, least significant first', async (t) test('a temporal column says what its cells count', async (t) => { const { conn } = await fresh(t) - await conn.exec( - "INSERT (e:event {id: 1, on: DATE '2024-01-01', at: LOCAL DATETIME '2024-01-02T03:04:05', " + - "took: DURATION 'PT1H'})", - ) const read = await conn.columnar( - 'MATCH (e:event) RETURN e.on AS on, e.at AS at, e.took AS took', + "RETURN DATE '2024-01-01' AS began, LOCAL DATETIME '2024-01-02T03:04:05' AS moment, " + + "DURATION 'PT1H' AS took", ) - const { on, at, took } = named(read) + const { began, moment, took } = named(read) - assert.equal(on.type, 'date') - assert.equal(on.unit, 'days') - assert.ok(on.values instanceof Int32Array) - assert.deepEqual([...on.values], [19_723]) + assert.equal(began.type, 'date') + assert.equal(began.unit, 'days') + assert.ok(began.values instanceof Int32Array) + assert.deepEqual([...began.values], [19_723]) - assert.equal(at.type, 'datetime') - assert.equal(at.unit, 'nanos') - assert.deepEqual([...at.values], [1_704_164_645_000_000_000n]) + assert.equal(moment.type, 'datetime') + assert.equal(moment.unit, 'nanos') + assert.deepEqual([...moment.values], [1_704_164_645_000_000_000n]) assert.equal(took.type, 'duration') assert.equal(took.unit, 'nanos') assert.deepEqual([...took.values], [3_600_000_000_000n]) }) +// A temporal column does not always arrive as cells. The engine's +// columnar sink fills a buffer for an integer, a real, a boolean and a +// string, and has no arm for days, nanoseconds or months, so a date or +// a duration reaching a projection any other way comes over as the +// values themselves. Which of the two a statement gets is the plan's +// business and not the caller's: the same property is cells in one +// statement and values in another, and adding an unrelated column to +// the projection is enough to move it. That is tamnd/zu#690. +// +// So `type` is what arrived rather than what the statement declared. +// A column saying `date` with nothing in `values` would be a switch +// that lands nowhere, and `type` exists to name the field that holds +// the values. The invariant is what is asserted here rather than which +// shape this particular statement produces, so the day the engine +// fills a buffer for every temporal column this test still passes and +// the one above it still says what a filled buffer holds. +test('a temporal column that arrived without cells says so', async (t) => { + const { conn } = await fresh(t) + await conn.exec( + "INSERT (e:event {id: 1, began: DATE '2024-01-01', moment: LOCAL DATETIME '2024-01-02T03:04:05', " + + "took: DURATION 'PT1H'})", + ) + const read = await conn.columnar( + 'MATCH (e:event) RETURN e.began AS began, e.moment AS moment, e.took AS took', + ) + const { began, moment, took } = named(read) + + for (const column of [began, moment, took]) { + assert.equal(column.length, 1) + if (column.type === 'value') { + assert.equal(column.values, null) + assert.equal(column.unit, null) + assert.equal(column.zone, null) + assert.ok(column.items !== null, `${column.name} is a value column holding nothing`) + } else { + assert.ok(column.values !== null, `${column.name} says ${column.type} and has no cells`) + assert.ok(column.unit !== null) + } + } + // Whichever way they came, they are the values that went in. + assert.equal((began.items?.[0] ?? { days: began.values?.[0] }).days, 19_723) + assert.equal((moment.items?.[0] ?? { nanos: moment.values?.[0] }).nanos, 1_704_164_645_000_000_000n) + assert.equal((took.items?.[0] ?? { nanos: took.values?.[0] }).nanos, 3_600_000_000_000n) +}) + test('a duration of months counts months rather than nanoseconds', async (t) => { const { conn } = await fresh(t) const read = await conn.columnar("RETURN DURATION 'P14M' AS every") @@ -209,21 +251,21 @@ test('a column with nothing null has no bitmap at all', async (t) => { test('a column of nothing but nulls has a length and no buffer', async (t) => { const { conn } = await fresh(t) - const read = await conn.columnar('RETURN null AS nothing') + const read = await conn.columnar('RETURN null AS blank') - const [nothing] = read.columns - assert.equal(nothing.type, 'null') - assert.equal(nothing.length, 1) - assert.equal(nothing.values, null) - assert.equal(nothing.items, null) + const [blank] = read.columns + assert.equal(blank.type, 'null') + assert.equal(blank.length, 1) + assert.equal(blank.values, null) + assert.equal(blank.items, null) }) test('what no buffer covers arrives as the values themselves', async (t) => { const { conn } = await twoPeople(t) const read = await conn.columnar( - 'MATCH (p:person) RETURN p AS who, [p.id, p.id] AS pair, {name: p.name} AS record', + 'MATCH (p:person) RETURN p AS who, [p.id, p.id] AS pair, {name: p.name} AS fields', ) - const { who, pair, record } = named(read) + const { who, pair, fields } = named(read) assert.equal(who.type, 'value') assert.equal(who.values, null) @@ -234,8 +276,8 @@ test('what no buffer covers arrives as the values themselves', async (t) => { assert.equal(pair.type, 'value') assert.deepEqual(pair.items[0], [1n, 1n]) - assert.equal(record.type, 'value') - assert.deepEqual(record.items[1], { name: 'zoe' }) + assert.equal(fields.type, 'value') + assert.deepEqual(fields.items[1], { name: 'zoe' }) }) test('a statement that matched nothing is columns of no rows', async (t) => { @@ -355,16 +397,17 @@ test('the buffers are handed over rather than shared, so two reads are two buffe test('a million rows come back down one buffer and the loop stays free', async (t) => { const { conn } = await fresh(t) - await conn.exec("INSERT (n:number {id: 1, at: 1})") + await conn.exec("INSERT (n:numbers {id: 1, moment: 1})") const rows = 1_000_000 - const appender = await conn.appender('number') + const appender = await conn.appender('numbers') for (let at = 2; at <= rows; at += 1) appender.appendRow([BigInt(at), BigInt(at)]) await appender.close() + const idle = await tickRate() let ticks = 0 const timer = setInterval(() => (ticks += 1), 1) const at = performance.now() - const read = await conn.columnar('MATCH (n:number) RETURN n.at AS at') + const read = await conn.columnar('MATCH (n:numbers) RETURN n.moment AS moment') const took = performance.now() - at clearInterval(timer) @@ -372,13 +415,17 @@ test('a million rows come back down one buffer and the loop stays free', async ( assert.equal(read.columns[0].values.length, rows) assert.equal(read.columns[0].values[rows - 1], BigInt(rows)) // The whole read is on the threadpool, so the timer kept firing - // throughout it rather than queueing behind it. The bar is a tick - // every ten milliseconds of the read and not a fixed count, because - // a blocked loop fires none however long the read takes and a fixed - // count turns every speedup into a failure. + // throughout it rather than queueing behind it. The bar is a share of + // what this loop manages with nothing to do rather than a rate of its + // own: a blocked loop fires no timer at all however long the read + // takes, and how fast a free one fires is the platform's business. A + // fifth, which is far enough below idle to survive a loaded machine + // and still a hundred times what a loop parked in the addon would + // give. assert.ok( - ticks > took / 10, - `the event loop ticked ${ticks} times in ${took.toFixed(0)} ms`, + ticks / took > idle / 5, + `the event loop ticked ${ticks} times in ${took.toFixed(0)} ms, ` + + `against ${(idle * took).toFixed(0)} with nothing to do`, ) }) @@ -418,16 +465,16 @@ function tableOf(read) { test('the columns become an Arrow table without being copied', async (t) => { const { conn } = await fresh(t) await conn.exec( - "INSERT (e:event {id: 1, name: 'ada', ratio: 1.5, hot: true, on: DATE '2024-01-01', " + + "INSERT (e:event {id: 1, name: 'ada', ratio: 1.5, hot: true, began: DATE '2024-01-01', " + "took: DURATION 'PT1H'})", ) await conn.exec( - "INSERT (e:event {id: 2, name: 'zoe', ratio: 2.5, hot: false, on: DATE '2024-02-01', " + + "INSERT (e:event {id: 2, name: 'zoe', ratio: 2.5, hot: false, began: DATE '2024-02-01', " + "took: DURATION 'PT2H'})", ) const read = await conn.columnar( 'MATCH (e:event) RETURN e.id AS id, e.name AS name, e.ratio AS ratio, e.hot AS hot, ' + - 'e.on AS on, e.took AS took', + 'e.began AS began, e.took AS took', ) const table = tableOf(read) @@ -439,7 +486,7 @@ test('the columns become an Arrow table without being copied', async (t) => { 'name:Utf8', 'ratio:Float64', 'hot:Bool', - 'on:Date32', + 'began:Date32', 'took:Duration', ], ) diff --git a/test/conformance-arrow.test.mjs b/test/conformance-arrow.test.mjs new file mode 100644 index 0000000..5fc09e8 --- /dev/null +++ b/test/conformance-arrow.test.mjs @@ -0,0 +1,174 @@ +// The Arrow half: what a case says an export gives, and what the export +// actually gave. +// +// This client hands out Arrow IPC bytes rather than a C Data Interface +// stream, so the schema is read out of the flatbuffer at the head of the +// buffer and turned back into the format strings a case is written in. +// That reader is the thing under test here, and it is tested against +// bytes the addon really produced rather than against bytes made up for +// the occasion, because a hand written buffer would only prove that the +// reader agrees with whoever wrote the buffer. +// +// The comparison is tested on its own, with no bytes at all, since it is +// strings and a walk and its wording goes into a report that is diffed +// against the reference runner's. + +import assert from 'node:assert/strict' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' + +import { connect, load } from 'zudb' + +import { THE_RESULT, exported, schemaSays } from '../conformance/arrow.mjs' + +// A field, written short because a schema is mostly nesting. +function field(name, format, ...children) { + return { name, format, children } +} + +// A connection on a database of its own, gone when the test ends. +async function open(t) { + const dir = await mkdtemp(join(tmpdir(), 'zu-arrow-')) + t.after(() => rm(dir, { recursive: true, force: true })) + const conn = await connect(join(dir, 'one.zu')) + t.after(() => conn.close()) + return conn +} + +test('a schema that matches says nothing', () => { + const held = [field('n', 'l'), field('l', '+l', field('item', 'u'))] + assert.equal(schemaSays(held, held), '') + assert.equal(schemaSays([], []), '') +}) + +// The place a difference is in, which is the result itself at the top +// and the case's own names joined by dots underneath. A schema printed +// in full at somebody is a report nobody reads. +test('a schema that differs says where and how', () => { + for (const [what, got, want, says] of [ + [ + 'more fields than the case wants', + [field('a', 'l'), field('b', 'l')], + [field('a', 'l')], + `arrow gives 2 fields in ${THE_RESULT} where the case wants 1`, + ], + [ + 'fewer fields than the case wants', + [], + [field('a', 'l')], + `arrow gives 0 fields in ${THE_RESULT} where the case wants 1`, + ], + [ + 'a field under another name', + [field('a', 'l')], + [field('b', 'l')], + `arrow field 1 in ${THE_RESULT} is named "a" where the case wants "b"`, + ], + [ + 'a field of another type', + [field('a', 'l')], + [field('a', 'g')], + 'arrow field "a" is "l" where the case wants "g"', + ], + [ + 'a list whose field is named something else', + [field('l', '+l', field('item', 'l'))], + [field('l', '+l', field('element', 'l'))], + 'arrow field 1 in "l" is named "item" where the case wants "element"', + ], + [ + 'a struct with a field missing from it', + [field('p', '+s', field('table', 'u'))], + [field('p', '+s', field('table', 'u'), field('offset', 'l'))], + 'arrow gives 1 fields in "p" where the case wants 2', + ], + [ + 'a field two deep', + [field('l', '+l', field('item', '+s', field('table', 'u')))], + [field('l', '+l', field('item', '+s', field('table', 'U')))], + 'arrow field "l.item.table" is "u" where the case wants "U"', + ], + [ + // The name is checked before the type, so a field that is both in + // the wrong place and of the wrong type reports the place. Naming + // the type first would report the same schema differently + // depending on which field the walk reached first. + 'a field that is wrong both ways', + [field('a', 'l')], + [field('b', 'g')], + `arrow field 1 in ${THE_RESULT} is named "a" where the case wants "b"`, + ], + ]) { + assert.equal(schemaSays(got, want), says, `${what} was reported with the wrong words`) + } +}) + +// From here down the addon produces the bytes. + +test('the schema of a real export reads back as format strings', async (t) => { + const conn = await open(t) + for (const [query, name, format] of [ + ['RETURN 1 AS n', 'n', 'l'], + ['RETURN 1.5 AS f', 'f', 'g'], + ["RETURN 'text' AS s", 's', 'u'], + ['RETURN true AS b', 'b', 'b'], + ]) { + const got = exported(await conn.arrow(query)) + assert.deepEqual( + got.fields, + [field(name, format)], + `${query} exported ${JSON.stringify(got.fields)}`, + ) + } +}) + +// A flatbuffer leaves out any field holding the schema's default for it, +// so a reader that defaults the wrong way reads a whole family of types +// as another one and says nothing about it. `is_signed` defaults to +// false, so an unsigned integer is written as an `Int` table with a +// width and nothing else, and a reader defaulting it to true reports +// every UINT64 in the export as `l`. A node's row offset is the unsigned +// column this engine actually exports, so this asserts the default +// through bytes rather than through a table of my own. +test('an unsigned column reads back unsigned', async (t) => { + const dir = await mkdtemp(join(tmpdir(), 'zu-arrow-')) + t.after(() => rm(dir, { recursive: true, force: true })) + const path = join(dir, 'people.zu') + await load(path, { + nodes: 'person', + rels: 'knows', + rows: 2, + columns: { name: ['ann', 'bo'] }, + edges: new Uint32Array([0, 1]), + }) + const conn = await connect(path) + t.after(() => conn.close()) + const got = exported(await conn.arrow("MATCH (n:person) WHERE n.name = 'ann' RETURN n")) + assert.equal(got.fields.length, 1) + const offset = got.fields[0].children.find((one) => one.name === 'offset') + assert.ok(offset !== undefined, `a node exported as ${JSON.stringify(got.fields[0])}`) + assert.equal(offset.format, 'L', 'an unsigned integer read back as a signed one') +}) + +// The buffer carries a row count beside the bytes and the bytes carry +// their own in the batches. A client where those two disagree has a bug +// no case asserting values would ever see, so the reader hands both back +// and the runner compares them. +test('an export counts its rows in the batches and beside them', async (t) => { + const conn = await open(t) + const got = exported(await conn.arrow('RETURN 1 AS n')) + assert.equal(got.rows, 1) + assert.equal(got.declared, 1, 'the buffer and its batches disagree about how many rows there are') +}) + +// A statement answering no columns at all is not a statement answering +// none of something, and the export has to come back rather than fail. +test('an export of a result with nothing in it still reads', async (t) => { + const conn = await open(t) + const got = exported(await conn.arrow('FINISH')) + assert.deepEqual(got.fields, []) + assert.equal(got.rows, 0) + assert.equal(got.declared, 0) +}) diff --git a/test/conformance-cases.test.mjs b/test/conformance-cases.test.mjs new file mode 100644 index 0000000..3079498 --- /dev/null +++ b/test/conformance-cases.test.mjs @@ -0,0 +1,510 @@ +// What a case file is allowed to say, and what it is not. +// +// A case is read once and run against every client, so a file that reads +// two ways is worse than a file that does not read at all. Most of this +// is therefore refusals, and each of them is a mistake somebody would +// otherwise make quietly: a case whose row has one value too few, a load +// whose column has one value too many, a name that is spelled two ways in +// two clients, a case that says both what it returns and what it raises. +// +// The messages are checked in full, for the reason they are in the reader +// test: they are diffed against the reference runner's report and a +// wording that drifted is a difference in the report that is not a +// difference in the answer. + +import assert from 'node:assert/strict' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' + +import { CorpusError, quote } from '../conformance/reader.mjs' +import { MAIN, SCHEMA, read, readDir } from '../conformance/cases.mjs' +import { same, show } from '../conformance/values.mjs' + +// The head of a file that reads, so that a test about one key is a test +// about that key. +const HEAD = `schema: ${SCHEMA}\nsuite: example\ndoc: what this suite is for\n` + +// A file holding one case, with the lines given put under `cases:`. +function suite(...lines) { + return HEAD + 'cases:\n' + lines.map((one) => ` ${one}\n`).join('') +} + +// The one case a file of one case holds. +function only(text) { + const held = read(text) + assert.equal(held.cases.length, 1, 'the file held more than one case') + return held.cases[0] +} + +// The message a file was refused with, and a failure when it was read. +function refused(text) { + try { + read(text) + assert.fail('this file was read, and it should have been refused') + } catch (err) { + assert.ok(err instanceof CorpusError, `read gave a ${err.name}, and everything here is a CorpusError`) + return err.message + } +} + +test('a suite is a header, an optional load and its cases', () => { + const held = read( + HEAD + + ['cases:', ' - name: one', ' doc: the first', ' query: RETURN 1', ' columns:', ' - n', + ' rows:', ' - values:', ' - type: INT64', ' value: "1"', + ' - name: two', ' doc: the second', ' query: RETURN 2', ' raises: 42601', ''].join('\n'), + ) + assert.equal(held.name, 'example') + assert.equal(held.doc, 'what this suite is for') + assert.equal(held.load, null, 'a suite with no `load:` came back with one') + assert.deepEqual(held.cases.map((one) => one.name), ['one', 'two']) + assert.deepEqual(held.cases[0].columns, ['n']) + assert.ok(same([[1n]], held.cases[0].rows)) + assert.equal(held.cases[0].raises, '') + assert.equal(held.cases[1].raises, '42601') + assert.equal(held.cases[1].hasColumns, false) + // The line a report cites is the line the case opened on, which is what + // makes a failure something to open a file at. + assert.equal(held.cases[0].line, 5) + assert.equal(held.cases[1].line, 14) +}) + +// The version exists so that a corpus unpacked from an old release says +// what it is instead of failing somewhere in the middle of a suite. +test('a file says which schema it is and is refused when it is another', () => { + assert.equal( + refused(`schema: ${SCHEMA + 1}\nsuite: example\ndoc: d\ncases:\n - name: one\n`), + `this is schema ${SCHEMA + 1} and the runner reads schema ${SCHEMA}`, + ) + assert.equal(refused('suite: example\ndoc: d\n'), 'the file does not open with `schema:`') + assert.equal(refused('schema: four\nsuite: example\ndoc: d\n'), '"four" is not a schema version') +}) + +test('a suite says what it is and what is in it', () => { + for (const [what, text, want] of [ + ['no suite name', `schema: ${SCHEMA}\ndoc: d\n`, 'line 1: no `suite:`'], + ['no doc', `schema: ${SCHEMA}\nsuite: example\n`, 'line 1: no `doc:`'], + ['no cases', HEAD, 'a suite with no `cases:`'], + ['an empty cases', `${HEAD}cases:\n`, '`cases:` is a sequence'], + ['a key nothing knows', `${HEAD}extra: 1\n`, 'line 1: a suite has no key "extra"'], + ]) { + assert.equal(refused(text), want, `${what} was refused with the wrong words`) + } +}) + +test('a case says what it is, what it runs and what that produces', () => { + const body = ['doc: d', 'query: RETURN 1', 'columns:', ' - n', 'rows:'] + for (const [what, lines, want] of [ + ['no name', ['- doc: d', ' query: RETURN 1'], 'line 5: no `name:`'], + ['no doc', ['- name: one', ' query: RETURN 1'], 'line 5: no `doc:`'], + ['no query', ['- name: one', ' doc: d'], 'line 5: no `query:`'], + [ + 'no expectation at all', + ['- name: one', ' doc: d', ' query: RETURN 1'], + 'line 5: a case says what it produces, with `columns:` and `rows:` or with `raises:`', + ], + [ + 'both an expectation and a condition', + ['- name: one', ...body.map((l) => ` ${l}`), ' raises: 42601'], + 'line 5: a case that raises has no rows, and one that returns rows does not raise', + ], + [ + 'columns with no rows', + ['- name: one', ' doc: d', ' query: RETURN 1', ' columns:', ' - n'], + 'line 5: `columns:` with no `rows:`. A case expecting nothing back writes `rows:` with an ' + + 'empty sequence under it.', + ], + ['a key nothing knows', ['- name: one', ' doc: d', ' qeury: RETURN 1'], 'line 5: a case has no key "qeury"'], + [ + 'a case that is not a mapping', + ['- just a scalar'], + 'line 5: a case is a mapping, and this is a scalar', + ], + ]) { + assert.equal(refused(suite(...lines)), want, `${what} was refused with the wrong words`) + } +}) + +// A name is what a report cites and what a binding's skip list names, so +// it is spelled one way and only one way. +test('a name is lower case words joined by dashes', () => { + for (const name of ['one', 'a-b-c', 'utf8-length', 'a1']) { + assert.equal(only(suite(`- name: ${name}`, ' doc: d', ' query: RETURN 1', ' raises: 42601')).name, name) + } + for (const name of ['One', 'a_b', 'a b', 'a.b', "'"]) { + assert.equal( + refused(suite(`- name: ${quote(name)}`, ' doc: d', ' query: RETURN 1', ' raises: 42601')), + `line 5: ${quote(name)} is a case name, which is lower case words joined by dashes`, + name, + ) + } +}) + +test('two cases of one name are refused', () => { + const one = ['- name: one', ' doc: d', ' query: RETURN 1', ' raises: 42601'] + assert.equal(refused(suite(...one, ...one)), 'two cases are called "one"') +}) + +// The code and not the message, because the code is the contract and the +// message is prose that will improve. +test('a condition is a GQLSTATUS and is checked for its shape', () => { + assert.equal(only(suite('- name: one', ' doc: d', ' query: RETURN 1', ' raises: 42601')).raises, '42601') + for (const code of ['4260', '426011', '4260a', 'syntax error', '42-01']) { + assert.equal( + refused(suite('- name: one', ' doc: d', ' query: RETURN 1', ` raises: ${quote(code)}`)), + `line 8: ${quote(code)} is not the shape of a GQLSTATUS, which is five characters of digits ` + + 'and capitals', + code, + ) + } +}) + +// FINISH answers no columns at all, which is not the same as a query +// whose columns held no rows, so both spellings have to read. +test('a case with no columns and a case with no rows are both cases', () => { + const noColumns = only(suite('- name: one', ' doc: d', ' query: FINISH', ' columns:', ' rows:')) + assert.equal(noColumns.hasColumns, true, 'an empty `columns:` was taken for an absent one') + assert.deepEqual(noColumns.columns, []) + assert.deepEqual(noColumns.rows, []) + + const noRows = only(suite('- name: one', ' doc: d', ' query: RETURN 1', ' columns:', ' - n', ' rows:')) + assert.deepEqual(noRows.columns, ['n']) + assert.deepEqual(noRows.rows, []) +}) + +test('a row that does not fit its columns is refused where it is written', () => { + assert.equal( + refused( + suite( + '- name: one', ' doc: d', ' query: RETURN 1, 2', ' columns:', ' - a', ' - b', + ' rows:', ' - values:', ' - type: INT64', ' value: "1"', + ), + ), + 'line 5: a row of 1 against 2 columns', + ) + for (const [what, lines, want] of [ + [ + 'a row that is not a values', + ['- name: one', ' doc: d', ' query: RETURN 1', ' columns:', ' - a', ' rows:', ' - 1'], + 'line 11: a row is a `values:` and the values under it', + ], + [ + 'a row with a key nothing knows', + ['- name: one', ' doc: d', ' query: RETURN 1', ' columns:', ' - a', ' rows:', ' - value:'], + 'line 11: a row has no key "value"', + ], + ]) { + assert.equal(refused(suite(...lines)), want, `${what} was refused with the wrong words`) + } +}) + +// A statement runs on `main` unless the case says otherwise, which is +// every case but the handful about transactions. Those need a second +// connection, because a transaction is only observable from outside it. +test('a case runs on main unless it names a connection', () => { + const plain = only(suite('- name: one', ' doc: d', ' query: RETURN 1', ' raises: 42601')) + assert.equal(plain.on, MAIN) + assert.deepEqual(plain.setup, []) + + const held = only( + suite( + '- name: one', ' doc: d', ' setup:', ' - CREATE NODE TABLE person (id INT64)', + ' - on: other', ' query: INSERT (:person {id: 1})', ' on: other', + ' query: RETURN 1', ' raises: 42601', + ), + ) + assert.equal(held.on, 'other') + assert.deepEqual(held.setup, [ + { on: MAIN, query: 'CREATE NODE TABLE person (id INT64)' }, + { on: 'other', query: 'INSERT (:person {id: 1})' }, + ]) +}) + +test('a setup statement is a line or a connection and a line', () => { + for (const [what, lines, want] of [ + [ + 'a mapping with no connection on it', + ['- name: one', ' doc: d', ' setup:', ' - query: RETURN 1', ' query: RETURN 1', ' raises: 42601'], + 'line 8: a setup statement written as a mapping names the connection it runs on', + ], + [ + 'a mapping with a key nothing knows', + ['- name: one', ' doc: d', ' setup:', ' - on: other', ' qeury: RETURN 1', ' query: RETURN 1', + ' raises: 42601'], + 'line 8: a setup statement has no key "qeury"', + ], + [ + 'a setup that is not a sequence', + ['- name: one', ' doc: d', ' setup: RETURN 1', ' query: RETURN 1', ' raises: 42601'], + 'line 5: `setup:` is a sequence of statements', + ], + [ + 'a connection named the wrong way', + ['- name: one', ' doc: d', ' on: Other', ' query: RETURN 1', ' raises: 42601'], + 'line 7: "Other" is a connection name, which is lower case words joined by dashes', + ], + ]) { + assert.equal(refused(suite(...lines)), want, `${what} was refused with the wrong words`) + } +}) + +// The other direction the encoding travels. A client that decodes a date +// correctly and encodes it a day early passes every case with no +// parameters in it, so the cases with parameters are the ones that catch +// it. +test('a parameter is a name and a value in the encoding', () => { + const held = only( + suite( + '- name: one', ' doc: d', ' params:', ' - name: n', ' type: INT64', ' value: "1"', + ' - name: s', ' type: STRING', ' value: text', ' query: RETURN $n', ' raises: 42601', + ), + ) + assert.deepEqual(held.params.map((one) => one.name), ['n', 's']) + assert.ok(same(1n, held.params[0].value), `n came to ${show(held.params[0].value)}`) + assert.ok(same('text', held.params[1].value)) + + for (const [what, lines, want] of [ + [ + 'a name a statement could not write', + ['- name: one', ' doc: d', ' params:', ' - name: n one', ' type: INT64', ' value: "1"', + ' query: RETURN $n', ' raises: 42601'], + 'line 8: "n one" is a parameter name, which is what a statement writes after the `$`', + ], + [ + 'two parameters of one name', + ['- name: one', ' doc: d', ' params:', ' - name: n', ' type: INT64', ' value: "1"', + ' - name: n', ' type: INT64', ' value: "2"', ' query: RETURN $n', ' raises: 42601'], + 'line 11: two parameters are called "n"', + ], + [ + 'a parameter with a key nothing knows', + ['- name: one', ' doc: d', ' params:', ' - nmae: n', ' type: INT64', ' value: "1"', + ' query: RETURN $n', ' raises: 42601'], + 'line 8: a parameter has no key "nmae"', + ], + [ + 'a params that is not a sequence', + ['- name: one', ' doc: d', ' params: n', ' query: RETURN $n', ' raises: 42601'], + 'line 7: `params:` is a sequence', + ], + ]) { + assert.equal(refused(suite(...lines)), want, `${what} was refused with the wrong words`) + } +}) + +// Everything else in the corpus is an expression, and an expression says +// what a value means on the way out and nothing about how it got in. A +// load is the other half. +test('a load is a table, its columns and the edges between its rows', () => { + const held = read( + HEAD + + ['load:', ' nodes: person', ' edges: knows', ' count: 2', ' columns:', ' - name: id', + ' type: INT64', ' values:', ' - "1"', ' - "2"', ' - name: tag', + ' type: STRING', ' values:', ' - a', ' - b', ' pairs:', ' - from: 0', + ' to: 1', 'cases:', ' - name: one', ' doc: d', ' query: RETURN 1', + ' raises: 42601', ''].join('\n'), + ) + const load = held.load + assert.equal(load.nodes, 'person') + assert.equal(load.edges, 'knows') + assert.equal(load.count, 2) + assert.deepEqual(load.columns.map((one) => [one.name, one.type]), [['id', 'INT64'], ['tag', 'STRING']]) + assert.ok(same([1n, 2n], load.columns[0].values)) + assert.ok(same(['a', 'b'], load.columns[1].values)) + assert.deepEqual(load.pairs, [[0, 1]]) +}) + +test('a load that does not add up is refused where it is written', () => { + const head = ['load:', ' nodes: person', ' edges: knows', ' count: 2'] + const tail = ['cases:', ' - name: one', ' doc: d', ' query: RETURN 1', ' raises: 42601'] + const file = (...lines) => `${HEAD}${[...head, ...lines, ...tail].join('\n')}\n` + const column = [' columns:', ' - name: id', ' type: INT64', ' values:', ' - "1"', ' - "2"'] + + for (const [what, lines, want] of [ + [ + 'a column with the wrong number of values', + [' columns:', ' - name: id', ' type: INT64', ' values:', ' - "1"'], + 'line 9: column "id" holds 1 values against the 2 rows the load declares', + ], + [ + 'a type nothing knows', + [' columns:', ' - name: id', ' type: INTEGER', ' values:', ' - 1', ' - 2'], + 'line 9: INTEGER is not a type this encoding knows', + ], + ['no columns at all', [], 'line 5: a load has `columns:`'], + [ + 'two columns of one name', + [...column, ' - name: id', ' type: STRING', ' values:', ' - a', ' - b'], + 'line 5: two columns are called "id"', + ], + [ + 'an edge past the end of the table', + [...column, ' pairs:', ' - from: 0', ' to: 2'], + 'line 15: `to: 2` against a table of 2 rows, which are numbered 0 to 1', + ], + [ + 'an edge with one end', + [...column, ' pairs:', ' - from: 0'], + 'line 15: an edge has a `to:` row number', + ], + [ + 'an edge with a key nothing knows', + [...column, ' pairs:', ' - form: 0', ' to: 1'], + 'line 15: an edge has no key "form"', + ], + [ + 'a load with a key nothing knows', + [...column, ' paris:', ' - from: 0'], + 'line 5: a load has no key "paris"', + ], + ]) { + assert.equal(refused(file(...lines)), want, `${what} was refused with the wrong words`) + } + + // A load of no rows is a load nothing can be read back from, which is a + // suite whose every case would pass by returning nothing. + assert.equal( + refused( + `${HEAD}${['load:', ' nodes: person', ' edges: knows', ' count: 0', ...column, ...tail].join('\n')}\n`, + ), + 'line 5: a load of no rows is a load nothing can be read back from', + ) + assert.equal( + refused(`${HEAD}${['load:', ' nodes: person', ' edges: knows', ...column, ...tail].join('\n')}\n`), + 'line 5: a load says how many rows it has, with `count:`', + ) + // A name a statement could not write, since the load goes in through a + // table the cases then name in their statements. + assert.equal( + refused( + `${HEAD}${['load:', ' nodes: a person', ' edges: knows', ' count: 2', ...column, ...tail].join('\n')}\n`, + ), + 'line 5: "a person" is not a table name', + ) +}) + +// A case may say what the export gives as well as what the rows are, and +// the two are checked against one statement. +test('a case may say what its export gives', () => { + const held = only( + suite( + '- name: one', ' doc: d', ' query: RETURN 1 AS n', ' columns:', ' - n', ' rows:', + ' arrow:', ' - name: n', ' format: l', + ), + ) + assert.deepEqual(held.arrow, { refused: false, fields: [{ name: 'n', format: 'l', children: [] }] }) + + // A result Arrow has no type for, which is a contract of its own and + // not a case that has nothing to say. + const no = only(suite('- name: one', ' doc: d', ' query: RETURN 1', ' columns:', ' - n', ' rows:', + ' arrow: refused')) + assert.deepEqual(no.arrow, { refused: true, fields: [] }) + + // A list names the field under it, rather than leaving it implied, so + // that a client naming it something else is a case that fails. + const nested = only( + suite( + '- name: one', ' doc: d', ' query: RETURN [1] AS l', ' columns:', ' - l', ' rows:', + ' arrow:', ' - name: l', ' format: +l', ' children:', ' - name: item', + ' format: l', + ), + ) + assert.deepEqual(nested.arrow.fields[0].children, [{ name: 'item', format: 'l', children: [] }]) + + for (const [what, lines, want] of [ + [ + 'a nested type with nothing under it', + ['- name: one', ' doc: d', ' query: RETURN [1] AS l', ' columns:', ' - l', ' rows:', + ' arrow:', ' - name: l', ' format: +l'], + 'line 12: "+l" is a nested type and the fields under it are part of it', + ], + [ + 'a flat type with fields under it', + ['- name: one', ' doc: d', ' query: RETURN 1 AS n', ' columns:', ' - n', ' rows:', + ' arrow:', ' - name: n', ' format: l', ' children:', ' - name: item', + ' format: l'], + 'line 12: "l" holds no fields, so nothing goes under it', + ], + [ + 'a word that is not refused', + ['- name: one', ' doc: d', ' query: RETURN 1', ' columns:', ' - n', ' rows:', ' arrow: no'], + 'line 11: `arrow:` is the columns the export gives, or `refused` for a result Arrow has no ' + + 'type for, and this is "no"', + ], + [ + 'a field with no format', + ['- name: one', ' doc: d', ' query: RETURN 1 AS n', ' columns:', ' - n', ' rows:', + ' arrow:', ' - name: n'], + 'line 12: an Arrow field has a `format:`', + ], + [ + 'a field with a key nothing knows', + ['- name: one', ' doc: d', ' query: RETURN 1 AS n', ' columns:', ' - n', ' rows:', + ' arrow:', ' - name: n', ' fromat: l'], + 'line 12: an Arrow field has no key "fromat"', + ], + ]) { + assert.equal(refused(suite(...lines)), want, `${what} was refused with the wrong words`) + } +}) + +// A directory of files, which is what the runner is handed. The file name +// and the suite name have to agree, because a report cites the suite and +// somebody looking for it opens the file. +test('a directory reads in a fixed order and the names have to agree', async (t) => { + const dir = await mkdtemp(join(tmpdir(), 'zu-cases-')) + t.after(() => rm(dir, { recursive: true, force: true })) + const one = (name) => + `schema: ${SCHEMA}\nsuite: ${name}\ndoc: d\ncases:\n - name: one\n doc: d\n query: RETURN 1\n` + + ' raises: 42601\n' + + await writeFile(join(dir, 'beta.yaml'), one('beta')) + await writeFile(join(dir, 'alpha.yaml'), one('alpha')) + // Not a case file, and not an error either, since a directory of cases + // has a README in it. + await writeFile(join(dir, 'README.md'), 'not a suite\n') + + const suites = await readDir(dir) + assert.deepEqual(suites.map((held) => held.name), ['alpha', 'beta']) + + await writeFile(join(dir, 'gamma.yaml'), one('delta')) + await assert.rejects(readDir(dir), (err) => { + assert.ok(err instanceof CorpusError) + assert.equal(err.message, `${join(dir, 'gamma.yaml')}: the suite calls itself "delta" and the file calls it "gamma"`) + return true + }) +}) + +test('a directory with no case files in it is refused', async (t) => { + const dir = await mkdtemp(join(tmpdir(), 'zu-cases-')) + t.after(() => rm(dir, { recursive: true, force: true })) + await assert.rejects(readDir(dir), (err) => { + assert.equal(err.message, `${dir}: no case files`) + return true + }) + const missing = join(dir, 'nowhere') + await assert.rejects(readDir(missing), (err) => { + assert.ok(err instanceof CorpusError) + assert.ok(err.message.startsWith(`${missing}: `), err.message) + return true + }) +}) + +// A refusal inside a file carries the file with it, since a report of +// twenty five suites that says "line 4" and nothing else is one nobody +// can act on. +test('a refusal inside a file says which file', async (t) => { + const dir = await mkdtemp(join(tmpdir(), 'zu-cases-')) + t.after(() => rm(dir, { recursive: true, force: true })) + const path = join(dir, 'broken.yaml') + await writeFile(path, `schema: ${SCHEMA}\nsuite: broken\ndoc: d\ncases:\n - name: One\n doc: d\n query: RETURN 1\n raises: 42601\n`) + await assert.rejects(readDir(dir), (err) => { + assert.equal( + err.message, + `${path}: line 5: "One" is a case name, which is lower case words joined by dashes`, + ) + return true + }) +}) diff --git a/test/conformance-reader.test.mjs b/test/conformance-reader.test.mjs new file mode 100644 index 0000000..5e2863a --- /dev/null +++ b/test/conformance-reader.test.mjs @@ -0,0 +1,264 @@ +// The corpus reader, tested against the subset it claims to read and +// against the constructs it claims to refuse. +// +// Both halves matter and the second one is the reason this file is long. +// A reader that accepts everything the corpus writes is half a reader: +// the other half is that a case using a block scalar, an anchor or a +// flow sequence is refused with a line number rather than read as +// something the author did not write. Five implementations of this +// subset exist and a construct one of them quietly accepts is a case +// that passes in one repository and fails in four. +// +// The refusal messages are checked in full rather than by substring, +// because they are diffed against the reference runner's and a wording +// that drifted would be a difference in the report that is not a +// difference in the answer. + +import assert from 'node:assert/strict' +import test from 'node:test' + +import { CorpusError, EMPTY, parse, quote } from '../conformance/reader.mjs' + +// The message a document was refused with, and a failure when it was not +// refused at all. +function refused(text) { + try { + const node = parse(text) + assert.fail(`parse read this as ${node.what()}, and it should have been refused`) + } catch (err) { + assert.ok(err instanceof CorpusError, `parse gave a ${err.name}, and everything here is a CorpusError`) + return err.message + } +} + +test('a mapping keeps its keys in the order they were written', () => { + const doc = parse('suite: string\ndoc: what a string does\nschema: 4\n') + const pairs = doc.map() + assert.ok(pairs !== null, `the document is ${doc.what()}, and it should be a mapping`) + assert.deepEqual(pairs.map((pair) => pair.key), ['suite', 'doc', 'schema']) + assert.equal(doc.get('doc').str(), 'what a string does') + assert.equal(doc.get('load'), null, 'get answered a key that is not in the mapping') +}) + +test('a sequence of mappings is one node per item', () => { + const items = parse( + ['cases:', ' - name: one', ' query: RETURN 1', ' - name: two', ' query: RETURN 2', ''].join('\n'), + ) + .get('cases') + .seq() + assert.equal(items.length, 2) + assert.deepEqual(items.map((item) => item.get('name').str()), ['one', 'two']) + // The line a refusal would cite is the line the item opened on and not + // the line the sequence did, which is the whole reason a node carries + // one. + assert.equal(items[1].line, 4) +}) + +// A `- ` and the key it opens are one line in the file and two lines by +// the time the parser sees them, and the split is what lets an item +// written on one line and an item written under its dash be the same +// shape. Both spellings are in the corpus. +test('a dash and its first key may share a line', () => { + const together = parse('cases:\n - name: one\n query: RETURN 1\n') + const apart = parse('cases:\n -\n name: one\n query: RETURN 1\n') + for (const doc of [together, apart]) { + const items = doc.get('cases').seq() + assert.equal(items.length, 1) + assert.equal(items[0].get('name').str(), 'one') + } +}) + +test('a scalar remembers whether it was quoted', () => { + const doc = parse('bare: 42\nsingle: \'42\'\ndouble: "42"\n') + for (const [key, quoted] of [ + ['bare', false], + ['single', true], + ['double', true], + ]) { + const held = doc.get(key).scalar() + assert.ok(held !== null, `${key} is ${doc.get(key).what()}`) + assert.equal(held.text, '42') + assert.equal(held.quoted, quoted, `${key} came back the other way round`) + } +}) + +test('a single quoted run escapes only by doubling the quote', () => { + const doc = parse("query: 'RETURN ''it''''s'' AS s'\n") + const held = doc.get('query').scalar() + assert.equal(held.quoted, true) + assert.equal(held.text, "RETURN 'it''s' AS s") + // A backslash inside a single quoted run is a backslash, which is what + // lets a case write a regular expression without doubling every one of + // them. + assert.equal(parse("query: 'a\\nb'\n").get('query').str(), 'a\\nb') +}) + +test('a double quoted run takes the escapes the corpus uses', () => { + const doc = parse('text: "a\\nb\\tc\\\\d\\"e\\r\\0f\\bg"\n') + assert.equal(doc.get('text').str(), 'a\nb\tc\\d"e\r\x00f\bg') +}) + +// A comment is dropped, and the three rules that keep the dropping from +// eating content are each worth a line: a # inside a word is part of the +// word, a quote inside a word is part of the word, and a quote that +// opens nothing that closes was not a run. +test('a comment goes and a hash inside a value stays', () => { + for (const [text, key, want] of [ + ['a: 1 # why\n', 'a', '1'], + ['# whole line\nb: 2\n', 'b', '2'], + ['c: person#1\n', 'c', 'person#1'], + ["d: 'a # b'\n", 'd', 'a # b'], + ["e: it's a plain scalar # and a comment\n", 'e', "it's a plain scalar"], + ["f: cast(' 42 ' AS INT64)\n", 'f', "cast(' 42 ' AS INT64)"], + ["g: RETURN 'a' AS a # a comment after a run that closed\n", 'g', "RETURN 'a' AS a"], + ]) { + assert.equal(parse(text).get(key).str(), want, `${quote(text)} read wrongly`) + } +}) + +// A key with nothing under it is a node rather than an error, because a +// case that expects no rows writes `rows:` and stops. Every accessor +// says no to it, so a `name:` somebody left blank is still caught. +test('a key with nothing under it is an empty node', () => { + const doc = parse('rows:\nname: after\n') + const empty = doc.get('rows') + assert.equal(empty.kind, EMPTY) + assert.equal(empty.what(), 'nothing') + assert.equal(empty.str(), null) + assert.equal(empty.seq(), null) + assert.equal(empty.map(), null) + assert.deepEqual(empty.seqOrEmpty(), []) + // The key after it is still read, so an empty value ends at its own + // line rather than swallowing what follows. + assert.equal(doc.get('name').str(), 'after') +}) + +test('unknown names the keys that are not expected', () => { + const doc = parse('name: one\nquery: RETURN 1\nqeury: RETURN 2\nrows:\n') + assert.deepEqual(doc.unknown('name', 'query', 'rows'), ['qeury']) + assert.deepEqual(doc.unknown('name', 'query', 'qeury', 'rows'), []) + // A scalar has no keys and is not a mapping, so it has no unknown ones + // either rather than being an error at this level. + assert.deepEqual(parse('just a scalar\n').unknown('name'), []) +}) + +test('what says which shape a node is', () => { + for (const [text, want] of [ + ['a plain scalar\n', 'a scalar'], + ['- one\n- two\n', 'a sequence'], + ['key: value\n', 'a mapping'], + ]) { + assert.equal(parse(text).what(), want) + } +}) + +// The constructs this reader will not read. Every one of them is real +// YAML that a general reader would take, and every one of them would +// mean a case says one thing to a reviewer and another to the runner. +test('the constructs this reader does not read', () => { + for (const [what, text, want] of [ + ['a tab', 'cases:\n\t- name: one\n', 'line 2: a tab at column 1, and indentation here is spaces'], + ['a document marker', '---\nschema: 4\n', 'line 1: "---" opens or closes a document, and a file here holds one'], + ['a document terminator', 'schema: 4\n...\n', 'line 2: "..." opens or closes a document, and a file here holds one'], + ['an odd indent', 'cases:\n - name: one\n', 'line 2: indented 3, and indentation here goes two spaces at a time'], + [ + 'a dash with two spaces after it', + 'cases:\n - name: one\n', + 'line 2: a `- ` takes exactly one space, so that what follows it lines up with the lines under it', + ], + [ + 'a sequence opening into a sequence', + 'cases:\n - - one\n', + 'line 2: a sequence opening straight into another one, which nothing here needs', + ], + ['a dash with nothing after it', 'cases:\n -\n', 'line 2: a `-` with nothing after it'], + [ + 'a flow sequence', + 'columns: [a, b]\n', + "line 1: a plain scalar opening with '[', which is a construct this reader does not read", + ], + [ + 'a flow mapping', + 'value: {type: INT64}\n', + "line 1: a plain scalar opening with '{', which is a construct this reader does not read", + ], + [ + 'an anchor', + 'row: &base one\n', + "line 1: a plain scalar opening with '&', which is a construct this reader does not read", + ], + [ + 'an alias', + 'row: *base\n', + "line 1: a plain scalar opening with '*', which is a construct this reader does not read", + ], + [ + 'a tag', + 'count: !!int 4\n', + "line 1: a plain scalar opening with '!', which is a construct this reader does not read", + ], + [ + 'a literal block scalar', + 'doc: |\n one\n', + "line 1: a plain scalar opening with '|', which is a construct this reader does not read", + ], + [ + 'a folded block scalar', + 'doc: >\n one\n', + "line 1: a plain scalar opening with '>', which is a construct this reader does not read", + ], + [ + 'a directive', + 'query: %YAML 1.2\n', + "line 1: a plain scalar opening with '%', which is a construct this reader does not read", + ], + ['a run that does not close', 'query: "RETURN 1\n', 'line 1: a " that opens and does not close on its line'], + ['two runs on one line', 'query: "a" and "b"\n', 'line 1: " and \\"b\\"" after the scalar ends'], + // The backslash escapes the quote that would have closed the run, so + // this is reported as a run left open rather than as a scalar ending + // in a backslash. Both messages are in the reader and this is the one + // that is reachable, since a backslash before the closing quote + // always takes the quote with it. + [ + 'a double quoted run whose last character escapes its quote', + 'query: "a\\"\n', + 'line 1: a " that opens and does not close on its line', + ], + ['an escape this reader has no rule for', 'query: "a\\x41b"\n', 'line 1: \\x is not an escape'], + ['a key set twice', 'name: one\nname: two\n', 'line 2: name is set twice in one mapping'], + [ + 'an indent under a key that is not two', + 'load:\n nodes: person\n', + 'line 2: indented 4, where what is under `load:` on line 1 is indented 2', + ], + [ + 'an indent under a dash that is not two', + 'cases:\n -\n name: one\n', + 'line 3: indented 6, where an item of the sequence on line 2 is indented 4', + ], + ['a first line that is indented', ' schema: 4\n', 'line 1: the first line is indented'], + ['a file with nothing in it', '# only a comment\n\n', 'the file has nothing in it'], + ['a line belonging to nothing above it', 'just a scalar\nand another\n', 'line 2: this belongs to nothing above it'], + ]) { + assert.equal(refused(text), want, `${what} was refused with the wrong words`) + } +}) + +// quote is Rust's {:?} and not JSON, because a refusal written in five +// languages and diffed across them cannot have one of them escaping a +// code point the others print. +test('quote writes a string the way the other runners do', () => { + for (const [text, want] of [ + ['plain', '"plain"'], + ['a "quoted" word', '"a \\"quoted\\" word"'], + ['a\\backslash', '"a\\\\backslash"'], + ['a\nb', '"a\\nb"'], + ['a\rb', '"a\\rb"'], + ['a\tb', '"a\\tb"'], + // The one JSON.stringify would escape and Rust would not. + ['héllo → 世界', '"héllo → 世界"'], + ['', '""'], + ]) { + assert.equal(quote(text), want) + } +}) diff --git a/test/conformance-runner.test.mjs b/test/conformance-runner.test.mjs new file mode 100644 index 0000000..e3ee022 --- /dev/null +++ b/test/conformance-runner.test.mjs @@ -0,0 +1,481 @@ +// The runner: what it does with a case, and what it says about one that +// did not do what the case said it would. +// +// Two halves. The first is the report, which is strings and counting and +// is diffed against the reference runner's output, so it is checked word +// for word. The second runs cases against this client for real, and is +// written to turn on the runner's own behaviour rather than on the +// engine's: a case here asserts the wrong answer on purpose and the test +// is that the runner noticed and said the right thing about it. Where a +// GQLSTATUS the engine chooses would otherwise be baked in, the test +// checks the shape of the line and not the code, because the code is the +// engine's to change and this file is about the runner. +// +// The corpus itself is run by conformance.test.mjs, which needs the case +// files and skips without them. This one needs nothing but the addon. + +import assert from 'node:assert/strict' +import { mkdtemp, readdir, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' + +import { read } from '../conformance/cases.mjs' +import { FAILED, PASSED, UNSUPPORTED, count, line, mark, run, summary } from '../conformance/runner.mjs' + +const HEAD = 'schema: 4\nsuite: example\ndoc: what this suite is for\n' + +// A suite of the cases given, each one a block of lines. +function suite(...cases) { + return read(HEAD + 'cases:\n' + cases.flat().map((one) => ` ${one}\n`).join('')) +} + +// A load, written once and put in front of a suite that needs one. +const LOAD = [ + 'load:', + ' nodes: person', + ' edges: knows', + ' count: 2', + ' columns:', + ' - name: id', + ' type: INT64', + ' values:', + ' - "1"', + ' - "2"', + ' - name: score', + ' type: FLOAT64', + ' values:', + ' - "1.0"', + ' - "2.5"', + ' - name: tag', + ' type: STRING', + ' values:', + ' - a', + ' - b', + ' pairs:', + ' - from: 0', + ' to: 1', +] + +// A directory to make the case databases under, gone when the test ends. +async function work(t) { + const dir = await mkdtemp(join(tmpdir(), 'zu-runner-')) + t.after(() => rm(dir, { recursive: true, force: true })) + return dir +} + +// Runs a suite and gives back its cases by name, so that a test names +// what it is asserting rather than counting positions. +async function ranBy(t, held) { + const ran = await run([held], await work(t)) + const by = new Map(ran.map((one) => [one.case, one])) + assert.equal(by.size, ran.length, 'two cases came back under one name') + return by +} + +test('a report spells an outcome the way the reference runner does', () => { + assert.equal(mark(PASSED), 'ok') + assert.equal(mark(FAILED), 'FAILED') + assert.equal(mark(UNSUPPORTED), 'unsupported') +}) + +test('a line is the suite, the case, the line and what happened', () => { + const one = { suite: 'string', case: 'utf8-length', line: 42, outcome: PASSED, detail: '' } + assert.equal(line(one), 'string/utf8-length line 42 ok') + // The detail goes after a colon, and a case with none has no colon at + // all rather than one with nothing behind it. + assert.equal( + line({ ...one, outcome: FAILED, detail: '1 rows where the case wants 2' }), + 'string/utf8-length line 42 FAILED: 1 rows where the case wants 2', + ) + assert.equal( + line({ ...one, outcome: UNSUPPORTED, detail: '42601 syntax error' }), + 'string/utf8-length line 42 unsupported: 42601 syntax error', + ) +}) + +test('a summary counts each outcome and the run as a whole', () => { + const ran = [ + { outcome: PASSED }, { outcome: PASSED }, { outcome: FAILED }, { outcome: UNSUPPORTED }, + ] + assert.equal(count(ran, PASSED), 2) + assert.equal(count(ran, FAILED), 1) + assert.equal(count(ran, UNSUPPORTED), 1) + assert.equal(summary(ran), '4 cases, 2 passed, 1 failed, 1 unsupported') + assert.equal(summary([]), '0 cases, 0 passed, 0 failed, 0 unsupported') +}) + +// From here down the addon runs statements for real. + +test('a case whose answer is what the engine gives passes', async (t) => { + const by = await ranBy( + t, + suite([ + '- name: one', + ' doc: a value comes back as itself', + ' query: RETURN 1 AS n', + ' columns:', + ' - n', + ' rows:', + ' - values:', + ' - type: INT64', + ' value: "1"', + ]), + ) + const one = by.get('one') + assert.equal(one.outcome, PASSED, one.detail) + assert.equal(one.detail, '') + assert.equal(one.suite, 'example') + assert.equal(one.line, 5) +}) + +// The wordings a failing case prints. Every one of these is driven by a +// statement the engine certainly answers, so what is under test is the +// runner's comparison and not the engine's opinion. +test('a case whose answer differs says what differs and stops there', async (t) => { + const by = await ranBy( + t, + suite( + [ + '- name: wrong-value', + ' doc: the value is not the one the engine gives', + ' query: RETURN 1 AS n', + ' columns:', + ' - n', + ' rows:', + ' - values:', + ' - type: INT64', + ' value: "2"', + ], + [ + '- name: wrong-name', + ' doc: the column is not the one the engine names', + ' query: RETURN 1 AS n', + ' columns:', + ' - m', + ' rows:', + ' - values:', + ' - type: INT64', + ' value: "1"', + ], + [ + '- name: wrong-count', + ' doc: there are fewer rows than the case wants', + ' query: RETURN 1 AS n', + ' columns:', + ' - n', + ' rows:', + ' - values:', + ' - type: INT64', + ' value: "1"', + ' - values:', + ' - type: INT64', + ' value: "1"', + ], + [ + '- name: wrong-type', + ' doc: the value is the right number under the wrong type', + ' query: RETURN 1 AS n', + ' columns:', + ' - n', + ' rows:', + ' - values:', + ' - type: FLOAT64', + ' value: "1.0"', + ], + ), + ) + for (const [name, want] of [ + ['wrong-value', 'row 1 column n is INT64 "1" where the case wants INT64 "2"'], + ['wrong-name', 'columns ["n"] where the case wants ["m"]'], + ['wrong-count', '1 rows where the case wants 2'], + // The type is part of the answer, so a whole 1 that came back as an + // integer does not satisfy a case wanting a float. This is the one + // comparison a runner written with == would get wrong in four + // languages out of five. + ['wrong-type', 'row 1 column n is INT64 "1" where the case wants FLOAT64 "1.0"'], + ]) { + const one = by.get(name) + assert.equal(one.outcome, FAILED, `${name} did not fail`) + assert.equal(one.detail, want, name) + } +}) + +// A condition is checked by its code. The code the engine picks for a +// broken statement is the engine's to change, so what is asserted here +// is that the runner compared the code it was given and reported the +// comparison, not which code that was. +test('a case that raises is checked against the code and not the message', async (t) => { + const by = await ranBy( + t, + suite( + [ + '- name: rows-where-a-condition-was-wanted', + ' doc: the statement answered where the case wants it to fail', + ' query: RETURN 1 AS n', + ' raises: 22003', + ], + [ + '- name: another-code', + ' doc: the statement failed with a code that is not the one wanted', + ' query: THIS IS NOT A STATEMENT', + ' raises: 22003', + ], + ), + ) + const rows = by.get('rows-where-a-condition-was-wanted') + assert.equal(rows.outcome, FAILED) + assert.equal(rows.detail, 'returned rows where the case wants 22003') + + const other = by.get('another-code') + assert.equal(other.outcome, FAILED, other.detail) + assert.match( + other.detail, + /^raised [0-9A-Z]{5} where the case wants 22003: /, + 'a mismatched condition did not print the code it got and the code it wanted', + ) +}) + +// A case the engine has not caught up to is not a failure. The corpus is +// the contract and the engine catches up to it, so a statement it cannot +// parse or has not implemented comes back as unsupported and a release +// branch is what decides whether that is allowed. +test('a case ahead of the engine is unsupported rather than failed', async (t) => { + const by = await ranBy( + t, + suite([ + '- name: ahead', + ' doc: a statement the engine does not parse', + ' query: THIS IS NOT A STATEMENT', + ' columns:', + ' - n', + ' rows:', + ]), + ) + const one = by.get('ahead') + assert.equal(one.outcome, UNSUPPORTED, one.detail) + assert.notEqual(one.detail, '', 'an unsupported case said nothing about why') +}) + +// The other kind of unsupported, which is this client's own limit rather +// than the engine's. A row comes back keyed by column name, so a result +// naming two columns the same is one this client cannot read +// positionally, and the corpus writes a case for exactly that. Reporting +// it unsupported names what the mapping costs, where failing it would +// read as the engine having answered wrongly and passing it would mean +// comparing against a row invented for the occasion. +test('a result this client cannot hold is unsupported rather than failed', async (t) => { + const by = await ranBy( + t, + suite([ + '- name: twice', + ' doc: two columns of one name', + ' query: RETURN 1 AS a, 2 AS a', + ' columns:', + ' - a', + ' - a', + ' rows:', + ' - values:', + ' - type: INT64', + ' value: "1"', + ' - type: INT64', + ' value: "2"', + ]), + ) + const one = by.get('twice') + assert.equal(one.outcome, UNSUPPORTED, one.detail) + assert.match(one.detail, /two columns of one name/) +}) + +test('a setup statement runs before the case and its failure is not a pass', async (t) => { + const by = await ranBy( + t, + suite( + [ + '- name: ran', + ' doc: the setup ran and the case did too', + ' setup:', + ' - RETURN 1', + ' query: RETURN 2 AS n', + ' columns:', + ' - n', + ' rows:', + ' - values:', + ' - type: INT64', + ' value: "2"', + ], + [ + '- name: broken-setup', + ' doc: the setup did not run, so the case says nothing about the statement', + ' setup:', + ' - THIS IS NOT A STATEMENT', + ' query: RETURN 1 AS n', + ' columns:', + ' - n', + ' rows:', + ' - values:', + ' - type: INT64', + ' value: "1"', + ], + ), + ) + assert.equal(by.get('ran').outcome, PASSED, by.get('ran').detail) + const broken = by.get('broken-setup') + // Never a pass, whichever way the setup went wrong, and the line says + // which of the setup statements it was. + assert.notEqual(broken.outcome, PASSED) + assert.match(broken.detail, /^setup 1[: ]/, broken.detail) +}) + +// A transaction is only observable from outside it, so a case that has +// something to say about one needs a second connection to say it to. The +// second is a duplicate of the first rather than a second open of the +// file, which is what makes the two share the write side. +test('a case may name a second connection and the two see one database', async (t) => { + const by = await ranBy( + t, + suite([ + '- name: two-connections', + ' doc: a statement on a connection the case named', + ' setup:', + ' - on: other', + ' query: RETURN 1', + ' on: other', + ' query: RETURN 1 AS n', + ' columns:', + ' - n', + ' rows:', + ' - values:', + ' - type: INT64', + ' value: "1"', + ]), + ) + const one = by.get('two-connections') + assert.equal(one.outcome, PASSED, one.detail) +}) + +// The bulk load is the other half of the corpus question: everything +// else asserts what a value means on the way out, and a load is a value +// going in by the path that builds the file rather than by a statement. +// +// What is asserted here is that the load went in, not what a statement +// then made of it: a suite whose load failed reports it with a detail +// that opens "the suite's load", and no case in a suite whose load +// failed says anything about the engine. +test('a suite with a load puts it in before the case runs', async (t) => { + const by = await ranBy( + t, + read( + HEAD + + [...LOAD, 'cases:', + ' - name: loaded', + ' doc: the load went in', + ' query: THIS IS NOT A STATEMENT', + ' columns:', + ' - n', + ' rows:', + ''].join('\n'), + ), + ) + const one = by.get('loaded') + assert.ok( + !one.detail.startsWith("the suite's load"), + `the load did not go in: ${one.detail}`, + ) +}) + +// A case may say what the export gives as well as what the rows are. +// What is under test here is that the runner exported at all and lined +// the fields up by name, since the Arrow type the engine picks for a +// column is the engine's and is asserted by the corpus itself. +test('a case may say what its export gives and the two are lined up by name', async (t) => { + const by = await ranBy( + t, + suite([ + '- name: wrong-field-name', + ' doc: the export gives a field the case named something else', + ' query: RETURN 1 AS n', + ' columns:', + ' - n', + ' rows:', + ' - values:', + ' - type: INT64', + ' value: "1"', + ' arrow:', + ' - name: wrong', + ' format: l', + ]), + ) + const one = by.get('wrong-field-name') + assert.equal(one.outcome, FAILED, one.detail) + assert.equal(one.detail, 'arrow field 1 in the result is named "n" where the case wants "wrong"') +}) + +// A failure leaves its database behind, because that is the one thing +// somebody reading the report will want to open. Everything else goes as +// it finishes, since a corpus of fourteen hundred cases is fourteen +// hundred files and holding them all until the run ends is gigabytes of +// a disk that has other work to do. +test('a run keeps the database of a case that failed and removes the rest', async (t) => { + const dir = await work(t) + const ran = await run( + [suite( + [ + '- name: passes', + ' doc: a case that passes', + ' query: RETURN 1 AS n', + ' columns:', + ' - n', + ' rows:', + ' - values:', + ' - type: INT64', + ' value: "1"', + ], + [ + '- name: fails', + ' doc: a case that does not', + ' query: RETURN 1 AS n', + ' columns:', + ' - n', + ' rows:', + ' - values:', + ' - type: INT64', + ' value: "2"', + ], + )], + dir, + ) + assert.equal(count(ran, FAILED), 1, summary(ran)) + const left = (await readdir(dir)).sort() + assert.ok(left.includes('example-fails.zu'), `${left.join(', ')} was left behind`) + assert.ok(!left.includes('example-passes.zu'), `${left.join(', ')} was left behind`) + // And the log beside it, since a log left under a name the next run + // creates again is a log that run would adopt. + assert.ok(!left.includes('example-passes.zu.wal'), `${left.join(', ')} was left behind`) +}) + +// The order is the order the cases were written in, because a report +// that is diffed against another runner's has to walk them the same way. +test('a run reports its cases in the order the file writes them', async (t) => { + const ran = await run( + [suite( + ['- name: first', ' doc: d', ' query: RETURN 1', ' raises: 42601'], + ['- name: second', ' doc: d', ' query: RETURN 1', ' raises: 42601'], + ['- name: third', ' doc: d', ' query: RETURN 1', ' raises: 42601'], + )], + await work(t), + ) + assert.deepEqual(ran.map((one) => one.case), ['first', 'second', 'third']) + // And each one is handed over as it finishes rather than at the end, + // which is what makes a run of fourteen hundred cases print as it goes. + const seen = [] + await run( + [suite( + ['- name: first', ' doc: d', ' query: RETURN 1', ' raises: 42601'], + ['- name: second', ' doc: d', ' query: RETURN 1', ' raises: 42601'], + )], + await work(t), + (one) => seen.push(one.case), + ) + assert.deepEqual(seen, ['first', 'second']) +}) diff --git a/test/conformance-temporal.test.mjs b/test/conformance-temporal.test.mjs new file mode 100644 index 0000000..3d5b34b --- /dev/null +++ b/test/conformance-temporal.test.mjs @@ -0,0 +1,399 @@ +// The temporal half of the encoding, tested in both directions. +// +// These four spellings are written out in the reader rather than handed +// to `Date.parse`, and the reason is the list of refusals here: a general +// reader takes `2024-1-1`, takes `Jan 1 2024`, and has its own opinion +// about what a date with no time in it means. A case that wrote one of +// those would read as a different instant in every client, which is the +// one failure a conformance corpus cannot report. +// +// The printing half is tested just as closely, because the report a +// failing case prints is diffed against the reference runner's. A time +// that prints six fraction digits here and nine there is a diff on every +// line of a report that agrees about every value in it. + +import assert from 'node:assert/strict' +import test from 'node:test' + +import { ZuDuration } from 'zudb' + +import { quote } from '../conformance/reader.mjs' +import { same, show } from '../conformance/values.mjs' +import { + NANOS_PER_DAY, + NANOS_PER_HOUR, + NANOS_PER_MINUTE, + NANOS_PER_SECOND, + clockNanos, + dateDays, + parseDate, + parseDuration, + parseLocalDateTime, + parseLocalTime, + parseZonedDateTime, + parseZonedTime, + showClock, + showDate, + showMonths, + showNanos, + showOffset, + showStamp, + splitOffset, + stampNanos, +} from '../conformance/temporal.mjs' + +test('a date is a count of days from the epoch', () => { + for (const [text, want] of [ + ['1970-01-01', 0], + ['1970-01-02', 1], + ['1969-12-31', -1], + ['2024-01-01', 19723], + ['2024-02-29', 19782], + ]) { + assert.equal(dateDays(text), want, text) + } + // A year under a hundred is that year and not that year plus nineteen + // hundred, which is what `Date.UTC` would make of it and the reason the + // reader builds the date the long way round. + assert.equal(showDate(dateDays('0024-01-01')), '0024-01-01') +}) + +test('a date the calendar does not have is refused', () => { + for (const text of [ + '2023-02-29', // not a leap year + '2023-02-30', + '2024-13-01', + '2024-00-01', + '2024-01-00', + '2024-01-32', + '2024-04-31', + ]) { + assert.equal(dateDays(text), undefined, `${quote(text)} was read as a date`) + } +}) + +// The spellings a general reader takes and this one does not. Every one +// of them is a date somebody could mean, and every one of them means +// something different in at least one of the five clients. +test('a date written any way but the one way is refused', () => { + for (const text of [ + '2024-1-01', // a month in one digit + '2024-01-1', + '24-01-01', // a year in two + '20240101', // the basic form + '2024/01/01', + '+2024-01-01', // an expanded year + '2024-01-01T00:00:00', // a datetime under a date + '2024-01-01 ', // trailing space, which nothing takes off by here + ' 2024-01-01', + '', + 'today', + ]) { + assert.equal(dateDays(text), undefined, `${quote(text)} was read as a date`) + } +}) + +test('a clock is nanoseconds since midnight', () => { + for (const [text, want] of [ + ['00:00:00', 0n], + ['12:34:56', 45296n * NANOS_PER_SECOND], + ['23:59:59', 86399n * NANOS_PER_SECOND], + ['00:00:00.5', 500000000n], + ['00:00:00.000000001', 1n], + ['00:00:00.123456789', 123456789n], + // A fraction is padded on the right, so a tenth is a tenth however + // many digits it was written in. + ['00:00:00.1', 100000000n], + ['00:00:00.100000000', 100000000n], + ]) { + assert.equal(clockNanos(text), want, text) + } +}) + +test('a clock outside the day, or spelt loosely, is refused', () => { + for (const text of [ + '24:00:00', // midnight at the far end, which the engine writes as the next day + '23:60:00', + // There is no leap second, because a time here is a count of + // nanoseconds since midnight and this is a second the count does not + // have. + '23:59:60', + '12:34', // no seconds + '1:34:56', // an hour in one digit + '12:34:56.', // a point and no fraction + '12:34:56.1234567890', // ten digits, which is finer than the engine counts + '12:34:56.abc', + '12-34-56', + '123456', + '', + ]) { + assert.equal(clockNanos(text), undefined, `${quote(text)} was read as a clock`) + } +}) + +test('a datetime is a date and a clock joined with a T', () => { + assert.equal(stampNanos('1970-01-01T00:00:00'), 0n) + assert.equal(stampNanos('1970-01-02T00:00:00'), NANOS_PER_DAY) + assert.equal(stampNanos('1969-12-31T23:59:59.999999999'), -1n) + assert.equal(stampNanos('2024-01-01T12:00:00'), 19723n * NANOS_PER_DAY + 12n * NANOS_PER_HOUR) + for (const text of [ + '1970-01-01 00:00:00', // a space where the T belongs + '1970-01-01t00:00:00', // the lower case one, which ISO 8601 allows and the engine does not print + '1970-01-01', + 'T00:00:00', + '1970-01-01T', + ]) { + assert.equal(stampNanos(text), undefined, `${quote(text)} was read as a datetime`) + } +}) + +test('an offset is minutes east of UTC and zero is Z', () => { + for (const [text, rest, offset] of [ + ['00:00:00Z', '00:00:00', 0], + // Both spellings of zero are read, since a case may assert either, + // and neither is kept apart from the other because the engine holds a + // count of minutes and prints zero as Z whichever way it went in. + ['00:00:00+00:00', '00:00:00', 0], + ['00:00:00+07:00', '00:00:00', 420], + ['00:00:00-05:30', '00:00:00', -330], + ['00:00:00+18:00', '00:00:00', 1080], + ['00:00:00-18:00', '00:00:00', -1080], + ]) { + assert.deepEqual(splitOffset(text), { rest, offset }, text) + } + for (const text of [ + '00:00:00+18:01', // past the standard's own limit + '00:00:00-18:01', + '00:00:00+00:60', + '00:00:00+0700', // the basic form + '00:00:00+07', // hours alone + '00:00:00z', + '00:00:00', // no offset at all + '+07:00', // an offset and nothing in front of it + ]) { + assert.equal(splitOffset(text), undefined, `${quote(text)} split into an offset`) + } +}) + +// A zoned time keeps the clock that was written and the offset beside it, +// so noon in Bangkok and five in the morning UTC are two values. A zoned +// datetime does the opposite, holding the instant in UTC, so the same two +// are one value with two spellings. That difference is the engine's and +// it is what these two assert. +test('a zoned time keeps its own clock and a zoned datetime keeps its instant', () => { + const bangkok = parseZonedTime('12:00:00+07:00') + const utc = parseZonedTime('05:00:00Z') + assert.equal(bangkok.nanos, 12n * NANOS_PER_HOUR) + assert.equal(bangkok.offset, 420) + assert.ok(!same(bangkok, utc), 'two zoned times an offset apart came out as one value') + + const there = parseZonedDateTime('1970-01-01T12:00:00+07:00') + const here = parseZonedDateTime('1970-01-01T05:00:00Z') + assert.equal(there.nanos, 5n * NANOS_PER_HOUR, 'the instant was not moved back by the offset') + assert.equal(there.offset, 420) + assert.equal(here.nanos, there.nanos, 'one instant in two zones came out as two instants') + // And the offset each was written with is still there, which is what + // keeps them apart as values. + assert.ok(!same(there, here)) +}) + +test('a local temporal has no offset on it and a zoned one must have', () => { + assert.equal(parseLocalTime('00:00:00').offset, null) + assert.equal(parseLocalDateTime('1970-01-01T00:00:00').offset, null) + assert.equal(parseLocalTime('00:00:00Z'), undefined, 'a local time took an offset') + assert.equal(parseLocalDateTime('1970-01-01T00:00:00Z'), undefined, 'a local datetime took an offset') + assert.equal(parseZonedTime('00:00:00'), undefined, 'a zoned time went without an offset') + assert.equal(parseZonedDateTime('1970-01-01T00:00:00'), undefined, 'a zoned datetime went without an offset') + // A space in front of the offset is left on the clock by the split and + // refused there, which is why it is not in the list of splits that give + // nothing back. + assert.equal(parseZonedTime('00:00:00 +07:00'), undefined) +}) + +// The two kinds the engine keeps apart. A month is not a number of days, +// so a duration is one or the other and never both, and a duration of +// nothing is two values here because the text says which one it is. +test('a duration is months or nanoseconds and the text says which', () => { + for (const [text, kind, count] of [ + ['P1Y', 'yearMonth', 12n], + ['P1Y2M', 'yearMonth', 14n], + ['P2M', 'yearMonth', 2n], + ['P0M', 'yearMonth', 0n], + ['-P1Y2M', 'yearMonth', -14n], + ['+P1Y', 'yearMonth', 12n], + ['P1D', 'dayTime', NANOS_PER_DAY], + ['P1W', 'dayTime', 7n * NANOS_PER_DAY], + ['P0D', 'dayTime', 0n], + ['PT0S', 'dayTime', 0n], + ['PT1H', 'dayTime', NANOS_PER_HOUR], + ['PT1M', 'dayTime', NANOS_PER_MINUTE], + ['PT1S', 'dayTime', NANOS_PER_SECOND], + ['PT1H30M', 'dayTime', NANOS_PER_HOUR + 30n * NANOS_PER_MINUTE], + ['P1DT2H3M4S', 'dayTime', NANOS_PER_DAY + 2n * NANOS_PER_HOUR + 3n * NANOS_PER_MINUTE + 4n * NANOS_PER_SECOND], + ['PT0.000000001S', 'dayTime', 1n], + ['PT1.5S', 'dayTime', NANOS_PER_SECOND + 500000000n], + ['-PT1H', 'dayTime', -NANOS_PER_HOUR], + ]) { + const got = parseDuration(text) + assert.ok(got instanceof ZuDuration, `${quote(text)} was not read as a duration`) + assert.equal(got.kind, kind, text) + assert.equal(kind === 'yearMonth' ? got.months : got.nanos, count, text) + } + // The pair the numbers cannot tell apart and the fields can, which is + // the reason the kind is read off the text rather than off the count. + assert.ok(!same(parseDuration('P0M'), parseDuration('PT0S'))) +}) + +test('a duration that mixes the two kinds is refused rather than guessed at', () => { + for (const text of ['P1MT1S', 'P1Y1D', 'P1YT1H', 'P1M1D']) { + assert.equal(parseDuration(text), undefined, `${quote(text)} was read as a duration`) + } +}) + +test('a fraction of anything but a second is refused', () => { + // A fraction of a year, a month, a week or a day is a length that + // depends on which one it lands on, so there is no count of nanoseconds + // that is right for all of them. + for (const text of ['P0.5Y', 'P1.5M', 'P0.5W', 'P0.5D', 'PT1.5H', 'PT1.5M']) { + assert.equal(parseDuration(text), undefined, `${quote(text)} was read as a duration`) + } + assert.equal(parseDuration('PT1.5S').nanos, NANOS_PER_SECOND + 500000000n) +}) + +test('a duration written loosely is refused', () => { + for (const text of [ + 'P', // nothing under it + 'PT', // a T and nothing after it + '', // nothing at all + 'T1H', // no P + '1D', + 'P1', // digits with no unit + 'PT1', // and the same on the other side + 'P1X', // a unit nothing knows + 'PT1D', // a day on the clock side + 'P1H', // an hour on the date side + 'PT1.S', // a point and no fraction + 'PT0.0000000001S', // ten digits + 'p1d', // the lower case one, which ISO 8601 allows and the engine does not print + 'P-1D', // a sign inside, where the whole duration carries the one sign + 'P 1D', + // Past what a signed 64 bit count holds, which is what the client's + // duration is, so this is refused here rather than handed to a + // constructor that would throw. + 'PT9223372036854775808S', + 'P999999999999999999999Y', + ]) { + assert.equal(parseDuration(text), undefined, `${quote(text)} was read as a duration`) + } +}) + +test('a date and a clock print the way the engine prints them', () => { + for (const [days, want] of [ + [0, '1970-01-01'], + [1, '1970-01-02'], + [-1, '1969-12-31'], + [19723, '2024-01-01'], + [19782, '2024-02-29'], + ]) { + assert.equal(showDate(days), want) + } + for (const [nanos, want] of [ + // Nine digits when there is a fraction and nothing when there is not, + // which is the engine's rule and not the shortest that reads back. + [0n, '00:00:00'], + [1n, '00:00:00.000000001'], + [100000000n, '00:00:00.100000000'], + [45296n * NANOS_PER_SECOND, '12:34:56'], + [86399999999999n, '23:59:59.999999999'], + ]) { + assert.equal(showClock(nanos), want) + } +}) + +// An instant before the epoch is on the day it is on and not the day +// after it, which is what the division rounding towards minus infinity is +// there for and the one thing about printing an instant that is easy to +// get wrong in every language that has a `%`. +test('an instant before the epoch prints on its own day', () => { + for (const [nanos, want] of [ + [0n, '1970-01-01T00:00:00'], + [-1n, '1969-12-31T23:59:59.999999999'], + [-NANOS_PER_DAY, '1969-12-31T00:00:00'], + [-NANOS_PER_DAY - 1n, '1969-12-30T23:59:59.999999999'], + [NANOS_PER_DAY, '1970-01-02T00:00:00'], + ]) { + assert.equal(showStamp(nanos), want) + } +}) + +test('an offset prints as Z at zero and with a sign anywhere else', () => { + for (const [offset, want] of [ + [0, 'Z'], + [420, '+07:00'], + [-330, '-05:30'], + [1080, '+18:00'], + [-1080, '-18:00'], + [1, '+00:01'], + [-1, '-00:01'], + ]) { + assert.equal(showOffset(offset), want) + } +}) + +test('a duration prints as the text that reads back to it', () => { + for (const [months, want] of [ + // A duration of nothing has to print as something, because a P on its + // own is not a value, and the something says which kind it was. + [0n, 'P0M'], + [1n, 'P1M'], + [12n, 'P1Y'], + [14n, 'P1Y2M'], + [-14n, '-P1Y2M'], + [-1n, '-P1M'], + ]) { + assert.equal(showMonths(months), want) + } + for (const [nanos, want] of [ + [0n, 'PT0S'], + [NANOS_PER_DAY, 'P1D'], + [NANOS_PER_DAY + 1n, 'P1DT0.000000001S'], + [NANOS_PER_HOUR, 'PT1H'], + [NANOS_PER_MINUTE, 'PT1M'], + [NANOS_PER_SECOND, 'PT1S'], + [NANOS_PER_HOUR + 30n * NANOS_PER_MINUTE, 'PT1H30M'], + [500000000n, 'PT0.500000000S'], + [-NANOS_PER_HOUR, '-PT1H'], + [NANOS_PER_DAY + 2n * NANOS_PER_HOUR + 3n * NANOS_PER_MINUTE + 4n * NANOS_PER_SECOND, 'P1DT2H3M4S'], + ]) { + assert.equal(showNanos(nanos), want) + } +}) + +// The property the whole file is for: what a case writes is what the +// report prints, so a failing case can be read against the file it came +// from without translating either one. +test('every temporal value written and printed comes back the same', () => { + for (const text of [ + '1970-01-01', + '2024-02-29', + '0024-01-01', + '9999-12-31', + ]) { + assert.equal(show(parseDate(text)), `DATE ${quote(text)}`) + } + for (const text of ['00:00:00', '12:34:56', '23:59:59.999999999', '00:00:00.100000000']) { + assert.equal(show(parseLocalTime(text)), `LOCALTIME ${quote(text)}`) + } + for (const text of ['00:00:00Z', '12:00:00+07:00', '12:00:00-05:30', '23:59:59.999999999+18:00']) { + assert.equal(show(parseZonedTime(text)), `ZONEDTIME ${quote(text)}`) + } + for (const text of ['1970-01-01T00:00:00', '1969-12-31T23:59:59.999999999', '2024-02-29T12:34:56.500000000']) { + assert.equal(show(parseLocalDateTime(text)), `LOCALDATETIME ${quote(text)}`) + } + for (const text of ['1970-01-01T00:00:00Z', '1970-01-01T12:00:00+07:00', '2024-01-01T00:00:00-05:30']) { + assert.equal(show(parseZonedDateTime(text)), `ZONEDDATETIME ${quote(text)}`) + } + for (const text of ['P0M', 'P1Y', 'P1Y2M', '-P1Y2M', 'PT0S', 'P1D', 'PT1H30M', 'P1DT2H3M4S', '-PT1H']) { + assert.equal(show(parseDuration(text)), `DURATION ${quote(text)}`) + } +}) diff --git a/test/conformance-values.test.mjs b/test/conformance-values.test.mjs new file mode 100644 index 0000000..99fc9ff --- /dev/null +++ b/test/conformance-values.test.mjs @@ -0,0 +1,466 @@ +// The corpus value encoding, tested on both sides of it. +// +// A case says what a statement produces by naming a type and a payload, +// and the whole point of naming the type is that a payload cannot be +// misread. So the tests here are mostly refusals: an INT64 written bare +// is refused because some reader will round it, a STRING written where a +// LIST belongs is refused, a payload out of its type's range is refused. +// A round trip through a reader that accepted all three would still look +// green. +// +// The other half is `show`, which is what a failure report prints. It is +// diffed against the Rust runner's line for line, so a float that +// switches to an exponent one power earlier here than there is a +// difference in the report that is not a difference in the answer. + +import assert from 'node:assert/strict' +import test from 'node:test' + +import { ZuDate, ZuDuration, ZuTime, ZuTimestamp } from 'zudb' + +import { parse, quote } from '../conformance/reader.mjs' +import { + EdgeAt, + NodeAt, + Walk, + cell, + decode, + form, + same, + show, + showFloat, +} from '../conformance/values.mjs' + +// The value a `type:`/`value:` mapping comes to, written the way a case +// writes it. +function value(text) { + return decode(parse(text)) +} + +// The message a `type:`/`value:` mapping was refused with, and a failure +// when it was read instead. +function declined(text) { + let node + try { + node = parse(text) + } catch (err) { + return err.message + } + try { + const got = decode(node) + assert.fail(`decode read this as ${show(got)}, and it should have been refused`) + } catch (err) { + return err.message + } +} + +test('every type says whether its payload is quoted', () => { + // The whole table, written out rather than iterated over, because the + // point of the test is that the table is this and not whatever the map + // happens to hold. + const bare = ['NULL', 'BOOL', 'INT8', 'INT16', 'INT32', 'UINT8', 'UINT16', 'UINT32', 'STRING', 'LIST', 'PATH'] + const quotedTypes = ['INT64', 'UINT64', 'FLOAT32', 'FLOAT64', 'BYTES', 'DATE', 'LOCALTIME', 'ZONEDTIME', + 'LOCALDATETIME', 'ZONEDDATETIME', 'DURATION', 'NODE', 'EDGE'] + for (const type of bare) assert.deepEqual(form(type), { quoted: false, known: true }, type) + for (const type of quotedTypes) assert.deepEqual(form(type), { quoted: true, known: true }, type) + // DECIMAL has a name and no value behind it, and is told apart from a + // typo so that the message says which of the two happened. + assert.equal(form('DECIMAL').known, false, 'DECIMAL is a type, and the engine has no value for one') +}) + +test('a payload is read as the type beside it', () => { + for (const [text, want] of [ + ['type: NULL\n', null], + ['type: BOOL\nvalue: true\n', true], + ['type: BOOL\nvalue: false\n', false], + ['type: INT8\nvalue: -128\n', -128n], + ['type: INT16\nvalue: 32767\n', 32767n], + ['type: INT32\nvalue: -2147483648\n', -2147483648n], + ['type: INT64\nvalue: "9223372036854775807"\n', 9223372036854775807n], + ['type: UINT8\nvalue: 255\n', 255n], + ['type: UINT16\nvalue: 65535\n', 65535n], + ['type: UINT32\nvalue: 4294967295\n', 4294967295n], + ['type: UINT64\nvalue: "0"\n', 0n], + ['type: FLOAT64\nvalue: "1.5"\n', 1.5], + ['type: FLOAT64\nvalue: "-0.0"\n', -0], + ['type: FLOAT64\nvalue: "inf"\n', Number.POSITIVE_INFINITY], + ['type: FLOAT64\nvalue: "-inf"\n', Number.NEGATIVE_INFINITY], + // A FLOAT32 is held as the double the single rounds to, since that is + // what comes back out of a column of them. + ['type: FLOAT32\nvalue: "0.1"\n', Math.fround(0.1)], + ['type: STRING\nvalue: plain\n', 'plain'], + ["type: STRING\nvalue: ''\n", ''], + ['type: BYTES\nvalue: "00AB00"\n', new Uint8Array([0, 0xab, 0])], + ['type: BYTES\nvalue: ""\n', new Uint8Array()], + ['type: NODE\nvalue: "person#1"\n', new NodeAt('person', 1n)], + ['type: EDGE\nvalue: "knows#0->2"\n', new EdgeAt('knows', 0n, 2n)], + ]) { + const got = value(text) + assert.ok(same(want, got), `${quote(text)} came to ${show(got)}, and it should be ${show(want)}`) + } +}) + +// A STRING is the one type that reads either way, because a string is +// what a plain scalar already is and a case quotes one only when it has +// to. Everything else is written one way and refused the other. +test('a string reads quoted or bare', () => { + assert.equal(value('type: STRING\nvalue: 42\n'), '42') + assert.equal(value('type: STRING\nvalue: "42"\n'), '42') +}) + +test('an integer is refused outside the range its type holds', () => { + for (const [type, text] of [ + ['INT8', '128'], + ['INT8', '-129'], + ['INT16', '32768'], + ['INT32', '2147483648'], + ['UINT8', '256'], + ['UINT8', '-1'], + ['UINT16', '65536'], + ['UINT32', '4294967296'], + ]) { + assert.equal(declined(`type: ${type}\nvalue: ${text}\n`), `line 2: ${quote(text)} is not a ${type}`) + } + // UINT64 stops at the signed maximum, because the engine's integer is + // signed and wrapping the top half into a negative would be a case that + // passes while meaning the opposite of what it says. + assert.equal( + declined('type: UINT64\nvalue: "9223372036854775808"\n'), + 'line 2: "9223372036854775808" is not a UINT64', + ) +}) + +// The rule the whole encoding exists for. A bare INT64 is a number some +// reader in some language rounds, and a bare NODE is a name and two +// numbers no reader has a scalar for, so the two are told apart. +test('a quoted type written bare is refused and says why', () => { + for (const [text, want] of [ + [ + 'type: INT64\nvalue: 1\n', + 'line 2: INT64 is written in quotes, because a bare 1 is a number and some reader of this file will round it', + ], + [ + 'type: FLOAT64\nvalue: 1.5\n', + 'line 2: FLOAT64 is written in quotes, because a bare 1.5 is a number and some reader of this file will round it', + ], + [ + 'type: NODE\nvalue: person#1\n', + 'line 2: NODE is written in quotes, because person#1 is a name and two numbers and no reader has a scalar for that', + ], + [ + 'type: EDGE\nvalue: knows#0->1\n', + 'line 2: EDGE is written in quotes, because knows#0->1 is a name and two numbers and no reader has a scalar for that', + ], + ]) { + assert.equal(declined(text), want) + } +}) + +test('a bare type written in quotes is refused', () => { + assert.equal( + declined('type: INT8\nvalue: "1"\n'), + 'line 2: INT8 is written without quotes, so that a reader cannot take it for a string', + ) +}) + +test('a value says what is wrong with it in the order that helps', () => { + for (const [what, text, want] of [ + ['a type nothing knows', 'type: INTEGER\nvalue: 1\n', 'line 1: INTEGER is not a type this encoding knows'], + [ + 'a type the encoding holds a name for', + 'type: DECIMAL\nvalue: "1.0"\n', + 'line 1: DECIMAL is a type the encoding reserves and the engine has no value for', + ], + // The type is the mistake and the missing payload is a consequence of + // it, so the type is what the message names. + ['a type nothing knows and no payload either', 'type: INTEGER\n', 'line 1: INTEGER is not a type this encoding knows'], + ['no type at all', 'value: 1\n', 'line 1: a value with no `type`'], + ['a type that is not a name', 'type:\n - INT8\nvalue: 1\n', 'line 1: a `type` that is not a name'], + ['no payload', 'type: INT8\n', 'line 1: a INT8 with no `value`'], + ['a payload under NULL', 'type: NULL\nvalue: 1\n', 'line 1: NULL carries no `value`'], + ['a key the encoding has no room for', 'type: INT8\nvalue: 1\nname: n\n', 'line 1: a value has no key "name"'], + [ + 'a sequence where a value belongs', + '- type: INT8\n', + 'line 1: a value is a mapping of `type` and `value`, and this is a sequence', + ], + [ + 'a scalar where a value belongs', + 'just a scalar\n', + 'line 1: a value is a mapping of `type` and `value`, and this is a scalar', + ], + ['a sequence under a scalar type', 'type: INT8\nvalue:\n - 1\n', 'line 3: a INT8 holds one scalar, and this is a sequence'], + ['a scalar under LIST', 'type: LIST\nvalue: 1\n', 'line 2: a LIST holds a sequence of values, and this is a scalar'], + ]) { + assert.equal(declined(text), want, `${what} was refused with the wrong words`) + } +}) + +test('a list holds values and the empty one has a spelling', () => { + const got = value( + ['type: LIST', 'value:', ' - type: INT8', ' value: 1', ' - type: NULL', ' - type: STRING', ' value: two', ''].join('\n'), + ) + assert.ok(same([1n, null, 'two'], got), `read ${show(got)}`) + // A `value:` with nothing under it, which is the empty list and a value + // a case asserts. + assert.ok(same([], value('type: LIST\nvalue:\n'))) +}) + +// A path alternates and ends at both ends with a node, so a sequence that +// does not is refused where it is written rather than at the comparison, +// which is the difference between a message naming a line and a report +// saying the row differs. +test('a path alternates node and edge or is refused', () => { + const oneHop = value( + [ + 'type: PATH', + 'value:', + ' - type: NODE', + ' value: "person#0"', + ' - type: EDGE', + ' value: "knows#0->1"', + ' - type: NODE', + ' value: "person#1"', + '', + ].join('\n'), + ) + assert.ok(oneHop instanceof Walk, `a one hop path read as ${show(oneHop)}`) + assert.equal(oneHop.elements.length, 3) + + for (const [what, text, want] of [ + [ + 'an even number of values', + 'type: PATH\nvalue:\n - type: NODE\n value: "person#0"\n - type: EDGE\n value: "knows#0->1"\n', + 'line 3: a PATH is a node, then an edge and a node for each hop, so it holds an odd number of values and this holds 2', + ], + [ + 'an edge where the walk starts', + 'type: PATH\nvalue:\n - type: EDGE\n value: "knows#0->1"\n', + 'line 3: a PATH alternates, so value 1 is an EDGE where it should be a NODE', + ], + [ + 'a node in the hop position', + 'type: PATH\nvalue:\n - type: NODE\n value: "person#0"\n - type: NODE\n value: "person#1"\n' + + ' - type: NODE\n value: "person#2"\n', + 'line 3: a PATH alternates, so value 2 is a NODE where it should be an EDGE', + ], + [ + 'something that is neither', + 'type: PATH\nvalue:\n - type: INT8\n value: 1\n', + 'line 3: a PATH alternates, so value 1 is neither a NODE nor an EDGE where it should be a NODE', + ], + ]) { + assert.equal(declined(text), want, `${what} was refused with the wrong words`) + } + // The empty path is refused too, since zero is an even number and a walk + // with no nodes in it is not a walk. + assert.match(declined('type: PATH\nvalue:\n'), /odd number/) +}) + +test('a node and an edge are a table name and row numbers', () => { + // Split from the right, so a table whose name holds a # still reads. + assert.ok(same(new NodeAt('od#d', 7n), value('type: NODE\nvalue: "od#d#7"\n'))) + for (const text of [ + 'person', // no offset + '#1', // no table + 'person#', // no digits + 'person#-1', // a sign, which BigInt would take + 'person#1_0', // an underscore, which BigInt would take too + 'person#a', // not a number + 'person#1->2', // an edge under a node's type + ]) { + assert.equal(declined(`type: NODE\nvalue: ${quote(text)}\n`), `line 2: ${quote(text)} is not a NODE`) + } + for (const text of [ + 'knows#0', // one row rather than two + 'knows#0->', // no second row + 'knows#->1', // no first row + 'knows#0-1', // the wrong arrow + '#0->1', // no table + 'knows#0->-1', // a sign + ]) { + assert.equal(declined(`type: EDGE\nvalue: ${quote(text)}\n`), `line 2: ${quote(text)} is not a EDGE`) + } +}) + +// An integer is written back out and compared, so that a spelling BigInt +// would take and no other reader would is refused. +test('an integer is refused when it is spelt unusually', () => { + for (const text of ['+1', '01', '1_0', '0x10']) { + assert.equal(declined(`type: INT8\nvalue: ${text}\n`), `line 2: ${quote(text)} is not a INT8`) + } + // Space around a bare payload never reaches here, because the reader + // takes it off along with the space after the colon. This is asserted + // rather than left implied, since it is the reason the list above has no + // padded spelling in it. + assert.ok(same(1n, value('type: INT8\nvalue: 1 \n'))) +}) + +// A float is exact here: `1` is an integer somebody meant to write as +// `1.0`, and `1e400` is `inf` under another name. `Number` takes four +// more spellings the other runners do not. +test('a float is refused when it is spelt unusually', () => { + for (const text of ['1', '-1', '1e400', '-1e400', 'Inf', 'infinity', 'nan', '0x1p-2', '1_0.0', '1.0f', '']) { + assert.equal(declined(`type: FLOAT64\nvalue: ${quote(text)}\n`), `line 2: ${quote(text)} is not a FLOAT64`) + } + for (const text of ['1.0', '-1.5', '1e10', '1E10', '1.5e-3', 'NaN', 'inf', '-inf']) { + assert.equal(typeof value(`type: FLOAT64\nvalue: ${quote(text)}\n`), 'number', text) + } +}) + +// Space anywhere in a byte string is dropped, which is what the +// standard's production allows and what lets a long literal be written in +// groups. Half a byte is refused. +test('a byte string is hexits in either case and space is dropped', () => { + for (const [text, want] of [ + ['00AB00', new Uint8Array([0, 0xab, 0])], + ['00ab00', new Uint8Array([0, 0xab, 0])], + ['00 AB 00', new Uint8Array([0, 0xab, 0])], + ['', new Uint8Array()], + ['FF', new Uint8Array([0xff])], + ]) { + const got = value(`type: BYTES\nvalue: ${quote(text)}\n`) + assert.ok(same(want, got), `BYTES ${quote(text)} read as ${show(got)}`) + } + for (const text of ['0', 'ABC', 'GG', '0x41', '00-AB']) { + assert.equal(declined(`type: BYTES\nvalue: ${quote(text)}\n`), `line 2: ${quote(text)} is not a BYTES`) + } +}) + +// Not ===, for three reasons: a float, because NaN is not equal to itself +// and -0 is equal to 0; the shapes that hold other values; and every +// temporal value, which is a class holding its count behind a getter. +test('same is equality except where equality is wrong', () => { + for (const [what, want, got, is] of [ + ['NaN against itself', Number.NaN, Number.NaN, true], + ['a negative zero against a positive one', -0, 0, false], + ['a positive zero against a negative one', 0, -0, false], + ['two ones', 1.0, 1.0, true], + ['an integer against a float', 1n, 1.0, false], + ['a float against an integer', 1.0, 1n, false], + ['a boolean against an integer', true, 1n, false], + ['nothing against nothing', null, null, true], + ['nothing against a value', null, 0n, false], + ['two lists', [1n, null], [1n, null], true], + ['lists of different lengths', [1n], [1n, null], false], + ['a list against a scalar', [1n], 1n, false], + ['a scalar against a list', 1n, [1n], false], + ['nested lists', [[1.0]], [[1.0]], true], + ['two walks', new Walk([new NodeAt('p', 0n)]), new Walk([new NodeAt('p', 0n)]), true], + ['a walk against a list', new Walk([new NodeAt('p', 0n)]), [new NodeAt('p', 0n)], false], + ['two records', { a: 1n }, { a: 1n }, true], + ['records of different sizes', { a: 1n }, { a: 1n, b: null }, false], + ['records with different names', { a: 1n }, { b: 1n }, false], + ['a record against a scalar', { a: 1n }, 1n, false], + ['a scalar against a record', 1n, { a: 1n }, false], + ['two byte strings', new Uint8Array([1, 2]), new Uint8Array([1, 2]), true], + ['byte strings that differ', new Uint8Array([1, 2]), new Uint8Array([1, 3]), false], + ['a byte string against a string', new Uint8Array([65]), 'A', false], + ['a string against a byte string', 'A', new Uint8Array([65]), false], + ['two dates', new ZuDate(1), new ZuDate(1), true], + ['dates that differ', new ZuDate(1), new ZuDate(2), false], + ['a local time against a zoned one', new ZuTime(0n), new ZuTime(0n, 0), false], + ['a year month against a day time', ZuDuration.ofMonths(0n), ZuDuration.ofNanos(0n), false], + ]) { + assert.equal(same(want, got), is, `${what} came out the other way round`) + } +}) + +// A value the runner did not put through `cell` prints as itself, under a +// name that is not a type, so a report carrying one cannot be mistaken +// for a case that could be pasted back in. +class Unconverted { + constructor() { + this.offset = 1n + } +} + +// `show` is what a failure report prints, in the encoding's own spelling +// so that a line can be pasted back into a case. +test('show writes a value the way a case would spell it', () => { + for (const [held, want] of [ + [null, 'NULL'], + [true, 'BOOL true'], + [false, 'BOOL false'], + [-7n, 'INT64 "-7"'], + [1.5, 'FLOAT64 "1.5"'], + ['a string', 'STRING "a string"'], + ['with "quotes"', 'STRING "with \\"quotes\\""'], + [new Uint8Array([0, 0xab]), 'BYTES "00AB"'], + [new Uint8Array(), 'BYTES ""'], + [new ZuDate(0), 'DATE "1970-01-01"'], + [new ZuTime(0n), 'LOCALTIME "00:00:00"'], + [new ZuTime(123456789n), 'LOCALTIME "00:00:00.123456789"'], + [new ZuTime(0n, 0), 'ZONEDTIME "00:00:00Z"'], + [new ZuTimestamp(0n), 'LOCALDATETIME "1970-01-01T00:00:00"'], + // The instant is UTC and the offset is what the case wrote, so the + // clock printed beside it is the instant moved into that zone, which + // is the wall clock the case reads back. + [new ZuTimestamp(0n, 60), 'ZONEDDATETIME "1970-01-01T01:00:00+01:00"'], + [ZuDuration.ofMonths(14n), 'DURATION "P1Y2M"'], + [ZuDuration.ofNanos(0n), 'DURATION "PT0S"'], + [[1n, null], 'LIST [INT64 "1", NULL]'], + [[], 'LIST []'], + [ + new Walk([new NodeAt('person', 0n), new EdgeAt('knows', 0n, 1n), new NodeAt('person', 1n)]), + 'PATH [NODE "person#0", EDGE "knows#0->1", NODE "person#1"]', + ], + [new NodeAt('person', 3n), 'NODE "person#3"'], + [new EdgeAt('knows', 3n, 4n), 'EDGE "knows#3->4"'], + [{ b: 2n, a: null }, 'RECORD {a: NULL, b: INT64 "2"}'], + [{}, 'RECORD {}'], + [new Unconverted(), '(Unconverted) {"offset":"1"}'], + ]) { + assert.equal(show(held), want) + } + // A record's names are sorted, because a failure that reorders its own + // fields between an engine and a case is a failure nobody can diff. + assert.equal(show({ z: null, a: null, m: null }), 'RECORD {a: NULL, m: NULL, z: NULL}') +}) + +// A float is printed the way Rust's {:?} writes one: the shortest text +// that reads back as the same double, always with a point or an exponent, +// switching to an exponent where Rust switches and writing the exponent +// bare rather than with a sign and a padding zero. +test('a float prints the way the reference runner prints it', () => { + for (const [held, want] of [ + [0, '0.0'], + [-0, '-0.0'], + [1, '1.0'], + [-1, '-1.0'], + [1.5, '1.5'], + [0.1, '0.1'], + [1 / 3, '0.3333333333333333'], + [100, '100.0'], + [1e15, '1000000000000000.0'], + // At ten to the sixteenth the digits go behind an exponent, which is + // where Rust switches and not where `toString` does. + [1e16, '1e16'], + [1e17, '1e17'], + [1.5e17, '1.5e17'], + [0.001, '0.001'], + [0.0001, '0.0001'], + // And below a ten thousandth, likewise. + [0.00001, '1e-5'], + [1.5e-5, '1.5e-5'], + [Number.MAX_VALUE, '1.7976931348623157e308'], + [Number.MIN_VALUE, '5e-324'], + [Number.NaN, 'NaN'], + [Number.POSITIVE_INFINITY, 'inf'], + [Number.NEGATIVE_INFINITY, '-inf'], + ]) { + assert.equal(showFloat(held), want, `${held} printed wrongly`) + } +}) + +// Everything a table holds is spelled the same on both sides and comes +// through untouched, which is what this half asserts. The graph values, +// which are the ones `cell` exists for, need an engine to make one, so +// they are checked in the live test below. +test('cell leaves everything that is not a graph value alone', () => { + for (const held of [1n, null, 'text', 1.5, true, new Uint8Array([1]), new ZuDate(3)]) { + assert.ok(same(held, cell(held)), `${show(held)} did not come through`) + } + assert.ok(same([1n, [2n]], cell([1n, [2n]]))) + assert.ok(same({ a: [1n] }, cell({ a: [1n] }))) +}) diff --git a/test/conformance.test.mjs b/test/conformance.test.mjs new file mode 100644 index 0000000..dc17903 --- /dev/null +++ b/test/conformance.test.mjs @@ -0,0 +1,72 @@ +// The shared cross-client corpus, run against this client. +// +// The cases live in the engine's repository and are versioned with it, +// so this test says where they are with an environment variable and +// skips without one. That is what zu-python does with ZU_CASES and what +// makes a checkout of this repository alone still `npm test` green: a +// client whose test suite cannot run without a second repository beside +// it is one nobody clones to fix a typo. +// +// CI sets the variable, having checked the engine out at the revision +// the addon was built from. Anything else compares a client against a +// corpus that is not the one it was built against, which reports the +// engine catching up to its own cases as this client failing. + +import assert from 'node:assert/strict' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' + +import { readDir } from '../conformance/cases.mjs' +import { FAILED, PASSED, UNSUPPORTED, count, line, run, summary } from '../conformance/runner.mjs' + +// Where the case files are, or undefined when nobody said. +const cases = process.env.ZU_CASES + +// A directory to make the case databases under, removed when the test +// ends. What is left in it at the end is the databases of the cases that +// failed, which is the point of keeping it until then. +async function work(t) { + const dir = await mkdtemp(join(tmpdir(), 'zu-corpus-')) + t.after(() => rm(dir, { recursive: true, force: true })) + return dir +} + +test('the corpus reads', { skip: cases ? false : 'ZU_CASES does not point at the case files' }, + async (t) => { + const suites = await readDir(cases) + const total = suites.reduce((sum, suite) => sum + suite.cases.length, 0) + assert.ok(total > 0, `${suites.length} suites and no cases in any of them`) + t.diagnostic(`${suites.length} suites, ${total} cases`) + }) + +// The run, which is the whole point of the directory. +// +// A case the engine has not caught up to is unsupported and is not a +// failure, because the corpus is the contract and the engine catches up +// to it. A case that fails is this client answering a question wrongly, +// and there is no allowance for one. +test('every case in the corpus passes or is ahead of the engine', + { skip: cases ? false : 'ZU_CASES does not point at the case files' }, + async (t) => { + const suites = await readDir(cases) + const ran = await run(suites, await work(t)) + const failed = ran.filter((one) => one.outcome === FAILED).map(line) + t.diagnostic(summary(ran)) + // The ones ahead of the engine are listed rather than counted, so + // that a release branch has something to read and so that a case + // quietly becoming unsupported is visible in the log. The corpus is + // run once and read twice, because running it is minutes. + for (const one of ran) { + if (one.outcome === UNSUPPORTED) t.diagnostic(` ${line(one)}`) + } + assert.deepEqual(failed, []) + // A run where nothing passed is a run that did not happen, which is + // what a corpus read from the wrong directory or an addon that + // answers nothing looks like from here. + assert.ok( + count(ran, PASSED) > 0, + 'no case passed, and a run where nothing passes is a run that did not happen', + ) + }) diff --git a/test/helper.mjs b/test/helper.mjs index 0643920..780654d 100644 --- a/test/helper.mjs +++ b/test/helper.mjs @@ -33,3 +33,26 @@ export async function twoPeople(t) { export function isZuError(err, name) { return guard(err) && err.name === name } + +// How often a one millisecond interval really fires on a loop with +// nothing else to do, in ticks per millisecond. +// +// A test that asks whether a read left the loop free counts ticks while +// the read runs, and a count on its own says nothing without knowing +// what a free loop would have managed. That is not one tick per +// millisecond everywhere: Windows fires a timer on its own tick, which +// is about sixteen milliseconds, so a read that takes sixty gets four +// ticks with the loop completely idle. Measuring it here is what lets +// the bar be a share of what this machine can do rather than a number +// that happens to hold on Linux, and it moves with the load on the +// machine as well, which is the other thing that makes a fixed bar +// flake. +export async function tickRate() { + let ticks = 0 + const timer = setInterval(() => (ticks += 1), 1) + const at = performance.now() + await new Promise((done) => setTimeout(done, 200)) + const took = performance.now() - at + clearInterval(timer) + return ticks / took +} diff --git a/test/load.test.mjs b/test/load.test.mjs index 9caaa90..ee1205a 100644 --- a/test/load.test.mjs +++ b/test/load.test.mjs @@ -189,18 +189,18 @@ test('a column is read out of a typed array as the numbers it holds', async (t) nodes: 'person', rels: 'knows', columns: { - small: new Int8Array([-1, 2]), + tiny: new Int8Array([-1, 2]), wide: new BigInt64Array([1n << 40n, -5n]), ratio: new Float32Array([0.5, 1.25]), - exact: new Float64Array([0.1, 2.5]), + precise: new Float64Array([0.1, 2.5]), }, }) const conn = await opened(t, path) const rows = await conn.query( - 'MATCH (p:person) RETURN p.small AS small, p.wide AS wide, p.ratio AS ratio, p.exact AS exact', + 'MATCH (p:person) RETURN p.tiny AS tiny, p.wide AS wide, p.ratio AS ratio, p.precise AS precise', ) assert.deepEqual( - rows.map((row) => [row.small, row.wide, row.ratio, row.exact]), + rows.map((row) => [row.tiny, row.wide, row.ratio, row.precise]), [ [-1n, 1n << 40n, 0.5, 0.1], [2n, -5n, 1.25, 2.5], @@ -211,7 +211,7 @@ test('a column is read out of a typed array as the numbers it holds', async (t) test('a column of every kind reads back as what it was', async (t) => { const path = await spot(t) const columns = { - count: [1, -2], + tally: [1, -2], ratio: [1.5, -0.25], flag: [true, false], name: ['ada', 'grace'], @@ -227,7 +227,7 @@ test('a column of every kind reads back as what it was', async (t) => { const rows = await conn.query( `MATCH (p:person) RETURN ${names.map((name) => `p.${name} AS ${name}`).join(', ')}`, ) - assert.equal(rows[0].count, 1n) + assert.equal(rows[0].tally, 1n) assert.equal(rows[0].ratio, 1.5) assert.equal(rows[0].flag, true) assert.equal(rows[0].name, 'ada') @@ -237,7 +237,7 @@ test('a column of every kind reads back as what it was', async (t) => { assert.equal(rows[0].took.nanos, 86_402_000_000_000n) assert.equal(rows[0].aged.months, 14n) assert.deepEqual( - [rows[1].count, rows[1].ratio, rows[1].flag, rows[1].name], + [rows[1].tally, rows[1].ratio, rows[1].flag, rows[1].name], [-2n, -0.25, false, 'grace'], ) }) diff --git a/test/progress.test.mjs b/test/progress.test.mjs index 0b59ca9..81dadb4 100644 --- a/test/progress.test.mjs +++ b/test/progress.test.mjs @@ -36,6 +36,13 @@ const run = promisify(execFile) const root = fileURLToPath(new URL('../', import.meta.url)) const sleep = (ms) => new Promise((wake) => setTimeout(wake, ms)) +// Whether this runtime can start a copy of itself that loads the addon. +// Node and Bun can. Deno cannot, twice over: spawning at all wants +// --allow-run, and a Deno child is granted nothing of what its parent +// holds, so the copy would fail to open the library rather than fail to +// exit, which is not the question the one test below is asking. +const SPAWNS = !process.versions.deno + // A database with a crowd in it, built by a load rather than by an // insert, because three thousand people written as one statement is a // megabyte of GQL and most of this file's time would go on parsing it. @@ -208,7 +215,7 @@ test('a watch wants a function, and an interval that is one', async (t) => { } }) -test('a watch does not hold the program open', async (t) => { +test('a watch does not hold the program open', { skip: !SPAWNS }, async (t) => { const { path } = await crowd(t) // A watch nobody stopped is a timer that would otherwise run for as diff --git a/test/register.test.mjs b/test/register.test.mjs index 2c54efb..da695d0 100644 --- a/test/register.test.mjs +++ b/test/register.test.mjs @@ -165,25 +165,25 @@ test('every kind of column a row can hold arrives as itself', async (t) => { await conn.register('kinds', { yes: [true, false], - small: new Int8Array([1, 2]), + tiny: new Int8Array([1, 2]), wide: new Uint32Array([3, 4]), narrow: new Float32Array([1.5, 2.5]), word: ['a', 'b'], - day: [new ZuDate(19723), new ZuDate(19754)], + dated: [new ZuDate(19723), new ZuDate(19754)], moment: [new ZuTimestamp(1_704_070_923_000_000_000n), new ZuTimestamp(0n)], span: [ZuDuration.ofNanos(90_000_000_000n), ZuDuration.ofNanos(0n)], }) const rows = await conn.query( - 'MATCH (k:kinds) RETURN k.yes AS yes, k.small AS small, k.wide AS wide, ' + - 'k.narrow AS narrow, k.word AS word, k.day AS day, k.moment AS moment, k.span AS span', + 'MATCH (k:kinds) RETURN k.yes AS yes, k.tiny AS tiny, k.wide AS wide, ' + + 'k.narrow AS narrow, k.word AS word, k.dated AS dated, k.moment AS moment, k.span AS span', ) assert.equal(rows[0].yes, true) - assert.equal(rows[0].small, 1n) + assert.equal(rows[0].tiny, 1n) assert.equal(rows[0].wide, 3n) assert.equal(rows[0].narrow, 1.5) assert.equal(rows[0].word, 'a') - assert.equal(rows[0].day.days, 19723) + assert.equal(rows[0].dated.days, 19723) assert.equal(rows[0].moment.nanos, 1_704_070_923_000_000_000n) assert.equal(rows[0].span.nanos, 90_000_000_000n) assert.equal(rows[1].yes, false) @@ -193,19 +193,19 @@ test('an arrow column of every width arrives as itself', async (t) => { const { conn } = await fresh(t) const table = tableFromArrays({ - small: new Int8Array([1, 2]), + tiny: new Int8Array([1, 2]), wide: new Uint32Array([3, 4]), narrow: new Float32Array([1.5, 2.5]), - big: new Float64Array([1.25, 2.25]), + huge: new Float64Array([1.25, 2.25]), }) assert.equal(await conn.register('widths', table), 2) const rows = await conn.query( - 'MATCH (w:widths) RETURN w.small AS small, w.wide AS wide, w.narrow AS narrow, w.big AS big', + 'MATCH (w:widths) RETURN w.tiny AS tiny, w.wide AS wide, w.narrow AS narrow, w.huge AS huge', ) - assert.equal(rows[0].small, 1n) + assert.equal(rows[0].tiny, 1n) assert.equal(rows[0].wide, 3n) assert.equal(rows[0].narrow, 1.5) - assert.equal(rows[0].big, 1.25) + assert.equal(rows[0].huge, 1.25) }) test('an object of plain arrays is copied because an array is not a column', async (t) => { diff --git a/test/temporal.test.mjs b/test/temporal.test.mjs index b0f608b..2c62a8f 100644 --- a/test/temporal.test.mjs +++ b/test/temporal.test.mjs @@ -242,14 +242,14 @@ test('a Temporal value goes out the way it came in', has, async (t) => { test('a statement and a stream spell temporal values the same way', has, async (t) => { const { conn } = await fresh(t, { temporal: true }) - await conn.exec("INSERT (d:day {id: 1, on: DATE '2024-01-01'})") + await conn.exec("INSERT (d:dated {id: 1, began: DATE '2024-01-01'})") - const rows = await conn.query('MATCH (d:day) RETURN d.on AS on') - assert.equal(tag(rows[0].on), '[object Temporal.PlainDate]') + const rows = await conn.query('MATCH (d:dated) RETURN d.began AS began') + assert.equal(tag(rows[0].began), '[object Temporal.PlainDate]') - const stream = conn.stream('MATCH (d:day) RETURN d.on AS on') + const stream = conn.stream('MATCH (d:dated) RETURN d.began AS began') const seen = [] - for await (const row of stream) seen.push(row.on) + for await (const row of stream) seen.push(row.began) assert.equal(seen.length, 1) assert.equal(tag(seen[0]), '[object Temporal.PlainDate]') assert.equal(seen[0].day, 1) diff --git a/test/values.test.mjs b/test/values.test.mjs index 99febe4..cf27534 100644 --- a/test/values.test.mjs +++ b/test/values.test.mjs @@ -46,6 +46,13 @@ test('a whole number binds as an integer and a fractional one as a float', async assert.equal(typeof (await roundTrip(conn, 1)), 'bigint') assert.equal(await roundTrip(conn, 1.5), 1.5) assert.equal(typeof (await roundTrip(conn, 1.5)), 'number') + + // Negative zero is whole and is still a float, because there is no + // integer that is negative zero and binding it as one throws away the + // sign the caller went out of their way to write. + const zero = await roundTrip(conn, -0) + assert.equal(typeof zero, 'number') + assert.ok(Object.is(zero, -0), `-0 came back as ${zero}`) }) test('a string, a boolean, a null and an undefined bind as themselves', async (t) => { diff --git a/types/header.d.ts b/types/header.d.ts index ccd123a..a6cebf7 100644 --- a/types/header.d.ts +++ b/types/header.d.ts @@ -79,6 +79,10 @@ export type ZuTemporalValue = typeof globalThis extends { * default and `Temporal` values on a connection opened with * `{ temporal: true }`. A time with an offset is the exception in both * directions: `Temporal` has no type for one, so it stays a `ZuTime`. + * + * BYTES is a `Uint8Array` and not a string. The bytes are octets and + * need not be text at all, so decoding them is the caller's call to + * make rather than this client's to make for them. */ export type ZuValue = | null @@ -94,6 +98,7 @@ export type ZuValue = | ZuTimestamp | ZuDuration | ZuTemporalValue + | Uint8Array | ZuValue[] | { [field: string]: ZuValue } @@ -102,11 +107,18 @@ export type ZuValue = * * Wider than what comes out, because a `number` that is whole binds as * INT64 and `undefined` binds as null, which is what makes an optional - * field of a plain object pass straight through. A `Temporal` value - * binds as the zu value it is on every connection, whether or not the - * connection asked for `Temporal` on the way out, because recognizing - * one costs a property read and refusing one would be a rule nobody - * could guess. + * field of a plain object pass straight through. Negative zero is the + * exception and binds as FLOAT64: no INT64 is negative zero, so binding + * it as one throws away the sign the caller went out of their way to + * write. A `Temporal` value binds as the zu value it is on every + * connection, whether or not the connection asked for `Temporal` on the + * way out, because recognizing one costs a property read and refusing + * one would be a rule nobody could guess. + * + * A `Uint8Array` binds as BYTES, and it is the only typed array that + * binds at all: an `Int32Array` is a buffer somebody meant to load + * rather than a value a statement holds, so it is refused instead of + * being read as the empty object it has no properties to be. */ export type ZuParam = | null @@ -120,6 +132,7 @@ export type ZuParam = | ZuTimestamp | ZuDuration | ZuTemporalValue + | Uint8Array | ZuParam[] | { [field: string]: ZuParam } @@ -155,10 +168,11 @@ export type ZuAppendValue = * One value of a registered frame's column, when the column is written * as a plain array. * - * The same values an appender takes, without the bytes: a column of - * BYTES is a column no statement can read back yet, so registering one - * would be naming data the caller cannot get at. There is no `null` - * either, for the reason there is none in a row of an appender. + * The same values an appender takes, without the bytes: a frame column + * is a run of values the engine reads where it lies, and byte strings + * are not a run of anything, so a BYTES column is refused rather than + * copied into a shape it does not have. There is no `null` either, for + * the reason there is none in a row of an appender. */ export type ZuFrameValue = | boolean @@ -299,6 +313,20 @@ export interface ZuNotice { * `zone`. `value` is the fallback for what no fixed width cell covers, * which is nodes, rels, paths, lists and records, and `null` is a * column that held nothing else. + * + * `bytes` arrives in the two buffers `string` arrives in and is not + * one: the bytes are octets and a reader that decoded them as text + * would be handed something it cannot decode, which is why the two + * have separate names for one layout. + * + * This is what arrived rather than what the statement declared, and the + * two differ for the temporal types today. The engine's columnar sink + * has no buffer for days, nanoseconds or months, so a date, a time or a + * duration sometimes comes over as the values themselves, and which of + * the two a statement gets is the plan's business rather than the + * caller's. Such a column is `value` with its `ZuDate` and `ZuDuration` + * objects in `items`, not `date` with an empty `values`, so that a + * switch on `type` always lands on a field that holds something. */ export type ZuColumnType = | 'null' @@ -306,6 +334,7 @@ export type ZuColumnType = | 'int' | 'float' | 'string' + | 'bytes' | 'date' | 'time' | 'datetime' @@ -319,8 +348,8 @@ export type ZuColumnType = * not apply, so reading one is a switch on `type` rather than a series * of tests for what is there. Which field carries the values follows * from the type: `values` for everything of a fixed width, `data` and - * `offsets` for strings, `items` for what no buffer covers, and none of - * them for a column of nulls. + * `offsets` for strings and byte strings, `items` for what no buffer + * covers, and none of them for a column of nulls. * * The buffers are the engine's own, handed over rather than copied, and * they are laid out the way Arrow lays them out: values end to end, a @@ -339,12 +368,16 @@ export interface ZuColumn { * booleans, least significant bit first. */ readonly values: BigInt64Array | Float64Array | Int32Array | Uint8Array | null - /** The bytes of every string end to end, for a string column. */ + /** + * The bytes of every value end to end, for a `string` or a `bytes` + * column. + */ readonly data: Uint8Array | null /** - * `length + 1` offsets into `data`, for a string column. Narrow until - * the bytes pass what a 32 bit offset addresses, which is the - * difference Arrow calls Utf8 against LargeUtf8. + * `length + 1` offsets into `data`, for a `string` or a `bytes` + * column. Narrow until the bytes pass what a 32 bit offset + * addresses, which is the difference Arrow calls Utf8 against + * LargeUtf8. */ readonly offsets: Int32Array | BigInt64Array | null /** The values themselves, for a column of type `value`. */ @@ -357,7 +390,11 @@ export interface ZuColumn { readonly validity: Uint8Array | null /** How many rows are null, which is zero when `validity` is null. */ readonly nulls: number - /** What one cell counts: `days`, `nanos` or `months`. */ + /** + * What one cell counts: `days`, `nanos` or `months`. Null where there + * are no cells to count, which includes a temporal column that + * arrived as a `value` column. + */ readonly unit: 'days' | 'nanos' | 'months' | null /** Minutes east of UTC, for a column of zoned times or datetimes. */ readonly zone: number | null