From 2d3039f5a8dffc85a04843401559961c5a133756 Mon Sep 17 00:00:00 2001 From: Tam Nguyen Duc <1218621+tamnd@users.noreply.github.com> Date: Tue, 25 Aug 2026 04:51:26 +0700 Subject: [PATCH] Read and bind a byte string (#37) The issue said the conformance reader was the only thing missing, and that was true of the reader and not of the client. The pin was at 6ee7d80, which is before the engine had a Value::Bytes at all, so there was nothing for to_py to return and nothing for a parameter to become. The pin moves to 230581c and the three places that touch a value gain the arm they were missing. Reading gives bytes and binding takes bytes. Not bytearray and not memoryview: a value that came out of a result is a reading of what the file holds and nothing in Python should be able to write through it, and a parameter is read after the call that takes it returns, so a buffer the caller can still write through is a promise this client would be taking on trust. One type in all three directions, which is the one the loader already names for a byte string column. numpy gets an object array. A byte string column keeps the two buffers a string column keeps, so the walk is the string walk without the UTF-8 check, but the array cannot be an S one: S pads every cell to the longest and drops trailing nulls, which is a different value from the one stored. The reader turns hexits into octets itself rather than calling bytes.fromhex. The two agree on everything except a vertical tab, which fromhex drops and Rust's is_ascii_whitespace does not, and which whitespace fromhex drops has changed across the Python versions this client supports. A reader of the shared corpus that accepts a shade more than the reference one is a reader that lets a malformed case through on one client and not on another, which is the failure the encoding exists to make impossible. The pin bump also made ten words reserved that were not before: on, at, number, nothing, record, small, count, big, day and exact. Four test files used four of them as an alias or a column name and are renamed. Nothing in the client changed for that, only what the tests are allowed to call things. A byte string can be stored and cannot be read back yet, so the loader's refusal at src/load.rs:148 stays where it is. ColType has no byte string in it and the row walk has no arm that produces one, which is tamnd/zu#728. The corpus is 1399 cases, 1393 passed, 0 failed, 6 unsupported, and all six are a time written to the nanosecond, which is a digit finer than a Python datetime holds. The six test files this touches are 360 passed and 2 skipped, the two skips being the corpus tests that want ZU_CASES. --- Cargo.lock | 74 ++++++++++++++++++++++++++------------- Cargo.toml | 6 ++-- README.md | 2 +- conformance/values.py | 58 ++++++++++++++++++++++++++++-- python/zudb/types.py | 5 ++- src/numpy.rs | 21 ++++++++++- src/value.rs | 22 ++++++++++-- tests/test_conformance.py | 54 ++++++++++++++++++++++++++++ tests/test_dbapi.py | 2 +- tests/test_html.py | 2 +- tests/test_load.py | 2 +- tests/test_register.py | 8 ++--- tests/test_values.py | 31 ++++++++++++++++ 13 files changed, 246 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a18318..e6b06cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -377,9 +377,9 @@ checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" [[package]] name = "foldhash" -version = "0.1.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "form_urlencoded" @@ -495,9 +495,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.5" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "foldhash", ] @@ -507,14 +507,17 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] [[package]] name = "hashlink" -version = "0.10.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +checksum = "32069d97bb81e38fa67eab65e3393bf804bb85969f2bc06bf13f64aef5aba248" dependencies = [ - "hashbrown 0.15.5", + "hashbrown 0.17.1", ] [[package]] @@ -770,9 +773,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libsqlite3-sys" -version = "0.35.0" +version = "0.38.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" dependencies = [ "cc", "pkg-config", @@ -1135,11 +1138,21 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror", +] + [[package]] name = "rusqlite" -version = "0.37.0" +version = "0.40.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" dependencies = [ "bitflags", "fallible-iterator", @@ -1147,6 +1160,7 @@ dependencies = [ "hashlink", "libsqlite3-sys", "smallvec", + "sqlite-wasm-rs", ] [[package]] @@ -1172,9 +1186,9 @@ checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "ruzstd" -version = "0.8.3" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7c1c839d570d835527c9a5e4db7cb2198683a988cb9d7293fc8674e6bd58fc8" +checksum = "a252f5e20f038fe7b4ea53e073e65398d652c864cc162fc77c56c2f13717b888" dependencies = [ "twox-hash", ] @@ -1259,6 +1273,18 @@ version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -1709,7 +1735,7 @@ dependencies = [ [[package]] name = "zu" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" +source = "git+https://github.com/tamnd/zu?rev=230581cdbeb832563d45c48e70436ff03d2d9b54#230581cdbeb832563d45c48e70436ff03d2d9b54" dependencies = [ "zu-common", "zu-encoding", @@ -1725,7 +1751,7 @@ dependencies = [ [[package]] name = "zu-arrow" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" +source = "git+https://github.com/tamnd/zu?rev=230581cdbeb832563d45c48e70436ff03d2d9b54#230581cdbeb832563d45c48e70436ff03d2d9b54" dependencies = [ "arrow", "zu-common", @@ -1735,7 +1761,7 @@ dependencies = [ [[package]] name = "zu-common" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" +source = "git+https://github.com/tamnd/zu?rev=230581cdbeb832563d45c48e70436ff03d2d9b54#230581cdbeb832563d45c48e70436ff03d2d9b54" dependencies = [ "thiserror", ] @@ -1743,7 +1769,7 @@ dependencies = [ [[package]] name = "zu-encoding" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" +source = "git+https://github.com/tamnd/zu?rev=230581cdbeb832563d45c48e70436ff03d2d9b54#230581cdbeb832563d45c48e70436ff03d2d9b54" dependencies = [ "ruzstd", "zu-common", @@ -1752,7 +1778,7 @@ dependencies = [ [[package]] name = "zu-exec" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" +source = "git+https://github.com/tamnd/zu?rev=230581cdbeb832563d45c48e70436ff03d2d9b54#230581cdbeb832563d45c48e70436ff03d2d9b54" dependencies = [ "zu-common", "zu-query", @@ -1762,7 +1788,7 @@ dependencies = [ [[package]] name = "zu-query" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" +source = "git+https://github.com/tamnd/zu?rev=230581cdbeb832563d45c48e70436ff03d2d9b54#230581cdbeb832563d45c48e70436ff03d2d9b54" dependencies = [ "crossbeam-deque", "zu-common", @@ -1773,7 +1799,7 @@ dependencies = [ [[package]] name = "zu-s3" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" +source = "git+https://github.com/tamnd/zu?rev=230581cdbeb832563d45c48e70436ff03d2d9b54#230581cdbeb832563d45c48e70436ff03d2d9b54" dependencies = [ "crc32c", "object_store", @@ -1784,7 +1810,7 @@ dependencies = [ [[package]] name = "zu-sqlite" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" +source = "git+https://github.com/tamnd/zu?rev=230581cdbeb832563d45c48e70436ff03d2d9b54#230581cdbeb832563d45c48e70436ff03d2d9b54" dependencies = [ "rusqlite", "zu-common", @@ -1794,7 +1820,7 @@ dependencies = [ [[package]] name = "zu-storage" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" +source = "git+https://github.com/tamnd/zu?rev=230581cdbeb832563d45c48e70436ff03d2d9b54#230581cdbeb832563d45c48e70436ff03d2d9b54" dependencies = [ "zu-common", "zu-encoding", @@ -1803,7 +1829,7 @@ dependencies = [ [[package]] name = "zu-vector" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" +source = "git+https://github.com/tamnd/zu?rev=230581cdbeb832563d45c48e70436ff03d2d9b54#230581cdbeb832563d45c48e70436ff03d2d9b54" dependencies = [ "zu-common", ] @@ -1811,7 +1837,7 @@ dependencies = [ [[package]] name = "zu-zu1" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" +source = "git+https://github.com/tamnd/zu?rev=230581cdbeb832563d45c48e70436ff03d2d9b54#230581cdbeb832563d45c48e70436ff03d2d9b54" dependencies = [ "crc32c", "loom", diff --git a/Cargo.toml b/Cargo.toml index e494801..8ee6cc2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,14 +18,14 @@ crate-type = ["cdylib"] # with (ADR 0002), so a revision is the honest way to say which one. # A local checkout is used instead with a `paths` override in # `.cargo/config.toml`, which is untracked on purpose. -zudb = { package = "zu", git = "https://github.com/tamnd/zu", rev = "6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" } -zu-common = { git = "https://github.com/tamnd/zu", rev = "6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" } +zudb = { package = "zu", git = "https://github.com/tamnd/zu", rev = "230581cdbeb832563d45c48e70436ff03d2d9b54" } +zu-common = { git = "https://github.com/tamnd/zu", rev = "230581cdbeb832563d45c48e70436ff03d2d9b54" } # The one translation from a result into Arrow, which lives in the # engine tree so that every client agrees about what a column becomes. # `ffi` is the only feature this client turns on: what Python wants is # the C Data Interface, which is how a result reaches pyarrow, pandas # and polars without a Python object per cell. -zu-arrow = { git = "https://github.com/tamnd/zu", rev = "6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a", features = ["ffi"] } +zu-arrow = { git = "https://github.com/tamnd/zu", rev = "230581cdbeb832563d45c48e70436ff03d2d9b54", features = ["ffi"] } # `extension-module` is asked for by maturin, in pyproject.toml, and # not here. Only the build backend knows how an extension is linked on # the platform it is building for, and a crate that turns the feature diff --git a/README.md b/README.md index d32f5a0..39bf8cb 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ result.fetchnumpy() # {"name": array([...]), "score": array([...])} `Result` implements `__arrow_c_stream__`, so anything that reads the protocol reads a result directly and none of the four methods above is needed: `pyarrow.table(result)` and `polars.DataFrame(result)` both work. Batches are 65,536 rows, or whatever `record_batches(rows_per_batch)` asks for, and the size costs nothing either way: the arrays are built once and a batch is a slice of them rather than a copy. A column holds one type, which the values decide, and integers beside floats are the one mixture that widens rather than being refused. A result that matched no rows still says what its columns hold, so a query that found nothing can still be written to Parquet or appended to a table that already exists. Nodes, rels and paths go across as structs. The translation runs with the GIL released, and on this machine 300,000 rows across three columns take 4.8 ms as Arrow against 86 ms as Python objects, and a single integer column takes 0.8 ms against 43 ms. That is a wider gap than it used to be because the engine now fills the column buffers during the scan rather than transposing the rows afterwards, so what is left here is putting an Arrow type around a buffer that already exists. -`fetchnumpy()` is the same columns as numpy arrays, keyed by column name, for the code that takes arrays rather than frames. It needs numpy and nothing else: an integer, float, datetime or duration column is the engine's own buffer moved into numpy and named, so there is no pass over the values and no second copy of the result in memory. Over a million rows in two columns on this machine it takes 9.9 ms, against 11.9 ms for building the Arrow table and calling `to_numpy` on each of its columns, and 199 ms for the same rows as tuples. Dates are `datetime64[D]`, datetimes `datetime64[ns]`, and durations and times of day `timedelta64[ns]`, the last of those being nanoseconds since midnight, which is what a clock reading is on a number line and the closest thing numpy has to one. A column with a null in it comes back as a `numpy.ma.masked_array`, since numpy has no missing integer, and the mask is built from the validity bitmap the engine already filled rather than by walking the column again. Strings, nodes, rels, paths, lists and records come back as object arrays with `None` in the cell, because an object array has somewhere to put one. +`fetchnumpy()` is the same columns as numpy arrays, keyed by column name, for the code that takes arrays rather than frames. It needs numpy and nothing else: an integer, float, datetime or duration column is the engine's own buffer moved into numpy and named, so there is no pass over the values and no second copy of the result in memory. Over a million rows in two columns on this machine it takes 9.9 ms, against 11.9 ms for building the Arrow table and calling `to_numpy` on each of its columns, and 199 ms for the same rows as tuples. Dates are `datetime64[D]`, datetimes `datetime64[ns]`, and durations and times of day `timedelta64[ns]`, the last of those being nanoseconds since midnight, which is what a clock reading is on a number line and the closest thing numpy has to one. A column with a null in it comes back as a `numpy.ma.masked_array`, since numpy has no missing integer, and the mask is built from the validity bitmap the engine already filled rather than by walking the column again. Strings, byte strings, nodes, rels, paths, lists and records come back as object arrays with `None` in the cell, because an object array has somewhere to put one. A byte string is an object array rather than an `S` one because `S` pads every cell to the longest and drops trailing nulls, which is a different value from the one stored. ## Reading a result as it arrives diff --git a/conformance/values.py b/conformance/values.py index 669c747..060acef 100644 --- a/conformance/values.py +++ b/conformance/values.py @@ -74,6 +74,11 @@ "FLOAT32": True, "FLOAT64": True, "STRING": False, + # A byte string is written in quotes because its hexits are digits + # as often as not: a bare 0041 is a number with a leading zero in + # one reader and the string it looks like in another, and neither of + # them is the two octets the case meant. + "BYTES": True, "DATE": True, "LOCALTIME": True, "ZONEDTIME": True, @@ -93,8 +98,10 @@ #: The types the encoding reserves a name for and the engine has no #: runtime value for yet, kept apart from an outright typo so that the -#: error says which of the two it is. -_RESERVED = ("DECIMAL", "BYTES") +#: error says which of the two it is. BYTES was here and is not any +#: more: the engine gained the value in tamnd/zu#543 and this reader +#: gained it with the pin that brought it in. +_RESERVED = ("DECIMAL",) #: The range each integer width holds, so that a case writing a value #: its own type cannot carry is refused rather than stored wider than it @@ -400,6 +407,8 @@ def _scalar(ty: str, text: str) -> object: return {"true": True, "false": False}.get(text, _NOT_ONE) if ty == "STRING": return text + if ty == "BYTES": + return _hexits(text) if ty in _RANGES: try: n = int(text) @@ -440,6 +449,46 @@ def _scalar(ty: str, text: str) -> object: return _NOT_ONE +#: What each hexit is worth, both cases of the six letters, because ISO +#: writes the literal in upper case and a case is free to write either. +#: A table rather than ``int(c, 16)``, which also takes the digits of +#: every other script and would read an Arabic-Indic three as a three. +_HEXITS = {c: n for n, c in enumerate("0123456789abcdef")} | { + c: n + 10 for n, c in enumerate("ABCDEF") +} + +#: What the reference reader drops between hexits, which is Rust's +#: `is_ascii_whitespace`: a vertical tab is not in it. +_SPACE = " \t\n\r\f" + + +def _hexits(text: str) -> object: + """The octets a run of hexits names, or ``_NOT_ONE`` for text that is + not a run of them or that names half a byte. + + Space is allowed anywhere and dropped, which is what the standard's + production allows and what lets a long literal be written in groups. + + Written out rather than handed to ``bytes.fromhex``, which is the + same function on two counts and not on a third: it drops a vertical + tab as well, and which whitespace it drops has changed between + Python versions this client supports. A second reader of the corpus + that accepts a shade more than the first is a reader that lets a + malformed case through on one client and not on another, which is + the failure this whole module exists to make impossible.""" + nibbles: list[int] = [] + for c in text: + if c in _SPACE: + continue + nibble = _HEXITS.get(c) + if nibble is None: + return _NOT_ONE + nibbles.append(nibble) + if len(nibbles) % 2: + return _NOT_ONE + return bytes((high << 4) | low for high, low in zip(nibbles[::2], nibbles[1::2], strict=True)) + + def _parse(text: str, fn) -> object: try: return fn(text) @@ -634,6 +683,11 @@ def show(value: object) -> str: return f'FLOAT64 "{_show_float(value)}"' if isinstance(value, str): return f"STRING {quote(value)}" + # Upper case because ISO writes the literal that way, and a report + # that is diffed against the reference one is comparing text: one + # case is one answer. + if isinstance(value, bytes): + return f'BYTES "{value.hex().upper()}"' if isinstance(value, Duration): return f'DURATION "{_show_duration(value)}"' if isinstance(value, TooFine): diff --git a/python/zudb/types.py b/python/zudb/types.py index 691706e..347cab0 100644 --- a/python/zudb/types.py +++ b/python/zudb/types.py @@ -20,13 +20,16 @@ #: because a list holds values and one of them may be a list. A #: ``timedelta`` goes in and never comes out: zu stores it as a day-time #: duration and hands one back, since a ``timedelta`` cannot hold every -#: duration zu can. +#: duration zu can. ``bytes`` and not ``bytearray``, because a parameter +#: is read after the call that takes it returns and a mutable buffer is a +#: promise the caller can break. Value: TypeAlias = ( None | bool | int | float | str + | bytes | datetime.date | datetime.time | datetime.datetime diff --git a/src/numpy.rs b/src/numpy.rs index 1ec865a..75121d9 100644 --- a/src/numpy.rs +++ b/src/numpy.rs @@ -31,7 +31,7 @@ use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::*; -use pyo3::types::PyDict; +use pyo3::types::{PyBytes, PyDict}; use zudb::query::QueryResult; use zudb::query::column::{Column, ColumnData, ColumnType, Offsets, Validity}; @@ -102,6 +102,25 @@ fn column<'py>( } return objects(py, out); } + // The same two buffers a string column keeps, over bytes that + // are not text, so the walk is the string walk without the + // UTF-8 check. An object array for the same reason: numpy has + // no dtype for a run of octets whose length varies by row, and + // `S` pads every cell to the longest one and drops trailing + // nulls, which is a different value from the one stored. + ColumnData::Bytes(octets) => { + let mut out = Vec::with_capacity(len); + let bytes = &octets.bytes; + let mut spans = spans(&octets.offsets); + for at in 0..len { + let (from, upto) = spans.next().unwrap_or((0, 0)); + out.push(match missing(&valid, at) { + true => py.None(), + false => PyBytes::new(py, &bytes[from..upto]).into_any().unbind(), + }); + } + return objects(py, out); + } ColumnData::Complex(values) => { let mut out = Vec::with_capacity(values.len()); for value in values { diff --git a/src/value.rs b/src/value.rs index fd49a86..0331381 100644 --- a/src/value.rs +++ b/src/value.rs @@ -10,7 +10,9 @@ use std::collections::HashMap; use pyo3::prelude::*; -use pyo3::types::{PyBool, PyDate, PyDateTime, PyDelta, PyDict, PyList, PyTime, PyTuple, PyTzInfo}; +use pyo3::types::{ + PyBool, PyBytes, PyDate, PyDateTime, PyDelta, PyDict, PyList, PyTime, PyTuple, PyTzInfo, +}; use zu_common::temporal::{NANOS_PER_DAY, NANOS_PER_MINUTE, civil_from_days, days_from_civil}; use zu_common::{DurationKind, Temporal}; use zudb::query::Value; @@ -312,6 +314,13 @@ pub fn to_py<'py>(py: Python<'py>, value: &Value, names: &Names) -> PyResult n.into_pyobject(py)?.into_any(), Value::Float(f) => f.into_pyobject(py)?.into_any(), Value::Str(s) => s.into_pyobject(py)?.into_any(), + // `bytes` and not `bytearray`, because a value that came out of + // a result is a reading of what the file holds and nothing in + // Python should be able to write through it. It is the same + // type this client takes for a byte string parameter and the + // same one the loader takes for a byte string column, so a + // round trip through any of the three is one type. + Value::Bytes(b) => PyBytes::new(py, b).into_any(), Value::Node { table, offset } => Node { table: names.node(*table), offset: *offset, @@ -510,6 +519,15 @@ fn nested(value: &Bound<'_, PyAny>, depth: usize) -> PyResult { if let Ok(s) = value.extract::() { return Ok(Value::Str(s)); } + // `bytes` and nothing else that holds octets. A `bytearray` is + // mutable and a `memoryview` is a window onto something that may be, + // and a parameter is read after this call returns, so taking either + // would be taking a promise the caller can break. It is the same + // type the loader takes for a byte string column, which is the point + // of picking one. + if let Ok(b) = value.cast::() { + return Ok(Value::Bytes(b.as_bytes().to_vec())); + } if let Ok(n) = value.extract::() { return Ok(Value::Int(n)); } @@ -634,6 +652,6 @@ fn refused(value: &Bound<'_, PyAny>) -> PyErr { .and_then(|name| name.extract::()) .unwrap_or_else(|_| "that".to_string()); pyo3::exceptions::PyTypeError::new_err(format!( - "a parameter cannot be a {name}: zu holds nulls, booleans, integers, floats, strings, lists, records, dates, times, datetimes and durations" + "a parameter cannot be a {name}: zu holds nulls, booleans, integers, floats, strings, byte strings, lists, records, dates, times, datetimes and durations" )) } diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 07e3102..97f2787 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -389,6 +389,60 @@ def test_a_graph_value_prints_the_way_a_case_writes_one() -> None: ) +def test_a_byte_string_is_read_from_its_hexits_in_either_case() -> None: + assert value('type: BYTES\nvalue: "00AB00"\n') == b"\x00\xab\x00" + assert value('type: BYTES\nvalue: "00ab00"\n') == b"\x00\xab\x00" + # Space anywhere and dropped, which is what lets a long literal be + # written in groups. All of the whitespace the reference reader + # drops, which is Rust's `is_ascii_whitespace`. + assert value('type: BYTES\nvalue: "00 AB 00"\n') == b"\x00\xab\x00" + assert value('type: BYTES\nvalue: "00\\tAB\\n00\\r\\f"\n') == b"\x00\xab\x00" + # The empty byte string is a value and a case is written for it, so + # it needs a spelling, and the spelling is nothing between quotes. + assert value('type: BYTES\nvalue: ""\n') == b"" + + +@pytest.mark.parametrize( + ("text", "want"), + [ + # Half a byte is not a byte string, and the two hexits a byte + # takes are the whole reason a case writes an even number. + ('type: BYTES\nvalue: "abc"\n', "is not a BYTES"), + ('type: BYTES\nvalue: "zz"\n', "is not a BYTES"), + # A digit of another script reads as a digit to `int` and to + # nothing else in the family, so it is refused here too. + ('type: BYTES\nvalue: "٣٣"\n', "is not a BYTES"), + # A backspace is a control character and not the whitespace the + # reference reader drops, so it is a character that is not a + # hexit rather than a gap between two of them. + ('type: BYTES\nvalue: "00\\b00"\n', "is not a BYTES"), + # And the quoting rule, which is the one the encoding exists for: + # a bare 0041 is a number with a leading zero in one reader and + # the string it looks like in another. + ("type: BYTES\nvalue: 0041\n", "BYTES is written in quotes"), + ], +) +def test_a_byte_string_written_wrong_is_refused(text: str, want: str) -> None: + with pytest.raises(reader.CorpusError) as raised: + value(text) + assert want in str(raised.value) + + +def test_a_byte_string_prints_the_way_a_case_writes_one() -> None: + assert values.show(b"\x00\xab\x00") == 'BYTES "00AB00"' + assert values.show(b"") == 'BYTES ""' + + +def test_a_byte_string_is_not_the_string_that_spells_the_same_octets() -> None: + # The case at `string.yaml` asserting that `X'0041' = 'A'` is not + # true is only worth running if this reader keeps the two apart, and + # a reader that decoded octets into a `str` somewhere would pass it + # for the wrong reason. + assert not values.same(b"A", "A") + assert not values.same("A", b"A") + assert values.same(b"A", b"A") + + def test_a_type_the_engine_cannot_hold_yet_says_so_rather_than_looking_like_a_typo() -> None: with pytest.raises(reader.CorpusError) as raised: value('type: DECIMAL\nvalue: "1.00"\n') diff --git a/tests/test_dbapi.py b/tests/test_dbapi.py index dd12244..5c5ddc8 100644 --- a/tests/test_dbapi.py +++ b/tests/test_dbapi.py @@ -125,7 +125,7 @@ def test_description_names_the_columns_and_their_types(conn: dbapi.Connection) - def test_a_column_that_is_null_the_whole_way_down_has_no_type(conn: dbapi.Connection) -> None: """There is nothing to read a type off, and a guess would be a lie.""" cur = conn.cursor() - cur.execute("MATCH (p:person) RETURN p.name AS name, null AS nothing") + cur.execute("MATCH (p:person) RETURN p.name AS name, null AS missing") assert [column[1] for column in cur.description] == [str, None] diff --git a/tests/test_html.py b/tests/test_html.py index a593cc0..3030d39 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -68,7 +68,7 @@ def test_a_boolean_is_not_a_number_to_line_up(empty: zudb.Connection) -> None: def test_a_null_says_so_rather_than_being_blank(empty: zudb.Connection) -> None: empty.execute("INSERT (p:person {uid: 1, name: 'ada'})") - html = empty.execute("MATCH (p:person) RETURN null AS nothing")._repr_html_() + html = empty.execute("MATCH (p:person) RETURN null AS missing")._repr_html_() assert cells(html) == ["null"] assert 'class="zu-null"' in html diff --git a/tests/test_load.py b/tests/test_load.py index c392615..8ffcb2f 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -108,7 +108,7 @@ def test_the_same_edge_twice_is_one_edge(tmp_path: Path) -> None: def test_a_column_of_every_kind_reads_back_as_what_it_was(tmp_path: Path) -> None: columns = { - "count": [1, -2], + "tally": [1, -2], "ratio": [1.5, -0.25], "flag": [True, False], "name": ["ada", "grace"], diff --git a/tests/test_register.py b/tests/test_register.py index fc4ce5e..c25a1d6 100644 --- a/tests/test_register.py +++ b/tests/test_register.py @@ -93,11 +93,11 @@ def test_every_kind_of_column_a_row_can_hold_arrives_as_itself(empty: zudb.Conne frame = pa.table( { "yes": pa.array([True, False]), - "small": pa.array([1, 2], pa.int8()), + "tiny": pa.array([1, 2], pa.int8()), "wide": pa.array([3, 4], pa.uint32()), "narrow": pa.array([1.5, 2.5], pa.float32()), "word": pa.array(["a", "b"]), - "day": pa.array([datetime.date(2024, 1, 1), datetime.date(2024, 2, 1)]), + "birthday": pa.array([datetime.date(2024, 1, 1), datetime.date(2024, 2, 1)]), "clock": pa.array([datetime.time(1, 2, 3), datetime.time(4, 5, 6)], pa.time64("us")), "moment": pa.array( [datetime.datetime(2024, 1, 1, 1, 2, 3), datetime.datetime(2024, 2, 1)] @@ -107,8 +107,8 @@ def test_every_kind_of_column_a_row_can_hold_arrives_as_itself(empty: zudb.Conne ) assert empty.register("kinds", frame) == 2 row = empty.execute( - "MATCH (k:kinds) RETURN k.yes AS yes, k.small AS small, k.wide AS wide, " - "k.narrow AS narrow, k.word AS word, k.day AS day, k.clock AS clock, " + "MATCH (k:kinds) RETURN k.yes AS yes, k.tiny AS tiny, k.wide AS wide, " + "k.narrow AS narrow, k.word AS word, k.birthday AS birthday, k.clock AS clock, " "k.moment AS moment, k.span AS span" ).fetchone() assert row == ( diff --git a/tests/test_values.py b/tests/test_values.py index a231e0a..eecfbea 100644 --- a/tests/test_values.py +++ b/tests/test_values.py @@ -29,6 +29,8 @@ def test_two_reads_of_one_node_are_equal_and_hash_alike(social: zudb.Connection) ("RETURN 1 AS v", 1), ("RETURN 1.5 AS v", 1.5), ("RETURN 'ada' AS v", "ada"), + ("RETURN X'00AB00' AS v", b"\x00\xab\x00"), + ("RETURN X'' AS v", b""), ("RETURN true AS v", True), ("RETURN false AS v", False), ("RETURN null AS v", None), @@ -45,6 +47,35 @@ def test_a_literal_reads_back_as_the_python_object_it_is( assert type(got) is type(answer) +def test_a_byte_string_is_bytes_and_not_the_string_that_spells_it( + empty: zudb.Connection, +) -> None: + """A byte string is octets and a string is characters, and the whole + reason the type exists is that the two are not the same value.""" + (same,) = empty.execute("RETURN X'0041' = 'A' AS v").fetchone() + assert same is not True + got = empty.execute("RETURN X'0041' AS v").fetchone()[0] + assert got == b"\x00A" + assert got != "\x00A" + + +def test_a_byte_string_goes_in_as_a_parameter_and_comes_back_the_same( + empty: zudb.Connection, +) -> None: + got = empty.execute("RETURN $b AS v", {"b": b"\xde\xad\xbe\xef"}).fetchone()[0] + assert got == b"\xde\xad\xbe\xef" + assert type(got) is bytes + + +def test_a_mutable_buffer_is_not_a_byte_string_parameter(empty: zudb.Connection) -> None: + """A parameter is read after the call that takes it returns, so a + buffer the caller can still write through is a promise this client + does not take. The refusal names the types it does take, which is + the shortest way to say `bytes(...)`.""" + with pytest.raises(TypeError, match="byte strings"): + empty.execute("RETURN $b AS v", {"b": bytearray(b"\x00")}) + + def test_a_duration_counts_months_or_nanoseconds_and_never_both() -> None: with pytest.raises(ValueError, match="never both"): zudb.Duration(months=1, nanoseconds=1)