From 837ca81cb7ecbb1e1b8e66eb0e1f40fb0ea18f71 Mon Sep 17 00:00:00 2001 From: Tam Nguyen Duc <1218621+tamnd@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:35:25 +0700 Subject: [PATCH] Read schema 4 of the corpus The corpus moved to schema 4 in the engine and this client was two schemas behind it, so the pin bump brings 1268 cases across a reader that knew about none of what the last two added. Three things arrive together because they arrive in the same files. Connections. A case may name the connection it runs on, and a setup statement may name its own, so that a case about a transaction can say what the other connection sees. A named connection is made the first time a case names one, by duplicating the connection the case opened with, because the two then share the write side and that is what a pool hands out. Opening the path twice would be two databases that happen to be the same file, which is a different question and not the one a case about a transaction asks. Graph values. A node is its table and the row it is, an edge is its table and the two rows it runs between, and a path is a sequence that alternates the two and holds an odd number of them. The engine's edge carries the position it holds among the edges out of its source, which the loader chose and no case writes down, so a value is normalised into the corpus's own shape before it is compared rather than compared against a number nobody picked. The export. A case may say what the result gives through Arrow, field by field and into the nested types, or say that Arrow refuses it. That is read off the C Data Interface with ctypes rather than through pyarrow, for two reasons. pyarrow is not installed to run the corpus, so a check that needed it would be a check that quietly does not run. And the C runner reads the interface directly too, so the two runners are looking at the same bytes and reporting them in the same words, which is the whole reason a case writes a format string down. The corpus run is a session fixture now. Every case gets a database of its own and there are over a thousand, so two tests reading one report were four gigabytes and three pytest generations of it, which filled a disk. One run, in a directory that goes when the run does. 1268 cases, 1262 passed, 0 failed, 6 unsupported. All six are a time written to the nanosecond, which is a digit finer than a Python datetime holds, and each one says so. --- Cargo.lock | 22 +-- Cargo.toml | 6 +- conformance/arrow.py | 309 +++++++++++++++++++++++++++++++++ conformance/cases.py | 94 ++++++++-- conformance/runner.py | 89 +++++++++- conformance/values.py | 171 ++++++++++++++++++- tests/test_conformance.py | 351 ++++++++++++++++++++++++++++++++++++-- 7 files changed, 981 insertions(+), 61 deletions(-) create mode 100644 conformance/arrow.py diff --git a/Cargo.lock b/Cargo.lock index 92d1c36..0a18318 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1709,7 +1709,7 @@ dependencies = [ [[package]] name = "zu" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "zu-common", "zu-encoding", @@ -1725,7 +1725,7 @@ dependencies = [ [[package]] name = "zu-arrow" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "arrow", "zu-common", @@ -1735,7 +1735,7 @@ dependencies = [ [[package]] name = "zu-common" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "thiserror", ] @@ -1743,7 +1743,7 @@ dependencies = [ [[package]] name = "zu-encoding" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "ruzstd", "zu-common", @@ -1752,7 +1752,7 @@ dependencies = [ [[package]] name = "zu-exec" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "zu-common", "zu-query", @@ -1762,7 +1762,7 @@ dependencies = [ [[package]] name = "zu-query" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "crossbeam-deque", "zu-common", @@ -1773,7 +1773,7 @@ dependencies = [ [[package]] name = "zu-s3" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "crc32c", "object_store", @@ -1784,7 +1784,7 @@ dependencies = [ [[package]] name = "zu-sqlite" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "rusqlite", "zu-common", @@ -1794,7 +1794,7 @@ dependencies = [ [[package]] name = "zu-storage" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "zu-common", "zu-encoding", @@ -1803,7 +1803,7 @@ dependencies = [ [[package]] name = "zu-vector" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "zu-common", ] @@ -1811,7 +1811,7 @@ dependencies = [ [[package]] name = "zu-zu1" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "crc32c", "loom", diff --git a/Cargo.toml b/Cargo.toml index afc30a5..0466f3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,14 +18,14 @@ crate-type = ["cdylib"] # with (ADR 0002), so a revision is the honest way to say which one. # A local checkout is used instead with a `paths` override in # `.cargo/config.toml`, which is untracked on purpose. -zudb = { package = "zu", git = "https://github.com/tamnd/zu", rev = "0698a4eccd31670f0a875b6d097d7753440a51b3" } -zu-common = { git = "https://github.com/tamnd/zu", rev = "0698a4eccd31670f0a875b6d097d7753440a51b3" } +zudb = { package = "zu", git = "https://github.com/tamnd/zu", rev = "6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" } +zu-common = { git = "https://github.com/tamnd/zu", rev = "6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" } # The one translation from a result into Arrow, which lives in the # engine tree so that every client agrees about what a column becomes. # `ffi` is the only feature this client turns on: what Python wants is # the C Data Interface, which is how a result reaches pyarrow, pandas # and polars without a Python object per cell. -zu-arrow = { git = "https://github.com/tamnd/zu", rev = "0698a4eccd31670f0a875b6d097d7753440a51b3", features = ["ffi"] } +zu-arrow = { git = "https://github.com/tamnd/zu", rev = "6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a", features = ["ffi"] } # `extension-module` is asked for by maturin, in pyproject.toml, and # not here. Only the build backend knows how an extension is linked on # the platform it is building for, and a crate that turns the feature diff --git a/conformance/arrow.py b/conformance/arrow.py new file mode 100644 index 0000000..29310c6 --- /dev/null +++ b/conformance/arrow.py @@ -0,0 +1,309 @@ +"""What a result looks like on the way out through Arrow. + +A client that reads rows one at a time and a client that exports a +million of them to a dataframe are the same client, and only one of +those paths is covered by a case that asserts values. The other one has +its own contract: a column of dates is a ``Date32`` and not a string of +digits, a year-month duration is a month-day-nano interval because that +is the interval every reader implements, a node is a struct of the name +of its table and the row it is, and a time with an offset is refused +rather than quietly moved to UTC. None of that shows up in a row a case +compares. + +So a case may say what the export gives as well as what the rows are, +and the runner checks both against one statement. What it checks is the +schema, field by field and into the nested types, and how many rows came +back through the stream. The schema is spelled in the C Data Interface's +own format strings, ``l`` for an int64 and ``+s`` for a struct, because +that is the one spelling every language sees the same. + +The schema is read off the interface itself rather than out of +``pyarrow``. Two reasons, and the second is the one that decided it. The +first is that ``pyarrow`` is not installed to run the corpus and a check +that needed it would be a check that quietly does not run. The second is +that the C Data Interface is what the C runner has at this point too, so +the two read the same bytes and report them in the same words, which is +the whole reason a format string is what the case writes down. + +Values are not read back here. A consumer that decoded every array by +hand in each of nine languages would be nine new decoders under test, +which is more of our own code and not more of the contract; the rows the +case already asserts are the same values by another road. +""" + +from __future__ import annotations + +import ctypes +from dataclasses import dataclass +from dataclasses import field as _field +from typing import Any + +from .reader import CorpusError, Node, quote + +__all__ = ["ArrowError", "Field", "Refused", "REFUSED", "RESULT", "parse", "exported", "schema"] + + +@dataclass(frozen=True) +class Field: + """One field of the schema an export gives, and the fields under it + when it is a struct or a list. + + A list has exactly one field under it, which Arrow names ``item``, + and a case writes that out rather than leaving it implied: a client + that named it ``element`` would export something no reader lines up + with what another client wrote.""" + + name: str + #: The C Data Interface format string, ``l`` for an int64, ``u`` for + #: a string, ``tsn:`` for a timestamp in nanoseconds with no zone. + format: str + children: tuple[Field, ...] = _field(default_factory=tuple) + + +class Refused: + """Arrow has no type for one of the columns, so there is no export to + describe. + + A time with an offset is the one a statement can write today: Arrow + has a time and a timestamp and nothing in between, and dropping the + offset would move the value.""" + + def __repr__(self) -> str: + return "refused" + + +#: The one value of the class above, so that what a case says about the +#: export is either a list of fields or this. +REFUSED = Refused() + +#: How a report names the whole result, which is the place the columns of +#: an export are in. +RESULT = "the result" + + +def parse(node: Node) -> list[Field] | Refused: + """The ``arrow:`` of a case, or what is wrong with it.""" + text = node.str_() + if text is not None: + if text == "refused": + return REFUSED + raise CorpusError( + f"line {node.line}: `arrow:` is the columns the export gives, or `refused` for a " + f"result Arrow has no type for, and this is {quote(text)}" + ) + return _fields(node) + + +def _fields(node: Node) -> list[Field]: + items = node.seq() + if items is None: + raise CorpusError( + f"line {node.line}: `arrow:` is a sequence of fields, and this is {node.what()}" + ) + return [_one(item) for item in items] + + +def _one(node: Node) -> Field: + line = node.line + if node.map() is None: + raise CorpusError( + f"line {line}: an Arrow field is a mapping of `name` and `format`, and this is " + f"{node.what()}" + ) + unknown = node.unknown(("name", "format", "children")) + if unknown: + raise CorpusError(f"line {line}: an Arrow field has no key {quote(unknown[0])}") + + def text(key: str) -> str: + value = node.get(key) + spelled = value.str_() if value is not None else None + if spelled is None: + raise CorpusError(f"line {line}: an Arrow field has a `{key}:`") + return spelled + + name = text("name") + fmt = text("format") + if not fmt: + raise CorpusError(f"line {line}: an empty format string is not a type Arrow has") + children_node = node.get("children") + children = tuple(_fields(children_node)) if children_node is not None else () + # A nested format is the one thing about a format string this reader + # knows, and it is worth knowing here: a case that wrote the fields + # of a struct under a `u` would be asserting something the export + # cannot produce, and finding that out at load time says so with a + # line number rather than as a failure in a report. + nested = fmt.startswith("+") + if nested and not children: + raise CorpusError( + f"line {line}: {quote(fmt)} is a nested type and the fields under it are part of it" + ) + if not nested and children: + raise CorpusError(f"line {line}: {quote(fmt)} holds no fields, so nothing goes under it") + return Field(name, fmt, children) + + +class _Schema(ctypes.Structure): + """``ArrowSchema``, laid out as the C Data Interface writes it.""" + + +_Schema._fields_ = [ + ("format", ctypes.c_char_p), + ("name", ctypes.c_char_p), + ("metadata", ctypes.c_char_p), + ("flags", ctypes.c_int64), + ("n_children", ctypes.c_int64), + ("children", ctypes.POINTER(ctypes.POINTER(_Schema))), + ("dictionary", ctypes.POINTER(_Schema)), + ("release", ctypes.CFUNCTYPE(None, ctypes.POINTER(_Schema))), + ("private_data", ctypes.c_void_p), +] + + +class _Array(ctypes.Structure): + """``ArrowArray``. Only the length is read: the values a case cares + about it already asserts as rows.""" + + +_Array._fields_ = [ + ("length", ctypes.c_int64), + ("null_count", ctypes.c_int64), + ("offset", ctypes.c_int64), + ("n_buffers", ctypes.c_int64), + ("n_children", ctypes.c_int64), + ("buffers", ctypes.POINTER(ctypes.c_void_p)), + ("children", ctypes.POINTER(ctypes.POINTER(_Array))), + ("dictionary", ctypes.POINTER(_Array)), + ("release", ctypes.CFUNCTYPE(None, ctypes.POINTER(_Array))), + ("private_data", ctypes.c_void_p), +] + + +class _Stream(ctypes.Structure): + """``ArrowArrayStream``, which is what a result hands over.""" + + +_Get = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.POINTER(_Stream), ctypes.POINTER(_Schema)) +_Next = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.POINTER(_Stream), ctypes.POINTER(_Array)) + +_Stream._fields_ = [ + ("get_schema", _Get), + ("get_next", _Next), + ("get_last_error", ctypes.CFUNCTYPE(ctypes.c_char_p, ctypes.POINTER(_Stream))), + ("release", ctypes.CFUNCTYPE(None, ctypes.POINTER(_Stream))), + ("private_data", ctypes.c_void_p), +] + + +class ArrowError(Exception): + """The stream said no, with what it said.""" + + +def _pointer(capsule: object, name: bytes, kind: Any) -> Any: + """The pointer inside a capsule, which is how the C Data Interface + travels between two Python objects that have never heard of each + other.""" + get = ctypes.pythonapi.PyCapsule_GetPointer + get.restype = ctypes.c_void_p + get.argtypes = [ctypes.py_object, ctypes.c_char_p] + address = get(capsule, name) + if not address: + raise ArrowError("the result gave a capsule with nothing in it") + return ctypes.cast(address, ctypes.POINTER(kind)) + + +def _walked(one: _Schema) -> Field: + """One field of an exported schema, and everything under it.""" + children = [_walked(one.children[i].contents) for i in range(one.n_children)] + return Field( + name=(one.name or b"").decode("utf-8"), + format=(one.format or b"").decode("utf-8"), + children=tuple(children), + ) + + +def exported(result: Any) -> tuple[list[Field], int]: + """The columns a result gives through Arrow and how many rows came + out of the stream. + + The stream is taken once and both answers come out of that one + taking, because a stream is consumed by reading it and a second + export would be a second statement in all but name. + + A column Arrow cannot hold is found when the stream is asked for, + before a row moves, and this client says so with the Python class the + mistake belongs to: a `TypeError` for a value of a type Arrow does + not have and a `ValueError` for one that will not fit. Both are the + export refusing, which is what a case writing `refused` means, so + both come back out of here under the one name and only around the + call that can raise them.""" + try: + capsule = result.__arrow_c_stream__() + except (TypeError, ValueError) as e: + raise ArrowError(str(e)) from e + stream = _pointer(capsule, b"arrow_array_stream", _Stream) + + def said(code: int) -> str: + text = stream.contents.get_last_error(stream) + return (text or b"").decode("utf-8", "replace") or f"errno {code}" + + schema_out = _Schema() + code = stream.contents.get_schema(stream, ctypes.byref(schema_out)) + if code != 0: + raise ArrowError(said(code)) + try: + # The stream's schema is a struct of the columns, so what the + # case is compared against is the fields under it. + top = _walked(schema_out) + finally: + if schema_out.release: + schema_out.release(ctypes.byref(schema_out)) + + count = 0 + while True: + batch = _Array() + code = stream.contents.get_next(stream, ctypes.byref(batch)) + if code != 0: + raise ArrowError(said(code)) + # A released batch is how the interface says there are no more. + if not batch.release: + break + count += batch.length + batch.release(ctypes.byref(batch)) + return list(top.children), count + + +def schema(got: list[Field], want: list[Field]) -> str | None: + """What the export gave that the case did not want, or ``None`` when + the two agree. + + The comparison walks the schema and the case's fields together and + stops at the first difference, for the reason the row comparison + does: the first is nearly always the cause of the rest.""" + return _fields_of("", got, want) + + +def _fields_of(prefix: str, got: list[Field], want: list[Field]) -> str | None: + """The fields under one place, where the place is the dotted path of + the field they are under and the empty one is the result itself.""" + place = RESULT if not prefix else quote(prefix) + if len(got) != len(want): + return f"arrow gives {len(got)} fields in {place} where the case wants {len(want)}" + for i, (one, expected) in enumerate(zip(got, want, strict=True)): + if one.name != expected.name: + return ( + f"arrow field {i + 1} in {place} is named {quote(one.name)} where the case wants " + f"{quote(expected.name)}" + ) + # The path is the case's own names joined with dots, which is how + # a field inside a path inside a column is pointed at without + # printing the whole schema at somebody. + path = expected.name if not prefix else f"{prefix}.{expected.name}" + if one.format != expected.format: + return ( + f"arrow field {quote(path)} is {quote(one.format)} where the case wants " + f"{quote(expected.format)}" + ) + why = _fields_of(path, list(one.children), list(expected.children)) + if why is not None: + return why + return None diff --git a/conformance/cases.py b/conformance/cases.py index ba1321d..c3b55ef 100644 --- a/conformance/cases.py +++ b/conformance/cases.py @@ -15,6 +15,13 @@ hands it to this client's own binding call, and asserts what came back. A client that decodes a date correctly and encodes it a day early passes every case that has no parameters in it. + +A case may say which connection each of its statements runs on, which is +how a case about a transaction is written: a transaction is only +observable from outside it, so a case that has to say what a commit +means needs a second connection to say it to. A case that says nothing +runs everything on one connection called ``main``, which is every case +but a handful. """ from __future__ import annotations @@ -22,21 +29,33 @@ from dataclasses import dataclass, field from pathlib import Path -from . import values +from . import arrow, values from .reader import CorpusError, Node, parse, quote -__all__ = ["SCHEMA", "Case", "Suite", "Column", "Load", "read_dir"] +__all__ = ["SCHEMA", "MAIN", "Case", "Step", "Suite", "Column", "Load", "read_dir"] #: The schema version a file declares. It exists so that a corpus #: unpacked from an old release tells a new runner what it is instead of #: failing in the middle. -SCHEMA = 3 +SCHEMA = 4 + +#: The connection a statement runs on when the case does not name one. +MAIN = "main" _SUITE_KEYS = ("schema", "suite", "doc", "load", "cases") -_CASE_KEYS = ("name", "doc", "setup", "params", "query", "columns", "rows", "raises") +_CASE_KEYS = ("name", "doc", "setup", "on", "params", "query", "columns", "rows", "raises", "arrow") _LOAD_KEYS = ("nodes", "edges", "count", "columns", "pairs") +@dataclass +class Step: + """A statement run before the one under test, and the connection it + runs on.""" + + on: str + query: str + + @dataclass class Case: """One statement and what it owes. @@ -49,11 +68,19 @@ class Case: doc: str query: str line: int - setup: list[str] = field(default_factory=list) + setup: list[Step] = field(default_factory=list) + #: The connection the statement under test runs on, which is + #: :data:`MAIN` unless the case says otherwise. + on: str = MAIN params: list[tuple[str, object]] = field(default_factory=list) columns: list[str] | None = None rows: list[list[object]] | None = None raises: str | None = None + #: What the same result looks like on the way out through Arrow, for + #: a case that says. Most do not: the export gives one answer per + #: column type and a handful of cases pin every one of them, so the + #: rest would be repeating a type the corpus already covers. + arrow: list[arrow.Field] | arrow.Refused | None = None @dataclass @@ -182,20 +209,22 @@ def _case(node: Node) -> Case: doc = _field(node, "doc") query = _field(node, "query") - setup: list[str] = [] + setup: list[Step] = [] setup_node = node.get("setup") if setup_node is not None: items = setup_node.seq() if items is None: raise CorpusError(f"line {line}: `setup:` is a sequence of statements") - for item in items: - text = item.str_() - if text is None: - raise CorpusError(f"line {item.line}: a setup statement is one line") - setup.append(text) + setup = [_step(item) for item in items] + + on_node = node.get("on") + on = _connection(on_node) if on_node is not None else MAIN params = _params(node) + arrow_node = node.get("arrow") + export = arrow.parse(arrow_node) if arrow_node is not None else None + raises_node = node.get("raises") columns_node = node.get("columns") if raises_node is not None and columns_node is not None: @@ -211,7 +240,7 @@ def _case(node: Node) -> Case: f"line {raises_node.line}: {quote(code)} is not the shape of a GQLSTATUS, which is " "five characters of digits and capitals" ) - return Case(name, doc, query, line, setup, params, raises=code) + return Case(name, doc, query, line, setup, on, params, raises=code, arrow=export) if columns_node is None: raise CorpusError( f"line {line}: a case says what it produces, with `columns:` and `rows:` or with " @@ -234,7 +263,46 @@ def _case(node: Node) -> Case: for row in rows: if len(row) != len(columns): raise CorpusError(f"line {line}: a row of {len(row)} against {len(columns)} columns") - return Case(name, doc, query, line, setup, params, columns=columns, rows=rows) + return Case(name, doc, query, line, setup, on, params, columns=columns, rows=rows, arrow=export) + + +def _step(node: Node) -> Step: + """One setup statement, which is a line of its own or a line and the + connection it runs on.""" + text = node.str_() + if text is not None: + return Step(MAIN, text) + if node.map() is None: + raise CorpusError( + f"line {node.line}: a setup statement is one line, or `on:` and `query:`, and this is " + f"{node.what()}" + ) + unknown = node.unknown(("on", "query")) + if unknown: + raise CorpusError(f"line {node.line}: a setup statement has no key {quote(unknown[0])}") + on_node = node.get("on") + if on_node is None: + raise CorpusError( + f"line {node.line}: a setup statement written as a mapping names the connection it " + "runs on" + ) + return Step(_connection(on_node), _field(node, "query")) + + +def _connection(node: Node) -> str: + """The name of a connection, spelled the way a case name is, because + a report cites it and a name a reader has to guess at is a report + that says less than it looks like it does.""" + name = node.str_() + if name is None: + raise CorpusError(f"line {node.line}: `on:` is the name of a connection") + spelled = name.isascii() and all(c.islower() or c.isdigit() or c == "-" for c in name) + if not name or not spelled: + raise CorpusError( + f"line {node.line}: {quote(name)} is a connection name, which is lower case words " + "joined by dashes" + ) + return name def _params(node: Node) -> list[tuple[str, object]]: diff --git a/conformance/runner.py b/conformance/runner.py index d267e8b..9d78aed 100644 --- a/conformance/runner.py +++ b/conformance/runner.py @@ -25,12 +25,14 @@ import tempfile from dataclasses import dataclass, field from pathlib import Path +from typing import Any import zudb -from .cases import Case, Load, Suite, read_dir -from .reader import CorpusError -from .values import same, show, too_fine, truncated +from . import arrow +from .cases import MAIN, Case, Load, Suite, read_dir +from .reader import CorpusError, quote +from .values import cell, same, show, too_fine, truncated __all__ = ["Ran", "Report", "run", "main"] @@ -115,10 +117,15 @@ def ran(outcome: str, detail: str = "") -> Ran: except zudb.Error as e: return ran(FAILED, f"opening {path}: {e}") + open_conns: list[tuple[str, zudb.Connection]] = [(MAIN, conn)] try: - for i, statement in enumerate(case.setup): + for i, step in enumerate(case.setup): try: - conn.execute(statement) + on = _connection(open_conns, step.on) + except zudb.Error as e: + return ran(FAILED, f"connecting as {quote(step.on)}: {_said(e)}") + try: + on.execute(step.query) except zudb.Error as e: # A setup that fails is not a result about the statement # under test, so it is never a pass and never a quiet @@ -127,9 +134,14 @@ def ran(outcome: str, detail: str = "") -> Ran: return ran(UNSUPPORTED, f"setup {i + 1}: {_said(e)}") return ran(FAILED, f"setup {i + 1} failed: {_said(e)}") + try: + on = _connection(open_conns, case.on) + except zudb.Error as e: + return ran(FAILED, f"connecting as {quote(case.on)}: {_said(e)}") + params = dict(case.params) if case.params else None try: - result = conn.execute(case.query, params) + result = on.execute(case.query, params) rows = result.fetchall() except zudb.Error as e: if case.raises is not None: @@ -151,9 +163,66 @@ def ran(outcome: str, detail: str = "") -> Ran: if case.raises is not None: return ran(FAILED, f"returned rows where the case wants {case.raises}") detail = _compare(case.columns or [], case.rows or [], result.columns, rows) + if detail is not None: + return ran(FAILED, detail) + # The export is checked on the result the rows were read from + # rather than on a second run of the statement, because it is the + # same result a client exports: one statement, two ways of + # reading what it gave back. + detail = _exported(case.arrow, result, len(rows)) return ran(PASSED) if detail is None else ran(FAILED, detail) finally: - conn.close() + # In reverse, so that the connection the case was opened with is + # the last one to go, which is the order the ones after it were + # made from it in. + for _, open_conn in reversed(open_conns): + open_conn.close() + + +def _connection(open_conns: list[tuple[str, Any]], name: str) -> Any: + """The connection a case named, made if this is the first mention of + it. + + A new one is a duplicate of the case's own rather than a second open + of the file, which is what a pool does: the two share the write side, + so each sees what the other has committed. Opening the path twice + would be two databases that happen to be the same file, which is a + different thing and not what a case about a transaction means.""" + for had, open_conn in open_conns: + if had == name: + return open_conn + made = open_conns[0][1].duplicate() + open_conns.append((name, made)) + return made + + +def _exported( + want: list[arrow.Field] | arrow.Refused | None, + result: Any, + rows: int, +) -> str | None: + """What the export gave that the case did not want, or ``None`` when + the case says nothing about it and when the two agree. + + A result Arrow has no type for is a refusal from the export rather + than a condition from the statement, so a case saying ``refused`` is + the case where the stream failing to open is the right answer.""" + if want is None: + return None + try: + got, given = arrow.exported(result) + except (arrow.ArrowError, zudb.Error) as e: + if isinstance(want, arrow.Refused): + return None + return f"arrow refused the result: {e}" + if isinstance(want, arrow.Refused): + return "arrow exported the result where the case wants a refusal" + detail = arrow.schema(got, want) + if detail is not None: + return detail + if given != rows: + return f"arrow gives {given} rows where the case wants {rows}" + return None def _apply(load: Load, path: Path) -> None: @@ -236,7 +305,11 @@ def _compare( if want_columns != got_columns: return f"columns {_names(got_columns)} where the case wants {_names(want_columns)}" for i, (want, got) in enumerate(zip(want_rows, got_rows, strict=False)): - for j, (a, b) in enumerate(zip(want, got, strict=False)): + for j, (a, raw) in enumerate(zip(want, got, strict=False)): + # Into the corpus's own shape first, because the engine's + # edge carries a field the corpus does not write and a + # comparison against it would fail on a number no case chose. + b = cell(raw) if not same(a, b): name = want_columns[j] if j < len(want_columns) else "?" return f"row {i + 1} column {name} is {show(b)} where the case wants {show(a)}" diff --git a/conformance/values.py b/conformance/values.py index 34395ce..669c747 100644 --- a/conformance/values.py +++ b/conformance/values.py @@ -13,6 +13,13 @@ reason and for ``NaN``, ``inf`` and ``-0.0``. A temporal value is a string, because YAML has no type that keeps an offset. +``NODE``, ``EDGE`` and ``PATH`` are the values a graph has and a table +does not, and they are written as names rather than as the numbers the +engine holds. A node is ``"person#1"``, the table it is a row of and +which row of it. An edge is ``"knows#0->1"``, its table and the two rows +it runs between. A path is a sequence, like a list, holding a node and +then an edge and a node for each hop. + Refusing the wrong form is half the point, and refusing it here is what makes this a second reader of the corpus rather than a consumer of it. @@ -36,7 +43,19 @@ from .reader import CorpusError, quote from .reader import Node as YamlNode -__all__ = ["decode", "typed", "payload", "same", "show", "form", "TooFine", "too_fine"] +__all__ = [ + "decode", + "typed", + "payload", + "cell", + "same", + "show", + "form", + "Edge", + "Walk", + "TooFine", + "too_fine", +] #: Whether a type's payload is written as a quoted string. ``False`` is #: a type a YAML scalar carries without loss, ``True`` is one it does @@ -62,12 +81,20 @@ "ZONEDDATETIME": True, "DURATION": True, "LIST": False, + # A node and an edge are written in quotes because what a case + # spells is a name and two numbers with punctuation between them, + # which is text in every reader and a number in none. + "NODE": True, + "EDGE": True, + # A path is a sequence, like a list, because that is what it is: the + # nodes and edges of a walk, in the order they were walked. + "PATH": False, } #: The types the encoding reserves a name for and the engine has no #: runtime value for yet, kept apart from an outright typo so that the #: error says which of the two it is. -_RESERVED = ("DECIMAL", "BYTES", "NODE", "EDGE", "PATH") +_RESERVED = ("DECIMAL", "BYTES") #: The range each integer width holds, so that a case writing a value #: its own type cannot carry is refused rather than stored wider than it @@ -86,6 +113,36 @@ } +@dataclass(frozen=True) +class Edge: + """An edge as a case names it: the rel table it is in and the two + rows it runs between. + + Not ``zudb.Rel``, which carries a fourth field the corpus does not + write. ``ord`` is where the edge's properties sit, which is its + place in the order the table was loaded in, and that is a number the + loader chose rather than one the case did. A pair may run more than + once, and a case that has to tell two parallel edges apart asserts a + property of them instead.""" + + table: str + src: int + dst: int + + +@dataclass(frozen=True) +class Walk: + """A path as a case writes it: nodes and edges alternating, a node + at each end. + + Not ``zudb.Path`` for the reason ``Edge`` is not ``zudb.Rel``, and + for one more: what a case compares is the walk, and two walks that + cross the same edges are the same walk whichever copy of a parallel + edge the engine happened to hand back.""" + + elements: tuple[object, ...] + + @dataclass(frozen=True) class TooFine: """A temporal value written finer than this client can hold. @@ -211,15 +268,16 @@ def payload(ty: str, value: YamlNode) -> object: if quoted_form is None: raise CorpusError(f"line {value.line}: {_unknown(ty)}") - if ty == "LIST": + if ty in ("LIST", "PATH"): # The empty list is a value worth a case and needs a spelling, # which is a `value:` with nothing under it. items = value.seq_or_empty() if items is None: raise CorpusError( - f"line {value.line}: a LIST holds a sequence of values, and this is {value.what()}" + f"line {value.line}: a {ty} holds a sequence of values, and this is {value.what()}" ) - return [decode(item) for item in items] + decoded = [decode(item) for item in items] + return decoded if ty == "LIST" else _walk(decoded, value.line) scalar = value.scalar() if scalar is None: @@ -230,6 +288,15 @@ def payload(ty: str, value: YamlNode) -> object: # is looked at, because a value that parses is exactly the case where # a silent misread would survive review. if quoted_form and not quoted: + # A node and an edge are quoted for a different reason from the + # numbers, so they are told a different reason. Both reasons are + # the same rule: a payload is quoted where a bare one would read + # as something else in some reader of this file. + if ty in ("NODE", "EDGE"): + raise CorpusError( + f"line {line}: {ty} is written in quotes, because {text} is a name and two " + "numbers and no reader has a scalar for that" + ) raise CorpusError( f"line {line}: {ty} is written in quotes, because a bare {text} is a number and some " "reader of this file will round it" @@ -239,12 +306,90 @@ def payload(ty: str, value: YamlNode) -> object: f"line {line}: {ty} is written without quotes, so that a reader cannot take it for a " "string" ) - out = _scalar(ty, text) + if ty == "NODE": + out = _node_at(text) + elif ty == "EDGE": + out = _edge_at(text) + else: + out = _scalar(ty, text) if out is _NOT_ONE: raise CorpusError(f"line {line}: {quote(text)} is not a {ty}") return out +def _walk(items: list[object], line: int) -> Walk: + """The nodes and edges of a walk, or what is wrong with the sequence + somebody wrote. + + A path alternates and ends at both ends with a node, so a sequence + that does not is a case that could never pass. Refusing it here + rather than at the comparison is the difference between a message + naming the line and a report saying the row differs.""" + if len(items) % 2 == 0: + raise CorpusError( + f"line {line}: a PATH is a node, then an edge and a node for each hop, so it holds an " + f"odd number of values and this holds {len(items)}" + ) + for i, item in enumerate(items): + want_node = i % 2 == 0 + if isinstance(item, Node): + ok, was = want_node, "a NODE" + elif isinstance(item, Edge): + ok, was = not want_node, "an EDGE" + else: + ok, was = False, "neither a NODE nor an EDGE" + if not ok: + wanted = "a NODE" if want_node else "an EDGE" + raise CorpusError( + f"line {line}: a PATH alternates, so value {i + 1} is {was} where it should be " + f"{wanted}" + ) + return Walk(tuple(items)) + + +def _node_at(text: str) -> object: + """A node, written as its table and the offset of its row: + ``person#1``. + + The table's name rather than its id, because the id is a number the + file decided and every client builds its own file. Split from the + right, so that a table whose name holds a ``#`` is still readable.""" + table, hash_, offset = text.rpartition("#") + if not hash_ or not table or not offset.isdigit(): + return _NOT_ONE + return Node(table, int(offset)) + + +def _edge_at(text: str) -> object: + """An edge, written as its table and the rows it runs between: + ``knows#0->1``.""" + table, hash_, ends = text.rpartition("#") + if not hash_ or not table: + return _NOT_ONE + src, arrow, dst = ends.partition("->") + if not arrow or not src.isdigit() or not dst.isdigit(): + return _NOT_ONE + return Edge(table, int(src), int(dst)) + + +def cell(value: object) -> object: + """The corpus's own shape for a value that came back from a + statement. + + Everything a table holds is spelled the same on both sides and comes + through untouched. A graph value is not: the engine's edge carries an + ``ord`` the corpus does not write, so an edge and a path are put into + the shapes above before anything is compared, which is what the Rust + runner's ``from_engine`` does for the same reason.""" + if isinstance(value, Rel): + return Edge(value.table, value.src, value.dst) + if isinstance(value, Path): + return Walk(tuple(cell(item) for item in value.elements)) + if isinstance(value, list): + return [cell(item) for item in value] + return value + + #: What `_scalar` gives back for text that does not spell a value of the #: type, which cannot be `None` because `None` is a value NULL spells. _NOT_ONE = object() @@ -468,6 +613,8 @@ def same(want: object, got: object) -> bool: return math.copysign(1.0, want) == math.copysign(1.0, got) and want == got if isinstance(want, list) and isinstance(got, list): return len(want) == len(got) and all(same(a, b) for a, b in zip(want, got, strict=True)) + if isinstance(want, Walk) and isinstance(got, Walk): + return same(list(want.elements), list(got.elements)) if isinstance(want, dict) and isinstance(got, dict): return list(want) == list(got) and all(same(want[k], got[k]) for k in want) return type(want) is type(got) and want == got @@ -505,8 +652,16 @@ def show(value: object) -> str: if isinstance(value, dict): fields = ", ".join(f"{name}: {show(v)}" for name, v in value.items()) return f"RECORD {{{fields}}}" - if isinstance(value, (Node, Rel, Path)): - return repr(value) + if isinstance(value, Node): + return f'NODE "{value.table}#{value.offset}"' + if isinstance(value, Edge): + return f'EDGE "{value.table}#{value.src}->{value.dst}"' + if isinstance(value, Walk): + return f"PATH [{', '.join(show(item) for item in value.elements)}]" + # A `zudb.Rel` or a `zudb.Path` reaching here is a value the runner + # did not put through `cell`, which is a defect in this runner rather + # than a wrong answer, so it prints as itself and says so by not + # looking like anything a case could be written from. return repr(value) diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 03027f3..07e3102 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -18,18 +18,21 @@ import os import subprocess import sys +import tempfile +from collections.abc import Iterator from pathlib import Path import pytest import zudb -from conformance import cases, reader, runner, values +from conformance import arrow, cases, reader, runner, values +from conformance.cases import MAIN CASES = os.environ.get("ZU_CASES") needs_cases = pytest.mark.skipif(not CASES, reason="ZU_CASES does not point at the case files") -HEAD = "schema: 3\nsuite: int\ndoc: the integer tower\n" +HEAD = "schema: 4\nsuite: int\ndoc: the integer tower\n" def rows_of(value: str, ty: str = "INT64", column: str = "n") -> str: @@ -318,6 +321,74 @@ def test_a_list_holds_encoded_values_and_not_bare_ones() -> None: assert value("type: LIST\nvalue:\n") == [] +def test_a_node_is_the_name_of_its_table_and_the_row_it_is() -> None: + assert value('type: NODE\nvalue: "person#1"\n') == zudb.Node("person", 1) + # The last hash is the separator, so a table whose name holds one is + # still read the way it was written. + assert value('type: NODE\nvalue: "a#b#2"\n') == zudb.Node("a#b", 2) + + +def test_an_edge_is_read_without_the_field_no_case_writes() -> None: + """The engine's edge carries the position it holds among the edges + out of its source, which the loader chose and no case picked. It is + dropped on both sides rather than guessed at on one.""" + assert value('type: EDGE\nvalue: "knows#0->1"\n') == values.Edge("knows", 0, 1) + assert values.cell(zudb.Rel("knows", 0, 1, 7)) == values.Edge("knows", 0, 1) + + +def test_a_path_alternates_nodes_and_edges_and_has_to_start_and_end_on_a_node() -> None: + walk = value( + 'type: PATH\nvalue:\n - type: NODE\n value: "person#0"\n' + ' - type: EDGE\n value: "knows#0->1"\n - type: NODE\n value: "person#1"\n' + ) + assert walk == values.Walk( + (zudb.Node("person", 0), values.Edge("knows", 0, 1), zudb.Node("person", 1)) + ) + # The one-node path is the shortest one there is, and it is a path. + assert value('type: PATH\nvalue:\n - type: NODE\n value: "person#0"\n') == values.Walk( + (zudb.Node("person", 0),) + ) + + +@pytest.mark.parametrize( + ("text", "want"), + [ + ("type: NODE\nvalue: person#1\n", "is a name and two numbers"), + ("type: EDGE\nvalue: knows#0->1\n", "is a name and two numbers"), + ('type: NODE\nvalue: "person"\n', '"person" is not a NODE'), + ('type: NODE\nvalue: "person#x"\n', '"person#x" is not a NODE'), + ('type: NODE\nvalue: "#1"\n', '"#1" is not a NODE'), + ('type: EDGE\nvalue: "knows#0"\n', '"knows#0" is not a EDGE'), + ('type: EDGE\nvalue: "knows#a->b"\n', '"knows#a->b" is not a EDGE'), + ('type: EDGE\nvalue: "knows#0->"\n', '"knows#0->" is not a EDGE'), + ( + 'type: PATH\nvalue:\n - type: NODE\n value: "person#0"\n' + ' - type: EDGE\n value: "knows#0->1"\n', + "an odd number", + ), + ( + 'type: PATH\nvalue:\n - type: EDGE\n value: "knows#0->1"\n' + ' - type: NODE\n value: "person#0"\n - type: EDGE\n value: "knows#0->1"\n', + "alternates", + ), + ("type: PATH\nvalue:\n", "an odd number"), + ], +) +def test_a_graph_value_written_wrong_is_refused_where_it_is_written(text: str, want: str) -> None: + with pytest.raises(reader.CorpusError) as raised: + value(text) + assert want in str(raised.value) + + +def test_a_graph_value_prints_the_way_a_case_writes_one() -> None: + assert values.show(zudb.Node("person", 1)) == 'NODE "person#1"' + assert values.show(values.Edge("knows", 0, 1)) == 'EDGE "knows#0->1"' + assert ( + values.show(values.Walk((zudb.Node("person", 0), values.Edge("knows", 0, 1)))) + == 'PATH [NODE "person#0", EDGE "knows#0->1"]' + ) + + def test_a_type_the_engine_cannot_hold_yet_says_so_rather_than_looking_like_a_typo() -> None: with pytest.raises(reader.CorpusError) as raised: value('type: DECIMAL\nvalue: "1.00"\n') @@ -492,7 +563,70 @@ def test_a_case_may_load_its_own_data_first() -> None: " rows:\n - values:\n - type: STRING\n value: a\n" ) assert len(case.setup) == 2 - assert case.setup[0].startswith("CREATE NODE TABLE") + assert case.setup[0].query.startswith("CREATE NODE TABLE") + # A setup written as a bare statement runs on the connection the case + # itself runs on, which is the one every case had before any of them + # named a second. + assert [step.on for step in case.setup] == [MAIN, MAIN] + assert case.on == MAIN + + +def test_a_case_may_say_which_connection_it_runs_on() -> None: + case = one( + " - name: on-another\n doc: a case run from a second connection\n setup:\n" + " - CREATE NODE TABLE Person(name STRING)\n" + " - on: writer\n query: INSERT (:Person {name: 'a'})\n" + " on: reader\n query: MATCH (p:Person) RETURN p.name AS name\n" + " columns:\n - name\n rows:\n - values:\n" + " - type: STRING\n value: a\n" + ) + assert [(step.on, step.query.split(" ")[0]) for step in case.setup] == [ + (MAIN, "CREATE"), + ("writer", "INSERT"), + ] + assert case.on == "reader" + + +def test_a_connection_is_named_the_way_everything_else_in_a_case_is() -> None: + """Lower case words joined by dashes, so that a report citing one + reads like the rest of a report and no client has to decide what to + do with a name another client would have written differently.""" + case = one( + " - name: a\n doc: d\n on: read-only-one\n query: RETURN 1\n raises: 22012\n" + ) + assert case.on == "read-only-one" + + +@pytest.mark.parametrize( + ("text", "want"), + [ + (" on: Writer\n", "lower case words joined by dashes"), + (" on: a_b\n", "lower case words joined by dashes"), + (" on:\n", "is the name of a connection"), + (" on:\n - a\n", "is the name of a connection"), + ], +) +def test_a_connection_written_wrong_is_refused_where_it_is_written(text: str, want: str) -> None: + with pytest.raises(reader.CorpusError) as raised: + suite(f" - name: a\n doc: d\n{text} query: RETURN 1\n raises: 22012\n") + assert want in str(raised.value) + + +@pytest.mark.parametrize( + ("text", "want"), + [ + (" - query: RETURN 1\n", "names the connection it runs on"), + (" - on: writer\n", "no `query:`"), + (" - on: writer\n stmt: RETURN 1\n", 'has no key "stmt"'), + (" - on: Writer\n query: RETURN 1\n", "lower case words joined by dashes"), + ], +) +def test_a_setup_step_written_as_a_mapping_says_both_halves(text: str, want: str) -> None: + with pytest.raises(reader.CorpusError) as raised: + suite( + f" - name: a\n doc: d\n setup:\n{text} query: RETURN 1\n raises: 22012\n" + ) + assert want in str(raised.value) def test_a_case_may_bind_parameters_and_they_keep_the_order_they_were_written_in() -> None: @@ -580,7 +714,7 @@ def test_a_parameter_a_statement_could_not_name_is_refused_where_it_is_written( def test_a_suite_may_load_a_table_every_case_in_it_reads_back() -> None: read = cases.read( - "schema: 3\nsuite: int\ndoc: d\nload:\n nodes: person\n edges: knows\n count: 1\n" + "schema: 4\nsuite: int\ndoc: d\nload:\n nodes: person\n edges: knows\n count: 1\n" ' columns:\n - name: age\n type: INT64\n values:\n - "30"\n' "cases:\n - name: a\n doc: d\n query: MATCH (p:person) RETURN p.age AS n\n" + rows_of("30") @@ -671,7 +805,7 @@ def test_the_same_case_name_twice_is_refused_because_a_report_cites_names() -> N def test_a_file_from_another_schema_says_so_rather_than_failing_in_the_middle() -> None: with pytest.raises(reader.CorpusError) as raised: cases.read("schema: 1\nsuite: int\ndoc: d\ncases:\n - name: a\n") - assert "schema 1 and the runner reads schema 3" in str(raised.value) + assert "schema 1 and the runner reads schema 4" in str(raised.value) def test_a_suite_whose_name_is_not_its_file_name_is_refused(tmp_path: Path) -> None: @@ -693,6 +827,90 @@ def test_a_directory_with_no_case_files_is_refused_rather_than_passing_empty( assert "no case files" in str(raised.value) +# What a case says about the export. + + +def test_a_case_may_say_what_the_export_gives_field_by_field() -> None: + case = one( + " - name: exported\n doc: a statement whose export is the point\n" + " query: RETURN 1 AS n\n columns:\n - n\n" + " rows:\n - values:\n - type: INT8\n value: 1\n" + " arrow:\n - name: n\n format: l\n" + ) + assert case.arrow == [arrow.Field("n", "l")] + + +def test_a_nested_field_carries_the_fields_under_it() -> None: + case = one( + " - name: exported\n doc: a list of strings, which is a field inside a field\n" + " query: RETURN ['a'] AS xs\n columns:\n - xs\n" + " rows:\n - values:\n - type: LIST\n value:\n" + " - type: STRING\n value: a\n" + " arrow:\n - name: xs\n format: +l\n children:\n" + " - name: item\n format: u\n" + ) + assert case.arrow == [arrow.Field("xs", "+l", (arrow.Field("item", "u"),))] + + +def test_a_result_arrow_has_no_type_for_is_written_as_a_refusal() -> None: + case = one( + " - name: refused\n doc: a time with an offset, which Arrow has no type for\n" + " query: RETURN 1 AS n\n raises: 22012\n arrow: refused\n" + ) + assert case.arrow is arrow.REFUSED + + +@pytest.mark.parametrize( + ("text", "want"), + [ + (" arrow: yes\n", "or `refused` for a result"), + (" arrow:\n - n\n", "an Arrow field is a mapping"), + (" arrow:\n - format: l\n", "an Arrow field has a `name:`"), + (" arrow:\n - name: n\n", "an Arrow field has a `format:`"), + (' arrow:\n - name: n\n format: ""\n', "not a type Arrow has"), + (" arrow:\n - name: n\n format: l\n kind: int\n", 'no key "kind"'), + ( + " arrow:\n - name: n\n format: +l\n", + "is a nested type and the fields under it are part of it", + ), + ( + " arrow:\n - name: n\n format: l\n children:\n" + " - name: item\n format: u\n", + "holds no fields, so nothing goes under it", + ), + ], +) +def test_an_export_written_wrong_is_refused_where_it_is_written(text: str, want: str) -> None: + with pytest.raises(reader.CorpusError) as raised: + suite(f" - name: a\n doc: d\n query: RETURN 1\n raises: 22012\n{text}") + assert want in str(raised.value) + + +def test_what_the_export_gave_that_the_case_did_not_want_is_said_the_way_rust_says_it() -> None: + """One wording per difference, checked here rather than only through + a run, because these are the lines the nine reports are diffed by.""" + n = arrow.Field("n", "l") + assert arrow.schema([n], [n]) is None + assert arrow.schema([n, n], [n]) == "arrow gives 2 fields in the result where the case wants 1" + assert ( + arrow.schema([arrow.Field("m", "l")], [n]) + == 'arrow field 1 in the result is named "m" where the case wants "n"' + ) + assert ( + arrow.schema([arrow.Field("n", "u")], [n]) + == 'arrow field "n" is "u" where the case wants "l"' + ) + nested = arrow.Field("xs", "+l", (arrow.Field("item", "u"),)) + assert ( + arrow.schema([arrow.Field("xs", "+l", (arrow.Field("item", "l"),))], [nested]) + == 'arrow field "xs.item" is "l" where the case wants "u"' + ) + assert ( + arrow.schema([arrow.Field("xs", "+l", ())], [nested]) + == 'arrow gives 0 fields in "xs" where the case wants 1' + ) + + # Running. @@ -700,7 +918,7 @@ def _write(tmp_path: Path, name: str, body: str) -> Path: directory = tmp_path / "cases" directory.mkdir(exist_ok=True) (directory / f"{name}.yaml").write_text( - f"schema: 3\nsuite: {name}\ndoc: a suite written by a test\ncases:\n{body}" + f"schema: 4\nsuite: {name}\ndoc: a suite written by a test\ncases:\n{body}" ) return directory @@ -805,7 +1023,7 @@ def test_a_suite_load_reaches_every_case_and_no_case_reaches_another(tmp_path: P directory = tmp_path / "cases" directory.mkdir() (directory / "graph.yaml").write_text( - "schema: 3\nsuite: graph\ndoc: a loaded suite\nload:\n nodes: person\n edges: knows\n" + "schema: 4\nsuite: graph\ndoc: a loaded suite\nload:\n nodes: person\n edges: knows\n" " count: 2\n columns:\n - name: name\n type: STRING\n values:\n" " - ada\n - bob\n pairs:\n - from: 0\n to: 1\n" "cases:\n" @@ -837,6 +1055,90 @@ def test_a_case_binds_its_parameters_through_this_client(tmp_path: Path) -> None assert report.count(runner.PASSED) == 1 +def test_a_second_connection_sees_what_the_first_one_committed(tmp_path: Path) -> None: + """The whole point of naming a connection: a write that came back + from one is a write the other reads. The two are duplicates of each + other and share the write side, so this is the answer a pool gives + and not the answer two opens of one file give.""" + report = _run( + tmp_path, + "two", + " - name: across\n doc: a committed write is another connection's to read\n" + ' setup:\n - on: writer\n query: "INSERT (:thing {n: 1})"\n' + " on: reader\n query: MATCH (t:thing) RETURN count(t) AS n\n" + rows_of("1"), + ) + assert report.summary() == "1 cases, 1 passed, 0 failed, 0 unsupported" + + +def test_a_case_naming_no_connection_runs_everything_on_the_one_it_opened( + tmp_path: Path, +) -> None: + report = _run( + tmp_path, + "one", + " - name: alone\n doc: a case that says nothing about connections\n" + ' setup:\n - "INSERT (:thing {n: 1})"\n' + " query: MATCH (t:thing) RETURN count(t) AS n\n" + rows_of("1"), + ) + assert report.count(runner.PASSED) == 1 + + +def test_the_export_is_checked_against_the_same_result_the_rows_came_from( + tmp_path: Path, +) -> None: + report = _run( + tmp_path, + "small", + " - name: one\n doc: an int64 column is an int64 on the way out\n" + " query: RETURN 1 AS n\n" + rows_of("1") + " arrow:\n - name: n\n" + " format: l\n", + ) + assert report.summary() == "1 cases, 1 passed, 0 failed, 0 unsupported" + + +def test_an_export_that_is_not_what_the_case_wants_is_a_failure_and_says_which_field( + tmp_path: Path, +) -> None: + report = _run( + tmp_path, + "small", + " - name: one\n doc: a case that wants the wrong Arrow type\n" + " query: RETURN 1 AS n\n" + rows_of("1") + " arrow:\n - name: n\n" + " format: u\n", + ) + assert report.failures()[0].detail == 'arrow field "n" is "l" where the case wants "u"' + + +def test_a_result_arrow_will_not_take_is_a_refusal_and_not_a_crash(tmp_path: Path) -> None: + """A time with an offset is the one a statement can write today. + Arrow has a time and a timestamp and nothing in between, so the + export says no, and a case saying `refused` is a case that passes on + it saying no.""" + body = ( + " - name: offset\n doc: a time with an offset has no Arrow type\n" + " query: RETURN ZONED TIME '12:34:56+07:00' AS n\n columns:\n - n\n" + " rows:\n - values:\n - type: ZONEDTIME\n" + ' value: "12:34:56+07:00"\n' + ) + assert _run(tmp_path, "zoned", body + " arrow: refused\n").count(runner.PASSED) == 1 + wanted = _run(tmp_path, "zoned", body + " arrow:\n - name: n\n format: l\n") + assert wanted.failures()[0].detail.startswith("arrow refused the result: ") + + +def test_an_export_giving_a_different_number_of_rows_is_reported( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The rows are counted off the stream rather than taken from the + rows already read, because an export that gave the right schema and + half the rows would pass every other check in here. The engine does + not disagree with itself about how many rows a result has, so the + only way to see the report is to make it disagree.""" + field = arrow.Field("n", "l") + monkeypatch.setattr(arrow, "exported", lambda result: ([field], 2)) + assert runner._exported([field], object(), 2) is None + assert runner._exported([field], object(), 3) == "arrow gives 2 rows where the case wants 3" + + def test_the_command_line_exits_zero_on_a_run_that_passes(tmp_path: Path) -> None: directory = _write( tmp_path, @@ -853,7 +1155,7 @@ def test_the_command_line_exits_one_on_a_corpus_it_cannot_read(tmp_path: Path) - disagree about what the run came to.""" directory = tmp_path / "cases" directory.mkdir() - (directory / "small.yaml").write_text("schema: 3\nsuite: small\ndoc: d\ncases:\n - name: a\n") + (directory / "small.yaml").write_text("schema: 4\nsuite: small\ndoc: d\ncases:\n - name: a\n") assert runner.main([str(directory), "--quiet"]) == 1 @@ -886,11 +1188,12 @@ def test_the_module_runs_as_a_command(tmp_path: Path) -> None: #: The cases this client cannot answer, and the only ones it may leave -#: unanswered. Written out rather than counted, so that a sixth one -#: arriving is a failure here and not a number nobody looks at. All five +#: unanswered. Written out rather than counted, so that a seventh one +#: arriving is a failure here and not a number nobody looks at. All six #: are a time to the nanosecond, which is a digit finer than Python's #: `datetime` holds. UNHELD = { + "arrow/a-local-time-is-nanoseconds-from-midnight", "param/localtime-to-the-nanosecond", "stored/a-localtime-column-keeps-every-digit", "stored/the-columns-of-one-row-belong-to-that-row", @@ -899,24 +1202,36 @@ def test_the_module_runs_as_a_command(tmp_path: Path) -> None: } +@pytest.fixture(scope="session") +def corpus() -> Iterator[runner.Report]: + """The whole corpus, run once for everything below that reads it. + + Every case gets a database of its own and there are over a thousand + of them, so a run is a couple of gigabytes that exist for as long as + it takes. Once per session rather than once per test, and under a + directory that goes with the run rather than one pytest keeps three + generations of: two tests reading one report is not a reason to fill + a disk twice over, and it is a disk this has already filled.""" + with tempfile.TemporaryDirectory(prefix="zudb-corpus-") as work: + yield runner.run(cases.read_dir(Path(CASES)), Path(work)) + + @needs_cases -def test_every_case_the_engine_ships_passes_through_this_client(tmp_path: Path) -> None: +def test_every_case_the_engine_ships_passes_through_this_client(corpus: runner.Report) -> None: """The corpus itself, which is the check the other two runners run and the reason this one exists.""" - report = runner.run(cases.read_dir(Path(CASES)), tmp_path) - assert report.count(runner.FAILED) == 0, [str(r) for r in report.failures()][:10] - unheld = {f"{r.suite}/{r.case}" for r in report.ran if r.outcome == runner.UNSUPPORTED} + assert corpus.count(runner.FAILED) == 0, [str(r) for r in corpus.failures()][:10] + unheld = {f"{r.suite}/{r.case}" for r in corpus.ran if r.outcome == runner.UNSUPPORTED} assert unheld == UNHELD - assert report.count(runner.PASSED) == len(report.ran) - len(UNHELD) + assert corpus.count(runner.PASSED) == len(corpus.ran) - len(UNHELD) @needs_cases def test_the_cases_this_client_cannot_hold_are_a_precision_and_not_a_wrong_answer( - tmp_path: Path, + corpus: runner.Report, ) -> None: """Every one of them says which value it is and why, because a case reported as unsupported with no reason is a case nobody revisits.""" - report = runner.run(cases.read_dir(Path(CASES)), tmp_path) - for ran in report.ran: + for ran in corpus.ran: if ran.outcome == runner.UNSUPPORTED: assert "finer" in ran.detail, str(ran)