diff --git a/Cargo.lock b/Cargo.lock index 92d1c36..0a18318 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1709,7 +1709,7 @@ dependencies = [ [[package]] name = "zu" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "zu-common", "zu-encoding", @@ -1725,7 +1725,7 @@ dependencies = [ [[package]] name = "zu-arrow" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "arrow", "zu-common", @@ -1735,7 +1735,7 @@ dependencies = [ [[package]] name = "zu-common" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "thiserror", ] @@ -1743,7 +1743,7 @@ dependencies = [ [[package]] name = "zu-encoding" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "ruzstd", "zu-common", @@ -1752,7 +1752,7 @@ dependencies = [ [[package]] name = "zu-exec" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "zu-common", "zu-query", @@ -1762,7 +1762,7 @@ dependencies = [ [[package]] name = "zu-query" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "crossbeam-deque", "zu-common", @@ -1773,7 +1773,7 @@ dependencies = [ [[package]] name = "zu-s3" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "crc32c", "object_store", @@ -1784,7 +1784,7 @@ dependencies = [ [[package]] name = "zu-sqlite" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "rusqlite", "zu-common", @@ -1794,7 +1794,7 @@ dependencies = [ [[package]] name = "zu-storage" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "zu-common", "zu-encoding", @@ -1803,7 +1803,7 @@ dependencies = [ [[package]] name = "zu-vector" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "zu-common", ] @@ -1811,7 +1811,7 @@ dependencies = [ [[package]] name = "zu-zu1" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +source = "git+https://github.com/tamnd/zu?rev=6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a#6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" dependencies = [ "crc32c", "loom", diff --git a/Cargo.toml b/Cargo.toml index afc30a5..0466f3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,14 +18,14 @@ crate-type = ["cdylib"] # with (ADR 0002), so a revision is the honest way to say which one. # A local checkout is used instead with a `paths` override in # `.cargo/config.toml`, which is untracked on purpose. -zudb = { package = "zu", git = "https://github.com/tamnd/zu", rev = "0698a4eccd31670f0a875b6d097d7753440a51b3" } -zu-common = { git = "https://github.com/tamnd/zu", rev = "0698a4eccd31670f0a875b6d097d7753440a51b3" } +zudb = { package = "zu", git = "https://github.com/tamnd/zu", rev = "6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" } +zu-common = { git = "https://github.com/tamnd/zu", rev = "6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a" } # The one translation from a result into Arrow, which lives in the # engine tree so that every client agrees about what a column becomes. # `ffi` is the only feature this client turns on: what Python wants is # the C Data Interface, which is how a result reaches pyarrow, pandas # and polars without a Python object per cell. -zu-arrow = { git = "https://github.com/tamnd/zu", rev = "0698a4eccd31670f0a875b6d097d7753440a51b3", features = ["ffi"] } +zu-arrow = { git = "https://github.com/tamnd/zu", rev = "6ee7d8019415ba16145e9324ccb16b4ffa1d6c9a", features = ["ffi"] } # `extension-module` is asked for by maturin, in pyproject.toml, and # not here. Only the build backend knows how an extension is linked on # the platform it is building for, and a crate that turns the feature diff --git a/conformance/arrow.py b/conformance/arrow.py new file mode 100644 index 0000000..29310c6 --- /dev/null +++ b/conformance/arrow.py @@ -0,0 +1,309 @@ +"""What a result looks like on the way out through Arrow. + +A client that reads rows one at a time and a client that exports a +million of them to a dataframe are the same client, and only one of +those paths is covered by a case that asserts values. The other one has +its own contract: a column of dates is a ``Date32`` and not a string of +digits, a year-month duration is a month-day-nano interval because that +is the interval every reader implements, a node is a struct of the name +of its table and the row it is, and a time with an offset is refused +rather than quietly moved to UTC. None of that shows up in a row a case +compares. + +So a case may say what the export gives as well as what the rows are, +and the runner checks both against one statement. What it checks is the +schema, field by field and into the nested types, and how many rows came +back through the stream. The schema is spelled in the C Data Interface's +own format strings, ``l`` for an int64 and ``+s`` for a struct, because +that is the one spelling every language sees the same. + +The schema is read off the interface itself rather than out of +``pyarrow``. Two reasons, and the second is the one that decided it. The +first is that ``pyarrow`` is not installed to run the corpus and a check +that needed it would be a check that quietly does not run. The second is +that the C Data Interface is what the C runner has at this point too, so +the two read the same bytes and report them in the same words, which is +the whole reason a format string is what the case writes down. + +Values are not read back here. A consumer that decoded every array by +hand in each of nine languages would be nine new decoders under test, +which is more of our own code and not more of the contract; the rows the +case already asserts are the same values by another road. +""" + +from __future__ import annotations + +import ctypes +from dataclasses import dataclass +from dataclasses import field as _field +from typing import Any + +from .reader import CorpusError, Node, quote + +__all__ = ["ArrowError", "Field", "Refused", "REFUSED", "RESULT", "parse", "exported", "schema"] + + +@dataclass(frozen=True) +class Field: + """One field of the schema an export gives, and the fields under it + when it is a struct or a list. + + A list has exactly one field under it, which Arrow names ``item``, + and a case writes that out rather than leaving it implied: a client + that named it ``element`` would export something no reader lines up + with what another client wrote.""" + + name: str + #: The C Data Interface format string, ``l`` for an int64, ``u`` for + #: a string, ``tsn:`` for a timestamp in nanoseconds with no zone. + format: str + children: tuple[Field, ...] = _field(default_factory=tuple) + + +class Refused: + """Arrow has no type for one of the columns, so there is no export to + describe. + + A time with an offset is the one a statement can write today: Arrow + has a time and a timestamp and nothing in between, and dropping the + offset would move the value.""" + + def __repr__(self) -> str: + return "refused" + + +#: The one value of the class above, so that what a case says about the +#: export is either a list of fields or this. +REFUSED = Refused() + +#: How a report names the whole result, which is the place the columns of +#: an export are in. +RESULT = "the result" + + +def parse(node: Node) -> list[Field] | Refused: + """The ``arrow:`` of a case, or what is wrong with it.""" + text = node.str_() + if text is not None: + if text == "refused": + return REFUSED + raise CorpusError( + f"line {node.line}: `arrow:` is the columns the export gives, or `refused` for a " + f"result Arrow has no type for, and this is {quote(text)}" + ) + return _fields(node) + + +def _fields(node: Node) -> list[Field]: + items = node.seq() + if items is None: + raise CorpusError( + f"line {node.line}: `arrow:` is a sequence of fields, and this is {node.what()}" + ) + return [_one(item) for item in items] + + +def _one(node: Node) -> Field: + line = node.line + if node.map() is None: + raise CorpusError( + f"line {line}: an Arrow field is a mapping of `name` and `format`, and this is " + f"{node.what()}" + ) + unknown = node.unknown(("name", "format", "children")) + if unknown: + raise CorpusError(f"line {line}: an Arrow field has no key {quote(unknown[0])}") + + def text(key: str) -> str: + value = node.get(key) + spelled = value.str_() if value is not None else None + if spelled is None: + raise CorpusError(f"line {line}: an Arrow field has a `{key}:`") + return spelled + + name = text("name") + fmt = text("format") + if not fmt: + raise CorpusError(f"line {line}: an empty format string is not a type Arrow has") + children_node = node.get("children") + children = tuple(_fields(children_node)) if children_node is not None else () + # A nested format is the one thing about a format string this reader + # knows, and it is worth knowing here: a case that wrote the fields + # of a struct under a `u` would be asserting something the export + # cannot produce, and finding that out at load time says so with a + # line number rather than as a failure in a report. + nested = fmt.startswith("+") + if nested and not children: + raise CorpusError( + f"line {line}: {quote(fmt)} is a nested type and the fields under it are part of it" + ) + if not nested and children: + raise CorpusError(f"line {line}: {quote(fmt)} holds no fields, so nothing goes under it") + return Field(name, fmt, children) + + +class _Schema(ctypes.Structure): + """``ArrowSchema``, laid out as the C Data Interface writes it.""" + + +_Schema._fields_ = [ + ("format", ctypes.c_char_p), + ("name", ctypes.c_char_p), + ("metadata", ctypes.c_char_p), + ("flags", ctypes.c_int64), + ("n_children", ctypes.c_int64), + ("children", ctypes.POINTER(ctypes.POINTER(_Schema))), + ("dictionary", ctypes.POINTER(_Schema)), + ("release", ctypes.CFUNCTYPE(None, ctypes.POINTER(_Schema))), + ("private_data", ctypes.c_void_p), +] + + +class _Array(ctypes.Structure): + """``ArrowArray``. Only the length is read: the values a case cares + about it already asserts as rows.""" + + +_Array._fields_ = [ + ("length", ctypes.c_int64), + ("null_count", ctypes.c_int64), + ("offset", ctypes.c_int64), + ("n_buffers", ctypes.c_int64), + ("n_children", ctypes.c_int64), + ("buffers", ctypes.POINTER(ctypes.c_void_p)), + ("children", ctypes.POINTER(ctypes.POINTER(_Array))), + ("dictionary", ctypes.POINTER(_Array)), + ("release", ctypes.CFUNCTYPE(None, ctypes.POINTER(_Array))), + ("private_data", ctypes.c_void_p), +] + + +class _Stream(ctypes.Structure): + """``ArrowArrayStream``, which is what a result hands over.""" + + +_Get = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.POINTER(_Stream), ctypes.POINTER(_Schema)) +_Next = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.POINTER(_Stream), ctypes.POINTER(_Array)) + +_Stream._fields_ = [ + ("get_schema", _Get), + ("get_next", _Next), + ("get_last_error", ctypes.CFUNCTYPE(ctypes.c_char_p, ctypes.POINTER(_Stream))), + ("release", ctypes.CFUNCTYPE(None, ctypes.POINTER(_Stream))), + ("private_data", ctypes.c_void_p), +] + + +class ArrowError(Exception): + """The stream said no, with what it said.""" + + +def _pointer(capsule: object, name: bytes, kind: Any) -> Any: + """The pointer inside a capsule, which is how the C Data Interface + travels between two Python objects that have never heard of each + other.""" + get = ctypes.pythonapi.PyCapsule_GetPointer + get.restype = ctypes.c_void_p + get.argtypes = [ctypes.py_object, ctypes.c_char_p] + address = get(capsule, name) + if not address: + raise ArrowError("the result gave a capsule with nothing in it") + return ctypes.cast(address, ctypes.POINTER(kind)) + + +def _walked(one: _Schema) -> Field: + """One field of an exported schema, and everything under it.""" + children = [_walked(one.children[i].contents) for i in range(one.n_children)] + return Field( + name=(one.name or b"").decode("utf-8"), + format=(one.format or b"").decode("utf-8"), + children=tuple(children), + ) + + +def exported(result: Any) -> tuple[list[Field], int]: + """The columns a result gives through Arrow and how many rows came + out of the stream. + + The stream is taken once and both answers come out of that one + taking, because a stream is consumed by reading it and a second + export would be a second statement in all but name. + + A column Arrow cannot hold is found when the stream is asked for, + before a row moves, and this client says so with the Python class the + mistake belongs to: a `TypeError` for a value of a type Arrow does + not have and a `ValueError` for one that will not fit. Both are the + export refusing, which is what a case writing `refused` means, so + both come back out of here under the one name and only around the + call that can raise them.""" + try: + capsule = result.__arrow_c_stream__() + except (TypeError, ValueError) as e: + raise ArrowError(str(e)) from e + stream = _pointer(capsule, b"arrow_array_stream", _Stream) + + def said(code: int) -> str: + text = stream.contents.get_last_error(stream) + return (text or b"").decode("utf-8", "replace") or f"errno {code}" + + schema_out = _Schema() + code = stream.contents.get_schema(stream, ctypes.byref(schema_out)) + if code != 0: + raise ArrowError(said(code)) + try: + # The stream's schema is a struct of the columns, so what the + # case is compared against is the fields under it. + top = _walked(schema_out) + finally: + if schema_out.release: + schema_out.release(ctypes.byref(schema_out)) + + count = 0 + while True: + batch = _Array() + code = stream.contents.get_next(stream, ctypes.byref(batch)) + if code != 0: + raise ArrowError(said(code)) + # A released batch is how the interface says there are no more. + if not batch.release: + break + count += batch.length + batch.release(ctypes.byref(batch)) + return list(top.children), count + + +def schema(got: list[Field], want: list[Field]) -> str | None: + """What the export gave that the case did not want, or ``None`` when + the two agree. + + The comparison walks the schema and the case's fields together and + stops at the first difference, for the reason the row comparison + does: the first is nearly always the cause of the rest.""" + return _fields_of("", got, want) + + +def _fields_of(prefix: str, got: list[Field], want: list[Field]) -> str | None: + """The fields under one place, where the place is the dotted path of + the field they are under and the empty one is the result itself.""" + place = RESULT if not prefix else quote(prefix) + if len(got) != len(want): + return f"arrow gives {len(got)} fields in {place} where the case wants {len(want)}" + for i, (one, expected) in enumerate(zip(got, want, strict=True)): + if one.name != expected.name: + return ( + f"arrow field {i + 1} in {place} is named {quote(one.name)} where the case wants " + f"{quote(expected.name)}" + ) + # The path is the case's own names joined with dots, which is how + # a field inside a path inside a column is pointed at without + # printing the whole schema at somebody. + path = expected.name if not prefix else f"{prefix}.{expected.name}" + if one.format != expected.format: + return ( + f"arrow field {quote(path)} is {quote(one.format)} where the case wants " + f"{quote(expected.format)}" + ) + why = _fields_of(path, list(one.children), list(expected.children)) + if why is not None: + return why + return None diff --git a/conformance/cases.py b/conformance/cases.py index ba1321d..c3b55ef 100644 --- a/conformance/cases.py +++ b/conformance/cases.py @@ -15,6 +15,13 @@ hands it to this client's own binding call, and asserts what came back. A client that decodes a date correctly and encodes it a day early passes every case that has no parameters in it. + +A case may say which connection each of its statements runs on, which is +how a case about a transaction is written: a transaction is only +observable from outside it, so a case that has to say what a commit +means needs a second connection to say it to. A case that says nothing +runs everything on one connection called ``main``, which is every case +but a handful. """ from __future__ import annotations @@ -22,21 +29,33 @@ from dataclasses import dataclass, field from pathlib import Path -from . import values +from . import arrow, values from .reader import CorpusError, Node, parse, quote -__all__ = ["SCHEMA", "Case", "Suite", "Column", "Load", "read_dir"] +__all__ = ["SCHEMA", "MAIN", "Case", "Step", "Suite", "Column", "Load", "read_dir"] #: The schema version a file declares. It exists so that a corpus #: unpacked from an old release tells a new runner what it is instead of #: failing in the middle. -SCHEMA = 3 +SCHEMA = 4 + +#: The connection a statement runs on when the case does not name one. +MAIN = "main" _SUITE_KEYS = ("schema", "suite", "doc", "load", "cases") -_CASE_KEYS = ("name", "doc", "setup", "params", "query", "columns", "rows", "raises") +_CASE_KEYS = ("name", "doc", "setup", "on", "params", "query", "columns", "rows", "raises", "arrow") _LOAD_KEYS = ("nodes", "edges", "count", "columns", "pairs") +@dataclass +class Step: + """A statement run before the one under test, and the connection it + runs on.""" + + on: str + query: str + + @dataclass class Case: """One statement and what it owes. @@ -49,11 +68,19 @@ class Case: doc: str query: str line: int - setup: list[str] = field(default_factory=list) + setup: list[Step] = field(default_factory=list) + #: The connection the statement under test runs on, which is + #: :data:`MAIN` unless the case says otherwise. + on: str = MAIN params: list[tuple[str, object]] = field(default_factory=list) columns: list[str] | None = None rows: list[list[object]] | None = None raises: str | None = None + #: What the same result looks like on the way out through Arrow, for + #: a case that says. Most do not: the export gives one answer per + #: column type and a handful of cases pin every one of them, so the + #: rest would be repeating a type the corpus already covers. + arrow: list[arrow.Field] | arrow.Refused | None = None @dataclass @@ -182,20 +209,22 @@ def _case(node: Node) -> Case: doc = _field(node, "doc") query = _field(node, "query") - setup: list[str] = [] + setup: list[Step] = [] setup_node = node.get("setup") if setup_node is not None: items = setup_node.seq() if items is None: raise CorpusError(f"line {line}: `setup:` is a sequence of statements") - for item in items: - text = item.str_() - if text is None: - raise CorpusError(f"line {item.line}: a setup statement is one line") - setup.append(text) + setup = [_step(item) for item in items] + + on_node = node.get("on") + on = _connection(on_node) if on_node is not None else MAIN params = _params(node) + arrow_node = node.get("arrow") + export = arrow.parse(arrow_node) if arrow_node is not None else None + raises_node = node.get("raises") columns_node = node.get("columns") if raises_node is not None and columns_node is not None: @@ -211,7 +240,7 @@ def _case(node: Node) -> Case: f"line {raises_node.line}: {quote(code)} is not the shape of a GQLSTATUS, which is " "five characters of digits and capitals" ) - return Case(name, doc, query, line, setup, params, raises=code) + return Case(name, doc, query, line, setup, on, params, raises=code, arrow=export) if columns_node is None: raise CorpusError( f"line {line}: a case says what it produces, with `columns:` and `rows:` or with " @@ -234,7 +263,46 @@ def _case(node: Node) -> Case: for row in rows: if len(row) != len(columns): raise CorpusError(f"line {line}: a row of {len(row)} against {len(columns)} columns") - return Case(name, doc, query, line, setup, params, columns=columns, rows=rows) + return Case(name, doc, query, line, setup, on, params, columns=columns, rows=rows, arrow=export) + + +def _step(node: Node) -> Step: + """One setup statement, which is a line of its own or a line and the + connection it runs on.""" + text = node.str_() + if text is not None: + return Step(MAIN, text) + if node.map() is None: + raise CorpusError( + f"line {node.line}: a setup statement is one line, or `on:` and `query:`, and this is " + f"{node.what()}" + ) + unknown = node.unknown(("on", "query")) + if unknown: + raise CorpusError(f"line {node.line}: a setup statement has no key {quote(unknown[0])}") + on_node = node.get("on") + if on_node is None: + raise CorpusError( + f"line {node.line}: a setup statement written as a mapping names the connection it " + "runs on" + ) + return Step(_connection(on_node), _field(node, "query")) + + +def _connection(node: Node) -> str: + """The name of a connection, spelled the way a case name is, because + a report cites it and a name a reader has to guess at is a report + that says less than it looks like it does.""" + name = node.str_() + if name is None: + raise CorpusError(f"line {node.line}: `on:` is the name of a connection") + spelled = name.isascii() and all(c.islower() or c.isdigit() or c == "-" for c in name) + if not name or not spelled: + raise CorpusError( + f"line {node.line}: {quote(name)} is a connection name, which is lower case words " + "joined by dashes" + ) + return name def _params(node: Node) -> list[tuple[str, object]]: diff --git a/conformance/runner.py b/conformance/runner.py index d267e8b..9d78aed 100644 --- a/conformance/runner.py +++ b/conformance/runner.py @@ -25,12 +25,14 @@ import tempfile from dataclasses import dataclass, field from pathlib import Path +from typing import Any import zudb -from .cases import Case, Load, Suite, read_dir -from .reader import CorpusError -from .values import same, show, too_fine, truncated +from . import arrow +from .cases import MAIN, Case, Load, Suite, read_dir +from .reader import CorpusError, quote +from .values import cell, same, show, too_fine, truncated __all__ = ["Ran", "Report", "run", "main"] @@ -115,10 +117,15 @@ def ran(outcome: str, detail: str = "") -> Ran: except zudb.Error as e: return ran(FAILED, f"opening {path}: {e}") + open_conns: list[tuple[str, zudb.Connection]] = [(MAIN, conn)] try: - for i, statement in enumerate(case.setup): + for i, step in enumerate(case.setup): try: - conn.execute(statement) + on = _connection(open_conns, step.on) + except zudb.Error as e: + return ran(FAILED, f"connecting as {quote(step.on)}: {_said(e)}") + try: + on.execute(step.query) except zudb.Error as e: # A setup that fails is not a result about the statement # under test, so it is never a pass and never a quiet @@ -127,9 +134,14 @@ def ran(outcome: str, detail: str = "") -> Ran: return ran(UNSUPPORTED, f"setup {i + 1}: {_said(e)}") return ran(FAILED, f"setup {i + 1} failed: {_said(e)}") + try: + on = _connection(open_conns, case.on) + except zudb.Error as e: + return ran(FAILED, f"connecting as {quote(case.on)}: {_said(e)}") + params = dict(case.params) if case.params else None try: - result = conn.execute(case.query, params) + result = on.execute(case.query, params) rows = result.fetchall() except zudb.Error as e: if case.raises is not None: @@ -151,9 +163,66 @@ def ran(outcome: str, detail: str = "") -> Ran: if case.raises is not None: return ran(FAILED, f"returned rows where the case wants {case.raises}") detail = _compare(case.columns or [], case.rows or [], result.columns, rows) + if detail is not None: + return ran(FAILED, detail) + # The export is checked on the result the rows were read from + # rather than on a second run of the statement, because it is the + # same result a client exports: one statement, two ways of + # reading what it gave back. + detail = _exported(case.arrow, result, len(rows)) return ran(PASSED) if detail is None else ran(FAILED, detail) finally: - conn.close() + # In reverse, so that the connection the case was opened with is + # the last one to go, which is the order the ones after it were + # made from it in. + for _, open_conn in reversed(open_conns): + open_conn.close() + + +def _connection(open_conns: list[tuple[str, Any]], name: str) -> Any: + """The connection a case named, made if this is the first mention of + it. + + A new one is a duplicate of the case's own rather than a second open + of the file, which is what a pool does: the two share the write side, + so each sees what the other has committed. Opening the path twice + would be two databases that happen to be the same file, which is a + different thing and not what a case about a transaction means.""" + for had, open_conn in open_conns: + if had == name: + return open_conn + made = open_conns[0][1].duplicate() + open_conns.append((name, made)) + return made + + +def _exported( + want: list[arrow.Field] | arrow.Refused | None, + result: Any, + rows: int, +) -> str | None: + """What the export gave that the case did not want, or ``None`` when + the case says nothing about it and when the two agree. + + A result Arrow has no type for is a refusal from the export rather + than a condition from the statement, so a case saying ``refused`` is + the case where the stream failing to open is the right answer.""" + if want is None: + return None + try: + got, given = arrow.exported(result) + except (arrow.ArrowError, zudb.Error) as e: + if isinstance(want, arrow.Refused): + return None + return f"arrow refused the result: {e}" + if isinstance(want, arrow.Refused): + return "arrow exported the result where the case wants a refusal" + detail = arrow.schema(got, want) + if detail is not None: + return detail + if given != rows: + return f"arrow gives {given} rows where the case wants {rows}" + return None def _apply(load: Load, path: Path) -> None: @@ -236,7 +305,11 @@ def _compare( if want_columns != got_columns: return f"columns {_names(got_columns)} where the case wants {_names(want_columns)}" for i, (want, got) in enumerate(zip(want_rows, got_rows, strict=False)): - for j, (a, b) in enumerate(zip(want, got, strict=False)): + for j, (a, raw) in enumerate(zip(want, got, strict=False)): + # Into the corpus's own shape first, because the engine's + # edge carries a field the corpus does not write and a + # comparison against it would fail on a number no case chose. + b = cell(raw) if not same(a, b): name = want_columns[j] if j < len(want_columns) else "?" return f"row {i + 1} column {name} is {show(b)} where the case wants {show(a)}" diff --git a/conformance/values.py b/conformance/values.py index 34395ce..669c747 100644 --- a/conformance/values.py +++ b/conformance/values.py @@ -13,6 +13,13 @@ reason and for ``NaN``, ``inf`` and ``-0.0``. A temporal value is a string, because YAML has no type that keeps an offset. +``NODE``, ``EDGE`` and ``PATH`` are the values a graph has and a table +does not, and they are written as names rather than as the numbers the +engine holds. A node is ``"person#1"``, the table it is a row of and +which row of it. An edge is ``"knows#0->1"``, its table and the two rows +it runs between. A path is a sequence, like a list, holding a node and +then an edge and a node for each hop. + Refusing the wrong form is half the point, and refusing it here is what makes this a second reader of the corpus rather than a consumer of it. @@ -36,7 +43,19 @@ from .reader import CorpusError, quote from .reader import Node as YamlNode -__all__ = ["decode", "typed", "payload", "same", "show", "form", "TooFine", "too_fine"] +__all__ = [ + "decode", + "typed", + "payload", + "cell", + "same", + "show", + "form", + "Edge", + "Walk", + "TooFine", + "too_fine", +] #: Whether a type's payload is written as a quoted string. ``False`` is #: a type a YAML scalar carries without loss, ``True`` is one it does @@ -62,12 +81,20 @@ "ZONEDDATETIME": True, "DURATION": True, "LIST": False, + # A node and an edge are written in quotes because what a case + # spells is a name and two numbers with punctuation between them, + # which is text in every reader and a number in none. + "NODE": True, + "EDGE": True, + # A path is a sequence, like a list, because that is what it is: the + # nodes and edges of a walk, in the order they were walked. + "PATH": False, } #: The types the encoding reserves a name for and the engine has no #: runtime value for yet, kept apart from an outright typo so that the #: error says which of the two it is. -_RESERVED = ("DECIMAL", "BYTES", "NODE", "EDGE", "PATH") +_RESERVED = ("DECIMAL", "BYTES") #: The range each integer width holds, so that a case writing a value #: its own type cannot carry is refused rather than stored wider than it @@ -86,6 +113,36 @@ } +@dataclass(frozen=True) +class Edge: + """An edge as a case names it: the rel table it is in and the two + rows it runs between. + + Not ``zudb.Rel``, which carries a fourth field the corpus does not + write. ``ord`` is where the edge's properties sit, which is its + place in the order the table was loaded in, and that is a number the + loader chose rather than one the case did. A pair may run more than + once, and a case that has to tell two parallel edges apart asserts a + property of them instead.""" + + table: str + src: int + dst: int + + +@dataclass(frozen=True) +class Walk: + """A path as a case writes it: nodes and edges alternating, a node + at each end. + + Not ``zudb.Path`` for the reason ``Edge`` is not ``zudb.Rel``, and + for one more: what a case compares is the walk, and two walks that + cross the same edges are the same walk whichever copy of a parallel + edge the engine happened to hand back.""" + + elements: tuple[object, ...] + + @dataclass(frozen=True) class TooFine: """A temporal value written finer than this client can hold. @@ -211,15 +268,16 @@ def payload(ty: str, value: YamlNode) -> object: if quoted_form is None: raise CorpusError(f"line {value.line}: {_unknown(ty)}") - if ty == "LIST": + if ty in ("LIST", "PATH"): # The empty list is a value worth a case and needs a spelling, # which is a `value:` with nothing under it. items = value.seq_or_empty() if items is None: raise CorpusError( - f"line {value.line}: a LIST holds a sequence of values, and this is {value.what()}" + f"line {value.line}: a {ty} holds a sequence of values, and this is {value.what()}" ) - return [decode(item) for item in items] + decoded = [decode(item) for item in items] + return decoded if ty == "LIST" else _walk(decoded, value.line) scalar = value.scalar() if scalar is None: @@ -230,6 +288,15 @@ def payload(ty: str, value: YamlNode) -> object: # is looked at, because a value that parses is exactly the case where # a silent misread would survive review. if quoted_form and not quoted: + # A node and an edge are quoted for a different reason from the + # numbers, so they are told a different reason. Both reasons are + # the same rule: a payload is quoted where a bare one would read + # as something else in some reader of this file. + if ty in ("NODE", "EDGE"): + raise CorpusError( + f"line {line}: {ty} is written in quotes, because {text} is a name and two " + "numbers and no reader has a scalar for that" + ) raise CorpusError( f"line {line}: {ty} is written in quotes, because a bare {text} is a number and some " "reader of this file will round it" @@ -239,12 +306,90 @@ def payload(ty: str, value: YamlNode) -> object: f"line {line}: {ty} is written without quotes, so that a reader cannot take it for a " "string" ) - out = _scalar(ty, text) + if ty == "NODE": + out = _node_at(text) + elif ty == "EDGE": + out = _edge_at(text) + else: + out = _scalar(ty, text) if out is _NOT_ONE: raise CorpusError(f"line {line}: {quote(text)} is not a {ty}") return out +def _walk(items: list[object], line: int) -> Walk: + """The nodes and edges of a walk, or what is wrong with the sequence + somebody wrote. + + A path alternates and ends at both ends with a node, so a sequence + that does not is a case that could never pass. Refusing it here + rather than at the comparison is the difference between a message + naming the line and a report saying the row differs.""" + if len(items) % 2 == 0: + raise CorpusError( + f"line {line}: a PATH is a node, then an edge and a node for each hop, so it holds an " + f"odd number of values and this holds {len(items)}" + ) + for i, item in enumerate(items): + want_node = i % 2 == 0 + if isinstance(item, Node): + ok, was = want_node, "a NODE" + elif isinstance(item, Edge): + ok, was = not want_node, "an EDGE" + else: + ok, was = False, "neither a NODE nor an EDGE" + if not ok: + wanted = "a NODE" if want_node else "an EDGE" + raise CorpusError( + f"line {line}: a PATH alternates, so value {i + 1} is {was} where it should be " + f"{wanted}" + ) + return Walk(tuple(items)) + + +def _node_at(text: str) -> object: + """A node, written as its table and the offset of its row: + ``person#1``. + + The table's name rather than its id, because the id is a number the + file decided and every client builds its own file. Split from the + right, so that a table whose name holds a ``#`` is still readable.""" + table, hash_, offset = text.rpartition("#") + if not hash_ or not table or not offset.isdigit(): + return _NOT_ONE + return Node(table, int(offset)) + + +def _edge_at(text: str) -> object: + """An edge, written as its table and the rows it runs between: + ``knows#0->1``.""" + table, hash_, ends = text.rpartition("#") + if not hash_ or not table: + return _NOT_ONE + src, arrow, dst = ends.partition("->") + if not arrow or not src.isdigit() or not dst.isdigit(): + return _NOT_ONE + return Edge(table, int(src), int(dst)) + + +def cell(value: object) -> object: + """The corpus's own shape for a value that came back from a + statement. + + Everything a table holds is spelled the same on both sides and comes + through untouched. A graph value is not: the engine's edge carries an + ``ord`` the corpus does not write, so an edge and a path are put into + the shapes above before anything is compared, which is what the Rust + runner's ``from_engine`` does for the same reason.""" + if isinstance(value, Rel): + return Edge(value.table, value.src, value.dst) + if isinstance(value, Path): + return Walk(tuple(cell(item) for item in value.elements)) + if isinstance(value, list): + return [cell(item) for item in value] + return value + + #: What `_scalar` gives back for text that does not spell a value of the #: type, which cannot be `None` because `None` is a value NULL spells. _NOT_ONE = object() @@ -468,6 +613,8 @@ def same(want: object, got: object) -> bool: return math.copysign(1.0, want) == math.copysign(1.0, got) and want == got if isinstance(want, list) and isinstance(got, list): return len(want) == len(got) and all(same(a, b) for a, b in zip(want, got, strict=True)) + if isinstance(want, Walk) and isinstance(got, Walk): + return same(list(want.elements), list(got.elements)) if isinstance(want, dict) and isinstance(got, dict): return list(want) == list(got) and all(same(want[k], got[k]) for k in want) return type(want) is type(got) and want == got @@ -505,8 +652,16 @@ def show(value: object) -> str: if isinstance(value, dict): fields = ", ".join(f"{name}: {show(v)}" for name, v in value.items()) return f"RECORD {{{fields}}}" - if isinstance(value, (Node, Rel, Path)): - return repr(value) + if isinstance(value, Node): + return f'NODE "{value.table}#{value.offset}"' + if isinstance(value, Edge): + return f'EDGE "{value.table}#{value.src}->{value.dst}"' + if isinstance(value, Walk): + return f"PATH [{', '.join(show(item) for item in value.elements)}]" + # A `zudb.Rel` or a `zudb.Path` reaching here is a value the runner + # did not put through `cell`, which is a defect in this runner rather + # than a wrong answer, so it prints as itself and says so by not + # looking like anything a case could be written from. return repr(value) diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 03027f3..07e3102 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -18,18 +18,21 @@ import os import subprocess import sys +import tempfile +from collections.abc import Iterator from pathlib import Path import pytest import zudb -from conformance import cases, reader, runner, values +from conformance import arrow, cases, reader, runner, values +from conformance.cases import MAIN CASES = os.environ.get("ZU_CASES") needs_cases = pytest.mark.skipif(not CASES, reason="ZU_CASES does not point at the case files") -HEAD = "schema: 3\nsuite: int\ndoc: the integer tower\n" +HEAD = "schema: 4\nsuite: int\ndoc: the integer tower\n" def rows_of(value: str, ty: str = "INT64", column: str = "n") -> str: @@ -318,6 +321,74 @@ def test_a_list_holds_encoded_values_and_not_bare_ones() -> None: assert value("type: LIST\nvalue:\n") == [] +def test_a_node_is_the_name_of_its_table_and_the_row_it_is() -> None: + assert value('type: NODE\nvalue: "person#1"\n') == zudb.Node("person", 1) + # The last hash is the separator, so a table whose name holds one is + # still read the way it was written. + assert value('type: NODE\nvalue: "a#b#2"\n') == zudb.Node("a#b", 2) + + +def test_an_edge_is_read_without_the_field_no_case_writes() -> None: + """The engine's edge carries the position it holds among the edges + out of its source, which the loader chose and no case picked. It is + dropped on both sides rather than guessed at on one.""" + assert value('type: EDGE\nvalue: "knows#0->1"\n') == values.Edge("knows", 0, 1) + assert values.cell(zudb.Rel("knows", 0, 1, 7)) == values.Edge("knows", 0, 1) + + +def test_a_path_alternates_nodes_and_edges_and_has_to_start_and_end_on_a_node() -> None: + walk = value( + 'type: PATH\nvalue:\n - type: NODE\n value: "person#0"\n' + ' - type: EDGE\n value: "knows#0->1"\n - type: NODE\n value: "person#1"\n' + ) + assert walk == values.Walk( + (zudb.Node("person", 0), values.Edge("knows", 0, 1), zudb.Node("person", 1)) + ) + # The one-node path is the shortest one there is, and it is a path. + assert value('type: PATH\nvalue:\n - type: NODE\n value: "person#0"\n') == values.Walk( + (zudb.Node("person", 0),) + ) + + +@pytest.mark.parametrize( + ("text", "want"), + [ + ("type: NODE\nvalue: person#1\n", "is a name and two numbers"), + ("type: EDGE\nvalue: knows#0->1\n", "is a name and two numbers"), + ('type: NODE\nvalue: "person"\n', '"person" is not a NODE'), + ('type: NODE\nvalue: "person#x"\n', '"person#x" is not a NODE'), + ('type: NODE\nvalue: "#1"\n', '"#1" is not a NODE'), + ('type: EDGE\nvalue: "knows#0"\n', '"knows#0" is not a EDGE'), + ('type: EDGE\nvalue: "knows#a->b"\n', '"knows#a->b" is not a EDGE'), + ('type: EDGE\nvalue: "knows#0->"\n', '"knows#0->" is not a EDGE'), + ( + 'type: PATH\nvalue:\n - type: NODE\n value: "person#0"\n' + ' - type: EDGE\n value: "knows#0->1"\n', + "an odd number", + ), + ( + 'type: PATH\nvalue:\n - type: EDGE\n value: "knows#0->1"\n' + ' - type: NODE\n value: "person#0"\n - type: EDGE\n value: "knows#0->1"\n', + "alternates", + ), + ("type: PATH\nvalue:\n", "an odd number"), + ], +) +def test_a_graph_value_written_wrong_is_refused_where_it_is_written(text: str, want: str) -> None: + with pytest.raises(reader.CorpusError) as raised: + value(text) + assert want in str(raised.value) + + +def test_a_graph_value_prints_the_way_a_case_writes_one() -> None: + assert values.show(zudb.Node("person", 1)) == 'NODE "person#1"' + assert values.show(values.Edge("knows", 0, 1)) == 'EDGE "knows#0->1"' + assert ( + values.show(values.Walk((zudb.Node("person", 0), values.Edge("knows", 0, 1)))) + == 'PATH [NODE "person#0", EDGE "knows#0->1"]' + ) + + def test_a_type_the_engine_cannot_hold_yet_says_so_rather_than_looking_like_a_typo() -> None: with pytest.raises(reader.CorpusError) as raised: value('type: DECIMAL\nvalue: "1.00"\n') @@ -492,7 +563,70 @@ def test_a_case_may_load_its_own_data_first() -> None: " rows:\n - values:\n - type: STRING\n value: a\n" ) assert len(case.setup) == 2 - assert case.setup[0].startswith("CREATE NODE TABLE") + assert case.setup[0].query.startswith("CREATE NODE TABLE") + # A setup written as a bare statement runs on the connection the case + # itself runs on, which is the one every case had before any of them + # named a second. + assert [step.on for step in case.setup] == [MAIN, MAIN] + assert case.on == MAIN + + +def test_a_case_may_say_which_connection_it_runs_on() -> None: + case = one( + " - name: on-another\n doc: a case run from a second connection\n setup:\n" + " - CREATE NODE TABLE Person(name STRING)\n" + " - on: writer\n query: INSERT (:Person {name: 'a'})\n" + " on: reader\n query: MATCH (p:Person) RETURN p.name AS name\n" + " columns:\n - name\n rows:\n - values:\n" + " - type: STRING\n value: a\n" + ) + assert [(step.on, step.query.split(" ")[0]) for step in case.setup] == [ + (MAIN, "CREATE"), + ("writer", "INSERT"), + ] + assert case.on == "reader" + + +def test_a_connection_is_named_the_way_everything_else_in_a_case_is() -> None: + """Lower case words joined by dashes, so that a report citing one + reads like the rest of a report and no client has to decide what to + do with a name another client would have written differently.""" + case = one( + " - name: a\n doc: d\n on: read-only-one\n query: RETURN 1\n raises: 22012\n" + ) + assert case.on == "read-only-one" + + +@pytest.mark.parametrize( + ("text", "want"), + [ + (" on: Writer\n", "lower case words joined by dashes"), + (" on: a_b\n", "lower case words joined by dashes"), + (" on:\n", "is the name of a connection"), + (" on:\n - a\n", "is the name of a connection"), + ], +) +def test_a_connection_written_wrong_is_refused_where_it_is_written(text: str, want: str) -> None: + with pytest.raises(reader.CorpusError) as raised: + suite(f" - name: a\n doc: d\n{text} query: RETURN 1\n raises: 22012\n") + assert want in str(raised.value) + + +@pytest.mark.parametrize( + ("text", "want"), + [ + (" - query: RETURN 1\n", "names the connection it runs on"), + (" - on: writer\n", "no `query:`"), + (" - on: writer\n stmt: RETURN 1\n", 'has no key "stmt"'), + (" - on: Writer\n query: RETURN 1\n", "lower case words joined by dashes"), + ], +) +def test_a_setup_step_written_as_a_mapping_says_both_halves(text: str, want: str) -> None: + with pytest.raises(reader.CorpusError) as raised: + suite( + f" - name: a\n doc: d\n setup:\n{text} query: RETURN 1\n raises: 22012\n" + ) + assert want in str(raised.value) def test_a_case_may_bind_parameters_and_they_keep_the_order_they_were_written_in() -> None: @@ -580,7 +714,7 @@ def test_a_parameter_a_statement_could_not_name_is_refused_where_it_is_written( def test_a_suite_may_load_a_table_every_case_in_it_reads_back() -> None: read = cases.read( - "schema: 3\nsuite: int\ndoc: d\nload:\n nodes: person\n edges: knows\n count: 1\n" + "schema: 4\nsuite: int\ndoc: d\nload:\n nodes: person\n edges: knows\n count: 1\n" ' columns:\n - name: age\n type: INT64\n values:\n - "30"\n' "cases:\n - name: a\n doc: d\n query: MATCH (p:person) RETURN p.age AS n\n" + rows_of("30") @@ -671,7 +805,7 @@ def test_the_same_case_name_twice_is_refused_because_a_report_cites_names() -> N def test_a_file_from_another_schema_says_so_rather_than_failing_in_the_middle() -> None: with pytest.raises(reader.CorpusError) as raised: cases.read("schema: 1\nsuite: int\ndoc: d\ncases:\n - name: a\n") - assert "schema 1 and the runner reads schema 3" in str(raised.value) + assert "schema 1 and the runner reads schema 4" in str(raised.value) def test_a_suite_whose_name_is_not_its_file_name_is_refused(tmp_path: Path) -> None: @@ -693,6 +827,90 @@ def test_a_directory_with_no_case_files_is_refused_rather_than_passing_empty( assert "no case files" in str(raised.value) +# What a case says about the export. + + +def test_a_case_may_say_what_the_export_gives_field_by_field() -> None: + case = one( + " - name: exported\n doc: a statement whose export is the point\n" + " query: RETURN 1 AS n\n columns:\n - n\n" + " rows:\n - values:\n - type: INT8\n value: 1\n" + " arrow:\n - name: n\n format: l\n" + ) + assert case.arrow == [arrow.Field("n", "l")] + + +def test_a_nested_field_carries_the_fields_under_it() -> None: + case = one( + " - name: exported\n doc: a list of strings, which is a field inside a field\n" + " query: RETURN ['a'] AS xs\n columns:\n - xs\n" + " rows:\n - values:\n - type: LIST\n value:\n" + " - type: STRING\n value: a\n" + " arrow:\n - name: xs\n format: +l\n children:\n" + " - name: item\n format: u\n" + ) + assert case.arrow == [arrow.Field("xs", "+l", (arrow.Field("item", "u"),))] + + +def test_a_result_arrow_has_no_type_for_is_written_as_a_refusal() -> None: + case = one( + " - name: refused\n doc: a time with an offset, which Arrow has no type for\n" + " query: RETURN 1 AS n\n raises: 22012\n arrow: refused\n" + ) + assert case.arrow is arrow.REFUSED + + +@pytest.mark.parametrize( + ("text", "want"), + [ + (" arrow: yes\n", "or `refused` for a result"), + (" arrow:\n - n\n", "an Arrow field is a mapping"), + (" arrow:\n - format: l\n", "an Arrow field has a `name:`"), + (" arrow:\n - name: n\n", "an Arrow field has a `format:`"), + (' arrow:\n - name: n\n format: ""\n', "not a type Arrow has"), + (" arrow:\n - name: n\n format: l\n kind: int\n", 'no key "kind"'), + ( + " arrow:\n - name: n\n format: +l\n", + "is a nested type and the fields under it are part of it", + ), + ( + " arrow:\n - name: n\n format: l\n children:\n" + " - name: item\n format: u\n", + "holds no fields, so nothing goes under it", + ), + ], +) +def test_an_export_written_wrong_is_refused_where_it_is_written(text: str, want: str) -> None: + with pytest.raises(reader.CorpusError) as raised: + suite(f" - name: a\n doc: d\n query: RETURN 1\n raises: 22012\n{text}") + assert want in str(raised.value) + + +def test_what_the_export_gave_that_the_case_did_not_want_is_said_the_way_rust_says_it() -> None: + """One wording per difference, checked here rather than only through + a run, because these are the lines the nine reports are diffed by.""" + n = arrow.Field("n", "l") + assert arrow.schema([n], [n]) is None + assert arrow.schema([n, n], [n]) == "arrow gives 2 fields in the result where the case wants 1" + assert ( + arrow.schema([arrow.Field("m", "l")], [n]) + == 'arrow field 1 in the result is named "m" where the case wants "n"' + ) + assert ( + arrow.schema([arrow.Field("n", "u")], [n]) + == 'arrow field "n" is "u" where the case wants "l"' + ) + nested = arrow.Field("xs", "+l", (arrow.Field("item", "u"),)) + assert ( + arrow.schema([arrow.Field("xs", "+l", (arrow.Field("item", "l"),))], [nested]) + == 'arrow field "xs.item" is "l" where the case wants "u"' + ) + assert ( + arrow.schema([arrow.Field("xs", "+l", ())], [nested]) + == 'arrow gives 0 fields in "xs" where the case wants 1' + ) + + # Running. @@ -700,7 +918,7 @@ def _write(tmp_path: Path, name: str, body: str) -> Path: directory = tmp_path / "cases" directory.mkdir(exist_ok=True) (directory / f"{name}.yaml").write_text( - f"schema: 3\nsuite: {name}\ndoc: a suite written by a test\ncases:\n{body}" + f"schema: 4\nsuite: {name}\ndoc: a suite written by a test\ncases:\n{body}" ) return directory @@ -805,7 +1023,7 @@ def test_a_suite_load_reaches_every_case_and_no_case_reaches_another(tmp_path: P directory = tmp_path / "cases" directory.mkdir() (directory / "graph.yaml").write_text( - "schema: 3\nsuite: graph\ndoc: a loaded suite\nload:\n nodes: person\n edges: knows\n" + "schema: 4\nsuite: graph\ndoc: a loaded suite\nload:\n nodes: person\n edges: knows\n" " count: 2\n columns:\n - name: name\n type: STRING\n values:\n" " - ada\n - bob\n pairs:\n - from: 0\n to: 1\n" "cases:\n" @@ -837,6 +1055,90 @@ def test_a_case_binds_its_parameters_through_this_client(tmp_path: Path) -> None assert report.count(runner.PASSED) == 1 +def test_a_second_connection_sees_what_the_first_one_committed(tmp_path: Path) -> None: + """The whole point of naming a connection: a write that came back + from one is a write the other reads. The two are duplicates of each + other and share the write side, so this is the answer a pool gives + and not the answer two opens of one file give.""" + report = _run( + tmp_path, + "two", + " - name: across\n doc: a committed write is another connection's to read\n" + ' setup:\n - on: writer\n query: "INSERT (:thing {n: 1})"\n' + " on: reader\n query: MATCH (t:thing) RETURN count(t) AS n\n" + rows_of("1"), + ) + assert report.summary() == "1 cases, 1 passed, 0 failed, 0 unsupported" + + +def test_a_case_naming_no_connection_runs_everything_on_the_one_it_opened( + tmp_path: Path, +) -> None: + report = _run( + tmp_path, + "one", + " - name: alone\n doc: a case that says nothing about connections\n" + ' setup:\n - "INSERT (:thing {n: 1})"\n' + " query: MATCH (t:thing) RETURN count(t) AS n\n" + rows_of("1"), + ) + assert report.count(runner.PASSED) == 1 + + +def test_the_export_is_checked_against_the_same_result_the_rows_came_from( + tmp_path: Path, +) -> None: + report = _run( + tmp_path, + "small", + " - name: one\n doc: an int64 column is an int64 on the way out\n" + " query: RETURN 1 AS n\n" + rows_of("1") + " arrow:\n - name: n\n" + " format: l\n", + ) + assert report.summary() == "1 cases, 1 passed, 0 failed, 0 unsupported" + + +def test_an_export_that_is_not_what_the_case_wants_is_a_failure_and_says_which_field( + tmp_path: Path, +) -> None: + report = _run( + tmp_path, + "small", + " - name: one\n doc: a case that wants the wrong Arrow type\n" + " query: RETURN 1 AS n\n" + rows_of("1") + " arrow:\n - name: n\n" + " format: u\n", + ) + assert report.failures()[0].detail == 'arrow field "n" is "l" where the case wants "u"' + + +def test_a_result_arrow_will_not_take_is_a_refusal_and_not_a_crash(tmp_path: Path) -> None: + """A time with an offset is the one a statement can write today. + Arrow has a time and a timestamp and nothing in between, so the + export says no, and a case saying `refused` is a case that passes on + it saying no.""" + body = ( + " - name: offset\n doc: a time with an offset has no Arrow type\n" + " query: RETURN ZONED TIME '12:34:56+07:00' AS n\n columns:\n - n\n" + " rows:\n - values:\n - type: ZONEDTIME\n" + ' value: "12:34:56+07:00"\n' + ) + assert _run(tmp_path, "zoned", body + " arrow: refused\n").count(runner.PASSED) == 1 + wanted = _run(tmp_path, "zoned", body + " arrow:\n - name: n\n format: l\n") + assert wanted.failures()[0].detail.startswith("arrow refused the result: ") + + +def test_an_export_giving_a_different_number_of_rows_is_reported( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The rows are counted off the stream rather than taken from the + rows already read, because an export that gave the right schema and + half the rows would pass every other check in here. The engine does + not disagree with itself about how many rows a result has, so the + only way to see the report is to make it disagree.""" + field = arrow.Field("n", "l") + monkeypatch.setattr(arrow, "exported", lambda result: ([field], 2)) + assert runner._exported([field], object(), 2) is None + assert runner._exported([field], object(), 3) == "arrow gives 2 rows where the case wants 3" + + def test_the_command_line_exits_zero_on_a_run_that_passes(tmp_path: Path) -> None: directory = _write( tmp_path, @@ -853,7 +1155,7 @@ def test_the_command_line_exits_one_on_a_corpus_it_cannot_read(tmp_path: Path) - disagree about what the run came to.""" directory = tmp_path / "cases" directory.mkdir() - (directory / "small.yaml").write_text("schema: 3\nsuite: small\ndoc: d\ncases:\n - name: a\n") + (directory / "small.yaml").write_text("schema: 4\nsuite: small\ndoc: d\ncases:\n - name: a\n") assert runner.main([str(directory), "--quiet"]) == 1 @@ -886,11 +1188,12 @@ def test_the_module_runs_as_a_command(tmp_path: Path) -> None: #: The cases this client cannot answer, and the only ones it may leave -#: unanswered. Written out rather than counted, so that a sixth one -#: arriving is a failure here and not a number nobody looks at. All five +#: unanswered. Written out rather than counted, so that a seventh one +#: arriving is a failure here and not a number nobody looks at. All six #: are a time to the nanosecond, which is a digit finer than Python's #: `datetime` holds. UNHELD = { + "arrow/a-local-time-is-nanoseconds-from-midnight", "param/localtime-to-the-nanosecond", "stored/a-localtime-column-keeps-every-digit", "stored/the-columns-of-one-row-belong-to-that-row", @@ -899,24 +1202,36 @@ def test_the_module_runs_as_a_command(tmp_path: Path) -> None: } +@pytest.fixture(scope="session") +def corpus() -> Iterator[runner.Report]: + """The whole corpus, run once for everything below that reads it. + + Every case gets a database of its own and there are over a thousand + of them, so a run is a couple of gigabytes that exist for as long as + it takes. Once per session rather than once per test, and under a + directory that goes with the run rather than one pytest keeps three + generations of: two tests reading one report is not a reason to fill + a disk twice over, and it is a disk this has already filled.""" + with tempfile.TemporaryDirectory(prefix="zudb-corpus-") as work: + yield runner.run(cases.read_dir(Path(CASES)), Path(work)) + + @needs_cases -def test_every_case_the_engine_ships_passes_through_this_client(tmp_path: Path) -> None: +def test_every_case_the_engine_ships_passes_through_this_client(corpus: runner.Report) -> None: """The corpus itself, which is the check the other two runners run and the reason this one exists.""" - report = runner.run(cases.read_dir(Path(CASES)), tmp_path) - assert report.count(runner.FAILED) == 0, [str(r) for r in report.failures()][:10] - unheld = {f"{r.suite}/{r.case}" for r in report.ran if r.outcome == runner.UNSUPPORTED} + assert corpus.count(runner.FAILED) == 0, [str(r) for r in corpus.failures()][:10] + unheld = {f"{r.suite}/{r.case}" for r in corpus.ran if r.outcome == runner.UNSUPPORTED} assert unheld == UNHELD - assert report.count(runner.PASSED) == len(report.ran) - len(UNHELD) + assert corpus.count(runner.PASSED) == len(corpus.ran) - len(UNHELD) @needs_cases def test_the_cases_this_client_cannot_hold_are_a_precision_and_not_a_wrong_answer( - tmp_path: Path, + corpus: runner.Report, ) -> None: """Every one of them says which value it is and why, because a case reported as unsupported with no reason is a case nobody revisits.""" - report = runner.run(cases.read_dir(Path(CASES)), tmp_path) - for ran in report.ran: + for ran in corpus.ran: if ran.outcome == runner.UNSUPPORTED: assert "finer" in ran.detail, str(ran)