diff --git a/README.md b/README.md index da6625f..b611812 100644 --- a/README.md +++ b/README.md @@ -327,6 +327,8 @@ with zudb.dbapi.connect("social.zu1") as conn: Parameters are `?`, which PEP 249 calls `qmark`, rewritten into the engine's own `$name` before the statement runs. The `named` style cannot work here: `:name` is how a pattern names a label, so `(p:person)` and `WHERE p.uid = :uid` cannot be told apart without parsing the statement, while `?` is a character GQL has no meaning for anywhere. A question mark inside a string, a quoted name or a comment is text somebody wrote and is left alone, and passing a dict instead of a sequence hands the statement over untouched, so `$name` still works for anyone writing zu statements rather than generating them. +`fetchone`, `fetchmany` and `fetchall` are all there, and the middle one is the one to reach for. It takes a block in a single call into the engine rather than a call per row, and `arraysize` is the size it takes when it is not told: PEP 249 sets that at one and nobody should leave it there. Over a million rows in two columns on this machine, reading them a row at a time takes 234 ms, and reading them in blocks of a thousand takes 75 ms, which is 4.3 million rows a second against 13.3 million. `fetchall` takes 125 ms and holds a million tuples at once, so blocks are both the faster way and the one that does not have to fit. Asking for more rows than are left gives what is left, asking for none gives none, and asking for fewer than none is refused rather than answered with an empty list, since an empty list is what the end of the rows looks like and a loop that believed it would stop early and quietly. + Transactions are implicit, which PEP 249 requires and the native client does not do: one opens before the first statement after each `commit` or `rollback`, and closing a connection rolls back what was not committed. `connect(..., autocommit=True)` turns that off and gives back the native behaviour, where every statement stands alone. The exception classes are both hierarchies at once. `zudb.Error` is the `Error` PEP 249 asks for, and a syntax error is a `zudb.SyntaxError` and a `dbapi.ProgrammingError` and the same object, carrying the same code, position and documentation link, so a driver-shaped library and code written against this client can catch the same failure in the same program. `cur.description` names the columns and gives the Python type of what is in them, read from the first rows, since a result declares no types of its own. `conn.zu` is the connection underneath and `cur.result` is the result the last statement gave back, so appenders, registered frames, `interrupt()` and `to_arrow()` are all still there. It is a layer, not a second client. @@ -351,7 +353,7 @@ Half of this package is compiled, which is the one thing an inspection cannot se ## What works today -The list above is what this client is for. What it does so far is the core of it: `connect`, `execute` and `sql` with named parameters, results that iterate and fetch, values as Python objects both ways including dates, times, datetimes and durations, `Node`, `Rel` and `Path` as classes, `load` for building a graph with edges in it, an appender for growing one, transactions as a context manager that commits at the end of a block and rolls back when it raises, every condition as an exception class carrying its code, its position and its documentation link, results as Arrow columns and as pandas and polars frames, `register` for putting a frame under a name a statement can match on and reading it where it lies, stubs inside the wheel with a gate that keeps them true, the GIL released around every statement, every load and every copy out, `Ctrl-C` and `interrupt()` stopping a statement without touching the connection under it, `zudb.aio` for the same calls awaited on an event loop, results, nodes, rels and paths that draw themselves in a notebook with `%gql` and `%%gql` to run statements in one, `zudb.dbapi` for code written against PEP 249, `prepare`, `explain` and `profile` for a statement compiled once, the plan it would run and the plan it did, `stream` for a result read as the engine makes it rather than after it has made all of it, and `cursor()` for a second connection made from the first, which is how a pool is written. Each one landed with the tests that say it works. +The list above is what this client is for. What it does so far is the core of it: `connect`, `execute` and `sql` with named parameters, results that iterate and fetch a row or a block at a time, values as Python objects both ways including dates, times, datetimes and durations, `Node`, `Rel` and `Path` as classes, `load` for building a graph with edges in it, an appender for growing one, transactions as a context manager that commits at the end of a block and rolls back when it raises, every condition as an exception class carrying its code, its position and its documentation link, results as Arrow columns and as pandas and polars frames, `register` for putting a frame under a name a statement can match on and reading it where it lies, stubs inside the wheel with a gate that keeps them true, the GIL released around every statement, every load and every copy out, `Ctrl-C` and `interrupt()` stopping a statement without touching the connection under it, `zudb.aio` for the same calls awaited on an event loop, results, nodes, rels and paths that draw themselves in a notebook with `%gql` and `%%gql` to run statements in one, `zudb.dbapi` for code written against PEP 249, `prepare`, `explain` and `profile` for a statement compiled once, the plan it would run and the plan it did, `stream` for a result read as the engine makes it rather than after it has made all of it, and `cursor()` for a second connection made from the first, which is how a pool is written. Each one landed with the tests that say it works. ## Wheels diff --git a/python/zudb/_zudb.pyi b/python/zudb/_zudb.pyi index b152a51..bd4c557 100644 --- a/python/zudb/_zudb.pyi +++ b/python/zudb/_zudb.pyi @@ -465,6 +465,9 @@ class Result: def fetchone(self) -> tuple[Value, ...] | None: """The next row, or `None` when there are no more.""" + def fetchmany(self, size: int = 1) -> list[tuple[Value, ...]]: + """The next `size` rows, or as many as are left.""" + # `Any` and not `pyarrow.Table`, because the wheel does not depend # on pyarrow and a stub that imported it would fail to resolve for # every caller who does not have it either. diff --git a/python/zudb/dbapi.py b/python/zudb/dbapi.py index e33118d..9140a2f 100644 --- a/python/zudb/dbapi.py +++ b/python/zudb/dbapi.py @@ -459,9 +459,10 @@ def __init__(self, connection: Connection) -> None: self._description: tuple[tuple[Any, ...], ...] | None = None self._rowcount = -1 #: Rows `fetchmany` takes when it is not told how many. One, - #: which PEP 249 asks for and nobody should leave alone: rows - #: are already in memory here, so a bigger number costs - #: nothing and saves calls. + #: which PEP 249 asks for and nobody should leave alone: the + #: rows are already in memory here and a block of them is one + #: call into the engine, so a bigger number costs nothing and + #: saves a crossing per row. self.arraysize = 1 @property @@ -557,29 +558,53 @@ def fetchone(self) -> tuple[Value, ...] | None: result = self._rows() if self._ahead: return self._ahead.popleft() - with _translating(): + # `_translating()` written out, and only here. It is a + # generator-based context manager, which costs about a + # microsecond to enter and leave, and this is the one call in + # the layer that a caller makes once per row: entering it a + # million times took longer than reading the million rows. + try: return result.fetchone() + except zudb.Error as failure: + _reraise(failure) def fetchmany(self, size: int | None = None) -> list[tuple[Value, ...]]: - """The next `size` rows, or as many as are left.""" - self._rows() + """The next `size` rows, or as many as are left. + + `size` defaults to `arraysize`, which PEP 249 sets at one. The + block is taken in a single call into the engine rather than a + call per row, so the number is worth raising: a page of a + thousand rows costs one crossing here and a thousand crossings + in a loop over `fetchone`. + + A negative `size` is a mistake and is refused. Returning an + empty list for it would look exactly like the end of the rows, + and the loop that asked would stop early and quietly. + """ + result = self._rows() wanted = self.arraysize if size is None else size + if wanted < 0: + raise ProgrammingError(f"fetchmany wants a number of rows, and {wanted} is not one") got: list[tuple[Value, ...]] = [] - while len(got) < wanted: - row = self.fetchone() - if row is None: - break - got.append(row) + while self._ahead and len(got) < wanted: + got.append(self._ahead.popleft()) + if len(got) < wanted: + with _translating(): + got.extend(result.fetchmany(wanted - len(got))) return got def fetchall(self) -> list[tuple[Value, ...]]: - """Every row that has not been fetched yet.""" + """Every row that has not been fetched yet. + + One crossing, like `fetchmany`: `len(result)` is every row the + statement produced, which is never fewer than the rows left, and + asking for more than are left gives what is left. + """ result = self._rows() taken = list(self._ahead) self._ahead.clear() with _translating(): - while (row := result.fetchone()) is not None: - taken.append(row) + taken.extend(result.fetchmany(len(result))) return taken def setinputsizes(self, sizes: Iterable[object]) -> None: diff --git a/src/conn.rs b/src/conn.rs index 910b88a..a22fb93 100644 --- a/src/conn.rs +++ b/src/conn.rs @@ -725,6 +725,41 @@ impl Result { self.row(py, row).map(Some) } + /// The next `size` rows, or as many as are left. + /// + /// The block form of `fetchone`, and the reason to prefer it is + /// the same one the streaming reader gives for reading batches: a + /// block is one call where a row is one call, and the call is what + /// costs. The position moves once, at the end, so a conversion + /// that fails leaves the result where it was rather than half a + /// block further on. + /// + /// Asking for more rows than are left gives what is left. Asking + /// for none gives none, which is what a loop over a page size read + /// from configuration wants; asking for fewer than none is a + /// mistake and says so. + #[pyo3(signature = (size = 1))] + fn fetchmany<'py>(&self, py: Python<'py>, size: isize) -> PyResult> { + if size < 0 { + return Err(pyo3::exceptions::PyValueError::new_err(format!( + "fetchmany wants a number of rows, and {size} is not one" + ))); + } + let mut next = self.next.lock().map_err(|_| { + pyo3::exceptions::PyRuntimeError::new_err("this result was left locked by a panic") + })?; + let from = *next; + let upto = from + .saturating_add(size as usize) + .min(self.result.rows.len()); + let rows = PyList::empty(py); + for row in &self.result.rows[from..upto] { + rows.append(self.row(py, row)?)?; + } + *next = upto; + Ok(rows) + } + /// The warnings the statement raised, if it raised any. A notice /// is a condition that did not stop the statement, so it arrives /// beside the rows rather than instead of them. diff --git a/tests/test_dbapi.py b/tests/test_dbapi.py index a913577..dd12244 100644 --- a/tests/test_dbapi.py +++ b/tests/test_dbapi.py @@ -71,6 +71,39 @@ def test_fetchmany_takes_arraysize_when_it_is_not_told(conn: dbapi.Connection) - assert cur.fetchmany() == [("grace",), ("kay",)] +def test_a_block_reaches_across_the_rows_read_to_type_the_columns( + conn: dbapi.Connection, +) -> None: + """One call, and the join in the middle of it does not show.""" + cur = conn.cursor() + cur.execute("MATCH (p:person) RETURN p.name AS name") + assert cur.fetchmany(3) == [("ada",), ("grace",), ("kay",)] + + +def test_a_block_bigger_than_what_is_left_gives_what_is_left(conn: dbapi.Connection) -> None: + cur = conn.cursor() + cur.execute("MATCH (p:person) RETURN p.name AS name") + assert cur.fetchmany(1000) == [("ada",), ("grace",), ("kay",)] + assert cur.fetchmany(1000) == [] + + +def test_a_block_of_no_rows_takes_none(conn: dbapi.Connection) -> None: + """A page size read from configuration can be zero, and it means zero.""" + cur = conn.cursor() + cur.execute("MATCH (p:person) RETURN p.name AS name") + assert cur.fetchmany(0) == [] + assert cur.fetchone() == ("ada",) + + +def test_a_block_of_fewer_than_no_rows_is_refused(conn: dbapi.Connection) -> None: + """An empty list would read as the end of the rows and end a loop early.""" + cur = conn.cursor() + cur.execute("MATCH (p:person) RETURN p.name AS name") + with pytest.raises(dbapi.ProgrammingError, match="-1 is not one"): + cur.fetchmany(-1) + assert cur.fetchall() == [("ada",), ("grace",), ("kay",)] + + def test_a_cursor_iterates(conn: dbapi.Connection) -> None: """An extension PEP 249 names, and the way anyone actually reads rows.""" cur = conn.cursor() diff --git a/tests/test_query.py b/tests/test_query.py index 26e2e74..4c04053 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -26,6 +26,41 @@ def test_fetchone_walks_the_rows_and_then_answers_none(social: zudb.Connection) assert rows.fetchone() is None +def test_fetchmany_takes_a_block_and_moves_the_cursor_once(social: zudb.Connection) -> None: + rows = social.execute("MATCH (p:person) RETURN p.uid AS uid ORDER BY uid") + assert rows.fetchmany(2) == [(10,), (20,)] + assert rows.fetchmany(2) == [(30,)] + assert rows.fetchmany(2) == [] + + +def test_fetchmany_takes_one_row_when_it_is_not_told_how_many(social: zudb.Connection) -> None: + rows = social.execute("MATCH (p:person) RETURN p.uid AS uid ORDER BY uid") + assert rows.fetchmany() == [(10,)] + assert rows.fetchone() == (20,) + assert rows.fetchmany() == [(30,)] + + +def test_fetchmany_and_fetchone_share_one_position(social: zudb.Connection) -> None: + rows = social.execute("MATCH (p:person) RETURN p.uid AS uid ORDER BY uid") + assert rows.fetchone() == (10,) + assert rows.fetchmany(10) == [(20,), (30,)] + assert rows.fetchone() is None + + +def test_fetchmany_of_no_rows_takes_none_and_leaves_the_position(social: zudb.Connection) -> None: + rows = social.execute("MATCH (p:person) RETURN p.uid AS uid ORDER BY uid") + assert rows.fetchmany(0) == [] + assert rows.fetchone() == (10,) + + +def test_fetchmany_of_fewer_than_no_rows_is_refused(social: zudb.Connection) -> None: + rows = social.execute("MATCH (p:person) RETURN p.uid AS uid ORDER BY uid") + with pytest.raises(ValueError, match="-1 is not one"): + rows.fetchmany(-1) + # Refused before anything moved, so the first row is still the first. + assert rows.fetchone() == (10,) + + def test_iterating_does_not_move_the_cursor(social: zudb.Connection) -> None: rows = social.execute("MATCH (p:person) RETURN p.uid AS uid ORDER BY uid") assert [uid for (uid,) in rows] == [10, 20, 30]