|
| 1 | +package dev.zudb.arrow; |
| 2 | + |
| 3 | +import dev.zudb.Connection; |
| 4 | +import dev.zudb.Result; |
| 5 | +import java.util.Objects; |
| 6 | +import org.apache.arrow.c.ArrowArrayStream; |
| 7 | +import org.apache.arrow.c.Data; |
| 8 | +import org.apache.arrow.memory.BufferAllocator; |
| 9 | +import org.apache.arrow.vector.ipc.ArrowReader; |
| 10 | + |
| 11 | +/** |
| 12 | + * A result as Arrow, without a copy on the way. |
| 13 | + * |
| 14 | + * <pre>{@code |
| 15 | + * try (BufferAllocator allocator = new RootAllocator(); |
| 16 | + * ArrowReader reader = Arrow.query(allocator, conn, "MATCH (p:Person) RETURN p.id AS id")) { |
| 17 | + * while (reader.loadNextBatch()) { |
| 18 | + * BigIntVector ids = (BigIntVector) reader.getVectorSchemaRoot().getVector(0); |
| 19 | + * for (int i = 0; i < ids.getValueCount(); i++) { |
| 20 | + * sum += ids.get(i); |
| 21 | + * } |
| 22 | + * } |
| 23 | + * } |
| 24 | + * }</pre> |
| 25 | + * |
| 26 | + * <p>Nothing on this path is proportional to the answer. The arrays that cross |
| 27 | + * are the buffers the engine's executor filled, at the addresses it filled |
| 28 | + * them at, and what an export costs is the schema, the stream and the pointers |
| 29 | + * in it. A million rows and ten thousand cost about the same. |
| 30 | + * |
| 31 | + * <p>That is also why an export spends the result. Once the buffers have left, |
| 32 | + * there is nothing on this side to read a second time, so the {@code Result} |
| 33 | + * handed to any of these is closed by the call and every buffer a columnar |
| 34 | + * reader borrowed from it before now belongs to the Arrow consumer. Closing it |
| 35 | + * again afterwards is the no-op it always was, so a try-with-resources around |
| 36 | + * it is still the right shape. |
| 37 | + * |
| 38 | + * <p>The reader owns what it was given and releases the stream when it closes, |
| 39 | + * which releases the result the stream was made from. Close the reader. |
| 40 | + * |
| 41 | + * <p>A result the engine had to build across its rows, which is anything with |
| 42 | + * an {@code ORDER BY}, has no buffers to move and is read into buffers of its |
| 43 | + * own on the way out. That is the fallback working rather than the fast path |
| 44 | + * failing, and it is still one pass and still correct. |
| 45 | + */ |
| 46 | +public final class Arrow { |
| 47 | + |
| 48 | + private Arrow() {} |
| 49 | + |
| 50 | + /** |
| 51 | + * Runs a statement and hands back its answer as Arrow. |
| 52 | + * |
| 53 | + * @param allocator what the Arrow side allocates from |
| 54 | + * @param conn the connection |
| 55 | + * @param statement the text |
| 56 | + * @return the reader, which the caller closes |
| 57 | + */ |
| 58 | + public static ArrowReader query(BufferAllocator allocator, Connection conn, String statement) { |
| 59 | + Objects.requireNonNull(conn, "conn"); |
| 60 | + Result result = conn.query(statement); |
| 61 | + try { |
| 62 | + return reader(allocator, result); |
| 63 | + } catch (RuntimeException | Error e) { |
| 64 | + result.close(); |
| 65 | + throw e; |
| 66 | + } |
| 67 | + } |
| 68 | + |
| 69 | + /** |
| 70 | + * A result already in hand, as Arrow, in batches of {@link |
| 71 | + * Result#DEFAULT_BATCH} rows. |
| 72 | + * |
| 73 | + * @param allocator what the Arrow side allocates from |
| 74 | + * @param result the result, which this call spends |
| 75 | + * @return the reader, which the caller closes |
| 76 | + */ |
| 77 | + public static ArrowReader reader(BufferAllocator allocator, Result result) { |
| 78 | + return reader(allocator, result, 0); |
| 79 | + } |
| 80 | + |
| 81 | + /** |
| 82 | + * The same, with the batch size named. |
| 83 | + * |
| 84 | + * @param allocator what the Arrow side allocates from |
| 85 | + * @param result the result, which this call spends |
| 86 | + * @param rowsPerBatch how many rows a consumer sees at a time, or zero for |
| 87 | + * {@link Result#DEFAULT_BATCH}. The batches are slices of arrays that |
| 88 | + * are already in memory, so this is about what a consumer likes to work |
| 89 | + * in and not about what gets allocated |
| 90 | + * @return the reader, which the caller closes |
| 91 | + */ |
| 92 | + public static ArrowReader reader(BufferAllocator allocator, Result result, long rowsPerBatch) { |
| 93 | + Objects.requireNonNull(allocator, "allocator"); |
| 94 | + Objects.requireNonNull(result, "result"); |
| 95 | + ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator); |
| 96 | + try { |
| 97 | + result.exportArrow(stream.memoryAddress(), rowsPerBatch); |
| 98 | + return Data.importArrayStream(allocator, stream); |
| 99 | + } catch (RuntimeException | Error e) { |
| 100 | + // A refusal leaves the struct as it was allocated, which is |
| 101 | + // released, so this frees the memory it sits in and calls nothing. |
| 102 | + // An import that failed leaves a live stream, and this is what |
| 103 | + // releases it. |
| 104 | + stream.close(); |
| 105 | + throw e; |
| 106 | + } |
| 107 | + } |
| 108 | +} |
0 commit comments