From 533fc542b3a905e67fae1779c6a30f61a4636fa1 Mon Sep 17 00:00:00 2001 From: ZhuchkaTriplesix Date: Wed, 26 Aug 2026 12:13:39 +0300 Subject: [PATCH] perf(memory): explore compact typed storage and lazy cell stringification for large results (#605) --- lib/core/database/compact_result_dataset.dart | 403 ++++++++++++++++++ .../database/compact_result_dataset_test.dart | 117 +++++ 2 files changed, 520 insertions(+) create mode 100644 lib/core/database/compact_result_dataset.dart create mode 100644 test/core/database/compact_result_dataset_test.dart diff --git a/lib/core/database/compact_result_dataset.dart b/lib/core/database/compact_result_dataset.dart new file mode 100644 index 0000000..ce08563 --- /dev/null +++ b/lib/core/database/compact_result_dataset.dart @@ -0,0 +1,403 @@ +import 'dart:collection'; +import 'dart:typed_data'; + +import 'result_row_string_convert.dart'; + +/// Supported compact column storage kinds. +enum CompactColumnKind { + int64, + float64, + string, + generic, +} + +/// Columnar storage interface for a single database column in a result set. +sealed class CompactColumn { + int get length; + CompactColumnKind get kind; + Object? rawValueAt(int index); + String stringValueAt(int index, [StringInternPool? pool]); + bool isNull(int index); +} + +/// Compact 64-bit integer column backed by [Int64List] and a packed null bitmap. +class Int64CompactColumn implements CompactColumn { + Int64CompactColumn({ + required this.values, + required this.nullBitmap, + required this.length, + }); + + final Int64List values; + final Uint8List nullBitmap; // 1 bit per row indicates null + @override + final int length; + + @override + CompactColumnKind get kind => CompactColumnKind.int64; + + @override + bool isNull(int index) { + final byteIndex = index >> 3; + final bitOffset = index & 7; + return (nullBitmap[byteIndex] & (1 << bitOffset)) != 0; + } + + @override + Object? rawValueAt(int index) { + if (isNull(index)) return null; + return values[index]; + } + + @override + String stringValueAt(int index, [StringInternPool? pool]) { + if (isNull(index)) return 'NULL'; + final val = values[index]; + if (val >= 0 && val <= 10 && pool != null) { + return pool.internObject(val); + } + return val.toString(); + } +} + +/// Compact 64-bit float column backed by [Float64List] and a packed null bitmap. +class Float64CompactColumn implements CompactColumn { + Float64CompactColumn({ + required this.values, + required this.nullBitmap, + required this.length, + }); + + final Float64List values; + final Uint8List nullBitmap; + @override + final int length; + + @override + CompactColumnKind get kind => CompactColumnKind.float64; + + @override + bool isNull(int index) { + final byteIndex = index >> 3; + final bitOffset = index & 7; + return (nullBitmap[byteIndex] & (1 << bitOffset)) != 0; + } + + @override + Object? rawValueAt(int index) { + if (isNull(index)) return null; + return values[index]; + } + + @override + String stringValueAt(int index, [StringInternPool? pool]) { + if (isNull(index)) return 'NULL'; + final val = values[index]; + if (val == val.toInt() && !val.isNaN && !val.isInfinite) { + return val.toInt().toString(); + } + return val.toString(); + } +} + +/// String column backed by interned String references. +class StringCompactColumn implements CompactColumn { + StringCompactColumn({ + required this.values, + required this.length, + }); + + final List values; + @override + final int length; + + @override + CompactColumnKind get kind => CompactColumnKind.string; + + @override + bool isNull(int index) => values[index] == null; + + @override + Object? rawValueAt(int index) => values[index]; + + @override + String stringValueAt(int index, [StringInternPool? pool]) { + final val = values[index]; + if (val == null) return 'NULL'; + return pool != null ? pool.intern(val) : val; + } +} + +/// Generic object column for composite or unparsed database types. +class GenericCompactColumn implements CompactColumn { + GenericCompactColumn({ + required this.values, + required this.length, + }); + + final List values; + @override + final int length; + + @override + CompactColumnKind get kind => CompactColumnKind.generic; + + @override + bool isNull(int index) => values[index] == null; + + @override + Object? rawValueAt(int index) => values[index]; + + @override + String stringValueAt(int index, [StringInternPool? pool]) { + final val = values[index]; + if (val == null) return 'NULL'; + return pool != null ? pool.internObject(val) : val.toString(); + } +} + +/// Memory-efficient columnar dataset for query result tables. +/// +/// Converts numeric columns to contiguous primitive TypedData arrays +/// and lazily stringifies cells on-demand for viewport rendering, +/// eliminating the allocation of millions of intermediate String objects in heap. +class CompactResultDataset { + CompactResultDataset({ + required this.columnNames, + required this.columns, + required this.rowCount, + StringInternPool? pool, + }) : pool = pool ?? StringInternPool(); + + final List columnNames; + final List columns; + final int rowCount; + final StringInternPool pool; + + int get columnCount => columnNames.length; + + /// Returns the raw cell value (int, double, String, or null). + Object? rawCellAt(int rowIndex, int colIndex) { + if (colIndex < 0 || colIndex >= columns.length) return null; + return columns[colIndex].rawValueAt(rowIndex); + } + + /// Lazily stringifies the cell value at [rowIndex], [colIndex]. + String cellAt(int rowIndex, int colIndex) { + if (colIndex < 0 || colIndex >= columns.length) return 'NULL'; + return columns[colIndex].stringValueAt(rowIndex, pool); + } + + /// Returns a full row of formatted strings for [rowIndex]. + List rowStrings(int rowIndex) { + return [ + for (var col = 0; col < columns.length; col++) + cellAt(rowIndex, col), + ]; + } + + /// Returns a full row of raw typed values for [rowIndex]. + List rawRow(int rowIndex) { + return [ + for (var col = 0; col < columns.length; col++) + rawCellAt(rowIndex, col), + ]; + } + + /// Exposes a lazy, unmodifiable `List>` row view that formats + /// cells on-the-fly when indexed by viewport-based virtual grids. + List> asLazyRowList() => _LazyDatasetRowList(this); + + /// Constructs a [CompactResultDataset] from raw rows, detecting column types + /// and packing numeric columns into [Int64List] / [Float64List]. + static CompactResultDataset fromRawRows( + List columnNames, + List> rawRows, { + StringInternPool? pool, + }) { + final rowCount = rawRows.length; + final colCount = columnNames.length; + final internPool = pool ?? StringInternPool(); + + if (rowCount == 0 || colCount == 0) { + return CompactResultDataset( + columnNames: columnNames, + columns: const [], + rowCount: 0, + pool: internPool, + ); + } + + // 1. Detect column types by inspecting non-null samples + final kinds = List.filled(colCount, CompactColumnKind.generic); + for (var col = 0; col < colCount; col++) { + var allInt = true; + var allFloat = true; + var allString = true; + var sampleCount = 0; + + for (var row = 0; row < rowCount; row++) { + final val = rawRows[row].length > col ? rawRows[row][col] : null; + if (val == null) continue; + sampleCount++; + + if (val is! int) { + allInt = false; + } + if (val is! num) { + allFloat = false; + } + if (val is! String) { + allString = false; + } + + if (sampleCount >= 60 && !allInt && !allFloat && !allString) { + break; + } + } + + if (sampleCount > 0) { + if (allInt) { + kinds[col] = CompactColumnKind.int64; + } else if (allFloat) { + kinds[col] = CompactColumnKind.float64; + } else if (allString) { + kinds[col] = CompactColumnKind.string; + } else { + kinds[col] = CompactColumnKind.generic; + } + } else { + kinds[col] = CompactColumnKind.string; + } + } + + // 2. Allocate packed columnar buffers + final compactColumns = []; + final bitmapBytes = (rowCount + 7) >> 3; + + for (var col = 0; col < colCount; col++) { + final kind = kinds[col]; + switch (kind) { + case CompactColumnKind.int64: + final values = Int64List(rowCount); + final nullBitmap = Uint8List(bitmapBytes); + for (var row = 0; row < rowCount; row++) { + final val = rawRows[row].length > col ? rawRows[row][col] : null; + if (val == null) { + final byte = row >> 3; + final bit = row & 7; + nullBitmap[byte] |= (1 << bit); + } else if (val is int) { + values[row] = val; + } else if (val is num) { + values[row] = val.toInt(); + } + } + compactColumns.add(Int64CompactColumn( + values: values, + nullBitmap: nullBitmap, + length: rowCount, + )); + + case CompactColumnKind.float64: + final values = Float64List(rowCount); + final nullBitmap = Uint8List(bitmapBytes); + for (var row = 0; row < rowCount; row++) { + final val = rawRows[row].length > col ? rawRows[row][col] : null; + if (val == null) { + final byte = row >> 3; + final bit = row & 7; + nullBitmap[byte] |= (1 << bit); + } else if (val is num) { + values[row] = val.toDouble(); + } + } + compactColumns.add(Float64CompactColumn( + values: values, + nullBitmap: nullBitmap, + length: rowCount, + )); + + case CompactColumnKind.string: + final values = List.filled(rowCount, null); + for (var row = 0; row < rowCount; row++) { + final val = rawRows[row].length > col ? rawRows[row][col] : null; + if (val is String) { + values[row] = internPool.intern(val); + } else if (val != null) { + values[row] = internPool.intern(val.toString()); + } + } + compactColumns.add(StringCompactColumn( + values: values, + length: rowCount, + )); + + case CompactColumnKind.generic: + final values = List.filled(rowCount, null); + for (var row = 0; row < rowCount; row++) { + final val = rawRows[row].length > col ? rawRows[row][col] : null; + values[row] = val; + } + compactColumns.add(GenericCompactColumn( + values: values, + length: rowCount, + )); + } + } + + return CompactResultDataset( + columnNames: columnNames, + columns: compactColumns, + rowCount: rowCount, + pool: internPool, + ); + } +} + +class _LazyDatasetRowList extends ListBase> { + _LazyDatasetRowList(this.dataset); + + final CompactResultDataset dataset; + + @override + int get length => dataset.rowCount; + + @override + set length(int newLength) => + throw UnsupportedError('Dataset rows are unmodifiable'); + + @override + List operator [](int index) { + RangeError.checkValidIndex(index, this, 'index', length); + return _LazyDatasetRow(dataset, index); + } + + @override + void operator []=(int index, List value) => + throw UnsupportedError('Dataset rows are unmodifiable'); +} + +class _LazyDatasetRow extends ListBase { + _LazyDatasetRow(this.dataset, this.rowIndex); + + final CompactResultDataset dataset; + final int rowIndex; + + @override + int get length => dataset.columnCount; + + @override + set length(int newLength) => + throw UnsupportedError('Row cells are unmodifiable'); + + @override + String operator [](int index) { + RangeError.checkValidIndex(index, this, 'index', length); + return dataset.cellAt(rowIndex, index); + } + + @override + void operator []=(int index, String value) => + throw UnsupportedError('Row cells are unmodifiable'); +} diff --git a/test/core/database/compact_result_dataset_test.dart b/test/core/database/compact_result_dataset_test.dart new file mode 100644 index 0000000..d42c0ee --- /dev/null +++ b/test/core/database/compact_result_dataset_test.dart @@ -0,0 +1,117 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:querya_desktop/core/database/compact_result_dataset.dart'; + +void main() { + group('CompactResultDataset', () { + test('packs typed columns into specialized compact columnar buffers', () { + final columns = ['id', 'amount', 'status', 'meta']; + final rows = >[ + [1, 19.99, 'active', {'role': 'admin'}], + [2, 49.50, 'pending', null], + [3, null, 'active', {'role': 'user'}], + [null, 0.0, null, null], + ]; + + final dataset = CompactResultDataset.fromRawRows(columns, rows); + + expect(dataset.rowCount, 4); + expect(dataset.columnCount, 4); + expect(dataset.columns[0].kind, CompactColumnKind.int64); + expect(dataset.columns[1].kind, CompactColumnKind.float64); + expect(dataset.columns[2].kind, CompactColumnKind.string); + expect(dataset.columns[3].kind, CompactColumnKind.generic); + + // Raw value tests + expect(dataset.rawCellAt(0, 0), 1); + expect(dataset.rawCellAt(0, 1), 19.99); + expect(dataset.rawCellAt(0, 2), 'active'); + expect(dataset.rawCellAt(0, 3), {'role': 'admin'}); + + // Null handling + expect(dataset.rawCellAt(2, 1), isNull); + expect(dataset.rawCellAt(3, 0), isNull); + expect(dataset.rawCellAt(3, 2), isNull); + + // Stringified lazy cells + expect(dataset.cellAt(0, 0), '1'); + expect(dataset.cellAt(0, 1), '19.99'); + expect(dataset.cellAt(0, 2), 'active'); + expect(dataset.cellAt(2, 1), 'NULL'); + expect(dataset.cellAt(3, 0), 'NULL'); + expect(dataset.cellAt(3, 2), 'NULL'); + expect(dataset.cellAt(3, 1), '0'); + }); + + test('asLazyRowList provides transparent List> indexable interface', () { + final columns = ['code', 'count']; + final rows = >[ + ['alpha', 10], + ['beta', 25], + ['gamma', null], + ]; + + final dataset = CompactResultDataset.fromRawRows(columns, rows); + final lazyRows = dataset.asLazyRowList(); + + expect(lazyRows.length, 3); + expect(lazyRows[0].length, 2); + expect(lazyRows[0][0], 'alpha'); + expect(lazyRows[0][1], '10'); + expect(lazyRows[1][0], 'beta'); + expect(lazyRows[1][1], '25'); + expect(lazyRows[2][0], 'gamma'); + expect(lazyRows[2][1], 'NULL'); + + expect(() => lazyRows[0][0] = 'modified', throwsUnsupportedError); + expect(() => lazyRows.add(['delta', '50']), throwsUnsupportedError); + }); + + test('handles empty dataset gracefully', () { + final dataset = CompactResultDataset.fromRawRows(['id', 'name'], []); + expect(dataset.rowCount, 0); + expect(dataset.asLazyRowList(), isEmpty); + }); + + test('100,000-row benchmark verifies packing throughput and fast viewport slicing', () { + final columns = ['id', 'user_id', 'balance', 'status', 'flag']; + final statuses = ['active', 'inactive', 'suspended', 'trial']; + + final rawData = List>.generate( + 100000, + (i) => [ + i + 1, + (i * 3) % 10000, + (i % 100) * 1.5, + statuses[i % statuses.length], + i % 2 == 0 ? 1 : 0, + ], + ); + + final packStopwatch = Stopwatch()..start(); + final dataset = CompactResultDataset.fromRawRows(columns, rawData); + packStopwatch.stop(); + + expect(dataset.rowCount, 100000); + expect(packStopwatch.elapsedMilliseconds, lessThan(300)); + + final lazyRows = dataset.asLazyRowList(); + + // Viewport simulation: slice 50 rows x 5 columns + final viewportStopwatch = Stopwatch()..start(); + final viewportSlice = >[]; + for (var r = 50000; r < 50050; r++) { + final row = []; + for (var c = 0; c < 5; c++) { + row.add(lazyRows[r][c]); + } + viewportSlice.add(row); + } + viewportStopwatch.stop(); + + expect(viewportSlice.length, 50); + expect(viewportStopwatch.elapsedMilliseconds, lessThan(5)); + expect(viewportSlice[0][0], '50001'); + expect(viewportSlice[0][3], 'active'); + }); + }); +}