From 83f8777b146e73035eca083a8f3e9ac5176fe445 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 15 Jan 2026 15:23:08 +0100 Subject: [PATCH 1/2] Merge pull request #94125 from ClickHouse/straddle Fix parquet reader v3 handling arrays that cross pages --- .../Formats/Impl/Parquet/Reader.cpp | 28 ++++--- src/Processors/Formats/Impl/Parquet/Reader.h | 3 +- .../00900_long_parquet_load_2.reference | 73 ++++++++++-------- .../0_stateless/00900_long_parquet_load_2.sh | 5 +- .../data_parquet/array_across_pages_1.parquet | Bin 0 -> 548 bytes .../data_parquet/array_across_pages_2.parquet | Bin 0 -> 622 bytes 6 files changed, 62 insertions(+), 47 deletions(-) create mode 100644 tests/queries/0_stateless/data_parquet/array_across_pages_1.parquet create mode 100644 tests/queries/0_stateless/data_parquet/array_across_pages_2.parquet diff --git a/src/Processors/Formats/Impl/Parquet/Reader.cpp b/src/Processors/Formats/Impl/Parquet/Reader.cpp index 24ba5277dd70..01cd1fedc8c5 100644 --- a/src/Processors/Formats/Impl/Parquet/Reader.cpp +++ b/src/Processors/Formats/Impl/Parquet/Reader.cpp @@ -1315,7 +1315,7 @@ void Reader::decodePrimitiveColumn(ColumnChunk & column, const PrimitiveColumnIn size_t end_row_idx = start_row_idx + num_rows; row_subidx += num_rows; - skipToRow(start_row_idx, column, column_info); + skipToRowOrNextPage(start_row_idx, column, column_info); while (true) // loop over pages { @@ -1331,7 +1331,7 @@ void Reader::decodePrimitiveColumn(ColumnChunk & column, const PrimitiveColumnIn /// Advance to next page. chassert(page.value_idx == page.num_values); - skipToRow(page.next_row_idx, column, column_info); + skipToRowOrNextPage(std::nullopt, column, column_info); chassert(page.value_idx == 0); } } @@ -1386,32 +1386,38 @@ void Reader::decodePrimitiveColumn(ColumnChunk & column, const PrimitiveColumnIn } } -void Reader::skipToRow(size_t row_idx, ColumnChunk & column, const PrimitiveColumnInfo & column_info) +void Reader::skipToRowOrNextPage(std::optional row_idx, ColumnChunk & column, const PrimitiveColumnInfo & column_info) { /// True if column.page is initialized and contains the requested row_idx. bool found_page = false; auto & page = column.page; - if (page.initialized && page.value_idx < page.num_values && page.end_row_idx.has_value() && *page.end_row_idx > row_idx) + if (!row_idx.has_value()) + chassert(page.initialized); + + if (row_idx.has_value() && page.initialized && page.value_idx < page.num_values && + page.end_row_idx.has_value() && *page.end_row_idx > *row_idx) /// Fast path: we're just continuing reading the same page as before. found_page = true; if (!found_page && !column.data_pages.empty()) { /// If we have offset index, find the row index there and jump to the correct page. + if (!row_idx.has_value()) + row_idx = column.data_pages[column.data_pages_idx].end_row_idx; while (column.data_pages_idx < column.data_pages.size() && - column.data_pages[column.data_pages_idx].end_row_idx <= row_idx) + column.data_pages[column.data_pages_idx].end_row_idx <= *row_idx) ++column.data_pages_idx; if (column.data_pages_idx == column.data_pages.size()) throw Exception(ErrorCodes::INCORRECT_DATA, "Parquet offset index covers too few rows"); const auto & page_info = column.data_pages[column.data_pages_idx]; size_t first_row_idx = size_t(page_info.meta->first_row_index); - if (first_row_idx > row_idx) + if (first_row_idx > *row_idx) throw Exception(ErrorCodes::LOGICAL_ERROR, "Row passes filters but its page was not selected for reading. This is a bug."); auto data = prefetcher.getRangeData(page_info.prefetch); const char * ptr = data.data(); - if (!initializeDataPage(ptr, ptr + data.size(), first_row_idx, page_info.end_row_idx, row_idx, column, column_info)) + if (!initializeDataPage(ptr, ptr + data.size(), first_row_idx, page_info.end_row_idx, *row_idx, column, column_info)) throw Exception(ErrorCodes::LOGICAL_ERROR, "Page doesn't contain requested row"); found_page = true; } @@ -1419,8 +1425,8 @@ void Reader::skipToRow(size_t row_idx, ColumnChunk & column, const PrimitiveColu while (true) { /// Skip rows inside the page. - if (page.initialized && page.value_idx < page.num_values && - skipRowsInPage(row_idx, page, column, column_info)) + if (row_idx.has_value() && page.initialized && page.value_idx < page.num_values && + skipRowsInPage(*row_idx, page, column, column_info)) return; if (found_page) @@ -1433,8 +1439,10 @@ void Reader::skipToRow(size_t row_idx, ColumnChunk & column, const PrimitiveColu chassert(column.next_page_offset <= all_pages.size()); const char * ptr = all_pages.data() + column.next_page_offset; const char * end = all_pages.data() + all_pages.size(); - initializeDataPage(ptr, end, page.next_row_idx, /*end_row_idx=*/ std::nullopt, row_idx, column, column_info); + initializeDataPage(ptr, end, page.next_row_idx, /*end_row_idx=*/ std::nullopt, row_idx.value_or(page.next_row_idx), column, column_info); column.next_page_offset = ptr - all_pages.data(); + if (!row_idx.has_value()) + return; } } diff --git a/src/Processors/Formats/Impl/Parquet/Reader.h b/src/Processors/Formats/Impl/Parquet/Reader.h index 296cb41419a2..ea0917fbf8bb 100644 --- a/src/Processors/Formats/Impl/Parquet/Reader.h +++ b/src/Processors/Formats/Impl/Parquet/Reader.h @@ -523,7 +523,8 @@ struct Reader void initializePrefetches(); double estimateAverageStringLengthPerRow(const ColumnChunk & column, const RowGroup & row_group) const; void decodeDictionaryPageImpl(const parq::PageHeader & header, std::span data, ColumnChunk & column, const PrimitiveColumnInfo & column_info); - void skipToRow(size_t row_idx, ColumnChunk & column, const PrimitiveColumnInfo & column_info); + /// If row_idx is provided, jump to the start of that row. Otherwise go to the start of next page. + void skipToRowOrNextPage(std::optional row_idx, ColumnChunk & column, const PrimitiveColumnInfo & column_info); std::tuple> decodeAndCheckPageHeader(const char * & data_ptr, const char * data_end) const; bool initializeDataPage(const char * & data_ptr, const char * data_end, size_t next_row_idx, std::optional end_row_idx, size_t target_row_idx, ColumnChunk & column, const PrimitiveColumnInfo & column_info); void decompressPageIfCompressed(PageState & page); diff --git a/tests/queries/0_stateless/00900_long_parquet_load_2.reference b/tests/queries/0_stateless/00900_long_parquet_load_2.reference index e3f9cd89e089..899a4e21cadc 100644 --- a/tests/queries/0_stateless/00900_long_parquet_load_2.reference +++ b/tests/queries/0_stateless/00900_long_parquet_load_2.reference @@ -111,6 +111,15 @@ 1 7 false 1 1 1 10 1.1 10.1 04/01/09 1 2009-04-01 00:01:00.000000000 1 10118128356831336697 \N \N \N \N \N \N \N \N \N \N \N \N 2 14757089928194662262 +=== Try load data from array_across_pages_1.parquet +0 [1,2] 1 5845551876324623491 + +\N [] 1 5845551876324623491 +=== Try load data from array_across_pages_2.parquet +0 [1,2] 1 5845551876324623491 +1 [3] 1 204286521036002596 + +\N [] 2 6049838397360626087 === Try load data from array_float.parquet 9 idx10 [10.2,8.2] 1 6500220947925738428 0 idx1 [] 1 12173579943307849648 @@ -455,11 +464,11 @@ \N \N 4 1916660908465950835 === Try load data from ipv6_bloom_filter.gz.parquet -0 zTNx7ꙕ 1 13288000489149796924 -4 w*I\n+ 1 9468751629168416770 -3 }ϔ8<\fA뙠Ѻ 1 17226734039706043359 -1  }B,K32 1 7253289996785367618 -2 ZD/y 1 842579928916594899 +0 z����TNx��7��ꙕ 1 13288000489149796924 +4 �w�*I���\n���+� 1 9468751629168416770 +3 }ϔ8<\fA뙠��Ѻ� 1 17226734039706043359 +1  �}�B�,K�32� 1 7253289996785367618 +2 �Z�D/�y���� 1 842579928916594899 \N \N 5 11185867936307116338 === Try load data from iris.parquet @@ -492,28 +501,28 @@ \N [] [] 3 15782817770995647175 === Try load data from multi_column_bf.gz.parquet -540 -62 60904 524 4294967391 13439599920079768466 false VIRE {"key":400, "value":"BNHSB"} GMKT MưFI6eĂ 1 9735371321288412055 -923 -58 46518 345 4294967327 18309803705868924873 true LJHD {"key":97, "value":"EIUNN"} WVRQ |wGZ_P 1 12513952248079036776 -988 -89 47241 756 4294967373 15522063381546272260 true CLDK {"key":604, "value":"JSQFX"} USNQ >8O;i, 1 4175486705904763621 -73 116 32428 513 4294967384 10876946806756585544 false SGPI {"key":711, "value":"XMEED"} IOIX d\rOFЂn\b 1 11079779081220916748 -639 -69 45537 315 4294967315 7263280072686713102 false CHFC {"key":48, "value":"MFGFA"} ZLBS ]Ʇ̠Bʹ (LuբW 1 1884003582258998694 -959 -51 56925 86 4294967348 2903746547904555720 true XHJC {"key":355, "value":"MQETE"} QIQA /<3@C x* 1 12551571122189109470 -286 7 32408 89 4294967349 11768163425287102515 true JBWQ {"key":63, "value":"LBRXP"} IHTI 2B=f*_ 1 2293773524378365089 -815 -16 37808 207 4294967396 18058293579070949154 true AEZA {"key":656, "value":"GOJYT"} ZODP eWE> 1 9939955730329575199 -212 -74 50116 784 4294967316 4639886571322169794 true JZKC {"key":844, "value":"WYFIG"} KELP w+ʰO] 1 5464573773970257898 -510 -44 14672 697 4294967323 3203265463862679482 true QPXM {"key":408, "value":"BJCSQ"} NMGV x|A2^qm 1 18363658913862052503 -827 38 52135 612 4294967349 2177159037635926390 false KWOV {"key":384, "value":"PLUUJ"} VACB >$=L$Q\t 1 1291085690902008617 -705 -96 7506 479 4294967337 1919118169111152342 true EGEU {"key":522, "value":"AOPEL"} TZFW sIޣr^3 1 12776128745828051091 -174 -38 10608 469 4294967369 8139895279544416045 false SUED {"key":348, "value":"KDIJQ"} VKSM v3A, 1 8457376369144183832 -89 79 31144 138 4294967379 3327168023216233174 true MRTX {"key":492, "value":"NGCXD"} GVCG y>LO(xج;& 1 9669048967548569557 -802 92 55937 83 4294967368 13260877643210989136 true FNXX {"key":566, "value":"KVFDZ"} CMMV m:uL{Tzf 1 8531841042433805304 -599 -107 36579 116 4294967311 4635153073869187643 true BURQ {"key":259, "value":"VNMQQ"} TMOJ YznM<~�8�O��;i�,�� 1 11820769459832951331 +73 116 32428 513 4294967384 10876946806756585544 false SGPI {"key":711,"value":"XMEED"} IOIX �d�\rO�FЂ���n\b 1 129876983795436829 +639 -69 45537 315 4294967315 7263280072686713102 false CHFC {"key":48,"value":"MFGFA"} ZLBS ]Ʇ̠Bʹ (LuբW 1 12009403283062034866 +959 -51 56925 86 4294967348 2903746547904555720 true XHJC {"key":355,"value":"MQETE"} QIQA /�<3@C�� x*�� 1 14720674036350698175 +286 7 32408 89 4294967349 11768163425287102515 true JBWQ {"key":63,"value":"LBRXP"} IHTI 2���B=�f*���_� 1 18315169877216931174 +815 -16 37808 207 4294967396 18058293579070949154 true AEZA {"key":656,"value":"GOJYT"} ZODP e�W�E����>�� 1 1240809798326395103 +212 -74 50116 784 4294967316 4639886571322169794 true JZKC {"key":844,"value":"WYFIG"} KELP ��w+ʰO�����]��� 1 13188467212433715822 +510 -44 14672 697 4294967323 3203265463862679482 true QPXM {"key":408,"value":"BJCSQ"} NMGV x��|��A2�^q���m� 1 1158043015045771815 +827 38 52135 612 4294967349 2177159037635926390 false KWOV {"key":384,"value":"PLUUJ"} VACB >�$=L����$Q�\t 1 12363779168489547769 +705 -96 7506 479 4294967337 1919118169111152342 true EGEU {"key":522,"value":"AOPEL"} TZFW ����s�Iޣ�r^�3 1 10667084832660097342 +174 -38 10608 469 4294967369 8139895279544416045 false SUED {"key":348,"value":"KDIJQ"} VKSM ���v3�A����,��� 1 7867030734613018460 +89 79 31144 138 4294967379 3327168023216233174 true MRTX {"key":492,"value":"NGCXD"} GVCG y�>L�O(�xج;&�� 1 17782050321393165287 +802 92 55937 83 4294967368 13260877643210989136 true FNXX {"key":566,"value":"KVFDZ"} CMMV m:�u��L{�Tz��f� 1 7067095719366985926 +599 -107 36579 116 4294967311 4635153073869187643 true BURQ {"key":259,"value":"VNMQQ"} TMOJ �Yz�n�M<��~&1 + SELECT _row_number, *, count(), sum(cityHash64(_row_number, *)) FROM file('$DATA_DIR/$NAME') GROUP BY all WITH TOTALS ORDER BY cityHash64(_row_number) LIMIT 20;" 2>&1 done diff --git a/tests/queries/0_stateless/data_parquet/array_across_pages_1.parquet b/tests/queries/0_stateless/data_parquet/array_across_pages_1.parquet new file mode 100644 index 0000000000000000000000000000000000000000..303a6819a065aaa3e200a56dc848aa1ddbd98072 GIT binary patch literal 548 zcmbV~ze@u#6vy9%coyx@Dmm`pCNCpTAJ90YZ96?|zebZ~Nbd3nkEe)3I7Y3GnL$|5Yu9!8Jjr2-ZJ;M{F| z)3OJkU_mf2Z|Scw1}w(5+57nBY1w21bt+TMGL>!mWQ;~}I+b5crDOsiib5pi=;Z49 zmvaC7;>|@R|1!&M<-9ep#wM(LX z5W8ZB(BBj-T_V~~*n}81P%N2engFxFCAkP;@P@D;X8xyQSAlCqFrSK zT+kKFO!W)c;!=I$0yh|mgFbT;*ZkYn&|IY0pWfgPIzPu!{LwHee_IF#& z;I!Bs^n`UT&U(#m$4X_^`8v0p+MwC0TfC7Ba@krwTNj&Ykx3;*E}!N3dXVASAno#? T5j0$$NhJ*VAn=J%!vFCDMD|?7 literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_parquet/array_across_pages_2.parquet b/tests/queries/0_stateless/data_parquet/array_across_pages_2.parquet new file mode 100644 index 0000000000000000000000000000000000000000..51740029cc3aa335e8758bdce377b481b5179ac5 GIT binary patch literal 622 zcmbV~ze~eF6vtoAGc73CDla4uB%p?N(WW+S3L+I9igZv1=_K0piqTfVw&K`9&|L=? z!Budy;OgRk;Hs-obQctS*H-A@j8Kbe7P&0dyEz)AX%BJIYiC^uQxqmCn|l{m zeagejYi^ z4OfMT;VC=38q^yhQ57|D(7;hm)mlSZi`x}DOfGw9>6OrhxXqbigGx)xnI^z*z%+2$ zgi*+~V3>5b08YWm(bgIRkIK1G$g6dKSJZZpfeX4qBdUc3Dsj2qRY3>HWe}B%>OI5FNbl<9FQ+fayv{$A-1|)4gCH5VY4o*V{udu4iBtkde)hBtXQcT xRLhoAN%&r-l+BdId`hH~3E^ckPPXi)os6Gy9lzpNTqm838}dQmUD~Ff_5*ggXEy)< literal 0 HcmV?d00001 From 3915ad3df1192e59305846b14d1daa14ae1d8197 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Wed, 22 Jul 2026 19:11:52 +0200 Subject: [PATCH 2/2] Scope test changes to 25.8: keep array_across_pages cases only Revert 00900_long_parquet_load_2.sh and reference to the antalya-25.8 versions, keeping only the new array_across_pages_* reference blocks. The upstream changes enabling JSON parsing and dropping session_timezone depend on features not present in 25.8 and broke the test on all configs. Co-Authored-By: Claude Fable 5 Signed-off-by: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> --- .../00900_long_parquet_load_2.reference | 64 +++++++++---------- .../0_stateless/00900_long_parquet_load_2.sh | 5 +- 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/tests/queries/0_stateless/00900_long_parquet_load_2.reference b/tests/queries/0_stateless/00900_long_parquet_load_2.reference index 899a4e21cadc..56dc8c5bae18 100644 --- a/tests/queries/0_stateless/00900_long_parquet_load_2.reference +++ b/tests/queries/0_stateless/00900_long_parquet_load_2.reference @@ -464,11 +464,11 @@ \N \N 4 1916660908465950835 === Try load data from ipv6_bloom_filter.gz.parquet -0 z����TNx��7��ꙕ 1 13288000489149796924 -4 �w�*I���\n���+� 1 9468751629168416770 -3 }ϔ8<\fA뙠��Ѻ� 1 17226734039706043359 -1  �}�B�,K�32� 1 7253289996785367618 -2 �Z�D/�y���� 1 842579928916594899 +0 zTNx7ꙕ 1 13288000489149796924 +4 w*I\n+ 1 9468751629168416770 +3 }ϔ8<\fA뙠Ѻ 1 17226734039706043359 +1  }B,K32 1 7253289996785367618 +2 ZD/y 1 842579928916594899 \N \N 5 11185867936307116338 === Try load data from iris.parquet @@ -501,28 +501,28 @@ \N [] [] 3 15782817770995647175 === Try load data from multi_column_bf.gz.parquet -540 -62 60904 524 4294967391 13439599920079768466 false VIRE {"key":400,"value":"BNHSB"} GMKT M�ư��FI�6e���Ă 1 9024461037616764855 -923 -58 46518 345 4294967327 18309803705868924873 true LJHD {"key":97,"value":"EIUNN"} WVRQ |�w�G����Z_P 1 13244549421489669952 -988 -89 47241 756 4294967373 15522063381546272260 true CLDK {"key":604,"value":"JSQFX"} USNQ �>�8�O��;i�,�� 1 11820769459832951331 -73 116 32428 513 4294967384 10876946806756585544 false SGPI {"key":711,"value":"XMEED"} IOIX �d�\rO�FЂ���n\b 1 129876983795436829 -639 -69 45537 315 4294967315 7263280072686713102 false CHFC {"key":48,"value":"MFGFA"} ZLBS ]Ʇ̠Bʹ (LuբW 1 12009403283062034866 -959 -51 56925 86 4294967348 2903746547904555720 true XHJC {"key":355,"value":"MQETE"} QIQA /�<3@C�� x*�� 1 14720674036350698175 -286 7 32408 89 4294967349 11768163425287102515 true JBWQ {"key":63,"value":"LBRXP"} IHTI 2���B=�f*���_� 1 18315169877216931174 -815 -16 37808 207 4294967396 18058293579070949154 true AEZA {"key":656,"value":"GOJYT"} ZODP e�W�E����>�� 1 1240809798326395103 -212 -74 50116 784 4294967316 4639886571322169794 true JZKC {"key":844,"value":"WYFIG"} KELP ��w+ʰO�����]��� 1 13188467212433715822 -510 -44 14672 697 4294967323 3203265463862679482 true QPXM {"key":408,"value":"BJCSQ"} NMGV x��|��A2�^q���m� 1 1158043015045771815 -827 38 52135 612 4294967349 2177159037635926390 false KWOV {"key":384,"value":"PLUUJ"} VACB >�$=L����$Q�\t 1 12363779168489547769 -705 -96 7506 479 4294967337 1919118169111152342 true EGEU {"key":522,"value":"AOPEL"} TZFW ����s�Iޣ�r^�3 1 10667084832660097342 -174 -38 10608 469 4294967369 8139895279544416045 false SUED {"key":348,"value":"KDIJQ"} VKSM ���v3�A����,��� 1 7867030734613018460 -89 79 31144 138 4294967379 3327168023216233174 true MRTX {"key":492,"value":"NGCXD"} GVCG y�>L�O(�xج;&�� 1 17782050321393165287 -802 92 55937 83 4294967368 13260877643210989136 true FNXX {"key":566,"value":"KVFDZ"} CMMV m:�u��L{�Tz��f� 1 7067095719366985926 -599 -107 36579 116 4294967311 4635153073869187643 true BURQ {"key":259,"value":"VNMQQ"} TMOJ �Yz�n�M<��~8O;i, 1 4175486705904763621 +73 116 32428 513 4294967384 10876946806756585544 false SGPI {"key":711, "value":"XMEED"} IOIX d\rOFЂn\b 1 11079779081220916748 +639 -69 45537 315 4294967315 7263280072686713102 false CHFC {"key":48, "value":"MFGFA"} ZLBS ]Ʇ̠Bʹ (LuբW 1 1884003582258998694 +959 -51 56925 86 4294967348 2903746547904555720 true XHJC {"key":355, "value":"MQETE"} QIQA /<3@C x* 1 12551571122189109470 +286 7 32408 89 4294967349 11768163425287102515 true JBWQ {"key":63, "value":"LBRXP"} IHTI 2B=f*_ 1 2293773524378365089 +815 -16 37808 207 4294967396 18058293579070949154 true AEZA {"key":656, "value":"GOJYT"} ZODP eWE> 1 9939955730329575199 +212 -74 50116 784 4294967316 4639886571322169794 true JZKC {"key":844, "value":"WYFIG"} KELP w+ʰO] 1 5464573773970257898 +510 -44 14672 697 4294967323 3203265463862679482 true QPXM {"key":408, "value":"BJCSQ"} NMGV x|A2^qm 1 18363658913862052503 +827 38 52135 612 4294967349 2177159037635926390 false KWOV {"key":384, "value":"PLUUJ"} VACB >$=L$Q\t 1 1291085690902008617 +705 -96 7506 479 4294967337 1919118169111152342 true EGEU {"key":522, "value":"AOPEL"} TZFW sIޣr^3 1 12776128745828051091 +174 -38 10608 469 4294967369 8139895279544416045 false SUED {"key":348, "value":"KDIJQ"} VKSM v3A, 1 8457376369144183832 +89 79 31144 138 4294967379 3327168023216233174 true MRTX {"key":492, "value":"NGCXD"} GVCG y>LO(xج;& 1 9669048967548569557 +802 92 55937 83 4294967368 13260877643210989136 true FNXX {"key":566, "value":"KVFDZ"} CMMV m:uL{Tzf 1 8531841042433805304 +599 -107 36579 116 4294967311 4635153073869187643 true BURQ {"key":259, "value":"VNMQQ"} TMOJ YznM<~&1 + SELECT _row_number, *, count(), sum(cityHash64(_row_number, *)) FROM file('$DATA_DIR/$NAME') GROUP BY all WITH TOTALS ORDER BY cityHash64(_row_number) LIMIT 20 SETTINGS input_format_parquet_enable_json_parsing=0, session_timezone='UTC';" 2>&1 done