Revert changes to IO/Parquet.hs

Author: jisantuc

dataframe-parquet/src/DataFrame/IO/Parquet.hs

@@ -401,21 +401,36 @@ getNonNullableColumn totalRows description chunks =
         PageDecoder a ->
         m Column
     go decoder =
-        foldNonNullable totalRows $
-            (\(vs, _, _) -> vs)
-                <$> Stream.unfoldMany (readPages description decoder) (Stream.fromList chunks)
+        foldNonNullable totalRows (foldColumnPagesM description decoder chunks)
+
+    -- Decode a non-nullable BYTE_ARRAY (UTF-8) column straight into a single
+    -- shared byte buffer + offsets ('PackedText'), instead of a boxed vector
+    -- of per-row 'Text'. Each page's decoded 'Text' values (which share the
+    -- chunk dictionary for dictionary-encoded pages) are appended by memcpy
+    -- into one builder across all pages/chunks, then frozen once. This is the
+    -- same representation the fast CSV reader uses and matches Arrow's string
+    -- layout: no retained per-row 'Text' headers, no eager UTF-8 validation.
+    goPackedText :: m Column
+    goPackedText = do
+        builder <- liftIO $ stToIO (newTextBuilder totalRows (totalRows * 8))
+        _ <-
+            foldColumnDataPagesM
+                description
+                chunks
+                ( \() (dict, enc, nPresent, valBytes, _, _) ->
+                    liftIO (appendStringPageIO builder dict enc nPresent valBytes)
+                )
+                ()
+        chunk <- liftIO $ stToIO (freezeTextChunk builder)
+        pure (mergeTextChunks [chunk])
 
     unboxedGo ::
         forall a.
         (Columnable a, VU.Unbox a) =>
         UnboxedPageDecoder a ->
         m Column
     unboxedGo decoder =
-        foldNonNullableUnboxed totalRows $
-            (\(vs, _, _) -> vs)
-                <$> Stream.unfoldMany
-                    (readPages description decoder)
-                    (Stream.fromList chunks)
+        foldNonNullableUnboxed totalRows (foldColumnPagesM description decoder chunks)
 
 -- | Decode an optional (nullable) column.
 {-# INLINEABLE getNullableColumn #-}
@@ -449,20 +464,36 @@ getNullableColumn totalRows description chunks =
         PageDecoder a ->
         m Column
     go decoder =
-        foldNullable maxDef totalRows $
-            (\(vs, ds, _) -> (vs, ds))
-                <$> Stream.unfoldMany (readPages description decoder) (Stream.fromList chunks)
+        foldNullable maxDef totalRows (foldColumnPagesM description decoder chunks)
+
+    -- Nullable BYTE_ARRAY (UTF-8): decode straight into a 'PackedText' (shared
+    -- byte buffer + offsets + validity bitmap) via the text builder, walking
+    -- def-levels to interleave nulls. Avoids the boxed @Vector Text@ the
+    -- generic 'foldNullable' path would build.
+    goPackedTextNullable :: m Column
+    goPackedTextNullable = do
+        builder <- liftIO $ stToIO (newTextBuilder totalRows (totalRows * 8))
+        _ <-
+            foldColumnDataPagesM
+                description
+                chunks
+                ( \() (dict, enc, nPresent, valBytes, defs, _) ->
+                    liftIO
+                        (appendNullableStringPageIO builder maxDef dict enc nPresent valBytes defs)
+                )
+                ()
+        chunk <- liftIO $ stToIO (freezeTextChunk builder)
+        pure (mergeTextChunks [chunk])
     unboxedGo ::
         forall a.
         (Columnable a, VU.Unbox a) =>
         UnboxedPageDecoder a ->
         m Column
     unboxedGo decoder =
-        foldNullableUnboxed maxDef totalRows $
-            (\(vs, ds, _) -> (vs, ds))
-                <$> Stream.unfoldMany
-                    (readPages description decoder)
-                    (Stream.fromList chunks)
+        foldNullableUnboxed
+            maxDef
+            totalRows
+            (foldColumnPagesM description decoder chunks)
 
 -- | Decode a repeated (list/nested) column.
 {-# INLINEABLE getRepeatedColumn #-}
@@ -501,8 +532,7 @@ getRepeatedColumn description chunks =
         PageDecoder a ->
         m Column
     go decoder =
-        foldRepeated maxRep maxDef $
-            Stream.unfoldMany (readPages description decoder) (Stream.fromList chunks)
+        foldRepeated maxRep maxDef (foldColumnPagesM description decoder chunks)
 
     unboxedGo ::
         forall a.
@@ -515,10 +545,7 @@ getRepeatedColumn description chunks =
         UnboxedPageDecoder a ->
         m Column
     unboxedGo decoder =
-        foldRepeatedUnboxed maxRep maxDef $
-            Stream.unfoldMany
-                (readPages description decoder)
-                (Stream.fromList chunks)
+        foldRepeatedUnboxed maxRep maxDef (foldColumnPagesM description decoder chunks)
 
 -- Options application -----------------------------------------------------