From 12b921e70c0fcefc7b0346eb1b107e6ffd3be30e Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 1 Jul 2026 16:29:56 -0700 Subject: [PATCH 1/7] GH-50326: [Python] Speed up to_pylist for list-like and string arrays Array.to_pylist() converts one element at a time: each row allocates a C++ Scalar (Array::GetScalar), a Python Scalar wrapper and, for list types, a Python Array wrapper for the row's values slice plus a fresh generator, before recursing per element. On top of the allocation cost itself, these GC-tracked wrappers repeatedly trigger collections that traverse the growing result list (~20% of runtime). This makes to_pylist on list-typed arrays several times slower than the bulk to_pandas conversion path. Add bulk to_pylist overrides: * ListArray / LargeListArray / FixedSizeListArray convert the referenced range of child values with a single recursive to_pylist call, then slice the resulting Python list per row using the raw offsets and the validity bitmap. MapArray keeps the generic scalar-based path (association-tuple / maps_as_pydicts duplicate-key semantics), as do the list-view types (overlapping views must not share sublist objects). * StringArray / LargeStringArray decode values directly from the data buffer (GetValue + PyUnicode_DecodeUTF8), matching StringScalar.as_py (= str(buf, 'utf8')) exactly. Semantics are unchanged; values inside numeric lists stay Python ints/None. Benchmarks (M4 Max, 2M rows of 2-element lists / 1M rows nested): list 1.93s -> 0.34s, list> 2.10s -> 0.65s, flat string (4M) 0.83s -> 0.05s. Co-authored-by: Isaac --- python/pyarrow/array.pxi | 258 +++++++++++++++++++++++++++++ python/pyarrow/tests/test_array.py | 32 ++++ 2 files changed, 290 insertions(+) diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 5fc74969abfd..0517f51fe56e 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -2776,6 +2776,59 @@ cdef class ListArray(BaseListArray): Concrete class for Arrow arrays of a list data type. """ + def to_pylist(self, *, maps_as_pydicts=None): + """ + Convert to a list of native Python objects. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + The default behavior (`None`), is to convert Arrow Map arrays to + Python association lists (list-of-tuples) in the same order as the + Arrow Map, as in [(key1, value1), (key2, value2), ...]. + + If 'lossy' or 'strict', convert Arrow Map arrays to native Python dicts. + + If 'lossy', whenever duplicate keys are detected, a warning will be printed. + The last seen value of a duplicate key will be in the Python dictionary. + If 'strict', this instead results in an exception being raised when detected. + + Returns + ------- + lst : list + """ + cdef: + CListArray* arr = self.ap + int64_t i, n, off0, start, end + self._assert_cpu() + n = arr.length() + if n == 0: + return [] + # Convert the range of child values referenced by this array in a + # single pass, then slice out each list. This avoids creating a + # Scalar wrapper, a Python Array wrapper and a values-array slice + # for every row (see GH-28694). + off0 = arr.value_offset(0) + child_py = pyarrow_wrap_array(arr.values()).slice( + off0, arr.value_offset(n) - off0 + ).to_pylist(maps_as_pydicts=maps_as_pydicts) + result = [] + if arr.null_count() == 0: + for i in range(n): + start = arr.value_offset(i) - off0 + end = arr.value_offset(i + 1) - off0 + result.append(child_py[start:end]) + else: + for i in range(n): + if arr.IsNull(i): + result.append(None) + else: + start = arr.value_offset(i) - off0 + end = arr.value_offset(i + 1) - off0 + result.append(child_py[start:end]) + return result + @staticmethod def from_arrays(offsets, values, DataType type=None, MemoryPool pool=None, mask=None): """ @@ -2961,6 +3014,56 @@ cdef class LargeListArray(BaseListArray): Identical to ListArray, but 64-bit offsets. """ + def to_pylist(self, *, maps_as_pydicts=None): + """ + Convert to a list of native Python objects. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + The default behavior (`None`), is to convert Arrow Map arrays to + Python association lists (list-of-tuples) in the same order as the + Arrow Map, as in [(key1, value1), (key2, value2), ...]. + + If 'lossy' or 'strict', convert Arrow Map arrays to native Python dicts. + + If 'lossy', whenever duplicate keys are detected, a warning will be printed. + The last seen value of a duplicate key will be in the Python dictionary. + If 'strict', this instead results in an exception being raised when detected. + + Returns + ------- + lst : list + """ + cdef: + CLargeListArray* arr = self.ap + int64_t i, n, off0, start, end + self._assert_cpu() + n = arr.length() + if n == 0: + return [] + # See ListArray.to_pylist for an explanation of the bulk conversion. + off0 = arr.value_offset(0) + child_py = pyarrow_wrap_array(arr.values()).slice( + off0, arr.value_offset(n) - off0 + ).to_pylist(maps_as_pydicts=maps_as_pydicts) + result = [] + if arr.null_count() == 0: + for i in range(n): + start = arr.value_offset(i) - off0 + end = arr.value_offset(i + 1) - off0 + result.append(child_py[start:end]) + else: + for i in range(n): + if arr.IsNull(i): + result.append(None) + else: + start = arr.value_offset(i) - off0 + end = arr.value_offset(i + 1) - off0 + result.append(child_py[start:end]) + return result + @staticmethod def from_arrays(offsets, values, DataType type=None, MemoryPool pool=None, mask=None): """ @@ -3551,6 +3654,34 @@ cdef class MapArray(ListArray): Concrete class for Arrow arrays of a map data type. """ + def to_pylist(self, *, maps_as_pydicts=None): + """ + Convert to a list of native Python objects. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + The default behavior (`None`), is to convert Arrow Map arrays to + Python association lists (list-of-tuples) in the same order as the + Arrow Map, as in [(key1, value1), (key2, value2), ...]. + + If 'lossy' or 'strict', convert Arrow Map arrays to native Python dicts. + + If 'lossy', whenever duplicate keys are detected, a warning will be printed. + The last seen value of a duplicate key will be in the Python dictionary. + If 'strict', this instead results in an exception being raised when detected. + + Returns + ------- + lst : list + """ + # Maps have per-entry key/value semantics (association tuples, + # optional dict conversion with duplicate-key detection) that the + # bulk path inherited from ListArray does not implement, so use + # the generic scalar-based conversion. + return Array.to_pylist(self, maps_as_pydicts=maps_as_pydicts) + @staticmethod def from_arrays(offsets, keys, items, DataType type=None, MemoryPool pool=None, mask=None): """ @@ -3688,6 +3819,56 @@ cdef class FixedSizeListArray(BaseListArray): Concrete class for Arrow arrays of a fixed size list data type. """ + def to_pylist(self, *, maps_as_pydicts=None): + """ + Convert to a list of native Python objects. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + The default behavior (`None`), is to convert Arrow Map arrays to + Python association lists (list-of-tuples) in the same order as the + Arrow Map, as in [(key1, value1), (key2, value2), ...]. + + If 'lossy' or 'strict', convert Arrow Map arrays to native Python dicts. + + If 'lossy', whenever duplicate keys are detected, a warning will be printed. + The last seen value of a duplicate key will be in the Python dictionary. + If 'strict', this instead results in an exception being raised when detected. + + Returns + ------- + lst : list + """ + cdef: + CFixedSizeListArray* arr = self.ap + int64_t i, n, off0, start, end + self._assert_cpu() + n = arr.length() + if n == 0: + return [] + # See ListArray.to_pylist for an explanation of the bulk conversion. + off0 = arr.value_offset(0) + child_py = pyarrow_wrap_array(arr.values()).slice( + off0, arr.value_offset(n) - off0 + ).to_pylist(maps_as_pydicts=maps_as_pydicts) + result = [] + if arr.null_count() == 0: + for i in range(n): + start = arr.value_offset(i) - off0 + end = arr.value_offset(i + 1) - off0 + result.append(child_py[start:end]) + else: + for i in range(n): + if arr.IsNull(i): + result.append(None) + else: + start = arr.value_offset(i) - off0 + end = arr.value_offset(i + 1) - off0 + result.append(child_py[start:end]) + return result + @staticmethod def from_arrays(values, list_size=None, DataType type=None, mask=None): """ @@ -3974,6 +4155,45 @@ cdef class StringArray(Array): Concrete class for Arrow arrays of string (or utf8) data type. """ + def to_pylist(self, *, maps_as_pydicts=None): + """ + Convert to a list of native Python objects. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Arrays. + + Returns + ------- + lst : list + """ + cdef: + CStringArray* arr = self.ap + int64_t i, n + int32_t length + const uint8_t* data + self._assert_cpu() + n = arr.length() + result = [] + # Decode values straight from the data buffer instead of creating + # a C++ Scalar and a Python Scalar wrapper per value (see GH-28694). + if arr.null_count() == 0: + for i in range(n): + data = arr.GetValue(i, &length) + result.append( + cp.PyUnicode_DecodeUTF8( data, length, NULL)) + else: + for i in range(n): + if arr.IsNull(i): + result.append(None) + else: + data = arr.GetValue(i, &length) + result.append( + cp.PyUnicode_DecodeUTF8( data, length, NULL)) + return result + @staticmethod def from_buffers(int length, Buffer value_offsets, Buffer data, Buffer null_bitmap=None, int null_count=-1, @@ -4006,6 +4226,44 @@ cdef class LargeStringArray(Array): Concrete class for Arrow arrays of large string (or utf8) data type. """ + def to_pylist(self, *, maps_as_pydicts=None): + """ + Convert to a list of native Python objects. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Arrays. + + Returns + ------- + lst : list + """ + cdef: + CLargeStringArray* arr = self.ap + int64_t i, n + int64_t length + const uint8_t* data + self._assert_cpu() + n = arr.length() + result = [] + # See StringArray.to_pylist for an explanation of the fast path. + if arr.null_count() == 0: + for i in range(n): + data = arr.GetValue(i, &length) + result.append( + cp.PyUnicode_DecodeUTF8( data, length, NULL)) + else: + for i in range(n): + if arr.IsNull(i): + result.append(None) + else: + data = arr.GetValue(i, &length) + result.append( + cp.PyUnicode_DecodeUTF8( data, length, NULL)) + return result + @staticmethod def from_buffers(int length, Buffer value_offsets, Buffer data, Buffer null_bitmap=None, int null_count=-1, diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index adc3e097b54a..ce1dcdee6b1d 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -465,6 +465,38 @@ def test_array_getitem_numpy_scalars(): assert arr[np.int32(idx)].as_py() == lst[idx] +def test_to_pylist_bulk_paths(): + # GH-50326: list-like and string arrays convert to Python objects in + # bulk instead of going through one Scalar per element; the result must + # match the per-scalar conversion exactly. + arrays = [ + pa.array([[1, None, 3], None, [], [4]], type=pa.list_(pa.int32())), + pa.array([["a", None], None, [], ["bcd", ""]], + type=pa.list_(pa.string())), + pa.array([["a", None], None, [], ["bcd", ""]], + type=pa.large_list(pa.large_string())), + pa.array([[1, None], None, [3, 4]], type=pa.list_(pa.int32(), 2)), + pa.array([[[1], [2, None]], None, [None, [3]]], + type=pa.list_(pa.list_(pa.int32()))), + pa.array([[("k1", 1), ("k2", None)], None, []], + type=pa.map_(pa.string(), pa.int32())), + pa.array(["a", None, "", "\N{GRINNING FACE} \N{SNOWMAN}"], + type=pa.string()), + pa.array(["a", None, "", "\N{GRINNING FACE} \N{SNOWMAN}"], + type=pa.large_string()), + pa.array([], type=pa.list_(pa.int32())), + pa.array([None, None], type=pa.list_(pa.string())), + ] + for arr in arrays: + for view in (arr, arr.slice(1), arr.slice(0, 2), arr.slice(2)): + assert view.to_pylist() == [x.as_py() for x in view] + + # Values inside numeric lists must stay Python ints/None, never floats + result = pa.array([[1, None, 3]], type=pa.list_(pa.int32())).to_pylist() + assert result == [[1, None, 3]] + assert [type(x) for x in result[0]] == [int, type(None), int] + + def test_array_slice(): arr = pa.array(range(10)) From 7a150778912d8899e83d5d67db62a0c91c6aa4d6 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 8 Jul 2026 10:47:28 -0700 Subject: [PATCH 2/7] GH-50326: [Python] Rework as a scalar-free _getitem_py used by to_pylist Per review feedback, replace the per-type to_pylist overrides with a general mechanism: * Array gains a cdef _getitem_py(i) returning self[i] as a Python object. The base implementation is GetScalar + Scalar.as_py, so any type without a specialization behaves exactly as before. * The baseline Array.to_pylist becomes a single loop over _getitem_py. maps_as_pydicts != None keeps the Scalar-based path (map->dict conversion has per-entry duplicate-key semantics). * Specializations avoid all per-element Scalar and per-row Array wrapper allocation: integers/floats (type_id switch on NumericArray; dates/times/timestamps fall through to the exact base), boolean, string/binary (+ large variants), list/large_list/fixed_size_list (per-row list built from the child's _getitem_py over the offset range, child wrapper cached on the array), map (list of key/value tuples), struct (dict per row; duplicate field names fall back so they raise ValueError like StructScalar.as_py). Benchmarks (M4 Max): flat int64 with nulls 4M ~0.39s -> 0.028s (~7ns per element, on par with ndarray.tolist); flat string 4M 0.83s -> 0.06s; list 2M 1.93s -> 0.46s; list> 1M 2.10s -> 0.40s; struct 1M 0.91s -> 0.07s; map 1M 2.77s -> 0.74s. Co-authored-by: Isaac --- python/pyarrow/array.pxi | 387 ++++++++++++++------------------------- python/pyarrow/lib.pxd | 3 + 2 files changed, 143 insertions(+), 247 deletions(-) diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 0517f51fe56e..2388d36be160 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -1864,7 +1864,19 @@ cdef class Array(_PandasConvertible): lst : list """ self._assert_cpu() - return [x.as_py(maps_as_pydicts=maps_as_pydicts) for x in self] + cdef int64_t i, n = self.length() + if maps_as_pydicts is not None: + # Converting maps to dicts has per-entry semantics (duplicate-key + # detection); use the Scalar-based conversion for exact behavior. + return [x.as_py(maps_as_pydicts=maps_as_pydicts) for x in self] + return [self._getitem_py(i) for i in range(n)] + + cdef object _getitem_py(self, int64_t i): + # Return self[i] as a Python object, without creating a Python Scalar + # (nor, for nested types, per-row Array wrappers) where a subclass + # provides a specialization; this base implementation goes through + # Scalar.as_py and thus preserves its semantics exactly (see GH-50326). + return self.getitem(i).as_py() def tolist(self): """ @@ -2444,6 +2456,11 @@ cdef class BooleanArray(Array): """ Concrete class for Arrow arrays of boolean data type. """ + + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + return ( self.ap).Value(i) @property def false_count(self): return ( self.ap).false_count() @@ -2458,6 +2475,34 @@ cdef class NumericArray(Array): A base class for Arrow numeric arrays. """ + cdef object _getitem_py(self, int64_t i): + cdef Type tid = self.ap.type_id() + if self.ap.IsNull(i): + return None + if tid == _Type_INT64: + return ( self.ap).Value(i) + elif tid == _Type_INT32: + return ( self.ap).Value(i) + elif tid == _Type_DOUBLE: + return ( self.ap).Value(i) + elif tid == _Type_FLOAT: + return ( self.ap).Value(i) + elif tid == _Type_INT16: + return ( self.ap).Value(i) + elif tid == _Type_INT8: + return ( self.ap).Value(i) + elif tid == _Type_UINT64: + return ( self.ap).Value(i) + elif tid == _Type_UINT32: + return ( self.ap).Value(i) + elif tid == _Type_UINT16: + return ( self.ap).Value(i) + elif tid == _Type_UINT8: + return ( self.ap).Value(i) + # Subclasses whose as_py returns non-primitive objects (dates, times, + # timestamps, durations, half floats, ...) use the exact Scalar path. + return Array._getitem_py(self, i) + cdef class IntegerArray(NumericArray): """ @@ -2776,58 +2821,15 @@ cdef class ListArray(BaseListArray): Concrete class for Arrow arrays of a list data type. """ - def to_pylist(self, *, maps_as_pydicts=None): - """ - Convert to a list of native Python objects. - - Parameters - ---------- - maps_as_pydicts : str, optional, default `None` - Valid values are `None`, 'lossy', or 'strict'. - The default behavior (`None`), is to convert Arrow Map arrays to - Python association lists (list-of-tuples) in the same order as the - Arrow Map, as in [(key1, value1), (key2, value2), ...]. - - If 'lossy' or 'strict', convert Arrow Map arrays to native Python dicts. - - If 'lossy', whenever duplicate keys are detected, a warning will be printed. - The last seen value of a duplicate key will be in the Python dictionary. - If 'strict', this instead results in an exception being raised when detected. - - Returns - ------- - lst : list - """ - cdef: - CListArray* arr = self.ap - int64_t i, n, off0, start, end - self._assert_cpu() - n = arr.length() - if n == 0: - return [] - # Convert the range of child values referenced by this array in a - # single pass, then slice out each list. This avoids creating a - # Scalar wrapper, a Python Array wrapper and a values-array slice - # for every row (see GH-28694). - off0 = arr.value_offset(0) - child_py = pyarrow_wrap_array(arr.values()).slice( - off0, arr.value_offset(n) - off0 - ).to_pylist(maps_as_pydicts=maps_as_pydicts) - result = [] - if arr.null_count() == 0: - for i in range(n): - start = arr.value_offset(i) - off0 - end = arr.value_offset(i + 1) - off0 - result.append(child_py[start:end]) - else: - for i in range(n): - if arr.IsNull(i): - result.append(None) - else: - start = arr.value_offset(i) - off0 - end = arr.value_offset(i + 1) - off0 - result.append(child_py[start:end]) - return result + cdef object _getitem_py(self, int64_t i): + cdef CListArray* arr = self.ap + if arr.IsNull(i): + return None + if self._children_cache is None: + self._children_cache = pyarrow_wrap_array(arr.values()) + cdef Array values = self._children_cache + cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) + return [values._getitem_py(j) for j in range(start, end)] @staticmethod def from_arrays(offsets, values, DataType type=None, MemoryPool pool=None, mask=None): @@ -3014,55 +3016,15 @@ cdef class LargeListArray(BaseListArray): Identical to ListArray, but 64-bit offsets. """ - def to_pylist(self, *, maps_as_pydicts=None): - """ - Convert to a list of native Python objects. - - Parameters - ---------- - maps_as_pydicts : str, optional, default `None` - Valid values are `None`, 'lossy', or 'strict'. - The default behavior (`None`), is to convert Arrow Map arrays to - Python association lists (list-of-tuples) in the same order as the - Arrow Map, as in [(key1, value1), (key2, value2), ...]. - - If 'lossy' or 'strict', convert Arrow Map arrays to native Python dicts. - - If 'lossy', whenever duplicate keys are detected, a warning will be printed. - The last seen value of a duplicate key will be in the Python dictionary. - If 'strict', this instead results in an exception being raised when detected. - - Returns - ------- - lst : list - """ - cdef: - CLargeListArray* arr = self.ap - int64_t i, n, off0, start, end - self._assert_cpu() - n = arr.length() - if n == 0: - return [] - # See ListArray.to_pylist for an explanation of the bulk conversion. - off0 = arr.value_offset(0) - child_py = pyarrow_wrap_array(arr.values()).slice( - off0, arr.value_offset(n) - off0 - ).to_pylist(maps_as_pydicts=maps_as_pydicts) - result = [] - if arr.null_count() == 0: - for i in range(n): - start = arr.value_offset(i) - off0 - end = arr.value_offset(i + 1) - off0 - result.append(child_py[start:end]) - else: - for i in range(n): - if arr.IsNull(i): - result.append(None) - else: - start = arr.value_offset(i) - off0 - end = arr.value_offset(i + 1) - off0 - result.append(child_py[start:end]) - return result + cdef object _getitem_py(self, int64_t i): + cdef CLargeListArray* arr = self.ap + if arr.IsNull(i): + return None + if self._children_cache is None: + self._children_cache = pyarrow_wrap_array(arr.values()) + cdef Array values = self._children_cache + cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) + return [values._getitem_py(j) for j in range(start, end)] @staticmethod def from_arrays(offsets, values, DataType type=None, MemoryPool pool=None, mask=None): @@ -3654,33 +3616,18 @@ cdef class MapArray(ListArray): Concrete class for Arrow arrays of a map data type. """ - def to_pylist(self, *, maps_as_pydicts=None): - """ - Convert to a list of native Python objects. - - Parameters - ---------- - maps_as_pydicts : str, optional, default `None` - Valid values are `None`, 'lossy', or 'strict'. - The default behavior (`None`), is to convert Arrow Map arrays to - Python association lists (list-of-tuples) in the same order as the - Arrow Map, as in [(key1, value1), (key2, value2), ...]. - - If 'lossy' or 'strict', convert Arrow Map arrays to native Python dicts. - - If 'lossy', whenever duplicate keys are detected, a warning will be printed. - The last seen value of a duplicate key will be in the Python dictionary. - If 'strict', this instead results in an exception being raised when detected. - - Returns - ------- - lst : list - """ - # Maps have per-entry key/value semantics (association tuples, - # optional dict conversion with duplicate-key detection) that the - # bulk path inherited from ListArray does not implement, so use - # the generic scalar-based conversion. - return Array.to_pylist(self, maps_as_pydicts=maps_as_pydicts) + cdef object _getitem_py(self, int64_t i): + cdef CListArray* arr = self.ap + if arr.IsNull(i): + return None + if self._children_cache is None: + self._children_cache = (self.keys, self.items) + cdef Array keys = ( self._children_cache)[0] + cdef Array items = ( self._children_cache)[1] + cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) + # Matches MapScalar.as_py with the default maps_as_pydicts=None: + # an association list of (key, value) tuples. + return [(keys._getitem_py(j), items._getitem_py(j)) for j in range(start, end)] @staticmethod def from_arrays(offsets, keys, items, DataType type=None, MemoryPool pool=None, mask=None): @@ -3819,55 +3766,15 @@ cdef class FixedSizeListArray(BaseListArray): Concrete class for Arrow arrays of a fixed size list data type. """ - def to_pylist(self, *, maps_as_pydicts=None): - """ - Convert to a list of native Python objects. - - Parameters - ---------- - maps_as_pydicts : str, optional, default `None` - Valid values are `None`, 'lossy', or 'strict'. - The default behavior (`None`), is to convert Arrow Map arrays to - Python association lists (list-of-tuples) in the same order as the - Arrow Map, as in [(key1, value1), (key2, value2), ...]. - - If 'lossy' or 'strict', convert Arrow Map arrays to native Python dicts. - - If 'lossy', whenever duplicate keys are detected, a warning will be printed. - The last seen value of a duplicate key will be in the Python dictionary. - If 'strict', this instead results in an exception being raised when detected. - - Returns - ------- - lst : list - """ - cdef: - CFixedSizeListArray* arr = self.ap - int64_t i, n, off0, start, end - self._assert_cpu() - n = arr.length() - if n == 0: - return [] - # See ListArray.to_pylist for an explanation of the bulk conversion. - off0 = arr.value_offset(0) - child_py = pyarrow_wrap_array(arr.values()).slice( - off0, arr.value_offset(n) - off0 - ).to_pylist(maps_as_pydicts=maps_as_pydicts) - result = [] - if arr.null_count() == 0: - for i in range(n): - start = arr.value_offset(i) - off0 - end = arr.value_offset(i + 1) - off0 - result.append(child_py[start:end]) - else: - for i in range(n): - if arr.IsNull(i): - result.append(None) - else: - start = arr.value_offset(i) - off0 - end = arr.value_offset(i + 1) - off0 - result.append(child_py[start:end]) - return result + cdef object _getitem_py(self, int64_t i): + cdef CFixedSizeListArray* arr = self.ap + if arr.IsNull(i): + return None + if self._children_cache is None: + self._children_cache = pyarrow_wrap_array(arr.values()) + cdef Array values = self._children_cache + cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) + return [values._getitem_py(j) for j in range(start, end)] @staticmethod def from_arrays(values, list_size=None, DataType type=None, mask=None): @@ -4155,44 +4062,15 @@ cdef class StringArray(Array): Concrete class for Arrow arrays of string (or utf8) data type. """ - def to_pylist(self, *, maps_as_pydicts=None): - """ - Convert to a list of native Python objects. - - Parameters - ---------- - maps_as_pydicts : str, optional, default `None` - Valid values are `None`, 'lossy', or 'strict'. - This parameter is ignored for non-nested Arrays. - - Returns - ------- - lst : list - """ + cdef object _getitem_py(self, int64_t i): cdef: - CStringArray* arr = self.ap - int64_t i, n int32_t length const uint8_t* data - self._assert_cpu() - n = arr.length() - result = [] - # Decode values straight from the data buffer instead of creating - # a C++ Scalar and a Python Scalar wrapper per value (see GH-28694). - if arr.null_count() == 0: - for i in range(n): - data = arr.GetValue(i, &length) - result.append( - cp.PyUnicode_DecodeUTF8( data, length, NULL)) - else: - for i in range(n): - if arr.IsNull(i): - result.append(None) - else: - data = arr.GetValue(i, &length) - result.append( - cp.PyUnicode_DecodeUTF8( data, length, NULL)) - return result + if self.ap.IsNull(i): + return None + data = ( self.ap).GetValue(i, &length) + # Matches StringScalar.as_py, which is str(buf, 'utf8'). + return cp.PyUnicode_DecodeUTF8( data, length, NULL) @staticmethod def from_buffers(int length, Buffer value_offsets, Buffer data, @@ -4226,43 +4104,14 @@ cdef class LargeStringArray(Array): Concrete class for Arrow arrays of large string (or utf8) data type. """ - def to_pylist(self, *, maps_as_pydicts=None): - """ - Convert to a list of native Python objects. - - Parameters - ---------- - maps_as_pydicts : str, optional, default `None` - Valid values are `None`, 'lossy', or 'strict'. - This parameter is ignored for non-nested Arrays. - - Returns - ------- - lst : list - """ + cdef object _getitem_py(self, int64_t i): cdef: - CLargeStringArray* arr = self.ap - int64_t i, n int64_t length const uint8_t* data - self._assert_cpu() - n = arr.length() - result = [] - # See StringArray.to_pylist for an explanation of the fast path. - if arr.null_count() == 0: - for i in range(n): - data = arr.GetValue(i, &length) - result.append( - cp.PyUnicode_DecodeUTF8( data, length, NULL)) - else: - for i in range(n): - if arr.IsNull(i): - result.append(None) - else: - data = arr.GetValue(i, &length) - result.append( - cp.PyUnicode_DecodeUTF8( data, length, NULL)) - return result + if self.ap.IsNull(i): + return None + data = ( self.ap).GetValue(i, &length) + return cp.PyUnicode_DecodeUTF8( data, length, NULL) @staticmethod def from_buffers(int length, Buffer value_offsets, Buffer data, @@ -4301,6 +4150,16 @@ cdef class BinaryArray(Array): """ Concrete class for Arrow arrays of variable-sized binary data type. """ + + cdef object _getitem_py(self, int64_t i): + cdef: + int32_t length + const uint8_t* data + if self.ap.IsNull(i): + return None + data = ( self.ap).GetValue(i, &length) + return cp.PyBytes_FromStringAndSize( data, length) + @property def total_values_length(self): """ @@ -4314,6 +4173,16 @@ cdef class LargeBinaryArray(Array): """ Concrete class for Arrow arrays of large variable-sized binary data type. """ + + cdef object _getitem_py(self, int64_t i): + cdef: + int64_t length + const uint8_t* data + if self.ap.IsNull(i): + return None + data = ( self.ap).GetValue(i, &length) + return cp.PyBytes_FromStringAndSize( data, length) + @property def total_values_length(self): """ @@ -4487,6 +4356,30 @@ cdef class StructArray(Array): Concrete class for Arrow arrays of a struct data type. """ + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef int64_t k, num_fields = self.type.num_fields + if self._children_cache is None: + names = [self.type.field(k).name for k in range(num_fields)] + if len(set(names)) != len(names): + # StructScalar.as_py raises ValueError on duplicate field + # names; mark the cache so we take the Scalar path below. + self._children_cache = (None, None) + else: + self._children_cache = ( + names, [self.field(k) for k in range(num_fields)]) + names = ( self._children_cache)[0] + if names is None: + return Array._getitem_py(self, i) + fields = ( self._children_cache)[1] + cdef Array field_arr + result = {} + for k in range(num_fields): + field_arr = fields[k] + result[names[k]] = field_arr._getitem_py(i) + return result + def field(self, index): """ Retrieves the child array belonging to field. diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd index 683faa7855c5..50a1ab58ef66 100644 --- a/python/pyarrow/lib.pxd +++ b/python/pyarrow/lib.pxd @@ -282,6 +282,8 @@ cdef class Array(_PandasConvertible): cdef: shared_ptr[CArray] sp_array CArray* ap + # Lazily wrapped child array(s) reused by _getitem_py (see GH-50326) + object _children_cache cdef readonly: DataType type @@ -290,6 +292,7 @@ cdef class Array(_PandasConvertible): cdef void init(self, const shared_ptr[CArray]& sp_array) except * cdef getitem(self, int64_t i) + cdef object _getitem_py(self, int64_t i) cdef int64_t length(self) cdef void _assert_cpu(self) except * From 985a2e3feaeb0a17d58450c42aad4554befa8021 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 8 Jul 2026 10:58:44 -0700 Subject: [PATCH 3/7] GH-50326: [Python] Fix autopep8 lint (missing blank line) Co-authored-by: Isaac --- python/pyarrow/array.pxi | 1 + 1 file changed, 1 insertion(+) diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 2388d36be160..ed6db497e0f2 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -2461,6 +2461,7 @@ cdef class BooleanArray(Array): if self.ap.IsNull(i): return None return ( self.ap).Value(i) + @property def false_count(self): return ( self.ap).false_count() From 728584f8e69309a81041478ed14e314d5cfc015d Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 8 Jul 2026 11:37:20 -0700 Subject: [PATCH 4/7] GH-50326: [Python] Address review: append new attribute, extend tests Move the _children_cache declaration after the pre-existing attributes so their offsets stay stable for extensions compiled against an older pyarrow, and extend test_to_pylist_bulk_paths with binary/large_binary (including embedded NUL bytes), list, wide-range integers, floats, boolean and struct coverage plus a duplicate-field-name ValueError assertion. Co-authored-by: Isaac --- python/pyarrow/lib.pxd | 8 ++++++-- python/pyarrow/tests/test_array.py | 24 +++++++++++++++++++++--- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd index 50a1ab58ef66..38f1ac69a807 100644 --- a/python/pyarrow/lib.pxd +++ b/python/pyarrow/lib.pxd @@ -282,14 +282,18 @@ cdef class Array(_PandasConvertible): cdef: shared_ptr[CArray] sp_array CArray* ap - # Lazily wrapped child array(s) reused by _getitem_py (see GH-50326) - object _children_cache cdef readonly: DataType type # To allow Table to propagate metadata to pandas.Series object _name + cdef: + # Lazily wrapped child array(s) reused by _getitem_py (see GH-50326). + # Appended after the pre-existing attributes to keep their offsets + # stable for extensions compiled against an older pyarrow. + object _children_cache + cdef void init(self, const shared_ptr[CArray]& sp_array) except * cdef getitem(self, int64_t i) cdef object _getitem_py(self, int64_t i) diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index ce1dcdee6b1d..6331409cdb8f 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -466,9 +466,9 @@ def test_array_getitem_numpy_scalars(): def test_to_pylist_bulk_paths(): - # GH-50326: list-like and string arrays convert to Python objects in - # bulk instead of going through one Scalar per element; the result must - # match the per-scalar conversion exactly. + # GH-50326: to_pylist converts through scalar-free _getitem_py + # specializations; the result must match the per-scalar conversion + # exactly. arrays = [ pa.array([[1, None, 3], None, [], [4]], type=pa.list_(pa.int32())), pa.array([["a", None], None, [], ["bcd", ""]], @@ -484,6 +484,18 @@ def test_to_pylist_bulk_paths(): type=pa.string()), pa.array(["a", None, "", "\N{GRINNING FACE} \N{SNOWMAN}"], type=pa.large_string()), + pa.array([b"a\x00b", None, b"", b"\xff"], type=pa.binary()), + pa.array([b"a\x00b", None, b""], type=pa.large_binary()), + pa.array([[b"x", None, b"\x00y"], None, []], + type=pa.list_(pa.binary())), + pa.array([1, None, -(2**62), 2**62], type=pa.int64()), + pa.array([0, None, 2**63 + 7], type=pa.uint64()), + pa.array([-128, 127, None], type=pa.int8()), + pa.array([1.5, None, -0.5], type=pa.float64()), + pa.array([1.5, None], type=pa.float32()), + pa.array([True, None, False], type=pa.bool_()), + pa.array([{"a": 1, "b": "x"}, None, {"a": None, "b": None}], + type=pa.struct([("a", pa.int32()), ("b", pa.string())])), pa.array([], type=pa.list_(pa.int32())), pa.array([None, None], type=pa.list_(pa.string())), ] @@ -496,6 +508,12 @@ def test_to_pylist_bulk_paths(): assert result == [[1, None, 3]] assert [type(x) for x in result[0]] == [int, type(None), int] + # Duplicate struct field names raise like StructScalar.as_py does + dup = pa.StructArray.from_arrays( + [pa.array([1, 2]), pa.array(["a", "b"])], names=["x", "x"]) + with pytest.raises(ValueError): + dup.to_pylist() + def test_array_slice(): arr = pa.array(range(10)) From 3d303ce48780d55660c8279184b5bc65c9ac6687 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 9 Jul 2026 07:47:39 -0700 Subject: [PATCH 5/7] GH-50326: [Python] Address review comments * Order the numeric type-id dispatch regularly (int8..uint64, float, double) instead of by expected hotness. * Use GetView(i) (std::string_view) instead of GetValue with an out parameter for string/binary values, and add StringViewArray / BinaryViewArray specializations on top of it. * Raise the duplicate-field-names ValueError directly in StructArray._getitem_py instead of falling back to the Scalar path, and assert the message in the test. * Add binary_view/string_view test coverage. * Add a TODO pointing at GH-50448 (per-range conversion follow-up). Co-authored-by: Isaac --- python/pyarrow/array.pxi | 88 ++++++++++++++-------------- python/pyarrow/includes/libarrow.pxd | 8 +++ 2 files changed, 52 insertions(+), 44 deletions(-) diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index ed6db497e0f2..71c18fbdfe44 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -1869,6 +1869,8 @@ cdef class Array(_PandasConvertible): # Converting maps to dicts has per-entry semantics (duplicate-key # detection); use the Scalar-based conversion for exact behavior. return [x.as_py(maps_as_pydicts=maps_as_pydicts) for x in self] + # TODO(GH-50448): convert per range instead of per element to cut + # the per-element call overhead further. return [self._getitem_py(i) for i in range(n)] cdef object _getitem_py(self, int64_t i): @@ -2480,26 +2482,26 @@ cdef class NumericArray(Array): cdef Type tid = self.ap.type_id() if self.ap.IsNull(i): return None - if tid == _Type_INT64: - return ( self.ap).Value(i) - elif tid == _Type_INT32: - return ( self.ap).Value(i) - elif tid == _Type_DOUBLE: - return ( self.ap).Value(i) - elif tid == _Type_FLOAT: - return ( self.ap).Value(i) + if tid == _Type_INT8: + return ( self.ap).Value(i) elif tid == _Type_INT16: return ( self.ap).Value(i) - elif tid == _Type_INT8: - return ( self.ap).Value(i) - elif tid == _Type_UINT64: - return ( self.ap).Value(i) - elif tid == _Type_UINT32: - return ( self.ap).Value(i) - elif tid == _Type_UINT16: - return ( self.ap).Value(i) + elif tid == _Type_INT32: + return ( self.ap).Value(i) + elif tid == _Type_INT64: + return ( self.ap).Value(i) elif tid == _Type_UINT8: return ( self.ap).Value(i) + elif tid == _Type_UINT16: + return ( self.ap).Value(i) + elif tid == _Type_UINT32: + return ( self.ap).Value(i) + elif tid == _Type_UINT64: + return ( self.ap).Value(i) + elif tid == _Type_FLOAT: + return ( self.ap).Value(i) + elif tid == _Type_DOUBLE: + return ( self.ap).Value(i) # Subclasses whose as_py returns non-primitive objects (dates, times, # timestamps, durations, half floats, ...) use the exact Scalar path. return Array._getitem_py(self, i) @@ -4064,14 +4066,11 @@ cdef class StringArray(Array): """ cdef object _getitem_py(self, int64_t i): - cdef: - int32_t length - const uint8_t* data if self.ap.IsNull(i): return None - data = ( self.ap).GetValue(i, &length) + cdef cpp_string_view view = ( self.ap).GetView(i) # Matches StringScalar.as_py, which is str(buf, 'utf8'). - return cp.PyUnicode_DecodeUTF8( data, length, NULL) + return cp.PyUnicode_DecodeUTF8(view.data(), view.size(), NULL) @staticmethod def from_buffers(int length, Buffer value_offsets, Buffer data, @@ -4106,13 +4105,10 @@ cdef class LargeStringArray(Array): """ cdef object _getitem_py(self, int64_t i): - cdef: - int64_t length - const uint8_t* data if self.ap.IsNull(i): return None - data = ( self.ap).GetValue(i, &length) - return cp.PyUnicode_DecodeUTF8( data, length, NULL) + cdef cpp_string_view view = ( self.ap).GetView(i) + return cp.PyUnicode_DecodeUTF8(view.data(), view.size(), NULL) @staticmethod def from_buffers(int length, Buffer value_offsets, Buffer data, @@ -4146,6 +4142,12 @@ cdef class StringViewArray(Array): Concrete class for Arrow arrays of string (or utf8) view data type. """ + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = ( self.ap).GetView(i) + return cp.PyUnicode_DecodeUTF8(view.data(), view.size(), NULL) + cdef class BinaryArray(Array): """ @@ -4153,13 +4155,10 @@ cdef class BinaryArray(Array): """ cdef object _getitem_py(self, int64_t i): - cdef: - int32_t length - const uint8_t* data if self.ap.IsNull(i): return None - data = ( self.ap).GetValue(i, &length) - return cp.PyBytes_FromStringAndSize( data, length) + cdef cpp_string_view view = ( self.ap).GetView(i) + return cp.PyBytes_FromStringAndSize(view.data(), view.size()) @property def total_values_length(self): @@ -4176,13 +4175,10 @@ cdef class LargeBinaryArray(Array): """ cdef object _getitem_py(self, int64_t i): - cdef: - int64_t length - const uint8_t* data if self.ap.IsNull(i): return None - data = ( self.ap).GetValue(i, &length) - return cp.PyBytes_FromStringAndSize( data, length) + cdef cpp_string_view view = ( self.ap).GetView(i) + return cp.PyBytes_FromStringAndSize(view.data(), view.size()) @property def total_values_length(self): @@ -4198,6 +4194,12 @@ cdef class BinaryViewArray(Array): Concrete class for Arrow arrays of variable-sized binary view data type. """ + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = ( self.ap).GetView(i) + return cp.PyBytes_FromStringAndSize(view.data(), view.size()) + cdef class DictionaryArray(Array): """ @@ -4364,15 +4366,13 @@ cdef class StructArray(Array): if self._children_cache is None: names = [self.type.field(k).name for k in range(num_fields)] if len(set(names)) != len(names): - # StructScalar.as_py raises ValueError on duplicate field - # names; mark the cache so we take the Scalar path below. - self._children_cache = (None, None) - else: - self._children_cache = ( - names, [self.field(k) for k in range(num_fields)]) + # Matches StructScalar.as_py + raise ValueError( + "Converting to Python dictionary is not supported when " + "duplicate field names are present") + self._children_cache = ( + names, [self.field(k) for k in range(num_fields)]) names = ( self._children_cache)[0] - if names is None: - return Array._getitem_py(self, i) fields = ( self._children_cache)[1] cdef Array field_arr result = {} diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 8b4786ecbf13..05467441ee19 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -946,6 +946,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: cdef cppclass CBinaryArray" arrow::BinaryArray"(CArray): const uint8_t* GetValue(int i, int32_t* length) + cpp_string_view GetView(int64_t i) shared_ptr[CBuffer] value_data() int32_t value_offset(int64_t i) int32_t value_length(int64_t i) @@ -953,6 +954,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: cdef cppclass CLargeBinaryArray" arrow::LargeBinaryArray"(CArray): const uint8_t* GetValue(int i, int64_t* length) + cpp_string_view GetView(int64_t i) shared_ptr[CBuffer] value_data() int64_t value_offset(int64_t i) int64_t value_length(int64_t i) @@ -977,6 +979,12 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: c_string GetString(int i) + cdef cppclass CBinaryViewArray" arrow::BinaryViewArray"(CArray): + cpp_string_view GetView(int64_t i) + + cdef cppclass CStringViewArray" arrow::StringViewArray"(CBinaryViewArray): + pass + cdef cppclass CStructArray" arrow::StructArray"(CArray): CStructArray(shared_ptr[CDataType]& type, int64_t length, vector[shared_ptr[CArray]]& children, From 0a1fee86324660606bcbe5348604224b3f431ee8 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 8 Jul 2026 15:20:43 -0700 Subject: [PATCH 6/7] GH-50429: [Python] Convert maps to dicts without per-element Scalars in to_pylist Thread maps_as_pydicts through the scalar-free _getitem_py mechanism instead of routing it to the Scalar-based path. MapArray builds the dict directly from the flattened keys/items children; when the dict size reveals duplicate keys, the row is redone with the per-key loop so the 'lossy' warnings and 'strict' KeyError match MapScalar.as_py exactly. Invalid values still raise the same ValueError when a map value is converted (including null rows), non-map arrays still ignore the option, and the option still propagates through nested types. Unspecialized types keep the exact Scalar fallback, which now receives the option. Benchmark (M4 Max, 1M rows of 2-entry maps, 10% nulls): to_pylist(maps_as_pydicts='lossy') 2.20s -> 0.10s (~21x). The dict form is also faster than the default association-list form (0.78s), which allocates a 2-tuple per entry. Co-authored-by: Isaac --- python/pyarrow/array.pxi | 85 ++++++++++++++++++++---------- python/pyarrow/lib.pxd | 2 +- python/pyarrow/tests/test_array.py | 30 +++++++++++ 3 files changed, 88 insertions(+), 29 deletions(-) diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 71c18fbdfe44..6931dc521086 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -1865,20 +1865,16 @@ cdef class Array(_PandasConvertible): """ self._assert_cpu() cdef int64_t i, n = self.length() - if maps_as_pydicts is not None: - # Converting maps to dicts has per-entry semantics (duplicate-key - # detection); use the Scalar-based conversion for exact behavior. - return [x.as_py(maps_as_pydicts=maps_as_pydicts) for x in self] # TODO(GH-50448): convert per range instead of per element to cut # the per-element call overhead further. - return [self._getitem_py(i) for i in range(n)] + return [self._getitem_py(i, maps_as_pydicts) for i in range(n)] - cdef object _getitem_py(self, int64_t i): + cdef object _getitem_py(self, int64_t i, object maps_as_pydicts): # Return self[i] as a Python object, without creating a Python Scalar # (nor, for nested types, per-row Array wrappers) where a subclass # provides a specialization; this base implementation goes through # Scalar.as_py and thus preserves its semantics exactly (see GH-50326). - return self.getitem(i).as_py() + return self.getitem(i).as_py(maps_as_pydicts=maps_as_pydicts) def tolist(self): """ @@ -2459,7 +2455,7 @@ cdef class BooleanArray(Array): Concrete class for Arrow arrays of boolean data type. """ - cdef object _getitem_py(self, int64_t i): + cdef object _getitem_py(self, int64_t i, object maps_as_pydicts): if self.ap.IsNull(i): return None return ( self.ap).Value(i) @@ -2478,7 +2474,7 @@ cdef class NumericArray(Array): A base class for Arrow numeric arrays. """ - cdef object _getitem_py(self, int64_t i): + cdef object _getitem_py(self, int64_t i, object maps_as_pydicts): cdef Type tid = self.ap.type_id() if self.ap.IsNull(i): return None @@ -2504,7 +2500,7 @@ cdef class NumericArray(Array): return ( self.ap).Value(i) # Subclasses whose as_py returns non-primitive objects (dates, times, # timestamps, durations, half floats, ...) use the exact Scalar path. - return Array._getitem_py(self, i) + return Array._getitem_py(self, i, maps_as_pydicts) cdef class IntegerArray(NumericArray): @@ -2824,7 +2820,7 @@ cdef class ListArray(BaseListArray): Concrete class for Arrow arrays of a list data type. """ - cdef object _getitem_py(self, int64_t i): + cdef object _getitem_py(self, int64_t i, object maps_as_pydicts): cdef CListArray* arr = self.ap if arr.IsNull(i): return None @@ -2832,7 +2828,7 @@ cdef class ListArray(BaseListArray): self._children_cache = pyarrow_wrap_array(arr.values()) cdef Array values = self._children_cache cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) - return [values._getitem_py(j) for j in range(start, end)] + return [values._getitem_py(j, maps_as_pydicts) for j in range(start, end)] @staticmethod def from_arrays(offsets, values, DataType type=None, MemoryPool pool=None, mask=None): @@ -3019,7 +3015,7 @@ cdef class LargeListArray(BaseListArray): Identical to ListArray, but 64-bit offsets. """ - cdef object _getitem_py(self, int64_t i): + cdef object _getitem_py(self, int64_t i, object maps_as_pydicts): cdef CLargeListArray* arr = self.ap if arr.IsNull(i): return None @@ -3027,7 +3023,7 @@ cdef class LargeListArray(BaseListArray): self._children_cache = pyarrow_wrap_array(arr.values()) cdef Array values = self._children_cache cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) - return [values._getitem_py(j) for j in range(start, end)] + return [values._getitem_py(j, maps_as_pydicts) for j in range(start, end)] @staticmethod def from_arrays(offsets, values, DataType type=None, MemoryPool pool=None, mask=None): @@ -3619,8 +3615,16 @@ cdef class MapArray(ListArray): Concrete class for Arrow arrays of a map data type. """ - cdef object _getitem_py(self, int64_t i): + cdef object _getitem_py(self, int64_t i, object maps_as_pydicts): cdef CListArray* arr = self.ap + cdef bint as_dicts = maps_as_pydicts is not None + if as_dicts and maps_as_pydicts != "lossy" and maps_as_pydicts != "strict": + # Matches MapScalar.as_py, which validates before the null check. + raise ValueError( + "Invalid value for 'maps_as_pydicts': " + + "valid values are 'lossy', 'strict' or `None` (default). " + + f"Received {maps_as_pydicts!r}." + ) if arr.IsNull(i): return None if self._children_cache is None: @@ -3628,9 +3632,34 @@ cdef class MapArray(ListArray): cdef Array keys = ( self._children_cache)[0] cdef Array items = ( self._children_cache)[1] cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) - # Matches MapScalar.as_py with the default maps_as_pydicts=None: - # an association list of (key, value) tuples. - return [(keys._getitem_py(j), items._getitem_py(j)) for j in range(start, end)] + if not as_dicts: + # Matches MapScalar.as_py with the default maps_as_pydicts=None: + # an association list of (key, value) tuples. + return [ + (keys._getitem_py(j, None), items._getitem_py(j, maps_as_pydicts)) + for j in range(start, end) + ] + cdef dict result = {} + for j in range(start, end): + result[keys._getitem_py(j, None)] = items._getitem_py(j, maps_as_pydicts) + if len(result) == end - start: + return result + # Duplicate keys: redo the row with the per-key loop so the 'lossy' + # warnings and 'strict' KeyError match MapScalar.as_py exactly. + result = {} + for j in range(start, end): + key = keys._getitem_py(j, None) + if key in result: + if maps_as_pydicts == "strict": + raise KeyError( + "Converting to Python dictionary is not supported in strict mode " + f"when duplicate keys are present (duplicate key was '{key}')." + ) + else: + warnings.warn( + f"Encountered key '{key}' which was already encountered.") + result[key] = items._getitem_py(j, maps_as_pydicts) + return result @staticmethod def from_arrays(offsets, keys, items, DataType type=None, MemoryPool pool=None, mask=None): @@ -3769,7 +3798,7 @@ cdef class FixedSizeListArray(BaseListArray): Concrete class for Arrow arrays of a fixed size list data type. """ - cdef object _getitem_py(self, int64_t i): + cdef object _getitem_py(self, int64_t i, object maps_as_pydicts): cdef CFixedSizeListArray* arr = self.ap if arr.IsNull(i): return None @@ -3777,7 +3806,7 @@ cdef class FixedSizeListArray(BaseListArray): self._children_cache = pyarrow_wrap_array(arr.values()) cdef Array values = self._children_cache cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) - return [values._getitem_py(j) for j in range(start, end)] + return [values._getitem_py(j, maps_as_pydicts) for j in range(start, end)] @staticmethod def from_arrays(values, list_size=None, DataType type=None, mask=None): @@ -4065,7 +4094,7 @@ cdef class StringArray(Array): Concrete class for Arrow arrays of string (or utf8) data type. """ - cdef object _getitem_py(self, int64_t i): + cdef object _getitem_py(self, int64_t i, object maps_as_pydicts): if self.ap.IsNull(i): return None cdef cpp_string_view view = ( self.ap).GetView(i) @@ -4104,7 +4133,7 @@ cdef class LargeStringArray(Array): Concrete class for Arrow arrays of large string (or utf8) data type. """ - cdef object _getitem_py(self, int64_t i): + cdef object _getitem_py(self, int64_t i, object maps_as_pydicts): if self.ap.IsNull(i): return None cdef cpp_string_view view = ( self.ap).GetView(i) @@ -4142,7 +4171,7 @@ cdef class StringViewArray(Array): Concrete class for Arrow arrays of string (or utf8) view data type. """ - cdef object _getitem_py(self, int64_t i): + cdef object _getitem_py(self, int64_t i, object maps_as_pydicts): if self.ap.IsNull(i): return None cdef cpp_string_view view = ( self.ap).GetView(i) @@ -4154,7 +4183,7 @@ cdef class BinaryArray(Array): Concrete class for Arrow arrays of variable-sized binary data type. """ - cdef object _getitem_py(self, int64_t i): + cdef object _getitem_py(self, int64_t i, object maps_as_pydicts): if self.ap.IsNull(i): return None cdef cpp_string_view view = ( self.ap).GetView(i) @@ -4174,7 +4203,7 @@ cdef class LargeBinaryArray(Array): Concrete class for Arrow arrays of large variable-sized binary data type. """ - cdef object _getitem_py(self, int64_t i): + cdef object _getitem_py(self, int64_t i, object maps_as_pydicts): if self.ap.IsNull(i): return None cdef cpp_string_view view = ( self.ap).GetView(i) @@ -4194,7 +4223,7 @@ cdef class BinaryViewArray(Array): Concrete class for Arrow arrays of variable-sized binary view data type. """ - cdef object _getitem_py(self, int64_t i): + cdef object _getitem_py(self, int64_t i, object maps_as_pydicts): if self.ap.IsNull(i): return None cdef cpp_string_view view = ( self.ap).GetView(i) @@ -4359,7 +4388,7 @@ cdef class StructArray(Array): Concrete class for Arrow arrays of a struct data type. """ - cdef object _getitem_py(self, int64_t i): + cdef object _getitem_py(self, int64_t i, object maps_as_pydicts): if self.ap.IsNull(i): return None cdef int64_t k, num_fields = self.type.num_fields @@ -4378,7 +4407,7 @@ cdef class StructArray(Array): result = {} for k in range(num_fields): field_arr = fields[k] - result[names[k]] = field_arr._getitem_py(i) + result[names[k]] = field_arr._getitem_py(i, maps_as_pydicts) return result def field(self, index): diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd index 38f1ac69a807..8e954323f8a9 100644 --- a/python/pyarrow/lib.pxd +++ b/python/pyarrow/lib.pxd @@ -296,7 +296,7 @@ cdef class Array(_PandasConvertible): cdef void init(self, const shared_ptr[CArray]& sp_array) except * cdef getitem(self, int64_t i) - cdef object _getitem_py(self, int64_t i) + cdef object _getitem_py(self, int64_t i, object maps_as_pydicts) cdef int64_t length(self) cdef void _assert_cpu(self) except * diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index 6331409cdb8f..190f39fb5cad 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -515,6 +515,36 @@ def test_to_pylist_bulk_paths(): dup.to_pylist() +def test_to_pylist_maps_as_pydicts(): + # GH-50429: maps_as_pydicts converts through the scalar-free path; the + # semantics must match MapScalar.as_py exactly. + map_type = pa.map_(pa.string(), pa.int32()) + arrays = [ + pa.array([[("k1", 1), ("k2", None)], None, []], type=map_type), + pa.array([[[("k", 1)], None], None], type=pa.list_(map_type)), + pa.array([[("o", [("i", 5)])]], + type=pa.map_(pa.string(), map_type)), + pa.array([{"m": [("k", 1)]}, None], + type=pa.struct([("m", map_type)])), + ] + for arr in arrays: + for mode in ("lossy", "strict"): + for view in (arr, arr.slice(1)): + expected = [x.as_py(maps_as_pydicts=mode) for x in view] + assert view.to_pylist(maps_as_pydicts=mode) == expected + + dup = pa.array([[("k", 1), ("k", 2)]], type=map_type) + with pytest.warns(UserWarning, match="already encountered"): + assert dup.to_pylist(maps_as_pydicts="lossy") == [{"k": 2}] + with pytest.raises(KeyError, match="strict mode"): + dup.to_pylist(maps_as_pydicts="strict") + with pytest.raises(ValueError, match="Invalid value for 'maps_as_pydicts'"): + dup.to_pylist(maps_as_pydicts="bogus") + # invalid values are only rejected when a map value is converted, + # matching the Scalar-based behavior + assert pa.array([1, 2]).to_pylist(maps_as_pydicts="bogus") == [1, 2] + + def test_array_slice(): arr = pa.array(range(10)) From 8626b97b1d00d88c9073e81e53ada24eaf5a34eb Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 9 Jul 2026 08:33:15 -0700 Subject: [PATCH 7/7] GH-50429: [Python] Detect duplicate keys before converting values Convert all keys first (as MapScalar.as_py does via keys()) and check for duplicates before converting any value, so that the 'lossy' warning and the 'strict' KeyError are emitted at the same point as in MapScalar.as_py even when a later value conversion raises. Add tests with a raising value placed after the duplicate key in both modes. lossy 1M rows: 0.16s (vs 2.20s Scalar-based; the extra keys pass costs ~0.06s vs the previous single-pass form). Co-authored-by: Isaac --- python/pyarrow/array.pxi | 23 ++++++++++++++--------- python/pyarrow/tests/test_array.py | 11 +++++++++++ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 6931dc521086..fb004243a209 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -3639,16 +3639,21 @@ cdef class MapArray(ListArray): (keys._getitem_py(j, None), items._getitem_py(j, maps_as_pydicts)) for j in range(start, end) ] + # Convert all keys first (as MapScalar.as_py does via keys()) and + # detect duplicates before converting any value, so that the 'lossy' + # warnings and the 'strict' KeyError are emitted at the same point as + # in MapScalar.as_py even when a later value conversion raises. + cdef int64_t count = end - start + cdef list keys_py = [keys._getitem_py(j, None) for j in range(start, end)] cdef dict result = {} - for j in range(start, end): - result[keys._getitem_py(j, None)] = items._getitem_py(j, maps_as_pydicts) - if len(result) == end - start: + cdef int64_t k + if len(set(keys_py)) == count: + for k in range(count): + result[keys_py[k]] = items._getitem_py(start + k, maps_as_pydicts) return result - # Duplicate keys: redo the row with the per-key loop so the 'lossy' - # warnings and 'strict' KeyError match MapScalar.as_py exactly. - result = {} - for j in range(start, end): - key = keys._getitem_py(j, None) + # Duplicate keys: per-key loop matching MapScalar.as_py exactly. + for k in range(count): + key = keys_py[k] if key in result: if maps_as_pydicts == "strict": raise KeyError( @@ -3658,7 +3663,7 @@ cdef class MapArray(ListArray): else: warnings.warn( f"Encountered key '{key}' which was already encountered.") - result[key] = items._getitem_py(j, maps_as_pydicts) + result[key] = items._getitem_py(start + k, maps_as_pydicts) return result @staticmethod diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index 190f39fb5cad..9128647e3dc7 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -538,6 +538,17 @@ def test_to_pylist_maps_as_pydicts(): assert dup.to_pylist(maps_as_pydicts="lossy") == [{"k": 2}] with pytest.raises(KeyError, match="strict mode"): dup.to_pylist(maps_as_pydicts="strict") + + # Duplicate keys must be detected before converting values: with a + # poison value *after* the duplicate, strict mode raises the outer + # duplicate-key error, and lossy mode still emits its warning first. + nested_map = pa.map_(pa.string(), map_type) + poison = pa.array( + [[("k1", [("a", 1)]), ("k1", [("d", 1), ("d", 2)])]], type=nested_map) + with pytest.raises(KeyError, match="duplicate key was 'k1'"): + poison.to_pylist(maps_as_pydicts="strict") + with pytest.warns(UserWarning, match="Encountered key 'k1'"): + assert poison.to_pylist(maps_as_pydicts="lossy") == [{"k1": {"d": 2}}] with pytest.raises(ValueError, match="Invalid value for 'maps_as_pydicts'"): dup.to_pylist(maps_as_pydicts="bogus") # invalid values are only rejected when a map value is converted,