From 616af4bcbf166259f3bf786f06408ede9c6f6e95 Mon Sep 17 00:00:00 2001
From: Onur Satici <onur@spiraldb.com>
Date: Tue, 2 Jun 2026 09:10:56 +0100
Subject: [PATCH 1/3] split registration to respect nested projection and
 filters

---
 vortex-array/src/dtype/field.rs               |  47 ++++-
 vortex-array/src/expr/analysis/mod.rs         |   2 +
 .../expr/analysis/referenced_field_paths.rs   | 172 ++++++++++++++++++
 vortex-layout/src/scan/scan_builder.rs        | 112 ++++++++----
 4 files changed, 295 insertions(+), 38 deletions(-)
 create mode 100644 vortex-array/src/expr/analysis/referenced_field_paths.rs

diff --git a/vortex-array/src/dtype/field.rs b/vortex-array/src/dtype/field.rs
index 45ddb70c9a7..0f2ffbd1574 100644
--- a/vortex-array/src/dtype/field.rs
+++ b/vortex-array/src/dtype/field.rs
@@ -238,7 +238,7 @@ impl Display for FieldPath {
     }
 }
 
-#[derive(Default, Clone, Debug)]
+#[derive(Default, Clone, Debug, PartialEq, Eq)]
 /// Contains a set of field paths, and can answer an efficient field path contains queries.
 pub struct FieldPathSet {
     /// While this is currently a set wrapper it can be replaced with a trie.
@@ -251,6 +251,30 @@ impl FieldPathSet {
     pub fn contains(&self, path: &FieldPath) -> bool {
         self.set.contains(path)
     }
+
+    /// Inserts a field path prefix unless an existing prefix already covers it.
+    ///
+    /// Any existing field paths covered by the new prefix are removed.
+    pub fn insert_prefix(&mut self, path: FieldPath) {
+        if self
+            .set
+            .iter()
+            .any(|existing| path.parts().starts_with(existing.parts()))
+        {
+            return;
+        }
+
+        self.set
+            .retain(|existing| !existing.parts().starts_with(path.parts()));
+        self.set.insert(path);
+    }
+
+    /// Inserts field path prefixes, retaining only the minimal covering set.
+    pub fn extend_prefixes(&mut self, paths: impl IntoIterator<Item = FieldPath>) {
+        for path in paths {
+            self.insert_prefix(path);
+        }
+    }
 }
 
 impl FromIterator<FieldPath> for FieldPathSet {
@@ -260,6 +284,15 @@ impl FromIterator<FieldPath> for FieldPathSet {
     }
 }
 
+impl IntoIterator for FieldPathSet {
+    type Item = FieldPath;
+    type IntoIter = <HashSet<FieldPath> as IntoIterator>::IntoIter;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.set.into_iter()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -418,4 +451,16 @@ mod tests {
         assert!(!path1.overlap(&path3));
         assert!(!path3.overlap(&path1));
     }
+
+    #[test]
+    fn insert_prefix_retains_minimal_covering_set() {
+        let mut paths = FieldPathSet::default();
+        paths.extend_prefixes([field_path!(a.b), field_path!(x), field_path!(a)]);
+        paths.insert_prefix(field_path!(a.c));
+
+        assert_eq!(
+            paths,
+            FieldPathSet::from_iter([field_path!(a), field_path!(x)])
+        );
+    }
 }
diff --git a/vortex-array/src/expr/analysis/mod.rs b/vortex-array/src/expr/analysis/mod.rs
index a0b07eb96e6..f5208a31be8 100644
--- a/vortex-array/src/expr/analysis/mod.rs
+++ b/vortex-array/src/expr/analysis/mod.rs
@@ -6,6 +6,7 @@ mod fallible;
 pub mod immediate_access;
 mod labeling;
 mod null_sensitive;
+mod referenced_field_paths;
 
 pub use annotation::*;
 pub use fallible::label_is_fallible;
@@ -13,3 +14,4 @@ pub use immediate_access::*;
 pub use labeling::*;
 pub use null_sensitive::BooleanLabels;
 pub use null_sensitive::label_null_sensitive;
+pub use referenced_field_paths::referenced_field_paths;
diff --git a/vortex-array/src/expr/analysis/referenced_field_paths.rs b/vortex-array/src/expr/analysis/referenced_field_paths.rs
new file mode 100644
index 00000000000..94444011c98
--- /dev/null
+++ b/vortex-array/src/expr/analysis/referenced_field_paths.rs
@@ -0,0 +1,172 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+use std::iter::once;
+
+use vortex_error::VortexResult;
+use vortex_error::vortex_err;
+
+use crate::dtype::DType;
+use crate::dtype::Field;
+use crate::dtype::FieldPath;
+use crate::dtype::FieldPathSet;
+use crate::expr::Expression;
+use crate::scalar_fn::fns::get_item::GetItem;
+use crate::scalar_fn::fns::root::Root;
+use crate::scalar_fn::fns::select::Select;
+
+/// Returns a prefix-minimal set of rooted field paths referenced by an expression.
+///
+/// A standalone root expression is represented by [`FieldPath::root`], which conservatively
+/// selects all fields. When one referenced path is a prefix of another, only the prefix is returned.
+/// Scalar functions other than `GetItem` and `Select` conservatively reference each complete child
+/// output.
+pub fn referenced_field_paths(expr: &Expression, scope: &DType) -> VortexResult<FieldPathSet> {
+    // Validate the whole expression so plain GetItem paths and Select paths behave consistently.
+    expr.return_dtype(scope)?;
+
+    let mut field_paths = FieldPathSet::default();
+    collect_referenced_field_paths(expr, scope, &FieldPath::root(), &mut field_paths)?;
+    Ok(field_paths)
+}
+
+fn collect_referenced_field_paths(
+    expr: &Expression,
+    scope: &DType,
+    requested_path: &FieldPath,
+    field_paths: &mut FieldPathSet,
+) -> VortexResult<()> {
+    if expr.is::<Root>() {
+        field_paths.insert_prefix(requested_path.clone());
+        return Ok(());
+    }
+
+    if let Some(field_name) = expr.as_opt::<GetItem>() {
+        let requested_path = FieldPath::from_iter(
+            once(Field::Name(field_name.clone())).chain(requested_path.parts().iter().cloned()),
+        );
+        return collect_referenced_field_paths(expr.child(0), scope, &requested_path, field_paths);
+    }
+
+    if let Some(selection) = expr.as_opt::<Select>() {
+        let child_dtype = expr.child(0).return_dtype(scope)?;
+        let child_fields = child_dtype
+            .as_struct_fields_opt()
+            .ok_or_else(|| vortex_err!("Select child is not a struct"))?;
+        let included_fields = selection.normalize_to_included_fields(child_fields.names())?;
+
+        if requested_path.is_root() {
+            for field_name in included_fields {
+                collect_referenced_field_paths(
+                    expr.child(0),
+                    scope,
+                    &FieldPath::from_name(field_name),
+                    field_paths,
+                )?;
+            }
+        } else if let Some(Field::Name(field_name)) = requested_path.parts().first()
+            && included_fields
+                .iter()
+                .any(|included| included == field_name)
+        {
+            collect_referenced_field_paths(expr.child(0), scope, requested_path, field_paths)?;
+        }
+        return Ok(());
+    }
+
+    for child in expr.children().iter() {
+        collect_referenced_field_paths(child, scope, &FieldPath::root(), field_paths)?;
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::dtype::Nullability::NonNullable;
+    use crate::dtype::PType::I32;
+    use crate::dtype::StructFields;
+    use crate::expr::get_item;
+    use crate::expr::pack;
+    use crate::expr::root;
+    use crate::expr::select;
+    use crate::expr::select_exclude;
+
+    fn scope() -> DType {
+        DType::Struct(
+            StructFields::from_iter([(
+                "a",
+                DType::Struct(
+                    StructFields::from_iter([("x", I32), ("y", I32)]),
+                    NonNullable,
+                ),
+            )]),
+            NonNullable,
+        )
+    }
+
+    #[test]
+    fn nested_select_preserves_field_path() -> VortexResult<()> {
+        let expr = select(["x"], get_item("a", root()));
+
+        assert_eq!(
+            referenced_field_paths(&expr, &scope())?,
+            FieldPathSet::from_iter([FieldPath::from_name("a").push("x")])
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn get_item_after_select_only_references_requested_field() -> VortexResult<()> {
+        let expr = get_item("x", select(["x", "y"], get_item("a", root())));
+
+        assert_eq!(
+            referenced_field_paths(&expr, &scope())?,
+            FieldPathSet::from_iter([FieldPath::from_name("a").push("x")])
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn select_exclude_references_included_fields() -> VortexResult<()> {
+        let expr = select_exclude(["y"], get_item("a", root()));
+
+        assert_eq!(
+            referenced_field_paths(&expr, &scope())?,
+            FieldPathSet::from_iter([FieldPath::from_name("a").push("x")])
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn ancestor_path_subsumes_descendant() -> VortexResult<()> {
+        let expr = pack(
+            [
+                ("a", get_item("a", root())),
+                ("x", get_item("x", get_item("a", root()))),
+            ],
+            NonNullable,
+        );
+
+        assert_eq!(
+            referenced_field_paths(&expr, &scope())?,
+            FieldPathSet::from_iter([FieldPath::from_name("a")])
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn root_references_all_fields() -> VortexResult<()> {
+        assert_eq!(
+            referenced_field_paths(&root(), &scope())?,
+            FieldPathSet::from_iter([FieldPath::root()])
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn invalid_get_item_path_returns_error() {
+        assert!(referenced_field_paths(&get_item("missing", root()), &scope()).is_err());
+    }
+}
diff --git a/vortex-layout/src/scan/scan_builder.rs b/vortex-layout/src/scan/scan_builder.rs
index a46bfa14e2f..550f67e5017 100644
--- a/vortex-layout/src/scan/scan_builder.rs
+++ b/vortex-layout/src/scan/scan_builder.rs
@@ -15,12 +15,9 @@ use futures::stream::BoxStream;
 use itertools::Itertools;
 use vortex_array::ArrayRef;
 use vortex_array::dtype::DType;
-use vortex_array::dtype::Field;
 use vortex_array::dtype::FieldMask;
-use vortex_array::dtype::FieldName;
-use vortex_array::dtype::FieldPath;
 use vortex_array::expr::Expression;
-use vortex_array::expr::analysis::immediate_access::immediate_scope_access;
+use vortex_array::expr::analysis::referenced_field_paths;
 use vortex_array::expr::root;
 use vortex_array::iter::ArrayIterator;
 use vortex_array::iter::ArrayIteratorAdapter;
@@ -267,9 +264,8 @@ impl<A: 'static + Send> ScanBuilder<A> {
             .transpose()?;
 
         // Construct field masks and compute the row splits of the scan.
-        let (filter_mask, projection_mask) =
-            filter_and_projection_masks(&projection, filter.as_ref(), layout_reader.dtype())?;
-        let field_mask: Vec<_> = [filter_mask, projection_mask].concat();
+        let field_mask =
+            referenced_field_masks(&projection, filter.as_ref(), layout_reader.dtype())?;
 
         let splits =
             if let Some(ranges) = attempt_split_ranges(&self.selection, self.row_range.as_ref()) {
@@ -412,40 +408,20 @@ impl<A: 'static + Send> Stream for LazyScanStream<A> {
 /// Compute masks of field paths referenced by the projection and filter in the scan.
 ///
 /// Projection and filter must be pre-simplified.
-pub fn filter_and_projection_masks(
+pub fn referenced_field_masks(
     projection: &Expression,
     filter: Option<&Expression>,
     dtype: &DType,
-) -> VortexResult<(Vec<FieldMask>, Vec<FieldMask>)> {
-    let Some(struct_dtype) = dtype.as_struct_fields_opt() else {
-        return Ok(match filter {
-            Some(_) => (vec![FieldMask::All], vec![FieldMask::All]),
-            None => (Vec::new(), vec![FieldMask::All]),
-        });
-    };
-    let projection_mask = immediate_scope_access(projection, struct_dtype);
-    Ok(match filter {
-        None => (
-            Vec::new(),
-            projection_mask.into_iter().map(to_field_mask).collect_vec(),
-        ),
-        Some(f) => {
-            let filter_mask = immediate_scope_access(f, struct_dtype);
-            let only_projection_mask = projection_mask
-                .difference(&filter_mask)
-                .cloned()
-                .map(to_field_mask)
-                .collect_vec();
-            (
-                filter_mask.into_iter().map(to_field_mask).collect_vec(),
-                only_projection_mask,
-            )
-        }
-    })
-}
+) -> VortexResult<Vec<FieldMask>> {
+    if dtype.as_struct_fields_opt().is_none() {
+        return Ok(vec![FieldMask::All]);
+    }
 
-fn to_field_mask(field: FieldName) -> FieldMask {
-    FieldMask::Prefix(FieldPath::from(Field::Name(field)))
+    let mut field_paths = referenced_field_paths(projection, dtype)?;
+    if let Some(filter) = filter {
+        field_paths.extend_prefixes(referenced_field_paths(filter, dtype)?);
+    }
+    Ok(field_paths.into_iter().map(FieldMask::Prefix).collect_vec())
 }
 
 #[cfg(test)]
@@ -470,9 +446,16 @@ mod test {
     use vortex_array::arrays::PrimitiveArray;
     use vortex_array::dtype::DType;
     use vortex_array::dtype::FieldMask;
+    use vortex_array::dtype::FieldPath;
     use vortex_array::dtype::Nullability;
     use vortex_array::dtype::PType;
+    use vortex_array::dtype::StructFields;
     use vortex_array::expr::Expression;
+    use vortex_array::expr::eq;
+    use vortex_array::expr::get_item;
+    use vortex_array::expr::is_not_null;
+    use vortex_array::expr::lit;
+    use vortex_array::expr::root;
     use vortex_error::VortexResult;
     use vortex_error::vortex_err;
     use vortex_io::runtime::BlockingRuntime;
@@ -480,6 +463,7 @@ mod test {
     use vortex_mask::Mask;
 
     use super::ScanBuilder;
+    use super::referenced_field_masks;
     use crate::ArrayFuture;
     use crate::LayoutReader;
     use crate::RowSplits;
@@ -487,6 +471,60 @@ mod test {
     use crate::scan::test::SCAN_SESSION;
     use crate::scan::test::session_with_handle;
 
+    fn nested_dtype() -> DType {
+        DType::Struct(
+            StructFields::from_iter([
+                (
+                    "a",
+                    DType::Struct(
+                        StructFields::from_iter([
+                            ("1", DType::Primitive(PType::I32, Nullability::NonNullable)),
+                            ("2", DType::Primitive(PType::I32, Nullability::NonNullable)),
+                        ]),
+                        Nullability::NonNullable,
+                    ),
+                ),
+                ("b", DType::Primitive(PType::I32, Nullability::NonNullable)),
+            ]),
+            Nullability::NonNullable,
+        )
+    }
+
+    #[test]
+    fn nested_projection_preserves_field_path_in_split_mask() -> VortexResult<()> {
+        let projection = get_item("1", get_item("a", root()));
+        let filter = eq(get_item("2", get_item("a", root())), lit(0_i32));
+
+        let field_masks = referenced_field_masks(&projection, Some(&filter), &nested_dtype())?;
+
+        assert_eq!(field_masks.len(), 2);
+        assert!(field_masks.contains(&FieldMask::Prefix(FieldPath::from_name("a").push("1"))));
+        assert!(field_masks.contains(&FieldMask::Prefix(FieldPath::from_name("a").push("2"))));
+        Ok(())
+    }
+
+    #[test]
+    fn filter_path_covers_nested_projection_path() -> VortexResult<()> {
+        let projection = get_item("1", get_item("a", root()));
+        let filter = is_not_null(get_item("a", root()));
+
+        let field_masks = referenced_field_masks(&projection, Some(&filter), &nested_dtype())?;
+
+        assert_eq!(field_masks, [FieldMask::Prefix(FieldPath::from_name("a"))]);
+        Ok(())
+    }
+
+    #[test]
+    fn parent_projection_path_covers_nested_filter_path() -> VortexResult<()> {
+        let projection = get_item("a", root());
+        let filter = is_not_null(get_item("1", get_item("a", root())));
+
+        let field_masks = referenced_field_masks(&projection, Some(&filter), &nested_dtype())?;
+
+        assert_eq!(field_masks, [FieldMask::Prefix(FieldPath::from_name("a"))]);
+        Ok(())
+    }
+
     #[derive(Debug)]
     struct CountingLayoutReader {
         name: Arc<str>,

From 0cff087db5f7dbe353f0ec7940ef06ba7af6a333 Mon Sep 17 00:00:00 2001
From: Onur Satici <onur@spiraldb.com>
Date: Wed, 3 Jun 2026 14:30:39 +0100
Subject: [PATCH 2/3] comments

Signed-off-by: Onur Satici <onur@spiraldb.com>
---
 vortex-array/src/dtype/field.rs               |   5 +
 .../expr/analysis/referenced_field_paths.rs   | 173 +++++++++++++-----
 2 files changed, 134 insertions(+), 44 deletions(-)

diff --git a/vortex-array/src/dtype/field.rs b/vortex-array/src/dtype/field.rs
index 0f2ffbd1574..8ce35928b4a 100644
--- a/vortex-array/src/dtype/field.rs
+++ b/vortex-array/src/dtype/field.rs
@@ -252,6 +252,11 @@ impl FieldPathSet {
         self.set.contains(path)
     }
 
+    /// Iterates over the field paths in the set.
+    pub fn iter(&self) -> impl Iterator<Item = &FieldPath> {
+        self.set.iter()
+    }
+
     /// Inserts a field path prefix unless an existing prefix already covers it.
     ///
     /// Any existing field paths covered by the new prefix are removed.
diff --git a/vortex-array/src/expr/analysis/referenced_field_paths.rs b/vortex-array/src/expr/analysis/referenced_field_paths.rs
index 94444011c98..767f1cbe274 100644
--- a/vortex-array/src/expr/analysis/referenced_field_paths.rs
+++ b/vortex-array/src/expr/analysis/referenced_field_paths.rs
@@ -11,6 +11,10 @@ use crate::dtype::Field;
 use crate::dtype::FieldPath;
 use crate::dtype::FieldPathSet;
 use crate::expr::Expression;
+use crate::expr::traversal::FoldDownContext;
+use crate::expr::traversal::FoldUp;
+use crate::expr::traversal::NodeExt;
+use crate::expr::traversal::NodeFolderContext;
 use crate::scalar_fn::fns::get_item::GetItem;
 use crate::scalar_fn::fns::root::Root;
 use crate::scalar_fn::fns::select::Select;
@@ -25,60 +29,128 @@ pub fn referenced_field_paths(expr: &Expression, scope: &DType) -> VortexResult<
     // Validate the whole expression so plain GetItem paths and Select paths behave consistently.
     expr.return_dtype(scope)?;
 
-    let mut field_paths = FieldPathSet::default();
-    collect_referenced_field_paths(expr, scope, &FieldPath::root(), &mut field_paths)?;
-    Ok(field_paths)
-}
+    let mut collector = ReferencedFieldPaths {
+        scope,
+        field_paths: FieldPathSet::default(),
+    };
+    expr.clone()
+        .fold_context(&vec![FieldPath::root()], &mut collector)?;
+    let field_paths = collector.field_paths;
 
-fn collect_referenced_field_paths(
-    expr: &Expression,
-    scope: &DType,
-    requested_path: &FieldPath,
-    field_paths: &mut FieldPathSet,
-) -> VortexResult<()> {
-    if expr.is::<Root>() {
-        field_paths.insert_prefix(requested_path.clone());
-        return Ok(());
-    }
+    // The top-level field of every referenced path must be one of the immediately accessed scope
+    // fields: this analysis only refines *which nested fields* are read, never which top-level
+    // fields. `FieldPath::root()` stands in for "all fields", so it expands to the whole scope.
+    #[cfg(debug_assertions)]
+    if let Some(scope_fields) = scope.as_struct_fields_opt() {
+        use vortex_utils::aliases::hash_set::HashSet;
+
+        use crate::dtype::FieldName;
+        use crate::expr::analysis::immediate_access::immediate_scope_access;
 
-    if let Some(field_name) = expr.as_opt::<GetItem>() {
-        let requested_path = FieldPath::from_iter(
-            once(Field::Name(field_name.clone())).chain(requested_path.parts().iter().cloned()),
+        let referenced_heads: HashSet<FieldName> = if field_paths.iter().any(FieldPath::is_root) {
+            scope_fields.names().iter().cloned().collect()
+        } else {
+            field_paths
+                .iter()
+                .filter_map(|path| match path.parts().first() {
+                    Some(Field::Name(name)) => Some(name.clone()),
+                    _ => None,
+                })
+                .collect()
+        };
+        debug_assert_eq!(
+            referenced_heads,
+            immediate_scope_access(expr, scope_fields),
+            "referenced field path heads must match the immediately accessed scope fields"
         );
-        return collect_referenced_field_paths(expr.child(0), scope, &requested_path, field_paths);
     }
 
-    if let Some(selection) = expr.as_opt::<Select>() {
-        let child_dtype = expr.child(0).return_dtype(scope)?;
-        let child_fields = child_dtype
-            .as_struct_fields_opt()
-            .ok_or_else(|| vortex_err!("Select child is not a struct"))?;
-        let included_fields = selection.normalize_to_included_fields(child_fields.names())?;
-
-        if requested_path.is_root() {
-            for field_name in included_fields {
-                collect_referenced_field_paths(
-                    expr.child(0),
-                    scope,
-                    &FieldPath::from_name(field_name),
-                    field_paths,
-                )?;
-            }
-        } else if let Some(Field::Name(field_name)) = requested_path.parts().first()
-            && included_fields
+    Ok(field_paths)
+}
+
+/// Threads the set of currently-requested field paths down the expression tree, narrowing it at
+/// each `GetItem`/`Select`, and records the rooted paths reached at each `Root` leaf.
+///
+/// Narrowing the requested path is only sound through `GetItem` (a genuine field access) and
+/// `Select` (a genuine column projection). Any other function is opaque—we cannot assume it
+/// preserves a field's provenance—so its children conservatively re-request the whole scope. This
+/// is what keeps an expression like `f($).x` reading every field of `$` rather than just `x`.
+struct ReferencedFieldPaths<'a> {
+    scope: &'a DType,
+    field_paths: FieldPathSet,
+}
+
+impl NodeFolderContext for ReferencedFieldPaths<'_> {
+    type NodeTy = Expression;
+    type Result = ();
+    type Context = Vec<FieldPath>;
+
+    fn visit_down(
+        &mut self,
+        requested: &Self::Context,
+        node: &Expression,
+    ) -> VortexResult<FoldDownContext<Self::Context, ()>> {
+        // A `Root` leaf resolves the requested paths against the actual scope.
+        if node.is::<Root>() {
+            self.field_paths.extend_prefixes(requested.iter().cloned());
+            return Ok(FoldDownContext::Skip(()));
+        }
+
+        // `GetItem` prepends its field to every requested path before descending into its child.
+        if let Some(field_name) = node.as_opt::<GetItem>() {
+            let prefixed = requested
                 .iter()
-                .any(|included| included == field_name)
-        {
-            collect_referenced_field_paths(expr.child(0), scope, requested_path, field_paths)?;
+                .map(|path| {
+                    FieldPath::from_iter(
+                        once(Field::Name(field_name.clone())).chain(path.parts().iter().cloned()),
+                    )
+                })
+                .collect();
+            return Ok(FoldDownContext::Continue(prefixed));
         }
-        return Ok(());
-    }
 
-    for child in expr.children().iter() {
-        collect_referenced_field_paths(child, scope, &FieldPath::root(), field_paths)?;
+        // `Select` keeps requested paths whose head is included, expanding a whole-scope request
+        // into one path per included field.
+        if let Some(selection) = node.as_opt::<Select>() {
+            let child_dtype = node.child(0).return_dtype(self.scope)?;
+            let child_fields = child_dtype
+                .as_struct_fields_opt()
+                .ok_or_else(|| vortex_err!("Select child is not a struct"))?;
+            let included_fields = selection.normalize_to_included_fields(child_fields.names())?;
+
+            let mut narrowed = Vec::new();
+            for path in requested {
+                if path.is_root() {
+                    narrowed.extend(included_fields.iter().cloned().map(FieldPath::from_name));
+                } else if let Some(Field::Name(field_name)) = path.parts().first()
+                    && included_fields
+                        .iter()
+                        .any(|included| included == field_name)
+                {
+                    narrowed.push(path.clone());
+                }
+            }
+
+            // Nothing is requested below this `Select`, so prune the subtree rather than letting an
+            // opaque child re-request the whole scope.
+            if narrowed.is_empty() {
+                return Ok(FoldDownContext::Skip(()));
+            }
+            return Ok(FoldDownContext::Continue(narrowed));
+        }
+
+        // Any other function conservatively references each child's complete output.
+        Ok(FoldDownContext::Continue(vec![FieldPath::root()]))
     }
 
-    Ok(())
+    fn visit_up(
+        &mut self,
+        _node: Expression,
+        _requested: &Self::Context,
+        _children: Vec<()>,
+    ) -> VortexResult<FoldUp<()>> {
+        Ok(FoldUp::Continue(()))
+    }
 }
 
 #[cfg(test)]
@@ -156,6 +228,19 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn get_item_through_opaque_fn_references_all_fields() -> VortexResult<()> {
+        // `pack` is opaque to the path analysis: a `GetItem` of its output must not be pushed down
+        // as a scope field access, so the wrapped `root()` conservatively references all fields.
+        let expr = get_item("x", pack([("x", root())], NonNullable));
+
+        assert_eq!(
+            referenced_field_paths(&expr, &scope())?,
+            FieldPathSet::from_iter([FieldPath::root()])
+        );
+        Ok(())
+    }
+
     #[test]
     fn root_references_all_fields() -> VortexResult<()> {
         assert_eq!(

From 27c5510c86fcff40e04a33278084c218a1ce2833 Mon Sep 17 00:00:00 2001
From: Onur Satici <onur@spiraldb.com>
Date: Wed, 3 Jun 2026 16:08:32 +0100
Subject: [PATCH 3/3] more comments

Signed-off-by: Onur Satici <onur@spiraldb.com>
---
 vortex-array/src/dtype/field.rs               | 67 ++++++++-------
 .../expr/analysis/referenced_field_paths.rs   | 81 ++++++++++---------
 vortex-layout/src/scan/scan_builder.rs        |  2 +-
 3 files changed, 84 insertions(+), 66 deletions(-)

diff --git a/vortex-array/src/dtype/field.rs b/vortex-array/src/dtype/field.rs
index 8ce35928b4a..84f631b5074 100644
--- a/vortex-array/src/dtype/field.rs
+++ b/vortex-array/src/dtype/field.rs
@@ -238,47 +238,56 @@ impl Display for FieldPath {
     }
 }
 
-#[derive(Default, Clone, Debug, PartialEq, Eq)]
-/// Contains a set of field paths, and can answer an efficient field path contains queries.
+#[derive(Default, Clone, Debug)]
+/// A set of field paths supporting efficient `contains` queries.
+///
+/// Paths are stored as inserted. Prefix-minimization—collapsing a path into an ancestor that
+/// already covers it—is deferred until the set is iterated via [`IntoIterator`], so insertion stays
+/// cheap.
 pub struct FieldPathSet {
-    /// While this is currently a set wrapper it can be replaced with a trie.
+    /// While this is currently a set wrapper it can be replaced with a trie, at which point the
+    /// deferred minimization in [`IntoIterator`] becomes cheap.
     // TODO(joe): this can be replaced with a `FieldPath` trie
     set: HashSet<FieldPath>,
 }
 
 impl FieldPathSet {
-    /// Checks if a set contains a field path
+    /// Checks if the set contains exactly this field path.
     pub fn contains(&self, path: &FieldPath) -> bool {
         self.set.contains(path)
     }
 
-    /// Iterates over the field paths in the set.
+    /// Iterates over the field paths in the set, as inserted (not prefix-minimized).
     pub fn iter(&self) -> impl Iterator<Item = &FieldPath> {
         self.set.iter()
     }
 
-    /// Inserts a field path prefix unless an existing prefix already covers it.
-    ///
-    /// Any existing field paths covered by the new prefix are removed.
-    pub fn insert_prefix(&mut self, path: FieldPath) {
-        if self
-            .set
+    /// Inserts a field path. Prefix-minimization is deferred until the set is iterated.
+    pub fn insert(&mut self, path: FieldPath) {
+        self.set.insert(path);
+    }
+}
+
+/// Reduces field paths to their minimal covering set: any path that has another path in the set as
+/// a prefix is redundant and dropped.
+fn minimal_covering_set(paths: impl IntoIterator<Item = FieldPath>) -> Vec<FieldPath> {
+    let mut covering: Vec<FieldPath> = Vec::new();
+    for path in paths {
+        if covering
             .iter()
             .any(|existing| path.parts().starts_with(existing.parts()))
         {
-            return;
+            continue;
         }
-
-        self.set
-            .retain(|existing| !existing.parts().starts_with(path.parts()));
-        self.set.insert(path);
+        covering.retain(|existing| !existing.parts().starts_with(path.parts()));
+        covering.push(path);
     }
+    covering
+}
 
-    /// Inserts field path prefixes, retaining only the minimal covering set.
-    pub fn extend_prefixes(&mut self, paths: impl IntoIterator<Item = FieldPath>) {
-        for path in paths {
-            self.insert_prefix(path);
-        }
+impl Extend<FieldPath> for FieldPathSet {
+    fn extend<T: IntoIterator<Item = FieldPath>>(&mut self, iter: T) {
+        self.set.extend(iter);
     }
 }
 
@@ -291,10 +300,11 @@ impl FromIterator<FieldPath> for FieldPathSet {
 
 impl IntoIterator for FieldPathSet {
     type Item = FieldPath;
-    type IntoIter = <HashSet<FieldPath> as IntoIterator>::IntoIter;
+    type IntoIter = std::vec::IntoIter<FieldPath>;
 
+    /// Iterates the prefix-minimal covering set: redundant descendants are dropped.
     fn into_iter(self) -> Self::IntoIter {
-        self.set.into_iter()
+        minimal_covering_set(self.set).into_iter()
     }
 }
 
@@ -458,14 +468,15 @@ mod tests {
     }
 
     #[test]
-    fn insert_prefix_retains_minimal_covering_set() {
+    fn iteration_yields_minimal_covering_set() {
         let mut paths = FieldPathSet::default();
-        paths.extend_prefixes([field_path!(a.b), field_path!(x), field_path!(a)]);
-        paths.insert_prefix(field_path!(a.c));
+        paths.extend([field_path!(a.b), field_path!(x), field_path!(a)]);
+        paths.insert(field_path!(a.c));
 
+        // Iteration collapses `a.b`/`a.c` into the covering `a`.
         assert_eq!(
-            paths,
-            FieldPathSet::from_iter([field_path!(a), field_path!(x)])
+            paths.into_iter().collect::<HashSet<_>>(),
+            HashSet::from_iter([field_path!(a), field_path!(x)])
         );
     }
 }
diff --git a/vortex-array/src/expr/analysis/referenced_field_paths.rs b/vortex-array/src/expr/analysis/referenced_field_paths.rs
index 767f1cbe274..5239e4a691c 100644
--- a/vortex-array/src/expr/analysis/referenced_field_paths.rs
+++ b/vortex-array/src/expr/analysis/referenced_field_paths.rs
@@ -1,8 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-use std::iter::once;
-
 use vortex_error::VortexResult;
 use vortex_error::vortex_err;
 
@@ -19,10 +17,11 @@ use crate::scalar_fn::fns::get_item::GetItem;
 use crate::scalar_fn::fns::root::Root;
 use crate::scalar_fn::fns::select::Select;
 
-/// Returns a prefix-minimal set of rooted field paths referenced by an expression.
+/// Returns the rooted field paths referenced by an expression.
 ///
-/// A standalone root expression is represented by [`FieldPath::root`], which conservatively
-/// selects all fields. When one referenced path is a prefix of another, only the prefix is returned.
+/// Iterating the returned set (via [`IntoIterator`]) yields the prefix-minimal covering set: when
+/// one referenced path is a prefix of another, only the prefix is kept. A standalone root
+/// expression is represented by [`FieldPath::root`], which conservatively selects all fields.
 /// Scalar functions other than `GetItem` and `Select` conservatively reference each complete child
 /// output.
 pub fn referenced_field_paths(expr: &Expression, scope: &DType) -> VortexResult<FieldPathSet> {
@@ -71,10 +70,14 @@ pub fn referenced_field_paths(expr: &Expression, scope: &DType) -> VortexResult<
 /// Threads the set of currently-requested field paths down the expression tree, narrowing it at
 /// each `GetItem`/`Select`, and records the rooted paths reached at each `Root` leaf.
 ///
-/// Narrowing the requested path is only sound through `GetItem` (a genuine field access) and
-/// `Select` (a genuine column projection). Any other function is opaque—we cannot assume it
-/// preserves a field's provenance—so its children conservatively re-request the whole scope. This
-/// is what keeps an expression like `f($).x` reading every field of `$` rather than just `x`.
+/// Paths are carried reversed so a `GetItem` can `push` its field instead of prepending it; they
+/// are reversed back to rooted order when recorded at a `Root`, and `Select` reads a path's head
+/// from its last element.
+///
+/// Narrowing is only sound through `GetItem` (a genuine field access) and `Select` (a genuine
+/// column projection). Any other function is opaque—we cannot assume it preserves a field's
+/// provenance—so its children conservatively re-request the whole scope, which is what keeps an
+/// expression like `f($).x` reading every field of `$` rather than just `x`.
 struct ReferencedFieldPaths<'a> {
     scope: &'a DType,
     field_paths: FieldPathSet,
@@ -90,27 +93,25 @@ impl NodeFolderContext for ReferencedFieldPaths<'_> {
         requested: &Self::Context,
         node: &Expression,
     ) -> VortexResult<FoldDownContext<Self::Context, ()>> {
-        // A `Root` leaf resolves the requested paths against the actual scope.
         if node.is::<Root>() {
-            self.field_paths.extend_prefixes(requested.iter().cloned());
+            self.field_paths.extend(
+                requested
+                    .iter()
+                    .map(|path| FieldPath::from_iter(path.parts().iter().rev().cloned())),
+            );
             return Ok(FoldDownContext::Skip(()));
         }
 
-        // `GetItem` prepends its field to every requested path before descending into its child.
         if let Some(field_name) = node.as_opt::<GetItem>() {
-            let prefixed = requested
+            let appended = requested
                 .iter()
-                .map(|path| {
-                    FieldPath::from_iter(
-                        once(Field::Name(field_name.clone())).chain(path.parts().iter().cloned()),
-                    )
-                })
+                .map(|path| path.clone().push(Field::Name(field_name.clone())))
                 .collect();
-            return Ok(FoldDownContext::Continue(prefixed));
+            return Ok(FoldDownContext::Continue(appended));
         }
 
-        // `Select` keeps requested paths whose head is included, expanding a whole-scope request
-        // into one path per included field.
+        // Keep requested paths whose head is included, expanding a whole-scope request into one
+        // path per included field.
         if let Some(selection) = node.as_opt::<Select>() {
             let child_dtype = node.child(0).return_dtype(self.scope)?;
             let child_fields = child_dtype
@@ -118,11 +119,11 @@ impl NodeFolderContext for ReferencedFieldPaths<'_> {
                 .ok_or_else(|| vortex_err!("Select child is not a struct"))?;
             let included_fields = selection.normalize_to_included_fields(child_fields.names())?;
 
-            let mut narrowed = Vec::new();
+            let mut narrowed = Vec::with_capacity(requested.len());
             for path in requested {
                 if path.is_root() {
                     narrowed.extend(included_fields.iter().cloned().map(FieldPath::from_name));
-                } else if let Some(Field::Name(field_name)) = path.parts().first()
+                } else if let Some(Field::Name(field_name)) = path.parts().last()
                     && included_fields
                         .iter()
                         .any(|included| included == field_name)
@@ -155,6 +156,8 @@ impl NodeFolderContext for ReferencedFieldPaths<'_> {
 
 #[cfg(test)]
 mod tests {
+    use vortex_utils::aliases::hash_set::HashSet;
+
     use super::*;
     use crate::dtype::Nullability::NonNullable;
     use crate::dtype::PType::I32;
@@ -178,13 +181,20 @@ mod tests {
         )
     }
 
+    /// Collects the prefix-minimal field paths referenced by `expr` against [`scope`].
+    fn referenced(expr: &Expression) -> VortexResult<HashSet<FieldPath>> {
+        Ok(referenced_field_paths(expr, &scope())?
+            .into_iter()
+            .collect())
+    }
+
     #[test]
     fn nested_select_preserves_field_path() -> VortexResult<()> {
         let expr = select(["x"], get_item("a", root()));
 
         assert_eq!(
-            referenced_field_paths(&expr, &scope())?,
-            FieldPathSet::from_iter([FieldPath::from_name("a").push("x")])
+            referenced(&expr)?,
+            HashSet::from_iter([FieldPath::from_name("a").push("x")])
         );
         Ok(())
     }
@@ -194,8 +204,8 @@ mod tests {
         let expr = get_item("x", select(["x", "y"], get_item("a", root())));
 
         assert_eq!(
-            referenced_field_paths(&expr, &scope())?,
-            FieldPathSet::from_iter([FieldPath::from_name("a").push("x")])
+            referenced(&expr)?,
+            HashSet::from_iter([FieldPath::from_name("a").push("x")])
         );
         Ok(())
     }
@@ -205,8 +215,8 @@ mod tests {
         let expr = select_exclude(["y"], get_item("a", root()));
 
         assert_eq!(
-            referenced_field_paths(&expr, &scope())?,
-            FieldPathSet::from_iter([FieldPath::from_name("a").push("x")])
+            referenced(&expr)?,
+            HashSet::from_iter([FieldPath::from_name("a").push("x")])
         );
         Ok(())
     }
@@ -222,8 +232,8 @@ mod tests {
         );
 
         assert_eq!(
-            referenced_field_paths(&expr, &scope())?,
-            FieldPathSet::from_iter([FieldPath::from_name("a")])
+            referenced(&expr)?,
+            HashSet::from_iter([FieldPath::from_name("a")])
         );
         Ok(())
     }
@@ -234,18 +244,15 @@ mod tests {
         // as a scope field access, so the wrapped `root()` conservatively references all fields.
         let expr = get_item("x", pack([("x", root())], NonNullable));
 
-        assert_eq!(
-            referenced_field_paths(&expr, &scope())?,
-            FieldPathSet::from_iter([FieldPath::root()])
-        );
+        assert_eq!(referenced(&expr)?, HashSet::from_iter([FieldPath::root()]));
         Ok(())
     }
 
     #[test]
     fn root_references_all_fields() -> VortexResult<()> {
         assert_eq!(
-            referenced_field_paths(&root(), &scope())?,
-            FieldPathSet::from_iter([FieldPath::root()])
+            referenced(&root())?,
+            HashSet::from_iter([FieldPath::root()])
         );
         Ok(())
     }
diff --git a/vortex-layout/src/scan/scan_builder.rs b/vortex-layout/src/scan/scan_builder.rs
index 550f67e5017..d916003fc78 100644
--- a/vortex-layout/src/scan/scan_builder.rs
+++ b/vortex-layout/src/scan/scan_builder.rs
@@ -419,7 +419,7 @@ pub fn referenced_field_masks(
 
     let mut field_paths = referenced_field_paths(projection, dtype)?;
     if let Some(filter) = filter {
-        field_paths.extend_prefixes(referenced_field_paths(filter, dtype)?);
+        field_paths.extend(referenced_field_paths(filter, dtype)?);
     }
     Ok(field_paths.into_iter().map(FieldMask::Prefix).collect_vec())
 }