Introduce a new JSON extension type in vortex-json

Cargo.toml

@@ -13,6 +13,7 @@ members = [
     "vortex-proto",
     "vortex-array",
     "vortex-tensor",
+    "vortex-json",
     "vortex-turboquant",
     "vortex-compressor",
     "vortex-btrblocks",
@@ -288,6 +289,7 @@ vortex-flatbuffers = { version = "0.1.0", path = "./vortex-flatbuffers", default
 vortex-fsst = { version = "0.1.0", path = "./encodings/fsst", default-features = false }
 vortex-io = { version = "0.1.0", path = "./vortex-io", default-features = false }
 vortex-ipc = { version = "0.1.0", path = "./vortex-ipc", default-features = false }
+vortex-json = { version = "0.1.0", path = "./vortex-json", default-features = false }
 vortex-layout = { version = "0.1.0", path = "./vortex-layout", default-features = false }
 vortex-mask = { version = "0.1.0", path = "./vortex-mask", default-features = false }
 vortex-metrics = { version = "0.1.0", path = "./vortex-metrics", default-features = false }

benchmarks/vector-search-bench/src/expression.rs

@@ -19,11 +19,11 @@
 //! size.
 
 use anyhow::Result;
+use vortex::array::EmptyMetadata;
 use vortex::array::expr::Expression;
 use vortex::array::expr::col;
 use vortex::array::expr::gt;
 use vortex::array::expr::lit;
-use vortex::array::extension::EmptyMetadata;
 use vortex::array::scalar::Scalar;
 use vortex::array::scalar_fn::EmptyOptions;
 use vortex::array::scalar_fn::ScalarFnVTableExt;

benchmarks/vector-search-bench/src/ingest.rs

@@ -21,6 +21,7 @@ use anyhow::Result;
 use anyhow::bail;
 use anyhow::ensure;
 use vortex::array::ArrayRef;
+use vortex::array::EmptyMetadata;
 use vortex::array::ExecutionCtx;
 use vortex::array::IntoArray;
 use vortex::array::arrays::ExtensionArray;
@@ -31,7 +32,6 @@ use vortex::array::arrays::StructArray;
 use vortex::array::arrays::extension::ExtensionArrayExt;
 use vortex::array::arrays::fixed_size_list::FixedSizeListArrayExt;
 use vortex::array::arrays::struct_::StructArrayExt;
-use vortex::array::extension::EmptyMetadata;
 use vortex::array::validity::Validity;
 use vortex::buffer::Buffer;
 use vortex::dtype::DType;

vortex-array/src/arrays/extension/compute/rules.rs

@@ -85,6 +85,7 @@ mod tests {
     use vortex_mask::Mask;
     use vortex_session::VortexSession;
 
+    use crate::EmptyMetadata;
     use crate::IntoArray;
     #[expect(deprecated)]
     use crate::ToCanonical as _;
@@ -105,7 +106,6 @@ mod tests {
     use crate::dtype::extension::ExtDTypeRef;
     use crate::dtype::extension::ExtId;
     use crate::dtype::extension::ExtVTable;
-    use crate::extension::EmptyMetadata;
     use crate::optimizer::ArrayOptimizer;
     use crate::scalar::Scalar;
     use crate::scalar::ScalarValue;

vortex-array/src/extension/mod.rs

@@ -3,19 +3,8 @@
 
 //! Extension types.
 
-use std::fmt;
-
 pub mod datetime;
 pub mod uuid;
 
 #[cfg(test)]
 mod tests;
-
-/// An empty metadata struct for extension dtypes that do not require any metadata.
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct EmptyMetadata;
-impl fmt::Display for EmptyMetadata {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "")
-    }
-}

vortex-array/src/metadata.rs

@@ -26,7 +26,7 @@ where
 }
 
 /// Empty array metadata
-#[derive(Debug)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct EmptyMetadata;
 
 impl SerializeMetadata for EmptyMetadata {
@@ -46,6 +46,12 @@ impl DeserializeMetadata for EmptyMetadata {
     }
 }
 
+impl std::fmt::Display for EmptyMetadata {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "")
+    }
+}
+
 /// A utility wrapper for raw metadata serialization. This delegates the serialiation step
 /// to the arrays' vtable.
 pub struct RawMetadata(pub Vec<u8>);

vortex-array/src/scalar/typed_view/extension/tests.rs

@@ -4,13 +4,13 @@
 use vortex_error::VortexResult;
 use vortex_error::vortex_bail;
 
+use crate::EmptyMetadata;
 use crate::dtype::DType;
 use crate::dtype::Nullability;
 use crate::dtype::PType;
 use crate::dtype::extension::ExtDType;
 use crate::dtype::extension::ExtId;
 use crate::dtype::extension::ExtVTable;
-use crate::extension::EmptyMetadata;
 use crate::scalar::Scalar;
 use crate::scalar::ScalarValue;
 

vortex-bench/src/vector_dataset/convert.rs

@@ -4,6 +4,7 @@
 // TODO(connor): Should we re-export this through `conversions.rs`?
 
 use vortex::array::ArrayRef;
+use vortex::array::EmptyMetadata;
 use vortex::array::IntoArray;
 use vortex::array::arrays::Chunked;
 use vortex::array::arrays::ChunkedArray;
@@ -23,7 +24,6 @@ use vortex::error::VortexExpect;
 use vortex::error::VortexResult;
 use vortex::error::vortex_bail;
 use vortex::error::vortex_err;
-use vortex::extension::EmptyMetadata;
 use vortex_tensor::vector::Vector;
 
 /// Rewrap a list-of-float column as a [`vortex_tensor::vector::Vector`] extension array.

vortex-duckdb/src/convert/dtype.rs

@@ -324,6 +324,7 @@ mod tests {
     use std::sync::Arc;
 
     use rstest::rstest;
+    use vortex::array::EmptyMetadata;
     use vortex::dtype::DType;
     use vortex::dtype::FieldName;
     use vortex::dtype::FieldNames;
@@ -334,7 +335,6 @@ mod tests {
     use vortex::dtype::extension::ExtId;
     use vortex::dtype::extension::ExtVTable;
     use vortex::error::VortexResult;
-    use vortex::extension::EmptyMetadata;
     use vortex::extension::datetime::Date;
     use vortex::extension::datetime::Time;
     use vortex::extension::datetime::Timestamp;

vortex-json/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "vortex-json"
+authors = { workspace = true }
+categories = { workspace = true }
+description = "Vortex JSON extension type"
+edition = { workspace = true }
+homepage = { workspace = true }
+include = { workspace = true }
+keywords = { workspace = true }
+license = { workspace = true }
+readme = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[lints]
+workspace = true
+
+[dependencies]
+vortex-array = { workspace = true, default-features = false }
+vortex-error = { workspace = true, default-features = false }

vortex-json/src/dtype.rs

@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+//! Extension dtype definition for the JSON type
+
+use vortex_array::EmptyMetadata;
+use vortex_array::dtype::extension::ExtDType;
+use vortex_array::dtype::extension::ExtId;
+use vortex_array::dtype::extension::ExtVTable;
+use vortex_array::scalar::ScalarValue;
+use vortex_error::VortexResult;
+use vortex_error::vortex_bail;
+use vortex_error::vortex_ensure;
+
+/// JSON logical type backed by UTF-8 string storage.
+#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
+pub struct Json;
+
+impl ExtVTable for Json {
+    type Metadata = EmptyMetadata;
+    type NativeValue<'a> = &'a str;
+
+    fn id(&self) -> ExtId {
+        ExtId::new("vortex.json")
+    }
+
+    fn serialize_metadata(&self, _metadata: &Self::Metadata) -> VortexResult<Vec<u8>> {
+        Ok(vec![])
+    }
+
+    fn deserialize_metadata(&self, metadata: &[u8]) -> VortexResult<Self::Metadata> {
+        vortex_ensure!(metadata.is_empty(), "JSON metadata must be empty");
+        Ok(EmptyMetadata)
+    }
+
+    fn validate_dtype(ext_dtype: &ExtDType<Self>) -> VortexResult<()> {
+        vortex_ensure!(
+            ext_dtype.storage_dtype().is_utf8(),
+            "JSON storage dtype must be utf8, got {}",
+            ext_dtype.storage_dtype()
+        );
+        Ok(())
+    }
+
+    fn unpack_native<'a>(
+        _ext_dtype: &'a ExtDType<Self>,
+        storage_value: &'a ScalarValue,
+    ) -> VortexResult<Self::NativeValue<'a>> {
+        let ScalarValue::Utf8(value) = storage_value else {
+            vortex_bail!("JSON storage scalar must be utf8, got {storage_value}");
+        };
+        Ok(value.as_str())
+    }
+}

vortex-json/src/lib.rs

@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+#![warn(missing_docs)]
+#![warn(clippy::missing_docs_in_private_items)]
+#![warn(clippy::missing_errors_doc)]
+#![warn(clippy::missing_panics_doc)]
+#![warn(clippy::missing_safety_doc)]
+
+//! Extension type and related functionality for a JSON extension type for Vortex.
+
+mod dtype;
+
+pub use dtype::Json;

vortex-tensor/src/encodings/turboquant/mod.rs

@@ -95,7 +95,7 @@
 //! use vortex_array::arrays::ExtensionArray;
 //! use vortex_array::arrays::FixedSizeListArray;
 //! use vortex_array::arrays::PrimitiveArray;
-//! use vortex_array::extension::EmptyMetadata;
+//! use vortex_array::EmptyMetadata;
 //! use vortex_array::session::ArraySession;
 //! use vortex_array::validity::Validity;
 //! use vortex_buffer::BufferMut;

vortex-tensor/src/encodings/turboquant/tests/structural.rs

@@ -224,10 +224,10 @@ fn dot_product_quantized_accuracy() -> VortexResult<()> {
     reason = "test uses known small dimensions"
 )]
 fn sorf_transform_roundtrip_isolation() -> VortexResult<()> {
+    use vortex_array::EmptyMetadata;
     use vortex_array::IntoArray;
     use vortex_array::arrays::dict::DictArray;
     use vortex_array::dtype::extension::ExtDType;
-    use vortex_array::extension::EmptyMetadata;
     use vortex_array::validity::Validity;
     use vortex_buffer::BufferMut;
 

vortex-tensor/src/scalar_fns/l2_norm.rs

@@ -254,6 +254,7 @@ mod tests {
     use rstest::rstest;
     use vortex_array::ArrayPlugin;
     use vortex_array::ArrayRef;
+    use vortex_array::EmptyMetadata;
     use vortex_array::IntoArray;
     use vortex_array::VortexSessionExecute;
     use vortex_array::arrays::Constant;
@@ -266,7 +267,6 @@ mod tests {
     use vortex_array::dtype::Nullability;
     use vortex_array::dtype::PType;
     use vortex_array::dtype::extension::ExtDType;
-    use vortex_array::extension::EmptyMetadata;
     use vortex_array::scalar::Scalar;
     use vortex_array::validity::Validity;
     use vortex_error::VortexResult;

vortex-tensor/src/scalar_fns/sorf_transform/tests.rs

@@ -9,6 +9,7 @@ use std::sync::Arc;
 
 use vortex_array::ArrayPlugin;
 use vortex_array::ArrayRef;
+use vortex_array::EmptyMetadata;
 use vortex_array::IntoArray;
 use vortex_array::VortexSessionExecute;
 use vortex_array::arrays::ExtensionArray;
@@ -22,7 +23,6 @@ use vortex_array::dtype::DType;
 use vortex_array::dtype::Nullability;
 use vortex_array::dtype::PType;
 use vortex_array::dtype::extension::ExtDType;
-use vortex_array::extension::EmptyMetadata;
 use vortex_array::validity::Validity;
 use vortex_buffer::Buffer;
 use vortex_buffer::BufferMut;

vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs

@@ -9,6 +9,7 @@ use num_traits::Float;
 use num_traits::FromPrimitive;
 use prost::Message;
 use vortex_array::ArrayRef;
+use vortex_array::EmptyMetadata;
 use vortex_array::ExecutionCtx;
 use vortex_array::IntoArray;
 use vortex_array::arrays::ExtensionArray;
@@ -28,7 +29,6 @@ use vortex_array::dtype::PType;
 use vortex_array::dtype::extension::ExtDType;
 use vortex_array::dtype::proto::dtype as pb;
 use vortex_array::expr::Expression;
-use vortex_array::extension::EmptyMetadata;
 use vortex_array::match_each_float_ptype;
 use vortex_array::scalar_fn::Arity;
 use vortex_array::scalar_fn::ChildName;

vortex-tensor/src/types/fixed_shape_tensor/matcher.rs

@@ -90,11 +90,11 @@ impl FixedShapeTensorMatcherMetadata<'_> {
 mod tests {
     use std::sync::Arc;
 
+    use vortex_array::EmptyMetadata;
     use vortex_array::dtype::DType;
     use vortex_array::dtype::Nullability;
     use vortex_array::dtype::PType;
     use vortex_array::dtype::extension::ExtDType;
-    use vortex_array::extension::EmptyMetadata;
     use vortex_error::VortexResult;
 
     use super::*;

vortex-tensor/src/types/vector/arrow.rs

@@ -15,6 +15,7 @@ use arrow_schema::DataType;
 use arrow_schema::Field;
 use arrow_schema::extension::EXTENSION_TYPE_NAME_KEY;
 use vortex_array::ArrayRef;
+use vortex_array::EmptyMetadata;
 use vortex_array::ExecutionCtx;
 use vortex_array::IntoArray;
 use vortex_array::arrays::ExtensionArray;
@@ -30,7 +31,6 @@ use vortex_array::dtype::DType;
 use vortex_array::dtype::arrow::FromArrowType;
 use vortex_array::dtype::extension::ExtDType;
 use vortex_array::dtype::extension::ExtVTable;
-use vortex_array::extension::EmptyMetadata;
 use vortex_error::VortexResult;
 use vortex_session::registry::CachedId;
 use vortex_session::registry::Id;
@@ -174,6 +174,7 @@ mod tests {
     use arrow_array::Float32Array;
     use arrow_array::Int32Array;
     use arrow_schema::Field;
+    use vortex_array::EmptyMetadata;
     use vortex_array::IntoArray;
     use vortex_array::VortexSessionExecute;
     use vortex_array::arrays::FixedSizeListArray;
@@ -188,7 +189,6 @@ mod tests {
     use vortex_array::dtype::PType;
     use vortex_array::dtype::StructFields;
     use vortex_array::dtype::extension::ExtDType;
-    use vortex_array::extension::EmptyMetadata;
     use vortex_array::validity::Validity;
     use vortex_buffer::buffer;
 

vortex-tensor/src/types/vector/matcher.rs

@@ -93,11 +93,11 @@ impl VectorMatcherMetadata {
 mod tests {
     use std::sync::Arc;
 
+    use vortex_array::EmptyMetadata;
     use vortex_array::dtype::DType;
     use vortex_array::dtype::Nullability;
     use vortex_array::dtype::PType;
     use vortex_array::dtype::extension::ExtDType;
-    use vortex_array::extension::EmptyMetadata;
     use vortex_error::VortexResult;
 
     use super::*;

vortex-tensor/src/types/vector/mod.rs

@@ -4,13 +4,13 @@
 //! Vector extension type for fixed-length float vectors (e.g., embeddings).
 
 use vortex_array::ArrayRef;
+use vortex_array::EmptyMetadata;
 use vortex_array::IntoArray;
 use vortex_array::arrays::ConstantArray;
 use vortex_array::arrays::ExtensionArray;
 use vortex_array::dtype::DType;
 use vortex_array::dtype::NativePType;
 use vortex_array::dtype::Nullability;
-use vortex_array::extension::EmptyMetadata;
 use vortex_array::scalar::PValue;
 use vortex_array::scalar::Scalar;
 use vortex_error::VortexResult;