WIP: JSON/Variant experiments

encodings/parquet-variant/Cargo.toml

@@ -12,7 +12,7 @@ readme = { workspace = true }
 repository = { workspace = true }
 rust-version = { workspace = true }
 version = { workspace = true }
-publish = false
+publish = true
 
 [lints]
 workspace = true

vortex-btrblocks/Cargo.toml

@@ -14,8 +14,11 @@ rust-version = { workspace = true }
 version = { workspace = true }
 
 [dependencies]
+arrow-array = { workspace = true, optional = true }
+arrow-schema = { workspace = true, optional = true }
 itertools = { workspace = true }
 num-traits = { workspace = true }
+parquet-variant-compute = { workspace = true, optional = true }
 pco = { workspace = true, optional = true }
 rand = { workspace = true }
 rustc-hash = { workspace = true }
@@ -31,9 +34,11 @@ vortex-fastlanes = { workspace = true }
 vortex-fsst = { workspace = true }
 vortex-mask = { workspace = true }
 vortex-onpair = { workspace = true, optional = true }
+vortex-parquet-variant = { workspace = true, optional = true }
 vortex-pco = { workspace = true, optional = true }
 vortex-runend = { workspace = true }
 vortex-sequence = { workspace = true }
+vortex-session = { workspace = true, optional = true }
 vortex-sparse = { workspace = true }
 vortex-tensor = { workspace = true, optional = true }
 vortex-utils = { workspace = true }
@@ -42,22 +47,32 @@ vortex-zstd = { workspace = true, optional = true }
 
 [dev-dependencies]
 divan = { workspace = true }
+rand = { workspace = true }
 rstest = { workspace = true }
 test-with = { workspace = true }
 vortex-array = { workspace = true, features = ["_test-harness"] }
 vortex-session = { workspace = true }
 
 [features]
-# This feature enabled unstable encodings for which we don't guarantee stability.
+
 unstable_encodings = [
     "dep:vortex-tensor",
     "dep:vortex-onpair",
     "vortex-zstd?/unstable_encodings",
 ]
+parquet-variant = [
+    "dep:arrow-array",
+    "dep:arrow-schema",
+    "dep:vortex-parquet-variant",
+    "dep:parquet-variant-compute",
+    "dep:vortex-session",
+    "zstd",
+]
 pco = ["dep:pco", "dep:vortex-pco"]
 zstd = ["dep:vortex-zstd"]
 
 [lints]
+# This feature enabled unstable encodings for which we don't guarantee stability.
 workspace = true
 
 [[bench]]

vortex-btrblocks/src/builder.rs

@@ -65,6 +65,7 @@ pub const ALL_SCHEMES: &[&dyn Scheme] = &[
     // Binary schemes.
     ////////////////////////////////////////////////////////////////////////////////////////////////
     &binary::BinaryDictScheme,
+    &binary::BinaryFSSTScheme,
     &binary::BinaryConstantScheme,
     // Decimal schemes.
     &decimal::DecimalScheme,
@@ -146,7 +147,9 @@ impl BtrBlocksCompressorBuilder {
     /// Panics if any of the compact schemes are already present.
     #[cfg(feature = "zstd")]
     pub fn with_compact(self) -> Self {
-        let builder = self.with_new_scheme(&string::ZstdScheme);
+        let builder = self
+            .with_new_scheme(&string::ZstdScheme)
+            .with_new_scheme(&binary::BinaryZstdScheme);
 
         #[cfg(feature = "pco")]
         let builder = builder
@@ -191,15 +194,20 @@ impl BtrBlocksCompressorBuilder {
             string::StringDictScheme.id(),
             string::FSSTScheme.id(),
             binary::BinaryDictScheme.id(),
+            binary::BinaryFSSTScheme.id(),
         ];
         #[cfg(feature = "unstable_encodings")]
         excluded.push(string::OnPairScheme.id());
         let builder = self.exclude_schemes(excluded);
 
         #[cfg(all(feature = "zstd", feature = "unstable_encodings"))]
-        let builder = builder.with_new_scheme(&string::ZstdBuffersScheme);
+        let builder = builder
+            .with_new_scheme(&string::ZstdBuffersScheme)
+            .with_new_scheme(&binary::BinaryZstdBuffersScheme);
         #[cfg(all(feature = "zstd", not(feature = "unstable_encodings")))]
-        let builder = builder.with_new_scheme(&string::ZstdScheme);
+        let builder = builder
+            .with_new_scheme(&string::ZstdScheme)
+            .with_new_scheme(&binary::BinaryZstdScheme);
 
         builder
     }

vortex-btrblocks/src/lib.rs

@@ -58,6 +58,8 @@ mod builder;
 mod canonical_compressor;
 /// Compression scheme implementations.
 pub mod schemes;
+#[cfg(feature = "parquet-variant")]
+pub mod variant;
 
 // Re-export framework types from vortex-compressor for backwards compatibility.
 // Btrblocks-specific exports.