Add support for bytes key btree indices

crates/core/src/host/instance_env.rs

@@ -23,7 +23,7 @@ use spacetimedb_lib::{http as st_http, ConnectionId, Identity, Timestamp};
 use spacetimedb_primitives::{ColId, ColList, IndexId, TableId};
 use spacetimedb_sats::{
     bsatn::{self, ToBsatn},
-    buffer::{CountWriter, TeeWriter},
+    buffer::CountWriter,
     AlgebraicValue, ProductValue,
 };
 use spacetimedb_schema::identifier::Identifier;
@@ -343,16 +343,16 @@ impl InstanceEnv {
     fn project_cols_bsatn(buffer: &mut [u8], cols: ColList, row_ref: RowRef<'_>) -> usize {
         // We get back a col-list with the columns with generated values.
         // Write those back to `buffer` and then the encoded length to `row_len`.
-        let counter = CountWriter::default();
-        let mut writer = TeeWriter::new(counter, buffer);
-        for col in cols.iter() {
-            // Read the column value to AV and then serialize.
-            let val = row_ref
-                .read_col::<AlgebraicValue>(col)
-                .expect("reading col as AV never panics");
-            bsatn::to_writer(&mut writer, &val).unwrap();
-        }
-        writer.w1.finish()
+        let (_, count) = CountWriter::run(buffer, |writer| {
+            for col in cols.iter() {
+                // Read the column value to AV and then serialize.
+                let val = row_ref
+                    .read_col::<AlgebraicValue>(col)
+                    .expect("reading col as AV never panics");
+                bsatn::to_writer(writer, &val).unwrap();
+            }
+        });
+        count
     }
 
     pub fn insert(&self, table_id: TableId, buffer: &mut [u8]) -> Result<usize, NodesError> {

crates/datastore/src/locking_tx_datastore/committed_state.rs

@@ -888,7 +888,8 @@ impl CommittedState {
 
             let index = table.new_index(&algo, is_unique)?;
             // SAFETY: `index` was derived from `table`.
-            unsafe { table.insert_index(blob_store, index_id, index) };
+            unsafe { table.insert_index(blob_store, index_id, index) }
+                .expect("rebuilding should not cause constraint violations");
             index_id_map.insert(index_id, table_id);
         }
         Ok(())

crates/datastore/src/locking_tx_datastore/mut_tx.rs

@@ -1482,6 +1482,7 @@ impl MutTxId {
                     .map_err(|IndexCannotSeekRange| IndexError::IndexCannotSeekRange(index_id))?;
                 IndexScanPointOrRange::Range(iter)
             }
+            PointOrRange::Unsupported => return Err(IndexError::IndexCannotSeekRange(index_id).into()),
         };
         Ok((table_id, iter))
     }

crates/sats/src/algebraic_value/de.rs

@@ -1,5 +1,5 @@
 use crate::array_value::{ArrayValueIntoIter, ArrayValueIterCloned};
-use crate::{de, AlgebraicValue, SumValue};
+use crate::{de, AlgebraicValue, ProductValue, SumValue};
 use crate::{i256, u256};
 use derive_more::From;
 
@@ -23,6 +23,11 @@ impl ValueDeserializer {
         // SAFETY: The conversion is OK due to `repr(transparent)`.
         unsafe { &*(val as *const AlgebraicValue as *const ValueDeserializer) }
     }
+
+    pub fn from_product_ref(prod: &ProductValue) -> RefProductAccess<'_> {
+        let vals = prod.elements.iter();
+        RefProductAccess { vals }
+    }
 }
 
 /// Errors that can occur when deserializing the `AlgebraicValue`.
@@ -348,7 +353,7 @@ impl<'de> de::Deserializer<'de> for &'de ValueDeserializer {
 }
 
 /// Defines deserialization for [`&'de ValueDeserializer`] where product elements are in the input.
-struct RefProductAccess<'a> {
+pub struct RefProductAccess<'a> {
     /// The element values of the product as an iterator of borrowed values.
     vals: std::slice::Iter<'a, AlgebraicValue>,
 }

crates/sats/src/buffer.rs

@@ -329,6 +329,14 @@ pub struct CountWriter {
 }
 
 impl CountWriter {
+    /// Run `work` on `writer`, but also count the number of bytes written.
+    pub fn run<W: BufWriter, R>(writer: W, work: impl FnOnce(&mut TeeWriter<W, CountWriter>) -> R) -> (R, usize) {
+        let counter = Self::default();
+        let mut writer = TeeWriter::new(writer, counter);
+        let ret = work(&mut writer);
+        (ret, writer.w2.finish())
+    }
+
     /// Consumes the counter and returns the final count.
     pub fn finish(self) -> usize {
         self.num_bytes

crates/sats/src/de/impls.rs

@@ -259,6 +259,17 @@ impl_deserialize!(
     de => Vec::<T>::validate(de)
 );
 
+/// The visitor merely valiates the slice.
+struct ValidatingSliceVisitor;
+
+impl<T: ToOwned + ?Sized> SliceVisitor<'_, T> for ValidatingSliceVisitor {
+    type Output = ();
+
+    fn visit<E: Error>(self, _: &T) -> Result<Self::Output, E> {
+        Ok(())
+    }
+}
+
 /// The visitor converts the slice to its owned version.
 struct OwnedSliceVisitor;
 
@@ -293,8 +304,16 @@ impl<const N: usize> SliceVisitor<'_, [u8]> for ByteArrayVisitor<N> {
     }
 }
 
-impl_deserialize!([] &'de str, de => de.deserialize_str(BorrowedSliceVisitor));
-impl_deserialize!([] &'de [u8], de => de.deserialize_bytes(BorrowedSliceVisitor));
+impl_deserialize!(
+    [] &'de str,
+    de => de.deserialize_str_slice(),
+    de => de.deserialize_str(ValidatingSliceVisitor)
+);
+impl_deserialize!(
+    [] &'de [u8],
+    de => de.deserialize_bytes(BorrowedSliceVisitor),
+    de => de.deserialize_bytes(ValidatingSliceVisitor)
+);
 
 /// The visitor returns the slice as-is and borrowed.
 pub(crate) struct BorrowedSliceVisitor;
@@ -610,7 +629,7 @@ impl<'de> DeserializeSeed<'de> for WithTypespace<'_, AlgebraicType> {
             AlgebraicType::U256 => u256::validate(de),
             AlgebraicType::F32 => f32::validate(de),
             AlgebraicType::F64 => f64::validate(de),
-            AlgebraicType::String => <Box<str>>::validate(de),
+            AlgebraicType::String => <&str>::validate(de),
         }
     }
 }

crates/sats/src/product_value.rs

@@ -69,6 +69,14 @@ impl From<ColId> for InvalidFieldError {
 }
 
 impl ProductValue {
+    /// Pushes a single value to he product.
+    pub fn push(self, val: impl Into<AlgebraicValue>) -> Self {
+        let mut vals: Vec<_> = self.elements.into();
+        vals.reserve(1);
+        vals.push(val.into());
+        Self::from(vals)
+    }
+
     /// Borrow the value at field of `self` identified by `col_pos`.
     ///
     /// The `name` is non-functional and is only used for error-messages.

crates/table/benches/page_manager.rs

@@ -763,7 +763,7 @@ fn make_table_with_index<R: IndexedRow>(unique: bool) -> (Table, IndexId) {
     let algo = BTreeAlgorithm { columns: cols }.into();
     let idx = tbl.new_index(&algo, unique).unwrap();
     // SAFETY: index was derived from the table.
-    unsafe { tbl.insert_index(&NullBlobStore, index_id, idx) };
+    unsafe { tbl.insert_index(&NullBlobStore, index_id, idx) }.unwrap();
 
     (tbl, index_id)
 }

crates/table/proptest-regressions/table.txt

@@ -12,3 +12,4 @@ cc 1f295db61a02ac3378f5ffcceb084637d2391bcc1758af6fb2df8355a713e998 # shrinks to
 cc 776d142680b35d7dad5b558fea7071b095f7e6a23c8549e9b32b452d5eebf92b # shrinks to (ty, val) = (ProductType { elements: [ProductTypeElement { name: None, algebraic_type: Builtin(String) }] }, ProductValue { elements: [String("\u{16af0}a®ਲ𒒀A 𑌅 ಎ꒐𑍇A A𐫫Aⷀ𑌵ૠ\u{b55} aㄱ \u{f99}a ")] })
 cc 66d99531b8e513d0fd558f492f708d110e1e117dfc7f3f42188bcc57c23bb89e # shrinks to (ty, val) = (ProductType { elements: [ProductTypeElement { name: None, algebraic_type: Builtin(Map(MapType { key_ty: Builtin(U8), ty: Builtin(Map(MapType { key_ty: Builtin(I32), ty: Builtin(F32) })) })) }] }, ProductValue { elements: [Map({U8(0): Map({I32(-4): F32(Total(0.0)), I32(-3): F32(Total(0.0)), I32(-2): F32(Total(-0.0)), I32(-1): F32(Total(-0.0)), I32(0): F32(Total(0.0)), I32(1): F32(Total(0.0)), I32(2): F32(Total(0.0)), I32(3): F32(Total(0.0))}), U8(1): Map({I32(-5): F32(Total(0.0)), I32(-4): F32(Total(0.0)), I32(-3): F32(Total(0.0)), I32(-2): F32(Total(0.0)), I32(-1): F32(Total(-0.0)), I32(0): F32(Total(0.0)), I32(1): F32(Total(0.0)), I32(2): F32(Total(0.0)), I32(3): F32(Total(0.0)), I32(4): F32(Total(0.0)), I32(5): F32(Total(0.0)), I32(6): F32(Total(0.0)), I32(7): F32(Total(0.0))}), U8(2): Map({I32(-3): F32(Total(-0.0)), I32(-2): F32(Total(0.0)), I32(-1): F32(Total(0.0)), I32(0): F32(Total(-0.0)), I32(1): F32(Total(0.0)), I32(2): F32(Total(0.0)), I32(3): F32(Total(0.0))}), U8(3): Map({I32(-10): F32(Total(0.0)), I32(-9): F32(Total(0.0)), I32(-8): F32(Total(-0.0)), I32(-7): F32(Total(0.0)), I32(-6): F32(Total(0.0)), I32(-5): F32(Total(0.0)), I32(-4): F32(Total(0.0)), I32(-3): F32(Total(0.0)), I32(-2): F32(Total(0.0)), I32(-1): F32(Total(0.0)), I32(0): F32(Total(0.0)), I32(1): F32(Total(0.0)), I32(2): F32(Total(0.0))}), U8(4): Map({I32(-7): F32(Total(0.0)), I32(-6): F32(Total(0.0)), I32(-5): F32(Total(0.0)), I32(-4): F32(Total(0.0)), I32(-3): F32(Total(0.0)), I32(-2): F32(Total(0.0)), I32(-1): F32(Total(0.0)), I32(0): F32(Total(0.0)), I32(1): F32(Total(0.0)), I32(2): F32(Total(0.0)), I32(3): F32(Total(0.0))}), U8(5): Map({I32(-9): F32(Total(0.0)), I32(-8): F32(Total(0.0)), I32(-7): F32(Total(0.0)), I32(-6): F32(Total(0.0)), I32(-5): F32(Total(0.0)), I32(-4): F32(Total(0.0)), I32(-3): F32(Total(0.0)), I32(-2): F32(Total(0.0)), I32(-1): F32(Total(0.0)), I32(0): F32(Total(0.0)), I32(1): F32(Total(0.0)), I32(2): F32(Total(0.0)), I32(3): F32(Total(0.0)), I32(4): F32(Total(0.0)), I32(5): F32(Total(0.0))}), U8(6): Map({I32(0): F32(Total(0.0)), I32(1): F32(Total(0.0)), I32(2): F32(Total(0.0))}), U8(7): Map({I32(-4): F32(Total(0.0)), I32(-3): F32(Total(0.0)), I32(-2): F32(Total(0.0)), I32(-1): F32(Total(0.0)), I32(0): F32(Total(0.0)), I32(1): F32(Total(0.0)), I32(2): F32(Total(-0.0)), I32(3): F32(Total(0.0))}), U8(8): Map({I32(-7): F32(Total(0.0)), I32(-6): F32(Total(-0.0)), I32(-5): F32(Total(0.0)), I32(-4): F32(Total(0.0)), I32(-3): F32(Total(0.0)), I32(-2): F32(Total(0.0)), I32(-1): F32(Total(0.0)), I32(0): F32(Total(0.0)), I32(1): F32(Total(0.0)), I32(2): F32(Total(0.0)), I32(3): F32(Total(0.0)), I32(4): F32(Total(0.0)), I32(5): F32(Total(0.0)), I32(6): F32(Total(-0.0)), I32(7): F32(Total(0.0))}), U8(9): Map({I32(-1349171619): F32(Total(418648100.0)), I32(-665792478): F32(Total(-5.3081414e23)), I32(-1): F32(Total(0.0)), I32(0): F32(Total(0.0)), I32(1): F32(Total(0.0)), I32(2): F32(Total(0.0)), I32(3): F32(Total(0.0)), I32(5): F32(Total(-0.0)), I32(906732021): F32(Total(1.952517e16)), I32(1965197035): F32(Total(1020.84216))}), U8(11): Map({I32(-7): F32(Total(0.0)), I32(-6): F32(Total(0.0)), I32(-5): F32(Total(0.0)), I32(-4): F32(Total(0.0)), I32(-3): F32(Total(0.0)), I32(-2): F32(Total(0.0)), I32(-1): F32(Total(0.0)), I32(0): F32(Total(0.0)), I32(1): F32(Total(0.0)), I32(2): F32(Total(0.0)), I32(3): F32(Total(0.0)), I32(4): F32(Total(0.0)), I32(5): F32(Total(0.0)), I32(6): F32(Total(0.0))})})] })
 cc 7f478c4dd0f24e715a74949c6d06af8ca2b4c8b82fae4f53c953a2b323cff851 # shrinks to (ty, val) = (ProductType { elements: [ProductTypeElement { name: None, algebraic_type: Builtin(Array(ArrayType { elem_ty: Builtin(Map(MapType { key_ty: Builtin(U64), ty: Builtin(Bool) })) })) }] }, ProductValue { elements: [Array([{U64(0): Bool(false), U64(1): Bool(false), U64(2): Bool(false), U64(3): Bool(false), U64(4): Bool(false), U64(5): Bool(false), U64(6): Bool(false), U64(7): Bool(false), U64(8): Bool(false), U64(9): Bool(false), U64(10): Bool(false), U64(11): Bool(false), U64(12): Bool(false), U64(13): Bool(false)}, {U64(0): Bool(false), U64(1): Bool(false), U64(2): Bool(false), U64(3): Bool(false), U64(4): Bool(false), U64(5): Bool(false), U64(6): Bool(false), U64(7): Bool(false), U64(8): Bool(false), U64(9): Bool(false)}, {U64(0): Bool(false), U64(1): Bool(false), U64(2): Bool(false), U64(3): Bool(false), U64(4): Bool(false), U64(5): Bool(false), U64(6): Bool(false), U64(7): Bool(false), U64(8): Bool(false), U64(9): Bool(false), U64(10): Bool(false), U64(11): Bool(false)}, {U64(0): Bool(false), U64(1): Bool(false), U64(2): Bool(false), U64(3): Bool(false), U64(4): Bool(false), U64(5): Bool(false), U64(6): Bool(false)}, {U64(0): Bool(false), U64(1): Bool(false), U64(2): Bool(false), U64(3): Bool(false), U64(4): Bool(false), U64(5): Bool(false), U64(6): Bool(false), U64(7): Bool(false), U64(8): Bool(false), U64(9): Bool(false), U64(10): Bool(false), U64(11): Bool(false), U64(12): Bool(false)}, {U64(0): Bool(false), U64(1): Bool(false), U64(2): Bool(false), U64(3): Bool(false), U64(4): Bool(false), U64(5): Bool(false), U64(6): Bool(false), U64(7): Bool(false), U64(8): Bool(false), U64(9): Bool(false)}, {U64(0): Bool(false), U64(1): Bool(false), U64(2): Bool(false), U64(3): Bool(false)}, {U64(0): Bool(false), U64(1): Bool(false), U64(2): Bool(false), U64(3): Bool(false), U64(4): Bool(false), U64(5): Bool(false), U64(6): Bool(false), U64(7): Bool(false), U64(8): Bool(false), U64(9): Bool(false), U64(10): Bool(false)}, {U64(0): Bool(false), U64(1): Bool(false), U64(2): Bool(false), U64(3): Bool(false), U64(4): Bool(false), U64(5): Bool(false), U64(6): Bool(false), U64(7): Bool(false), U64(8): Bool(false), U64(9): Bool(false), U64(10): Bool(false)}, {U64(0): Bool(false), U64(1): Bool(false), U64(2): Bool(false), U64(3): Bool(false), U64(4): Bool(false)}, {U64(0): Bool(false), U64(1): Bool(false), U64(2): Bool(false), U64(3): Bool(false), U64(4): Bool(false), U64(5): Bool(false)}, {U64(0): Bool(false), U64(1): Bool(false)}])] })
+cc 01bfd4449bee7eaa0b61b60792baed8d52d3589f4a5bb313bf057194a6248a83

crates/table/proptest-regressions/table_index/mod.txt

@@ -10,3 +10,5 @@ cc c1e4c959a32f6ab8ef9c4e29d39a24ec47cb03524584606a7f1fa4563f0f8cca # shrinks to
 cc 4cb325be8b24c9efa5b1f20b9504d044d9dd110eb9e99355de4ca42f9cfc20b4 # shrinks to (ty, cols, pv) = (ProductType {None: Sum(SumType {"variant_0": Product(ProductType {})})}, [ColId(0)], ProductValue { elements: [Sum(SumValue { tag: 0, value: Product(ProductValue { elements: [] }) })] }), is_unique = false
 cc a166a3c619c7cae3938f4e0cfb4e7a96cddfbb7943efd0b74e8cbb99d7a1e6a8 # shrinks to (ty, cols, pv) = (ProductType {None: U8}, [ColId(0)], ProductValue { elements: [U8(0)] }), kind = Direct
 cc 05390c104810e7086fa5d3f3cac7f491a377ae6ba64431661fd94662e28d1fca # shrinks to (ty, cols, pv) = (ProductType {None: Sum(SumType {"variant_0": Product(ProductType {})})}, [ColId(0)], ProductValue { elements: [Sum(SumValue { tag: 0, value: Product(ProductValue { elements: [] }) })] }), kind = Direct
+cc 3b8115315ff39f3268c02ccd659b82444e6d0b12aae7e6c0feba956875dab5ab # shrinks to is_unique = false, (prefix_ty, prefix_val) = (Array(ArrayType { elem_ty: Product(ProductType {"field_0": String}) }), Array([ProductValue { elements: [String("")] }])), (middle_ty, [start, middle, end]) = (Bool, [Bool(false), Bool(false), Bool(false)]), (suffix_ty, suffix_val) = (Bool, Bool(false))
+cc aa8f01aec687cbe6ad36acc77c8c484ccd323a53dca132c6eb490104563d17af # shrinks to is_unique = false, index_kind = BTree, (prefix_ty, prefix_val) = (Bool, Bool(false)), (middle_ty, [included, excluded]) = (Product(ProductType {}), [Product(ProductValue { elements: [] }), Product(ProductValue { elements: [] })]), (suffix_ty, suffix_val) = (Bool, Bool(false))