Support fine-grained SVS index

src/VecSim/algorithms/svs/svs_extensions.h

@@ -10,18 +10,28 @@
 #pragma once
 #include "VecSim/algorithms/svs/svs_utils.h"
 #include "svs/extensions/vamana/scalar.h"
+// Tells SQDataset that the concurrent SegmentedBlocked tag is a blocked allocator, so
+// that resize()/compact() are not constrained away. Without it a scalar-quantized
+// concurrent index fails to compile the moment the index grows.
+#include "svs/concurrent/extensions/scalar.h"
 
 #if HAVE_SVS_LVQ
 #include SVS_LVQ_HEADER
 #include SVS_LEANVEC_HEADER
+// The same treatment for the LVQ and LeanVec datasets, each of which keeps its own private
+// copy of the blocked-allocator trait. LeanVec additionally picks the allocator for its inner
+// datasets through that trait, and its blocked branch hardcodes svs::data::Blocked -- so
+// without this the inner storage would quietly lose the grow-stable guarantee.
+#include "svs/concurrent/extensions/lvq.h"
+#include "svs/concurrent/extensions/leanvec.h"
 #endif // HAVE_SVS_LVQ
 
 // Scalar Quantization traits for SVS
 template <typename DataType>
 struct SVSStorageTraits<DataType, 1, 0, false> {
     using element_type = std::int8_t;
     using allocator_type = svs_details::SVSAllocator<element_type>;
-    using blocked_type = svs::data::Blocked<svs::AllocatorHandle<element_type>>;
+    using blocked_type = svs::concurrent::SegmentedBlocked<svs::AllocatorHandle<element_type>>;
     using index_storage_type =
         svs::quantization::scalar::SQDataset<element_type, svs::Dynamic, blocked_type>;
 
@@ -32,7 +42,7 @@ struct SVSStorageTraits<DataType, 1, 0, false> {
         // SVS block size is a power of two, so we can use it directly
         auto svs_bs = svs_details::SVSBlockSize(block_size, element_size(dim));
         allocator_type data_allocator{std::move(allocator)};
-        return svs::make_blocked_allocator_handle({svs_bs}, data_allocator);
+        return svs_details::make_segmented_blocked_allocator_handle({svs_bs}, data_allocator);
     }
 
     static constexpr VecSimSvsQuantBits get_compression_mode() { return VecSimSvsQuant_Scalar; }
@@ -89,7 +99,7 @@ template <typename DataType, size_t QuantBits, size_t ResidualBits>
 struct SVSStorageTraits<DataType, QuantBits, ResidualBits, false,
                         std::enable_if_t<(QuantBits > 1)>> {
     using allocator_type = svs_details::SVSAllocator<std::byte>;
-    using blocked_type = svs::data::Blocked<svs::AllocatorHandle<std::byte>>;
+    using blocked_type = svs::concurrent::SegmentedBlocked<svs::AllocatorHandle<std::byte>>;
     using strategy_type = typename svs_details::LVQSelector<QuantBits>::strategy;
     using index_storage_type =
         svs::quantization::lvq::LVQDataset<QuantBits, ResidualBits, svs::Dynamic, strategy_type,
@@ -118,7 +128,7 @@ struct SVSStorageTraits<DataType, QuantBits, ResidualBits, false,
         auto elem_size = std::max(primary_element_size(dim), residual_element_size(dim));
         auto svs_bs = svs_details::SVSBlockSize(block_size, elem_size);
         allocator_type data_allocator{std::move(allocator)};
-        return svs::make_blocked_allocator_handle({svs_bs}, data_allocator);
+        return svs_details::make_segmented_blocked_allocator_handle({svs_bs}, data_allocator);
     }
 
     template <svs::data::ImmutableMemoryDataset Dataset, svs::threads::ThreadPool Pool>
@@ -181,7 +191,7 @@ struct SVSStorageTraits<DataType, QuantBits, ResidualBits, false,
 template <typename DataType, size_t QuantBits, size_t ResidualBits>
 struct SVSStorageTraits<DataType, QuantBits, ResidualBits, true> {
     using allocator_type = svs_details::SVSAllocator<std::byte>;
-    using blocked_type = svs::data::Blocked<svs::AllocatorHandle<std::byte>>;
+    using blocked_type = svs::concurrent::SegmentedBlocked<svs::AllocatorHandle<std::byte>>;
     using index_storage_type = svs::leanvec::LeanDataset<svs::leanvec::UsingLVQ<QuantBits>,
                                                          svs::leanvec::UsingLVQ<ResidualBits>,
                                                          svs::Dynamic, svs::Dynamic, blocked_type>;
@@ -212,7 +222,7 @@ struct SVSStorageTraits<DataType, QuantBits, ResidualBits, true> {
         auto elem_size = std::max(primary_element_size(dim), secondary_element_size(dim));
         auto svs_bs = svs_details::SVSBlockSize(block_size, elem_size);
         allocator_type data_allocator{std::move(allocator)};
-        return svs::make_blocked_allocator_handle({svs_bs}, data_allocator);
+        return svs_details::make_segmented_blocked_allocator_handle({svs_bs}, data_allocator);
     }
 
     template <svs::data::ImmutableMemoryDataset Dataset, svs::threads::ThreadPool Pool>

src/VecSim/algorithms/svs/svs_serializer_impl.h

@@ -93,17 +93,17 @@ void SVSIndex<MetricType, DataType, isMulti, QuantBits, ResidualBits, IsLeanVec>
     compareMetadataFile(folder_path + "/metadata");
 
     if constexpr (isMulti) {
-        auto loaded = svs::index::vamana::auto_multi_dynamic_assemble(
+        auto loaded = svs::concurrent::auto_multi_dynamic_assemble(
             folder_path + "/config",
             SVS_LAZY(graph_builder_t::load(folder_path + "/graph", this->blockSize,
                                            this->buildParams, this->getAllocator())),
             SVS_LAZY(storage_traits_t::load(folder_path + "/data", this->blockSize, this->dim,
                                             this->getAllocator())),
             distance_f(), std::move(threadpool_handle),
-            svs::index::vamana::MultiMutableVamanaLoad::FROM_MULTI, logger_);
+            svs::concurrent::MultiMutableVamanaLoad::FROM_MULTI, logger_);
         impl_ = std::make_unique<impl_type>(std::move(loaded));
     } else {
-        auto loaded = svs::index::vamana::auto_dynamic_assemble(
+        auto loaded = svs::concurrent::auto_dynamic_assemble(
             folder_path + "/config",
             SVS_LAZY(graph_builder_t::load(folder_path + "/graph", this->blockSize,
                                            this->buildParams, this->getAllocator())),
@@ -112,6 +112,7 @@ void SVSIndex<MetricType, DataType, isMulti, QuantBits, ResidualBits, IsLeanVec>
             distance_f(), std::move(threadpool_handle), false, logger_);
         impl_ = std::make_unique<impl_type>(std::move(loaded));
     }
+    setReady();
 }
 
 template <typename MetricType, typename DataType, bool isMulti, size_t QuantBits,