diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a11ee6..064a8d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,14 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up minimum supported Python + uses: actions/setup-python@v5 + with: + python-version: '3.9' + + - name: Check benchmark tools on Python 3.9 + run: python3 -m py_compile tools/convert_ann_benchmarks.py + - name: Validate .asf.yaml run: pip install pyyaml -q && python3 tools/validate_asf_yaml.py @@ -181,6 +189,8 @@ jobs: "$(pwd)/target/release/libpaimon_vindex_jni.so" java -cp java/target/test-classes:java/target/classes org.apache.paimon.index.vector.VectorIndexNativeHandleSafetyTest \ "$(pwd)/target/release/libpaimon_vindex_jni.so" + env: + RAYON_NUM_THREADS: 4 python-build: runs-on: ubuntu-latest @@ -278,7 +288,7 @@ jobs: reader = VectorIndexReader(Input(output.getvalue())) reader.optimize_for_search() - result_ids, distances = reader.search(data[0], SearchParams(top_k=2, nprobe=2)) + result_ids, distances = reader.search(data[0], SearchParams.ivf(top_k=2, nprobe=2)) assert result_ids[0] == 1, result_ids assert np.isfinite(distances[0]), distances reader.close() @@ -316,7 +326,7 @@ jobs: reader = VectorIndexReader(Input(output.getvalue())) reader.optimize_for_search() - result_ids, distances = reader.search(data[0], SearchParams(top_k=2, nprobe=2)) + result_ids, distances = reader.search(data[0], SearchParams.ivf(top_k=2, nprobe=2)) assert result_ids[0] == 1, result_ids assert np.isfinite(distances[0]), distances reader.close() diff --git a/Cargo.lock b/Cargo.lock index 6ea5337..1c69535 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -659,9 +659,11 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "paimon-vindex-core" -version = "0.2.0" +version = "0.3.0" dependencies = [ "criterion", + "half", + "libc", "matrixmultiply", "nalgebra", "rand", @@ -671,7 +673,7 @@ dependencies = [ [[package]] name = "paimon-vindex-ffi" -version = "0.2.0" +version = "0.3.0" dependencies = [ "cbindgen", "paimon-vindex-core", @@ -679,7 +681,7 @@ dependencies = [ [[package]] name = "paimon-vindex-jni" -version = "0.2.0" +version = "0.3.0" dependencies = [ "jni", "paimon-vindex-core", diff --git a/README.md b/README.md index f385253..953c53e 100644 --- a/README.md +++ b/README.md @@ -24,21 +24,31 @@ Apache Paimon Vector Index is a pure Rust vector indexing library designed for Apache Paimon and data lake storage such as S3, HDFS, and OSS. Its seek-based -readers load only the IVF lists selected by a query. - -The library supports IVF-FLAT, IVF-PQ, IVF-RQ, IVF-HNSW-FLAT, and -IVF-HNSW-SQ through shared Rust, C, C++, Java/JNI, and Python APIs. +readers load only the index pages needed by each query. + +The library supports IVF-FLAT, IVF-SQ, IVF-PQ, IVF-RQ, and DiskANN through +shared Rust, C, C++, Java/JNI, and Python APIs. IVF-RQ stores deterministic +rotated residuals as 1–8 bit planes (4 by default), uses a bounded sign-plane +coarse pass before full refinement, and exposes the persisted width as +`rq_bits` metadata. DiskANN combines +a Vamana graph, resident PQ codes, paged adjacency/raw vectors, and F16 or F32 +reranking for indexes queried from local SSD or object storage. Its parallel +builder uses contiguous fixed-capacity adjacency, adaptive dense/sparse visited +state, heap frontiers, and reusable prune scratch; query-local adjacency +retention is bounded and the shared cold cache is sharded for concurrent hits. ## Documentation - [Index selection and architecture](docs/index.html): compare all index families and open the detailed page for each implementation. +- [DiskANN positioning and tuning](docs/diskann.html): decide when to use + DiskANN and configure build, search, local-SSD, and object-store parameters. - [API and language bindings](docs/api.html): lifecycle, query parameters, warm-up, Rust, C, C++, Java, Python, and metadata filter pushdown. - [Development and benchmarks](docs/development.html): workspace layout, build and test commands, ANN benchmarks, and storage compatibility checks. -- [Storage format specification](STORAGE_FORMAT.md): normative v1 binary layout - and compatibility policy. +- [Storage format specification](core/STORAGE_FORMAT.md): normative v1 binary + layout and compatibility policy. GitHub shows committed HTML files as source. To view the styled documentation, clone the repository and serve `docs/` from the repository root: diff --git a/STORAGE_FORMAT.md b/STORAGE_FORMAT.md index a24b084..0206a9b 100644 --- a/STORAGE_FORMAT.md +++ b/STORAGE_FORMAT.md @@ -19,301 +19,9 @@ # Vector Index Storage Format -This document describes the v1 on-disk formats written by the Rust core -library. Version 1 is the first release format. Pre-release layouts are not part -of the compatibility contract. +The normative storage-format specification is shipped with the Rust core crate: +[core/STORAGE_FORMAT.md](core/STORAGE_FORMAT.md). -## Compatibility Policy - -- All multi-byte integers and `f32` values are little-endian. -- The unified reader dispatches by the first 4-byte magic value. -- Magic names below show the `u32` constants in human-readable big-endian form. - Because the fields are little-endian, the raw file bytes for those constants - appear in reverse ASCII order. -- Readers reject unknown magic values, unknown versions, unknown required flags, - non-zero reserved bytes, invalid section sizes, negative counts, and malformed - list payload metadata. -- Incompatible on-disk changes require a new format version. Version 1 readers - do not attempt to read future versions. -- Reserved bytes are written as zero and must be read back as zero. Future - extensions must use flags or a new format version rather than repurposing - non-zero reserved bytes within v1. -- Index files have no outer container, footer, checksum, compression envelope, - or schema registry. The complete file starts at byte offset 0 with one of the - headers below. -- File integrity, including length and checksum validation, is guaranteed by - the outer Paimon file/manifest layer rather than by an embedded index footer. -- Roaring row-id filters are a query-time API payload. They are not embedded in - any index file format. - -## Common Encodings - -### Delta-Varint IDs - -IVF-PQ, IVF-FLAT, and IVF-RQ v1 sort each non-empty list by signed row id before writing. -The first id is stored as `base_id: i64`. The id stream then stores one unsigned -LEB128 varint per id, including the first id's zero delta. Each delta is computed -with wrapping unsigned subtraction from the previous signed id. Readers reject a -decoded sequence that is not monotonically non-decreasing in signed order. - -### HNSW Graph Section - -IVF-HNSW-FLAT and IVF-HNSW-SQ store one graph section per non-empty list. The -section starts with a fixed header, followed by a contiguous sequence of -unsigned LEB128 varints. Neighbor ids within each adjacency group are sorted by -local vector id and stored as unsigned deltas from the previous neighbor id, -with an initial previous id of `0`: - -| Field | Count | -| --- | --- | -| `graph_magic` | 1 little-endian `u32`, `HWGR` (`0x48574752`) | -| `graph_version` | 1 little-endian `u32`, currently `1` | -| `graph_flags` | 1 little-endian `u32`; bit 0 delta-varint adjacency is required | -| `graph_count` | 1 varint | -| `entry_point` | 1 varint | -| `max_observed_level` | 1 varint | -| `level[node]` | `graph_count` varints | -| `degree[node][level]` followed by neighbor id deltas | one group for each node level | - -Each node has levels `0..=level[node]`. A level-0 node may have at most `2 * m` -neighbors, and higher levels may have at most `m` neighbors. - -## IVF-PQ v1 - -Magic: `IVPQ` (`0x49565051`). Version: `1`. Header size: 64 bytes. - -| Offset | Size | Type | Field | -| ---: | ---: | --- | --- | -| 0 | 4 | `u32` | magic | -| 4 | 4 | `u32` | version | -| 8 | 4 | `i32` | dimension `d` | -| 12 | 4 | `i32` | IVF list count `nlist` | -| 16 | 4 | `i32` | PQ subquantizer count `m` | -| 20 | 4 | `i32` | centroid count per subquantizer `ksub` | -| 24 | 4 | `i32` | subvector dimension `dsub` | -| 28 | 4 | `u32` | metric (`0=L2`, `1=InnerProduct`, `2=Cosine`) | -| 32 | 8 | `i64` | total vector count | -| 40 | 4 | `u32` | flags | -| 44 | 20 | bytes | reserved | - -Flags: - -| Bit | Meaning | -| ---: | --- | -| 0 | OPQ rotation matrix is present | -| 1 | PQ codes are trained/stored by residual | -| 2 | delta-varint ids are used; required in v1 | -| 3 | PQ codes are transposed by subquantizer; required in v1 | - -Sections after the header: - -1. Optional OPQ rotation matrix: `d * d` `f32` values when flag bit 0 is set. -2. IVF coarse centroids: `nlist * d` `f32` values. -3. PQ centroids: `m * ksub * dsub` `f32` values. -4. Offset table: `nlist` entries of `(offset: i64, count: i32, id_bytes_len: i32)`. -5. List payloads. - -For each non-empty list payload: - -| Field | Type | Notes | -| --- | --- | --- | -| `base_id` | `i64` | first sorted row id | -| `id_bytes_len` | `i32` | byte length of encoded id stream | -| `id_bytes` | bytes | delta-varint ids | -| `codes` | bytes | transposed PQ codes | - -For 8-bit PQ, each vector has `m` code bytes and the stored code layout is -`codes[sub][vector]`. For 4-bit PQ, each byte stores two subquantizers and the -stored layout is `codes[pair][vector]`. - -## IVF-FLAT v1 - -Magic: `IVFL` (`0x4956464C`). Version: `1`. Header size: 64 bytes. - -| Offset | Size | Type | Field | -| ---: | ---: | --- | --- | -| 0 | 4 | `u32` | magic | -| 4 | 4 | `u32` | version | -| 8 | 4 | `i32` | dimension `d` | -| 12 | 4 | `i32` | IVF list count `nlist` | -| 16 | 4 | `u32` | metric (`0=L2`, `1=InnerProduct`, `2=Cosine`) | -| 20 | 8 | `i64` | total vector count | -| 28 | 4 | `u32` | flags | -| 32 | 32 | bytes | reserved | - -Flags: - -| Bit | Meaning | -| ---: | --- | -| 0 | delta-varint ids are used; required in v1 | - -Sections after the header: - -1. IVF coarse centroids: `nlist * d` `f32` values. -2. Offset table: `nlist` entries of `(offset: i64, count: i32, id_bytes_len: i32)`. -3. List payloads. - -For each non-empty list payload: - -| Field | Type | Notes | -| --- | --- | --- | -| `base_id` | `i64` | first sorted row id | -| `id_bytes_len` | `i32` | byte length of encoded id stream | -| `id_bytes` | bytes | delta-varint ids | -| `vectors` | `count * d` `f32` | raw stored vectors | - -## IVF-RQ v1 - -Magic: `IVRQ` (`0x49565251`). Version: `1`. Header size: 64 bytes. - -| Offset | Size | Type | Field | -| ---: | ---: | --- | --- | -| 0 | 4 | `u32` | magic | -| 4 | 4 | `u32` | version | -| 8 | 4 | `i32` | dimension `d` | -| 12 | 4 | `i32` | IVF list count `nlist` | -| 16 | 4 | `u32` | metric (`0=L2`, `1=InnerProduct`, `2=Cosine`) | -| 20 | 4 | `u32` | flags | -| 24 | 8 | `i64` | total vector count | -| 32 | 8 | `u64` | deterministic rotation seed | -| 40 | 4 | `u32` | deterministic rotation rounds | -| 44 | 4 | `i32` | bytes per primary sign-code plane, `ceil(d / 8)` | -| 48 | 4 | `u32` | RQ `num_bits`; currently `1`, reserved values `2`, `4`, and `8` | -| 52 | 4 | `u32` | `rotation_type`; currently `1` for deterministic Kac rotation | -| 56 | 4 | `u32` | `factor_layout`; currently `1` for three RaBitQ `f32` factors | -| 60 | 4 | `u32` | RQ format flags | - -Flags: - -| Bit | Meaning | -| ---: | --- | -| 0 | delta-varint ids are used; required in v1 | - -RQ format flags: - -| Bit | Meaning | -| ---: | --- | -| 0 | optional `ex_codes` section is present | -| 1 | optional `error_factor` section is present | - -Sections after the header: - -1. IVF coarse centroids: `nlist * d` `f32` values. -2. Offset table: `nlist` entries of `(offset: i64, count: i32, id_bytes_len: i32)`. -3. List payloads. - -For each non-empty list payload: - -| Field | Type | Notes | -| --- | --- | --- | -| `base_id` | `i64` | first sorted row id | -| `id_bytes_len` | `i32` | byte length of encoded id stream | -| `id_bytes` | bytes | delta-varint ids | -| `codes` | `count * ceil(d / 8)` bytes | primary RaBitQ sign-code plane over rotated residuals | -| `ex_codes` | optional bytes | reserved for additional code planes when `num_bits > 1`; present when RQ format flag bit 0 is set | -| `factors` | `count * 3` `f32` | per-vector `(residual_norm_sqr, vector_norm_sqr, dp_multiplier)` correction factors | -| `error_factor` | optional `count` `f32` | reserved per-vector refinement factor; present when RQ format flag bit 1 is set | - -The deterministic rotation is derived from `(d, rotation_seed, rotation_rounds)` -and applied to both indexed residuals and query residuals. The rotation parameters -are stored in the header so independently written files can use the same -distance estimator after being read by any v1 reader. - -Current writers emit `num_bits=1`, `rotation_type=1`, `factor_layout=1`, and no -optional RQ sections. Current readers validate these format fields and reject -reserved multi-bit or optional-section encodings until the corresponding scanner -is implemented. - -`query_bits` is a search-time IVF-RQ parameter and is not serialized. `0` uses -the default float-query byte-LUT estimator; `4` and `8` quantize the rotated -query residual into sign and magnitude bit planes for bitwise/popcount scanning. - -## IVF-HNSW-FLAT v1 - -Magic: `IHFL` (`0x4948464C`). Version: `1`. Header size: 64 bytes. - -| Offset | Size | Type | Field | -| ---: | ---: | --- | --- | -| 0 | 4 | `u32` | magic | -| 4 | 4 | `u32` | version | -| 8 | 4 | `i32` | dimension `d` | -| 12 | 4 | `i32` | IVF list count `nlist` | -| 16 | 4 | `u32` | metric (`0=L2`, `1=InnerProduct`, `2=Cosine`) | -| 20 | 8 | `i64` | total vector count | -| 28 | 4 | `i32` | HNSW `m` | -| 32 | 4 | `i32` | HNSW `ef_construction` | -| 36 | 4 | `i32` | HNSW `max_level` | -| 40 | 4 | `u32` | flags | -| 44 | 20 | bytes | reserved | - -Flags: - -| Bit | Meaning | -| ---: | --- | -| 0 | sorted delta-varint ids are stored; required in v1 | -| 1 | HNSW graph section uses the v1 delta-varint graph encoding; required in v1 | - -Sections after the header: - -1. IVF coarse centroids: `nlist * d` `f32` values. -2. Offset table: `nlist` entries of - `(offset: i64, count: i32, graph_bytes_len: i32, payload_bytes_len: i64)`. -3. List payloads. - -For each non-empty list payload: - -| Field | Type | Notes | -| --- | --- | --- | -| `base_id` | `i64` | first sorted row id | -| `id_bytes_len` | `i32` | byte length of encoded id stream | -| `id_bytes` | bytes | delta-varint ids | -| `vectors` | `count * d` `f32` | raw stored vectors | -| `graph` | bytes | HNSW graph section | - -## IVF-HNSW-SQ v1 - -Magic: `IHSQ` (`0x49485351`). Version: `1`. Header size: 64 bytes. - -| Offset | Size | Type | Field | -| ---: | ---: | --- | --- | -| 0 | 4 | `u32` | magic | -| 4 | 4 | `u32` | version | -| 8 | 4 | `i32` | dimension `d` | -| 12 | 4 | `i32` | IVF list count `nlist` | -| 16 | 4 | `u32` | metric (`0=L2`, `1=InnerProduct`, `2=Cosine`) | -| 20 | 8 | `i64` | total vector count | -| 28 | 4 | `i32` | HNSW `m` | -| 32 | 4 | `i32` | HNSW `ef_construction` | -| 36 | 4 | `i32` | HNSW `max_level` | -| 40 | 4 | `f32` | global minimum SQ bound summary | -| 44 | 4 | `f32` | global maximum SQ bound summary | -| 48 | 4 | `u32` | flags | -| 52 | 12 | bytes | reserved | - -Flags: - -| Bit | Meaning | -| ---: | --- | -| 0 | sorted delta-varint ids are stored; required in v1 | -| 1 | HNSW graph section uses the v1 delta-varint graph encoding; required in v1 | - -Sections after the header: - -1. Global SQ min bounds: `d` `f32` values. -2. Global SQ max bounds: `d` `f32` values. -3. Per-list SQ bounds: for each list, `d` min `f32` values followed by `d` - max `f32` values. -4. IVF coarse centroids: `nlist * d` `f32` values. -5. Offset table: `nlist` entries of - `(offset: i64, count: i32, graph_bytes_len: i32, payload_bytes_len: i64)`. -6. List payloads. - -For each non-empty list payload: - -| Field | Type | Notes | -| --- | --- | --- | -| `base_id` | `i64` | first sorted row id | -| `id_bytes_len` | `i32` | byte length of encoded id stream | -| `id_bytes` | bytes | delta-varint ids | -| `codes` | bytes | scalar quantized residual codes, `count * d` bytes | -| `graph` | bytes | HNSW graph section over decoded vectors | + +For the stable DiskANN layout, see +[DiskANN v1](core/STORAGE_FORMAT.md#diskann-v1). diff --git a/c/test_vindex.c b/c/test_vindex.c index 1cb05cb..fbd1c93 100644 --- a/c/test_vindex.c +++ b/c/test_vindex.c @@ -196,7 +196,7 @@ static void run_roundtrip( uintptr_t num_options, uint32_t expected_index_type, uintptr_t expected_pq_m, - uintptr_t expected_hnsw_m) { + uintptr_t expected_pq_bits) { PaimonVindexTrainerHandle *trainer = paimon_vindex_trainer_open(keys, values, num_options); if (trainer == NULL) { @@ -262,61 +262,83 @@ static void run_roundtrip( ASSERT_EQ_I64(metadata.index_type, expected_index_type); ASSERT_EQ_I64(metadata.metric, PAIMON_VINDEX_METRIC_L2); ASSERT_EQ_I64(metadata.dimension, ROUNDTRIP_DIMENSION); - ASSERT_EQ_I64(metadata.nlist, 4); + ASSERT_EQ_I64( + metadata.nlist, + expected_index_type == PAIMON_VINDEX_INDEX_TYPE_DISKANN ? 1 : 4); ASSERT_EQ_I64(metadata.total_vectors, ROUNDTRIP_VECTOR_COUNT); ASSERT_EQ_I64(metadata.pq_m, expected_pq_m); - ASSERT_EQ_I64(metadata.hnsw_m, expected_hnsw_m); + ASSERT_EQ_I64(metadata.pq_bits, expected_pq_bits); + ASSERT_EQ_I64( + metadata.rq_bits, + expected_index_type == PAIMON_VINDEX_INDEX_TYPE_IVF_RQ ? 5 : 0); + if (expected_index_type == PAIMON_VINDEX_INDEX_TYPE_DISKANN) { + ASSERT_EQ_I64(metadata.diskann_max_degree, 8); + ASSERT_EQ_I64(metadata.diskann_build_search_list_size, 16); + ASSERT_TRUE(fabsf(metadata.diskann_alpha - 1.2f) < 1e-6f); + struct PaimonVindexReadPlan read_plan = {0}; + if (paimon_vindex_reader_read_plan(reader, &read_plan) != 0) { + fail_ffi("reader read plan failed"); + } + ASSERT_TRUE(read_plan.window_bytes > 0); + ASSERT_EQ_I64( + read_plan.memory_budget_bytes, + (uintptr_t) 4 * 1024 * 1024 * 1024); + } if (paimon_vindex_reader_optimize_for_search(reader) != 0) { fail_ffi("reader optimize_for_search failed"); } - float query[ROUNDTRIP_DIMENSION]; fill_query(query, 0.0f); + if (expected_index_type == PAIMON_VINDEX_INDEX_TYPE_DISKANN) { + uintptr_t calibrated_width = 0; + if (paimon_vindex_reader_calibrate_search_width( + reader, query, 1, 2, &calibrated_width) != 0) { + fail_ffi("reader calibrate_search_width failed"); + } + ASSERT_TRUE( + calibrated_width == 100 || + calibrated_width == 200 || + calibrated_width == 400); + } int64_t result_ids[2] = {0}; float result_distances[2] = {0}; - struct PaimonVindexSearchParams search_params = {2, 4, 16, 0}; + struct PaimonVindexSearchParams search_params = { + .top_k = 2, + .search_width = expected_index_type == PAIMON_VINDEX_INDEX_TYPE_DISKANN + ? PAIMON_VINDEX_SEARCH_WIDTH_AUTO + : PAIMON_VINDEX_SEARCH_WIDTH_IVF_NPROBE, + .width = expected_index_type == PAIMON_VINDEX_INDEX_TYPE_DISKANN ? 0 : 4}; if (paimon_vindex_reader_search( reader, query, search_params, result_ids, result_distances, 2) != 0) { fail_ffi("reader search failed"); } + if (expected_index_type == PAIMON_VINDEX_INDEX_TYPE_DISKANN && + paimon_vindex_reader_warmup_queries(reader, query, 1, 32) != 0) { + fail_ffi("reader warmup_queries failed"); + } assert_id_in_cluster(result_ids[0], 0); ASSERT_TRUE(isfinite(result_distances[0])); if (expected_index_type == PAIMON_VINDEX_INDEX_TYPE_IVF_PQ) { ASSERT_TRUE(buf.max_read_request_count > 1); } - if (expected_index_type == PAIMON_VINDEX_INDEX_TYPE_IVF_RQ) { - search_params.query_bits = 4; - if (paimon_vindex_reader_search( - reader, query, search_params, result_ids, result_distances, 2) != 0) { - fail_ffi("reader search with query bits failed"); - } - assert_id_in_cluster(result_ids[0], 0); - ASSERT_TRUE(isfinite(result_distances[0])); - } - float queries[2 * ROUNDTRIP_DIMENSION]; fill_query(queries, 0.0f); fill_query(queries + ROUNDTRIP_DIMENSION, 20.0f); int64_t batch_ids[2] = {0}; float batch_distances[2] = {0}; - struct PaimonVindexSearchParams batch_params = {1, 4, 16, 0}; + struct PaimonVindexSearchParams batch_params = { + .top_k = 1, + .search_width = expected_index_type == PAIMON_VINDEX_INDEX_TYPE_DISKANN + ? PAIMON_VINDEX_SEARCH_WIDTH_DISKANN_L_SEARCH + : PAIMON_VINDEX_SEARCH_WIDTH_IVF_NPROBE, + .width = expected_index_type == PAIMON_VINDEX_INDEX_TYPE_DISKANN ? 100 : 4}; if (paimon_vindex_reader_search_batch( reader, queries, 2, batch_params, batch_ids, batch_distances, 2) != 0) { fail_ffi("reader search batch failed"); } assert_id_in_cluster(batch_ids[0], 0); assert_id_in_cluster(batch_ids[1], 1); - if (expected_index_type == PAIMON_VINDEX_INDEX_TYPE_IVF_RQ) { - batch_params.query_bits = 8; - if (paimon_vindex_reader_search_batch( - reader, queries, 2, batch_params, batch_ids, batch_distances, 2) != 0) { - fail_ffi("reader search batch with query bits failed"); - } - assert_id_in_cluster(batch_ids[0], 0); - assert_id_in_cluster(batch_ids[1], 1); - } - paimon_vindex_reader_free(reader); free(buf.data); free(data); @@ -410,48 +432,55 @@ static void test_supported_index_roundtrips(void) { 0, 0); - const char *pq_keys[] = {"index.type", "dimension", "nlist", "metric", "pq.m"}; - const char *pq_values[] = {"ivf_pq", "8", "4", "l2", "4"}; + const char *pq_keys[] = {"index.type", "dimension", "nlist", "metric"}; + const char *pq_values[] = {"ivf_pq", "8", "4", "l2"}; run_roundtrip( "ivf_pq_roundtrip", pq_keys, pq_values, - 5, - PAIMON_VINDEX_INDEX_TYPE_IVF_PQ, 4, - 0); + PAIMON_VINDEX_INDEX_TYPE_IVF_PQ, + 2, + 8); - const char *rq_keys[] = {"index.type", "dimension", "nlist", "metric"}; - const char *rq_values[] = {"ivf_rq", "8", "4", "l2"}; + const char *rq_keys[] = {"index.type", "dimension", "nlist", "rq.bits", "metric"}; + const char *rq_values[] = {"ivf_rq", "8", "4", "5", "l2"}; run_roundtrip( "ivf_rq_roundtrip", rq_keys, rq_values, - 4, + 5, PAIMON_VINDEX_INDEX_TYPE_IVF_RQ, 0, 0); - const char *hnsw_flat_keys[] = {"index.type", "dimension", "nlist", "metric", "hnsw.m"}; - const char *hnsw_flat_values[] = {"ivf_hnsw_flat", "8", "4", "l2", "4"}; + const char *sq_keys[] = {"index.type", "dimension", "nlist", "metric"}; + const char *sq_values[] = {"ivf_sq", "8", "4", "l2"}; run_roundtrip( - "ivf_hnsw_flat_roundtrip", - hnsw_flat_keys, - hnsw_flat_values, - 5, - PAIMON_VINDEX_INDEX_TYPE_IVF_HNSW_FLAT, + "ivf_sq_roundtrip", + sq_keys, + sq_values, + 4, + PAIMON_VINDEX_INDEX_TYPE_IVF_SQ, 0, - 4); - - const char *hnsw_sq_keys[] = {"index.type", "dimension", "nlist", "metric", "hnsw.m"}; - const char *hnsw_sq_values[] = {"ivf_hnsw_sq", "8", "4", "l2", "4"}; + 8); + + const char *diskann_keys[] = { + "index.type", + "dimension", + "metric", + "pq.m", + "pq.bits", + "diskann.max-degree", + "diskann.build-search-list-size"}; + const char *diskann_values[] = {"diskann", "8", "l2", "4", "4", "8", "16"}; run_roundtrip( - "ivf_hnsw_sq_roundtrip", - hnsw_sq_keys, - hnsw_sq_values, - 5, - PAIMON_VINDEX_INDEX_TYPE_IVF_HNSW_SQ, - 0, + "diskann_roundtrip", + diskann_keys, + diskann_values, + 7, + PAIMON_VINDEX_INDEX_TYPE_DISKANN, + 4, 4); } diff --git a/core/Cargo.toml b/core/Cargo.toml index ae44437..4732c86 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -17,12 +17,17 @@ [package] name = "paimon-vindex-core" -version = "0.2.0" +version = "0.3.0" edition = "2021" description = "Apache Paimon Vector Index — core Rust library" license = "Apache-2.0" +readme = "README.md" +repository = "https://github.com/apache/paimon-vector-index" +homepage = "https://paimon.apache.org" +documentation = "https://docs.rs/paimon-vindex-core" [dependencies] +half = "2" nalgebra = "0.33" rand = "0.8" rayon = "1.10" @@ -31,6 +36,7 @@ roaring = "0.11" [dev-dependencies] criterion = "0.5" +libc = "0.2" [[bench]] name = "pq4_bench" @@ -45,5 +51,5 @@ name = "ann_bench" harness = false [[bench]] -name = "ivfhnswsq_filter_bench" +name = "diskann_bench" harness = false diff --git a/core/README.md b/core/README.md new file mode 100644 index 0000000..940d417 --- /dev/null +++ b/core/README.md @@ -0,0 +1,35 @@ + + +# Apache Paimon Vector Index Core + +`paimon-vindex-core` contains the Rust implementations and seek-based readers +for IVF-FLAT, IVF-SQ, IVF-PQ, IVF-RQ, and DiskANN. + +The DiskANN and Vamana code is an independent Apache-licensed implementation +based on the published algorithms and this project's existing storage +abstractions. It does not incorporate source code from Microsoft's +[MIT-licensed DiskANN repository](https://github.com/microsoft/DiskANN). +The implementation supports L2, inner-product, and cosine search with the same +lower-is-better distance semantics as the IVF indexes. + +The crate ships its [normative v1 storage-format specification](STORAGE_FORMAT.md) +and byte-exact fixtures. Project documentation, language bindings, and +contribution guidance live in the +[Apache Paimon Vector Index repository](https://github.com/apache/paimon-vector-index). diff --git a/core/STORAGE_FORMAT.md b/core/STORAGE_FORMAT.md new file mode 100644 index 0000000..9616d59 --- /dev/null +++ b/core/STORAGE_FORMAT.md @@ -0,0 +1,455 @@ + + +# Vector Index Storage Format + +This document describes the v1 on-disk formats written by the +`paimon-vindex-core` crate. Version 1 is the first release format. Pre-release +layouts are not part of the compatibility contract. + +## Compatibility Policy + +- All multi-byte integers and `f32` values are little-endian. +- The unified reader dispatches by the first 4-byte magic value. +- Magic names below show the `u32` constants in human-readable big-endian form. + Because the fields are little-endian, the raw file bytes for those constants + appear in reverse ASCII order. +- Readers reject unknown magic values, unknown versions, unknown required flags, + non-zero reserved bytes, invalid section sizes, negative counts, and malformed + list payload metadata. +- Incompatible on-disk changes require a new format version. Version 1 readers + do not attempt to read future versions. +- Reserved bytes are written as zero and must be read back as zero. They cannot + acquire meaning within v1. A new field, flag meaning, or reserved-byte use + requires a new format version unless that format explicitly defined the flag + as optional from its first v1 release. +- Index files have no outer container, footer, checksum, compression envelope, + or schema registry. The complete file starts at byte offset 0 with one of the + headers below. +- File integrity, including length and checksum validation, is guaranteed by + the outer Paimon file/manifest layer rather than by an embedded index footer. +- Roaring row-id filters are a query-time API payload. They are not embedded in + any index file format. + +## Common Encodings + +### Delta-Varint IDs + +IVF-PQ, IVF-FLAT, IVF-RQ, and IVF-SQ v1 sort each non-empty list by signed row +id before writing. The first id is stored as `base_id: i64`. The id stream then +stores one unsigned LEB128 varint per id, including the first id's zero delta. +Each delta is computed with wrapping unsigned subtraction from the previous +signed id. Readers reject a decoded sequence that is not monotonically +non-decreasing in signed order. + +## DiskANN v1 + +Raw magic bytes: `DANN` (`u32` value `0x4E4E4144`). Version: `1`. Header size: +256 bytes. DiskANN v1 is little-endian, supports L2, inner product, and cosine, +uses dense `u32` internal node IDs, and applies one BFS locality permutation +consistently to every section. + +| Offset | Size | Type | Field | +| ---: | ---: | --- | --- | +| 0 | 4 | `u32` | magic | +| 4 | 4 | `u32` | version | +| 8 | 4 | `u32` | header size, `256` | +| 12 | 4 | `u32` | flags | +| 16 | 4 | `u32` | dimension | +| 20 | 4 | `u32` | metric (`0=L2`, `1=InnerProduct`, `2=Cosine`) | +| 24 | 8 | `u64` | vector count | +| 32 | 4 | `u32` | entry node | +| 36 | 4 | `u32` | maximum graph degree `R` | +| 40 | 4 | `u32` | build search-list size `Lbuild` | +| 44 | 4 | `f32` | robust-prune alpha | +| 48 | 8 | `u64` | build seed | +| 56 | 4 | `u32` | PQ subquantizer count `m` | +| 60 | 4 | `u32` | PQ bits, required `4` or `8` | +| 64 | 4 | `u32` | logical page size, required `4096` | +| 68 | 4 | `u32` | per-node adjacency locator payload size, required `4` | +| 72 | 4 | `u32` | adjacency locator encoding, required `3` | +| 76 | 4 | `u32` | raw-vector encoding: `1=f32`, `2=IEEE 754 binary16` | +| 80 | 4 | `u32` | raw-vector record size, `dimension * element_size` | +| 84 | 4 | `u32` | section count, required `7` | +| 88 | 8 | `u64` | exact total file length | +| 96 | 112 | seven `(offset: u64, length: u64)` pairs | sections in the order below | +| 208 | 48 | bytes | reserved, required zero | + +Bits 0, 2, 3, and 4 are required: BFS layout, adaptive adjacency encoding, PQ +codes, and row-ID order. Exactly one storage-layout bit is required: bit 1 for +separate adjacency/vector sections or bit 5 for interleaved vector/adjacency +records. Unknown flags are rejected. Consequently, adding any DiskANN flag that +a v1 writer may emit requires version 2; it is not a compatible v1 extension. + +The following header invariants are part of v1 and are enforced symmetrically by +the writer and reader: + +- `1 <= dimension <= 1024`; +- `metric` is `0`, `1`, or `2`; +- `1 <= vector_count <= u32::MAX` and `entry_node < vector_count`; +- `1 <= pq_m <= dimension`, and `pq_bits` is 4 or 8; +- `1 <= R <= 1023`, `Lbuild >= R`, and `Lbuild <= u32::MAX`; +- `alpha` is finite and at least 1; +- `raw_vector_encoding` is `1` or `2`, and `vector_record_size` is exactly + `dimension * 4` or `dimension * 2`, respectively; and +- every stored PQ centroid and decoded raw-vector component is finite. A writer + using binary16 additionally rejects finite `f32` inputs whose conversion + would overflow the finite binary16 range. + +Inner-product results use negative dot product so lower values remain better. +Inner-product Vamana construction uses the metric-specific occluding prune rule +rather than the L2 triangle-inequality rule. Cosine training and indexed vectors +are normalized before PQ encoding and persistence; queries are normalized +before graph traversal, and final distances retain the public `1 - cosine` +semantics. Zero vectors remain zero and have cosine distance `1`. + +Let `E` be the raw-vector element size (`4` for `f32`, `2` for binary16). The +interleaved writer requires `E * dimension + 4 * R <= 4096`. This +content-independent bound guarantees that even a raw-`u32` maximum-degree +adjacency list fits beside its vector without changing layout based on +compression results. + +The seven sections are: + +1. Self-describing PQ codebook at absolute offset 4096, described below. +2. Row IDs, adaptively encoded in dense node order as described below. +3. PQ codes, `m` bytes per node for 8-bit or `ceil(m / 2)` bytes per node for 4-bit. + In 4-bit mode, each byte stores the earlier subquantizer in its low nibble + and the next subquantizer in its high nibble. When `m` is odd, the unused + high nibble of every final byte is required to be zero. +4. Row-ID order, one `u32` node ID per node, sorted by `(row_id, node_id)`. +5. Block-compressed adjacency index described below. +6. 4096-byte-aligned adaptively encoded adjacency pages, optionally containing + the interleaved raw-vector records described below. +7. Dense raw-vector records for the separate layout; a zero-length section + whose offset equals the file length for the interleaved layout. + +The PQ-codebook section starts with this 32-byte header: + +| Offset | Size | Type | Field | +| ---: | ---: | --- | --- | +| 0 | 4 | `u32` | magic `DPQ1` (`0x31515044`) | +| 4 | 4 | `u32` | codebook version, required `1` | +| 8 | 4 | `u32` | dimension, equal to the file header | +| 12 | 4 | `u32` | subquantizer count `m`, equal to the file header | +| 16 | 4 | `u32` | PQ bits, equal to the file header | +| 20 | 4 | `u32` | centroid count per chunk, `1 << pq_bits` | +| 24 | 4 | `u32` | chunk-offset count, `m + 1` | +| 28 | 4 | bytes | reserved, required zero | + +It is followed by exactly `m + 1` little-endian `u32` component offsets. They +start at zero, end at `dimension`, and are strictly increasing. The current +writer creates balanced contiguous chunks: the first `dimension % m` chunks +have `floor(dimension / m) + 1` components and the rest have +`floor(dimension / m)`. Readers use the persisted offsets rather than +re-deriving that policy. + +The remaining payload contains exactly `dimension * (1 << pq_bits)` finite +little-endian `f32` centroid components. Its order is +`centroid[chunk][code][component-within-chunk]`: components are contiguous, +followed by code, followed by chunk. Chunk `s` begins at centroid-component +offset `chunk_offsets[s] * (1 << pq_bits)`. The section has no trailing bytes. + +The row-ID section starts with this 32-byte header: + +| Offset | Size | Type | Field | +| ---: | ---: | --- | --- | +| 0 | 4 | `u32` | encoding: `0=raw i64`, `1=global FOR bit-pack` | +| 4 | 4 | `u32` | bit width: raw requires `64`; FOR requires `0..63` | +| 8 | 8 | `u64` | row-ID count, equal to vector count | +| 16 | 8 | `i64` | base: raw requires `0`; FOR stores the minimum row ID | +| 24 | 8 | bytes | reserved, required zero | + +Raw encoding appends exactly `8 * N` little-endian bytes. FOR appends exactly +`ceil(N * bit_width / 8)` bytes containing unsigned `row_id - base` deltas in +dense node order, least-significant bit first. Unused high bits in the final +byte are zero. Width zero has no payload and maps every node to `base`. The +writer selects the minimum width and falls back to raw only for a 64-bit span. +The reader retains the packed payload and performs O(1) random row-ID lookup; +it validates the exact length, metadata, tail bits, and decoded `i64` range. + +For `B = ceil(N / 16)`, the adjacency index contains three contiguous arrays: + +1. `B` little-endian `u64` block base offsets relative to the start of the + adjacency-page section; +2. `N` little-endian `u16` byte offsets relative to the corresponding + 16-node block base; and +3. `N` little-endian `u16` values whose bit 15 selects raw `u32` adjacency + encoding and whose bits 0–14 store degree. + +The section length is exactly `8 * B + 4 * N` bytes. Node `i` resolves to +`block_base[i / 16] + relative_offset[i]`; division and remainder by 4096 +produce its page index and byte offset. The first relative offset in every +block is zero. A 16-node block can advance at most 15 pages because every list +is page-contained, so the largest valid relative offset is `65535`. + +The codebook, encoded row IDs, PQ codes, row-ID order, and adjacency index are +contiguous. The reader probes the declared file tail, loads required resident +sections directly into their final representations, and loads row-ID order +lazily for sparse filtered queries. The adjacency section begins at the next +4096-byte boundary. Writers fill the bytes from the 256-byte header to the +codebook, and the alignment gap before adjacency, with zero. These alignment +bytes carry no v1 semantics and readers ignore them; they cannot be repurposed +without a new format version. +Neighbor IDs are strictly increasing and packed by actual degree. Each list +uses canonical unsigned delta LEB128 when that is strictly smaller than raw +little-endian `u32`; otherwise bit 15 in its locator selects raw encoding. The +first varint is the first absolute neighbor ID (a delta from zero), and later +varints are positive deltas from the preceding ID. Empty lists use delta mode +and no payload bytes. This adaptive choice guarantees that a list and the +complete adjacency payload never exceed the fixed-`u32` representation. + +A list never crosses a logical-page boundary. In the separate layout, adjacent +locator ranges are contiguous. In the interleaved layout, each page record is +`[dimension * E raw-vector bytes][encoded adjacency bytes]`, and the locator +points to the first adjacency byte; adjacent records are contiguous. +Raw-vector bytes use the header encoding and little-endian scalar +representation. The remaining adjacency-page tail is zero. The resident block +bases, relative offsets, and degree/encoding values are structurally validated +before graph search. `optimize_for_search` validates every preloaded page in +parallel before publishing the hot prefix; a cold page is validated by shared +single-flight work on first access. Payload validation decodes each list to +establish its exact end and checks raw-vector finiteness, canonical varints, the +uniquely minimal adaptive mode, neighbor IDs, contiguity, and the zero +adjacency-page tail. + +In the separate layout, raw-vector record `i` begins at +`vectors.offset + i * vector_record_size`; the section length is exactly +`vector_count * vector_record_size`, with no record or page padding. A record +contains `dimension` little-endian `f32` values when the encoding is `1`, or +`dimension` little-endian IEEE 754 binary16 bit patterns when the encoding is +`2`. Runtime readers group +`max(1, floor(profile_window_bytes / vector_record_size))` complete consecutive +records into one read window and clip the final window to the section end. A +record may therefore cross a 4096-byte address boundary, but never crosses its +runtime read window. Readers validate every consumed component before distance +evaluation. The exact derived lengths, ordering, file length, locator bounds, +degrees, neighbor IDs, duplicate/self edges, finite codebook/vector values, +row-ID encoding, and permutation are validated by the reader. + +DiskANN v1 intentionally has no embedded checksum. The enclosing Paimon +file/manifest contract owns object length and checksum validation; the DiskANN +reader owns all structural and semantic checks described above. + +The `Memory`, `LocalStorage`, `RemoteStorage`, and `ObjectStore` profiles do not +change this physical format. They group 4096-byte adjacency pages into runtime +read windows of 4096, 16384, 32768, or 65536 bytes respectively. Separate raw +vectors use the complete-record grouping described above with the same profile +window-byte targets. + +## IVF-PQ v1 + +Magic: `IVPQ` (`0x49565051`). Version: `1`. Header size: 64 bytes. + +| Offset | Size | Type | Field | +| ---: | ---: | --- | --- | +| 0 | 4 | `u32` | magic | +| 4 | 4 | `u32` | version | +| 8 | 4 | `i32` | dimension `d` | +| 12 | 4 | `i32` | IVF list count `nlist` | +| 16 | 4 | `i32` | PQ subquantizer count `m` | +| 20 | 4 | `i32` | centroid count per subquantizer `ksub` | +| 24 | 4 | `i32` | subvector dimension `dsub` | +| 28 | 4 | `u32` | metric (`0=L2`, `1=InnerProduct`, `2=Cosine`) | +| 32 | 8 | `i64` | total vector count | +| 40 | 4 | `u32` | flags | +| 44 | 20 | bytes | reserved | + +Flags: + +| Bit | Meaning | +| ---: | --- | +| 0 | OPQ rotation matrix is present | +| 1 | PQ codes are trained/stored by residual | +| 2 | delta-varint ids are used; required in v1 | +| 3 | PQ codes are transposed by subquantizer; required in v1 | + +Sections after the header: + +1. Optional OPQ rotation matrix: `d * d` `f32` values when flag bit 0 is set. +2. IVF coarse centroids: `nlist * d` `f32` values. +3. PQ centroids: `m * ksub * dsub` `f32` values. +4. Offset table: `nlist` entries of `(offset: i64, count: i32, id_bytes_len: i32)`. +5. List payloads. + +For each non-empty list payload: + +| Field | Type | Notes | +| --- | --- | --- | +| `base_id` | `i64` | first sorted row id | +| `id_bytes_len` | `i32` | byte length of encoded id stream | +| `id_bytes` | bytes | delta-varint ids | +| `codes` | bytes | transposed PQ codes | + +For 8-bit PQ, each vector has `m` code bytes and the stored code layout is +`codes[sub][vector]`. For 4-bit PQ, each byte stores two subquantizers and the +stored layout is `codes[pair][vector]`. + +## IVF-FLAT v1 + +Magic: `IVFL` (`0x4956464C`). Version: `1`. Header size: 64 bytes. + +| Offset | Size | Type | Field | +| ---: | ---: | --- | --- | +| 0 | 4 | `u32` | magic | +| 4 | 4 | `u32` | version | +| 8 | 4 | `i32` | dimension `d` | +| 12 | 4 | `i32` | IVF list count `nlist` | +| 16 | 4 | `u32` | metric (`0=L2`, `1=InnerProduct`, `2=Cosine`) | +| 20 | 8 | `i64` | total vector count | +| 28 | 4 | `u32` | flags | +| 32 | 32 | bytes | reserved | + +Flags: + +| Bit | Meaning | +| ---: | --- | +| 0 | delta-varint ids are used; required in v1 | + +Sections after the header: + +1. IVF coarse centroids: `nlist * d` `f32` values. +2. Offset table: `nlist` entries of `(offset: i64, count: i32, id_bytes_len: i32)`. +3. List payloads. + +For each non-empty list payload: + +| Field | Type | Notes | +| --- | --- | --- | +| `base_id` | `i64` | first sorted row id | +| `id_bytes_len` | `i32` | byte length of encoded id stream | +| `id_bytes` | bytes | delta-varint ids | +| `vectors` | `count * d` `f32` | raw stored vectors | + +## IVF-RQ v1 + +Magic: `IVRQ` (`0x49565251`). Version: `1`. Header size: 64 bytes. + +| Offset | Size | Type | Field | +| ---: | ---: | --- | --- | +| 0 | 4 | `u32` | magic | +| 4 | 4 | `u32` | version | +| 8 | 4 | `i32` | logical dimension `d` | +| 12 | 4 | `i32` | `padded_d`, the next multiple of 64 | +| 16 | 4 | `i32` | IVF list count `nlist` | +| 20 | 4 | `u32` | metric (`0=L2`, `1=InnerProduct`, `2=Cosine`) | +| 24 | 4 | `u32` | required layout flags | +| 28 | 4 | `u32` | persisted RQ bit width, in `1..=8` | +| 32 | 8 | `i64` | total vector count | +| 40 | 8 | `u64` | deterministic rotation seed | +| 48 | 4 | `u32` | deterministic rotation rounds; `4` | +| 52 | 4 | `i32` | bytes per bit plane, `padded_d / 8` | +| 56 | 4 | `u32` | `rotation_type`; `2` for sign + 64-wide normalized FHT + permutation | +| 60 | 4 | `u32` | `factor_layout`; `3` for compact incremental coarse/full factors | + +Flags: + +| Bit | Meaning | +| ---: | --- | +| 0 | delta-varint ids are used; required in v1 | +| 1 | codes are transposed within 32-vector blocks; required in v1 | +| 2 | factors use structure-of-arrays layout within each block; required in v1 | + +Sections after the header: + +1. IVF coarse centroids: `nlist * d` `f32` values. +2. Offset table: `nlist` entries of `(offset: i64, count: i32, id_bytes_len: i32)`. +3. List payloads. + +For each non-empty list payload: + +| Field | Type | Notes | +| --- | --- | --- | +| `base_id` | `i64` | first sorted row id | +| `id_bytes_len` | `i32` | byte length of encoded id stream | +| `code_bytes_len` | `i32` | exact blocked-code byte length | +| `id_bytes` | bytes | delta-varint ids | +| `codes` | `count * bits * (padded_d / 8)` bytes | MSB-first bit planes; within every up-to-32-vector block the order is plane, byte position, lane | +| `factors` | `count * fields` `f32` | block-SoA fields; 2 coarse fields for one bit, otherwise 3 coarse plus 2 full fields | + +The coarse factor fields are `(f_add, f_rescale, f_error)` when multiple bit +planes require a deterministic reconstruction-error lower bound. A one-bit +file stores only `(f_add, f_rescale)` because it has no later refinement stage. +For multi-bit files, candidates that can still enter Top-K are refined with +every plane and the full `(f_add, f_rescale)` estimate. The full +reconstruction-error factor is intentionally omitted because the final stage +does not compute another lower bound. + +The orthogonal transform is reconstructed from `(d, rotation_seed, +rotation_rounds)`. It pads with zeros to `padded_d` and applies four rounds of +random signs, normalized 64-wide FHT, and permutation. The Reader rotates each +query once and reuses its byte LUT across every selected list. + +The pre-release one-bit/Kac/factor-layout-1 representation used the same magic +and version but was never published. v1 Readers intentionally reject it through +the required padded dimension, flags, rotation type, and factor layout checks. +There is no query-side bit-width parameter; the file fixes the representation. + +## IVF-SQ v1 + +Magic: `IVSQ` (`0x49565351`). Version: `1`. Header size: 64 bytes. IVF-SQ uses +one unsigned 8-bit code per residual dimension and scans every code in each +selected IVF list. + +| Offset | Size | Type | Field | +| ---: | ---: | --- | --- | +| 0 | 4 | `u32` | magic | +| 4 | 4 | `u32` | version | +| 8 | 4 | `i32` | dimension `d` | +| 12 | 4 | `i32` | IVF list count `nlist` | +| 16 | 4 | `u32` | metric (`0=L2`, `1=InnerProduct`, `2=Cosine`) | +| 20 | 8 | `i64` | total vector count | +| 28 | 4 | `u32` | SQ bits, required `8` | +| 32 | 4 | `u32` | flags | +| 36 | 4 | `f32` | global minimum SQ bound summary | +| 40 | 4 | `f32` | global maximum SQ bound summary | +| 44 | 20 | bytes | reserved | + +Flags: + +| Bit | Meaning | +| ---: | --- | +| 0 | sorted delta-varint ids are stored; required in v1 | +| 1 | codes use 32-row blocked dimension-major layout; required in v1 | + +Sections after the header: + +1. Global SQ min bounds: `d` `f32` values. +2. Global SQ max bounds: `d` `f32` values. +3. Per-list SQ bounds: for each list, `d` min `f32` values followed by `d` + max `f32` values. +4. IVF coarse centroids: `nlist * d` `f32` values. +5. Offset table: `nlist` entries of + `(offset: i64, count: i32, id_bytes_len: i32)`. +6. List payloads. + +For each non-empty list payload: + +| Field | Type | Notes | +| --- | --- | --- | +| `codes` | bytes | `count * d` scalar codes; within every up-to-32-row block the order is dimension, then row lane | +| `base_id` | `i64` | first sorted row id | +| `id_bytes_len` | `i32` | byte length of encoded id stream | +| `id_bytes` | bytes | delta-varint ids | + +The global bounds provide a fallback for empty training lists. Non-empty lists +normally use their own per-dimension residual bounds. A reader validates all +offsets, counts, encoded-ID sizes, and SQ bounds before exposing the index. +Putting blocked codes first lets the reader retain the list payload allocation +as the scan buffer after decoding and truncating the trailing ID section. diff --git a/core/benches/ann_bench.rs b/core/benches/ann_bench.rs index 4f060e8..5d14976 100644 --- a/core/benches/ann_bench.rs +++ b/core/benches/ann_bench.rs @@ -15,163 +15,600 @@ // specific language governing permissions and limitations // under the License. -use paimon_vindex_core::distance::MetricType; -use paimon_vindex_core::hnsw::HnswBuildParams; -use paimon_vindex_core::io::{write_index, IVFPQIndexReader, PosWriter}; -use paimon_vindex_core::ivfhnswflat::IVFHNSWFlatIndex; -use paimon_vindex_core::ivfhnswflat_io::{ - search_batch_ivfhnswflat_reader, write_ivfhnswflat_index, IVFHNSWFlatIndexReader, +#[path = "support/ann_bench_support.rs"] +mod ann_bench_support; + +use ann_bench_support::{ + add_fixed_round_latency, default_training_vector_count, inspect_i32_records, + inspect_public_dataset, parse_storage_case_names, resolve_shape_value, should_isolate_indexes, }; -use paimon_vindex_core::ivfhnswsq::IVFHNSWSQIndex; -use paimon_vindex_core::ivfhnswsq_io::{ - search_batch_ivfhnswsq_reader, write_ivfhnswsq_index, IVFHNSWSQIndexReader, +use paimon_vindex_core::diskann::{ + DiskAnnBuildDistance, DiskAnnBuildParams, DiskAnnRawVectorEncoding, }; -use paimon_vindex_core::ivfpq::{search_batch_reader, IVFPQIndex}; -use paimon_vindex_core::ivfrq::IVFRQIndex; -use paimon_vindex_core::ivfrq_io::{ - search_batch_ivfrq_reader_with_query_bits, write_ivfrq_index, IVFRQIndexReader, +use paimon_vindex_core::distance::{fvec_l2sqr, MetricType}; +use paimon_vindex_core::index::{ + infer_pq_m, VectorIndexConfig, VectorIndexReader, VectorIndexReaderOptions, VectorIndexTrainer, + VectorIndexWriter, VectorSearchParams, DEFAULT_PQ_CODE_RATIO, }; -use paimon_vindex_core::rq::is_supported_query_bits; +use paimon_vindex_core::io::{PosWriter, ReadRequest, SeekRead, SeekReadCapabilities}; +use paimon_vindex_core::rq::{is_supported_rq_bits, DEFAULT_RQ_BITS}; +use rayon::prelude::*; +use rayon::{ThreadPool, ThreadPoolBuilder}; use std::env; -use std::fs; -use std::io; +use std::fs::{self, File}; +use std::io::{self, BufReader, Read, Write}; use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; use std::time::{Duration, Instant}; +const ISOLATED_CHILD_ENV: &str = "ANN_BENCH_ISOLATED_CHILD"; +const SUPPRESS_CSV_HEADER_ENV: &str = "ANN_BENCH_SUPPRESS_CSV_HEADER"; + fn main() -> Result<(), Box> { - let cfg = Config::from_env()?; - let dataset = Dataset::clustered(cfg.n, cfg.nq, cfg.d, cfg.clusters, cfg.seed); - let ids: Vec = (0..cfg.n as i64).collect(); - let workspace = prepare_workspace(&cfg.output_dir)?; + if run_isolated_public_indexes()? { + return Ok(()); + } + run_benchmark() +} + +fn run_isolated_public_indexes() -> Result> { + let dataset_paths = DatasetPaths::from_env()?; + let indexes = selected_indexes()?; + let is_child = env::var_os(ISOLATED_CHILD_ENV).is_some(); + let reuses_index = env::var_os("ANN_REUSE_INDEX_PATH").is_some(); + if !should_isolate_indexes( + dataset_paths.is_some(), + indexes.len(), + is_child, + reuses_index, + ) { + return Ok(false); + } println!("{}", CsvRow::header()); - run_paimon_ivfpq(&cfg, &dataset, &ids, &workspace)?; - run_paimon_ivfrq(&cfg, &dataset, &ids, &workspace)?; - run_paimon_ivfhnswflat(&cfg, &dataset, &ids, &workspace)?; - run_paimon_ivfhnswsq(&cfg, &dataset, &ids, &workspace)?; + io::stdout().flush()?; + let executable = env::current_exe()?; + for index in indexes { + eprintln!("running public benchmark in isolated process: {index}"); + let status = Command::new(&executable) + .env("ANN_INDEXES", &index) + .env(ISOLATED_CHILD_ENV, "1") + .env(SUPPRESS_CSV_HEADER_ENV, "1") + .status()?; + if !status.success() { + return Err( + format!("isolated ANN benchmark process for {index} exited with {status}").into(), + ); + } + } + Ok(true) +} + +fn run_benchmark() -> Result<(), Box> { + let config = Config::from_env()?; + eprintln!( + "workload: n={} train_n={} d={} pq_m={} raw_bytes={} ({:.3} GiB), indexes={}, storage_cases={}", + config.n, + config.train_n, + config.d, + config.pq_m, + config.raw_dataset_bytes()?, + config.raw_dataset_bytes()? as f64 / GIB as f64, + config.indexes.join(","), + config + .storage_cases + .iter() + .map(|storage| storage.name) + .collect::>() + .join(",") + ); + let dataset_started = Instant::now(); + let dataset = Dataset::load(&config)?; + eprintln!( + "loaded dataset={} ({:.3} GiB) in {} ms", + config.dataset_name, + config.raw_dataset_bytes()? as f64 / GIB as f64, + dataset_started.elapsed().as_millis() + ); + let ground_truth = if let Some(ground_truth) = dataset.ground_truth.clone() { + eprintln!( + "loaded published ground truth: {} queries, top-{}", + ground_truth.len(), + ground_truth.first().map_or(0, Vec::len) + ); + ground_truth + } else { + let truth_started = Instant::now(); + let ground_truth = exact_ground_truth(&dataset, config.k); + eprintln!( + "exact ground truth: {} queries in {} ms", + config.nq, + truth_started.elapsed().as_millis() + ); + ground_truth + }; + if env::var_os(SUPPRESS_CSV_HEADER_ENV).is_none() { + println!("{}", CsvRow::header()); + } + if let Some(path) = &config.reuse_index_path { + let mut specs = index_specs(&config); + if specs.len() != 1 { + return Err("ANN_REUSE_INDEX_PATH requires exactly one ANN_INDEXES value".into()); + } + let spec = specs.pop().expect("validated one ANN index specification"); + let built = BuiltIndex { + name: spec.name, + path: path.clone(), + searches: spec.searches, + build_time_ms: 0, + train_time_ms: 0, + add_time_ms: 0, + write_time_ms: 0, + file_bytes: fs::metadata(path)?.len(), + peak_rss_bytes: peak_resident_set_bytes()?, + }; + run_built_index(&config, &dataset, &ground_truth, &built)?; + return Ok(()); + } + + let ids = (0..config.n as i64).collect::>(); + let workspace = prepare_workspace(&config.output_dir)?; + for spec in index_specs(&config) { + let built = build_index(&config, &dataset, &ids, &workspace, spec)?; + eprintln!( + "built {} in {} ms: train={} ms, add={} ms, write={} ms ({} bytes, peak RSS {} bytes)", + built.name, + built.build_time_ms, + built.train_time_ms, + built.add_time_ms, + built.write_time_ms, + built.file_bytes, + built.peak_rss_bytes + ); + run_built_index(&config, &dataset, &ground_truth, &built)?; + if !config.keep_indexes { + fs::remove_file(&built.path)?; + } + } + Ok(()) +} + +fn run_built_index( + config: &Config, + dataset: &Dataset, + ground_truth: &[Vec], + built: &BuiltIndex, +) -> Result<(), Box> { + for search in &built.searches { + for storage in &config.storage_cases { + run_query_case(config, dataset, ground_truth, built, *search, *storage)?; + } + } Ok(()) } +const GIB: u64 = 1024 * 1024 * 1024; +const ALL_INDEX_NAMES: [&str; 5] = ["IVF_FLAT", "IVF_SQ", "IVF_PQ", "IVF_RQ", "DISKANN"]; + #[derive(Clone)] struct Config { + dataset_name: String, + dataset_paths: Option, n: usize, + train_n: usize, nq: usize, d: usize, k: usize, nlist: usize, nprobe: usize, pq_m: usize, - hnsw_m: usize, - hnsw_ef_construction: usize, - hnsw_ef_search: usize, - rq_query_bits: usize, + diskann_l_searches: Vec, + diskann_memory_budget_bytes: usize, + diskann_build_distance: DiskAnnBuildDistance, + diskann_raw_vector_encoding: DiskAnnRawVectorEncoding, + rq_bits: usize, clusters: usize, + noise_dimensions: usize, seed: u64, + indexes: Vec, + storage_cases: Vec, + keep_indexes: bool, + reuse_index_path: Option, output_dir: PathBuf, } impl Config { fn from_env() -> Result> { - let n = read_env("ANN_N", 10_000)?; - let nq = read_env("ANN_NQ", 100)?; - let d = read_env("ANN_D", 128)?; + let dataset_paths = DatasetPaths::from_env()?; + let public_shape = dataset_paths + .as_ref() + .map(|paths| inspect_public_dataset(&paths.base, &paths.queries, &paths.ground_truth)) + .transpose()?; + let dataset_name = env::var("ANN_DATASET_NAME").unwrap_or_else(|_| { + if dataset_paths.is_some() { + "public-fvecs".to_string() + } else { + "clustered-synthetic".to_string() + } + }); + if dataset_name.contains(',') || dataset_name.contains('\n') { + return Err("ANN_DATASET_NAME must not contain commas or newlines".into()); + } + let d = resolve_shape_value( + "ANN_D", + read_optional_env("ANN_D")?, + public_shape.map(|shape| shape.dimension), + 64, + )?; + let n = match env::var("ANN_DATA_GIB") { + Ok(value) => { + if public_shape.is_some() { + return Err( + "ANN_DATA_GIB cannot be combined with public fvecs dataset paths".into(), + ); + } + let gib = value.parse::()?; + if !gib.is_finite() || gib <= 0.0 { + return Err("ANN_DATA_GIB must be finite and greater than zero".into()); + } + let requested_bytes = gib * GIB as f64; + if requested_bytes > usize::MAX as f64 { + return Err("ANN_DATA_GIB exceeds addressable memory".into()); + } + let vector_bytes = d + .checked_mul(size_of::()) + .ok_or("ANN_D byte size overflows usize")?; + (requested_bytes as usize) / vector_bytes + } + Err(env::VarError::NotPresent) => resolve_shape_value( + "ANN_N", + read_optional_env("ANN_N")?, + public_shape.map(|shape| shape.vector_count), + 20_000, + )?, + Err(error) => return Err(Box::new(error)), + }; + let nq = resolve_shape_value( + "ANN_NQ", + read_optional_env("ANN_NQ")?, + public_shape.map(|shape| shape.query_count), + 64, + )?; let k = read_env("ANN_K", 10)?; let nlist = read_env("ANN_NLIST", 64)?; let nprobe = read_env("ANN_NPROBE", 8)?; - let pq_m = read_env("ANN_PQ_M", 16)?; - let hnsw_m = read_env("ANN_HNSW_M", 20)?; - let hnsw_ef_construction = read_env("ANN_HNSW_EF_CONSTRUCTION", 150)?; - let hnsw_ef_search = read_env("ANN_HNSW_EF_SEARCH", 80)?; - let rq_query_bits = read_env("ANN_RQ_QUERY_BITS", 0)?; + let train_n = match read_optional_env("ANN_TRAIN_N")? { + Some(train_n) => train_n, + None => default_training_vector_count(n, nlist)?, + }; + if let Some(shape) = public_shape { + if shape.ground_truth_width < k { + return Err(format!( + "public ground-truth width {} is smaller than ANN_K={k}", + shape.ground_truth_width + ) + .into()); + } + } + let pq_m = match env::var("ANN_PQ_M") { + Ok(value) => value.parse()?, + Err(env::VarError::NotPresent) => { + let code_ratio = read_env("ANN_PQ_CODE_RATIO", DEFAULT_PQ_CODE_RATIO)?; + infer_pq_m(d, 8, code_ratio)? + } + Err(error) => return Err(Box::new(error)), + }; + let diskann_l_search = read_env("ANN_DISKANN_L_SEARCH", 100)?; + let diskann_l_searches = + read_usize_list_env("ANN_DISKANN_L_SEARCHES", &[diskann_l_search])?; + let diskann_memory_budget_bytes = read_env( + "ANN_DISKANN_MEMORY_BUDGET_BYTES", + DiskAnnBuildParams::default().memory_budget_bytes, + )?; + let diskann_build_distance = match env::var("ANN_DISKANN_BUILD_DISTANCE") + .unwrap_or_else(|_| "product_quantized".to_string()) + .to_ascii_lowercase() + .as_str() + { + "full_precision" | "full-precision" => DiskAnnBuildDistance::FullPrecision, + "product_quantized" | "product-quantized" | "pq" => { + DiskAnnBuildDistance::ProductQuantized + } + value => { + return Err(format!( + "ANN_DISKANN_BUILD_DISTANCE must be full_precision or product_quantized, got {value}" + ) + .into()) + } + }; + let diskann_raw_vector_encoding = match env::var("ANN_DISKANN_RAW_VECTOR_ENCODING") + .unwrap_or_else(|_| { + match DiskAnnBuildParams::default().raw_vector_encoding { + DiskAnnRawVectorEncoding::F32 => "f32", + DiskAnnRawVectorEncoding::F16 => "f16", + } + .to_string() + }) + .to_ascii_lowercase() + .as_str() + { + "f32" => DiskAnnRawVectorEncoding::F32, + "f16" => DiskAnnRawVectorEncoding::F16, + value => { + return Err(format!( + "ANN_DISKANN_RAW_VECTOR_ENCODING must be f32 or f16, got {value}" + ) + .into()) + } + }; + let rq_bits = read_env("ANN_RQ_BITS", DEFAULT_RQ_BITS)?; let clusters = read_env("ANN_CLUSTERS", 32)?; + let noise_dimensions = read_env("ANN_NOISE_DIMENSIONS", d)?; let seed = read_env("ANN_SEED", 42)?; + let indexes = selected_indexes()?; + let storage_cases = selected_storage_cases()?; + let keep_indexes = read_bool_env("ANN_KEEP_INDEXES", false)?; + let reuse_index_path = env::var("ANN_REUSE_INDEX_PATH").ok().map(PathBuf::from); let output_dir = env::var("ANN_OUTPUT_DIR") .map(PathBuf::from) .unwrap_or_else(|_| env::temp_dir().join("paimon-ann-bench")); - if n == 0 || nq == 0 || d == 0 || k == 0 || nlist == 0 || nprobe == 0 { + if n == 0 + || train_n == 0 + || nq == 0 + || d == 0 + || k == 0 + || nlist == 0 + || nprobe == 0 + || clusters == 0 + || noise_dimensions == 0 + { return Err( - "ANN_N, ANN_NQ, ANN_D, ANN_K, ANN_NLIST, and ANN_NPROBE must be > 0".into(), + "ANN_N/ANN_DATA_GIB, ANN_TRAIN_N, ANN_NQ, ANN_D, ANN_K, ANN_NLIST, ANN_NPROBE, ANN_CLUSTERS, and ANN_NOISE_DIMENSIONS must be > 0".into(), ); } + if noise_dimensions > d { + return Err(format!( + "ANN_NOISE_DIMENSIONS ({noise_dimensions}) must be <= ANN_D ({d})" + ) + .into()); + } + if train_n > n { + return Err(format!("ANN_TRAIN_N ({train_n}) must be <= vector count ({n})").into()); + } if nlist > n { - return Err(format!("ANN_NLIST ({}) must be <= ANN_N ({})", nlist, n).into()); + return Err(format!("ANN_NLIST ({nlist}) must be <= ANN_N ({n})").into()); } if nprobe > nlist { - return Err(format!("ANN_NPROBE ({}) must be <= ANN_NLIST ({})", nprobe, nlist).into()); + return Err(format!("ANN_NPROBE ({nprobe}) must be <= ANN_NLIST ({nlist})").into()); } - if d % pq_m != 0 { - return Err(format!("ANN_D ({}) must be divisible by ANN_PQ_M ({})", d, pq_m).into()); + let needs_pq = indexes + .iter() + .any(|name| matches!(name.as_str(), "IVF_PQ" | "DISKANN")); + if needs_pq && pq_m == 0 { + return Err("ANN_PQ_M must be > 0 for IVF_PQ or DISKANN".into()); } - if d % 8 != 0 { - return Err(format!("ANN_D ({}) must be divisible by 8 for IVF_RQ", d).into()); + if needs_pq && !d.is_multiple_of(pq_m) { + return Err(format!("ANN_D ({d}) must be divisible by ANN_PQ_M ({pq_m})").into()); } - if !is_supported_query_bits(rq_query_bits) { - return Err(format!( - "ANN_RQ_QUERY_BITS ({}) must be one of 0, 4, or 8", - rq_query_bits - ) - .into()); + if !is_supported_rq_bits(rq_bits) { + return Err(format!("ANN_RQ_BITS ({rq_bits}) must be in 1..=8").into()); } Ok(Self { + dataset_name, + dataset_paths, n, + train_n, nq, d, k, nlist, nprobe, pq_m, - hnsw_m, - hnsw_ef_construction, - hnsw_ef_search, - rq_query_bits, + diskann_l_searches, + diskann_memory_budget_bytes, + diskann_build_distance, + diskann_raw_vector_encoding, + rq_bits, clusters, + noise_dimensions, seed, + indexes, + storage_cases, + keep_indexes, + reuse_index_path, output_dir, }) } - fn hnsw_params(&self) -> HnswBuildParams { - HnswBuildParams { - m: self.hnsw_m, - ef_construction: self.hnsw_ef_construction, - ..HnswBuildParams::default() - } - .sanitized() + fn raw_dataset_bytes(&self) -> Result> { + self.n + .checked_mul(self.d) + .and_then(|elements| elements.checked_mul(size_of::())) + .ok_or_else(|| "ANN raw dataset byte size overflows usize".into()) } } struct Dataset { data: Vec, queries: Vec, + dimension: usize, + ground_truth: Option>>, } impl Dataset { - fn clustered(n: usize, nq: usize, d: usize, clusters: usize, seed: u64) -> Self { - let mut rng = Lcg::new(seed); - let mut centers = vec![0.0f32; clusters * d]; + fn load(config: &Config) -> io::Result { + if let Some(paths) = &config.dataset_paths { + return Self::fvecs(config, paths); + } + Self::clustered(config) + } + + fn fvecs(config: &Config, paths: &DatasetPaths) -> io::Result { + let (dimension, data) = read_fvecs(&paths.base)?; + let (query_dimension, queries) = read_fvecs(&paths.queries)?; + if dimension != config.d || query_dimension != config.d { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "ANN_D={} does not match base/query fvec dimensions {dimension}/{query_dimension}", + config.d + ), + )); + } + let n = data.len() / dimension; + let nq = queries.len() / dimension; + if n != config.n || nq != config.nq { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "ANN_N/ANN_NQ={}/{} do not match base/query fvec counts {n}/{nq}", + config.n, config.nq + ), + )); + } + let ground_truth = read_ivecs(&paths.ground_truth)?; + if ground_truth.len() != config.nq { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "ANN_NQ={} does not match ground-truth count {}", + config.nq, + ground_truth.len() + ), + )); + } + let truth_width = ground_truth.first().map_or(0, Vec::len); + if truth_width < config.k || ground_truth.iter().any(|row| row.len() != truth_width) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "ground truth must have a fixed width of at least ANN_K={}", + config.k + ), + )); + } + if let Some(invalid) = ground_truth + .iter() + .flatten() + .find(|row_id| **row_id < 0 || **row_id as usize >= config.n) + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("ground-truth row ID {invalid} is outside [0, {})", config.n), + )); + } + Ok(Self { + data, + queries, + dimension, + ground_truth: Some(ground_truth), + }) + } + + fn clustered(config: &Config) -> io::Result { + let mut rng = Lcg::new(config.seed); + let center_elements = config + .clusters + .checked_mul(config.d) + .ok_or_else(|| io::Error::other("ANN center shape overflows usize"))?; + let mut centers = try_zeroed_f32(center_elements, "cluster centers")?; for value in &mut centers { - *value = rng.next_f32() * 30.0; + *value = rng.next_signed_f32() * 15.0; } - let mut data = vec![0.0f32; n * d]; - for i in 0..n { - let cluster = i % clusters; - for j in 0..d { - data[i * d + j] = centers[cluster * d + j] + rng.next_f32(); + let data_elements = config + .n + .checked_mul(config.d) + .ok_or_else(|| io::Error::other("ANN dataset shape overflows usize"))?; + let mut data = try_zeroed_f32(data_elements, "dataset")?; + for row in 0..config.n { + let cluster = row % config.clusters; + for component in 0..config.d { + let noise = if is_noise_dimension(component, config.d, config.noise_dimensions) { + rng.next_signed_f32() + } else { + 0.0 + }; + data[row * config.d + component] = centers[cluster * config.d + component] + noise; } } - let mut queries = vec![0.0f32; nq * d]; - for qi in 0..nq { - let source = (qi * 9973) % n; - queries[qi * d..(qi + 1) * d].copy_from_slice(&data[source * d..(source + 1) * d]); + let query_elements = config + .nq + .checked_mul(config.d) + .ok_or_else(|| io::Error::other("ANN query shape overflows usize"))?; + let mut queries = try_zeroed_f32(query_elements, "queries")?; + for query in 0..config.nq { + let cluster = (query * 17 + 3) % config.clusters; + for component in 0..config.d { + let noise = if is_noise_dimension(component, config.d, config.noise_dimensions) { + rng.next_signed_f32() + } else { + 0.0 + }; + queries[query * config.d + component] = + centers[cluster * config.d + component] + noise; + } + } + Ok(Self { + data, + queries, + dimension: config.d, + ground_truth: None, + }) + } +} + +#[derive(Clone)] +struct DatasetPaths { + base: PathBuf, + queries: PathBuf, + ground_truth: PathBuf, +} + +impl DatasetPaths { + fn from_env() -> Result, Box> { + match ( + env::var_os("ANN_BASE_FVECS"), + env::var_os("ANN_QUERY_FVECS"), + env::var_os("ANN_GROUND_TRUTH_IVECS"), + ) { + (Some(base), Some(queries), Some(ground_truth)) => Ok(Some(Self { + base: PathBuf::from(base), + queries: PathBuf::from(queries), + ground_truth: PathBuf::from(ground_truth), + })), + (None, None, None) => Ok(None), + _ => Err( + "set ANN_BASE_FVECS, ANN_QUERY_FVECS, and ANN_GROUND_TRUTH_IVECS together".into(), + ), } + } +} - Self { data, queries } +fn is_noise_dimension(component: usize, dimension: usize, noise_dimensions: usize) -> bool { + if noise_dimensions == dimension { + return true; } + let component = component as u128; + let noise_dimensions = noise_dimensions as u128; + let dimension = dimension as u128; + (component + 1) * noise_dimensions / dimension != component * noise_dimensions / dimension +} + +fn try_zeroed_f32(elements: usize, name: &str) -> io::Result> { + let mut values = Vec::new(); + values + .try_reserve_exact(elements) + .map_err(|error| io::Error::other(format!("failed to allocate ANN {name}: {error}")))?; + values.resize(elements, 0.0); + Ok(values) } struct Lcg { @@ -183,350 +620,607 @@ impl Lcg { Self { state: seed } } - fn next_f32(&mut self) -> f32 { - self.state = self.state.wrapping_mul(6364136223846793005).wrapping_add(1); - ((self.state >> 33) as f32) / (u32::MAX as f32) * 2.0 - 1.0 + fn next_signed_f32(&mut self) -> f32 { + self.state = self + .state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1); + (self.state >> 32) as f32 / u32::MAX as f32 * 2.0 - 1.0 } } -struct CsvRow { - engine: &'static str, - index: &'static str, - n: usize, - nq: usize, - d: usize, - k: usize, - nlist: usize, - nprobe: usize, - ef_search: Option, - build_ms: u128, - read_ms: u128, - first_query_ms: u128, - search_ms: u128, - qps: f64, - disk_bytes: u64, - disk_scope: &'static str, - note: &'static str, +struct IndexSpec { + name: &'static str, + config: VectorIndexConfig, + searches: Vec, } -impl CsvRow { - fn header() -> &'static str { - "engine,index,n,nq,d,k,nlist,nprobe,ef_search,build_ms,read_ms,first_query_ms,search_ms,qps,disk_bytes,disk_scope,note" - } -} - -impl std::fmt::Display for CsvRow { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{},{},{},{},{},{},{},{},{},{},{},{},{},{:.2},{},{},{}", - self.engine, - self.index, - self.n, - self.nq, - self.d, - self.k, - self.nlist, - self.nprobe, - self.ef_search - .map(|value| value.to_string()) - .unwrap_or_default(), - self.build_ms, - self.read_ms, - self.first_query_ms, - self.search_ms, - self.qps, - self.disk_bytes, - self.disk_scope, - self.note, - ) - } +fn index_specs(config: &Config) -> Vec { + let ivf_search = VectorSearchParams::new(config.k, config.nprobe); + vec![ + IndexSpec { + name: "IVF_FLAT", + config: VectorIndexConfig::IvfFlat { + dimension: config.d, + nlist: config.nlist, + metric: MetricType::L2, + }, + searches: vec![ivf_search], + }, + IndexSpec { + name: "IVF_SQ", + config: VectorIndexConfig::IvfSq { + dimension: config.d, + nlist: config.nlist, + metric: MetricType::L2, + }, + searches: vec![ivf_search], + }, + IndexSpec { + name: "IVF_PQ", + config: VectorIndexConfig::IvfPq { + dimension: config.d, + nlist: config.nlist, + m: config.pq_m, + metric: MetricType::L2, + use_opq: false, + }, + searches: vec![ivf_search], + }, + IndexSpec { + name: "IVF_RQ", + config: VectorIndexConfig::IvfRq { + dimension: config.d, + nlist: config.nlist, + bits: config.rq_bits, + metric: MetricType::L2, + }, + searches: vec![ivf_search], + }, + IndexSpec { + name: "DISKANN", + config: VectorIndexConfig::DiskAnn { + dimension: config.d, + metric: MetricType::L2, + pq_m: config.pq_m, + pq_bits: 8, + build: DiskAnnBuildParams { + memory_budget_bytes: config.diskann_memory_budget_bytes, + build_distance: config.diskann_build_distance, + raw_vector_encoding: config.diskann_raw_vector_encoding, + ..DiskAnnBuildParams::default() + }, + }, + searches: config + .diskann_l_searches + .iter() + .map(|l_search| VectorSearchParams::with_l_search(config.k, *l_search)) + .collect(), + }, + ] + .into_iter() + .filter(|spec| config.indexes.iter().any(|name| name == spec.name)) + .collect() } -fn run_paimon_ivfpq( - cfg: &Config, +struct BuiltIndex { + name: &'static str, + path: PathBuf, + searches: Vec, + build_time_ms: u128, + train_time_ms: u128, + add_time_ms: u128, + write_time_ms: u128, + file_bytes: u64, + peak_rss_bytes: u64, +} + +fn build_index( + config: &Config, dataset: &Dataset, ids: &[i64], workspace: &Path, -) -> io::Result<()> { - let path = workspace.join("paimon_ivfpq.index"); - let start = Instant::now(); - let mut index = IVFPQIndex::new(cfg.d, cfg.nlist, cfg.pq_m, MetricType::L2, false); - index.train(&dataset.data, cfg.n); - index.add(&dataset.data, ids, cfg.n); - index.build_search_structures(); - index.build_precomputed_table(); - write_to_file(&path, |writer| write_index(&index, writer))?; - let build = start.elapsed(); - drop(index); - - let start = Instant::now(); - let file = fs::File::open(&path)?; - let mut reader = IVFPQIndexReader::open(file)?; - reader.ensure_loaded()?; - let read = start.elapsed(); - - let first_query = - time_first_query(|| reader.search(&dataset.queries[..cfg.d], cfg.k, cfg.nprobe))?; - let search = time_search(|| { - search_batch_reader(&mut reader, &dataset.queries, cfg.nq, cfg.k, cfg.nprobe).map(|_| ()) - })?; - - print_row( - cfg, - "paimon", - "IVF_PQ", - None, - build, - read, - first_query, - search, - path.metadata()?.len(), - "index_bytes", - "", + spec: IndexSpec, +) -> io::Result { + let path = workspace.join(format!("{}.index", spec.name.to_ascii_lowercase())); + let started = Instant::now(); + let training_elements = config + .train_n + .checked_mul(config.d) + .ok_or_else(|| io::Error::other("ANN training shape overflows usize"))?; + eprintln!( + "training {} with {} sampled vectors", + spec.name, config.train_n ); - Ok(()) + let train_started = Instant::now(); + let training = VectorIndexTrainer::train( + spec.config, + &dataset.data[..training_elements], + config.train_n, + )?; + let train_time_ms = train_started.elapsed().as_millis(); + let mut writer = VectorIndexWriter::new(training); + eprintln!("adding {} vectors to {}", config.n, spec.name); + let add_started = Instant::now(); + writer.add_vectors(ids, &dataset.data, config.n)?; + let add_time_ms = add_started.elapsed().as_millis(); + let mut file = File::create(&path)?; + eprintln!("building and serializing {}", spec.name); + let write_started = Instant::now(); + writer.write(&mut PosWriter::new(&mut file))?; + file.sync_all()?; + let write_time_ms = write_started.elapsed().as_millis(); + let build_time_ms = started.elapsed().as_millis(); + let file_bytes = file.metadata()?.len(); + let peak_rss_bytes = peak_resident_set_bytes()?; + Ok(BuiltIndex { + name: spec.name, + path, + searches: spec.searches, + build_time_ms, + train_time_ms, + add_time_ms, + write_time_ms, + file_bytes, + peak_rss_bytes, + }) } -fn run_paimon_ivfrq( - cfg: &Config, +#[derive(Clone, Copy)] +struct StorageCase { + name: &'static str, + latency: Duration, + virtualize_sequential_latency: bool, +} + +impl StorageCase { + fn local_ssd() -> Self { + Self { + name: "local_ssd_warm_cache", + latency: Duration::ZERO, + virtualize_sequential_latency: false, + } + } + + fn remote_cache_2ms() -> Self { + Self { + name: "remote_cache_2ms", + latency: Duration::from_millis(2), + virtualize_sequential_latency: false, + } + } + + fn object_store_20ms() -> Self { + Self { + name: "object_store_20ms", + latency: Duration::from_millis(20), + virtualize_sequential_latency: true, + } + } + + fn from_name(name: &str) -> Self { + match name { + "local_ssd_warm_cache" => Self::local_ssd(), + "remote_cache_2ms" => Self::remote_cache_2ms(), + "object_store_20ms" => Self::object_store_20ms(), + _ => unreachable!("storage case names are validated before conversion"), + } + } +} + +fn run_query_case( + config: &Config, dataset: &Dataset, - ids: &[i64], - workspace: &Path, -) -> io::Result<()> { - let path = workspace.join("paimon_ivfrq.index"); - let start = Instant::now(); - let mut index = IVFRQIndex::new(cfg.d, cfg.nlist, MetricType::L2); - index.train(&dataset.data, cfg.n); - index.add(&dataset.data, ids, cfg.n); - write_to_file(&path, |writer| write_ivfrq_index(&index, writer))?; - let build = start.elapsed(); - drop(index); - - let start = Instant::now(); - let file = fs::File::open(&path)?; - let mut reader = IVFRQIndexReader::open(file)?; - reader.ensure_loaded()?; - let read = start.elapsed(); - - let first_query = time_first_query(|| { - reader.search_with_query_bits( - &dataset.queries[..cfg.d], - cfg.k, - cfg.nprobe, - cfg.rq_query_bits, - ) - })?; - let search = time_search(|| { - search_batch_ivfrq_reader_with_query_bits( - &mut reader, - &dataset.queries, - cfg.nq, - cfg.k, - cfg.nprobe, - cfg.rq_query_bits, + ground_truth: &[Vec], + index: &BuiltIndex, + search: VectorSearchParams, + storage: StorageCase, +) -> Result<(), Box> { + eprintln!( + "querying {} with storage={} l_search={}", + index.name, + storage.name, + search.configured_diskann_l_search().unwrap_or(0) + ); + let io_pool = if index.name == "DISKANN" { + Some(Arc::new( + ThreadPoolBuilder::new() + .num_threads(rayon::current_num_threads()) + .thread_name(|index| format!("ann-bench-io-{index}")) + .build()?, + )) + } else { + None + }; + let stats = Arc::new(Mutex::new(IoStats::default())); + let source = InstrumentedFile { + file: Arc::new(File::open(&index.path)?), + io_pool: io_pool.clone(), + stats: Arc::clone(&stats), + latency: storage.latency, + latency_enabled: Arc::new(AtomicBool::new(true)), + }; + let latency_enabled = Arc::clone(&source.latency_enabled); + if storage.virtualize_sequential_latency { + latency_enabled.store(false, Ordering::Relaxed); + } + let optimize_started = Instant::now(); + let mut reader = VectorIndexReader::open_with_options( + source, + VectorIndexReaderOptions::new(4 * 1024 * 1024 * 1024), + )?; + reader.optimize_for_search()?; + let optimize_elapsed = optimize_started.elapsed(); + let optimize_stats = snapshot_stats(&stats); + let optimize_ms = if storage.virtualize_sequential_latency { + add_fixed_round_latency(optimize_elapsed, optimize_stats.rounds, storage.latency) + .as_millis() + } else { + optimize_elapsed.as_millis() + }; + + let first_rounds_before = snapshot_stats(&stats).rounds; + let first_started = Instant::now(); + reader.search(&dataset.queries[..config.d], search)?; + let first_elapsed = first_started.elapsed(); + let first_query_us = if storage.virtualize_sequential_latency { + add_fixed_round_latency( + first_elapsed, + snapshot_stats(&stats) + .rounds + .saturating_sub(first_rounds_before), + storage.latency, ) - .map(|_| ()) - })?; - - print_row( - cfg, - "paimon", - "IVF_RQ", - None, - build, - read, - first_query, - search, - path.metadata()?.len(), - "index_bytes", - rq_note(cfg.rq_query_bits), + .as_micros() + } else { + first_elapsed.as_micros() + }; + + eprintln!( + "running sequential queries for {} storage={} l_search={}", + index.name, + storage.name, + search.configured_diskann_l_search().unwrap_or(0) + ); + reset_stats(&stats); + let mut latencies = Vec::with_capacity(config.nq); + let mut virtual_sequential_elapsed = Duration::ZERO; + let sequential_started = Instant::now(); + for (query_index, query) in dataset.queries.chunks_exact(config.d).enumerate() { + let rounds_before = snapshot_stats(&stats).rounds; + let started = Instant::now(); + reader.search(query, search).map_err(|error| { + io::Error::new( + error.kind(), + format!("sequential query {query_index} failed: {error}"), + ) + })?; + let elapsed = started.elapsed(); + let elapsed = if storage.virtualize_sequential_latency { + add_fixed_round_latency( + elapsed, + snapshot_stats(&stats).rounds.saturating_sub(rounds_before), + storage.latency, + ) + } else { + elapsed + }; + virtual_sequential_elapsed = virtual_sequential_elapsed.saturating_add(elapsed); + latencies.push(elapsed); + } + let sequential_elapsed = if storage.virtualize_sequential_latency { + virtual_sequential_elapsed + } else { + sequential_started.elapsed() + }; + let sequential_stats = snapshot_stats(&stats); + + eprintln!( + "running batch queries for {} storage={} l_search={}", + index.name, + storage.name, + search.configured_diskann_l_search().unwrap_or(0) + ); + let batch_stats = Arc::new(Mutex::new(IoStats::default())); + let batch_source = InstrumentedFile { + file: Arc::new(File::open(&index.path)?), + io_pool, + stats: Arc::clone(&batch_stats), + latency: storage.latency, + latency_enabled: Arc::new(AtomicBool::new(!storage.virtualize_sequential_latency)), + }; + let batch_latency_enabled = Arc::clone(&batch_source.latency_enabled); + let mut batch_reader = VectorIndexReader::open_with_options( + batch_source, + VectorIndexReaderOptions::new(4 * 1024 * 1024 * 1024), + )?; + batch_reader.optimize_for_search()?; + batch_latency_enabled.store(true, Ordering::Relaxed); + reset_stats(&batch_stats); + let batch_started = Instant::now(); + let (result_ids, _) = batch_reader.search_batch(&dataset.queries, config.nq, search)?; + let batch_elapsed = batch_started.elapsed(); + let batch_stats = snapshot_stats(&batch_stats); + + let recall_at_10 = recall_at_k(&result_ids, ground_truth, config.k); + let p50 = percentile(&latencies, 50); + let p95 = percentile(&latencies, 95); + let build_distance = match config.diskann_build_distance { + DiskAnnBuildDistance::FullPrecision => "full_precision", + DiskAnnBuildDistance::ProductQuantized => "product_quantized", + }; + let raw_vector_encoding = match (index.name, config.diskann_raw_vector_encoding) { + ("DISKANN", DiskAnnRawVectorEncoding::F32) => "f32", + ("DISKANN", DiskAnnRawVectorEncoding::F16) => "f16", + _ => "none", + }; + println!( + "{dataset},{index},{storage},{n},{train_n},{raw_dataset_bytes},{nq},{d},{k},{nlist},{nprobe},{pq_m},{rq_bits},{build_distance},{raw_vector_encoding},{l_search},{build_ms},{train_ms},{add_ms},{write_ms},{peak_rss_bytes},{optimize_ms},{optimize_rounds},{optimize_ranges},{optimize_bytes},{file_bytes},{recall:.4},{first_us},{p50_us},{p95_us},{sequential_qps:.2},{seq_rounds},{seq_ranges},{seq_bytes},{batch_ms},{batch_qps:.2},{batch_rounds},{batch_ranges},{batch_bytes}", + dataset = config.dataset_name, + index = index.name, + storage = storage.name, + n = config.n, + train_n = config.train_n, + raw_dataset_bytes = config.raw_dataset_bytes()?, + nq = config.nq, + d = config.d, + k = config.k, + nlist = if index.name == "DISKANN" { + 1 + } else { + config.nlist + }, + nprobe = search.configured_ivf_nprobe().unwrap_or(0), + pq_m = config.pq_m, + rq_bits = if index.name == "IVF_RQ" { config.rq_bits } else { 0 }, + build_distance = build_distance, + raw_vector_encoding = raw_vector_encoding, + l_search = search.configured_diskann_l_search().unwrap_or(0), + build_ms = index.build_time_ms, + train_ms = index.train_time_ms, + add_ms = index.add_time_ms, + write_ms = index.write_time_ms, + peak_rss_bytes = index.peak_rss_bytes, + optimize_rounds = optimize_stats.rounds, + optimize_ranges = optimize_stats.ranges, + optimize_bytes = optimize_stats.bytes, + file_bytes = index.file_bytes, + recall = recall_at_10, + first_us = first_query_us, + p50_us = p50.as_micros(), + p95_us = p95.as_micros(), + sequential_qps = config.nq as f64 / sequential_elapsed.as_secs_f64(), + seq_rounds = sequential_stats.rounds, + seq_ranges = sequential_stats.ranges, + seq_bytes = sequential_stats.bytes, + batch_ms = batch_elapsed.as_millis(), + batch_qps = config.nq as f64 / batch_elapsed.as_secs_f64(), + batch_rounds = batch_stats.rounds, + batch_ranges = batch_stats.ranges, + batch_bytes = batch_stats.bytes, ); Ok(()) } -fn rq_note(query_bits: usize) -> &'static str { - match query_bits { - 0 => "1bit query_bits=0", - 4 => "1bit query_bits=4", - 8 => "1bit query_bits=8", - _ => "1bit query_bits=invalid", +struct CsvRow; + +impl CsvRow { + fn header() -> &'static str { + "dataset,index,storage,n,train_n,raw_dataset_bytes,nq,d,k,nlist,nprobe,pq_m,rq_bits,diskann_build_distance,diskann_raw_vector_encoding,l_search,build_ms,train_ms,add_ms,write_ms,peak_rss_bytes,optimize_ms,optimize_pread_rounds,optimize_pread_ranges,optimize_pread_bytes,file_bytes,recall_at_10,first_query_us,p50_query_us,p95_query_us,sequential_qps,sequential_pread_rounds,sequential_pread_ranges,sequential_pread_bytes,batch_ms,batch_qps,batch_pread_rounds,batch_pread_ranges,batch_pread_bytes" } } -fn run_paimon_ivfhnswflat( - cfg: &Config, - dataset: &Dataset, - ids: &[i64], - workspace: &Path, -) -> io::Result<()> { - let path = workspace.join("paimon_ivfhnswflat.index"); - let start = Instant::now(); - let mut index = IVFHNSWFlatIndex::new(cfg.d, cfg.nlist, MetricType::L2, cfg.hnsw_params()); - index.train(&dataset.data, cfg.n); - index.add(&dataset.data, ids, cfg.n); - index.build_graphs()?; - write_to_file(&path, |writer| write_ivfhnswflat_index(&index, writer))?; - let build = start.elapsed(); - drop(index); - - let start = Instant::now(); - let file = fs::File::open(&path)?; - let mut reader = IVFHNSWFlatIndexReader::open(file)?; - reader.ensure_loaded()?; - let read = start.elapsed(); - - let first_query = time_first_query(|| { - reader.search( - &dataset.queries[..cfg.d], - cfg.k, - cfg.nprobe, - cfg.hnsw_ef_search, - ) - })?; - let search = time_search(|| { - search_batch_ivfhnswflat_reader( - &mut reader, - &dataset.queries, - cfg.nq, - cfg.k, - cfg.nprobe, - cfg.hnsw_ef_search, - ) - .map(|_| ()) - })?; - - print_row( - cfg, - "paimon", - "IVF_HNSW_FLAT", - Some(cfg.hnsw_ef_search), - build, - read, - first_query, - search, - path.metadata()?.len(), - "index_bytes", - "", - ); +fn exact_ground_truth(dataset: &Dataset, k: usize) -> Vec> { + dataset + .queries + .par_chunks_exact(dataset.dimension) + .map(|query| { + let mut distances = dataset + .data + .chunks_exact(dataset.dimension) + .enumerate() + .map(|(row, vector)| (fvec_l2sqr(query, vector), row as i64)) + .collect::>(); + let retained = k.min(distances.len()); + if retained != 0 { + distances.select_nth_unstable_by(retained - 1, |left, right| { + left.0 + .total_cmp(&right.0) + .then_with(|| left.1.cmp(&right.1)) + }); + distances.truncate(retained); + distances.sort_unstable_by(|left, right| { + left.0 + .total_cmp(&right.0) + .then_with(|| left.1.cmp(&right.1)) + }); + } + distances.into_iter().map(|(_, row)| row).collect() + }) + .collect() +} + +fn recall_at_k(actual: &[i64], expected: &[Vec], k: usize) -> f64 { + let mut hits = 0usize; + let mut total = 0usize; + for (actual, expected) in actual.chunks_exact(k).zip(expected) { + let expected = &expected[..expected.len().min(k)]; + total += expected.len(); + hits += actual + .iter() + .filter(|row_id| **row_id >= 0 && expected.contains(row_id)) + .count(); + } + if total == 0 { + 1.0 + } else { + hits as f64 / total as f64 + } +} + +fn percentile(samples: &[Duration], percentile: usize) -> Duration { + if samples.is_empty() { + return Duration::ZERO; + } + let mut samples = samples.to_vec(); + samples.sort_unstable(); + samples[(samples.len() - 1) * percentile.min(100) / 100] +} + +#[derive(Debug, Clone, Copy, Default)] +struct IoStats { + rounds: usize, + ranges: usize, + bytes: usize, +} + +fn reset_stats(stats: &Mutex) { + *stats.lock().unwrap() = IoStats::default(); +} + +fn snapshot_stats(stats: &Mutex) -> IoStats { + *stats.lock().unwrap() +} + +#[derive(Clone)] +struct InstrumentedFile { + file: Arc, + io_pool: Option>, + stats: Arc>, + latency: Duration, + latency_enabled: Arc, +} + +impl SeekRead for InstrumentedFile { + fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { + let range_count = ranges.len(); + let byte_count = ranges.iter().map(|range| range.buf.len()).sum::(); + if self.latency_enabled.load(Ordering::Relaxed) && !self.latency.is_zero() { + thread::sleep(self.latency); + } + let mut read_ranges = || { + ranges + .par_iter_mut() + .try_for_each(|range| read_exact_at(&self.file, range.buf, range.pos)) + }; + if let Some(io_pool) = &self.io_pool { + io_pool.install(read_ranges)?; + } else { + read_ranges()?; + } + let mut stats = self.stats.lock().unwrap(); + stats.rounds += 1; + stats.ranges += range_count; + stats.bytes += byte_count; + Ok(()) + } + + fn try_clone_reader(&self) -> io::Result> { + Ok(Some(self.clone())) + } + + fn read_capabilities(&self) -> SeekReadCapabilities { + SeekReadCapabilities { + estimated_random_read_latency_nanos: u64::try_from(self.latency.as_nanos()) + .unwrap_or(u64::MAX), + ..SeekReadCapabilities::default() + } + } +} + +#[cfg(unix)] +fn read_exact_at(file: &File, mut buf: &mut [u8], mut pos: u64) -> io::Result<()> { + use std::os::unix::fs::FileExt; + while !buf.is_empty() { + let read = file.read_at(buf, pos)?; + if read == 0 { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "benchmark positional read reached EOF", + )); + } + pos = pos + .checked_add(read as u64) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "read offset overflow"))?; + buf = &mut buf[read..]; + } Ok(()) } -fn run_paimon_ivfhnswsq( - cfg: &Config, - dataset: &Dataset, - ids: &[i64], - workspace: &Path, -) -> io::Result<()> { - let path = workspace.join("paimon_ivfhnswsq.index"); - let start = Instant::now(); - let mut index = IVFHNSWSQIndex::new(cfg.d, cfg.nlist, MetricType::L2, cfg.hnsw_params()); - index.train(&dataset.data, cfg.n); - index.add(&dataset.data, ids, cfg.n); - index.build_graphs()?; - write_to_file(&path, |writer| write_ivfhnswsq_index(&index, writer))?; - let build = start.elapsed(); - drop(index); - - let start = Instant::now(); - let file = fs::File::open(&path)?; - let mut reader = IVFHNSWSQIndexReader::open(file)?; - reader.ensure_loaded()?; - let read = start.elapsed(); - - let first_query = time_first_query(|| { - reader.search( - &dataset.queries[..cfg.d], - cfg.k, - cfg.nprobe, - cfg.hnsw_ef_search, - ) - })?; - let search = time_search(|| { - search_batch_ivfhnswsq_reader( - &mut reader, - &dataset.queries, - cfg.nq, - cfg.k, - cfg.nprobe, - cfg.hnsw_ef_search, - ) - .map(|_| ()) - })?; - - print_row( - cfg, - "paimon", - "IVF_HNSW_SQ", - Some(cfg.hnsw_ef_search), - build, - read, - first_query, - search, - path.metadata()?.len(), - "index_bytes", - "", - ); +#[cfg(windows)] +fn read_exact_at(file: &File, mut buf: &mut [u8], mut pos: u64) -> io::Result<()> { + use std::os::windows::fs::FileExt; + while !buf.is_empty() { + let read = file.seek_read(buf, pos)?; + if read == 0 { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "benchmark positional read reached EOF", + )); + } + pos = pos + .checked_add(read as u64) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "read offset overflow"))?; + buf = &mut buf[read..]; + } Ok(()) } -fn write_to_file( - path: &Path, - write: impl FnOnce(&mut PosWriter<&mut fs::File>) -> io::Result<()>, -) -> io::Result<()> { - let mut file = fs::File::create(path)?; - let mut writer = PosWriter::new(&mut file); - write(&mut writer) -} - -fn time_first_query(query: impl FnOnce() -> io::Result) -> io::Result { - let start = Instant::now(); - query()?; - Ok(start.elapsed()) -} - -fn time_search(search: impl FnOnce() -> io::Result<()>) -> io::Result { - let start = Instant::now(); - search()?; - Ok(start.elapsed()) -} - -#[allow(clippy::too_many_arguments)] -fn print_row( - cfg: &Config, - engine: &'static str, - index: &'static str, - ef_search: Option, - build: Duration, - read: Duration, - first_query: Duration, - search: Duration, - disk_bytes: u64, - disk_scope: &'static str, - note: &'static str, -) { - println!( - "{}", - CsvRow { - engine, - index, - n: cfg.n, - nq: cfg.nq, - d: cfg.d, - k: cfg.k, - nlist: cfg.nlist, - nprobe: cfg.nprobe, - ef_search, - build_ms: build.as_millis(), - read_ms: read.as_millis(), - first_query_ms: first_query.as_millis(), - search_ms: search.as_millis(), - qps: cfg.nq as f64 / search.as_secs_f64(), - disk_bytes, - disk_scope, - note, +fn read_fvecs(path: &Path) -> io::Result<(usize, Vec)> { + let (rows, dimension) = inspect_i32_records(path)?; + let elements = rows + .checked_mul(dimension) + .ok_or_else(|| io::Error::other("fvec shape overflows usize"))?; + let mut values = Vec::new(); + values + .try_reserve_exact(elements) + .map_err(|error| io::Error::other(format!("failed to allocate fvec data: {error}")))?; + let mut reader = BufReader::with_capacity(8 * 1024 * 1024, File::open(path)?); + let mut header = [0u8; 4]; + let mut row = vec![0u8; dimension * size_of::()]; + for row_index in 0..rows { + reader.read_exact(&mut header)?; + let actual_dimension = i32::from_le_bytes(header); + if actual_dimension != dimension as i32 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "fvec row {row_index} has dimension {actual_dimension}, expected {dimension}" + ), + )); } - ); + reader.read_exact(&mut row)?; + values.extend( + row.chunks_exact(4) + .map(|bytes| f32::from_le_bytes(bytes.try_into().expect("four-byte chunk"))), + ); + } + Ok((dimension, values)) +} + +fn read_ivecs(path: &Path) -> io::Result>> { + let (rows, width) = inspect_i32_records(path)?; + let mut result = Vec::with_capacity(rows); + let mut reader = BufReader::with_capacity(1024 * 1024, File::open(path)?); + let mut header = [0u8; 4]; + let mut row = vec![0u8; width * size_of::()]; + for row_index in 0..rows { + reader.read_exact(&mut header)?; + let actual_width = i32::from_le_bytes(header); + if actual_width != width as i32 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("ivec row {row_index} has width {actual_width}, expected {width}"), + )); + } + reader.read_exact(&mut row)?; + result.push( + row.chunks_exact(4) + .map(|bytes| i32::from_le_bytes(bytes.try_into().expect("four-byte chunk")) as i64) + .collect(), + ); + } + Ok(result) } fn read_env(name: &str, default: T) -> Result> @@ -537,10 +1231,114 @@ where match env::var(name) { Ok(value) => Ok(value.parse()?), Err(env::VarError::NotPresent) => Ok(default), - Err(err) => Err(Box::new(err)), + Err(error) => Err(Box::new(error)), } } +fn read_optional_env(name: &str) -> Result, Box> +where + T: std::str::FromStr, + T::Err: std::error::Error + 'static, +{ + match env::var(name) { + Ok(value) => Ok(Some(value.parse()?)), + Err(env::VarError::NotPresent) => Ok(None), + Err(error) => Err(Box::new(error)), + } +} + +fn read_bool_env(name: &str, default: bool) -> Result> { + match env::var(name) { + Ok(value) => match value.trim().to_ascii_lowercase().as_str() { + "1" | "true" | "yes" => Ok(true), + "0" | "false" | "no" => Ok(false), + _ => Err(format!("{name} must be one of 1, 0, true, false, yes, or no").into()), + }, + Err(env::VarError::NotPresent) => Ok(default), + Err(error) => Err(Box::new(error)), + } +} + +fn read_usize_list_env( + name: &str, + default: &[usize], +) -> Result, Box> { + let value = match env::var(name) { + Ok(value) => value, + Err(env::VarError::NotPresent) => return Ok(default.to_vec()), + Err(error) => return Err(Box::new(error)), + }; + let mut values = Vec::new(); + for item in value.split(',') { + let parsed = item.trim().parse::()?; + if parsed == 0 { + return Err(format!("{name} values must be > 0").into()); + } + if !values.contains(&parsed) { + values.push(parsed); + } + } + if values.is_empty() { + return Err(format!("{name} must contain at least one value").into()); + } + Ok(values) +} + +fn selected_indexes() -> Result, Box> { + let value = env::var("ANN_INDEXES").unwrap_or_else(|_| "all".to_string()); + if value.trim().eq_ignore_ascii_case("all") { + return Ok(ALL_INDEX_NAMES + .iter() + .map(|name| (*name).to_string()) + .collect()); + } + let mut selected = Vec::new(); + for value in value.split(',') { + let normalized = value.trim().to_ascii_uppercase().replace('-', "_"); + if !ALL_INDEX_NAMES.contains(&normalized.as_str()) { + return Err(format!( + "unknown ANN_INDEXES value '{value}'; expected all or a comma-separated subset of {}", + ALL_INDEX_NAMES.join(",") + ) + .into()); + } + if !selected.contains(&normalized) { + selected.push(normalized); + } + } + if selected.is_empty() { + return Err("ANN_INDEXES must select at least one index".into()); + } + Ok(selected) +} + +fn selected_storage_cases() -> Result, Box> { + let value = env::var("ANN_STORAGE_CASES").ok(); + Ok(parse_storage_case_names(value.as_deref())? + .into_iter() + .map(StorageCase::from_name) + .collect()) +} + +#[cfg(unix)] +fn peak_resident_set_bytes() -> io::Result { + let mut usage = std::mem::MaybeUninit::::zeroed(); + let result = unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) }; + if result != 0 { + return Err(io::Error::last_os_error()); + } + let max_rss = unsafe { usage.assume_init() }.ru_maxrss as u64; + #[cfg(target_os = "macos")] + return Ok(max_rss); + #[cfg(not(target_os = "macos"))] + return Ok(max_rss.saturating_mul(1024)); +} + +#[cfg(not(unix))] +fn peak_resident_set_bytes() -> io::Result { + Ok(0) +} + fn prepare_workspace(output_dir: &Path) -> io::Result { let workspace = output_dir.join(format!("{}", std::process::id())); if workspace.exists() { diff --git a/core/benches/diskann_bench.rs b/core/benches/diskann_bench.rs new file mode 100644 index 0000000..535df90 --- /dev/null +++ b/core/benches/diskann_bench.rs @@ -0,0 +1,1035 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use paimon_vindex_core::diskann::{ + DiskAnnBuildDistance, DiskAnnBuildParams, DiskAnnBuildStats, DiskAnnIndex, + DiskAnnRawVectorEncoding, DiskAnnStorageLayout, +}; +use paimon_vindex_core::diskann_io::{ + write_diskann_index_with_stats, DiskAnnHeader, DISKANN_HEADER_SIZE, DISKANN_PAGE_SIZE, +}; +use paimon_vindex_core::distance::{fvec_l2sqr, MetricType}; +use paimon_vindex_core::index::{ + infer_pq_m, VectorIndexReader, VectorIndexReaderOptions, VectorSearchParams, + DEFAULT_PQ_CODE_RATIO, +}; +use paimon_vindex_core::io::{PosWriter, ReadRequest, SeekRead, SeekReadCapabilities}; +use rayon::prelude::*; +use roaring::RoaringTreemap; +use std::env; +use std::fs::{File, OpenOptions}; +use std::io::{self, BufReader, Read}; +use std::mem::MaybeUninit; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +#[derive(Clone, Copy)] +enum BenchmarkReadTier { + Memory, + LocalStorage, + RemoteStorage, + ObjectStore, +} + +fn main() -> Result<(), Box> { + let dataset = Dataset::from_env_or_smoke()?; + let ids = (0..dataset.base.len() / dataset.dimension) + .map(|id| id as i64) + .collect::>(); + let pq_bits = env::var("DISKANN_BENCH_PQ_BITS") + .unwrap_or_else(|_| "8".to_string()) + .parse::()?; + if !matches!(pq_bits, 4 | 8) { + return Err("DISKANN_BENCH_PQ_BITS must be 4 or 8".into()); + } + let pq_m = match env::var("DISKANN_BENCH_PQ_M") { + Ok(value) => value.parse::()?, + Err(env::VarError::NotPresent) => { + let code_ratio = env::var("DISKANN_BENCH_PQ_CODE_RATIO") + .unwrap_or_else(|_| DEFAULT_PQ_CODE_RATIO.to_string()) + .parse::()?; + infer_pq_m(dataset.dimension, pq_bits, code_ratio)? + } + Err(error) => return Err(Box::new(error)), + }; + if pq_m == 0 || pq_m > dataset.dimension { + return Err("DISKANN_BENCH_PQ_M must be positive and not exceed the dimension".into()); + } + let storage_layout = match env::var("DISKANN_BENCH_STORAGE_LAYOUT") + .unwrap_or_else(|_| "compact".to_string()) + .as_str() + { + "compact" => DiskAnnStorageLayout::Compact, + "interleaved" => DiskAnnStorageLayout::Interleaved, + _ => return Err("DISKANN_BENCH_STORAGE_LAYOUT must be compact or interleaved".into()), + }; + let raw_vector_encoding = match env::var("DISKANN_BENCH_RAW_VECTOR_ENCODING") + .unwrap_or_else(|_| "f32".to_string()) + .as_str() + { + "f32" => DiskAnnRawVectorEncoding::F32, + "f16" => DiskAnnRawVectorEncoding::F16, + _ => return Err("DISKANN_BENCH_RAW_VECTOR_ENCODING must be f32 or f16".into()), + }; + let build_distance = match env::var("DISKANN_BENCH_BUILD_DISTANCE") + .unwrap_or_else(|_| "product_quantized".to_string()) + .as_str() + { + "full_precision" => DiskAnnBuildDistance::FullPrecision, + "product_quantized" => DiskAnnBuildDistance::ProductQuantized, + _ => { + return Err( + "DISKANN_BENCH_BUILD_DISTANCE must be full_precision or product_quantized".into(), + ) + } + }; + let build_params = DiskAnnBuildParams { + storage_layout, + raw_vector_encoding, + build_distance, + ..DiskAnnBuildParams::default() + }; + let build_start = Instant::now(); + let mut index = DiskAnnIndex::with_pq_bits( + dataset.dimension, + MetricType::L2, + pq_m, + pq_bits, + build_params, + ); + let pq_training_started = Instant::now(); + index.train(&dataset.base, ids.len())?; + let pq_training_time = pq_training_started.elapsed(); + index.add(&dataset.base, &ids); + let (index_file, mut output) = TemporaryIndexFile::create()?; + let build_stats = write_diskann_index_with_stats(&index, &mut PosWriter::new(&mut output))?; + output.sync_all()?; + let file_bytes = output.metadata()?.len(); + drop(output); + let (adjacency_section_bytes, adjacency_pages) = + read_adjacency_layout_metrics(index_file.path())?; + let build_time = build_start.elapsed(); + let peak_rss_bytes = peak_resident_set_bytes()?; + let concurrency = env::var("DISKANN_BENCH_CONCURRENCY") + .unwrap_or_else(|_| "1".to_string()) + .parse::()?; + if concurrency == 0 { + return Err("DISKANN_BENCH_CONCURRENCY must be greater than zero".into()); + } + let reader_memory_budget_bytes = env::var("DISKANN_BENCH_READER_MEMORY_BUDGET_BYTES") + .unwrap_or_else(|_| (4usize * 1024 * 1024 * 1024).to_string()) + .parse::()?; + let warmup_query_count = env::var("DISKANN_BENCH_WARMUP_QUERIES") + .unwrap_or_else(|_| "8".to_string()) + .parse::()? + .min(dataset.queries.len() / dataset.dimension); + + println!( + "read_tier,storage_layout,raw_vector_encoding,build_distance,l_search,n,nq,d,pq_bits,k,concurrency,reader_memory_budget_bytes,warmup_query_count,warmup_ms,warmup_pread_rounds,warmup_pread_ranges,warmup_pread_bytes,recall_at_1,recall_at_10,build_ms,graph_shards,pq_train_ms,pq_encode_ms,vamana_init_ms,vamana_pass_one_ms,vamana_pass_two_ms,connectivity_repair_ms,locality_remap_ms,resident_serialize_ms,adjacency_serialize_ms,vector_serialize_ms,peak_rss_bytes,first_query_us,p50_query_us,p95_query_us,p99_query_us,warm_query_us,qps,pread_rounds,pread_ranges,max_ranges_per_round,max_in_flight_rounds,max_in_flight_ranges,pread_bytes,pread_wait_ms,adjacency_cache_hits,adjacency_cache_misses,adjacency_cache_waits,adjacency_cache_evictions,adjacency_cache_lock_acquisitions,adjacency_cache_lock_wait_ns,query_adjacency_cache_peak_bytes,query_adjacency_cache_evictions,rerank_candidate_references,rerank_unique_windows,raw_vector_cache_hits,raw_vector_cache_misses,raw_vector_cache_evictions,parallel_session_queries,simulated_rtt_ms,adjacency_section_bytes,adjacency_pages,file_bytes" + ); + for read_tier in [ + BenchmarkReadTier::Memory, + BenchmarkReadTier::LocalStorage, + BenchmarkReadTier::RemoteStorage, + BenchmarkReadTier::ObjectStore, + ] { + for l_search in [50, 100, 200] { + run_profile( + &dataset, + ProfileRun { + index_path: index_file.path(), + file_bytes, + adjacency_section_bytes, + adjacency_pages, + pq_bits, + storage_layout, + raw_vector_encoding, + build_distance, + read_tier, + l_search, + concurrency, + reader_memory_budget_bytes, + warmup_query_count, + build_time, + pq_training_time, + build_stats, + peak_rss_bytes, + }, + )?; + } + } + if env::var_os("DISKANN_BENCH_FILTERED_MATRIX").is_some() { + run_filtered_matrix(&dataset, index_file.path(), reader_memory_budget_bytes)?; + } + Ok(()) +} + +fn read_adjacency_layout_metrics(path: &Path) -> io::Result<(u64, u64)> { + let mut header_bytes = [0u8; DISKANN_HEADER_SIZE]; + File::open(path)?.read_exact(&mut header_bytes)?; + let header = DiskAnnHeader::decode(&header_bytes)?; + let adjacency_section_bytes = header.sections.adjacency.length; + Ok(( + adjacency_section_bytes, + adjacency_section_bytes / u64::from(DISKANN_PAGE_SIZE), + )) +} + +fn run_filtered_matrix( + dataset: &Dataset, + index_path: &Path, + reader_memory_budget_bytes: usize, +) -> Result<(), Box> { + let stats = Arc::new(Mutex::new(ReadStats::default())); + let source = InstrumentedStore { + inner: BenchmarkFileSource::open(index_path)?, + stats: Arc::clone(&stats), + round_trip_latency: Duration::ZERO, + estimated_random_read_latency_nanos: 0, + }; + let mut adaptive = VectorIndexReader::open_with_options( + source, + VectorIndexReaderOptions::new(reader_memory_budget_bytes), + )?; + adaptive.optimize_for_search()?; + let mut exhaustive = VectorIndexReader::open_with_options( + BenchmarkFileSource::open(index_path)?, + VectorIndexReaderOptions::new(reader_memory_budget_bytes), + )?; + exhaustive.optimize_for_search()?; + let vector_count = dataset.base.len() / dataset.dimension; + let query_count = dataset.queries.len() / dataset.dimension; + let params = VectorSearchParams::with_l_search(10, 200); + println!( + "filtered_distribution,selectivity_percent,matching_nodes,actual_strategy,graph_fallbacks,recall_at_10_vs_exhaustive,elapsed_ms,pread_rounds,pread_ranges,max_ranges_per_round,pread_bytes,pq_distance_evaluations,pq_code_loads,query_chunks,rerank_chunks,rerank_candidate_references,rerank_unique_windows,adjacency_cache_hits,adjacency_cache_misses,adjacency_cache_waits,adjacency_cache_evictions,raw_vector_cache_hits,raw_vector_cache_misses,raw_vector_cache_evictions,parallel_exact_rerank_chunks,parallel_exact_rerank_references" + ); + for distribution in ["random", "clustered"] { + let mut ordered_nodes = (0..vector_count).collect::>(); + if distribution == "random" { + ordered_nodes.sort_unstable_by_key(|node| benchmark_mix(*node as u64)); + } + for selectivity_basis_points in [1usize, 10, 100, 1000, 5000, 10_000] { + let matching_count = vector_count + .saturating_mul(selectivity_basis_points) + .div_ceil(10_000) + .max(1); + let mut filter = RoaringTreemap::new(); + filter.extend( + ordered_nodes[..matching_count] + .iter() + .map(|node| *node as u64), + ); + let mut filter_bytes = Vec::new(); + filter.serialize_into(&mut filter_bytes)?; + *stats.lock().unwrap() = ReadStats::default(); + let started = Instant::now(); + let (adaptive_ids, _) = adaptive.search_batch_with_roaring_filter( + &dataset.queries, + query_count, + params, + &filter_bytes, + )?; + let elapsed = started.elapsed(); + let (exhaustive_ids, _) = exhaustive.search_batch_with_roaring_filter( + &dataset.queries, + query_count, + params, + &filter_bytes, + )?; + let recall = recall_against_exhaustive(&adaptive_ids, &exhaustive_ids, params.top_k); + let search_stats = adaptive + .diskann_search_stats() + .expect("DiskANN benchmark reader diagnostics"); + let strategy = if search_stats.filtered_graph_queries == 0 { + "scan" + } else if search_stats.filtered_graph_fallbacks == 0 { + "graph" + } else { + "graph_fallback" + }; + let snapshot = *stats.lock().unwrap(); + println!( + "{distribution},{selectivity:.2},{matching_count},{strategy},{fallbacks},{recall:.4},{elapsed_ms},{rounds},{ranges},{max_ranges},{bytes},{pq_evaluations},{pq_loads},{query_chunks},{rerank_chunks},{candidate_references},{unique_windows},{adjacency_hits},{adjacency_misses},{adjacency_waits},{adjacency_evictions},{cache_hits},{cache_misses},{cache_evictions},{parallel_rerank_chunks},{parallel_rerank_references}", + selectivity = selectivity_basis_points as f64 / 100.0, + fallbacks = search_stats.filtered_graph_fallbacks, + elapsed_ms = elapsed.as_millis(), + rounds = snapshot.rounds, + ranges = snapshot.ranges, + max_ranges = snapshot.max_ranges_per_round, + bytes = snapshot.bytes, + pq_evaluations = search_stats.pq_distance_evaluations, + pq_loads = search_stats.pq_code_loads, + query_chunks = search_stats.query_chunks, + rerank_chunks = search_stats.rerank_chunks, + candidate_references = search_stats.rerank_candidate_references, + unique_windows = search_stats.rerank_unique_windows, + adjacency_hits = search_stats.adjacency_cache_hits, + adjacency_misses = search_stats.adjacency_cache_misses, + adjacency_waits = search_stats.adjacency_cache_waits, + adjacency_evictions = search_stats.adjacency_cache_evictions, + cache_hits = search_stats.raw_vector_cache_hits, + cache_misses = search_stats.raw_vector_cache_misses, + cache_evictions = search_stats.raw_vector_cache_evictions, + parallel_rerank_chunks = search_stats.parallel_exact_rerank_chunks, + parallel_rerank_references = search_stats.parallel_exact_rerank_references, + ); + if search_stats.filtered_graph_queries != 0 + && env::var_os("DISKANN_BENCH_ACCEPTANCE").is_some() + && recall + 0.01 + f64::EPSILON < 1.0 + { + return Err(format!( + "filtered {distribution} Recall@10 {:.4} is more than 0.01 below exhaustive at {:.2}% selectivity", + recall, + selectivity_basis_points as f64 / 100.0 + ) + .into()); + } + } + } + Ok(()) +} + +fn benchmark_mix(mut value: u64) -> u64 { + value ^= value >> 30; + value = value.wrapping_mul(0xbf58_476d_1ce4_e5b9); + value ^= value >> 27; + value = value.wrapping_mul(0x94d0_49bb_1331_11eb); + value ^ (value >> 31) +} + +fn recall_against_exhaustive(actual: &[i64], expected: &[i64], top_k: usize) -> f64 { + let mut hits = 0usize; + let mut total = 0usize; + for (actual, expected) in actual.chunks_exact(top_k).zip(expected.chunks_exact(top_k)) { + let expected = expected + .iter() + .copied() + .filter(|row_id| *row_id >= 0) + .collect::>(); + total += expected.len(); + hits += actual + .iter() + .filter(|row_id| **row_id >= 0 && expected.contains(row_id)) + .count(); + } + if total == 0 { + 1.0 + } else { + hits as f64 / total as f64 + } +} + +struct ProfileRun<'a> { + index_path: &'a Path, + file_bytes: u64, + adjacency_section_bytes: u64, + adjacency_pages: u64, + pq_bits: usize, + storage_layout: DiskAnnStorageLayout, + raw_vector_encoding: DiskAnnRawVectorEncoding, + build_distance: DiskAnnBuildDistance, + read_tier: BenchmarkReadTier, + l_search: usize, + concurrency: usize, + reader_memory_budget_bytes: usize, + warmup_query_count: usize, + build_time: Duration, + pq_training_time: Duration, + build_stats: DiskAnnBuildStats, + peak_rss_bytes: u64, +} + +fn run_profile(dataset: &Dataset, run: ProfileRun<'_>) -> Result<(), Box> { + let ProfileRun { + index_path, + file_bytes, + adjacency_section_bytes, + adjacency_pages, + pq_bits, + storage_layout, + raw_vector_encoding, + build_distance, + read_tier, + l_search, + concurrency: requested_concurrency, + reader_memory_budget_bytes, + warmup_query_count, + build_time, + pq_training_time, + build_stats, + peak_rss_bytes, + } = run; + let stats = Arc::new(Mutex::new(ReadStats::default())); + let round_trip_latency = Duration::from_millis( + env::var("DISKANN_BENCH_SIMULATED_RTT_MS") + .unwrap_or_else(|_| "0".to_string()) + .parse::()?, + ); + let query_count = dataset.queries.len() / dataset.dimension; + let concurrency = requested_concurrency + .min(query_count.max(1)) + .min(rayon::current_num_threads()); + let source = InstrumentedStore { + inner: BenchmarkFileSource::open(index_path)?, + stats: Arc::clone(&stats), + round_trip_latency, + estimated_random_read_latency_nanos: match read_tier { + BenchmarkReadTier::Memory => 1, + BenchmarkReadTier::LocalStorage => 100_000, + BenchmarkReadTier::RemoteStorage => 2_000_000, + BenchmarkReadTier::ObjectStore => 20_000_000, + }, + }; + let mut reader = VectorIndexReader::open_with_options( + source, + VectorIndexReaderOptions::new(reader_memory_budget_bytes), + )?; + *stats.lock().unwrap() = ReadStats::default(); + let warmup_started = Instant::now(); + reader.warmup_queries( + &dataset.queries[..warmup_query_count * dataset.dimension], + warmup_query_count, + l_search, + )?; + let warmup_time = warmup_started.elapsed(); + let warmup_snapshot = *stats.lock().unwrap(); + *stats.lock().unwrap() = ReadStats::default(); + + let params = VectorSearchParams::with_l_search(10, l_search); + let started = Instant::now(); + let mut outcomes = Vec::with_capacity(query_count); + let first_query_started = Instant::now(); + let (first_ids, _) = reader.search(&dataset.queries[..dataset.dimension], params)?; + let first_search_stats = reader + .diskann_search_stats() + .expect("DiskANN benchmark reader diagnostics"); + let mut adjacency_cache_hits = first_search_stats.adjacency_cache_hits; + let mut adjacency_cache_misses = first_search_stats.adjacency_cache_misses; + let mut adjacency_cache_waits = first_search_stats.adjacency_cache_waits; + let mut adjacency_cache_evictions = first_search_stats.adjacency_cache_evictions; + let mut adjacency_cache_lock_acquisitions = + first_search_stats.adjacency_cache_lock_acquisitions; + let mut adjacency_cache_lock_wait_nanos = first_search_stats.adjacency_cache_lock_wait_nanos; + let mut query_adjacency_cache_peak_bytes = first_search_stats.query_adjacency_cache_peak_bytes; + let mut query_adjacency_cache_evictions = first_search_stats.query_adjacency_cache_evictions; + let mut rerank_candidate_references = first_search_stats.rerank_candidate_references; + let mut rerank_unique_windows = first_search_stats.rerank_unique_windows; + let mut raw_vector_cache_hits = first_search_stats.raw_vector_cache_hits; + let mut raw_vector_cache_misses = first_search_stats.raw_vector_cache_misses; + let mut raw_vector_cache_evictions = first_search_stats.raw_vector_cache_evictions; + let mut parallel_session_queries = first_search_stats.parallel_session_queries; + outcomes.push(QueryOutcome { + query_index: 0, + latency: first_query_started.elapsed(), + result_ids: first_ids, + }); + if concurrency == 1 { + for query_index in 1..query_count { + let query = &dataset.queries + [query_index * dataset.dimension..(query_index + 1) * dataset.dimension]; + let query_started = Instant::now(); + let (result_ids, _) = reader.search(query, params)?; + let search_stats = reader + .diskann_search_stats() + .expect("DiskANN benchmark reader diagnostics"); + adjacency_cache_hits = + adjacency_cache_hits.saturating_add(search_stats.adjacency_cache_hits); + adjacency_cache_misses = + adjacency_cache_misses.saturating_add(search_stats.adjacency_cache_misses); + adjacency_cache_waits = + adjacency_cache_waits.saturating_add(search_stats.adjacency_cache_waits); + adjacency_cache_evictions = + adjacency_cache_evictions.saturating_add(search_stats.adjacency_cache_evictions); + adjacency_cache_lock_acquisitions = adjacency_cache_lock_acquisitions + .saturating_add(search_stats.adjacency_cache_lock_acquisitions); + adjacency_cache_lock_wait_nanos = adjacency_cache_lock_wait_nanos + .saturating_add(search_stats.adjacency_cache_lock_wait_nanos); + query_adjacency_cache_peak_bytes = + query_adjacency_cache_peak_bytes.max(search_stats.query_adjacency_cache_peak_bytes); + query_adjacency_cache_evictions = query_adjacency_cache_evictions + .saturating_add(search_stats.query_adjacency_cache_evictions); + rerank_candidate_references = rerank_candidate_references + .saturating_add(search_stats.rerank_candidate_references); + rerank_unique_windows = + rerank_unique_windows.saturating_add(search_stats.rerank_unique_windows); + raw_vector_cache_hits = + raw_vector_cache_hits.saturating_add(search_stats.raw_vector_cache_hits); + raw_vector_cache_misses = + raw_vector_cache_misses.saturating_add(search_stats.raw_vector_cache_misses); + raw_vector_cache_evictions = + raw_vector_cache_evictions.saturating_add(search_stats.raw_vector_cache_evictions); + parallel_session_queries = + parallel_session_queries.saturating_add(search_stats.parallel_session_queries); + outcomes.push(QueryOutcome { + query_index, + latency: query_started.elapsed(), + result_ids, + }); + } + } else { + for batch_start in (1..query_count).step_by(concurrency) { + let batch_end = (batch_start + concurrency).min(query_count); + let batch_count = batch_end - batch_start; + let batch_queries = + &dataset.queries[batch_start * dataset.dimension..batch_end * dataset.dimension]; + let batch_started = Instant::now(); + let (batch_ids, _) = reader.search_batch(batch_queries, batch_count, params)?; + let search_stats = reader + .diskann_search_stats() + .expect("DiskANN benchmark reader diagnostics"); + adjacency_cache_hits = + adjacency_cache_hits.saturating_add(search_stats.adjacency_cache_hits); + adjacency_cache_misses = + adjacency_cache_misses.saturating_add(search_stats.adjacency_cache_misses); + adjacency_cache_waits = + adjacency_cache_waits.saturating_add(search_stats.adjacency_cache_waits); + adjacency_cache_evictions = + adjacency_cache_evictions.saturating_add(search_stats.adjacency_cache_evictions); + adjacency_cache_lock_acquisitions = adjacency_cache_lock_acquisitions + .saturating_add(search_stats.adjacency_cache_lock_acquisitions); + adjacency_cache_lock_wait_nanos = adjacency_cache_lock_wait_nanos + .saturating_add(search_stats.adjacency_cache_lock_wait_nanos); + query_adjacency_cache_peak_bytes = + query_adjacency_cache_peak_bytes.max(search_stats.query_adjacency_cache_peak_bytes); + query_adjacency_cache_evictions = query_adjacency_cache_evictions + .saturating_add(search_stats.query_adjacency_cache_evictions); + rerank_candidate_references = rerank_candidate_references + .saturating_add(search_stats.rerank_candidate_references); + rerank_unique_windows = + rerank_unique_windows.saturating_add(search_stats.rerank_unique_windows); + raw_vector_cache_hits = + raw_vector_cache_hits.saturating_add(search_stats.raw_vector_cache_hits); + raw_vector_cache_misses = + raw_vector_cache_misses.saturating_add(search_stats.raw_vector_cache_misses); + raw_vector_cache_evictions = + raw_vector_cache_evictions.saturating_add(search_stats.raw_vector_cache_evictions); + parallel_session_queries = + parallel_session_queries.saturating_add(search_stats.parallel_session_queries); + let batch_latency = batch_started.elapsed(); + for batch_index in 0..batch_count { + outcomes.push(QueryOutcome { + query_index: batch_start + batch_index, + latency: batch_latency, + result_ids: batch_ids + [batch_index * params.top_k..(batch_index + 1) * params.top_k] + .to_vec(), + }); + } + } + } + let elapsed = started.elapsed(); + outcomes.sort_unstable_by_key(|outcome| outcome.query_index); + let first_query = outcomes + .first() + .map(|outcome| outcome.latency) + .unwrap_or_default(); + let mut recall_1_hits = 0usize; + let mut recall_10_hits = 0usize; + let mut recall_10_total = 0usize; + let mut query_latencies = Vec::with_capacity(query_count); + for outcome in &outcomes { + query_latencies.push(outcome.latency); + let truth = &dataset.ground_truth[outcome.query_index]; + recall_1_hits += + usize::from(outcome.result_ids.first().copied() == truth.first().map(|id| *id as i64)); + let truth_at_10 = truth.iter().take(10).copied().collect::>(); + recall_10_total += truth_at_10.len(); + recall_10_hits += outcome + .result_ids + .iter() + .filter(|&&id| id >= 0 && truth_at_10.contains(&(id as u32))) + .count(); + } + let warm_query = if query_count > 1 { + query_latencies.iter().skip(1).copied().sum::() / (query_count - 1) as u32 + } else { + first_query + }; + let snapshot = *stats.lock().unwrap(); + let recall_at_1 = recall_1_hits as f64 / query_count as f64; + let recall_at_10 = if recall_10_total == 0 { + 0.0 + } else { + recall_10_hits as f64 / recall_10_total as f64 + }; + let read_tier_name = match read_tier { + BenchmarkReadTier::Memory => "memory", + BenchmarkReadTier::LocalStorage => "local_storage", + BenchmarkReadTier::RemoteStorage => "remote_storage", + BenchmarkReadTier::ObjectStore => "object_store", + }; + let p50_query = percentile(&query_latencies, 50); + let p95_query = percentile(&query_latencies, 95); + let p99_query = percentile(&query_latencies, 99); + let observed_peak_rss_bytes = peak_resident_set_bytes()?.max(peak_rss_bytes); + println!( + "{profile},{storage_layout},{raw_vector_encoding},{build_distance},{l_search},{n},{nq},{d},{pq_bits},10,{concurrency},{reader_memory_budget_bytes},{warmup_query_count},{warmup_ms},{warmup_rounds},{warmup_ranges},{warmup_bytes},{recall_1:.4},{recall_10:.4},{build_ms},{graph_shards},{pq_train_ms},{pq_encode_ms},{vamana_init_ms},{vamana_pass_one_ms},{vamana_pass_two_ms},{connectivity_repair_ms},{locality_remap_ms},{resident_serialize_ms},{adjacency_serialize_ms},{vector_serialize_ms},{peak_rss},{first_us},{p50_us},{p95_us},{p99_us},{warm_us},{qps:.2},{rounds},{ranges},{round_qd},{in_flight_rounds},{in_flight_ranges},{read_bytes},{wait_ms:.3},{adjacency_hits},{adjacency_misses},{adjacency_waits},{adjacency_evictions},{adjacency_lock_acquisitions},{adjacency_lock_wait_nanos},{query_adjacency_peak_bytes},{query_adjacency_evictions},{rerank_candidate_references},{rerank_unique_windows},{cache_hits},{cache_misses},{cache_evictions},{parallel_session_queries},{rtt_ms},{adjacency_section_bytes},{adjacency_pages},{file_bytes}", + profile = read_tier_name, + storage_layout = match storage_layout { + DiskAnnStorageLayout::Compact => "compact", + DiskAnnStorageLayout::Interleaved => "interleaved", + }, + raw_vector_encoding = match raw_vector_encoding { + DiskAnnRawVectorEncoding::F32 => "f32", + DiskAnnRawVectorEncoding::F16 => "f16", + }, + build_distance = match build_distance { + DiskAnnBuildDistance::FullPrecision => "full_precision", + DiskAnnBuildDistance::ProductQuantized => "product_quantized", + }, + l_search = l_search, + n = dataset.base.len() / dataset.dimension, + nq = query_count, + d = dataset.dimension, + warmup_query_count = warmup_query_count, + warmup_ms = warmup_time.as_millis(), + warmup_rounds = warmup_snapshot.rounds, + warmup_ranges = warmup_snapshot.ranges, + warmup_bytes = warmup_snapshot.bytes, + recall_1 = recall_at_1, + recall_10 = recall_at_10, + build_ms = build_time.as_millis(), + graph_shards = build_stats.graph_shards, + pq_train_ms = pq_training_time.as_millis(), + pq_encode_ms = build_stats.pq_encoding.as_millis(), + vamana_init_ms = build_stats.vamana_initialization.as_millis(), + vamana_pass_one_ms = build_stats.vamana_pass_one.as_millis(), + vamana_pass_two_ms = build_stats.vamana_pass_two.as_millis(), + connectivity_repair_ms = build_stats.connectivity_repair.as_millis(), + locality_remap_ms = build_stats.locality_remap.as_millis(), + resident_serialize_ms = build_stats.resident_serialization.as_millis(), + adjacency_serialize_ms = build_stats.adjacency_serialization.as_millis(), + vector_serialize_ms = build_stats.vector_serialization.as_millis(), + peak_rss = observed_peak_rss_bytes, + first_us = first_query.as_micros(), + p50_us = p50_query.as_micros(), + p95_us = p95_query.as_micros(), + p99_us = p99_query.as_micros(), + warm_us = warm_query.as_micros(), + qps = query_count as f64 / elapsed.as_secs_f64(), + rounds = snapshot.rounds, + ranges = snapshot.ranges, + round_qd = snapshot.max_ranges_per_round, + in_flight_rounds = snapshot.max_in_flight_rounds, + in_flight_ranges = snapshot.max_in_flight_ranges, + read_bytes = snapshot.bytes, + wait_ms = snapshot.wait.as_secs_f64() * 1000.0, + adjacency_hits = adjacency_cache_hits, + adjacency_misses = adjacency_cache_misses, + adjacency_waits = adjacency_cache_waits, + adjacency_evictions = adjacency_cache_evictions, + adjacency_lock_acquisitions = adjacency_cache_lock_acquisitions, + adjacency_lock_wait_nanos = adjacency_cache_lock_wait_nanos, + query_adjacency_peak_bytes = query_adjacency_cache_peak_bytes, + query_adjacency_evictions = query_adjacency_cache_evictions, + rerank_candidate_references = rerank_candidate_references, + rerank_unique_windows = rerank_unique_windows, + cache_hits = raw_vector_cache_hits, + cache_misses = raw_vector_cache_misses, + cache_evictions = raw_vector_cache_evictions, + rtt_ms = round_trip_latency.as_millis(), + ); + + if env::var_os("DISKANN_BENCH_ACCEPTANCE").is_some() { + if recall_at_10 < 0.90 { + return Err(format!("Recall@10 {:.4} is below 0.90", recall_at_10).into()); + } + if build_time > Duration::from_secs(2 * 60 * 60) { + return Err("DiskANN build exceeded two hours".into()); + } + if observed_peak_rss_bytes > 4 * 1024 * 1024 * 1024 { + return Err( + format!("DiskANN peak RSS {} exceeds 4 GiB", observed_peak_rss_bytes).into(), + ); + } + if matches!( + read_tier, + BenchmarkReadTier::RemoteStorage | BenchmarkReadTier::ObjectStore + ) && snapshot.rounds > query_count.saturating_mul(8) + { + return Err(format!( + "coalesced-range reads {} exceed seven graph plus one rerank round per query", + snapshot.rounds + ) + .into()); + } + } + Ok(()) +} + +struct QueryOutcome { + query_index: usize, + latency: Duration, + result_ids: Vec, +} + +/// Benchmark-only adapter for exercising local positional I/O. +/// +/// Production integrations provide their own `SeekRead` implementation so the +/// core API does not expose or assume `std::fs::File`. +struct BenchmarkFileSource { + file: File, +} + +impl BenchmarkFileSource { + fn open(path: impl AsRef) -> io::Result { + Ok(Self { + file: File::open(path)?, + }) + } +} + +impl SeekRead for BenchmarkFileSource { + fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { + #[cfg(any(unix, windows))] + { + ranges + .par_iter_mut() + .try_for_each(|range| benchmark_read_exact_at(&self.file, range.buf, range.pos)) + } + + #[cfg(not(any(unix, windows)))] + { + let old_pos = std::io::Seek::stream_position(&mut self.file)?; + for range in ranges { + std::io::Seek::seek(&mut self.file, std::io::SeekFrom::Start(range.pos))?; + std::io::Read::read_exact(&mut self.file, range.buf)?; + } + std::io::Seek::seek(&mut self.file, std::io::SeekFrom::Start(old_pos))?; + Ok(()) + } + } + + fn try_clone_reader(&self) -> io::Result> { + Ok(Some(Self { + file: self.file.try_clone()?, + })) + } +} + +#[cfg(unix)] +fn benchmark_read_exact_at(file: &File, buf: &mut [u8], pos: u64) -> io::Result<()> { + use std::os::unix::fs::FileExt; + file.read_exact_at(buf, pos) +} + +#[cfg(windows)] +fn benchmark_read_exact_at(file: &File, mut buf: &mut [u8], mut pos: u64) -> io::Result<()> { + use std::os::windows::fs::FileExt; + + while !buf.is_empty() { + let read = file.seek_read(buf, pos)?; + if read == 0 { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "failed to fill whole buffer", + )); + } + pos = pos + .checked_add(read as u64) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "read offset overflow"))?; + buf = &mut buf[read..]; + } + Ok(()) +} + +#[derive(Clone, Copy, Default)] +struct ReadStats { + rounds: usize, + ranges: usize, + bytes: usize, + wait: Duration, + max_ranges_per_round: usize, + in_flight_rounds: usize, + in_flight_ranges: usize, + max_in_flight_rounds: usize, + max_in_flight_ranges: usize, +} + +struct InstrumentedStore { + inner: BenchmarkFileSource, + stats: Arc>, + round_trip_latency: Duration, + estimated_random_read_latency_nanos: u64, +} + +impl SeekRead for InstrumentedStore { + fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { + let range_count = ranges.len(); + let byte_count = ranges.iter().map(|range| range.buf.len()).sum::(); + { + let mut stats = self.stats.lock().unwrap(); + stats.in_flight_rounds += 1; + stats.in_flight_ranges += range_count; + stats.max_in_flight_rounds = stats.max_in_flight_rounds.max(stats.in_flight_rounds); + stats.max_in_flight_ranges = stats.max_in_flight_ranges.max(stats.in_flight_ranges); + } + let started = Instant::now(); + if !self.round_trip_latency.is_zero() { + thread::sleep(self.round_trip_latency); + } + let result = self.inner.pread(ranges); + let mut stats = self.stats.lock().unwrap(); + stats.in_flight_rounds -= 1; + stats.in_flight_ranges -= range_count; + stats.rounds += 1; + stats.ranges += range_count; + stats.max_ranges_per_round = stats.max_ranges_per_round.max(range_count); + stats.bytes += byte_count; + stats.wait += started.elapsed(); + result + } + + fn try_clone_reader(&self) -> io::Result> { + let Some(inner) = self.inner.try_clone_reader()? else { + return Ok(None); + }; + Ok(Some(Self { + inner, + stats: Arc::clone(&self.stats), + round_trip_latency: self.round_trip_latency, + estimated_random_read_latency_nanos: self.estimated_random_read_latency_nanos, + })) + } + + fn read_capabilities(&self) -> SeekReadCapabilities { + SeekReadCapabilities { + estimated_random_read_latency_nanos: self.estimated_random_read_latency_nanos, + ..SeekReadCapabilities::default() + } + } +} + +struct TemporaryIndexFile { + path: PathBuf, +} + +impl TemporaryIndexFile { + fn create() -> io::Result<(Self, File)> { + let directory = env::var_os("DISKANN_BENCH_INDEX_DIR") + .map(PathBuf::from) + .unwrap_or_else(env::temp_dir); + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(io::Error::other)? + .as_nanos(); + for attempt in 0..100 { + let path = directory.join(format!( + "paimon-vindex-diskann-{}-{}-{}.dann", + std::process::id(), + timestamp, + attempt + )); + match OpenOptions::new() + .create_new(true) + .read(true) + .write(true) + .open(&path) + { + Ok(file) => return Ok((Self { path }, file)), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error), + } + } + Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "failed to allocate a unique DiskANN benchmark index file", + )) + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TemporaryIndexFile { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.path); + } +} + +fn percentile(samples: &[Duration], percentile: usize) -> Duration { + if samples.is_empty() { + return Duration::ZERO; + } + let mut sorted = samples.to_vec(); + sorted.sort_unstable(); + let index = (sorted.len() - 1) * percentile.min(100) / 100; + sorted[index] +} + +#[cfg(unix)] +fn peak_resident_set_bytes() -> io::Result { + let mut usage = MaybeUninit::::zeroed(); + let status = unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) }; + if status != 0 { + return Err(io::Error::last_os_error()); + } + let max_rss = unsafe { usage.assume_init() }.ru_maxrss as u64; + #[cfg(target_os = "macos")] + return Ok(max_rss); + #[cfg(not(target_os = "macos"))] + return Ok(max_rss.saturating_mul(1024)); +} + +#[cfg(not(unix))] +fn peak_resident_set_bytes() -> io::Result { + Ok(0) +} + +struct Dataset { + dimension: usize, + base: Vec, + queries: Vec, + ground_truth: Vec>, +} + +impl Dataset { + fn from_env_or_smoke() -> Result> { + match ( + env::var_os("DISKANN_BASE_FVECS"), + env::var_os("DISKANN_QUERY_FVECS"), + env::var_os("DISKANN_GROUND_TRUTH_IVECS"), + ) { + (Some(base), Some(queries), Some(truth)) => { + let (dimension, base) = read_fvecs(Path::new(&base))?; + let (query_dimension, queries) = read_fvecs(Path::new(&queries))?; + if query_dimension != dimension { + return Err("base/query fvec dimensions differ".into()); + } + let ground_truth = read_ivecs(Path::new(&truth))?; + if ground_truth.len() != queries.len() / dimension { + return Err("query and ground-truth counts differ".into()); + } + Ok(Self { + dimension, + base, + queries, + ground_truth, + }) + } + (None, None, None) => Self::smoke(), + _ => Err("set all three DISKANN_* dataset paths or none of them".into()), + } + } + + fn smoke() -> Result> { + let n = env::var("DISKANN_BENCH_SMOKE_N") + .unwrap_or_else(|_| "512".to_string()) + .parse::()?; + let nq = env::var("DISKANN_BENCH_SMOKE_NQ") + .unwrap_or_else(|_| "16".to_string()) + .parse::()?; + let dimension = env::var("DISKANN_BENCH_SMOKE_DIMENSION") + .unwrap_or_else(|_| "32".to_string()) + .parse::()?; + if n == 0 || nq == 0 || dimension == 0 { + return Err("DiskANN smoke n, nq, and dimension must be positive".into()); + } + let mut state = 42u64; + let mut base = vec![0.0f32; n * dimension]; + for value in &mut base { + state = state.wrapping_mul(6364136223846793005).wrapping_add(1); + *value = (state >> 32) as f32 / u32::MAX as f32; + } + let mut queries = Vec::with_capacity(nq * dimension); + let mut ground_truth = Vec::with_capacity(nq); + for query in 0..nq { + let source = query * 31 % n; + let query_vector = &base[source * dimension..(source + 1) * dimension]; + queries.extend_from_slice(query_vector); + let mut exact = base + .chunks_exact(dimension) + .enumerate() + .map(|(node, vector)| (fvec_l2sqr(query_vector, vector), node as u32)) + .collect::>(); + exact.select_nth_unstable_by(10.min(n).saturating_sub(1), |left, right| { + left.0 + .total_cmp(&right.0) + .then_with(|| left.1.cmp(&right.1)) + }); + exact.truncate(10.min(n)); + exact.sort_unstable_by(|left, right| { + left.0 + .total_cmp(&right.0) + .then_with(|| left.1.cmp(&right.1)) + }); + ground_truth.push(exact.into_iter().map(|(_, node)| node).collect()); + } + Ok(Self { + dimension, + base, + queries, + ground_truth, + }) + } +} + +fn read_fvecs(path: &Path) -> io::Result<(usize, Vec)> { + let rows = read_i32_records(path)?; + let dimension = rows.first().map(Vec::len).unwrap_or(0); + if dimension == 0 || rows.iter().any(|row| row.len() != dimension) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid fvecs shape", + )); + } + Ok(( + dimension, + rows.into_iter() + .flatten() + .map(|bits| f32::from_bits(bits as u32)) + .collect(), + )) +} + +fn read_ivecs(path: &Path) -> io::Result>> { + Ok(read_i32_records(path)? + .into_iter() + .map(|row| row.into_iter().map(|value| value as u32).collect()) + .collect()) +} + +fn read_i32_records(path: &Path) -> io::Result>> { + let mut reader = BufReader::new(File::open(path)?); + let mut rows = Vec::new(); + loop { + let mut length = [0u8; 4]; + match reader.read_exact(&mut length) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => break, + Err(error) => return Err(error), + } + let length = i32::from_le_bytes(length); + if length <= 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid record length", + )); + } + let mut row = Vec::with_capacity(length as usize); + for _ in 0..length { + let mut value = [0u8; 4]; + reader.read_exact(&mut value)?; + row.push(i32::from_le_bytes(value)); + } + rows.push(row); + } + Ok(rows) +} diff --git a/core/benches/ivfhnswsq_filter_bench.rs b/core/benches/ivfhnswsq_filter_bench.rs deleted file mode 100644 index ab82310..0000000 --- a/core/benches/ivfhnswsq_filter_bench.rs +++ /dev/null @@ -1,314 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use paimon_vindex_core::distance::MetricType; -use paimon_vindex_core::hnsw::HnswBuildParams; -use paimon_vindex_core::index::{ - VectorIndexConfig, VectorIndexReader, VectorIndexTrainer, VectorIndexWriter, VectorSearchParams, -}; -use paimon_vindex_core::io::PosWriter; -use roaring::RoaringTreemap; -use std::env; -use std::io::{self, Cursor}; -use std::time::{Duration, Instant}; - -fn main() -> Result<(), Box> { - let cfg = Config::from_env()?; - cfg.validate()?; - - let (data, queries) = generate_clustered_data(cfg.n, cfg.nq, cfg.d, cfg.clusters, cfg.seed); - let ids: Vec = (0..cfg.n as i64).collect(); - - let start = Instant::now(); - let training = VectorIndexTrainer::train( - VectorIndexConfig::IvfHnswSq { - dimension: cfg.d, - nlist: cfg.nlist, - metric: MetricType::L2, - hnsw: cfg.hnsw_params(), - }, - &data, - cfg.n, - )?; - let mut writer = VectorIndexWriter::new(training); - writer.add_vectors(&ids, &data, cfg.n)?; - let mut index_bytes = Vec::new(); - writer.write(&mut PosWriter::new(&mut index_bytes))?; - let build = start.elapsed(); - - println!( - "n={},nq={},d={},k={},nlist={},nprobe={},ef_search={},index_bytes={},build_ms={}", - cfg.n, - cfg.nq, - cfg.d, - cfg.k, - cfg.nlist, - cfg.nprobe, - cfg.ef_search, - index_bytes.len(), - build.as_millis() - ); - println!("filter_stride,allowed,optimized,warmup_ms,search_ms,us_per_query,qps"); - - for stride in &cfg.filter_strides { - let filter_bytes = filter_bytes(cfg.n, *stride)?; - let allowed = cfg.n.div_ceil(*stride); - let baseline = run_case(&cfg, &index_bytes, &queries, &filter_bytes, false)?; - let optimized = run_case(&cfg, &index_bytes, &queries, &filter_bytes, true)?; - assert_same_results(*stride, &baseline.result, &optimized.result); - print_row( - *stride, - allowed, - false, - baseline.warmup, - baseline.search, - cfg.nq, - ); - print_row( - *stride, - allowed, - true, - optimized.warmup, - optimized.search, - cfg.nq, - ); - } - - Ok(()) -} - -struct Config { - n: usize, - nq: usize, - d: usize, - k: usize, - nlist: usize, - nprobe: usize, - ef_search: usize, - hnsw_m: usize, - hnsw_ef_construction: usize, - hnsw_max_level: usize, - clusters: usize, - seed: u64, - filter_strides: Vec, -} - -impl Config { - fn from_env() -> Result> { - Ok(Self { - n: read_env("FILTER_BENCH_N", 50_000)?, - nq: read_env("FILTER_BENCH_NQ", 500)?, - d: read_env("FILTER_BENCH_D", 128)?, - k: read_env("FILTER_BENCH_K", 10)?, - nlist: read_env("FILTER_BENCH_NLIST", 64)?, - nprobe: read_env("FILTER_BENCH_NPROBE", 32)?, - ef_search: read_env("FILTER_BENCH_EF_SEARCH", 80)?, - hnsw_m: read_env("FILTER_BENCH_HNSW_M", 20)?, - hnsw_ef_construction: read_env("FILTER_BENCH_HNSW_EF_CONSTRUCTION", 150)?, - hnsw_max_level: read_env("FILTER_BENCH_HNSW_MAX_LEVEL", 7)?, - clusters: read_env("FILTER_BENCH_CLUSTERS", 32)?, - seed: read_env("FILTER_BENCH_SEED", 42)?, - filter_strides: read_strides("FILTER_BENCH_FILTER_STRIDES", &[1, 4, 16, 64])?, - }) - } - - fn validate(&self) -> Result<(), Box> { - if self.n == 0 || self.nq == 0 || self.d == 0 || self.k == 0 { - return Err("FILTER_BENCH_N, NQ, D, and K must be greater than 0".into()); - } - if self.nlist == 0 || self.nprobe == 0 || self.nprobe > self.nlist || self.nlist > self.n { - return Err("FILTER_BENCH_NLIST/NPROBE must satisfy 0 < nprobe <= nlist <= n".into()); - } - if self.clusters == 0 { - return Err("FILTER_BENCH_CLUSTERS must be greater than 0".into()); - } - if self.filter_strides.is_empty() || self.filter_strides.contains(&0) { - return Err("FILTER_BENCH_FILTER_STRIDES must contain positive integers".into()); - } - Ok(()) - } - - fn hnsw_params(&self) -> HnswBuildParams { - HnswBuildParams { - m: self.hnsw_m, - ef_construction: self.hnsw_ef_construction, - max_level: self.hnsw_max_level, - } - .sanitized() - } -} - -struct CaseResult { - warmup: Duration, - search: Duration, - result: (Vec, Vec), -} - -fn run_case( - cfg: &Config, - index_bytes: &[u8], - queries: &[f32], - filter_bytes: &[u8], - optimized: bool, -) -> io::Result { - let mut reader = VectorIndexReader::open(Cursor::new(index_bytes.to_vec()))?; - let warmup_start = Instant::now(); - if optimized { - reader.optimize_for_search()?; - } - let warmup = warmup_start.elapsed(); - - let params = VectorSearchParams::with_ef_search(cfg.k, cfg.nprobe, cfg.ef_search); - let _ = reader.search_batch_with_roaring_filter(queries, cfg.nq, params, filter_bytes)?; - - let start = Instant::now(); - let result = reader.search_batch_with_roaring_filter(queries, cfg.nq, params, filter_bytes)?; - let search = start.elapsed(); - Ok(CaseResult { - warmup, - search, - result, - }) -} - -fn filter_bytes(n: usize, stride: usize) -> io::Result> { - let mut filter = RoaringTreemap::new(); - for id in (0..n as u64).step_by(stride) { - filter.insert(id); - } - let mut bytes = Vec::new(); - filter.serialize_into(&mut bytes)?; - Ok(bytes) -} - -fn assert_same_results( - stride: usize, - expected: &(Vec, Vec), - actual: &(Vec, Vec), -) { - assert_eq!( - actual.0, expected.0, - "ids should match for stride {}", - stride - ); - assert_eq!( - actual.1.len(), - expected.1.len(), - "distance count should match for stride {}", - stride - ); - for (actual, expected) in actual.1.iter().zip(expected.1.iter()) { - assert!( - (actual - expected).abs() < 1e-4, - "distance {} should match {} for stride {}", - actual, - expected, - stride - ); - } -} - -fn print_row( - filter_stride: usize, - allowed: usize, - optimized: bool, - warmup: Duration, - search: Duration, - nq: usize, -) { - println!( - "{},{},{},{},{},{:.2},{:.2}", - filter_stride, - allowed, - optimized, - warmup.as_millis(), - search.as_millis(), - search.as_secs_f64() * 1_000_000.0 / nq as f64, - nq as f64 / search.as_secs_f64(), - ); -} - -fn generate_clustered_data( - n: usize, - nq: usize, - d: usize, - clusters: usize, - seed: u64, -) -> (Vec, Vec) { - let mut rng = Lcg::new(seed); - let mut centers = vec![0.0f32; clusters * d]; - for value in &mut centers { - *value = rng.next_f32() * 30.0; - } - - let mut data = vec![0.0f32; n * d]; - for i in 0..n { - let cluster = i % clusters; - for j in 0..d { - data[i * d + j] = centers[cluster * d + j] + rng.next_f32(); - } - } - - let mut queries = vec![0.0f32; nq * d]; - for qi in 0..nq { - let source = (qi * 9973) % n; - queries[qi * d..(qi + 1) * d].copy_from_slice(&data[source * d..(source + 1) * d]); - } - (data, queries) -} - -fn read_env(name: &str, default: T) -> Result> -where - T: std::str::FromStr, - T::Err: std::error::Error + 'static, -{ - match env::var(name) { - Ok(value) => Ok(value.parse()?), - Err(env::VarError::NotPresent) => Ok(default), - Err(err) => Err(Box::new(err)), - } -} - -fn read_strides(name: &str, default: &[usize]) -> Result, Box> { - match env::var(name) { - Ok(value) => value - .split(',') - .map(|part| { - part.trim() - .parse::() - .map_err(|e| -> Box { Box::new(e) }) - }) - .collect(), - Err(env::VarError::NotPresent) => Ok(default.to_vec()), - Err(err) => Err(Box::new(err)), - } -} - -struct Lcg { - state: u64, -} - -impl Lcg { - fn new(seed: u64) -> Self { - Self { state: seed } - } - - fn next_f32(&mut self) -> f32 { - self.state = self.state.wrapping_mul(6364136223846793005).wrapping_add(1); - ((self.state >> 33) as f32) / (u32::MAX as f32) * 2.0 - 1.0 - } -} diff --git a/core/benches/recall_bench.rs b/core/benches/recall_bench.rs index aa6896a..d2ec25a 100644 --- a/core/benches/recall_bench.rs +++ b/core/benches/recall_bench.rs @@ -16,15 +16,12 @@ // under the License. use paimon_vindex_core::distance::{fvec_distance, MetricType}; -use paimon_vindex_core::hnsw::HnswBuildParams; use paimon_vindex_core::io::{write_index, PosWriter}; use paimon_vindex_core::ivfflat::IVFFlatIndex; use paimon_vindex_core::ivfflat_io::write_ivfflat_index; -use paimon_vindex_core::ivfhnswflat::IVFHNSWFlatIndex; -use paimon_vindex_core::ivfhnswflat_io::write_ivfhnswflat_index; -use paimon_vindex_core::ivfhnswsq::IVFHNSWSQIndex; -use paimon_vindex_core::ivfhnswsq_io::write_ivfhnswsq_index; use paimon_vindex_core::ivfpq::IVFPQIndex; +use paimon_vindex_core::ivfsq::IVFSQIndex; +use paimon_vindex_core::ivfsq_io::write_ivfsq_index; use std::collections::HashSet; use std::time::Instant; @@ -38,7 +35,6 @@ fn main() { nlist: 64, pq_m: 8, nprobes: &[1, 4, 8, 16, 32, 64], - hnsw_search_efs: &[80], }); println!(); @@ -52,7 +48,6 @@ fn main() { nlist: 8, pq_m: 8, nprobes: &[1, 2, 4, 8], - hnsw_search_efs: &[80, 160, 320], }); } @@ -65,7 +60,6 @@ struct Scenario<'a> { nlist: usize, pq_m: usize, nprobes: &'a [usize], - hnsw_search_efs: &'a [usize], } fn run_scenario(s: Scenario<'_>) { @@ -103,7 +97,7 @@ fn run_scenario(s: Scenario<'_>) { println!("build IVF-FLAT: {:.2}s", start.elapsed().as_secs_f64()); let start = Instant::now(); - let mut ivfsq = IVFHNSWSQIndex::new(s.d, s.nlist, MetricType::L2, HnswBuildParams::default()); + let mut ivfsq = IVFSQIndex::new(s.d, s.nlist, MetricType::L2); ivfsq.train(&data, s.n); ivfsq.add(&data, &ids, s.n); println!("build IVF-SQ scan: {:.2}s", start.elapsed().as_secs_f64()); @@ -111,11 +105,8 @@ fn run_scenario(s: Scenario<'_>) { println!(); println!("baseline exact scans over stored representations"); - println!( - "index nprobe ef recall@{} query_ms us/query", - s.k - ); - println!("--------- ------ ------ --------- -------- --------"); + println!("index nprobe recall@{} query_ms us/query", s.k); + println!("--------- ------ --------- -------- --------"); for &nprobe in s.nprobes { let mut distances = vec![0.0f32; s.nq * s.k]; @@ -126,7 +117,6 @@ fn run_scenario(s: Scenario<'_>) { print_row( "IVF-PQ", nprobe, - None, recall_at_k(&labels, &ground_truth, s.nq, s.k), elapsed, s.nq, @@ -140,7 +130,6 @@ fn run_scenario(s: Scenario<'_>) { print_row( "IVF-FLAT", nprobe, - None, recall_at_k(&labels, &ground_truth, s.nq, s.k), elapsed, s.nq, @@ -149,154 +138,44 @@ fn run_scenario(s: Scenario<'_>) { let mut distances = vec![0.0f32; s.nq * s.k]; let mut labels = vec![0i64; s.nq * s.k]; let start = Instant::now(); - ivfsq.search(queries, s.nq, s.k, nprobe, s.k, &mut distances, &mut labels); + ivfsq.search(queries, s.nq, s.k, nprobe, &mut distances, &mut labels); let elapsed = start.elapsed(); print_row( "IVF-SQ", nprobe, - None, recall_at_k(&labels, &ground_truth, s.nq, s.k), elapsed, s.nq, ); } - - let hnsw_params = HnswBuildParams::default(); - println!(); - println!( - "hnsw params: m={}, ef_construction={}", - hnsw_params.m, hnsw_params.ef_construction - ); - - let start = Instant::now(); - let mut ivfhnswflat = IVFHNSWFlatIndex::new(s.d, s.nlist, MetricType::L2, hnsw_params); - ivfhnswflat.train(&data, s.n); - ivfhnswflat.add(&data, &ids, s.n); - ivfhnswflat.build_graphs().unwrap(); - println!("build IVF-HNSW-FLAT: {:.2}s", start.elapsed().as_secs_f64()); - - let start = Instant::now(); - let mut ivfhnswsq = IVFHNSWSQIndex::new(s.d, s.nlist, MetricType::L2, hnsw_params); - ivfhnswsq.train(&data, s.n); - ivfhnswsq.add(&data, &ids, s.n); - ivfhnswsq.build_graphs().unwrap(); - println!("build IVF-HNSW-SQ: {:.2}s", start.elapsed().as_secs_f64()); - print_hnsw_sizes(&ivfhnswflat, &ivfhnswsq); - - println!( - "index nprobe ef recall@{} query_ms us/query", - s.k - ); - println!("--------- ------ ------ --------- -------- --------"); - - for &nprobe in s.nprobes { - for &ef_search in s.hnsw_search_efs { - let mut distances = vec![0.0f32; s.nq * s.k]; - let mut labels = vec![0i64; s.nq * s.k]; - let start = Instant::now(); - ivfhnswflat.search( - queries, - s.nq, - s.k, - nprobe, - ef_search, - &mut distances, - &mut labels, - ); - let elapsed = start.elapsed(); - print_row( - "IVF-HNSW", - nprobe, - Some(ef_search), - recall_at_k(&labels, &ground_truth, s.nq, s.k), - elapsed, - s.nq, - ); - - let mut distances = vec![0.0f32; s.nq * s.k]; - let mut labels = vec![0i64; s.nq * s.k]; - let start = Instant::now(); - ivfhnswsq.search( - queries, - s.nq, - s.k, - nprobe, - ef_search, - &mut distances, - &mut labels, - ); - let elapsed = start.elapsed(); - print_row( - "IVF-HSQ", - nprobe, - Some(ef_search), - recall_at_k(&labels, &ground_truth, s.nq, s.k), - elapsed, - s.nq, - ); - } - } } -fn print_base_sizes(ivfpq: &IVFPQIndex, ivfflat: &IVFFlatIndex, ivfsq: &IVFHNSWSQIndex) { +fn print_base_sizes(ivfpq: &IVFPQIndex, ivfflat: &IVFFlatIndex, ivfsq: &IVFSQIndex) { let mut pq = Vec::new(); write_index(ivfpq, &mut PosWriter::new(&mut pq)).unwrap(); let mut flat = Vec::new(); write_ivfflat_index(ivfflat, &mut PosWriter::new(&mut flat)).unwrap(); + let mut sq = Vec::new(); + write_ivfsq_index(ivfsq, &mut PosWriter::new(&mut sq)).unwrap(); println!( - "baseline sizes: IVF-PQ={:.2} MiB, IVF-FLAT={:.2} MiB, IVF-SQ payload~{:.2} MiB", + "serialized sizes: IVF-PQ={:.2} MiB, IVF-FLAT={:.2} MiB, IVF-SQ={:.2} MiB", bytes_to_mib(pq.len()), bytes_to_mib(flat.len()), - bytes_to_mib(ivfsq_payload_bytes(ivfsq)) - ); -} - -fn print_hnsw_sizes(ivfhnswflat: &IVFHNSWFlatIndex, ivfhnswsq: &IVFHNSWSQIndex) { - let mut hnswflat = Vec::new(); - write_ivfhnswflat_index(ivfhnswflat, &mut PosWriter::new(&mut hnswflat)).unwrap(); - let mut hnswsq = Vec::new(); - write_ivfhnswsq_index(ivfhnswsq, &mut PosWriter::new(&mut hnswsq)).unwrap(); - - println!( - "serialized sizes: IVF-HNSW-FLAT={:.2} MiB, IVF-HNSW-SQ={:.2} MiB", - bytes_to_mib(hnswflat.len()), - bytes_to_mib(hnswsq.len()) + bytes_to_mib(sq.len()) ); } -fn ivfsq_payload_bytes(index: &IVFHNSWSQIndex) -> usize { - let id_bytes: usize = index.ids.iter().map(|ids| ids.len() * 8).sum(); - let code_bytes: usize = index.codes.iter().map(Vec::len).sum(); - let centroid_bytes = index.quantizer_centroids.len() * std::mem::size_of::(); - let global_sq_bytes = (index.sq.mins.len() + index.sq.maxs.len()) * std::mem::size_of::(); - let list_sq_bytes: usize = index - .list_sqs - .iter() - .map(|sq| (sq.mins.len() + sq.maxs.len()) * std::mem::size_of::()) - .sum(); - id_bytes + code_bytes + centroid_bytes + global_sq_bytes + list_sq_bytes -} - fn bytes_to_mib(bytes: usize) -> f64 { bytes as f64 / 1024.0 / 1024.0 } -fn print_row( - index: &str, - nprobe: usize, - ef: Option, - recall: f64, - elapsed: std::time::Duration, - nq: usize, -) { +fn print_row(index: &str, nprobe: usize, recall: f64, elapsed: std::time::Duration, nq: usize) { let ms = elapsed.as_secs_f64() * 1000.0; - let ef = ef.map(|v| v.to_string()).unwrap_or_else(|| "-".to_string()); println!( - "{:<9} {:>6} {:>6} {:>8.2}% {:>8.2} {:>8.1}", + "{:<9} {:>6} {:>8.2}% {:>8.2} {:>8.1}", index, nprobe, - ef, recall * 100.0, ms, ms * 1000.0 / nq as f64 diff --git a/core/benches/support/ann_bench_support.rs b/core/benches/support/ann_bench_support.rs new file mode 100644 index 0000000..649f3c4 --- /dev/null +++ b/core/benches/support/ann_bench_support.rs @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared support for the public ANN benchmark and its focused tests. + +use std::fs::File; +use std::io::{self, BufReader, Read}; +use std::path::Path; +use std::time::Duration; + +pub(crate) const DEFAULT_TRAINING_VECTORS: usize = 65_536; +pub(crate) const ALL_STORAGE_CASE_NAMES: [&str; 3] = [ + "local_ssd_warm_cache", + "remote_cache_2ms", + "object_store_20ms", +]; +const TRAINING_VECTORS_PER_LIST: usize = 64; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct PublicDatasetShape { + pub(crate) vector_count: usize, + pub(crate) query_count: usize, + pub(crate) dimension: usize, + pub(crate) ground_truth_width: usize, +} + +pub(crate) fn inspect_public_dataset( + base: &Path, + queries: &Path, + ground_truth: &Path, +) -> io::Result { + let (vector_count, dimension) = inspect_i32_records(base)?; + let (query_count, query_dimension) = inspect_i32_records(queries)?; + if query_dimension != dimension { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("base/query fvec dimensions differ: {dimension}/{query_dimension}"), + )); + } + let (ground_truth_count, ground_truth_width) = inspect_i32_records(ground_truth)?; + if ground_truth_count != query_count { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("query/ground-truth counts differ: {query_count}/{ground_truth_count}"), + )); + } + Ok(PublicDatasetShape { + vector_count, + query_count, + dimension, + ground_truth_width, + }) +} + +pub(crate) fn resolve_shape_value( + name: &str, + explicit: Option, + inferred: Option, + generated_default: usize, +) -> io::Result { + match (explicit, inferred) { + (Some(value), Some(inferred)) if value != inferred => Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name}={value} does not match public dataset shape {inferred}"), + )), + (Some(value), _) => Ok(value), + (None, Some(inferred)) => Ok(inferred), + (None, None) => Ok(generated_default), + } +} + +pub(crate) fn default_training_vector_count( + vector_count: usize, + nlist: usize, +) -> io::Result { + let list_training_vectors = nlist + .checked_mul(TRAINING_VECTORS_PER_LIST) + .ok_or_else(|| io::Error::other("ANN training sample count overflows usize"))?; + Ok(vector_count.min(DEFAULT_TRAINING_VECTORS.max(list_training_vectors))) +} + +pub(crate) fn should_isolate_indexes( + has_public_dataset: bool, + selected_index_count: usize, + is_child: bool, + reuses_index: bool, +) -> bool { + has_public_dataset && selected_index_count > 1 && !is_child && !reuses_index +} + +pub(crate) fn parse_storage_case_names(value: Option<&str>) -> io::Result> { + let value = value.unwrap_or("all"); + if value.trim().eq_ignore_ascii_case("all") { + return Ok(ALL_STORAGE_CASE_NAMES.to_vec()); + } + let mut selected = Vec::new(); + for item in value.split(',') { + let normalized = item.trim().to_ascii_lowercase().replace('-', "_"); + let Some(name) = ALL_STORAGE_CASE_NAMES + .iter() + .copied() + .find(|name| *name == normalized) + else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "unknown ANN_STORAGE_CASES value '{item}'; expected all or a comma-separated subset of {}", + ALL_STORAGE_CASE_NAMES.join(",") + ), + )); + }; + if !selected.contains(&name) { + selected.push(name); + } + } + if selected.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "ANN_STORAGE_CASES must select at least one storage case", + )); + } + Ok(selected) +} + +pub(crate) fn add_fixed_round_latency( + elapsed: Duration, + rounds: usize, + latency_per_round: Duration, +) -> Duration { + let rounds = u32::try_from(rounds).unwrap_or(u32::MAX); + elapsed.saturating_add(latency_per_round.saturating_mul(rounds)) +} + +pub(crate) fn inspect_i32_records(path: &Path) -> io::Result<(usize, usize)> { + let file = File::open(path)?; + let file_bytes = file.metadata()?.len(); + let mut reader = BufReader::new(file); + let mut header = [0u8; 4]; + reader.read_exact(&mut header)?; + let width = i32::from_le_bytes(header); + if width <= 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("{} has invalid record width {width}", path.display()), + )); + } + let record_bytes = 4u64 + .checked_add( + (width as u64) + .checked_mul(4) + .ok_or_else(|| io::Error::other("record byte size overflow"))?, + ) + .ok_or_else(|| io::Error::other("record byte size overflow"))?; + if file_bytes == 0 || file_bytes % record_bytes != 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "{} size {file_bytes} is not a multiple of record size {record_bytes}", + path.display() + ), + )); + } + let rows = usize::try_from(file_bytes / record_bytes) + .map_err(|_| io::Error::other("record count exceeds usize"))?; + Ok((rows, width as usize)) +} diff --git a/core/src/autotune.rs b/core/src/autotune.rs new file mode 100644 index 0000000..a7adba0 --- /dev/null +++ b/core/src/autotune.rs @@ -0,0 +1,470 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::diskann::{ + DiskAnnBuildDistance, DiskAnnBuildParams, DiskAnnRawVectorEncoding, DiskAnnStorageLayout, +}; +use crate::index::IndexType; +use crate::read_options::DeploymentProfile; +use crate::rq::padded_dimension; +use std::io; + +pub const MIN_IVF_TRAINING_VECTORS: usize = 65_536; +pub const IVF_TRAINING_VECTORS_PER_LIST: usize = 64; +pub const DEFAULT_IVF_LIST_FRACTION: usize = 16; +pub const DEFAULT_IVF_MIN_NPROBE: usize = 8; +pub const DEFAULT_IVF_CANDIDATES_PER_RESULT: usize = 4; + +/// A user setting which is either resolved by the planner or explicitly pinned. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum AutoValue { + #[default] + Auto, + Explicit(T), +} + +impl AutoValue { + pub fn explicit(self) -> Option { + match self { + Self::Auto => None, + Self::Explicit(value) => Some(value), + } + } +} + +/// Accuracy, storage, and build constraints used by optional offline tuning. +/// +/// Recall is deliberately optional: callers without representative queries and +/// ground truth can still resolve deterministic storage constraints, but must +/// not claim a measured recall guarantee. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct TuningObjective { + pub target_recall: Option, + pub max_bytes_per_vector: Option, + pub max_build_seconds: Option, + pub deployment_profile: DeploymentProfile, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct RecallCandidate { + pub value: T, + pub measured_recall: f32, + pub bytes_per_vector: usize, + pub build_seconds: f64, +} + +/// Selects the smallest candidate satisfying every supplied measured target. +/// +/// Ties prefer lower build time. No fallback is hidden: callers get `None` +/// when the calibration sample does not contain a candidate meeting the goal. +pub fn select_calibrated_candidate( + candidates: &[RecallCandidate], + objective: TuningObjective, +) -> Option> { + candidates + .iter() + .copied() + .filter(|candidate| { + objective + .target_recall + .is_none_or(|target| candidate.measured_recall >= target) + && objective + .max_bytes_per_vector + .is_none_or(|limit| candidate.bytes_per_vector <= limit) + && objective + .max_build_seconds + .is_none_or(|limit| candidate.build_seconds <= limit) + }) + .min_by(|left, right| { + left.bytes_per_vector + .cmp(&right.bytes_per_vector) + .then_with(|| left.build_seconds.total_cmp(&right.build_seconds)) + .then_with(|| right.measured_recall.total_cmp(&left.measured_recall)) + }) +} + +/// Chooses the nearest power-of-two list count around `sqrt(vector_count)`. +/// +/// At least 64 training vectors per list are retained when the corpus is large +/// enough. Tiny corpora degrade to one list instead of creating poorly trained +/// centroids. +pub fn infer_ivf_nlist(vector_count: usize) -> io::Result { + if vector_count == 0 { + return Err(invalid_input( + "expected vector count must be greater than 0 for automatic nlist", + )); + } + + let root = (vector_count as f64).sqrt().max(1.0); + let lower = floor_power_of_two(root.floor() as usize); + let upper = lower.checked_mul(2).unwrap_or(lower); + let mut candidate = if root - lower as f64 <= upper as f64 - root { + lower + } else { + upper + }; + let training_cap = (vector_count / IVF_TRAINING_VECTORS_PER_LIST).max(1); + candidate = candidate.min(floor_power_of_two(training_cap)); + Ok(candidate.max(1)) +} + +pub fn default_training_vector_count(vector_count: usize, nlist: usize) -> io::Result { + if vector_count == 0 { + return Err(invalid_input("vector count must be greater than 0")); + } + if nlist == 0 { + return Err(invalid_input("nlist must be greater than 0")); + } + let per_list = nlist + .checked_mul(IVF_TRAINING_VECTORS_PER_LIST) + .ok_or_else(|| invalid_input("automatic training vector count overflows usize"))?; + Ok(vector_count.min(MIN_IVF_TRAINING_VECTORS.max(per_list))) +} + +/// Resolves the initial number of IVF lists to probe. +/// +/// The policy scans at least 1/16 of coarse lists and enough average list rows +/// for four candidates per requested result. Filtering scales this initial +/// width by inverse selectivity; search wrappers may still expand progressively +/// when invalid/padded results remain. +pub fn infer_ivf_nprobe( + nlist: usize, + vector_count: usize, + top_k: usize, + matching_count: Option, +) -> io::Result { + if nlist == 0 { + return Err(invalid_input("nlist must be greater than 0")); + } + if vector_count == 0 { + return Err(invalid_input("vector count must be greater than 0")); + } + if top_k == 0 { + return Err(invalid_input("top_k must be greater than 0")); + } + + let average_list_rows = vector_count.div_ceil(nlist).max(1); + let candidate_rows = top_k + .checked_mul(DEFAULT_IVF_CANDIDATES_PER_RESULT) + .ok_or_else(|| invalid_input("automatic nprobe candidate count overflows usize"))?; + let candidate_lists = candidate_rows.div_ceil(average_list_rows); + let mut nprobe = DEFAULT_IVF_MIN_NPROBE + .max(nlist.div_ceil(DEFAULT_IVF_LIST_FRACTION)) + .max(candidate_lists) + .min(nlist); + + if let Some(matching_count) = matching_count { + if matching_count == 0 { + return Ok(1); + } + nprobe = ((nprobe as u128) + .saturating_mul(vector_count as u128) + .div_ceil(matching_count as u128) + .min(nlist as u128)) as usize; + } + Ok(nprobe.clamp(1, nlist)) +} + +pub fn infer_diskann_l_search(top_k: usize) -> io::Result { + if top_k == 0 { + return Err(invalid_input("top_k must be greater than 0")); + } + Ok(top_k.saturating_mul(2).max(100)) +} + +/// Returns the largest supported RQ bit width fitting an approximate per-row +/// budget. Multi-bit RQ stores five f32 factors (20 bytes) in addition to +/// padded bitplanes; one-bit RQ needs only two factors (8 bytes). +pub fn infer_rq_bits(dimension: usize, max_bytes_per_vector: usize) -> io::Result { + if dimension == 0 { + return Err(invalid_input("dimension must be greater than 0")); + } + let padded = padded_dimension(dimension); + for bits in (1..=8).rev() { + let factor_bytes = if bits == 1 { 8 } else { 20 }; + let code_bytes = padded + .checked_mul(bits) + .and_then(|value| value.checked_div(8)) + .and_then(|value| value.checked_add(factor_bytes)) + .ok_or_else(|| invalid_input("RQ row byte estimate overflows usize"))?; + if code_bytes <= max_bytes_per_vector { + return Ok(bits); + } + } + Err(invalid_input(format!( + "max bytes per vector {max_bytes_per_vector} cannot fit one-bit RQ codes and factors" + ))) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DiskAnnBuildPreset { + FastBuild, + Balanced, + HighRecall, +} + +pub fn diskann_build_preset( + preset: DiskAnnBuildPreset, + dimension: usize, + deployment_profile: DeploymentProfile, + memory_budget_bytes: usize, + seed: u64, +) -> io::Result { + if dimension == 0 { + return Err(invalid_input("dimension must be greater than 0")); + } + if memory_budget_bytes == 0 { + return Err(invalid_input( + "DiskANN memory budget must be greater than 0", + )); + } + let (max_degree, build_search_list_size, alpha, raw_vector_encoding, build_distance) = + match preset { + DiskAnnBuildPreset::FastBuild => ( + 48, + 64, + 1.15, + DiskAnnRawVectorEncoding::F16, + DiskAnnBuildDistance::ProductQuantized, + ), + DiskAnnBuildPreset::Balanced => ( + 64, + 100, + 1.2, + DiskAnnRawVectorEncoding::F16, + DiskAnnBuildDistance::ProductQuantized, + ), + DiskAnnBuildPreset::HighRecall => ( + 96, + 200, + 1.2, + DiskAnnRawVectorEncoding::F32, + DiskAnnBuildDistance::FullPrecision, + ), + }; + let record_bytes = dimension + .checked_mul(raw_vector_encoding.element_size()) + .and_then(|bytes| bytes.checked_add(max_degree * size_of::())) + .ok_or_else(|| invalid_input("DiskANN interleaved row size overflows usize"))?; + let storage_layout = match deployment_profile { + DeploymentProfile::Memory | DeploymentProfile::LocalStorage if record_bytes <= 4096 => { + DiskAnnStorageLayout::Interleaved + } + DeploymentProfile::Auto + | DeploymentProfile::Memory + | DeploymentProfile::LocalStorage + | DeploymentProfile::RemoteStorage + | DeploymentProfile::ObjectStore => DiskAnnStorageLayout::Compact, + }; + Ok(DiskAnnBuildParams { + max_degree, + build_search_list_size, + alpha, + seed, + memory_budget_bytes, + storage_layout, + raw_vector_encoding, + build_distance, + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct IndexRecommendation { + pub index_type: IndexType, + pub reason: &'static str, +} + +/// Advisory only: index type changes persistence, accuracy, and latency +/// semantics, so callers must explicitly accept this recommendation. +pub fn recommend_index( + vector_count: usize, + dimension: usize, + objective: TuningObjective, +) -> io::Result { + if vector_count == 0 || dimension == 0 { + return Err(invalid_input( + "vector count and dimension must be greater than 0", + )); + } + if vector_count <= 100_000 { + return Ok(IndexRecommendation { + index_type: IndexType::IvfFlat, + reason: "small corpus favors exact values and simple construction", + }); + } + if objective + .max_bytes_per_vector + .is_some_and(|bytes| bytes <= 32) + { + return Ok(IndexRecommendation { + index_type: IndexType::IvfPq, + reason: "strict row-size budget favors product quantization", + }); + } + if matches!( + objective.deployment_profile, + DeploymentProfile::RemoteStorage | DeploymentProfile::ObjectStore + ) { + return Ok(IndexRecommendation { + index_type: IndexType::IvfRq, + reason: "remote storage favors a compact one-round IVF scan", + }); + } + if objective.target_recall.is_some_and(|recall| recall >= 0.95) { + return Ok(IndexRecommendation { + index_type: IndexType::DiskAnn, + reason: "high recall on memory or local storage favors graph traversal", + }); + } + Ok(IndexRecommendation { + index_type: IndexType::IvfSq, + reason: "balanced default favors one-byte scalar codes and one-round reads", + }) +} + +fn floor_power_of_two(value: usize) -> usize { + if value <= 1 { + 1 + } else { + 1usize << (usize::BITS - 1 - value.leading_zeros()) + } +} + +fn invalid_input(message: impl Into) -> io::Error { + io::Error::new(io::ErrorKind::InvalidInput, message.into()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn automatic_nlist_is_power_of_two_and_keeps_training_density() { + assert_eq!(infer_ivf_nlist(1_000_000).unwrap(), 1024); + assert_eq!(infer_ivf_nlist(1_183_514).unwrap(), 1024); + assert_eq!(infer_ivf_nlist(10).unwrap(), 1); + assert!(infer_ivf_nlist(0).is_err()); + } + + #[test] + fn balanced_diskann_uses_compact_f16_rerank_vectors() { + let build = diskann_build_preset( + DiskAnnBuildPreset::Balanced, + 960, + DeploymentProfile::RemoteStorage, + 8 * 1024 * 1024 * 1024, + 42, + ) + .unwrap(); + + assert_eq!(build.raw_vector_encoding, DiskAnnRawVectorEncoding::F16); + assert_eq!( + DiskAnnBuildParams::default().raw_vector_encoding, + DiskAnnRawVectorEncoding::F16 + ); + } + + #[test] + fn automatic_training_count_is_bounded_and_overflow_checked() { + assert_eq!( + default_training_vector_count(1_000_000, 1024).unwrap(), + 65_536 + ); + assert_eq!( + default_training_vector_count(1_000_000, 4096).unwrap(), + 262_144 + ); + assert_eq!(default_training_vector_count(10_000, 1024).unwrap(), 10_000); + assert!(default_training_vector_count(usize::MAX, usize::MAX).is_err()); + } + + #[test] + fn automatic_nprobe_scales_for_lists_topk_and_filters() { + assert_eq!(infer_ivf_nprobe(64, 1_000_000, 10, None).unwrap(), 8); + assert_eq!(infer_ivf_nprobe(1024, 1_000_000, 10, None).unwrap(), 64); + assert_eq!( + infer_ivf_nprobe(1024, 1_000_000, 10, Some(10_000)).unwrap(), + 1024 + ); + assert_eq!(infer_ivf_nprobe(1024, 1_000_000, 10, Some(0)).unwrap(), 1); + } + + #[test] + fn calibrated_candidate_never_hides_an_unsatisfied_target() { + let candidates = [ + RecallCandidate { + value: 4, + measured_recall: 0.90, + bytes_per_vector: 48, + build_seconds: 2.0, + }, + RecallCandidate { + value: 6, + measured_recall: 0.96, + bytes_per_vector: 72, + build_seconds: 3.0, + }, + ]; + let selected = select_calibrated_candidate( + &candidates, + TuningObjective { + target_recall: Some(0.95), + max_bytes_per_vector: Some(80), + ..TuningObjective::default() + }, + ) + .unwrap(); + assert_eq!(selected.value, 6); + assert!(select_calibrated_candidate( + &candidates, + TuningObjective { + target_recall: Some(0.99), + ..TuningObjective::default() + } + ) + .is_none()); + } + + #[test] + fn rq_budget_accounts_for_padding_and_fixed_factors() { + assert_eq!(infer_rq_bits(100, 88).unwrap(), 4); + assert_eq!(infer_rq_bits(100, 72).unwrap(), 3); + assert!(infer_rq_bits(100, 23).is_err()); + } + + #[test] + fn diskann_preset_chooses_layout_from_deployment_profile() { + let local = diskann_build_preset( + DiskAnnBuildPreset::Balanced, + 128, + DeploymentProfile::LocalStorage, + 1 << 30, + 42, + ) + .unwrap(); + assert_eq!(local.storage_layout, DiskAnnStorageLayout::Interleaved); + let remote = diskann_build_preset( + DiskAnnBuildPreset::Balanced, + 128, + DeploymentProfile::RemoteStorage, + 1 << 30, + 42, + ) + .unwrap(); + assert_eq!(remote.storage_layout, DiskAnnStorageLayout::Compact); + } +} diff --git a/core/src/diskann.rs b/core/src/diskann.rs new file mode 100644 index 0000000..4b2cb95 --- /dev/null +++ b/core/src/diskann.rs @@ -0,0 +1,1094 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::distance::{preprocess_vectors, MetricType}; +use crate::kmeans::KMeansConfig; +use crate::pq::ProductQuantizer; +use crate::vamana::{ + estimate_sharded_vamana_memory_bytes, estimate_vamana_memory_bytes, VamanaGraph, +}; +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; +use std::borrow::Cow; +use std::collections::VecDeque; +use std::io; +use std::time::{Duration, Instant}; + +pub(crate) const DISKANN_ADJACENCY_LOCATOR_NODE_BYTES: usize = 4; +pub(crate) const DISKANN_ADJACENCY_LOCATOR_BLOCK_NODES: usize = 16; +/// Match the proven DiskANN training bound: more samples materially increase +/// memory and training time without consistently improving the codebook. +pub const DISKANN_MAX_PQ_TRAINING_VECTORS: usize = 50_000; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DiskAnnStorageLayout { + /// Keep compressed adjacency pages and dense raw-vector records in separate sections. + Compact, + /// Store each raw vector immediately before its compressed adjacency list. + Interleaved, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u32)] +pub enum DiskAnnRawVectorEncoding { + /// Preserve indexed vectors and final distances as little-endian `f32`. + F32 = 1, + /// Store little-endian IEEE 754 binary16 values for approximate final reranking. + F16 = 2, +} + +impl DiskAnnRawVectorEncoding { + pub(crate) const fn element_size(self) -> usize { + match self { + Self::F32 => size_of::(), + Self::F16 => size_of::(), + } + } + + pub(crate) const fn from_code(code: u32) -> Option { + match code { + 1 => Some(Self::F32), + 2 => Some(Self::F16), + _ => None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DiskAnnBuildDistance { + /// Use full-precision distances for graph traversal and robust pruning. + FullPrecision, + /// Use PQ distances for graph traversal and full precision for robust pruning. + ProductQuantized, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct DiskAnnBuildParams { + pub max_degree: usize, + pub build_search_list_size: usize, + pub alpha: f32, + pub seed: u64, + pub memory_budget_bytes: usize, + pub storage_layout: DiskAnnStorageLayout, + pub raw_vector_encoding: DiskAnnRawVectorEncoding, + pub build_distance: DiskAnnBuildDistance, +} + +impl Default for DiskAnnBuildParams { + fn default() -> Self { + Self { + max_degree: 64, + build_search_list_size: 100, + alpha: 1.2, + seed: 42, + memory_budget_bytes: 8 * 1024 * 1024 * 1024, + storage_layout: DiskAnnStorageLayout::Compact, + raw_vector_encoding: DiskAnnRawVectorEncoding::F16, + build_distance: DiskAnnBuildDistance::ProductQuantized, + } + } +} + +pub(crate) fn validate_diskann_format_configuration( + dimension: usize, + pq_m: usize, + pq_bits: usize, + build: DiskAnnBuildParams, +) -> io::Result<()> { + if dimension == 0 { + return Err(invalid_input("DiskANN dimension must be greater than 0")); + } + if dimension > 1024 { + return Err(invalid_input("DiskANN v1 dimension must be at most 1024")); + } + if pq_m == 0 { + return Err(invalid_input("DiskANN pq.m must be greater than 0")); + } + if pq_m > dimension { + return Err(invalid_input(format!( + "DiskANN pq.m {} must not exceed dimension {}", + pq_m, dimension + ))); + } + if !matches!(pq_bits, 4 | 8) { + return Err(invalid_input("DiskANN pq.bits must be 4 or 8")); + } + if build.max_degree == 0 { + return Err(invalid_input( + "DiskANN maximum degree must be greater than 0", + )); + } + if build.max_degree > 1023 { + return Err(invalid_input(format!( + "DiskANN adjacency list size {} exceeds the v1 1023-neighbor page limit", + build.max_degree.saturating_mul(size_of::()) + ))); + } + if build.build_search_list_size < build.max_degree { + return Err(invalid_input(format!( + "DiskANN build search-list size {} must be at least maximum degree {}", + build.build_search_list_size, build.max_degree + ))); + } + if u32::try_from(build.build_search_list_size).is_err() { + return Err(invalid_input("DiskANN build search-list size exceeds u32")); + } + if !build.alpha.is_finite() || build.alpha < 1.0 { + return Err(invalid_input("DiskANN alpha must be at least 1 and finite")); + } + let interleaved_record_bytes = dimension + .checked_mul(build.raw_vector_encoding.element_size()) + .and_then(|vector_bytes| { + build + .max_degree + .checked_mul(size_of::()) + .and_then(|adjacency_bytes| vector_bytes.checked_add(adjacency_bytes)) + }); + if build.storage_layout == DiskAnnStorageLayout::Interleaved + && interleaved_record_bytes.is_none_or(|record_bytes| record_bytes > 4096) + { + return Err(invalid_input( + "DiskANN interleaved raw vector and maximum adjacency list must fit in one page", + )); + } + Ok(()) +} + +pub(crate) fn validate_diskann_training_budget( + dimension: usize, + metric: MetricType, + pq_m: usize, + pq_bits: usize, + memory_budget_bytes: usize, +) -> io::Result<()> { + let minimum_training_vectors = 1usize << pq_bits; + pq_training_plan_with_sample_buffers( + dimension, + pq_m, + minimum_training_vectors, + minimum_training_vectors, + memory_budget_bytes, + usize::from(metric == MetricType::Cosine) + 1, + ) + .map(|_| ()) + .map_err(|error| { + invalid_input(format!( + "DiskANN memory budget cannot fit minimum PQ training: {error}" + )) + }) +} + +pub(crate) fn diskann_training_sample_limit( + dimension: usize, + metric: MetricType, + pq_m: usize, + pq_bits: usize, + memory_budget_bytes: usize, +) -> io::Result { + pq_training_plan_with_sample_buffers( + dimension, + pq_m, + 1usize << pq_bits, + DISKANN_MAX_PQ_TRAINING_VECTORS, + memory_budget_bytes, + usize::from(metric == MetricType::Cosine) + 1, + ) + .map(|plan| plan.sample_count) +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct DiskAnnBuildStats { + /// One for the normal parallel build; greater than one when the memory + /// budget selected overlapping shard construction. + pub graph_shards: usize, + pub total: Duration, + pub pq_encoding: Duration, + pub vamana_initialization: Duration, + pub vamana_pass_one: Duration, + pub vamana_pass_two: Duration, + pub connectivity_repair: Duration, + pub locality_remap: Duration, + pub resident_serialization: Duration, + pub adjacency_serialization: Duration, + pub vector_serialization: Duration, +} + +impl DiskAnnBuildStats { + pub fn accounted_duration(self) -> Duration { + [ + self.pq_encoding, + self.vamana_initialization, + self.vamana_pass_one, + self.vamana_pass_two, + self.connectivity_repair, + self.locality_remap, + self.resident_serialization, + self.adjacency_serialization, + self.vector_serialization, + ] + .into_iter() + .sum() + } +} + +pub struct DiskAnnIndex { + pub d: usize, + pub metric: MetricType, + pub pq: ProductQuantizer, + pub build_params: DiskAnnBuildParams, + pub ids: Vec, + pub vectors: Vec, +} + +impl DiskAnnIndex { + pub fn new( + d: usize, + metric: MetricType, + pq_m: usize, + build_params: DiskAnnBuildParams, + ) -> Self { + Self::with_pq_bits(d, metric, pq_m, 8, build_params) + } + + pub fn with_pq_bits( + d: usize, + metric: MetricType, + pq_m: usize, + pq_bits: usize, + build_params: DiskAnnBuildParams, + ) -> Self { + Self { + d, + metric, + pq: ProductQuantizer::with_nbits_balanced(d, pq_m, pq_bits), + build_params, + ids: Vec::new(), + vectors: Vec::new(), + } + } + + pub fn train(&mut self, data: &[f32], n: usize) -> io::Result<()> { + if n == 0 { + return Err(invalid_input( + "DiskANN training vector count must be greater than zero", + )); + } + let expected_len = n + .checked_mul(self.d) + .ok_or_else(|| invalid_input("DiskANN training data length overflows usize"))?; + if data.len() != expected_len { + return Err(invalid_input(format!( + "DiskANN training data length {} does not match n * dimension {}", + data.len(), + expected_len + ))); + } + let plan = self.training_plan(n)?; + let sample = + bounded_pq_training_sample(data, n, self.d, plan.sample_count, self.build_params.seed); + let sample = sample + .as_deref() + .unwrap_or(&data[..plan.sample_count.saturating_mul(self.d)]); + let processed = self.preprocess_vectors(sample, plan.sample_count); + self.pq.train_hot_start_with_parallelism( + &processed, + plan.sample_count, + &KMeansConfig::default(), + false, + plan.parallelism, + ); + Ok(()) + } + + fn training_plan(&self, n: usize) -> io::Result { + pq_training_plan_with_sample_buffers( + self.d, + self.pq.m, + self.pq.ksub, + n, + self.build_params.memory_budget_bytes, + usize::from(self.metric == MetricType::Cosine) + 1, + ) + } + + pub fn add(&mut self, data: &[f32], ids: &[i64]) { + self.ids.extend_from_slice(ids); + self.vectors + .extend_from_slice(self.preprocess_vectors(data, ids.len()).as_ref()); + } + + pub fn estimate_build_memory_bytes(&self) -> io::Result { + let n = self.ids.len(); + let workers = rayon::current_num_threads().max(1); + let raw_vectors = checked_bytes(self.vectors.len(), size_of::(), "raw vectors")?; + let row_ids = checked_bytes(n, size_of::(), "row IDs")?; + let row_id_encoding_scratch = row_id_encoding_scratch_bytes(n)?; + let pq_codes = checked_bytes(n, self.pq.code_size(), "PQ codes")?; + let pq_codebook = checked_bytes(self.pq.centroids.len(), size_of::(), "PQ codebook")?; + let pq_build_distances = if self.build_params.build_distance + == DiskAnnBuildDistance::ProductQuantized + { + self.pq + .m + .checked_mul(self.pq.ksub) + .and_then(|value| value.checked_mul(self.pq.ksub)) + .and_then(|value| value.checked_mul(size_of::())) + .ok_or_else(|| invalid_input("DiskANN PQ build-distance table size overflows"))? + } else { + 0 + }; + let row_id_order = checked_bytes(n, size_of::(), "row-ID order")?; + let adjacency_index = checked_bytes( + n, + DISKANN_ADJACENCY_LOCATOR_NODE_BYTES, + "adjacency locators", + )? + .checked_add(checked_bytes( + n.div_ceil(DISKANN_ADJACENCY_LOCATOR_BLOCK_NODES), + size_of::(), + "adjacency locator block offsets", + )?) + .ok_or_else(|| invalid_input("DiskANN adjacency index size overflows usize"))?; + let vamana = estimate_vamana_memory_bytes( + n, + self.build_params.max_degree, + self.build_params.build_search_list_size, + workers, + ) + .ok_or_else(|| invalid_input("DiskANN Vamana memory estimate overflows usize"))?; + let graph_stage_peak = vamana.build_peak_bytes.max(vamana.remap_peak_bytes); + + [ + raw_vectors, + row_ids, + row_id_encoding_scratch, + pq_codes, + pq_codebook, + pq_build_distances, + row_id_order, + adjacency_index, + graph_stage_peak, + ] + .into_iter() + .try_fold(0usize, |total, value| { + total + .checked_add(value) + .ok_or_else(|| invalid_input("DiskANN memory estimate overflows usize")) + }) + } + + fn graph_build_shard_count(&self) -> io::Result { + let estimated = self.estimate_build_memory_bytes()?; + if estimated <= self.build_params.memory_budget_bytes { + return Ok(1); + } + let n = self.ids.len(); + let workers = rayon::current_num_threads().max(1); + let vamana = estimate_vamana_memory_bytes( + n, + self.build_params.max_degree, + self.build_params.build_search_list_size, + workers, + ) + .ok_or_else(|| invalid_input("DiskANN Vamana memory estimate overflows usize"))?; + let graph_peak = vamana.build_peak_bytes.max(vamana.remap_peak_bytes); + let fixed_bytes = estimated + .checked_sub(graph_peak) + .ok_or_else(|| invalid_input("DiskANN fixed memory estimate underflows"))?; + let max_shards = 64.min(n / 2); + for shard_count in 2..=max_shards { + let Some(sharded_graph) = estimate_sharded_vamana_memory_bytes( + n, + self.d, + self.build_params.max_degree.min(n.saturating_sub(1)), + shard_count, + if self.build_params.build_distance == DiskAnnBuildDistance::ProductQuantized { + self.pq.code_size() + } else { + 0 + }, + ) else { + continue; + }; + if fixed_bytes + .checked_add(sharded_graph) + .is_some_and(|peak| peak <= self.build_params.memory_budget_bytes) + { + return Ok(shard_count); + } + } + Err(invalid_input(format!( + "DiskANN estimated build memory {} exceeds memory budget {}; overlapping sharded build also cannot fit", + estimated, self.build_params.memory_budget_bytes + ))) + } + + pub(crate) fn validate_for_write(&self) -> io::Result<()> { + validate_diskann_format_configuration(self.d, self.pq.m, self.pq.nbits, self.build_params)?; + validate_diskann_training_budget( + self.d, + self.metric, + self.pq.m, + self.pq.nbits, + self.build_params.memory_budget_bytes, + )?; + if self.build_params.memory_budget_bytes == 0 { + return Err(invalid_input( + "DiskANN memory budget must be greater than zero", + )); + } + if self.ids.is_empty() || u32::try_from(self.ids.len()).is_err() { + return Err(invalid_input( + "DiskANN vector count must be between 1 and u32::MAX", + )); + } + let expected_vectors = self + .ids + .len() + .checked_mul(self.d) + .ok_or_else(|| invalid_input("DiskANN vector shape overflows usize"))?; + if self.vectors.len() != expected_vectors { + return Err(invalid_input(format!( + "DiskANN vector length {} does not match {} row IDs * dimension {}", + self.vectors.len(), + self.ids.len(), + self.d + ))); + } + if let Some(offset) = self.vectors.iter().position(|value| !value.is_finite()) { + return Err(invalid_input(format!( + "DiskANN vector data contains a non-finite value at offset {}", + offset + ))); + } + if self.build_params.raw_vector_encoding == DiskAnnRawVectorEncoding::F16 { + if let Some(offset) = self + .vectors + .iter() + .position(|&value| !half::f16::from_f32(value).is_finite()) + { + return Err(invalid_input(format!( + "DiskANN vector data at offset {} is outside the finite f16 range", + offset + ))); + } + } + + let expected_ksub = 1usize + .checked_shl(self.pq.nbits as u32) + .ok_or_else(|| invalid_input("DiskANN PQ centroid count overflows usize"))?; + let expected_centroids = self + .d + .checked_mul(expected_ksub) + .ok_or_else(|| invalid_input("DiskANN PQ codebook shape overflows usize"))?; + if self.pq.d != self.d + || self.pq.ksub != expected_ksub + || self.pq.centroids.len() != expected_centroids + || !self.pq.has_valid_layout() + { + return Err(invalid_input("DiskANN PQ codebook shape is invalid")); + } + if let Some(offset) = self + .pq + .centroids + .iter() + .position(|value| !value.is_finite()) + { + return Err(invalid_input(format!( + "DiskANN PQ codebook contains a non-finite value at offset {}", + offset + ))); + } + Ok(()) + } + + pub(crate) fn prepare_build(&self) -> io::Result { + self.validate_for_write()?; + let graph_shards = self.graph_build_shard_count()?; + + let pq_started = Instant::now(); + let mut pq_codes = vec![0u8; self.ids.len() * self.pq.code_size()]; + self.pq + .encode_batch(&self.vectors, self.ids.len(), &mut pq_codes); + let pq_encoding = pq_started.elapsed(); + let (mut graph, vamana_stats) = if graph_shards > 1 { + match self.build_params.build_distance { + DiskAnnBuildDistance::FullPrecision => VamanaGraph::build_sharded_with_stats( + &self.vectors, + self.ids.len(), + self.d, + self.graph_metric(), + self.build_params, + graph_shards, + )?, + DiskAnnBuildDistance::ProductQuantized => VamanaGraph::build_sharded_with_pq_stats( + &self.vectors, + &self.pq, + &pq_codes, + self.ids.len(), + self.d, + self.graph_metric(), + self.build_params, + graph_shards, + )?, + } + } else { + match self.build_params.build_distance { + DiskAnnBuildDistance::FullPrecision => VamanaGraph::build_with_stats( + &self.vectors, + self.ids.len(), + self.d, + self.graph_metric(), + self.build_params, + )?, + DiskAnnBuildDistance::ProductQuantized => VamanaGraph::build_with_pq_stats( + &self.vectors, + &self.pq, + &pq_codes, + self.ids.len(), + self.d, + self.graph_metric(), + self.build_params, + )?, + } + }; + let locality_started = Instant::now(); + let permutation = bfs_locality_permutation(&graph); + remap_graph_in_place(&mut graph, &permutation); + let locality_remap = locality_started.elapsed(); + Ok(PreparedDiskAnn { + graph, + permutation, + pq_codes, + stats: DiskAnnBuildStats { + graph_shards, + pq_encoding, + vamana_initialization: vamana_stats.initialization, + vamana_pass_one: vamana_stats.pass_one, + vamana_pass_two: vamana_stats.pass_two, + connectivity_repair: vamana_stats.connectivity_repair, + locality_remap, + ..DiskAnnBuildStats::default() + }, + }) + } + + fn graph_metric(&self) -> MetricType { + match self.metric { + // Cosine vectors are normalized during training and ingestion, so + // squared L2 has identical ordering and a well-behaved prune ratio. + MetricType::Cosine => MetricType::L2, + metric => metric, + } + } + + fn preprocess_vectors<'a>(&self, data: &'a [f32], count: usize) -> Cow<'a, [f32]> { + if self.metric == MetricType::Cosine { + Cow::Owned(preprocess_vectors(data, count, self.d, self.metric)) + } else { + Cow::Borrowed(&data[..count * self.d]) + } + } +} + +fn bounded_pq_training_sample( + data: &[f32], + count: usize, + dimension: usize, + sample_limit: usize, + seed: u64, +) -> Option> { + if count <= sample_limit { + return None; + } + let mut sample = Vec::with_capacity(sample_limit.saturating_mul(dimension)); + sample.extend_from_slice(&data[..sample_limit * dimension]); + let mut rng = StdRng::seed_from_u64(seed); + for (stream_index, vector) in data + .chunks_exact(dimension) + .enumerate() + .skip(sample_limit) + .take(count - sample_limit) + { + let replacement = rng.gen_range(0..=stream_index); + if replacement < sample_limit { + let start = replacement * dimension; + sample[start..start + dimension].copy_from_slice(vector); + } + } + Some(sample) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct PqTrainingPlan { + sample_count: usize, + parallelism: usize, +} + +#[cfg(test)] +fn pq_training_plan( + dimension: usize, + pq_m: usize, + ksub: usize, + vector_count: usize, + memory_budget_bytes: usize, +) -> io::Result { + pq_training_plan_with_sample_buffers( + dimension, + pq_m, + ksub, + vector_count, + memory_budget_bytes, + 1, + ) +} + +fn pq_training_plan_with_sample_buffers( + dimension: usize, + pq_m: usize, + ksub: usize, + vector_count: usize, + memory_budget_bytes: usize, + sample_buffer_count: usize, +) -> io::Result { + if vector_count == 0 { + return Ok(PqTrainingPlan { + sample_count: 0, + parallelism: 1, + }); + } + let max_sample = vector_count.min(DISKANN_MAX_PQ_TRAINING_VECTORS); + let min_sample = vector_count.min(ksub.max(1)); + let max_chunk_dimension = dimension.div_ceil(pq_m.max(1)); + let current_parallelism = rayon::current_num_threads().max(1).min(pq_m.max(1)); + let peak_for = |sample_count: usize, parallelism: usize| -> Option { + let sample_bytes = sample_count + .checked_mul(dimension)? + .checked_mul(size_of::())? + .checked_mul(sample_buffer_count)?; + let codebook_bytes = dimension.checked_mul(ksub)?.checked_mul(size_of::())?; + let subvector_copy_bytes = sample_count + .checked_mul(max_chunk_dimension)? + .checked_mul(2 * size_of::())?; + let assignment_bytes = sample_count.checked_mul(size_of::() + size_of::())?; + let score_matrix_bytes = sample_count + .checked_mul(ksub)? + .min(4 * 1024 * 1024) + .checked_mul(size_of::())?; + let centroid_scratch_bytes = max_chunk_dimension + .checked_mul(ksub)? + .checked_mul(3 * size_of::())?; + let per_task = [ + subvector_copy_bytes, + assignment_bytes, + score_matrix_bytes, + centroid_scratch_bytes, + ] + .into_iter() + .try_fold(0usize, |total, bytes| total.checked_add(bytes))?; + sample_bytes + .checked_add(codebook_bytes)? + .checked_add(per_task.checked_mul(parallelism)?) + }; + if peak_for(min_sample, 1).is_none_or(|peak| peak > memory_budget_bytes) { + return Err(invalid_input(format!( + "{} bytes is below the estimated one-worker PQ training peak", + memory_budget_bytes + ))); + } + + let mut low = min_sample; + let mut high = max_sample; + while low < high { + let middle = low + (high - low).div_ceil(2); + if peak_for(middle, 1).is_some_and(|peak| peak <= memory_budget_bytes) { + low = middle; + } else { + high = middle - 1; + } + } + let sample_count = low; + let parallelism = (1..=current_parallelism) + .rev() + .find(|¶llelism| { + peak_for(sample_count, parallelism).is_some_and(|peak| peak <= memory_budget_bytes) + }) + .unwrap_or(1); + Ok(PqTrainingPlan { + sample_count, + parallelism, + }) +} + +fn checked_bytes(count: usize, item_size: usize, name: &str) -> io::Result { + count + .checked_mul(item_size) + .ok_or_else(|| invalid_input(format!("DiskANN {} byte size overflows usize", name))) +} + +fn row_id_encoding_scratch_bytes(count: usize) -> io::Result { + checked_bytes(count, size_of::(), "row-ID encoding scratch") +} + +fn invalid_input(message: impl Into) -> io::Error { + io::Error::new(io::ErrorKind::InvalidInput, message.into()) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct LocalityPermutation { + pub(crate) new_to_old: Vec, + pub(crate) old_to_new: Vec, +} + +pub(crate) fn bfs_locality_permutation(graph: &VamanaGraph) -> LocalityPermutation { + let count = graph.adjacency.len(); + let mut visited = vec![false; count]; + let mut new_to_old = Vec::with_capacity(count); + let entry = graph.entry_node as usize; + if entry < count { + visited[entry] = true; + let mut queue = VecDeque::from([entry]); + while let Some(node) = queue.pop_front() { + new_to_old.push(node as u32); + for &neighbor in &graph.adjacency[node] { + let neighbor = neighbor as usize; + if neighbor < count && !visited[neighbor] { + visited[neighbor] = true; + queue.push_back(neighbor); + } + } + } + } + for (node, was_visited) in visited.into_iter().enumerate() { + if !was_visited { + new_to_old.push(node as u32); + } + } + + let mut old_to_new = vec![0u32; count]; + for (new_id, &old_id) in new_to_old.iter().enumerate() { + old_to_new[old_id as usize] = new_id as u32; + } + LocalityPermutation { + new_to_old, + old_to_new, + } +} + +pub(crate) fn remap_graph_in_place(graph: &mut VamanaGraph, permutation: &LocalityPermutation) { + graph + .adjacency + .permute_and_map_neighbors(&permutation.old_to_new); + graph.entry_node = permutation.old_to_new[graph.entry_node as usize]; +} + +pub(crate) struct PreparedDiskAnn { + pub(crate) graph: VamanaGraph, + pub(crate) permutation: LocalityPermutation, + pub(crate) pq_codes: Vec, + pub(crate) stats: DiskAnnBuildStats, +} + +impl PreparedDiskAnn { + pub(crate) fn row_id(&self, index: &DiskAnnIndex, new_id: usize) -> i64 { + index.ids[self.permutation.new_to_old[new_id] as usize] + } + + pub(crate) fn vector<'a>(&self, index: &'a DiskAnnIndex, new_id: usize) -> &'a [f32] { + let old_id = self.permutation.new_to_old[new_id] as usize; + &index.vectors[old_id * index.d..(old_id + 1) * index.d] + } + + pub(crate) fn pq_code<'a>(&'a self, index: &DiskAnnIndex, new_id: usize) -> &'a [u8] { + let old_id = self.permutation.new_to_old[new_id] as usize; + let code_size = index.pq.code_size(); + &self.pq_codes[old_id * code_size..(old_id + 1) * code_size] + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn low_level_diskann_training_uses_the_same_bounded_deterministic_reservoir() { + let dimension = 2; + let count = DISKANN_MAX_PQ_TRAINING_VECTORS + 257; + let data = (0..count * dimension) + .map(|offset| offset as f32) + .collect::>(); + + let first = bounded_pq_training_sample( + &data, + count, + dimension, + DISKANN_MAX_PQ_TRAINING_VECTORS, + 73, + ) + .unwrap(); + let second = bounded_pq_training_sample( + &data, + count, + dimension, + DISKANN_MAX_PQ_TRAINING_VECTORS, + 73, + ) + .unwrap(); + let different_seed = bounded_pq_training_sample( + &data, + count, + dimension, + DISKANN_MAX_PQ_TRAINING_VECTORS, + 74, + ) + .unwrap(); + + assert_eq!(first.len(), DISKANN_MAX_PQ_TRAINING_VECTORS * dimension); + assert_eq!(first, second); + assert_ne!(first, different_seed); + assert_ne!(first, data[..DISKANN_MAX_PQ_TRAINING_VECTORS * dimension]); + assert!(bounded_pq_training_sample( + &data[..DISKANN_MAX_PQ_TRAINING_VECTORS * dimension], + DISKANN_MAX_PQ_TRAINING_VECTORS, + dimension, + DISKANN_MAX_PQ_TRAINING_VECTORS, + 73 + ) + .is_none()); + } + + #[test] + fn diskann_pq_training_plan_throttles_parallelism_and_samples_to_budget() { + let unrestricted = + pq_training_plan(1024, 256, 256, 50_000, 8 * 1024 * 1024 * 1024).unwrap(); + assert_eq!(unrestricted.sample_count, 50_000); + assert_eq!( + unrestricted.parallelism, + rayon::current_num_threads().min(256) + ); + + let bounded = pq_training_plan(1024, 256, 256, 50_000, 64 * 1024 * 1024).unwrap(); + assert!(bounded.sample_count < unrestricted.sample_count); + assert!(bounded.parallelism <= unrestricted.parallelism); + assert!(pq_training_plan(1024, 1, 256, 256, 1).is_err()); + } + + #[test] + fn diskann_pq_training_plan_accounts_for_retained_and_normalized_samples() { + let budget = 128 * 1024 * 1024; + let retained = + pq_training_plan_with_sample_buffers(1024, 256, 256, 50_000, budget, 1).unwrap(); + let cosine = + pq_training_plan_with_sample_buffers(1024, 256, 256, 50_000, budget, 2).unwrap(); + assert!(retained.sample_count < 50_000); + assert!(cosine.sample_count < retained.sample_count); + } + + #[test] + fn low_level_diskann_training_uses_metric_aware_budget_and_propagates_failure() { + let budget = 128 * 1024 * 1024; + let build_params = DiskAnnBuildParams { + memory_budget_bytes: budget, + ..DiskAnnBuildParams::default() + }; + let l2 = DiskAnnIndex::new(1024, MetricType::L2, 256, build_params); + let cosine = DiskAnnIndex::new(1024, MetricType::Cosine, 256, build_params); + let l2_plan = l2.training_plan(50_000).unwrap(); + let cosine_plan = cosine.training_plan(50_000).unwrap(); + + assert_eq!( + l2_plan, + pq_training_plan_with_sample_buffers(1024, 256, 256, 50_000, budget, 1).unwrap() + ); + assert_eq!( + cosine_plan, + pq_training_plan_with_sample_buffers(1024, 256, 256, 50_000, budget, 2).unwrap() + ); + assert!(cosine_plan.sample_count < l2_plan.sample_count); + + let mut infeasible = DiskAnnIndex::new( + 8, + MetricType::Cosine, + 2, + DiskAnnBuildParams { + memory_budget_bytes: 1, + ..DiskAnnBuildParams::default() + }, + ); + let error = infeasible + .train(&[0.0; 8], 1) + .expect_err("an infeasible low-level training budget must be returned"); + assert!(error.to_string().contains("PQ training peak")); + } + + #[test] + fn diskann_index_rejects_build_exceeding_memory_budget() { + let build_params = DiskAnnBuildParams { + memory_budget_bytes: 1, + ..DiskAnnBuildParams::default() + }; + let mut index = DiskAnnIndex::new(8, MetricType::L2, 2, build_params); + index.add(&[0.0; 8], &[7]); + + let error = index + .graph_build_shard_count() + .expect_err("build should fail before graph allocation"); + assert!(error.to_string().contains("memory budget")); + } + + #[test] + fn diskann_memory_estimate_accounts_for_one_compact_graph_during_remap() { + let count = 1024; + let max_degree = 1024; + let estimate = estimate_vamana_memory_bytes(count, max_degree, 1024, 1) + .unwrap() + .remap_peak_bytes; + let one_graph = count * (max_degree * size_of::() + size_of::()); + + assert!( + estimate >= one_graph && estimate < 2 * one_graph, + "in-place remapping should retain one compact graph; estimate={estimate}, one_graph={one_graph}" + ); + } + + #[test] + fn diskann_memory_estimate_reserves_row_id_encoding_scratch() { + assert_eq!(row_id_encoding_scratch_bytes(1024).unwrap(), 8 * 1024); + assert!(row_id_encoding_scratch_bytes(usize::MAX).is_err()); + } + + #[test] + fn diskann_memory_budget_automatically_selects_overlapping_shards() { + // Pin the worker count so this test exercises the same memory-plan + // relationship on small CI runners and developer machines. + rayon::ThreadPoolBuilder::new() + .num_threads(12) + .build() + .unwrap() + .install(|| { + let count = 2_048; + let dimension = 8; + let params = DiskAnnBuildParams { + max_degree: 256, + build_search_list_size: 256, + ..DiskAnnBuildParams::default() + }; + let mut index = DiskAnnIndex::new(dimension, MetricType::L2, 2, params); + index.ids = (0..count as i64).collect(); + index.vectors = vec![0.0; count * dimension]; + let full_peak = index.estimate_build_memory_bytes().unwrap(); + index.build_params.memory_budget_bytes = full_peak - 1; + + let shard_count = index.graph_build_shard_count().unwrap(); + + assert!(shard_count > 1); + assert!(shard_count <= 64); + }); + } + + #[test] + fn diskann_locality_permutation_uses_bfs_then_unreachable_ids() { + let graph = + VamanaGraph::from_adjacency(2, vec![vec![], vec![], vec![3, 1], vec![4], vec![]]); + + let permutation = bfs_locality_permutation(&graph); + + assert_eq!(permutation.new_to_old, vec![2, 3, 1, 4, 0]); + assert_eq!(permutation.old_to_new, vec![4, 2, 0, 1, 3]); + } + + #[test] + fn diskann_locality_remaps_entry_and_neighbor_ids() { + let mut graph = + VamanaGraph::from_adjacency(2, vec![vec![], vec![], vec![3, 1], vec![4], vec![]]); + let permutation = bfs_locality_permutation(&graph); + + remap_graph_in_place(&mut graph, &permutation); + + assert_eq!(graph.entry_node, 0); + assert_eq!( + graph + .adjacency + .iter() + .map(<[u32]>::to_vec) + .collect::>(), + vec![vec![1, 2], vec![3], vec![], vec![], vec![]] + ); + } + + #[test] + fn diskann_prepare_build_keeps_rows_codes_and_vectors_aligned() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 29) % 113) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + + let prepared = index.prepare_build().unwrap(); + + assert_eq!(prepared.graph.entry_node, 0); + assert_eq!(prepared.pq_codes.len(), indexed_count * 2); + for new_id in 0..indexed_count { + let old_id = prepared.permutation.new_to_old[new_id] as usize; + assert_eq!(prepared.row_id(&index, new_id), ids[old_id]); + assert_eq!( + prepared.vector(&index, new_id), + &data[old_id * dimension..(old_id + 1) * dimension] + ); + assert_eq!(prepared.pq_code(&index, new_id).len(), 2); + } + } + + #[test] + fn diskann_cosine_normalizes_vectors_before_pq_encoding_and_persistence() { + let mut index = + DiskAnnIndex::with_pq_bits(2, MetricType::Cosine, 1, 4, DiskAnnBuildParams::default()); + + index.add(&[3.0, 4.0, 0.0, 0.0], &[10, 11]); + + assert_eq!(index.vectors, vec![0.6, 0.8, 0.0, 0.0]); + } + + #[test] + fn diskann_only_allocates_metric_preprocessing_for_cosine() { + let data = [3.0, 4.0]; + for metric in [MetricType::L2, MetricType::InnerProduct] { + let index = DiskAnnIndex::with_pq_bits(2, metric, 1, 4, DiskAnnBuildParams::default()); + assert!(matches!( + index.preprocess_vectors(&data, 1), + Cow::Borrowed(_) + )); + } + let index = + DiskAnnIndex::with_pq_bits(2, MetricType::Cosine, 1, 4, DiskAnnBuildParams::default()); + assert!(matches!(index.preprocess_vectors(&data, 1), Cow::Owned(_))); + } +} diff --git a/core/src/diskann_io.rs b/core/src/diskann_io.rs new file mode 100644 index 0000000..bb20884 --- /dev/null +++ b/core/src/diskann_io.rs @@ -0,0 +1,5433 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! DiskANN v1 persistence and readers. +//! +//! The normative byte layout is documented in the repository's +//! [storage-format specification](https://github.com/apache/paimon-vector-index/blob/main/core/STORAGE_FORMAT.md#diskann-v1). + +use crate::diskann::{ + validate_diskann_format_configuration, DiskAnnBuildParams, DiskAnnBuildStats, DiskAnnIndex, + DiskAnnRawVectorEncoding, DiskAnnStorageLayout, PreparedDiskAnn, + DISKANN_ADJACENCY_LOCATOR_BLOCK_NODES as ADJACENCY_LOCATOR_BLOCK_NODES, + DISKANN_ADJACENCY_LOCATOR_NODE_BYTES, +}; +use crate::diskann_search::{DiskAnnQueryScratch, DiskAnnSearchStats}; +use crate::distance::MetricType; +use crate::io::{ReadRequest, SeekRead, SeekReadCapabilities, SeekWrite}; +use crate::pq::ProductQuantizer; +use crate::read_options::{ + DeploymentProfile, ReadPlan, ResolvedVectorIndexReaderOptions, VectorIndexReadPlan, + VectorIndexReaderOptions, +}; +use rayon::prelude::*; +use std::collections::{HashMap, HashSet}; +use std::io; +use std::ops::{Index, IndexMut}; +use std::sync::atomic::{AtomicU8, AtomicUsize, Ordering as AtomicOrdering}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +pub const DISKANN_MAGIC: u32 = 0x4E4E4144; // "DANN" +pub const DISKANN_VERSION: u32 = 1; +pub const DISKANN_HEADER_SIZE: usize = 256; +pub const DISKANN_PAGE_SIZE: u32 = 4096; +const FLAG_BFS_LAYOUT: u32 = 1 << 0; +const FLAG_SEPARATE_ADJACENCY_AND_VECTORS: u32 = 1 << 1; +const FLAG_ADAPTIVE_ADJACENCY: u32 = 1 << 2; +const FLAG_PQ_CODES: u32 = 1 << 3; +const FLAG_ROW_ID_ORDER: u32 = 1 << 4; +const FLAG_INTERLEAVED_ADJACENCY_AND_VECTORS: u32 = 1 << 5; +pub const DISKANN_REQUIRED_FLAGS: u32 = + FLAG_BFS_LAYOUT | FLAG_ADAPTIVE_ADJACENCY | FLAG_PQ_CODES | FLAG_ROW_ID_ORDER; +const DISKANN_SUPPORTED_FLAGS: u32 = DISKANN_REQUIRED_FLAGS + | FLAG_SEPARATE_ADJACENCY_AND_VECTORS + | FLAG_INTERLEAVED_ADJACENCY_AND_VECTORS; +const SECTION_COUNT: usize = 7; +const ADJACENCY_LOCATOR_SIZE: u32 = DISKANN_ADJACENCY_LOCATOR_NODE_BYTES as u32; +const ADJACENCY_LOCATOR_ENCODING: u32 = 3; +const ADJACENCY_LOCATOR_BLOCK_BASE_SIZE: usize = size_of::(); +const ADJACENCY_RAW_U32_FLAG: u16 = 1 << 15; +const ADJACENCY_DEGREE_MASK: u16 = ADJACENCY_RAW_U32_FLAG - 1; +const ROW_ID_SECTION_HEADER_SIZE: usize = 32; +const ROW_ID_ENCODING_RAW_I64: u32 = 0; +const ROW_ID_ENCODING_FOR_BITPACK: u32 = 1; +const PQ_CODEBOOK_MAGIC: u32 = 0x3151_5044; // "DPQ1" +const PQ_CODEBOOK_VERSION: u32 = 1; +const PQ_CODEBOOK_HEADER_SIZE: usize = 32; +const DISKANN_WRITE_BUFFER_SIZE: usize = 1024 * 1024; +const DISKANN_RESIDENT_DECODE_BUFFER_SIZE: usize = 1024 * 1024; +const DISKANN_ADJACENCY_PRELOAD_ALIGNMENT: usize = 64 * 1024; +const AUTO_PROFILE_MEMORY_LATENCY_THRESHOLD: Duration = Duration::from_micros(50); +const AUTO_PROFILE_LOCAL_LATENCY_THRESHOLD: Duration = Duration::from_micros(750); +const AUTO_PROFILE_REMOTE_LATENCY_THRESHOLD: Duration = Duration::from_millis(3); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SectionRange { + pub offset: u64, + pub length: u64, +} + +impl SectionRange { + pub const fn new(offset: u64, length: u64) -> Self { + Self { offset, length } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DiskAnnSections { + pub codebook: SectionRange, + pub row_ids: SectionRange, + pub pq_codes: SectionRange, + pub row_id_order: SectionRange, + pub adjacency_index: SectionRange, + pub adjacency: SectionRange, + pub vectors: SectionRange, +} + +impl DiskAnnSections { + fn from_array(sections: [SectionRange; SECTION_COUNT]) -> Self { + Self { + codebook: sections[0], + row_ids: sections[1], + pq_codes: sections[2], + row_id_order: sections[3], + adjacency_index: sections[4], + adjacency: sections[5], + vectors: sections[6], + } + } + + fn as_array(self) -> [SectionRange; SECTION_COUNT] { + [ + self.codebook, + self.row_ids, + self.pq_codes, + self.row_id_order, + self.adjacency_index, + self.adjacency, + self.vectors, + ] + } +} + +impl Index for DiskAnnSections { + type Output = SectionRange; + + fn index(&self, index: usize) -> &Self::Output { + match index { + 0 => &self.codebook, + 1 => &self.row_ids, + 2 => &self.pq_codes, + 3 => &self.row_id_order, + 4 => &self.adjacency_index, + 5 => &self.adjacency, + 6 => &self.vectors, + _ => panic!("DiskANN section index {index} is out of range"), + } + } +} + +impl IndexMut for DiskAnnSections { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + match index { + 0 => &mut self.codebook, + 1 => &mut self.row_ids, + 2 => &mut self.pq_codes, + 3 => &mut self.row_id_order, + 4 => &mut self.adjacency_index, + 5 => &mut self.adjacency, + 6 => &mut self.vectors, + _ => panic!("DiskANN section index {index} is out of range"), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct AdjacencyLocator { + pub page_index: u32, + pub byte_offset: u16, + degree_and_flags: u16, +} + +impl AdjacencyLocator { + fn new( + page_index: u32, + byte_offset: u16, + degree: usize, + encoding: AdjacencyListEncoding, + ) -> io::Result { + let degree = u16::try_from(degree) + .map_err(|_| invalid_input("DiskANN adjacency degree exceeds u16"))?; + if degree > ADJACENCY_DEGREE_MASK { + return Err(invalid_input( + "DiskANN adjacency degree exceeds locator capacity", + )); + } + let encoding_flag = match encoding { + AdjacencyListEncoding::DeltaVarint => 0, + AdjacencyListEncoding::RawU32 => ADJACENCY_RAW_U32_FLAG, + }; + Ok(Self { + page_index, + byte_offset, + degree_and_flags: degree | encoding_flag, + }) + } + + pub(crate) fn degree(self) -> usize { + usize::from(self.degree_and_flags & ADJACENCY_DEGREE_MASK) + } + + pub(crate) fn encoding(self) -> AdjacencyListEncoding { + if self.degree_and_flags & ADJACENCY_RAW_U32_FLAG == 0 { + AdjacencyListEncoding::DeltaVarint + } else { + AdjacencyListEncoding::RawU32 + } + } +} + +#[derive(Debug)] +struct AdjacencyIndex { + block_offsets: Box<[u64]>, + relative_offsets: Box<[u16]>, + degree_and_flags: Box<[u16]>, +} + +impl AdjacencyIndex { + #[cfg(test)] + fn from_locators(locators: &[AdjacencyLocator]) -> io::Result { + let block_count = locators.len().div_ceil(ADJACENCY_LOCATOR_BLOCK_NODES); + let mut block_offsets = Vec::new(); + block_offsets + .try_reserve_exact(block_count) + .map_err(|_| invalid_data("DiskANN adjacency block-offset allocation failed"))?; + let mut relative_offsets = Vec::new(); + relative_offsets + .try_reserve_exact(locators.len()) + .map_err(|_| invalid_data("DiskANN adjacency relative-offset allocation failed"))?; + let mut degree_and_flags = Vec::new(); + degree_and_flags + .try_reserve_exact(locators.len()) + .map_err(|_| invalid_data("DiskANN adjacency metadata allocation failed"))?; + + for (node, locator) in locators.iter().copied().enumerate() { + let absolute_offset = adjacency_locator_absolute_offset(locator)?; + if node.is_multiple_of(ADJACENCY_LOCATOR_BLOCK_NODES) { + block_offsets.push(absolute_offset); + } + let block_offset = *block_offsets + .last() + .expect("each adjacency locator belongs to a block"); + let relative_offset = absolute_offset + .checked_sub(block_offset) + .and_then(|offset| u16::try_from(offset).ok()) + .ok_or_else(|| { + invalid_data("DiskANN adjacency locator exceeds its block offset range") + })?; + relative_offsets.push(relative_offset); + degree_and_flags.push(locator.degree_and_flags); + } + Ok(Self { + block_offsets: block_offsets.into_boxed_slice(), + relative_offsets: relative_offsets.into_boxed_slice(), + degree_and_flags: degree_and_flags.into_boxed_slice(), + }) + } + + fn len(&self) -> usize { + self.relative_offsets.len() + } + + fn locator(&self, node: usize) -> Option { + let relative_offset = u64::from(*self.relative_offsets.get(node)?); + let degree_and_flags = *self.degree_and_flags.get(node)?; + let block_offset = *self + .block_offsets + .get(node / ADJACENCY_LOCATOR_BLOCK_NODES)?; + let absolute_offset = block_offset.checked_add(relative_offset)?; + let page_index = u32::try_from(absolute_offset / u64::from(DISKANN_PAGE_SIZE)).ok()?; + let byte_offset = u16::try_from(absolute_offset % u64::from(DISKANN_PAGE_SIZE)).ok()?; + Some(AdjacencyLocator { + page_index, + byte_offset, + degree_and_flags, + }) + } + + fn partition_point(&self, mut predicate: impl FnMut(AdjacencyLocator) -> bool) -> usize { + let mut left = 0; + let mut right = self.len(); + while left < right { + let middle = left + (right - left) / 2; + let locator = self + .locator(middle) + .expect("validated DiskANN adjacency index"); + if predicate(locator) { + left = middle + 1; + } else { + right = middle; + } + } + left + } +} + +fn adjacency_locator_absolute_offset(locator: AdjacencyLocator) -> io::Result { + u64::from(locator.page_index) + .checked_mul(u64::from(DISKANN_PAGE_SIZE)) + .and_then(|offset| offset.checked_add(u64::from(locator.byte_offset))) + .ok_or_else(|| invalid_data("DiskANN adjacency locator offset overflows")) +} + +fn adjacency_index_serialized_len(vector_count: usize) -> io::Result { + let block_count = vector_count.div_ceil(ADJACENCY_LOCATOR_BLOCK_NODES); + let block_bytes = block_count + .checked_mul(ADJACENCY_LOCATOR_BLOCK_BASE_SIZE) + .ok_or_else(|| invalid_input("DiskANN adjacency block-offset size overflows usize"))?; + let locator_bytes = vector_count + .checked_mul(ADJACENCY_LOCATOR_SIZE as usize) + .ok_or_else(|| invalid_input("DiskANN adjacency locator size overflows usize"))?; + u64::try_from( + block_bytes + .checked_add(locator_bytes) + .ok_or_else(|| invalid_input("DiskANN adjacency index size overflows usize"))?, + ) + .map_err(|_| invalid_input("DiskANN adjacency index size exceeds u64")) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum AdjacencyListEncoding { + DeltaVarint, + RawU32, +} + +fn encode_adjacency_list( + neighbors: &[u32], + encoded: &mut Vec, +) -> io::Result { + encoded.clear(); + let (encoding, encoded_len) = plan_adjacency_list(neighbors)?; + encoded + .try_reserve_exact(encoded_len) + .map_err(|_| invalid_input("DiskANN adjacency allocation failed"))?; + match encoding { + AdjacencyListEncoding::DeltaVarint => { + let mut previous = 0u32; + for &neighbor in neighbors { + append_u32_varint(encoded, neighbor - previous); + previous = neighbor; + } + } + AdjacencyListEncoding::RawU32 => { + for &neighbor in neighbors { + encoded.extend_from_slice(&neighbor.to_le_bytes()); + } + } + } + Ok(encoding) +} + +fn plan_adjacency_list(neighbors: &[u32]) -> io::Result<(AdjacencyListEncoding, usize)> { + let raw_len = neighbors + .len() + .checked_mul(size_of::()) + .ok_or_else(|| invalid_input("DiskANN adjacency list size overflows usize"))?; + let delta_len = adjacency_delta_varint_len(neighbors) + .ok_or_else(|| invalid_input("DiskANN adjacency neighbors must be strictly increasing"))?; + if neighbors.is_empty() || delta_len < raw_len { + return Ok((AdjacencyListEncoding::DeltaVarint, delta_len)); + } + Ok((AdjacencyListEncoding::RawU32, raw_len)) +} + +fn adjacency_delta_varint_len(neighbors: &[u32]) -> Option { + let mut previous = 0u32; + let mut encoded_len = 0usize; + for (index, &neighbor) in neighbors.iter().enumerate() { + if index != 0 && neighbor <= previous { + return None; + } + encoded_len = encoded_len.checked_add(u32_varint_len(neighbor - previous))?; + previous = neighbor; + } + Some(encoded_len) +} + +fn u32_varint_len(value: u32) -> usize { + let significant_bits = (u32::BITS - value.leading_zeros()).max(1); + significant_bits.div_ceil(7) as usize +} + +pub(crate) fn decode_adjacency_list( + bytes: &[u8], + degree: usize, + encoding: AdjacencyListEncoding, + neighbors: &mut Vec, +) -> io::Result { + neighbors.clear(); + neighbors + .try_reserve(degree) + .map_err(|_| invalid_data("DiskANN adjacency decode allocation failed"))?; + match encoding { + AdjacencyListEncoding::DeltaVarint => { + let mut position = 0usize; + let mut previous = 0u32; + for _ in 0..degree { + let delta = read_u32_varint(bytes, &mut position)?; + let neighbor = previous + .checked_add(delta) + .ok_or_else(|| invalid_data("DiskANN adjacency delta overflows u32"))?; + neighbors.push(neighbor); + previous = neighbor; + } + Ok(position) + } + AdjacencyListEncoding::RawU32 => { + let encoded_len = degree + .checked_mul(size_of::()) + .ok_or_else(|| invalid_data("DiskANN raw adjacency size overflows usize"))?; + let encoded = bytes + .get(..encoded_len) + .ok_or_else(|| invalid_data("DiskANN raw adjacency list is truncated"))?; + neighbors.extend(encoded.chunks_exact(4).map(|value| { + u32::from_le_bytes(value.try_into().expect("fixed adjacency neighbor")) + })); + Ok(encoded_len) + } + } +} + +fn append_u32_varint(encoded: &mut Vec, mut value: u32) { + while value >= 0x80 { + encoded.push((value as u8 & 0x7f) | 0x80); + value >>= 7; + } + encoded.push(value as u8); +} + +fn read_u32_varint(bytes: &[u8], position: &mut usize) -> io::Result { + let mut value = 0u32; + let start = *position; + for shift in (0..=28).step_by(7) { + let byte = *bytes + .get(*position) + .ok_or_else(|| invalid_data("DiskANN adjacency varint is truncated"))?; + *position += 1; + if shift == 28 && byte > 0x0f { + return Err(invalid_data("DiskANN adjacency varint exceeds u32")); + } + value |= u32::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + if *position - start > 1 && byte == 0 { + return Err(invalid_data("DiskANN adjacency varint is not canonical")); + } + return Ok(value); + } + } + Err(invalid_data("DiskANN adjacency varint exceeds five bytes")) +} + +#[derive(Debug, Clone, PartialEq)] +pub struct DiskAnnHeader { + pub flags: u32, + pub dimension: u32, + pub metric: u32, + pub vector_count: u64, + pub entry_node: u32, + pub max_degree: u32, + pub build_search_list_size: u32, + pub alpha: f32, + pub seed: u64, + pub pq_m: u32, + pub pq_bits: u32, + pub page_size: u32, + pub adjacency_locator_size: u32, + pub adjacency_locator_encoding: u32, + pub raw_vector_encoding: u32, + pub vector_record_size: u32, + pub file_len: u64, + pub sections: DiskAnnSections, +} + +pub struct DiskAnnIndexReader { + reader: R, + pub header: DiskAnnHeader, + resident: Option>, + options: ResolvedVectorIndexReaderOptions, + read_capabilities: SeekReadCapabilities, + effective_read_tier: DeploymentProfile, + random_read_latency: Duration, + hot_adjacency: Arc<[u8]>, + row_id_order: Arc>, + pub(crate) query_scratch: Box, + pub(crate) last_search_stats: DiskAnnSearchStats, + pub(crate) batch_workers: Vec>, + pub(crate) calibrated_l_search: Option, +} + +struct DiskAnnResidentData { + pq: ProductQuantizer, + row_ids: RowIdStorage, + pq_codes: Vec, + adjacency_index: AdjacencyIndex, + adjacency_validation: AdjacencyValidationCache, + adjacency_cache: SharedWindowCache, + raw_vector_cache: SharedWindowCache, +} + +#[derive(Clone, Copy, Default)] +struct OffsetLruLink { + older: Option, + newer: Option, +} + +#[derive(Default)] +pub(crate) struct OffsetLru { + links: HashMap, + oldest: Option, + newest: Option, +} + +impl OffsetLru { + pub(crate) fn touch(&mut self, offset: u64) { + if self.newest == Some(offset) { + return; + } + let already_present = match self.links.entry(offset) { + std::collections::hash_map::Entry::Occupied(_) => true, + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert(OffsetLruLink::default()); + false + } + }; + if already_present { + self.detach(offset); + } + let older = self.newest; + if let Some(older) = older { + self.links + .get_mut(&older) + .expect("DiskANN LRU newest offset must exist") + .newer = Some(offset); + } else { + self.oldest = Some(offset); + } + let link = self + .links + .get_mut(&offset) + .expect("DiskANN LRU touched offset must exist"); + link.older = older; + link.newer = None; + self.newest = Some(offset); + } + + pub(crate) fn remove(&mut self, offset: u64) { + if self.links.contains_key(&offset) { + self.detach(offset); + self.links.remove(&offset); + } + } + + pub(crate) fn pop_oldest(&mut self) -> Option { + let offset = self.oldest?; + self.remove(offset); + Some(offset) + } + + pub(crate) fn clear(&mut self) { + self.links.clear(); + self.oldest = None; + self.newest = None; + } + + #[cfg(test)] + pub(crate) fn is_empty(&self) -> bool { + self.links.is_empty() + } + + #[cfg(test)] + pub(crate) fn len(&self) -> usize { + self.links.len() + } + + #[cfg(test)] + pub(crate) fn oldest_offsets(&self) -> Vec { + let mut offsets = Vec::with_capacity(self.links.len()); + let mut current = self.oldest; + while let Some(offset) = current { + offsets.push(offset); + current = self.links.get(&offset).and_then(|link| link.newer); + } + debug_assert_eq!(offsets.len(), self.links.len()); + offsets + } + + fn detach(&mut self, offset: u64) { + let link = *self + .links + .get(&offset) + .expect("DiskANN LRU detached offset must exist"); + if let Some(older) = link.older { + self.links + .get_mut(&older) + .expect("DiskANN LRU older offset must exist") + .newer = link.newer; + } else { + self.oldest = link.newer; + } + if let Some(newer) = link.newer { + self.links + .get_mut(&newer) + .expect("DiskANN LRU newer offset must exist") + .older = link.older; + } else { + self.newest = link.older; + } + } +} + +pub(crate) enum SharedWindowCacheLookup { + Hit(Arc>), + Reserved, + Loading, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct CacheLockMetrics { + pub(crate) acquisitions: usize, + pub(crate) wait_nanos: u64, +} + +struct SharedWindowCacheState { + entries: HashMap>>, + loading: HashSet, + recency: OffsetLru, + retained_bytes: usize, +} + +struct SharedWindowCacheShard { + capacity_bytes: AtomicUsize, + state: Mutex, + waiters: Condvar, +} + +const SHARED_WINDOW_CACHE_SHARDS: usize = 16; +const MAX_SHARED_READ_WINDOW_BYTES: usize = 64 * 1024; + +pub(crate) struct SharedWindowCache { + shards: Box<[SharedWindowCacheShard]>, +} + +impl SharedWindowCache { + fn new(capacity_bytes: usize) -> Self { + Self::new_with_max_shards(capacity_bytes, SHARED_WINDOW_CACHE_SHARDS) + } + + fn new_with_max_shards(capacity_bytes: usize, max_shards: usize) -> Self { + let max_shards = max_shards.max(1); + let shard_count = if capacity_bytes >= max_shards * MAX_SHARED_READ_WINDOW_BYTES { + max_shards + } else { + 1 + }; + let base_capacity = capacity_bytes / shard_count; + let remainder = capacity_bytes % shard_count; + let shards = (0..shard_count) + .map(|shard| SharedWindowCacheShard { + capacity_bytes: AtomicUsize::new(base_capacity + usize::from(shard < remainder)), + state: Mutex::new(SharedWindowCacheState { + entries: HashMap::new(), + loading: HashSet::new(), + recency: OffsetLru::default(), + retained_bytes: 0, + }), + waiters: Condvar::new(), + }) + .collect::>() + .into_boxed_slice(); + Self { shards } + } + + fn shard_index(&self, offset: u64) -> usize { + let page = offset / u64::from(DISKANN_PAGE_SIZE); + let mut mixed = page.wrapping_add(0x9e37_79b9_7f4a_7c15); + mixed = (mixed ^ (mixed >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + mixed = (mixed ^ (mixed >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + ((mixed ^ (mixed >> 31)) as usize) % self.shards.len() + } + + fn shard(&self, offset: u64) -> &SharedWindowCacheShard { + &self.shards[self.shard_index(offset)] + } + + #[cfg(test)] + fn shard_count(&self) -> usize { + self.shards.len() + } + + fn total_capacity(&self) -> usize { + self.shards + .iter() + .map(|shard| shard.capacity_bytes.load(AtomicOrdering::Relaxed)) + .sum() + } + + fn add_lock_metrics(total: &mut CacheLockMetrics, metrics: CacheLockMetrics) { + total.acquisitions = total.acquisitions.saturating_add(metrics.acquisitions); + total.wait_nanos = total.wait_nanos.saturating_add(metrics.wait_nanos); + } + + fn set_total_capacity(&self, capacity_bytes: usize) -> io::Result<()> { + let base_capacity = capacity_bytes / self.shards.len(); + let remainder = capacity_bytes % self.shards.len(); + for (shard_index, shard) in self.shards.iter().enumerate() { + let shard_capacity = base_capacity + usize::from(shard_index < remainder); + shard + .capacity_bytes + .store(shard_capacity, AtomicOrdering::Relaxed); + let (mut state, _) = Self::lock_state(shard)?; + while state.retained_bytes > shard_capacity { + let Some(oldest) = state.recency.pop_oldest() else { + break; + }; + if let Some(evicted) = state.entries.remove(&oldest) { + state.retained_bytes = state.retained_bytes.saturating_sub(evicted.capacity()); + } + } + } + Ok(()) + } + + fn lock_state( + shard: &SharedWindowCacheShard, + ) -> io::Result<(MutexGuard<'_, SharedWindowCacheState>, CacheLockMetrics)> { + let started = Instant::now(); + let state = shard + .state + .lock() + .map_err(|_| invalid_data("DiskANN shared window cache state is poisoned"))?; + Ok(( + state, + CacheLockMetrics { + acquisitions: 1, + wait_nanos: u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX), + }, + )) + } + + fn remove_loading(&self, offsets: &[u64], shard_index: usize) -> io::Result { + let shard = &self.shards[shard_index]; + let (mut state, metrics) = Self::lock_state(shard)?; + for offset in offsets { + if self.shard_index(*offset) == shard_index { + state.loading.remove(offset); + } + } + shard.waiters.notify_all(); + Ok(metrics) + } + + pub(crate) fn lookup_or_reserve( + &self, + offset: u64, + length: usize, + ) -> io::Result<(SharedWindowCacheLookup, CacheLockMetrics)> { + let shard = self.shard(offset); + let (mut state, metrics) = Self::lock_state(shard)?; + if let Some(payload) = state.entries.get(&offset).cloned() { + if payload.len() == length { + state.recency.touch(offset); + return Ok((SharedWindowCacheLookup::Hit(payload), metrics)); + } + state.entries.remove(&offset); + state.recency.remove(offset); + state.retained_bytes = state.retained_bytes.saturating_sub(payload.capacity()); + } + if state.loading.contains(&offset) { + return Ok((SharedWindowCacheLookup::Loading, metrics)); + } + state.loading.insert(offset); + Ok((SharedWindowCacheLookup::Reserved, metrics)) + } + + pub(crate) fn publish( + &self, + offset: u64, + payload: Arc>, + ) -> io::Result<(usize, CacheLockMetrics)> { + let shard = self.shard(offset); + let (mut state, metrics) = Self::lock_state(shard)?; + state.loading.remove(&offset); + if let Some(previous) = state.entries.insert(offset, Arc::clone(&payload)) { + state.retained_bytes = state.retained_bytes.saturating_sub(previous.capacity()); + } + state.retained_bytes = state.retained_bytes.saturating_add(payload.capacity()); + state.recency.touch(offset); + let mut evictions = 0usize; + let capacity_bytes = shard.capacity_bytes.load(AtomicOrdering::Relaxed); + while state.retained_bytes > capacity_bytes { + let Some(oldest) = state.recency.pop_oldest() else { + break; + }; + if let Some(evicted) = state.entries.remove(&oldest) { + state.retained_bytes = state.retained_bytes.saturating_sub(evicted.capacity()); + evictions = evictions.saturating_add(1); + } + } + shard.waiters.notify_all(); + Ok((evictions, metrics)) + } + + pub(crate) fn cancel(&self, offsets: &[u64]) -> io::Result { + let mut metrics = CacheLockMetrics::default(); + for shard_index in 0..self.shards.len() { + if offsets + .iter() + .any(|offset| self.shard_index(*offset) == shard_index) + { + Self::add_lock_metrics(&mut metrics, self.remove_loading(offsets, shard_index)?); + } + } + Ok(metrics) + } + + pub(crate) fn wait_for( + &self, + offset: u64, + length: usize, + ) -> io::Result<(Option>>, CacheLockMetrics)> { + let shard = self.shard(offset); + let (mut state, metrics) = Self::lock_state(shard)?; + while state.loading.contains(&offset) { + state = shard + .waiters + .wait(state) + .map_err(|_| invalid_data("DiskANN shared window cache state is poisoned"))?; + } + let payload = state + .entries + .get(&offset) + .filter(|payload| payload.len() == length) + .cloned(); + if payload.is_some() { + state.recency.touch(offset); + } + Ok((payload, metrics)) + } +} + +const ADJACENCY_PAGE_UNVALIDATED: u8 = 0; +const ADJACENCY_PAGE_VALIDATING: u8 = 1; +const ADJACENCY_PAGE_VALID: u8 = 2; +const ADJACENCY_PAGE_INVALID: u8 = 3; + +#[derive(Clone)] +struct CachedValidationError { + kind: io::ErrorKind, + message: String, +} + +impl CachedValidationError { + fn from_error(error: &io::Error) -> Self { + Self { + kind: error.kind(), + message: error.to_string(), + } + } + + fn to_error(&self) -> io::Error { + io::Error::new(self.kind, self.message.clone()) + } +} + +struct AdjacencyValidationCache { + states: Box<[AtomicU8]>, + errors: Mutex>, + wait_lock: Mutex<()>, + waiters: Condvar, +} + +impl AdjacencyValidationCache { + fn new(page_count: usize) -> io::Result { + let mut states = Vec::new(); + states + .try_reserve_exact(page_count) + .map_err(|_| invalid_data("DiskANN adjacency validation cache allocation failed"))?; + states.extend((0..page_count).map(|_| AtomicU8::new(ADJACENCY_PAGE_UNVALIDATED))); + Ok(Self { + states: states.into_boxed_slice(), + errors: Mutex::new(HashMap::new()), + wait_lock: Mutex::new(()), + waiters: Condvar::new(), + }) + } + + fn get_or_validate( + &self, + page_index: usize, + validate: impl FnOnce() -> io::Result<()>, + ) -> io::Result<()> { + let state = self + .states + .get(page_index) + .ok_or_else(|| invalid_data("DiskANN adjacency page is out of range"))?; + loop { + match state.load(AtomicOrdering::Acquire) { + ADJACENCY_PAGE_UNVALIDATED => { + if state + .compare_exchange( + ADJACENCY_PAGE_UNVALIDATED, + ADJACENCY_PAGE_VALIDATING, + AtomicOrdering::AcqRel, + AtomicOrdering::Acquire, + ) + .is_err() + { + continue; + } + let mut claim = AdjacencyValidationClaim { + state, + wait_lock: &self.wait_lock, + waiters: &self.waiters, + published: false, + }; + return match validate() { + Ok(()) => { + claim.publish(ADJACENCY_PAGE_VALID); + Ok(()) + } + Err(error) => { + self.errors + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(page_index, CachedValidationError::from_error(&error)); + claim.publish(ADJACENCY_PAGE_INVALID); + Err(error) + } + }; + } + ADJACENCY_PAGE_VALIDATING => { + let guard = self + .wait_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _guard = self + .waiters + .wait_while(guard, |_| { + state.load(AtomicOrdering::Acquire) == ADJACENCY_PAGE_VALIDATING + }) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + ADJACENCY_PAGE_VALID => return Ok(()), + ADJACENCY_PAGE_INVALID => { + let errors = self + .errors + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + return Err(errors.get(&page_index).map_or_else( + || invalid_data("DiskANN adjacency page failed validation"), + CachedValidationError::to_error, + )); + } + _ => return Err(invalid_data("invalid DiskANN adjacency validation state")), + } + } + } +} + +struct AdjacencyValidationClaim<'a> { + state: &'a AtomicU8, + wait_lock: &'a Mutex<()>, + waiters: &'a Condvar, + published: bool, +} + +impl AdjacencyValidationClaim<'_> { + fn publish(&mut self, state: u8) { + let _guard = self + .wait_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + self.state.store(state, AtomicOrdering::Release); + self.published = true; + self.waiters.notify_all(); + } +} + +impl Drop for AdjacencyValidationClaim<'_> { + fn drop(&mut self) { + if !self.published { + let _guard = self + .wait_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + self.state + .store(ADJACENCY_PAGE_UNVALIDATED, AtomicOrdering::Release); + self.waiters.notify_all(); + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum RowIdStorage { + Raw(Vec), + ForBitPacked { + base: i64, + bit_width: u8, + count: usize, + payload: Vec, + }, +} + +impl RowIdStorage { + #[cfg(test)] + fn encode(row_ids: Vec) -> io::Result { + Self::encode_from_fn(row_ids.len(), |node| row_ids[node]) + } + + fn encode_from_fn(count: usize, row_id_at: impl Fn(usize) -> i64) -> io::Result { + if count == 0 { + return Err(invalid_input("DiskANN row IDs must not be empty")); + } + let mut base = row_id_at(0); + let mut maximum = base; + for node in 1..count { + let row_id = row_id_at(node); + base = base.min(row_id); + maximum = maximum.max(row_id); + } + let span = u64::try_from(maximum as i128 - base as i128) + .expect("the difference between two i64 values fits in u64"); + let bit_width = if span == 0 { + 0 + } else { + (u64::BITS - span.leading_zeros()) as u8 + }; + if bit_width == u64::BITS as u8 { + let mut row_ids = Vec::new(); + row_ids + .try_reserve_exact(count) + .map_err(|_| invalid_input("DiskANN raw row-ID allocation failed"))?; + row_ids.extend((0..count).map(row_id_at)); + return Ok(Self::Raw(row_ids)); + } + + let payload_len = packed_row_id_payload_len(count, bit_width)?; + let mut payload = Vec::new(); + payload + .try_reserve_exact(payload_len) + .map_err(|_| invalid_input("DiskANN packed row-ID allocation failed"))?; + payload.resize(payload_len, 0); + for node in 0..count { + let row_id = row_id_at(node); + let delta = u64::try_from(row_id as i128 - base as i128) + .expect("the row ID is not below the selected base"); + pack_row_id_delta(&mut payload, node, bit_width, delta)?; + } + Ok(Self::ForBitPacked { + base, + bit_width, + count, + payload, + }) + } + + fn len(&self) -> usize { + match self { + Self::Raw(row_ids) => row_ids.len(), + Self::ForBitPacked { count, .. } => *count, + } + } + + #[cfg(test)] + fn bit_width(&self) -> u8 { + match self { + Self::Raw(_) => u64::BITS as u8, + Self::ForBitPacked { bit_width, .. } => *bit_width, + } + } + + fn get(&self, node: usize) -> Option { + match self { + Self::Raw(row_ids) => row_ids.get(node).copied(), + Self::ForBitPacked { + base, + bit_width, + count, + payload, + } => { + if node >= *count { + return None; + } + let delta = unpack_row_id_delta(payload, node, *bit_width)?; + i64::try_from(*base as i128 + delta as i128).ok() + } + } + } + + fn try_for_each( + &self, + mut visitor: impl FnMut(usize, i64) -> io::Result<()>, + ) -> io::Result<()> { + match self { + Self::Raw(row_ids) => { + for (node, &row_id) in row_ids.iter().enumerate() { + visitor(node, row_id)?; + } + } + Self::ForBitPacked { + base, + bit_width, + count, + payload, + } => { + let mut bit_offset = 0usize; + for node in 0..*count { + let delta = unpack_row_id_delta_at_bit_offset(payload, bit_offset, *bit_width) + .ok_or_else(|| { + invalid_data("DiskANN packed row-ID payload is truncated") + })?; + let row_id = i64::try_from(*base as i128 + delta as i128) + .map_err(|_| invalid_data("DiskANN packed row ID overflows i64"))?; + visitor(node, row_id)?; + bit_offset = bit_offset + .checked_add(*bit_width as usize) + .ok_or_else(|| invalid_data("DiskANN packed row-ID offset overflows"))?; + } + } + } + Ok(()) + } + + fn payload_len(&self) -> io::Result { + match self { + Self::Raw(row_ids) => row_ids + .len() + .checked_mul(size_of::()) + .ok_or_else(|| invalid_input("DiskANN raw row-ID length overflows usize")), + Self::ForBitPacked { payload, .. } => Ok(payload.len()), + } + } + + fn serialized_len(&self) -> io::Result { + ROW_ID_SECTION_HEADER_SIZE + .checked_add(self.payload_len()?) + .ok_or_else(|| invalid_input("DiskANN row-ID section length overflows usize")) + } + + fn section_header(&self) -> [u8; ROW_ID_SECTION_HEADER_SIZE] { + let mut bytes = [0u8; ROW_ID_SECTION_HEADER_SIZE]; + match self { + Self::Raw(row_ids) => { + put_u32(&mut bytes, 0, ROW_ID_ENCODING_RAW_I64); + put_u32(&mut bytes, 4, u64::BITS); + put_u64(&mut bytes, 8, row_ids.len() as u64); + } + Self::ForBitPacked { + base, + bit_width, + count, + .. + } => { + put_u32(&mut bytes, 0, ROW_ID_ENCODING_FOR_BITPACK); + put_u32(&mut bytes, 4, *bit_width as u32); + put_u64(&mut bytes, 8, *count as u64); + put_u64(&mut bytes, 16, *base as u64); + } + } + bytes + } +} + +fn packed_row_id_payload_len(count: usize, bit_width: u8) -> io::Result { + count + .checked_mul(bit_width as usize) + .and_then(|bits| bits.checked_add(7)) + .map(|bits| bits / 8) + .ok_or_else(|| invalid_input("DiskANN packed row-ID length overflows usize")) +} + +fn raw_row_id_section_len(count: usize) -> io::Result { + count + .checked_mul(size_of::()) + .and_then(|payload| payload.checked_add(ROW_ID_SECTION_HEADER_SIZE)) + .ok_or_else(|| invalid_input("DiskANN raw row-ID section length overflows usize")) +} + +fn pack_row_id_delta(payload: &mut [u8], node: usize, bit_width: u8, delta: u64) -> io::Result<()> { + if bit_width == 0 { + return Ok(()); + } + let bit_offset = node + .checked_mul(bit_width as usize) + .ok_or_else(|| invalid_input("DiskANN packed row-ID offset overflows usize"))?; + let byte_offset = bit_offset / 8; + let shift = bit_offset % 8; + let byte_count = (shift + bit_width as usize).div_ceil(8); + let encoded = (delta as u128) << shift; + let destination = payload + .get_mut(byte_offset..byte_offset + byte_count) + .ok_or_else(|| invalid_input("DiskANN packed row-ID payload is truncated"))?; + for (index, byte) in destination.iter_mut().enumerate() { + *byte |= (encoded >> (index * 8)) as u8; + } + Ok(()) +} + +fn unpack_row_id_delta(payload: &[u8], node: usize, bit_width: u8) -> Option { + let bit_offset = node.checked_mul(bit_width as usize)?; + unpack_row_id_delta_at_bit_offset(payload, bit_offset, bit_width) +} + +fn unpack_row_id_delta_at_bit_offset( + payload: &[u8], + bit_offset: usize, + bit_width: u8, +) -> Option { + if bit_width == 0 { + return Some(0); + } + let byte_offset = bit_offset / 8; + let shift = bit_offset % 8; + let byte_count = (shift + bit_width as usize).div_ceil(8); + let source = payload.get(byte_offset..byte_offset + byte_count)?; + let encoded = source + .iter() + .enumerate() + .fold(0u128, |value, (index, &byte)| { + value | ((byte as u128) << (index * 8)) + }); + let mask = (1u128 << bit_width) - 1; + Some(((encoded >> shift) & mask) as u64) +} + +#[derive(Debug, Clone, Copy)] +struct RowIdSectionHeader { + encoding: u32, + bit_width: u8, + count: usize, + base: i64, +} + +fn decode_row_id_section_header( + bytes: &[u8], + section_len: usize, + expected_count: usize, +) -> io::Result { + if bytes.len() < ROW_ID_SECTION_HEADER_SIZE { + return Err(invalid_data("DiskANN row-ID section header is truncated")); + } + if bytes[24..ROW_ID_SECTION_HEADER_SIZE] + .iter() + .any(|&byte| byte != 0) + { + return Err(invalid_data( + "DiskANN row-ID section reserved bytes must be zero", + )); + } + let count = usize::try_from(get_u64(bytes, 8)) + .map_err(|_| invalid_data("DiskANN row-ID count exceeds usize"))?; + if count != expected_count { + return Err(invalid_data("DiskANN row-ID count does not match header")); + } + let encoding = get_u32(bytes, 0); + let width = get_u32(bytes, 4); + let base = get_u64(bytes, 16) as i64; + let (bit_width, payload_len) = match encoding { + ROW_ID_ENCODING_RAW_I64 => { + if width != u64::BITS || base != 0 { + return Err(invalid_data("invalid DiskANN raw row-ID metadata")); + } + ( + u64::BITS as u8, + count + .checked_mul(size_of::()) + .ok_or_else(|| invalid_data("DiskANN raw row-ID length overflows usize"))?, + ) + } + ROW_ID_ENCODING_FOR_BITPACK => { + let bit_width = u8::try_from(width) + .map_err(|_| invalid_data("invalid DiskANN FOR row-ID bit width"))?; + if bit_width >= u64::BITS as u8 { + return Err(invalid_data("invalid DiskANN FOR row-ID bit width")); + } + let payload_len = packed_row_id_payload_len(count, bit_width) + .map_err(|_| invalid_data("DiskANN packed row-ID length overflows usize"))?; + (bit_width, payload_len) + } + _ => return Err(invalid_data("unsupported DiskANN row-ID encoding")), + }; + if ROW_ID_SECTION_HEADER_SIZE.checked_add(payload_len) != Some(section_len) { + return Err(invalid_data("invalid DiskANN row-ID payload length")); + } + Ok(RowIdSectionHeader { + encoding, + bit_width, + count, + base, + }) +} + +fn validate_row_id_storage(storage: &RowIdStorage) -> io::Result<()> { + if let RowIdStorage::ForBitPacked { + bit_width, + count, + payload, + .. + } = storage + { + let used_bits = count + .checked_mul(*bit_width as usize) + .ok_or_else(|| invalid_data("DiskANN packed row-ID length overflows usize"))?; + let tail_bits = used_bits % 8; + if tail_bits != 0 + && payload + .last() + .is_some_and(|&byte| byte & !((1u8 << tail_bits) - 1) != 0) + { + return Err(invalid_data("DiskANN packed row-ID tail bits must be zero")); + } + } + storage.try_for_each(|_, _| Ok(())) +} + +#[cfg(test)] +fn encode_row_id_section(storage: &RowIdStorage) -> io::Result> { + let section_len = storage.serialized_len()?; + let mut bytes = Vec::new(); + bytes + .try_reserve_exact(section_len) + .map_err(|_| invalid_input("DiskANN row-ID section allocation failed"))?; + bytes.extend_from_slice(&storage.section_header()); + match storage { + RowIdStorage::Raw(row_ids) => row_ids + .iter() + .for_each(|row_id| bytes.extend_from_slice(&row_id.to_le_bytes())), + RowIdStorage::ForBitPacked { payload, .. } => bytes.extend_from_slice(payload), + } + Ok(bytes) +} + +#[cfg(test)] +fn decode_row_id_section(bytes: &[u8], expected_count: usize) -> io::Result { + let header = decode_row_id_section_header(bytes, bytes.len(), expected_count)?; + let payload = &bytes[ROW_ID_SECTION_HEADER_SIZE..]; + let storage = match header.encoding { + ROW_ID_ENCODING_RAW_I64 => { + let mut row_ids = Vec::new(); + row_ids + .try_reserve_exact(header.count) + .map_err(|_| invalid_data("DiskANN raw row-ID allocation failed"))?; + row_ids.extend(payload.chunks_exact(8).map(|value| { + i64::from_le_bytes(value.try_into().expect("validated eight-byte row ID")) + })); + RowIdStorage::Raw(row_ids) + } + ROW_ID_ENCODING_FOR_BITPACK => { + let mut packed = Vec::new(); + packed + .try_reserve_exact(payload.len()) + .map_err(|_| invalid_data("DiskANN packed row-ID allocation failed"))?; + packed.extend_from_slice(payload); + RowIdStorage::ForBitPacked { + base: header.base, + bit_width: header.bit_width, + count: header.count, + payload: packed, + } + } + _ => unreachable!("row-ID encoding was validated"), + }; + validate_row_id_storage(&storage)?; + Ok(storage) +} + +#[derive(Default)] +enum RowIdOrderState { + #[default] + NotLoaded, + Loaded(Arc<[u32]>), + UnavailableByBudget, +} + +fn classify_read_tier(random_read_latency: Duration) -> DeploymentProfile { + if random_read_latency < AUTO_PROFILE_MEMORY_LATENCY_THRESHOLD { + DeploymentProfile::Memory + } else if random_read_latency < AUTO_PROFILE_LOCAL_LATENCY_THRESHOLD { + DeploymentProfile::LocalStorage + } else if random_read_latency < AUTO_PROFILE_REMOTE_LATENCY_THRESHOLD { + DeploymentProfile::RemoteStorage + } else { + DeploymentProfile::ObjectStore + } +} + +impl DiskAnnIndexReader { + pub fn open(reader: R) -> io::Result { + Self::open_with_options(reader, VectorIndexReaderOptions::default()) + } + + pub fn open_with_options(mut reader: R, options: VectorIndexReaderOptions) -> io::Result { + let read_capabilities = reader.read_capabilities(); + let mut bytes = [0u8; DISKANN_HEADER_SIZE]; + let header_read_started = Instant::now(); + reader + .pread(&mut [ReadRequest::new(0, &mut bytes)]) + .map_err(|error| map_read_error(error, "header"))?; + let measured_header_read_latency = header_read_started.elapsed(); + let header = DiskAnnHeader::decode(&bytes)?; + let random_read_latency = if read_capabilities.estimated_random_read_latency_nanos > 0 { + Duration::from_nanos(read_capabilities.estimated_random_read_latency_nanos) + } else { + measured_header_read_latency.max(Duration::from_nanos(1)) + }; + let effective_read_tier = classify_read_tier(random_read_latency); + let options = options.resolve_cache_budgets( + effective_read_tier, + resident_steady_bytes(&header)?, + usize::try_from(header.sections.adjacency.length).unwrap_or(usize::MAX), + usize::try_from(header.sections.vectors.length).unwrap_or(usize::MAX), + ); + Ok(Self { + reader, + header, + resident: None, + options, + read_capabilities, + effective_read_tier, + random_read_latency, + hot_adjacency: Arc::from([]), + row_id_order: Arc::new(Mutex::new(RowIdOrderState::NotLoaded)), + query_scratch: Box::default(), + last_search_stats: DiskAnnSearchStats::default(), + batch_workers: Vec::new(), + calibrated_l_search: None, + }) + } + + pub fn ensure_resident(&mut self) -> io::Result<()> { + if self.resident.is_some() { + return Ok(()); + } + let peak_bytes = resident_peak_bytes(&self.header)?; + if peak_bytes > self.options.max_resident_bytes { + return Err(invalid_data(format!( + "DiskANN resident warmup requires {} bytes, exceeding reader budget {}", + peak_bytes, self.options.max_resident_bytes + ))); + } + + let (mut pq, row_ids, pq_codes, adjacency_index) = + read_resident_sections(&mut self.reader, &self.header)?; + pq.try_rebuild_norms_cache() + .map_err(|_| invalid_data("DiskANN PQ norms allocation failed"))?; + validate_pq_code_padding(&self.header, &pq_codes)?; + let adjacency_validation = + AdjacencyValidationCache::new(adjacency_page_count(&self.header)?)?; + self.resident = Some(Arc::new(DiskAnnResidentData { + pq, + row_ids, + pq_codes, + adjacency_index, + adjacency_validation, + adjacency_cache: SharedWindowCache::new(self.options.adjacency_cache_bytes), + raw_vector_cache: SharedWindowCache::new(self.options.raw_vector_cache_bytes), + })); + Ok(()) + } + + pub fn pq(&self) -> io::Result<&ProductQuantizer> { + Ok(&self.resident()?.pq) + } + + pub fn row_id(&self, node: usize) -> io::Result { + self.resident()? + .row_ids + .get(node) + .ok_or_else(|| invalid_data("DiskANN row-ID node is out of range")) + } + + pub fn row_id_count(&self) -> io::Result { + Ok(self.resident()?.row_ids.len()) + } + + pub(crate) fn try_for_each_row_id( + &self, + visitor: impl FnMut(usize, i64) -> io::Result<()>, + ) -> io::Result<()> { + self.resident()?.row_ids.try_for_each(visitor) + } + + pub fn pq_codes(&self) -> io::Result<&[u8]> { + Ok(&self.resident()?.pq_codes) + } + + pub(crate) fn adjacency_locator(&self, node: usize) -> io::Result { + self.resident()? + .adjacency_index + .locator(node) + .ok_or_else(|| invalid_data("DiskANN adjacency index is truncated")) + } + + pub(crate) fn adjacency_cache(&self) -> io::Result<&SharedWindowCache> { + Ok(&self.resident()?.adjacency_cache) + } + + pub(crate) fn raw_vector_cache(&self) -> io::Result<&SharedWindowCache> { + Ok(&self.resident()?.raw_vector_cache) + } + + fn resize_shared_cache_budgets(&self, total_bytes: usize) -> io::Result<()> { + let desired_adjacency = self.options.adjacency_cache_bytes; + let desired_raw = self.options.raw_vector_cache_bytes; + let desired_total = desired_adjacency.saturating_add(desired_raw); + let total_bytes = total_bytes.min(desired_total); + let adjacency_bytes = if desired_total == 0 { + 0 + } else { + usize::try_from( + (total_bytes as u128 * desired_adjacency as u128) / desired_total as u128, + ) + .unwrap_or(total_bytes) + }; + let raw_bytes = total_bytes.saturating_sub(adjacency_bytes); + let resident = self.resident()?; + resident + .adjacency_cache + .set_total_capacity(adjacency_bytes)?; + resident.raw_vector_cache.set_total_capacity(raw_bytes) + } + + fn loaded_row_id_order_bytes(&self) -> io::Result { + let state = self + .row_id_order + .lock() + .map_err(|_| invalid_data("DiskANN row-ID lookup state is poisoned"))?; + Ok(match &*state { + RowIdOrderState::Loaded(order) => order + .len() + .checked_mul(size_of::()) + .ok_or_else(|| invalid_data("DiskANN row-ID order size overflows usize"))?, + RowIdOrderState::NotLoaded | RowIdOrderState::UnavailableByBudget => 0, + }) + } + + pub(crate) fn ensure_row_id_order(&mut self) -> io::Result>> { + self.ensure_resident()?; + let mut state = self + .row_id_order + .lock() + .map_err(|_| invalid_data("DiskANN row-ID lookup state is poisoned"))?; + match &*state { + RowIdOrderState::Loaded(order) => return Ok(Some(order.clone())), + RowIdOrderState::UnavailableByBudget => return Ok(None), + RowIdOrderState::NotLoaded => {} + } + let peak_bytes = row_id_order_peak_bytes(&self.header, self.hot_adjacency.len())?; + if peak_bytes > self.options.max_resident_bytes { + *state = RowIdOrderState::UnavailableByBudget; + return Ok(None); + } + // Reserve the decode peak before allocating the immutable lookup. + // Cache hits remain lock-free with respect to this budget operation; + // only cache publication reads the adjusted capacity. + self.resize_shared_cache_budgets(self.options.max_resident_bytes - peak_bytes)?; + let order = read_u32_section( + &mut self.reader, + self.header.sections.row_id_order, + "row-ID order", + )?; + validate_row_id_order(&self.resident()?.row_ids, &order)?; + let order: Arc<[u32]> = Arc::from(order); + *state = RowIdOrderState::Loaded(order.clone()); + let steady_with_order = resident_steady_bytes(&self.header)? + .checked_add(self.hot_adjacency.len()) + .and_then(|bytes| { + order + .len() + .checked_mul(size_of::()) + .and_then(|order_bytes| bytes.checked_add(order_bytes)) + }) + .ok_or_else(|| invalid_data("DiskANN filtered resident size overflows usize"))?; + self.resize_shared_cache_budgets( + self.options + .max_resident_bytes + .saturating_sub(steady_with_order), + )?; + Ok(Some(order)) + } + + pub fn optimize_for_search(&mut self) -> io::Result<()> { + self.ensure_resident()?; + if self.options.adjacency_preload_bytes == 0 || !self.hot_adjacency.is_empty() { + return Ok(()); + } + let adjacency = self.header.sections.adjacency; + let requested_len = self + .options + .adjacency_preload_bytes + .min(adjacency.length as usize); + let requested_len = requested_len + .div_ceil(DISKANN_ADJACENCY_PRELOAD_ALIGNMENT) + .saturating_mul(DISKANN_ADJACENCY_PRELOAD_ALIGNMENT) + .min(adjacency.length as usize); + let row_id_order_bytes = self.loaded_row_id_order_bytes()?; + let available_bytes = self + .options + .max_resident_bytes + .saturating_sub(resident_steady_bytes(&self.header)?) + .saturating_sub(row_id_order_bytes); + let mut preload_len = requested_len.min(available_bytes); + if preload_len < adjacency.length as usize { + preload_len = preload_len / DISKANN_ADJACENCY_PRELOAD_ALIGNMENT + * DISKANN_ADJACENCY_PRELOAD_ALIGNMENT; + } + if preload_len == 0 { + self.resize_shared_cache_budgets(available_bytes)?; + return Ok(()); + } + self.resize_shared_cache_budgets(available_bytes.saturating_sub(preload_len))?; + let mut payload = vec![0u8; preload_len]; + self.reader + .pread(&mut [ReadRequest::new(adjacency.offset, &mut payload)]) + .map_err(|error| map_read_error(error, "adjacency preload"))?; + let payload: Arc<[u8]> = Arc::from(payload); + let resident = self + .resident + .as_ref() + .expect("resident sections were loaded before adjacency preload"); + payload + .par_chunks_exact(DISKANN_PAGE_SIZE as usize) + .enumerate() + .try_for_each(|(page_index, page)| { + resident + .adjacency_validation + .get_or_validate(page_index, || { + validate_adjacency_page_payload( + &self.header, + &resident.adjacency_index, + page_index, + page, + ) + }) + })?; + self.hot_adjacency = payload; + Ok(()) + } + + #[cfg(test)] + pub(crate) fn effective_read_tier(&self) -> DeploymentProfile { + self.effective_read_tier + } + + #[cfg(test)] + pub(crate) fn random_read_latency(&self) -> Duration { + self.random_read_latency + } + + pub(crate) const fn options(&self) -> ResolvedVectorIndexReaderOptions { + self.options + } + + pub fn read_capabilities(&self) -> SeekReadCapabilities { + self.read_capabilities + } + + pub fn vector_read_plan(&self) -> VectorIndexReadPlan { + let plan = self.read_plan(); + let (adjacency_cache_bytes, raw_vector_cache_bytes) = + self.resident.as_ref().map_or((0, 0), |resident| { + ( + resident.adjacency_cache.total_capacity(), + resident.raw_vector_cache.total_capacity(), + ) + }); + VectorIndexReadPlan { + random_read_latency_nanos: u64::try_from(self.random_read_latency.as_nanos()) + .unwrap_or(u64::MAX), + window_bytes: plan.window_bytes, + max_ranges_per_read: self.read_capabilities.max_ranges_per_pread, + graph_beam_width: plan.graph_beam_width, + filtered_graph_beam_width: plan.filtered_graph_beam_width, + adjacency_preload_bytes: self.hot_adjacency.len(), + adjacency_cache_bytes, + raw_vector_cache_bytes, + memory_budget_bytes: self.options.max_resident_bytes, + } + } + + pub(crate) fn read_plan(&self) -> ReadPlan { + self.options() + .read_tier + .read_plan() + .with_capabilities(self.read_capabilities) + } + + pub(crate) fn limit_raw_vector_cache_bytes(&mut self, limit: usize) { + self.options.raw_vector_cache_bytes = self.options.raw_vector_cache_bytes.min(limit); + } + + pub fn last_search_stats(&self) -> DiskAnnSearchStats { + self.last_search_stats + } + + pub(crate) fn pread_ranges(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { + let max_ranges = self.read_capabilities.max_ranges_per_pread; + if max_ranges == 0 || ranges.len() <= max_ranges { + return self + .reader + .pread(ranges) + .map_err(|error| map_read_error(error, "paged section")); + } + for chunk in ranges.chunks_mut(max_ranges) { + self.reader + .pread(chunk) + .map_err(|error| map_read_error(error, "paged section"))?; + } + Ok(()) + } + + pub(crate) fn hot_adjacency_window(&self, offset: u64, length: usize) -> Option<&[u8]> { + let relative = + usize::try_from(offset.checked_sub(self.header.sections.adjacency.offset)?).ok()?; + let end = relative.checked_add(length)?; + self.hot_adjacency.get(relative..end) + } + + pub(crate) fn adjacency_fully_preloaded(&self) -> bool { + u64::try_from(self.hot_adjacency.len()).ok() == Some(self.header.sections.adjacency.length) + } + + pub(crate) fn try_clone_for_search(&mut self) -> io::Result> { + self.optimize_for_search()?; + self.try_clone_with_shared_state() + } + + pub(crate) fn try_clone_for_filtered_search(&mut self) -> io::Result> { + self.ensure_resident()?; + self.try_clone_with_shared_state() + } + + fn try_clone_with_shared_state(&self) -> io::Result> { + let Some(reader) = self.reader.try_clone_reader()? else { + return Ok(None); + }; + Ok(Some(Self { + reader, + header: self.header.clone(), + resident: self.resident.clone(), + options: self.options, + read_capabilities: self.read_capabilities, + effective_read_tier: self.effective_read_tier, + random_read_latency: self.random_read_latency, + hot_adjacency: self.hot_adjacency.clone(), + row_id_order: self.row_id_order.clone(), + query_scratch: Box::default(), + last_search_stats: DiskAnnSearchStats::default(), + batch_workers: Vec::new(), + calibrated_l_search: self.calibrated_l_search, + })) + } + + pub(crate) fn refresh_shared_state_from(&mut self, source: &Self) { + self.header = source.header.clone(); + self.resident = source.resident.clone(); + self.options = source.options; + self.read_capabilities = source.read_capabilities; + self.effective_read_tier = source.effective_read_tier; + self.random_read_latency = source.random_read_latency; + self.calibrated_l_search = source.calibrated_l_search; + self.hot_adjacency = source.hot_adjacency.clone(); + self.row_id_order = source.row_id_order.clone(); + } + + pub(crate) fn validate_adjacency_page(&self, page_index: usize, page: &[u8]) -> io::Result<()> { + let resident = self.resident()?; + resident + .adjacency_validation + .get_or_validate(page_index, || { + validate_adjacency_page_payload( + &self.header, + &resident.adjacency_index, + page_index, + page, + ) + }) + } + + fn resident(&self) -> io::Result<&DiskAnnResidentData> { + self.resident + .as_deref() + .ok_or_else(|| invalid_data("DiskANN resident sections are not loaded")) + } +} + +fn validate_adjacency_page_payload( + header: &DiskAnnHeader, + locators: &AdjacencyIndex, + page_index: usize, + page: &[u8], +) -> io::Result<()> { + if page.len() != DISKANN_PAGE_SIZE as usize { + return Err(invalid_data("DiskANN adjacency page is truncated")); + } + let first = locators.partition_point(|locator| (locator.page_index as usize) < page_index); + let end = locators.partition_point(|locator| (locator.page_index as usize) <= page_index); + if first == end { + return Err(invalid_data("DiskANN adjacency page has no indexed nodes")); + } + let vector_count = header.vector_count as usize; + let record_prefix = if header.is_interleaved() { + header.vector_record_size as usize + } else { + 0 + }; + let mut used_end = 0usize; + let mut neighbors = Vec::new(); + for node in 0..end - first { + let source = first + node; + let locator = locators + .locator(source) + .ok_or_else(|| invalid_data("DiskANN adjacency index is truncated"))?; + let start = locator.byte_offset as usize; + let record_start = start + .checked_sub(record_prefix) + .ok_or_else(|| invalid_data("DiskANN interleaved vector offset underflows"))?; + if record_start != used_end { + return Err(invalid_data( + "DiskANN adjacency page records must be contiguous", + )); + } + if header.is_interleaved() { + let vector = page + .get(record_start..start) + .ok_or_else(|| invalid_data("DiskANN interleaved raw vector is truncated"))?; + validate_raw_vector_bytes(vector, header.raw_vector_encoding())?; + } + let bytes = page + .get(start..) + .ok_or_else(|| invalid_data("DiskANN adjacency list is truncated"))?; + let consumed = + decode_adjacency_list(bytes, locator.degree(), locator.encoding(), &mut neighbors)?; + let list_end = start + .checked_add(consumed) + .ok_or_else(|| invalid_data("DiskANN adjacency locator range overflows"))?; + if let Some(next) = locators + .locator(source + 1) + .filter(|next| next.page_index == locator.page_index) + { + if list_end.checked_add(record_prefix) != Some(next.byte_offset as usize) { + return Err(invalid_data( + "DiskANN adjacency locator ranges must be contiguous", + )); + } + } + let mut previous = None; + for &neighbor in &neighbors { + if neighbor as usize >= vector_count { + return Err(invalid_data("DiskANN adjacency neighbor is out of range")); + } + if neighbor as usize == source { + return Err(invalid_data("DiskANN adjacency contains a self edge")); + } + if previous.is_some_and(|previous| neighbor <= previous) { + return Err(invalid_data( + "DiskANN adjacency neighbors must be strictly increasing", + )); + } + previous = Some(neighbor); + } + let raw_len = neighbors + .len() + .checked_mul(size_of::()) + .ok_or_else(|| invalid_data("DiskANN adjacency list size overflows usize"))?; + let delta_len = adjacency_delta_varint_len(&neighbors) + .expect("validated DiskANN adjacency neighbors are strictly increasing"); + let minimal_encoding = if neighbors.is_empty() || delta_len < raw_len { + AdjacencyListEncoding::DeltaVarint + } else { + AdjacencyListEncoding::RawU32 + }; + if locator.encoding() != minimal_encoding { + return Err(invalid_data( + "DiskANN adjacency list does not use its minimal encoding", + )); + } + used_end = list_end; + } + if page[used_end..].iter().any(|&byte| byte != 0) { + return Err(invalid_data("DiskANN adjacency page tail must be zero")); + } + Ok(()) +} + +fn validate_raw_vector_bytes(vector: &[u8], encoding: DiskAnnRawVectorEncoding) -> io::Result<()> { + let has_non_finite = match encoding { + DiskAnnRawVectorEncoding::F32 => vector.chunks_exact(size_of::()).any(|value| { + !f32::from_le_bytes(value.try_into().expect("four-byte vector component")).is_finite() + }), + DiskAnnRawVectorEncoding::F16 => vector.chunks_exact(size_of::()).any(|value| { + !half::f16::from_bits(u16::from_le_bytes( + value.try_into().expect("two-byte vector component"), + )) + .is_finite() + }), + }; + if !vector.len().is_multiple_of(encoding.element_size()) || has_non_finite { + return Err(invalid_data( + "DiskANN interleaved raw vector contains a non-finite or malformed component", + )); + } + Ok(()) +} + +impl DiskAnnHeader { + pub fn for_layout( + dimension: usize, + vector_count: usize, + entry_node: u32, + pq_m: usize, + build: DiskAnnBuildParams, + ) -> io::Result { + Self::for_layout_with_pq_bits(dimension, vector_count, entry_node, pq_m, 8, build) + } + + pub fn for_layout_with_pq_bits( + dimension: usize, + vector_count: usize, + entry_node: u32, + pq_m: usize, + pq_bits: usize, + build: DiskAnnBuildParams, + ) -> io::Result { + let row_ids_len = raw_row_id_section_len(vector_count)?; + Self::for_layout_with_adjacency_pages_and_pq_bits( + dimension, + vector_count, + entry_node, + pq_m, + pq_bits, + MetricType::L2, + build, + row_ids_len, + 1, + ) + } + + #[cfg(test)] + fn for_layout_with_adjacency_pages( + dimension: usize, + vector_count: usize, + entry_node: u32, + pq_m: usize, + build: DiskAnnBuildParams, + row_ids_len: usize, + adjacency_pages: usize, + ) -> io::Result { + Self::for_layout_with_adjacency_pages_and_pq_bits( + dimension, + vector_count, + entry_node, + pq_m, + 8, + MetricType::L2, + build, + row_ids_len, + adjacency_pages, + ) + } + + #[allow(clippy::too_many_arguments)] + fn for_layout_with_adjacency_pages_and_pq_bits( + dimension: usize, + vector_count: usize, + entry_node: u32, + pq_m: usize, + pq_bits: usize, + metric: MetricType, + build: DiskAnnBuildParams, + row_ids_len: usize, + adjacency_pages: usize, + ) -> io::Result { + validate_diskann_format_configuration(dimension, pq_m, pq_bits, build)?; + if vector_count == 0 || u32::try_from(vector_count).is_err() { + return Err(invalid_input( + "DiskANN vector count must be between 1 and u32::MAX", + )); + } + if entry_node as usize >= vector_count { + return Err(invalid_input("DiskANN entry node is out of range")); + } + if adjacency_pages == 0 || u32::try_from(adjacency_pages).is_err() { + return Err(invalid_input( + "DiskANN adjacency page count must be between 1 and u32::MAX", + )); + } + let maximum_row_ids_len = raw_row_id_section_len(vector_count)?; + if !(ROW_ID_SECTION_HEADER_SIZE..=maximum_row_ids_len).contains(&row_ids_len) { + return Err(invalid_input("invalid DiskANN row-ID section length")); + } + let dimension_u32 = + u32::try_from(dimension).map_err(|_| invalid_input("DiskANN dimension exceeds u32"))?; + let pq_m_u32 = + u32::try_from(pq_m).map_err(|_| invalid_input("DiskANN pq.m exceeds u32"))?; + let pq_bits_u32 = + u32::try_from(pq_bits).map_err(|_| invalid_input("DiskANN pq.bits exceeds u32"))?; + let pq_ksub = diskann_pq_ksub(pq_bits_u32)?; + let pq_code_size = diskann_pq_code_size(pq_m, pq_bits_u32)?; + let max_degree = u32::try_from(build.max_degree) + .map_err(|_| invalid_input("DiskANN maximum degree exceeds u32"))?; + if max_degree == 0 || max_degree > 1023 { + return Err(invalid_input( + "DiskANN maximum degree must be between 1 and 1023", + )); + } + let build_search_list_size = u32::try_from(build.build_search_list_size) + .map_err(|_| invalid_input("DiskANN build search-list size exceeds u32"))?; + let raw_vector_encoding = build.raw_vector_encoding as u32; + let vector_record_size = dimension_u32 + .checked_mul(build.raw_vector_encoding.element_size() as u32) + .ok_or_else(|| invalid_input("DiskANN vector record size overflows u32"))?; + if vector_record_size == 0 || vector_record_size > DISKANN_PAGE_SIZE { + return Err(invalid_input( + "DiskANN raw vector does not fit in a logical page", + )); + } + if build.storage_layout == DiskAnnStorageLayout::Interleaved { + let maximum_adjacency_bytes = max_degree + .checked_mul(size_of::() as u32) + .ok_or_else(|| invalid_input("DiskANN adjacency record size overflows u32"))?; + if vector_record_size + .checked_add(maximum_adjacency_bytes) + .is_none_or(|record_size| record_size > DISKANN_PAGE_SIZE) + { + return Err(invalid_input( + "DiskANN interleaved raw vector and maximum adjacency list do not fit in one page", + )); + } + } + + let codebook_len = pq_codebook_serialized_len(dimension, pq_m, pq_ksub)?; + let row_ids_len = row_ids_len as u64; + let pq_codes_len = checked_mul_u64(vector_count as u64, pq_code_size as u64, "PQ codes")?; + let row_id_order_len = checked_mul_u64(vector_count as u64, 4, "row-ID order")?; + let adjacency_index_len = adjacency_index_serialized_len(vector_count)?; + let adjacency_len = checked_mul_u64( + adjacency_pages as u64, + DISKANN_PAGE_SIZE as u64, + "adjacency pages", + )?; + let vectors_len = match build.storage_layout { + DiskAnnStorageLayout::Compact => checked_mul_u64( + vector_count as u64, + vector_record_size as u64, + "raw vector records", + )?, + DiskAnnStorageLayout::Interleaved => 0, + }; + + let codebook = SectionRange::new(DISKANN_PAGE_SIZE as u64, codebook_len); + let row_ids = SectionRange::new(section_end(codebook)?, row_ids_len); + let pq_codes = SectionRange::new(section_end(row_ids)?, pq_codes_len); + let row_id_order = SectionRange::new(section_end(pq_codes)?, row_id_order_len); + let adjacency_index = SectionRange::new(section_end(row_id_order)?, adjacency_index_len); + let adjacency = SectionRange::new( + align_up(section_end(adjacency_index)?, DISKANN_PAGE_SIZE as u64)?, + adjacency_len, + ); + let vectors = SectionRange::new(section_end(adjacency)?, vectors_len); + let file_len = section_end(vectors)?; + + Ok(Self { + flags: DISKANN_REQUIRED_FLAGS + | match build.storage_layout { + DiskAnnStorageLayout::Compact => FLAG_SEPARATE_ADJACENCY_AND_VECTORS, + DiskAnnStorageLayout::Interleaved => FLAG_INTERLEAVED_ADJACENCY_AND_VECTORS, + }, + dimension: dimension_u32, + metric: metric as u32, + vector_count: vector_count as u64, + entry_node, + max_degree, + build_search_list_size, + alpha: build.alpha, + seed: build.seed, + pq_m: pq_m_u32, + pq_bits: pq_bits_u32, + page_size: DISKANN_PAGE_SIZE, + adjacency_locator_size: ADJACENCY_LOCATOR_SIZE, + adjacency_locator_encoding: ADJACENCY_LOCATOR_ENCODING, + raw_vector_encoding, + vector_record_size, + file_len, + sections: DiskAnnSections { + codebook, + row_ids, + pq_codes, + row_id_order, + adjacency_index, + adjacency, + vectors, + }, + }) + } + + pub fn encode(&self) -> [u8; DISKANN_HEADER_SIZE] { + let mut bytes = [0u8; DISKANN_HEADER_SIZE]; + put_u32(&mut bytes, 0, DISKANN_MAGIC); + put_u32(&mut bytes, 4, DISKANN_VERSION); + put_u32(&mut bytes, 8, DISKANN_HEADER_SIZE as u32); + put_u32(&mut bytes, 12, self.flags); + put_u32(&mut bytes, 16, self.dimension); + put_u32(&mut bytes, 20, self.metric); + put_u64(&mut bytes, 24, self.vector_count); + put_u32(&mut bytes, 32, self.entry_node); + put_u32(&mut bytes, 36, self.max_degree); + put_u32(&mut bytes, 40, self.build_search_list_size); + put_u32(&mut bytes, 44, self.alpha.to_bits()); + put_u64(&mut bytes, 48, self.seed); + put_u32(&mut bytes, 56, self.pq_m); + put_u32(&mut bytes, 60, self.pq_bits); + put_u32(&mut bytes, 64, self.page_size); + put_u32(&mut bytes, 68, self.adjacency_locator_size); + put_u32(&mut bytes, 72, self.adjacency_locator_encoding); + put_u32(&mut bytes, 76, self.raw_vector_encoding); + put_u32(&mut bytes, 80, self.vector_record_size); + put_u32(&mut bytes, 84, SECTION_COUNT as u32); + put_u64(&mut bytes, 88, self.file_len); + for (index, section) in self.sections.as_array().into_iter().enumerate() { + let offset = 96 + index * 16; + put_u64(&mut bytes, offset, section.offset); + put_u64(&mut bytes, offset + 8, section.length); + } + bytes + } + + pub fn decode(bytes: &[u8]) -> io::Result { + if bytes.len() < DISKANN_HEADER_SIZE { + return Err(invalid_data(format!( + "DiskANN header is truncated: {} bytes", + bytes.len() + ))); + } + if get_u32(bytes, 0) != DISKANN_MAGIC { + return Err(invalid_data("invalid DiskANN magic")); + } + if get_u32(bytes, 4) != DISKANN_VERSION { + return Err(invalid_data("unsupported DiskANN version")); + } + if get_u32(bytes, 8) != DISKANN_HEADER_SIZE as u32 { + return Err(invalid_data("invalid DiskANN header size")); + } + let flags = get_u32(bytes, 12); + if flags & DISKANN_REQUIRED_FLAGS != DISKANN_REQUIRED_FLAGS + || flags & !DISKANN_SUPPORTED_FLAGS != 0 + || (flags & FLAG_SEPARATE_ADJACENCY_AND_VECTORS != 0) + == (flags & FLAG_INTERLEAVED_ADJACENCY_AND_VECTORS != 0) + { + return Err(invalid_data("invalid DiskANN required flags")); + } + if get_u32(bytes, 84) != SECTION_COUNT as u32 { + return Err(invalid_data("invalid DiskANN section count")); + } + if bytes[208..DISKANN_HEADER_SIZE] + .iter() + .any(|&byte| byte != 0) + { + return Err(invalid_data("DiskANN reserved header bytes must be zero")); + } + let mut sections = [SectionRange::new(0, 0); SECTION_COUNT]; + for (index, section) in sections.iter_mut().enumerate() { + let offset = 96 + index * 16; + *section = SectionRange::new(get_u64(bytes, offset), get_u64(bytes, offset + 8)); + } + let header = Self { + flags, + dimension: get_u32(bytes, 16), + metric: get_u32(bytes, 20), + vector_count: get_u64(bytes, 24), + entry_node: get_u32(bytes, 32), + max_degree: get_u32(bytes, 36), + build_search_list_size: get_u32(bytes, 40), + alpha: f32::from_bits(get_u32(bytes, 44)), + seed: get_u64(bytes, 48), + pq_m: get_u32(bytes, 56), + pq_bits: get_u32(bytes, 60), + page_size: get_u32(bytes, 64), + adjacency_locator_size: get_u32(bytes, 68), + adjacency_locator_encoding: get_u32(bytes, 72), + raw_vector_encoding: get_u32(bytes, 76), + vector_record_size: get_u32(bytes, 80), + file_len: get_u64(bytes, 88), + sections: DiskAnnSections::from_array(sections), + }; + if MetricType::from_code(header.metric).is_none() + || !matches!(header.pq_bits, 4 | 8) + || header.page_size != DISKANN_PAGE_SIZE + || DiskAnnRawVectorEncoding::from_code(header.raw_vector_encoding).is_none() + { + return Err(invalid_data("unsupported DiskANN v1 layout")); + } + if !header.alpha.is_finite() { + return Err(invalid_data("DiskANN alpha must be finite")); + } + header.validate_layout()?; + Ok(header) + } + + fn validate_layout(&self) -> io::Result<()> { + if self.dimension == 0 || self.dimension > 1024 { + return Err(invalid_data("invalid DiskANN dimension")); + } + if self.vector_count == 0 || self.vector_count > u32::MAX as u64 { + return Err(invalid_data("invalid DiskANN vector count")); + } + if self.entry_node as u64 >= self.vector_count { + return Err(invalid_data("invalid DiskANN entry node")); + } + if self.pq_m == 0 || self.pq_m > self.dimension { + return Err(invalid_data("invalid DiskANN PQ shape")); + } + if self.max_degree == 0 + || self.max_degree > 1023 + || self.build_search_list_size < self.max_degree + { + return Err(invalid_data("invalid DiskANN build parameters")); + } + if self.alpha < 1.0 { + return Err(invalid_data("invalid DiskANN alpha")); + } + let vector_count = usize::try_from(self.vector_count) + .map_err(|_| invalid_data("DiskANN vector count exceeds usize"))?; + let adjacency_pages = + usize::try_from(self.sections.adjacency.length / DISKANN_PAGE_SIZE as u64) + .map_err(|_| invalid_data("DiskANN adjacency page count exceeds usize"))?; + if adjacency_pages == 0 + || !self + .sections + .adjacency + .length + .is_multiple_of(DISKANN_PAGE_SIZE as u64) + { + return Err(invalid_data("invalid DiskANN adjacency section length")); + } + let expected = Self::for_layout_with_adjacency_pages_and_pq_bits( + self.dimension as usize, + vector_count, + self.entry_node, + self.pq_m as usize, + self.pq_bits as usize, + self.metric_type(), + DiskAnnBuildParams { + max_degree: self.max_degree as usize, + build_search_list_size: self.build_search_list_size as usize, + alpha: self.alpha, + seed: self.seed, + memory_budget_bytes: 1, + storage_layout: self.storage_layout(), + raw_vector_encoding: self.raw_vector_encoding(), + ..DiskAnnBuildParams::default() + }, + usize::try_from(self.sections.row_ids.length) + .map_err(|_| invalid_data("DiskANN row-ID section exceeds usize"))?, + adjacency_pages, + ) + .map_err(|error| invalid_data(format!("invalid DiskANN layout: {}", error)))?; + if self.adjacency_locator_size != expected.adjacency_locator_size + || self.adjacency_locator_encoding != expected.adjacency_locator_encoding + || self.raw_vector_encoding != expected.raw_vector_encoding + || self.vector_record_size != expected.vector_record_size + || self.sections != expected.sections + || self.file_len != expected.file_len + { + return Err(invalid_data("invalid DiskANN section layout")); + } + Ok(()) + } + + pub(crate) fn storage_layout(&self) -> DiskAnnStorageLayout { + if self.flags & FLAG_INTERLEAVED_ADJACENCY_AND_VECTORS != 0 { + DiskAnnStorageLayout::Interleaved + } else { + DiskAnnStorageLayout::Compact + } + } + + pub(crate) fn is_interleaved(&self) -> bool { + self.storage_layout() == DiskAnnStorageLayout::Interleaved + } + + pub(crate) fn raw_vector_encoding(&self) -> DiskAnnRawVectorEncoding { + DiskAnnRawVectorEncoding::from_code(self.raw_vector_encoding) + .expect("validated DiskANN raw-vector encoding") + } + + pub(crate) fn metric_type(&self) -> MetricType { + MetricType::from_code(self.metric).expect("validated DiskANN metric") + } +} + +pub fn write_diskann_index(index: &DiskAnnIndex, out: &mut dyn SeekWrite) -> io::Result<()> { + write_diskann_index_with_stats(index, out).map(|_| ()) +} + +pub fn write_diskann_index_with_stats( + index: &DiskAnnIndex, + out: &mut dyn SeekWrite, +) -> io::Result { + let total_started = Instant::now(); + if out.pos() != 0 { + return Err(invalid_input("DiskANN output must start at offset zero")); + } + let prepared = index.prepare_build()?; + let mut stats = prepared.stats; + let interleaved_vector_bytes = match index.build_params.storage_layout { + DiskAnnStorageLayout::Compact => 0, + DiskAnnStorageLayout::Interleaved => index + .d + .checked_mul(index.build_params.raw_vector_encoding.element_size()) + .ok_or_else(|| invalid_input("DiskANN raw vector size overflows usize"))?, + }; + let adjacency_layout = plan_adjacency_layout(&prepared, interleaved_vector_bytes)?; + let row_ids = + RowIdStorage::encode_from_fn(index.ids.len(), |new_id| prepared.row_id(index, new_id))?; + let row_ids_len = row_ids.serialized_len()?; + let header = DiskAnnHeader::for_layout_with_adjacency_pages_and_pq_bits( + index.d, + index.ids.len(), + prepared.graph.entry_node, + index.pq.m, + index.pq.nbits, + index.metric, + index.build_params, + row_ids_len, + adjacency_layout.page_count, + )?; + let resident_started = Instant::now(); + out.write_all(&header.encode())?; + pad_to(out, header.sections.codebook.offset)?; + { + let mut resident_writer = ChunkedSectionWriter::new(out); + write_pq_codebook(&index.pq, &mut resident_writer)?; + write_row_id_section(&row_ids, &mut resident_writer)?; + for new_id in 0..index.ids.len() { + resident_writer.write_bytes(prepared.pq_code(index, new_id))?; + } + let mut row_id_order = (0..index.ids.len() as u32).collect::>(); + row_id_order.sort_unstable_by(|&left, &right| { + prepared + .row_id(index, left as usize) + .cmp(&prepared.row_id(index, right as usize)) + .then_with(|| left.cmp(&right)) + }); + for node in row_id_order { + resident_writer.write_bytes(&node.to_le_bytes())?; + } + write_adjacency_index(&adjacency_layout, &mut resident_writer)?; + resident_writer.finish()?; + } + stats.resident_serialization = resident_started.elapsed(); + let adjacency_started = Instant::now(); + pad_to(out, header.sections.adjacency.offset)?; + { + let mut adjacency_writer = ChunkedSectionWriter::new(out); + write_adjacency_pages(index, &prepared, &adjacency_layout, &mut adjacency_writer)?; + adjacency_writer.finish()?; + } + stats.adjacency_serialization = adjacency_started.elapsed(); + let vector_started = Instant::now(); + if !header.is_interleaved() { + write_vector_records(index, &prepared, &header, out)?; + } + stats.vector_serialization = vector_started.elapsed(); + if out.pos() != header.file_len { + return Err(invalid_input(format!( + "DiskANN writer ended at {}, expected {}", + out.pos(), + header.file_len + ))); + } + stats.total = total_started.elapsed(); + Ok(stats) +} + +struct AdjacencyLayout { + locators: Vec, + page_count: usize, +} + +fn write_adjacency_index( + layout: &AdjacencyLayout, + writer: &mut ChunkedSectionWriter<'_>, +) -> io::Result<()> { + for block_start in (0..layout.locators.len()).step_by(ADJACENCY_LOCATOR_BLOCK_NODES) { + let block_offset = adjacency_locator_absolute_offset(layout.locators[block_start])?; + writer.write_bytes(&block_offset.to_le_bytes())?; + } + for (node, locator) in layout.locators.iter().copied().enumerate() { + let block_start = node / ADJACENCY_LOCATOR_BLOCK_NODES * ADJACENCY_LOCATOR_BLOCK_NODES; + let block_offset = adjacency_locator_absolute_offset(layout.locators[block_start])?; + let relative_offset = adjacency_locator_absolute_offset(locator)? + .checked_sub(block_offset) + .and_then(|offset| u16::try_from(offset).ok()) + .ok_or_else(|| { + invalid_input("DiskANN adjacency locator exceeds its block offset range") + })?; + writer.write_bytes(&relative_offset.to_le_bytes())?; + } + for locator in &layout.locators { + writer.write_bytes(&locator.degree_and_flags.to_le_bytes())?; + } + Ok(()) +} + +fn plan_adjacency_layout( + prepared: &PreparedDiskAnn, + interleaved_vector_bytes: usize, +) -> io::Result { + let mut locators = Vec::new(); + locators + .try_reserve_exact(prepared.graph.adjacency.len()) + .map_err(|_| invalid_input("DiskANN adjacency locator allocation failed"))?; + let mut page_index = 0u32; + let mut byte_offset = 0usize; + for neighbors in prepared.graph.adjacency.iter() { + let (encoding, list_bytes) = plan_adjacency_list(neighbors)?; + let record_bytes = interleaved_vector_bytes + .checked_add(list_bytes) + .ok_or_else(|| invalid_input("DiskANN interleaved record size overflows usize"))?; + if record_bytes > DISKANN_PAGE_SIZE as usize { + return Err(invalid_input( + "DiskANN vector and adjacency list do not fit in a logical page", + )); + } + if byte_offset + record_bytes > DISKANN_PAGE_SIZE as usize { + page_index = page_index + .checked_add(1) + .ok_or_else(|| invalid_input("DiskANN adjacency page count exceeds u32"))?; + byte_offset = 0; + } + locators.push(AdjacencyLocator::new( + page_index, + u16::try_from(byte_offset + interleaved_vector_bytes) + .map_err(|_| invalid_input("DiskANN adjacency byte offset exceeds u16"))?, + neighbors.len(), + encoding, + )?); + byte_offset += record_bytes; + } + Ok(AdjacencyLayout { + locators, + page_count: page_index as usize + 1, + }) +} + +fn write_adjacency_pages( + index: &DiskAnnIndex, + prepared: &PreparedDiskAnn, + layout: &AdjacencyLayout, + writer: &mut ChunkedSectionWriter<'_>, +) -> io::Result<()> { + let mut page = vec![0u8; DISKANN_PAGE_SIZE as usize]; + let mut current_page = 0u32; + let mut encoded = Vec::new(); + let mut encoded_vector = Vec::new(); + for (node, locator) in layout.locators.iter().enumerate() { + while locator.page_index > current_page { + writer.write_bytes(&page)?; + page.fill(0); + current_page += 1; + } + let encoding = encode_adjacency_list(&prepared.graph.adjacency[node], &mut encoded)?; + if encoding != locator.encoding() { + return Err(invalid_input( + "DiskANN adjacency encoding changed after layout planning", + )); + } + let start = locator.byte_offset as usize; + if index.build_params.storage_layout == DiskAnnStorageLayout::Interleaved { + encode_raw_vector( + prepared.vector(index, node), + index.build_params.raw_vector_encoding, + &mut encoded_vector, + )?; + let vector_start = start + .checked_sub(encoded_vector.len()) + .ok_or_else(|| invalid_input("DiskANN interleaved vector offset underflows"))?; + page[vector_start..start].copy_from_slice(&encoded_vector); + } + let end = start + .checked_add(encoded.len()) + .ok_or_else(|| invalid_input("DiskANN adjacency page range overflows"))?; + page.get_mut(start..end) + .ok_or_else(|| invalid_input("DiskANN adjacency list crosses a logical page"))? + .copy_from_slice(&encoded); + } + writer.write_bytes(&page)?; + Ok(()) +} + +fn write_vector_records( + index: &DiskAnnIndex, + prepared: &PreparedDiskAnn, + header: &DiskAnnHeader, + out: &mut dyn SeekWrite, +) -> io::Result<()> { + let mut writer = ChunkedSectionWriter::new(out); + let mut encoded = Vec::new(); + for new_id in 0..index.ids.len() { + encode_raw_vector( + prepared.vector(index, new_id), + header.raw_vector_encoding(), + &mut encoded, + )?; + writer.write_bytes(&encoded)?; + } + writer.finish() +} + +fn encode_raw_vector( + vector: &[f32], + encoding: DiskAnnRawVectorEncoding, + encoded: &mut Vec, +) -> io::Result<()> { + encoded.clear(); + let encoded_len = vector + .len() + .checked_mul(encoding.element_size()) + .ok_or_else(|| invalid_input("DiskANN raw-vector encoding size overflows usize"))?; + encoded + .try_reserve_exact(encoded_len) + .map_err(|_| invalid_input("DiskANN raw-vector encoding allocation failed"))?; + match encoding { + DiskAnnRawVectorEncoding::F32 => { + for &value in vector { + encoded.extend_from_slice(&value.to_bits().to_le_bytes()); + } + } + DiskAnnRawVectorEncoding::F16 => { + for &value in vector { + let value = half::f16::from_f32(value); + if !value.is_finite() { + return Err(invalid_input( + "DiskANN vector value is outside the finite f16 range", + )); + } + encoded.extend_from_slice(&value.to_bits().to_le_bytes()); + } + } + } + Ok(()) +} + +struct ChunkedSectionWriter<'a> { + out: &'a mut dyn SeekWrite, + buffer: Vec, +} + +impl<'a> ChunkedSectionWriter<'a> { + fn new(out: &'a mut dyn SeekWrite) -> Self { + Self { + out, + buffer: Vec::with_capacity(DISKANN_WRITE_BUFFER_SIZE), + } + } + + fn write_bytes(&mut self, mut bytes: &[u8]) -> io::Result<()> { + while !bytes.is_empty() { + if self.buffer.len() == DISKANN_WRITE_BUFFER_SIZE { + self.flush()?; + } + let available = DISKANN_WRITE_BUFFER_SIZE - self.buffer.len(); + let chunk = available.min(bytes.len()); + self.buffer.extend_from_slice(&bytes[..chunk]); + bytes = &bytes[chunk..]; + } + Ok(()) + } + + fn flush(&mut self) -> io::Result<()> { + if !self.buffer.is_empty() { + self.out.write_all(&self.buffer)?; + self.buffer.clear(); + } + Ok(()) + } + + fn finish(mut self) -> io::Result<()> { + self.flush() + } +} + +fn write_pq_codebook( + pq: &ProductQuantizer, + writer: &mut ChunkedSectionWriter<'_>, +) -> io::Result<()> { + if !pq.has_valid_layout() { + return Err(invalid_input("DiskANN PQ codebook layout is invalid")); + } + let mut header = [0u8; PQ_CODEBOOK_HEADER_SIZE]; + put_u32(&mut header, 0, PQ_CODEBOOK_MAGIC); + put_u32(&mut header, 4, PQ_CODEBOOK_VERSION); + put_u32( + &mut header, + 8, + u32::try_from(pq.d).map_err(|_| invalid_input("DiskANN PQ dimension exceeds u32"))?, + ); + put_u32( + &mut header, + 12, + u32::try_from(pq.m).map_err(|_| invalid_input("DiskANN PQ m exceeds u32"))?, + ); + put_u32( + &mut header, + 16, + u32::try_from(pq.nbits).map_err(|_| invalid_input("DiskANN PQ bits exceeds u32"))?, + ); + put_u32( + &mut header, + 20, + u32::try_from(pq.ksub).map_err(|_| invalid_input("DiskANN PQ ksub exceeds u32"))?, + ); + put_u32( + &mut header, + 24, + u32::try_from(pq.chunk_offsets.len()) + .map_err(|_| invalid_input("DiskANN PQ chunk-offset count exceeds u32"))?, + ); + writer.write_bytes(&header)?; + for &offset in &pq.chunk_offsets { + writer.write_bytes( + &u32::try_from(offset) + .map_err(|_| invalid_input("DiskANN PQ chunk offset exceeds u32"))? + .to_le_bytes(), + )?; + } + for &value in &pq.centroids { + writer.write_bytes(&value.to_le_bytes())?; + } + Ok(()) +} + +fn write_row_id_section( + storage: &RowIdStorage, + writer: &mut ChunkedSectionWriter<'_>, +) -> io::Result<()> { + writer.write_bytes(&storage.section_header())?; + match storage { + RowIdStorage::Raw(row_ids) => { + for row_id in row_ids { + writer.write_bytes(&row_id.to_le_bytes())?; + } + } + RowIdStorage::ForBitPacked { payload, .. } => writer.write_bytes(payload)?, + } + Ok(()) +} + +fn pad_to(out: &mut dyn SeekWrite, target: u64) -> io::Result<()> { + let padding = target + .checked_sub(out.pos()) + .ok_or_else(|| invalid_input("DiskANN writer passed a section offset"))?; + let mut remaining = + usize::try_from(padding).map_err(|_| invalid_input("DiskANN padding exceeds usize"))?; + const ZEROES: [u8; 4096] = [0; 4096]; + while remaining != 0 { + let chunk = remaining.min(ZEROES.len()); + out.write_all(&ZEROES[..chunk])?; + remaining -= chunk; + } + Ok(()) +} + +fn resident_peak_bytes(header: &DiskAnnHeader) -> io::Result { + let serialized_codebook = section_len_usize(header.sections.codebook, "PQ codebook")?; + let decoded_codebook = decoded_pq_codebook_bytes(header)?; + let serialized_row_ids = section_len_usize(header.sections.row_ids, "row IDs")?; + let decoded_row_ids = resident_row_id_bytes(header)?; + let pq_codes = section_len_usize(header.sections.pq_codes, "PQ codes")?; + let serialized_adjacency_index = + section_len_usize(header.sections.adjacency_index, "adjacency index")?; + let adjacency_validation = adjacency_validation_bytes(header)?; + let pq_norms = (header.pq_m as usize) + .checked_mul(diskann_pq_ksub(header.pq_bits)?) + .and_then(|count| count.checked_mul(size_of::())) + .ok_or_else(|| invalid_data("DiskANN PQ norms size overflows usize"))?; + let serialized_payloads = checked_resident_sum([ + serialized_codebook, + serialized_row_ids, + pq_codes, + serialized_adjacency_index, + 1, + ])?; + let codebook_decode = serialized_payloads + .checked_add(decoded_codebook) + .ok_or_else(|| invalid_data("DiskANN resident peak size overflows usize"))?; + // `Vec::split_off` temporarily retains both the serialized row-ID + // allocation and its payload while the raw/packed representation is + // materialized. + let row_id_decode = checked_resident_sum([ + serialized_row_ids, + pq_codes, + serialized_adjacency_index, + decoded_codebook, + decoded_row_ids, + 1, + ])?; + let adjacency_decode = checked_resident_sum([ + pq_codes, + serialized_adjacency_index, + decoded_codebook, + decoded_row_ids, + serialized_adjacency_index, + 1, + ])?; + let steady = checked_resident_sum([ + decoded_codebook, + decoded_row_ids, + pq_codes, + pq_norms, + serialized_adjacency_index, + adjacency_validation, + ])?; + Ok(codebook_decode + .max(row_id_decode) + .max(adjacency_decode) + .max(steady)) +} + +fn resident_steady_bytes(header: &DiskAnnHeader) -> io::Result { + let pq_norms = (header.pq_m as usize) + .checked_mul(diskann_pq_ksub(header.pq_bits)?) + .and_then(|count| count.checked_mul(size_of::())) + .ok_or_else(|| invalid_data("DiskANN PQ norms size overflows usize"))?; + [ + decoded_pq_codebook_bytes(header)?, + resident_row_id_bytes(header)?, + section_len_usize(header.sections.pq_codes, "PQ codes")?, + section_len_usize(header.sections.adjacency_index, "adjacency index")?, + adjacency_validation_bytes(header)?, + pq_norms, + ] + .into_iter() + .try_fold(0usize, |total, bytes| { + total + .checked_add(bytes) + .ok_or_else(|| invalid_data("DiskANN resident size overflows usize")) + }) +} + +fn checked_resident_sum(parts: [usize; N]) -> io::Result { + parts.into_iter().try_fold(0usize, |total, bytes| { + total + .checked_add(bytes) + .ok_or_else(|| invalid_data("DiskANN resident size overflows usize")) + }) +} + +fn decoded_pq_codebook_bytes(header: &DiskAnnHeader) -> io::Result { + let offsets = (header.pq_m as usize) + .checked_add(1) + .and_then(|count| count.checked_mul(size_of::())) + .ok_or_else(|| invalid_data("DiskANN decoded PQ offset size overflows usize"))?; + let centroids = (header.dimension as usize) + .checked_mul(diskann_pq_ksub(header.pq_bits)?) + .and_then(|count| count.checked_mul(size_of::())) + .ok_or_else(|| invalid_data("DiskANN decoded PQ centroid size overflows usize"))?; + offsets + .checked_add(centroids) + .ok_or_else(|| invalid_data("DiskANN decoded PQ codebook size overflows usize")) +} + +fn adjacency_page_count(header: &DiskAnnHeader) -> io::Result { + let adjacency_bytes = section_len_usize(header.sections.adjacency, "adjacency")?; + if !adjacency_bytes.is_multiple_of(DISKANN_PAGE_SIZE as usize) { + return Err(invalid_data( + "DiskANN adjacency section is not page-aligned", + )); + } + Ok(adjacency_bytes / DISKANN_PAGE_SIZE as usize) +} + +fn adjacency_validation_bytes(header: &DiskAnnHeader) -> io::Result { + adjacency_page_count(header)? + .checked_mul(size_of::()) + .ok_or_else(|| invalid_data("DiskANN adjacency validation size overflows usize")) +} + +fn resident_row_id_bytes(header: &DiskAnnHeader) -> io::Result { + section_len_usize(header.sections.row_ids, "row IDs")? + .checked_sub(ROW_ID_SECTION_HEADER_SIZE) + .ok_or_else(|| invalid_data("DiskANN row-ID section header is truncated")) +} + +fn row_id_order_peak_bytes(header: &DiskAnnHeader, hot_adjacency: usize) -> io::Result { + let order = section_len_usize(header.sections.row_id_order, "row-ID order")?; + let scratch = order.min(DISKANN_RESIDENT_DECODE_BUFFER_SIZE); + [ + resident_steady_bytes(header)?, + hot_adjacency, + order, + scratch, + ] + .into_iter() + .try_fold(0usize, |total, bytes| { + total + .checked_add(bytes) + .ok_or_else(|| invalid_data("DiskANN row-ID lookup peak size overflows usize")) + }) +} + +fn validate_row_id_order(row_ids: &RowIdStorage, order: &[u32]) -> io::Result<()> { + if order.len() != row_ids.len() { + return Err(invalid_data( + "DiskANN row-ID order has an invalid node count", + )); + } + let mut previous = None; + for &node in order { + let node_index = node as usize; + let row_id = row_ids + .get(node_index) + .ok_or_else(|| invalid_data("DiskANN row-ID order node is out of range"))?; + let key = (row_id, node); + if previous.is_some_and(|previous| key <= previous) { + return Err(invalid_data( + "DiskANN row-ID order must be globally strictly increasing", + )); + } + previous = Some(key); + } + Ok(()) +} + +fn decode_adjacency_index(bytes: &[u8], header: &DiskAnnHeader) -> io::Result { + let section = header.sections.adjacency_index; + let section_len = section_len_usize(section, "adjacency index")?; + let locator_count = header.vector_count as usize; + let expected_len = usize::try_from(adjacency_index_serialized_len(locator_count)?) + .map_err(|_| invalid_data("DiskANN adjacency index exceeds usize"))?; + if section_len != expected_len { + return Err(invalid_data( + "DiskANN adjacency index has an invalid byte length", + )); + } + let block_count = locator_count.div_ceil(ADJACENCY_LOCATOR_BLOCK_NODES); + let block_bytes = block_count + .checked_mul(size_of::()) + .ok_or_else(|| invalid_data("DiskANN adjacency block-offset size overflows usize"))?; + let relative_bytes = locator_count + .checked_mul(size_of::()) + .ok_or_else(|| invalid_data("DiskANN adjacency relative-offset size overflows usize"))?; + if bytes.len() != section_len { + return Err(invalid_data("DiskANN adjacency index payload is truncated")); + } + let relative_offset = block_bytes; + let metadata_offset = relative_offset + .checked_add(relative_bytes) + .ok_or_else(|| invalid_data("DiskANN adjacency metadata position overflows"))?; + let mut block_offsets = Vec::new(); + block_offsets + .try_reserve_exact(block_count) + .map_err(|_| invalid_data("DiskANN adjacency block-offset allocation failed"))?; + block_offsets.extend(bytes[..relative_offset].chunks_exact(8).map(|value| { + u64::from_le_bytes(value.try_into().expect("validated eight-byte block offset")) + })); + let mut relative_offsets = Vec::new(); + relative_offsets + .try_reserve_exact(locator_count) + .map_err(|_| invalid_data("DiskANN adjacency relative-offset allocation failed"))?; + relative_offsets.extend( + bytes[relative_offset..metadata_offset] + .chunks_exact(2) + .map(|value| { + u16::from_le_bytes( + value + .try_into() + .expect("validated two-byte relative offset"), + ) + }), + ); + let mut degree_and_flags = Vec::new(); + degree_and_flags + .try_reserve_exact(locator_count) + .map_err(|_| invalid_data("DiskANN adjacency metadata allocation failed"))?; + degree_and_flags.extend(bytes[metadata_offset..].chunks_exact(2).map(|value| { + u16::from_le_bytes( + value + .try_into() + .expect("validated two-byte locator metadata"), + ) + })); + let index = AdjacencyIndex { + block_offsets: block_offsets.into_boxed_slice(), + relative_offsets: relative_offsets.into_boxed_slice(), + degree_and_flags: degree_and_flags.into_boxed_slice(), + }; + validate_adjacency_index(header, &index)?; + Ok(index) +} + +fn validate_adjacency_index(header: &DiskAnnHeader, locators: &AdjacencyIndex) -> io::Result<()> { + if locators.len() != header.vector_count as usize + || locators.degree_and_flags.len() != locators.len() + || locators.block_offsets.len() != locators.len().div_ceil(ADJACENCY_LOCATOR_BLOCK_NODES) + { + return Err(invalid_data( + "DiskANN adjacency index has an invalid locator count", + )); + } + let page_count = u32::try_from(header.sections.adjacency.length / DISKANN_PAGE_SIZE as u64) + .map_err(|_| invalid_data("DiskANN adjacency page count exceeds u32"))?; + let mut previous_page = 0u32; + let mut previous_offset = 0usize; + let mut previous_degree = 0usize; + let mut previous_encoding = AdjacencyListEncoding::DeltaVarint; + let record_prefix = if header.is_interleaved() { + header.vector_record_size as usize + } else { + 0 + }; + for node in 0..locators.len() { + if node.is_multiple_of(ADJACENCY_LOCATOR_BLOCK_NODES) + && locators.relative_offsets[node] != 0 + { + return Err(invalid_data( + "DiskANN adjacency locator block must start at relative offset zero", + )); + } + let locator = locators + .locator(node) + .ok_or_else(|| invalid_data("DiskANN adjacency locator offset is invalid"))?; + if locator.page_index >= page_count { + return Err(invalid_data( + "DiskANN adjacency locator page is out of range", + )); + } + if node == 0 && locator.page_index != 0 { + return Err(invalid_data( + "DiskANN adjacency locator must start on page zero", + )); + } + if locator.page_index < previous_page + || locator.page_index > previous_page + u32::from(node != 0) + { + return Err(invalid_data( + "DiskANN adjacency locator pages are not monotonic", + )); + } + if locator.degree() as u32 > header.max_degree { + return Err(invalid_data("DiskANN adjacency degree exceeds maximum")); + } + let offset = locator.byte_offset as usize; + if offset < record_prefix || offset > DISKANN_PAGE_SIZE as usize { + return Err(invalid_data( + "DiskANN adjacency locator offset exceeds its page", + )); + } + if locator.encoding() == AdjacencyListEncoding::RawU32 { + let end = offset + .checked_add(locator.degree() * size_of::()) + .ok_or_else(|| invalid_data("DiskANN adjacency locator range overflows"))?; + if end > DISKANN_PAGE_SIZE as usize { + return Err(invalid_data("DiskANN adjacency locator crosses a page")); + } + } + if node == 0 && offset != record_prefix { + return Err(invalid_data( + "DiskANN adjacency index has an invalid first record offset", + )); + } + if locator.page_index == previous_page { + if node != 0 { + match previous_encoding { + AdjacencyListEncoding::DeltaVarint => { + let minimum_offset = + previous_offset + usize::from(previous_degree != 0) + record_prefix; + if offset < minimum_offset { + return Err(invalid_data("DiskANN adjacency locator ranges overlap")); + } + } + AdjacencyListEncoding::RawU32 => { + let expected_offset = previous_offset + .checked_add(previous_degree * size_of::()) + .and_then(|offset| offset.checked_add(record_prefix)) + .ok_or_else(|| { + invalid_data("DiskANN adjacency locator range overflows") + })?; + if offset != expected_offset { + return Err(invalid_data( + "DiskANN adjacency locator ranges must be contiguous", + )); + } + } + } + } + } else { + if offset != record_prefix { + return Err(invalid_data( + "DiskANN adjacency page has an invalid first record offset", + )); + } + } + previous_page = locator.page_index; + previous_offset = offset; + previous_degree = locator.degree(); + previous_encoding = locator.encoding(); + } + if previous_page.checked_add(1) != Some(page_count) { + return Err(invalid_data( + "DiskANN adjacency pages are not fully indexed", + )); + } + Ok(()) +} + +fn section_len_usize(section: SectionRange, name: &str) -> io::Result { + usize::try_from(section.length) + .map_err(|_| invalid_data(format!("DiskANN {} section exceeds usize", name))) +} + +fn decode_pq_codebook(bytes: &[u8], header: &DiskAnnHeader) -> io::Result { + if bytes.len() < PQ_CODEBOOK_HEADER_SIZE { + return Err(invalid_data("DiskANN PQ codebook header is truncated")); + } + if get_u32(bytes, 0) != PQ_CODEBOOK_MAGIC + || get_u32(bytes, 4) != PQ_CODEBOOK_VERSION + || get_u32(bytes, 8) != header.dimension + || get_u32(bytes, 12) != header.pq_m + || get_u32(bytes, 16) != header.pq_bits + || get_u32(bytes, 20) != diskann_pq_ksub(header.pq_bits)? as u32 + || get_u32(bytes, 24) != header.pq_m + 1 + || get_u32(bytes, 28) != 0 + { + return Err(invalid_data("invalid DiskANN PQ codebook metadata")); + } + let offset_count = header.pq_m as usize + 1; + let offsets_bytes = offset_count + .checked_mul(size_of::()) + .ok_or_else(|| invalid_data("DiskANN PQ chunk-offset size overflows usize"))?; + let centroids_offset = PQ_CODEBOOK_HEADER_SIZE + .checked_add(offsets_bytes) + .ok_or_else(|| invalid_data("DiskANN PQ codebook offset overflows usize"))?; + let expected_len = usize::try_from(pq_codebook_serialized_len( + header.dimension as usize, + header.pq_m as usize, + diskann_pq_ksub(header.pq_bits)?, + )?) + .map_err(|_| invalid_data("DiskANN PQ codebook size exceeds usize"))?; + if bytes.len() != expected_len { + return Err(invalid_data("invalid DiskANN PQ codebook length")); + } + let mut chunk_offsets = Vec::new(); + chunk_offsets + .try_reserve_exact(offset_count) + .map_err(|_| invalid_data("DiskANN PQ chunk-offset allocation failed"))?; + for encoded in bytes[PQ_CODEBOOK_HEADER_SIZE..centroids_offset].chunks_exact(4) { + chunk_offsets.push(u32::from_le_bytes( + encoded.try_into().expect("four-byte PQ chunk offset"), + ) as usize); + } + let mut pq = ProductQuantizer::try_with_chunk_offsets( + header.dimension as usize, + header.pq_bits as usize, + chunk_offsets, + ) + .map_err(invalid_data)?; + let mut centroids = Vec::new(); + centroids + .try_reserve_exact(header.dimension as usize * pq.ksub) + .map_err(|_| invalid_data("DiskANN PQ centroid allocation failed"))?; + for encoded in bytes[centroids_offset..].chunks_exact(4) { + let value = + f32::from_le_bytes(encoded.try_into().expect("validated four-byte PQ centroid")); + if !value.is_finite() { + return Err(invalid_data("DiskANN PQ centroids must be finite")); + } + centroids.push(value); + } + pq.centroids = centroids; + if !pq.has_valid_layout() { + return Err(invalid_data("invalid DiskANN PQ codebook layout")); + } + Ok(pq) +} + +fn decode_row_id_section_owned( + mut bytes: Vec, + expected_count: usize, +) -> io::Result { + let header = decode_row_id_section_header(&bytes, bytes.len(), expected_count)?; + let payload = bytes.split_off(ROW_ID_SECTION_HEADER_SIZE); + drop(bytes); + let storage = match header.encoding { + ROW_ID_ENCODING_RAW_I64 => { + let mut row_ids = Vec::new(); + row_ids + .try_reserve_exact(header.count) + .map_err(|_| invalid_data("DiskANN raw row-ID allocation failed"))?; + row_ids.extend(payload.chunks_exact(8).map(|value| { + i64::from_le_bytes(value.try_into().expect("validated eight-byte row ID")) + })); + RowIdStorage::Raw(row_ids) + } + ROW_ID_ENCODING_FOR_BITPACK => RowIdStorage::ForBitPacked { + base: header.base, + bit_width: header.bit_width, + count: header.count, + payload, + }, + _ => unreachable!("row-ID encoding was validated"), + }; + validate_row_id_storage(&storage)?; + Ok(storage) +} + +fn read_u32_section( + reader: &mut R, + section: SectionRange, + name: &str, +) -> io::Result> { + let section_len = section_len_usize(section, name)?; + if !section_len.is_multiple_of(size_of::()) { + return Err(invalid_data(format!( + "DiskANN {} section is not u32-aligned", + name + ))); + } + let mut values = Vec::new(); + values + .try_reserve_exact(section_len / size_of::()) + .map_err(|_| invalid_data(format!("DiskANN {} allocation failed", name)))?; + let scratch_len = section_len + .min(DISKANN_RESIDENT_DECODE_BUFFER_SIZE) + .max(size_of::()); + let mut scratch = vec![0u8; scratch_len]; + let mut loaded = 0usize; + while loaded < section_len { + let chunk_len = (section_len - loaded).min(scratch.len()); + let chunk = &mut scratch[..chunk_len]; + reader + .pread(&mut [ReadRequest::new(section.offset + loaded as u64, chunk)]) + .map_err(|error| map_read_error(error, name))?; + values.extend( + chunk.chunks_exact(4).map(|bytes| { + u32::from_le_bytes(bytes.try_into().expect("validated four-byte value")) + }), + ); + loaded += chunk_len; + } + Ok(values) +} + +fn read_resident_sections( + reader: &mut R, + header: &DiskAnnHeader, +) -> io::Result<(ProductQuantizer, RowIdStorage, Vec, AdjacencyIndex)> { + let sections = [ + (header.sections.codebook, "PQ codebook"), + (header.sections.row_ids, "row IDs"), + (header.sections.pq_codes, "PQ codes"), + (header.sections.adjacency_index, "adjacency index"), + ( + SectionRange::new(header.file_len - 1, 1), + "file length probe", + ), + ]; + let mut payloads = Vec::with_capacity(sections.len()); + for (section, name) in sections { + let len = section_len_usize(section, name)?; + let mut payload = Vec::new(); + payload + .try_reserve_exact(len) + .map_err(|_| invalid_data(format!("DiskANN {name} allocation failed")))?; + payload.resize(len, 0); + payloads.push(payload); + } + let max_ranges = match reader.read_capabilities().max_ranges_per_pread { + 0 => usize::MAX, + value => value, + }; + let mut start = 0usize; + while start < payloads.len() { + let end = payloads.len().min(start.saturating_add(max_ranges)); + let mut requests = sections[start..end] + .iter() + .zip(&mut payloads[start..end]) + .map(|((section, _), payload)| ReadRequest::new(section.offset, payload)) + .collect::>(); + reader + .pread(&mut requests) + .map_err(|error| map_read_error(error, "resident sections"))?; + start = end; + } + let mut payloads = payloads.into_iter(); + let pq = decode_pq_codebook( + &payloads.next().expect("five resident section payloads"), + header, + )?; + let row_ids = decode_row_id_section_owned( + payloads.next().expect("five resident section payloads"), + header.vector_count as usize, + )?; + let pq_codes = payloads.next().expect("five resident section payloads"); + let adjacency_index = decode_adjacency_index( + &payloads.next().expect("five resident section payloads"), + header, + )?; + let _file_tail = payloads.next().expect("file length probe payload"); + Ok((pq, row_ids, pq_codes, adjacency_index)) +} + +fn put_u32(bytes: &mut [u8], offset: usize, value: u32) { + bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +#[cfg(test)] +fn put_u16(bytes: &mut [u8], offset: usize, value: u16) { + bytes[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +#[cfg(test)] +fn get_u16(bytes: &[u8], offset: usize) -> u16 { + u16::from_le_bytes( + bytes[offset..offset + 2] + .try_into() + .expect("fixed u16 field"), + ) +} + +fn put_u64(bytes: &mut [u8], offset: usize, value: u64) { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn get_u32(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes( + bytes[offset..offset + 4] + .try_into() + .expect("fixed u32 field"), + ) +} + +fn get_u64(bytes: &[u8], offset: usize) -> u64 { + u64::from_le_bytes( + bytes[offset..offset + 8] + .try_into() + .expect("fixed u64 field"), + ) +} + +fn invalid_data(message: impl Into) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message.into()) +} + +fn invalid_input(message: impl Into) -> io::Error { + io::Error::new(io::ErrorKind::InvalidInput, message.into()) +} + +fn map_read_error(error: io::Error, section: &str) -> io::Error { + if error.kind() == io::ErrorKind::UnexpectedEof { + invalid_data(format!("DiskANN {} is truncated", section)) + } else { + error + } +} + +fn checked_mul_u64(left: u64, right: u64, name: &str) -> io::Result { + left.checked_mul(right) + .ok_or_else(|| invalid_input(format!("DiskANN {} length overflows u64", name))) +} + +fn diskann_pq_ksub(pq_bits: u32) -> io::Result { + match pq_bits { + 4 => Ok(16), + 8 => Ok(256), + _ => Err(invalid_input("DiskANN pq.bits must be 4 or 8")), + } +} + +fn diskann_pq_code_size(pq_m: usize, pq_bits: u32) -> io::Result { + match pq_bits { + 4 => Ok(pq_m.div_ceil(2)), + 8 => Ok(pq_m), + _ => Err(invalid_input("DiskANN pq.bits must be 4 or 8")), + } +} + +fn pq_codebook_serialized_len(dimension: usize, pq_m: usize, ksub: usize) -> io::Result { + let offsets = pq_m + .checked_add(1) + .and_then(|count| count.checked_mul(size_of::())) + .ok_or_else(|| invalid_input("DiskANN PQ chunk-offset size overflows usize"))?; + let centroids = dimension + .checked_mul(ksub) + .and_then(|count| count.checked_mul(size_of::())) + .ok_or_else(|| invalid_input("DiskANN PQ centroid size overflows usize"))?; + u64::try_from( + PQ_CODEBOOK_HEADER_SIZE + .checked_add(offsets) + .and_then(|bytes| bytes.checked_add(centroids)) + .ok_or_else(|| invalid_input("DiskANN PQ codebook size overflows usize"))?, + ) + .map_err(|_| invalid_input("DiskANN PQ codebook size exceeds u64")) +} + +fn validate_pq_code_padding(header: &DiskAnnHeader, codes: &[u8]) -> io::Result<()> { + if header.pq_bits != 4 || header.pq_m.is_multiple_of(2) { + return Ok(()); + } + let code_size = diskann_pq_code_size(header.pq_m as usize, header.pq_bits)?; + if codes + .chunks_exact(code_size) + .any(|code| code.last().is_some_and(|byte| byte & 0xf0 != 0)) + { + return Err(invalid_data( + "DiskANN odd 4-bit PQ codes must zero the unused high nibble", + )); + } + Ok(()) +} + +fn section_end(section: SectionRange) -> io::Result { + section + .offset + .checked_add(section.length) + .ok_or_else(|| invalid_input("DiskANN section end overflows u64")) +} + +fn align_up(value: u64, alignment: u64) -> io::Result { + let mask = alignment - 1; + value + .checked_add(mask) + .map(|value| value & !mask) + .ok_or_else(|| invalid_input("DiskANN section alignment overflows u64")) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::diskann::{DiskAnnBuildParams, DiskAnnIndex, DiskAnnStorageLayout}; + use crate::distance::MetricType; + use crate::io::{PosWriter, ReadRequest, SeekRead}; + use crate::read_options::DeploymentProfile; + use std::io::Cursor; + use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; + use std::sync::mpsc; + use std::sync::{Arc, Mutex}; + use std::thread; + + #[test] + fn diskann_shared_adjacency_cache_reports_lock_acquisitions() { + let cache = SharedWindowCache::new(4096); + + let (lookup, metrics) = cache.lookup_or_reserve(7, 4).unwrap(); + + assert!(matches!(lookup, SharedWindowCacheLookup::Reserved)); + assert_eq!(metrics.acquisitions, 1); + cache.cancel(&[7]).unwrap(); + } + + #[test] + fn diskann_shared_adjacency_cache_shards_production_sized_budgets() { + let cache = SharedWindowCache::new(16 * 1024 * 1024); + + assert_eq!(cache.shard_count(), 16); + } + + #[test] + fn diskann_shared_window_cache_hashes_coalesced_offsets_across_all_shards() { + let cache = SharedWindowCache::new(16 * 1024 * 1024); + let shard_indexes = (0..80) + .map(|window| cache.shard_index(window * 64 * 1024)) + .collect::>(); + + assert_eq!(shard_indexes.len(), SHARED_WINDOW_CACHE_SHARDS); + } + + #[test] + fn diskann_shared_adjacency_cache_keeps_one_full_coalesced_window_per_shard() { + let cache = SharedWindowCache::new(64 * 1024); + + assert_eq!(cache.shard_count(), 1); + } + + #[test] + fn diskann_shared_adjacency_cache_is_bounded_and_lru() { + let cache = SharedWindowCache::new(8); + assert!(matches!( + cache.lookup_or_reserve(0, 4).unwrap().0, + SharedWindowCacheLookup::Reserved + )); + assert_eq!(cache.publish(0, Arc::new(vec![0u8; 4])).unwrap().0, 0); + assert!(matches!( + cache.lookup_or_reserve(4, 4).unwrap().0, + SharedWindowCacheLookup::Reserved + )); + assert_eq!(cache.publish(4, Arc::new(vec![1u8; 4])).unwrap().0, 0); + assert!(matches!( + cache.lookup_or_reserve(0, 4).unwrap().0, + SharedWindowCacheLookup::Hit(_) + )); + assert!(matches!( + cache.lookup_or_reserve(8, 4).unwrap().0, + SharedWindowCacheLookup::Reserved + )); + + assert_eq!(cache.publish(8, Arc::new(vec![2u8; 4])).unwrap().0, 1); + assert!(matches!( + cache.lookup_or_reserve(0, 4).unwrap().0, + SharedWindowCacheLookup::Hit(_) + )); + assert!(matches!( + cache.lookup_or_reserve(4, 4).unwrap().0, + SharedWindowCacheLookup::Reserved + )); + cache.cancel(&[4]).unwrap(); + } + + #[test] + fn diskann_shared_adjacency_cache_charges_allocation_capacity() { + let cache = SharedWindowCache::new(8); + assert!(matches!( + cache.lookup_or_reserve(0, 4).unwrap().0, + SharedWindowCacheLookup::Reserved + )); + let mut payload = Vec::with_capacity(64); + payload.resize(4, 0); + + assert_eq!(cache.publish(0, Arc::new(payload)).unwrap().0, 1); + assert!(matches!( + cache.lookup_or_reserve(0, 4).unwrap().0, + SharedWindowCacheLookup::Reserved + )); + cache.cancel(&[0]).unwrap(); + } + + #[test] + fn diskann_offset_lru_supports_constant_time_oldest_eviction() { + let mut lru = OffsetLru::default(); + lru.touch(10); + lru.touch(20); + lru.touch(30); + lru.touch(10); + assert_eq!(lru.oldest_offsets(), vec![20, 30, 10]); + + lru.remove(30); + assert_eq!(lru.pop_oldest(), Some(20)); + assert_eq!(lru.pop_oldest(), Some(10)); + assert_eq!(lru.pop_oldest(), None); + } + + #[test] + fn diskann_shared_adjacency_cache_waits_for_one_loader() { + let cache = Arc::new(SharedWindowCache::new(4096)); + assert!(matches!( + cache.lookup_or_reserve(7, 4).unwrap().0, + SharedWindowCacheLookup::Reserved + )); + assert!(matches!( + cache.lookup_or_reserve(7, 4).unwrap().0, + SharedWindowCacheLookup::Loading + )); + let (started_tx, started_rx) = mpsc::channel(); + let waiter_cache = Arc::clone(&cache); + let waiter = thread::spawn(move || { + started_tx.send(()).unwrap(); + waiter_cache.wait_for(7, 4).unwrap().0.unwrap() + }); + started_rx.recv().unwrap(); + + cache.publish(7, Arc::new(vec![3u8; 4])).unwrap(); + + assert_eq!(&*waiter.join().unwrap(), &[3u8; 4]); + } + + #[test] + fn diskann_shared_adjacency_cache_cancel_releases_waiters() { + let cache = SharedWindowCache::new(4096); + assert!(matches!( + cache.lookup_or_reserve(7, 4).unwrap().0, + SharedWindowCacheLookup::Reserved + )); + + cache.cancel(&[7]).unwrap(); + + assert!(cache.wait_for(7, 4).unwrap().0.is_none()); + assert!(matches!( + cache.lookup_or_reserve(7, 4).unwrap().0, + SharedWindowCacheLookup::Reserved + )); + cache.cancel(&[7]).unwrap(); + } + + #[test] + fn diskann_shared_adjacency_validation_is_single_flight() { + let cache = Arc::new(AdjacencyValidationCache::new(1).unwrap()); + let validation_count = Arc::new(AtomicUsize::new(0)); + let (started_tx, started_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + + let first_cache = Arc::clone(&cache); + let first_count = Arc::clone(&validation_count); + let first = thread::spawn(move || { + first_cache + .get_or_validate(0, || { + first_count.fetch_add(1, AtomicOrdering::SeqCst); + started_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + Ok(()) + }) + .unwrap(); + }); + started_rx.recv().unwrap(); + + let second_cache = Arc::clone(&cache); + let second_count = Arc::clone(&validation_count); + let second = thread::spawn(move || { + second_cache + .get_or_validate(0, || { + second_count.fetch_add(1, AtomicOrdering::SeqCst); + Ok(()) + }) + .unwrap(); + }); + + release_tx.send(()).unwrap(); + first.join().unwrap(); + second.join().unwrap(); + assert_eq!(validation_count.load(AtomicOrdering::SeqCst), 1); + } + + #[test] + fn diskann_shared_adjacency_validation_caches_failures() { + let cache = AdjacencyValidationCache::new(1).unwrap(); + let validation_count = AtomicUsize::new(0); + + let first = cache + .get_or_validate(0, || { + validation_count.fetch_add(1, AtomicOrdering::SeqCst); + Err(invalid_data("corrupt immutable adjacency page")) + }) + .expect_err("corrupt page must fail validation"); + let second = cache + .get_or_validate(0, || { + validation_count.fetch_add(1, AtomicOrdering::SeqCst); + Ok(()) + }) + .expect_err("cached corrupt page must keep failing"); + + assert_eq!(validation_count.load(AtomicOrdering::SeqCst), 1); + assert_eq!(second.kind(), first.kind()); + assert_eq!(second.to_string(), first.to_string()); + } + + #[derive(Clone)] + struct RecordingReader { + inner: Cursor>, + reads: Arc>>, + } + + #[derive(Clone)] + struct RoundRecordingReader { + inner: Cursor>, + rounds: ReadRounds, + max_ranges_per_pread: usize, + } + + type ReadRounds = Arc>>>; + + #[derive(Clone)] + struct LatencyHintReader { + inner: Cursor>, + estimated_random_read_latency_nanos: u64, + capability_calls: Arc, + read_calls: Arc, + } + + #[derive(Default)] + struct CountingWriter { + bytes: Vec, + write_calls: usize, + write_lengths: Vec, + } + + impl SeekWrite for CountingWriter { + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + self.write_calls += 1; + self.write_lengths.push(buf.len()); + self.bytes.extend_from_slice(buf); + Ok(()) + } + + fn pos(&self) -> u64 { + self.bytes.len() as u64 + } + } + + impl SeekRead for RecordingReader { + fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { + for range in ranges { + self.reads + .lock() + .unwrap() + .push((range.pos, range.buf.len())); + self.inner.set_position(range.pos); + io::Read::read_exact(&mut self.inner, range.buf)?; + } + Ok(()) + } + + fn try_clone_reader(&self) -> io::Result> { + Ok(Some(self.clone())) + } + } + + impl SeekRead for RoundRecordingReader { + fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { + self.rounds.lock().unwrap().push( + ranges + .iter() + .map(|range| (range.pos, range.buf.len())) + .collect(), + ); + for range in ranges { + self.inner.set_position(range.pos); + io::Read::read_exact(&mut self.inner, range.buf)?; + } + Ok(()) + } + + fn try_clone_reader(&self) -> io::Result> { + Ok(Some(self.clone())) + } + + fn read_capabilities(&self) -> SeekReadCapabilities { + SeekReadCapabilities { + max_ranges_per_pread: self.max_ranges_per_pread, + ..SeekReadCapabilities::default() + } + } + } + + impl SeekRead for LatencyHintReader { + fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { + self.read_calls.fetch_add(1, AtomicOrdering::SeqCst); + for range in ranges { + self.inner.set_position(range.pos); + io::Read::read_exact(&mut self.inner, range.buf)?; + } + Ok(()) + } + + fn read_capabilities(&self) -> SeekReadCapabilities { + self.capability_calls.fetch_add(1, AtomicOrdering::SeqCst); + SeekReadCapabilities { + estimated_random_read_latency_nanos: self.estimated_random_read_latency_nanos, + ..SeekReadCapabilities::default() + } + } + + fn try_clone_reader(&self) -> io::Result> { + Ok(Some(self.clone())) + } + } + + fn initialize_empty_adjacency_index(bytes: &mut [u8], header: &DiskAnnHeader) { + let codebook = header.sections.codebook.offset as usize; + put_u32(bytes, codebook, PQ_CODEBOOK_MAGIC); + put_u32(bytes, codebook + 4, PQ_CODEBOOK_VERSION); + put_u32(bytes, codebook + 8, header.dimension); + put_u32(bytes, codebook + 12, header.pq_m); + put_u32(bytes, codebook + 16, header.pq_bits); + put_u32(bytes, codebook + 20, 1 << header.pq_bits); + put_u32(bytes, codebook + 24, header.pq_m + 1); + let dimension = header.dimension as usize; + let pq_m = header.pq_m as usize; + let wide_chunks = dimension % pq_m; + let narrow_width = dimension / pq_m; + let mut chunk_offset = 0usize; + for chunk in 0..=pq_m { + put_u32( + bytes, + codebook + PQ_CODEBOOK_HEADER_SIZE + chunk * size_of::(), + chunk_offset as u32, + ); + if chunk < pq_m { + chunk_offset += narrow_width + usize::from(chunk < wide_chunks); + } + } + + let row_ids = header.sections.row_ids.offset as usize; + put_u32(bytes, row_ids, ROW_ID_ENCODING_RAW_I64); + put_u32(bytes, row_ids + 4, u64::BITS); + put_u64(bytes, row_ids + 8, header.vector_count); + let page_count = (header.sections.adjacency.length / DISKANN_PAGE_SIZE as u64) as usize; + let base = header.sections.adjacency_index.offset as usize; + let vector_count = header.vector_count as usize; + let block_count = vector_count.div_ceil(ADJACENCY_LOCATOR_BLOCK_NODES); + let relative_base = base + block_count * size_of::(); + let metadata_base = relative_base + vector_count * size_of::(); + for block in 0..block_count { + let node = block * ADJACENCY_LOCATOR_BLOCK_NODES; + let block_offset = node.min(page_count - 1) as u64 * u64::from(DISKANN_PAGE_SIZE); + put_u64(bytes, base + block * size_of::(), block_offset); + } + for node in 0..header.vector_count as usize { + let block_node = node / ADJACENCY_LOCATOR_BLOCK_NODES * ADJACENCY_LOCATOR_BLOCK_NODES; + let block_page = block_node.min(page_count - 1); + let page = node.min(page_count - 1); + let relative_offset = (page - block_page) * DISKANN_PAGE_SIZE as usize; + put_u16( + bytes, + relative_base + node * size_of::(), + relative_offset as u16, + ); + put_u16(bytes, metadata_base + node * size_of::(), 0); + } + } + + fn serialized_adjacency_locator( + bytes: &[u8], + header: &DiskAnnHeader, + node: usize, + ) -> AdjacencyLocator { + let vector_count = header.vector_count as usize; + let block_count = vector_count.div_ceil(ADJACENCY_LOCATOR_BLOCK_NODES); + let base = header.sections.adjacency_index.offset as usize; + let relative_base = base + block_count * size_of::(); + let metadata_base = relative_base + vector_count * size_of::(); + let block_offset = get_u64( + bytes, + base + node / ADJACENCY_LOCATOR_BLOCK_NODES * size_of::(), + ); + let relative_offset = u64::from(get_u16(bytes, relative_base + node * size_of::())); + let absolute_offset = block_offset + relative_offset; + AdjacencyLocator { + page_index: (absolute_offset / u64::from(DISKANN_PAGE_SIZE)) as u32, + byte_offset: (absolute_offset % u64::from(DISKANN_PAGE_SIZE)) as u16, + degree_and_flags: get_u16(bytes, metadata_base + node * size_of::()), + } + } + + #[test] + fn diskann_block_locator_index_roundtrips_block_and_u16_boundaries() { + let mut locators = (0..17) + .map(|node| { + AdjacencyLocator::new( + node.min(16) as u32, + 0, + node % 5, + if node % 2 == 0 { + AdjacencyListEncoding::DeltaVarint + } else { + AdjacencyListEncoding::RawU32 + }, + ) + .unwrap() + }) + .collect::>(); + locators[15].byte_offset = DISKANN_PAGE_SIZE as u16 - 1; + + let index = AdjacencyIndex::from_locators(&locators).unwrap(); + + assert_eq!(adjacency_index_serialized_len(locators.len()).unwrap(), 84); + assert_eq!( + index.block_offsets.as_ref(), + &[0, 16 * DISKANN_PAGE_SIZE as u64] + ); + assert_eq!(index.relative_offsets[15], u16::MAX); + assert_eq!(index.len(), locators.len()); + for (node, expected) in locators.iter().copied().enumerate() { + assert_eq!(index.locator(node), Some(expected)); + } + assert_eq!(index.locator(locators.len()), None); + } + + #[test] + fn diskann_adjacency_codec_selects_the_smaller_encoding_and_roundtrips() { + let compact = [100, 101, 105, 110]; + assert_eq!( + plan_adjacency_list(&compact).unwrap(), + (AdjacencyListEncoding::DeltaVarint, 4) + ); + let mut encoded = Vec::new(); + let encoding = encode_adjacency_list(&compact, &mut encoded).unwrap(); + assert_eq!(encoding, AdjacencyListEncoding::DeltaVarint); + assert_eq!(encoded, [100, 1, 4, 5]); + let mut decoded = Vec::new(); + let consumed = + decode_adjacency_list(&encoded, compact.len(), encoding, &mut decoded).unwrap(); + assert_eq!(consumed, encoded.len()); + assert_eq!(decoded, compact); + + let sparse = [1 << 31, u32::MAX]; + assert_eq!( + plan_adjacency_list(&sparse).unwrap(), + (AdjacencyListEncoding::RawU32, 8) + ); + let encoding = encode_adjacency_list(&sparse, &mut encoded).unwrap(); + assert_eq!(encoding, AdjacencyListEncoding::RawU32); + assert_eq!( + encoded, + [ + 0x00, 0x00, 0x00, 0x80, // 1 << 31 + 0xff, 0xff, 0xff, 0xff, // u32::MAX + ] + ); + let consumed = + decode_adjacency_list(&encoded, sparse.len(), encoding, &mut decoded).unwrap(); + assert_eq!(consumed, encoded.len()); + assert_eq!(decoded, sparse); + } + + #[test] + fn diskann_adjacency_delta_varint_rejects_corrupt_encodings() { + let mut decoded = Vec::new(); + let truncated = + decode_adjacency_list(&[0x80], 1, AdjacencyListEncoding::DeltaVarint, &mut decoded) + .unwrap_err(); + assert!(truncated.to_string().contains("truncated")); + + let non_canonical = decode_adjacency_list( + &[0x80, 0x00], + 1, + AdjacencyListEncoding::DeltaVarint, + &mut decoded, + ) + .unwrap_err(); + assert!(non_canonical.to_string().contains("canonical")); + + let exceeds_u32 = decode_adjacency_list( + &[0xff, 0xff, 0xff, 0xff, 0x10], + 1, + AdjacencyListEncoding::DeltaVarint, + &mut decoded, + ) + .unwrap_err(); + assert!(exceeds_u32.to_string().contains("exceeds u32")); + + let delta_overflow = decode_adjacency_list( + &[0xff, 0xff, 0xff, 0xff, 0x0f, 0x01], + 2, + AdjacencyListEncoding::DeltaVarint, + &mut decoded, + ) + .unwrap_err(); + assert!(delta_overflow.to_string().contains("overflows u32")); + } + + #[test] + fn diskann_writer_uses_adaptive_adjacency_format_and_fewer_pages() { + let dimension = 8; + let count = 512; + let data = (0..count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (0..count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 16, + build_search_list_size: 32, + seed: 73, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, count).unwrap(); + index.add(&data, &ids); + + let prepared = index.prepare_build().unwrap(); + let mut fixed_pages = 1usize; + let mut fixed_offset = 0usize; + for neighbors in prepared.graph.adjacency.iter() { + let bytes = size_of_val(neighbors); + if fixed_offset + bytes > DISKANN_PAGE_SIZE as usize { + fixed_pages += 1; + fixed_offset = 0; + } + fixed_offset += bytes; + } + + let mut bytes = Vec::new(); + let stats = + write_diskann_index_with_stats(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let header = DiskAnnHeader::decode(&bytes[..DISKANN_HEADER_SIZE]).unwrap(); + let adaptive_pages = header.sections.adjacency.length as usize / DISKANN_PAGE_SIZE as usize; + + assert!(stats.accounted_duration() <= stats.total); + assert_eq!(header.adjacency_locator_encoding, 3); + assert!( + adaptive_pages < fixed_pages, + "adaptive={adaptive_pages}, fixed={fixed_pages}" + ); + } + + #[test] + fn diskann_row_id_codec_roundtrips_random_access() { + let cases = [ + (0, vec![5, 5, 5]), + (1, vec![-5, -4, -5]), + (7, vec![-64, 63, 0]), + (8, vec![-64, 64, 0]), + (9, vec![-64, 192, 7]), + (20, vec![-7, (1_i64 << 19) - 7, 42]), + (63, vec![i64::MIN, -1, i64::MIN]), + ]; + + for (expected_width, row_ids) in cases { + let storage = RowIdStorage::encode(row_ids.clone()).unwrap(); + assert_eq!(storage.bit_width(), expected_width); + assert_eq!(storage.len(), row_ids.len()); + assert_eq!( + (0..row_ids.len()) + .map(|node| storage.get(node).unwrap()) + .collect::>(), + row_ids + ); + assert_eq!(storage.get(row_ids.len()), None); + } + } + + #[test] + fn diskann_row_id_sequential_decoder_handles_raw_and_packed_widths() { + let cases = [ + vec![5, 5, 5], + vec![-5, -4, -5], + vec![-64, 63, 0], + vec![-64, 64, 0], + vec![-64, 192, 7], + vec![-7, (1_i64 << 19) - 7, 42], + vec![i64::MIN, -1, i64::MIN], + vec![i64::MIN, 0, i64::MAX], + ]; + + for row_ids in cases { + let storage = RowIdStorage::encode(row_ids.clone()).unwrap(); + let mut decoded = Vec::new(); + + storage + .try_for_each(|node, row_id| { + decoded.push((node, row_id)); + Ok(()) + }) + .unwrap(); + + assert_eq!(decoded, row_ids.into_iter().enumerate().collect::>()); + } + } + + #[test] + fn diskann_row_id_codec_uses_raw_fallback_for_full_width_span() { + let row_ids = [i64::MIN, 0, i64::MAX]; + + let storage = RowIdStorage::encode(row_ids.to_vec()).unwrap(); + + assert!(matches!(storage, RowIdStorage::Raw(_))); + assert_eq!(storage.bit_width(), 64); + assert_eq!(storage.get(0), Some(i64::MIN)); + assert_eq!(storage.get(2), Some(i64::MAX)); + } + + #[test] + fn diskann_row_id_section_roundtrips_self_describing_encoding() { + let storage = RowIdStorage::encode(vec![-7, 0, 17, -7]).unwrap(); + + let bytes = encode_row_id_section(&storage).unwrap(); + let decoded = decode_row_id_section(&bytes, 4).unwrap(); + + assert_eq!(get_u32(&bytes, 0), ROW_ID_ENCODING_FOR_BITPACK); + assert_eq!(get_u32(&bytes, 4), 5); + assert_eq!(get_u64(&bytes, 8), 4); + assert_eq!(get_u64(&bytes, 16) as i64, -7); + assert!(bytes[24..32].iter().all(|&byte| byte == 0)); + assert_eq!(decoded, storage); + } + + #[test] + fn diskann_row_id_section_rejects_corrupt_metadata_and_tail_bits() { + let storage = RowIdStorage::encode(vec![10, 11, 10]).unwrap(); + let valid = encode_row_id_section(&storage).unwrap(); + let mut corruptions = Vec::new(); + + let mut unknown_encoding = valid.clone(); + put_u32(&mut unknown_encoding, 0, 99); + corruptions.push(unknown_encoding); + let mut invalid_width = valid.clone(); + put_u32(&mut invalid_width, 4, 64); + corruptions.push(invalid_width); + let mut invalid_count = valid.clone(); + put_u64(&mut invalid_count, 8, 4); + corruptions.push(invalid_count); + let mut nonzero_reserved = valid.clone(); + nonzero_reserved[24] = 1; + corruptions.push(nonzero_reserved); + let mut nonzero_tail = valid.clone(); + *nonzero_tail.last_mut().unwrap() |= 0x80; + corruptions.push(nonzero_tail); + + for bytes in corruptions { + let error = decode_row_id_section(&bytes, 3) + .expect_err("corrupt row-ID metadata must fail closed"); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + } + } + + #[test] + fn diskann_raw_row_id_section_roundtrips_and_validates_exact_layout() { + let storage = RowIdStorage::encode(vec![i64::MIN, 0, i64::MAX]).unwrap(); + let valid = encode_row_id_section(&storage).unwrap(); + + assert_eq!(get_u32(&valid, 0), ROW_ID_ENCODING_RAW_I64); + assert_eq!(get_u32(&valid, 4), 64); + assert_eq!(decode_row_id_section(&valid, 3).unwrap(), storage); + + let mut nonzero_base = valid.clone(); + put_u64(&mut nonzero_base, 16, 1); + assert!(decode_row_id_section(&nonzero_base, 3).is_err()); + assert!(decode_row_id_section(&valid[..valid.len() - 1], 3).is_err()); + let mut extended = valid; + extended.push(0); + assert!(decode_row_id_section(&extended, 3).is_err()); + } + + #[test] + fn diskann_for_row_id_section_rejects_decoded_i64_overflow() { + let storage = RowIdStorage::encode(vec![i64::MAX - 1, i64::MAX]).unwrap(); + let mut bytes = encode_row_id_section(&storage).unwrap(); + put_u64(&mut bytes, 16, i64::MAX as u64); + + let error = decode_row_id_section(&bytes, 2) + .expect_err("base plus packed delta above i64::MAX must fail closed"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("overflows")); + } + + #[test] + fn diskann_header_roundtrips_fixed_256_byte_layout() { + let header = DiskAnnHeader { + flags: DISKANN_REQUIRED_FLAGS | FLAG_SEPARATE_ADJACENCY_AND_VECTORS, + dimension: 128, + metric: 0, + vector_count: 1, + entry_node: 0, + max_degree: 64, + build_search_list_size: 100, + alpha: 1.2, + seed: 42, + pq_m: 16, + pq_bits: 8, + page_size: 4096, + adjacency_locator_size: 4, + adjacency_locator_encoding: 3, + raw_vector_encoding: DiskAnnRawVectorEncoding::F32 as u32, + vector_record_size: 512, + file_len: 143872, + sections: DiskAnnSections { + codebook: SectionRange::new(4096, 131172), + row_ids: SectionRange::new(135268, 40), + pq_codes: SectionRange::new(135308, 16), + row_id_order: SectionRange::new(135324, 4), + adjacency_index: SectionRange::new(135328, 12), + adjacency: SectionRange::new(139264, 4096), + vectors: SectionRange::new(143360, 512), + }, + }; + + let encoded = header.encode(); + assert_eq!(encoded.len(), 256); + assert_eq!(&encoded[..4], b"DANN"); + assert!(encoded[208..].iter().all(|&byte| byte == 0)); + assert_eq!(DiskAnnHeader::decode(&encoded).unwrap(), header); + } + + #[test] + fn diskann_header_accepts_public_metrics_and_rejects_unknown_codes() { + let header = DiskAnnHeader::for_layout(8, 2, 0, 2, DiskAnnBuildParams::default()).unwrap(); + for metric in [MetricType::L2, MetricType::InnerProduct, MetricType::Cosine] { + let mut bytes = header.encode(); + put_u32(&mut bytes, 20, metric as u32); + assert_eq!(DiskAnnHeader::decode(&bytes).unwrap().metric_type(), metric); + } + let mut bytes = header.encode(); + put_u32(&mut bytes, 20, 3); + assert!(DiskAnnHeader::decode(&bytes).is_err()); + } + + #[test] + fn diskann_header_constructor_rejects_layouts_the_reader_would_reject() { + let invalid_builds = [ + DiskAnnBuildParams { + max_degree: 2, + build_search_list_size: 1, + ..DiskAnnBuildParams::default() + }, + DiskAnnBuildParams { + max_degree: 1, + build_search_list_size: 1, + alpha: 0.5, + ..DiskAnnBuildParams::default() + }, + DiskAnnBuildParams { + max_degree: 1, + build_search_list_size: 1, + alpha: f32::NAN, + ..DiskAnnBuildParams::default() + }, + ]; + for build in invalid_builds { + assert!( + DiskAnnHeader::for_layout(8, 2, 0, 2, build).is_err(), + "header constructor must enforce reader build-parameter invariants" + ); + } + assert!( + DiskAnnHeader::for_layout(8, 2, 2, 2, DiskAnnBuildParams::default()).is_err(), + "entry node must be in range" + ); + assert!( + DiskAnnHeader::for_layout(8, 2, 0, 9, DiskAnnBuildParams::default()).is_err(), + "PQ shape must be valid" + ); + } + + #[test] + fn diskann_writer_rejects_non_finite_persisted_values_before_output() { + fn one_vector_index() -> DiskAnnIndex { + let mut index = DiskAnnIndex::new( + 1, + MetricType::L2, + 1, + DiskAnnBuildParams { + max_degree: 1, + build_search_list_size: 1, + ..DiskAnnBuildParams::default() + }, + ); + index.pq.centroids = (0..256).map(|code| code as f32).collect(); + index.pq.rebuild_norms_cache(); + index.ids = vec![7]; + index.vectors = vec![0.0]; + index + } + + let mut invalid_codebook = one_vector_index(); + invalid_codebook.pq.centroids[0] = f32::NAN; + let mut codebook_output = Vec::new(); + assert!( + write_diskann_index(&invalid_codebook, &mut PosWriter::new(&mut codebook_output)) + .is_err() + ); + assert!(codebook_output.is_empty()); + + let mut invalid_vector = one_vector_index(); + invalid_vector.vectors[0] = f32::INFINITY; + let mut vector_output = Vec::new(); + assert!( + write_diskann_index(&invalid_vector, &mut PosWriter::new(&mut vector_output)).is_err() + ); + assert!(vector_output.is_empty()); + + let mut f16_overflow = one_vector_index(); + f16_overflow.build_params.raw_vector_encoding = DiskAnnRawVectorEncoding::F16; + f16_overflow.vectors[0] = 70_000.0; + let mut f16_output = Vec::new(); + let error = write_diskann_index(&f16_overflow, &mut PosWriter::new(&mut f16_output)) + .expect_err("finite f32 outside binary16 range must be rejected"); + assert!(error.to_string().contains("finite f16 range")); + assert!(f16_output.is_empty()); + + let mut invalid_pq_shape = one_vector_index(); + invalid_pq_shape.pq.chunk_offsets[1] = 0; + let mut pq_shape_output = Vec::new(); + assert!( + write_diskann_index(&invalid_pq_shape, &mut PosWriter::new(&mut pq_shape_output)) + .is_err() + ); + assert!(pq_shape_output.is_empty()); + + let mut invalid_build = one_vector_index(); + invalid_build.build_params.build_search_list_size = 0; + let mut build_output = Vec::new(); + assert!( + write_diskann_index(&invalid_build, &mut PosWriter::new(&mut build_output)).is_err() + ); + assert!(build_output.is_empty()); + } + + #[test] + fn diskann_header_requires_exactly_one_storage_layout() { + let mut bytes = DiskAnnHeader::for_layout(8, 2, 0, 2, DiskAnnBuildParams::default()) + .unwrap() + .encode(); + put_u32(&mut bytes, 12, DISKANN_REQUIRED_FLAGS); + assert!(DiskAnnHeader::decode(&bytes).is_err()); + + put_u32( + &mut bytes, + 12, + DISKANN_REQUIRED_FLAGS + | FLAG_SEPARATE_ADJACENCY_AND_VECTORS + | FLAG_INTERLEAVED_ADJACENCY_AND_VECTORS, + ); + assert!(DiskAnnHeader::decode(&bytes).is_err()); + } + + #[test] + fn diskann_header_rejects_version_flags_reserved_and_shape_corruption() { + let valid = DiskAnnHeader::for_layout(8, 2, 0, 2, DiskAnnBuildParams::default()) + .unwrap() + .encode(); + let corruptions: &[(usize, u32, &str)] = &[ + (4, DISKANN_VERSION + 1, "version"), + (8, DISKANN_HEADER_SIZE as u32 - 1, "header size"), + (12, get_u32(&valid, 12) | (1 << 31), "unknown feature flag"), + (76, 0, "raw-vector encoding"), + (80, 31, "raw-vector record size"), + (84, SECTION_COUNT as u32 - 1, "section count"), + ]; + for &(offset, value, name) in corruptions { + let mut bytes = valid; + put_u32(&mut bytes, offset, value); + assert!( + DiskAnnHeader::decode(&bytes).is_err(), + "{name} corruption must fail closed" + ); + } + + let mut reserved = valid; + reserved[208] = 1; + assert!( + DiskAnnHeader::decode(&reserved).is_err(), + "non-zero reserved bytes must fail closed" + ); + } + + #[test] + fn diskann_read_tier_latency_classifier_has_stable_boundaries() { + assert_eq!( + classify_read_tier(AUTO_PROFILE_MEMORY_LATENCY_THRESHOLD - Duration::from_nanos(1),), + DeploymentProfile::Memory + ); + assert_eq!( + classify_read_tier(AUTO_PROFILE_MEMORY_LATENCY_THRESHOLD), + DeploymentProfile::LocalStorage + ); + assert_eq!( + classify_read_tier(AUTO_PROFILE_LOCAL_LATENCY_THRESHOLD - Duration::from_nanos(1),), + DeploymentProfile::LocalStorage + ); + assert_eq!( + classify_read_tier(AUTO_PROFILE_LOCAL_LATENCY_THRESHOLD), + DeploymentProfile::RemoteStorage + ); + assert_eq!( + classify_read_tier(AUTO_PROFILE_REMOTE_LATENCY_THRESHOLD - Duration::from_nanos(1),), + DeploymentProfile::RemoteStorage + ); + assert_eq!( + classify_read_tier(AUTO_PROFILE_REMOTE_LATENCY_THRESHOLD), + DeploymentProfile::ObjectStore + ); + } + + #[test] + fn diskann_read_plan_uses_latency_hint_or_mandatory_header_read() { + let header = DiskAnnHeader::for_layout(8, 2, 0, 2, DiskAnnBuildParams::default()).unwrap(); + let mut bytes = vec![0u8; header.file_len as usize]; + bytes[..DISKANN_HEADER_SIZE].copy_from_slice(&header.encode()); + initialize_empty_adjacency_index(&mut bytes, &header); + + let hinted_capability_calls = Arc::new(AtomicUsize::new(0)); + let hinted_read_calls = Arc::new(AtomicUsize::new(0)); + let mut hinted = DiskAnnIndexReader::open_with_options( + LatencyHintReader { + inner: Cursor::new(bytes.clone()), + estimated_random_read_latency_nanos: 20_000_000, + capability_calls: Arc::clone(&hinted_capability_calls), + read_calls: Arc::clone(&hinted_read_calls), + }, + VectorIndexReaderOptions::default(), + ) + .unwrap(); + let initial_plan = hinted.vector_read_plan(); + assert_eq!(initial_plan.random_read_latency_nanos, 20_000_000); + assert_eq!(initial_plan.window_bytes, 64 * 1024); + assert_eq!(hinted_read_calls.load(AtomicOrdering::SeqCst), 1); + hinted.search(&[0.0; 8], 1, 10).unwrap(); + hinted.search(&[0.0; 8], 1, 10).unwrap(); + let clone = hinted.try_clone_for_search().unwrap().unwrap(); + + assert_eq!(hinted.effective_read_tier(), DeploymentProfile::ObjectStore); + assert_eq!(clone.effective_read_tier(), DeploymentProfile::ObjectStore); + assert_eq!(hinted.random_read_latency(), Duration::from_millis(20)); + let plan = hinted.vector_read_plan(); + assert_eq!(plan.random_read_latency_nanos, 20_000_000); + assert_eq!(plan.window_bytes, 64 * 1024); + assert_eq!(plan.graph_beam_width, 16); + assert_eq!(plan.filtered_graph_beam_width, 4); + assert_eq!(plan.memory_budget_bytes, 4 * 1024 * 1024 * 1024); + assert!(hinted_capability_calls.load(AtomicOrdering::SeqCst) >= 1); + + let measured_capability_calls = Arc::new(AtomicUsize::new(0)); + let measured_read_calls = Arc::new(AtomicUsize::new(0)); + let mut measured = DiskAnnIndexReader::open(LatencyHintReader { + inner: Cursor::new(bytes), + estimated_random_read_latency_nanos: 0, + capability_calls: Arc::clone(&measured_capability_calls), + read_calls: Arc::clone(&measured_read_calls), + }) + .unwrap(); + assert_eq!(measured_read_calls.load(AtomicOrdering::SeqCst), 1); + measured.search(&[0.0; 8], 1, 10).unwrap(); + assert_ne!(measured.effective_read_tier(), DeploymentProfile::Auto); + assert!(measured.random_read_latency() > Duration::ZERO); + assert!(measured_capability_calls.load(AtomicOrdering::SeqCst) >= 1); + } + + #[test] + fn diskann_header_calculates_aligned_section_layout() { + let header = DiskAnnHeader::for_layout( + 128, + 31, + 0, + 16, + DiskAnnBuildParams { + raw_vector_encoding: DiskAnnRawVectorEncoding::F32, + ..DiskAnnBuildParams::default() + }, + ) + .unwrap(); + + assert_eq!(header.adjacency_locator_size, 4); + assert_eq!(header.adjacency_locator_encoding, 3); + assert_eq!( + header.raw_vector_encoding, + DiskAnnRawVectorEncoding::F32 as u32 + ); + assert_eq!(header.vector_record_size, 512); + assert_eq!( + header.sections, + DiskAnnSections { + codebook: SectionRange::new(4096, 131172), + row_ids: SectionRange::new(135268, 280), + pq_codes: SectionRange::new(135548, 496), + row_id_order: SectionRange::new(136044, 124), + adjacency_index: SectionRange::new(136168, 140), + adjacency: SectionRange::new(139264, 4096), + vectors: SectionRange::new(143360, 15872), + } + ); + assert_eq!(header.file_len, 159232); + } + + #[test] + fn diskann_interleaved_layout_rejects_non_finite_raw_vectors_during_warmup() { + let dimension = 8; + let count = 64; + let data = (0..count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (0..count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + storage_layout: DiskAnnStorageLayout::Interleaved, + raw_vector_encoding: DiskAnnRawVectorEncoding::F16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, count).unwrap(); + index.add(&data, &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + + let mut locator_reader = DiskAnnIndexReader::open(Cursor::new(bytes.clone())).unwrap(); + locator_reader.ensure_resident().unwrap(); + let locator = locator_reader.adjacency_locator(0).unwrap(); + let header = locator_reader.header.clone(); + let vector_offset = header.sections.adjacency.offset as usize + + locator.page_index as usize * DISKANN_PAGE_SIZE as usize + + locator.byte_offset as usize + - header.vector_record_size as usize; + bytes[vector_offset..vector_offset + 2] + .copy_from_slice(&half::f16::NAN.to_bits().to_le_bytes()); + + let mut reader = DiskAnnIndexReader::open_with_options( + Cursor::new(bytes), + VectorIndexReaderOptions::with_cache_budgets( + DeploymentProfile::Auto, + header.sections.adjacency.length as usize, + 16 * 1024 * 1024, + 4 * 1024 * 1024 * 1024, + 8 * 1024 * 1024, + ), + ) + .unwrap(); + let error = reader + .optimize_for_search() + .expect_err("non-finite interleaved raw vectors must fail closed"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("non-finite")); + } + + #[test] + fn diskann_header_uses_packed_4bit_pq_layout() { + let header = DiskAnnHeader::for_layout_with_pq_bits( + 128, + 31, + 0, + 16, + 4, + DiskAnnBuildParams::default(), + ) + .unwrap(); + + assert_eq!(header.pq_bits, 4); + assert_eq!( + header.sections.codebook.length, + (PQ_CODEBOOK_HEADER_SIZE + 17 * size_of::() + 128 * 16 * 4) as u64 + ); + assert_eq!(header.sections.pq_codes.length, 31 * 8); + assert_eq!(DiskAnnHeader::decode(&header.encode()).unwrap(), header); + } + + #[test] + fn diskann_header_rejects_previous_adjacency_locator_encoding() { + let header = DiskAnnHeader::for_layout(8, 2, 0, 2, DiskAnnBuildParams::default()).unwrap(); + let mut encoded = header.encode(); + put_u32(&mut encoded, 72, 2); + + let error = DiskAnnHeader::decode(&encoded) + .expect_err("the unpublished fixed-u32 locator encoding must be rejected"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("layout")); + } + + #[test] + fn diskann_header_uses_encoded_row_id_section_length() { + let build = DiskAnnBuildParams::default(); + let row_ids_len = ROW_ID_SECTION_HEADER_SIZE + 78; + + let header = + DiskAnnHeader::for_layout_with_adjacency_pages(128, 31, 0, 16, build, row_ids_len, 1) + .unwrap(); + + assert_eq!(header.sections.row_ids.length, row_ids_len as u64); + assert_eq!( + header.sections.pq_codes.offset, + header.sections.row_ids.offset + row_ids_len as u64 + ); + assert!(DiskAnnHeader::for_layout_with_adjacency_pages( + 128, + 31, + 0, + 16, + build, + ROW_ID_SECTION_HEADER_SIZE - 1, + 1, + ) + .is_err()); + } + + #[test] + fn diskann_writer_serializes_aligned_and_consistently_remapped_sections() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + raw_vector_encoding: DiskAnnRawVectorEncoding::F32, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + + let header = DiskAnnHeader::decode(&bytes[..DISKANN_HEADER_SIZE]).unwrap(); + assert_eq!(bytes.len() as u64, header.file_len); + assert_eq!(header.sections.codebook.offset, 4096); + assert_eq!(header.sections.adjacency.offset % 4096, 0); + assert_eq!(header.sections.vectors.offset % 4096, 0); + + assert!( + header.sections.row_ids.length < raw_row_id_section_len(indexed_count).unwrap() as u64 + ); + let row_offset = header.sections.row_ids.offset as usize; + let row_end = section_end(header.sections.row_ids).unwrap() as usize; + let row_ids = decode_row_id_section(&bytes[row_offset..row_end], indexed_count).unwrap(); + let first_row_id = row_ids.get(0).unwrap(); + let old_id = (first_row_id - 1000) as usize; + let vector_offset = header.sections.vectors.offset as usize; + let first_value = + f32::from_le_bytes(bytes[vector_offset..vector_offset + 4].try_into().unwrap()); + assert_eq!(first_value, data[old_id * dimension]); + + assert_eq!( + header.sections.row_id_order.length, + indexed_count as u64 * 4 + ); + assert_eq!( + header.sections.adjacency_index.length, + adjacency_index_serialized_len(indexed_count).unwrap() + ); + + let mut reader = DiskAnnIndexReader::open(Cursor::new(bytes)).unwrap(); + reader.ensure_resident().unwrap(); + assert_eq!(reader.row_id(0).unwrap(), first_row_id); + assert_eq!(reader.row_id_count().unwrap(), indexed_count); + } + + #[test] + fn diskann_writer_chunks_all_sections_into_bounded_write_calls() { + let dimension = 8; + let training_count = 1024; + let indexed_count = 1024; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 64, + build_search_list_size: 100, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut writer = CountingWriter::default(); + + write_diskann_index(&index, &mut writer).unwrap(); + + let header = DiskAnnHeader::decode(&writer.bytes[..DISKANN_HEADER_SIZE]).unwrap(); + let adjacency_pages = + header.sections.adjacency.length as usize / DISKANN_PAGE_SIZE as usize; + assert!(adjacency_pages > 1); + let page_sized_writes = writer + .write_lengths + .iter() + .filter(|&&length| length == DISKANN_PAGE_SIZE as usize) + .count(); + assert!( + page_sized_writes < adjacency_pages, + "adjacency serialization issued one output call per logical page" + ); + let full_chunks = writer.bytes.len().div_ceil(DISKANN_WRITE_BUFFER_SIZE); + assert!( + writer.write_calls <= full_chunks + 8, + "DiskANN serialization crossed the output boundary {} times for {} chunks", + writer.write_calls, + full_chunks + ); + } + + #[test] + fn diskann_row_id_order_is_persisted_by_row_id_then_node() { + let dimension = 8; + let training_count = 256; + let indexed_count = 8; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = vec![9, -1, 9, 3, 7, 3, 11, 7]; + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 4, + build_search_list_size: 8, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let header = DiskAnnHeader::decode(&bytes[..DISKANN_HEADER_SIZE]).unwrap(); + let row_ids = decode_row_id_section( + &bytes[header.sections.row_ids.offset as usize + ..section_end(header.sections.row_ids).unwrap() as usize], + indexed_count, + ) + .unwrap(); + let order = bytes[header.sections.row_id_order.offset as usize + ..section_end(header.sections.row_id_order).unwrap() as usize] + .chunks_exact(4) + .map(|value| u32::from_le_bytes(value.try_into().unwrap())) + .collect::>(); + + validate_row_id_order(&row_ids, &order).unwrap(); + let ordered_keys = order + .iter() + .map(|&node| (row_ids.get(node as usize).unwrap(), node)) + .collect::>(); + assert_eq!( + ordered_keys.iter().map(|key| key.0).collect::>(), + vec![-1, 3, 3, 7, 7, 9, 9, 11] + ); + assert!(ordered_keys.windows(2).all(|pair| pair[0] < pair[1])); + } + + #[test] + fn diskann_row_id_order_corruption_fails_closed() { + let header = DiskAnnHeader::for_layout( + 8, + 2, + 0, + 2, + DiskAnnBuildParams { + max_degree: 1, + build_search_list_size: 2, + ..DiskAnnBuildParams::default() + }, + ) + .unwrap(); + let mut bytes = vec![0u8; header.file_len as usize]; + bytes[..DISKANN_HEADER_SIZE].copy_from_slice(&header.encode()); + initialize_empty_adjacency_index(&mut bytes, &header); + let order = header.sections.row_id_order.offset as usize; + bytes[order..order + 4].copy_from_slice(&0u32.to_le_bytes()); + bytes[order + 4..order + 8].copy_from_slice(&0u32.to_le_bytes()); + let mut reader = DiskAnnIndexReader::open(Cursor::new(bytes)).unwrap(); + + let error = reader + .ensure_row_id_order() + .expect_err("duplicate row-order nodes must fail closed"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("strictly increasing")); + } + + #[test] + fn diskann_row_id_order_budget_fallback_is_cached_without_reading_lookup() { + let header = DiskAnnHeader::for_layout( + 8, + 64, + 0, + 2, + DiskAnnBuildParams { + max_degree: 1, + build_search_list_size: 2, + ..DiskAnnBuildParams::default() + }, + ) + .unwrap(); + let budget = resident_peak_bytes(&header).unwrap(); + let mut bytes = vec![0u8; header.file_len as usize]; + bytes[..DISKANN_HEADER_SIZE].copy_from_slice(&header.encode()); + initialize_empty_adjacency_index(&mut bytes, &header); + let reads = Arc::new(Mutex::new(Vec::new())); + let mut reader = DiskAnnIndexReader::open_with_options( + RecordingReader { + inner: Cursor::new(bytes), + reads: Arc::clone(&reads), + }, + VectorIndexReaderOptions::with_cache_budgets( + DeploymentProfile::Auto, + 16 * 1024 * 1024, + 16 * 1024 * 1024, + budget, + 8 * 1024 * 1024, + ), + ) + .unwrap(); + reader.hot_adjacency = Arc::from(vec![0u8; 16 * 1024]); + + assert!(reader.ensure_row_id_order().unwrap().is_none()); + let reads_after_first_attempt = reads.lock().unwrap().len(); + assert!(reader.ensure_row_id_order().unwrap().is_none()); + + assert_eq!(reads.lock().unwrap().len(), reads_after_first_attempt); + assert!(!reads + .lock() + .unwrap() + .iter() + .any(|(offset, _)| *offset == header.sections.row_id_order.offset)); + } + + #[test] + fn diskann_row_id_order_reserves_budget_from_shared_caches() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (0..indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let header = DiskAnnHeader::decode(&bytes[..DISKANN_HEADER_SIZE]).unwrap(); + let order_bytes = header.sections.row_id_order.length as usize; + let minimum_peak = resident_peak_bytes(&header) + .unwrap() + .max(row_id_order_peak_bytes(&header, 0).unwrap()); + let budget = minimum_peak + 4 * 1024; + let mut reader = DiskAnnIndexReader::open_with_options( + Cursor::new(bytes), + VectorIndexReaderOptions::with_cache_budgets( + DeploymentProfile::Auto, + 0, + 16 * 1024, + budget, + 16 * 1024, + ), + ) + .unwrap(); + reader.ensure_resident().unwrap(); + let initial_capacity = reader + .adjacency_cache() + .unwrap() + .total_capacity() + .saturating_add(reader.raw_vector_cache().unwrap().total_capacity()); + assert_eq!(initial_capacity, 32 * 1024); + + assert!(reader.ensure_row_id_order().unwrap().is_some()); + let final_capacity = reader + .adjacency_cache() + .unwrap() + .total_capacity() + .saturating_add(reader.raw_vector_cache().unwrap().total_capacity()); + assert_eq!( + final_capacity, + (budget - resident_steady_bytes(&header).unwrap() - order_bytes).min(32 * 1024) + ); + let effective_plan = reader.vector_read_plan(); + assert_eq!(effective_plan.adjacency_preload_bytes, 0); + assert_eq!( + effective_plan + .adjacency_cache_bytes + .saturating_add(effective_plan.raw_vector_cache_bytes), + final_capacity + ); + assert!(resident_steady_bytes(&header).unwrap() + order_bytes + final_capacity <= budget); + } + + #[test] + fn diskann_header_rejects_overlapping_sections() { + let mut header = + DiskAnnHeader::for_layout(128, 31, 0, 16, DiskAnnBuildParams::default()).unwrap(); + header.sections.row_ids.offset -= 1; + + let error = DiskAnnHeader::decode(&header.encode()) + .expect_err("overlapping sections should fail closed"); + + assert!(error.to_string().contains("section layout")); + } + + #[test] + fn diskann_reader_loads_resident_sections_in_one_multi_range_round() { + let dimension = 8; + let training_count = 256; + let indexed_count = 24; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + + let rounds = Arc::new(Mutex::new(Vec::new())); + let recording = RoundRecordingReader { + inner: Cursor::new(bytes.clone()), + rounds: Arc::clone(&rounds), + max_ranges_per_pread: 0, + }; + let mut reader = DiskAnnIndexReader::open_with_options( + recording, + VectorIndexReaderOptions::with_cache_budgets( + DeploymentProfile::LocalStorage, + 0, + 0, + 4 * 1024 * 1024 * 1024, + 0, + ), + ) + .unwrap(); + assert_eq!( + rounds.lock().unwrap().as_slice(), + &[vec![(0, DISKANN_HEADER_SIZE)]] + ); + + reader.ensure_resident().unwrap(); + + assert_eq!( + rounds.lock().unwrap().as_slice(), + &[ + vec![(0, DISKANN_HEADER_SIZE)], + vec![ + ( + reader.header.sections.codebook.offset, + reader.header.sections.codebook.length as usize, + ), + ( + reader.header.sections.row_ids.offset, + reader.header.sections.row_ids.length as usize, + ), + ( + reader.header.sections.pq_codes.offset, + reader.header.sections.pq_codes.length as usize, + ), + ( + reader.header.sections.adjacency_index.offset, + reader.header.sections.adjacency_index.length as usize, + ), + (reader.header.file_len - 1, 1), + ], + ] + ); + assert_eq!(reader.row_id_count().unwrap(), indexed_count); + assert_eq!(reader.pq_codes().unwrap().len(), indexed_count * 2); + assert_eq!(reader.pq().unwrap().centroids, index.pq.centroids); + + let limited_rounds = Arc::new(Mutex::new(Vec::new())); + let limited_recording = RoundRecordingReader { + inner: Cursor::new(bytes), + rounds: Arc::clone(&limited_rounds), + max_ranges_per_pread: 2, + }; + let mut limited_reader = DiskAnnIndexReader::open_with_options( + limited_recording, + VectorIndexReaderOptions::with_cache_budgets( + DeploymentProfile::LocalStorage, + 0, + 0, + 4 * 1024 * 1024 * 1024, + 0, + ), + ) + .unwrap(); + limited_reader.ensure_resident().unwrap(); + assert_eq!( + limited_rounds + .lock() + .unwrap() + .iter() + .map(Vec::len) + .collect::>(), + vec![1, 2, 2, 1] + ); + assert_eq!(limited_reader.row_id_count().unwrap(), indexed_count); + assert_eq!(limited_reader.pq_codes().unwrap().len(), indexed_count * 2); + } + + #[test] + fn diskann_reader_rejects_malformed_adjacency_degree() { + let header = DiskAnnHeader::for_layout( + 8, + 2, + 0, + 2, + DiskAnnBuildParams { + max_degree: 1, + build_search_list_size: 2, + ..DiskAnnBuildParams::default() + }, + ) + .unwrap(); + let mut bytes = vec![0u8; header.file_len as usize]; + bytes[..DISKANN_HEADER_SIZE].copy_from_slice(&header.encode()); + initialize_empty_adjacency_index(&mut bytes, &header); + let locator_offset = header.sections.adjacency_index.offset as usize; + bytes[locator_offset + 12..locator_offset + 14] + .copy_from_slice(&((header.max_degree + 1) as u16).to_le_bytes()); + let mut reader = DiskAnnIndexReader::open(Cursor::new(bytes)).unwrap(); + + let error = reader + .ensure_resident() + .expect_err("degree above R must fail closed"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("degree")); + } + + #[test] + fn diskann_adjacency_index_rejects_gaps_and_page_crossing() { + let header = DiskAnnHeader::for_layout( + 8, + 2, + 0, + 2, + DiskAnnBuildParams { + max_degree: 2, + build_search_list_size: 2, + ..DiskAnnBuildParams::default() + }, + ) + .unwrap(); + let gap = AdjacencyIndex::from_locators(&[ + AdjacencyLocator::new(0, 0, 1, AdjacencyListEncoding::RawU32).unwrap(), + AdjacencyLocator::new(0, 8, 0, AdjacencyListEncoding::DeltaVarint).unwrap(), + ]) + .unwrap(); + let crossing = AdjacencyIndex::from_locators(&[ + AdjacencyLocator::new(0, 4092, 2, AdjacencyListEncoding::RawU32).unwrap(), + AdjacencyLocator::new(0, 4092, 0, AdjacencyListEncoding::DeltaVarint).unwrap(), + ]) + .unwrap(); + + assert!(validate_adjacency_index(&header, &gap) + .unwrap_err() + .to_string() + .contains("contiguous")); + assert!(validate_adjacency_index(&header, &crossing) + .unwrap_err() + .to_string() + .contains("crosses")); + } + + #[test] + fn diskann_adjacency_page_rejects_noncanonical_varints_and_encoded_gaps() { + let header = DiskAnnHeader::for_layout( + 8, + 2, + 0, + 2, + DiskAnnBuildParams { + max_degree: 2, + build_search_list_size: 2, + ..DiskAnnBuildParams::default() + }, + ) + .unwrap(); + let locators = AdjacencyIndex::from_locators(&[ + AdjacencyLocator::new(0, 0, 1, AdjacencyListEncoding::DeltaVarint).unwrap(), + AdjacencyLocator::new(0, 2, 0, AdjacencyListEncoding::DeltaVarint).unwrap(), + ]) + .unwrap(); + let mut page = vec![0u8; DISKANN_PAGE_SIZE as usize]; + page[..2].copy_from_slice(&[0x81, 0x00]); + + let noncanonical = + validate_adjacency_page_payload(&header, &locators, 0, &page).unwrap_err(); + assert!(noncanonical.to_string().contains("canonical")); + + page[..2].copy_from_slice(&[0x01, 0x00]); + let gap = validate_adjacency_page_payload(&header, &locators, 0, &page).unwrap_err(); + assert!(gap.to_string().contains("contiguous")); + } + + #[test] + fn diskann_adjacency_page_rejects_nonminimal_adaptive_encoding() { + let header = DiskAnnHeader::for_layout( + 8, + 2, + 0, + 2, + DiskAnnBuildParams { + max_degree: 2, + build_search_list_size: 2, + ..DiskAnnBuildParams::default() + }, + ) + .unwrap(); + let locators = AdjacencyIndex::from_locators(&[ + AdjacencyLocator::new(0, 0, 1, AdjacencyListEncoding::RawU32).unwrap(), + AdjacencyLocator::new(0, 4, 0, AdjacencyListEncoding::DeltaVarint).unwrap(), + ]) + .unwrap(); + let mut page = vec![0u8; DISKANN_PAGE_SIZE as usize]; + put_u32(&mut page, 0, 1); + + let error = validate_adjacency_page_payload(&header, &locators, 0, &page) + .expect_err("raw u32 must be rejected when delta-varint is smaller"); + + assert!(error.to_string().contains("minimal")); + } + + #[test] + fn diskann_adjacency_page_rejects_unsorted_or_self_neighbors() { + let dimension = 8; + let training_count = 256; + let indexed_count = 16; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (0..indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 4, + build_search_list_size: 8, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let header = DiskAnnHeader::decode(&bytes[..DISKANN_HEADER_SIZE]).unwrap(); + let (node, locator) = (0..indexed_count) + .find_map(|node| { + let locator = serialized_adjacency_locator(&bytes, &header, node); + (locator.degree() > 0).then_some((node, locator)) + }) + .unwrap(); + let neighbor_offset = header.sections.adjacency.offset as usize + + locator.page_index as usize * DISKANN_PAGE_SIZE as usize + + locator.byte_offset as usize; + match locator.encoding() { + AdjacencyListEncoding::DeltaVarint => { + bytes[neighbor_offset] = node as u8; + } + AdjacencyListEncoding::RawU32 => { + put_u32(&mut bytes, neighbor_offset, node as u32); + } + } + let page_start = header.sections.adjacency.offset as usize + + locator.page_index as usize * DISKANN_PAGE_SIZE as usize; + let page = bytes[page_start..page_start + DISKANN_PAGE_SIZE as usize].to_vec(); + let mut reader = DiskAnnIndexReader::open(Cursor::new(bytes)).unwrap(); + reader.ensure_resident().unwrap(); + + let error = reader + .validate_adjacency_page(locator.page_index as usize, &page) + .expect_err("self edges must fail closed"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("self edge")); + } + + #[test] + fn diskann_reader_records_shared_adjacency_validation_in_resident_data() { + let header = DiskAnnHeader::for_layout( + 8, + 2, + 0, + 2, + DiskAnnBuildParams { + max_degree: 1, + build_search_list_size: 2, + ..DiskAnnBuildParams::default() + }, + ) + .unwrap(); + let mut bytes = vec![0u8; header.file_len as usize]; + bytes[..DISKANN_HEADER_SIZE].copy_from_slice(&header.encode()); + initialize_empty_adjacency_index(&mut bytes, &header); + let mut reader = DiskAnnIndexReader::open(Cursor::new(bytes)).unwrap(); + reader.ensure_resident().unwrap(); + let page = vec![0u8; DISKANN_PAGE_SIZE as usize]; + + reader.validate_adjacency_page(0, &page).unwrap(); + + assert_eq!( + reader + .resident + .as_ref() + .unwrap() + .adjacency_validation + .states[0] + .load(AtomicOrdering::Acquire), + ADJACENCY_PAGE_VALID + ); + } + + #[test] + fn diskann_reader_rejects_non_finite_codebook_during_warmup() { + let header = DiskAnnHeader::for_layout( + 8, + 2, + 0, + 2, + DiskAnnBuildParams { + max_degree: 1, + build_search_list_size: 2, + ..DiskAnnBuildParams::default() + }, + ) + .unwrap(); + let mut bytes = vec![0u8; header.file_len as usize]; + bytes[..DISKANN_HEADER_SIZE].copy_from_slice(&header.encode()); + initialize_empty_adjacency_index(&mut bytes, &header); + let first_centroid = header.sections.codebook.offset as usize + + PQ_CODEBOOK_HEADER_SIZE + + (header.pq_m as usize + 1) * size_of::(); + bytes[first_centroid..first_centroid + 4].copy_from_slice(&f32::NAN.to_le_bytes()); + let mut reader = DiskAnnIndexReader::open(Cursor::new(bytes)).unwrap(); + + let error = reader + .ensure_resident() + .expect_err("non-finite PQ centroid must fail closed"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("finite")); + } + + #[test] + fn diskann_reader_maps_truncated_header_to_invalid_data() { + let error = match DiskAnnIndexReader::open(Cursor::new(b"DANN".to_vec())) { + Ok(_) => panic!("truncated header must fail closed"), + Err(error) => error, + }; + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("truncated")); + } + + #[test] + fn diskann_reader_rejects_resident_sections_above_reader_budget_before_reading() { + let header = DiskAnnHeader::for_layout( + 8, + 2, + 0, + 2, + DiskAnnBuildParams { + max_degree: 1, + build_search_list_size: 2, + ..DiskAnnBuildParams::default() + }, + ) + .unwrap(); + let reads = Arc::new(Mutex::new(Vec::new())); + let recording = RecordingReader { + inner: Cursor::new(header.encode().to_vec()), + reads: Arc::clone(&reads), + }; + let mut reader = + DiskAnnIndexReader::open_with_options(recording, VectorIndexReaderOptions::new(1)) + .unwrap(); + + let error = reader + .ensure_resident() + .expect_err("resident data above the configured budget must fail closed"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("resident")); + assert_eq!( + reads.lock().unwrap().as_slice(), + &[(0, DISKANN_HEADER_SIZE)] + ); + } + + #[test] + fn diskann_adjacency_preload_rounds_up_for_every_read_tier() { + let header = DiskAnnHeader::for_layout_with_adjacency_pages( + 8, + 5_000, + 0, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + raw_row_id_section_len(5_000).unwrap(), + 32, + ) + .unwrap(); + let mut bytes = vec![0u8; header.file_len as usize]; + bytes[..DISKANN_HEADER_SIZE].copy_from_slice(&header.encode()); + initialize_empty_adjacency_index(&mut bytes, &header); + for read_tier in [ + DeploymentProfile::Memory, + DeploymentProfile::LocalStorage, + DeploymentProfile::RemoteStorage, + DeploymentProfile::ObjectStore, + ] { + let reads = Arc::new(Mutex::new(Vec::new())); + let recording = RecordingReader { + inner: Cursor::new(bytes.clone()), + reads: Arc::clone(&reads), + }; + let mut reader = DiskAnnIndexReader::open_with_options( + recording, + VectorIndexReaderOptions::with_cache_budgets( + read_tier, + 4096, + 16 * 1024 * 1024, + 4 * 1024 * 1024 * 1024, + 8 * 1024 * 1024, + ), + ) + .unwrap(); + + reader.optimize_for_search().unwrap(); + + assert_eq!( + reads.lock().unwrap().last().copied(), + Some((header.sections.adjacency.offset, 64 * 1024)), + "{read_tier:?} must honor adjacency_preload_bytes" + ); + } + } + + #[test] + fn diskann_adjacency_preload_validates_pages_during_warmup() { + let header = DiskAnnHeader::for_layout( + 8, + 2, + 0, + 2, + DiskAnnBuildParams { + max_degree: 1, + build_search_list_size: 2, + ..DiskAnnBuildParams::default() + }, + ) + .unwrap(); + let mut bytes = vec![0u8; header.file_len as usize]; + bytes[..DISKANN_HEADER_SIZE].copy_from_slice(&header.encode()); + initialize_empty_adjacency_index(&mut bytes, &header); + bytes[header.sections.adjacency.offset as usize] = 1; + let mut reader = DiskAnnIndexReader::open_with_options( + Cursor::new(bytes), + VectorIndexReaderOptions::with_cache_budgets( + DeploymentProfile::Auto, + DISKANN_PAGE_SIZE as usize, + 16 * 1024 * 1024, + 4 * 1024 * 1024 * 1024, + 8 * 1024 * 1024, + ), + ) + .unwrap(); + + let error = reader + .optimize_for_search() + .expect_err("preloaded adjacency pages must be validated during warmup"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("tail")); + assert!(reader.hot_adjacency.is_empty()); + } + + #[test] + fn diskann_adjacency_preload_stays_within_resident_budget() { + let header = DiskAnnHeader::for_layout_with_adjacency_pages( + 8, + 5_000, + 0, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + raw_row_id_section_len(5_000).unwrap(), + 64, + ) + .unwrap(); + assert!(header.sections.adjacency.length > 128 * 1024); + let max_resident_bytes = resident_peak_bytes(&header).unwrap() + 64 * 1024; + let available_preload_bytes = max_resident_bytes - resident_steady_bytes(&header).unwrap(); + let expected_preload_bytes = available_preload_bytes / (64 * 1024) * (64 * 1024); + assert_eq!(expected_preload_bytes, 64 * 1024); + let mut bytes = vec![0u8; header.file_len as usize]; + bytes[..DISKANN_HEADER_SIZE].copy_from_slice(&header.encode()); + initialize_empty_adjacency_index(&mut bytes, &header); + let mut reader = DiskAnnIndexReader::open_with_options( + Cursor::new(bytes), + VectorIndexReaderOptions::with_cache_budgets( + DeploymentProfile::Auto, + 128 * 1024, + 16 * 1024 * 1024, + max_resident_bytes, + 8 * 1024 * 1024, + ), + ) + .unwrap(); + + reader.optimize_for_search().unwrap(); + + assert_eq!(reader.hot_adjacency.len(), expected_preload_bytes); + assert!( + resident_steady_bytes(&header).unwrap() + reader.hot_adjacency.len() + <= max_resident_bytes + ); + } + + #[test] + fn diskann_resident_peak_accounts_for_batched_decode_buffers() { + let vector_count = 1_000_000; + let header = DiskAnnHeader::for_layout_with_adjacency_pages( + 128, + vector_count, + 0, + 32, + DiskAnnBuildParams::default(), + raw_row_id_section_len(vector_count).unwrap(), + 1, + ) + .unwrap(); + let decoded_codebook_bytes = 128 * diskann_pq_ksub(header.pq_bits).unwrap() * 4 + + (header.pq_m as usize + 1) * size_of::(); + let decoded_row_id_bytes = vector_count * size_of::(); + let row_decode_phase_bytes = header.sections.row_ids.length as usize + + header.sections.pq_codes.length as usize + + header.sections.adjacency_index.length as usize + + decoded_codebook_bytes + + decoded_row_id_bytes; + + assert!( + resident_peak_bytes(&header).unwrap() >= row_decode_phase_bytes, + "the budget must cover serialized row IDs and their decoded representation at once" + ); + } + + #[test] + fn diskann_search_clones_share_resident_and_hot_adjacency() { + let header = DiskAnnHeader::for_layout_with_adjacency_pages( + 8, + 5_000, + 0, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + raw_row_id_section_len(5_000).unwrap(), + 32, + ) + .unwrap(); + let mut bytes = vec![0u8; header.file_len as usize]; + bytes[..DISKANN_HEADER_SIZE].copy_from_slice(&header.encode()); + initialize_empty_adjacency_index(&mut bytes, &header); + let mut reader = DiskAnnIndexReader::open_with_options( + RecordingReader { + inner: Cursor::new(bytes), + reads: Arc::new(Mutex::new(Vec::new())), + }, + VectorIndexReaderOptions::with_cache_budgets( + DeploymentProfile::ObjectStore, + 4096, + 16 * 1024 * 1024, + 4 * 1024 * 1024 * 1024, + 8 * 1024 * 1024, + ), + ) + .unwrap(); + reader.optimize_for_search().unwrap(); + + let clone = reader.try_clone_for_search().unwrap().unwrap(); + + assert!(Arc::ptr_eq( + reader.resident.as_ref().unwrap(), + clone.resident.as_ref().unwrap() + )); + assert!(Arc::ptr_eq(&reader.hot_adjacency, &clone.hot_adjacency)); + } + + #[test] + fn diskann_reader_budget_bounds_decoded_peak_not_only_serialized_bytes() { + let header = DiskAnnHeader::for_layout( + 8, + 2, + 0, + 2, + DiskAnnBuildParams { + max_degree: 1, + build_search_list_size: 2, + ..DiskAnnBuildParams::default() + }, + ) + .unwrap(); + let serialized_resident_bytes = (section_end(header.sections.adjacency_index).unwrap() + - header.sections.codebook.offset) as usize; + let mut bytes = vec![0u8; header.file_len as usize]; + bytes[..DISKANN_HEADER_SIZE].copy_from_slice(&header.encode()); + let mut reader = DiskAnnIndexReader::open_with_options( + Cursor::new(bytes), + VectorIndexReaderOptions::new(serialized_resident_bytes), + ) + .unwrap(); + + let error = reader + .ensure_resident() + .expect_err("budget must include decoded vectors, PQ norms, and load scratch"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("reader budget")); + } + + #[test] + fn diskann_memory_accounts_for_row_id_payload_without_section_header() { + let row_ids_len = ROW_ID_SECTION_HEADER_SIZE + 78; + let header = DiskAnnHeader::for_layout_with_adjacency_pages( + 128, + 31, + 0, + 16, + DiskAnnBuildParams::default(), + row_ids_len, + 1, + ) + .unwrap(); + let non_row_id_bytes = decoded_pq_codebook_bytes(&header).unwrap() + + header.sections.pq_codes.length as usize + + header.sections.adjacency_index.length as usize + + adjacency_validation_bytes(&header).unwrap() + + header.pq_m as usize * 256 * size_of::(); + + assert_eq!( + resident_steady_bytes(&header).unwrap() - non_row_id_bytes, + row_ids_len - ROW_ID_SECTION_HEADER_SIZE + ); + } +} diff --git a/core/src/diskann_search.rs b/core/src/diskann_search.rs new file mode 100644 index 0000000..e1a557b --- /dev/null +++ b/core/src/diskann_search.rs @@ -0,0 +1,7046 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::diskann::DiskAnnRawVectorEncoding; +use crate::diskann_io::{ + decode_adjacency_list, CacheLockMetrics, OffsetLru, SectionRange, SharedWindowCacheLookup, + DISKANN_PAGE_SIZE, +}; +use crate::distance::{ + fvec_distance, fvec_l2sqr, pq_distance_four_codes, pq_distance_from_table, preprocess_vectors, + MetricType, +}; +use crate::index_io_util::decode_roaring_filter; +use crate::io::{ReadRequest, SeekRead}; +use crate::read_options::ReadPlan; +use crate::sparse_table::{estimated_memory_bytes as sparse_table_memory_bytes, SparseTable}; +use half::prelude::{HalfBitsSliceExt, HalfFloatSliceExt}; +use rayon::prelude::*; +use roaring::{RoaringBitmap, RoaringTreemap}; +use std::borrow::Cow; +use std::cmp::{Ordering, Reverse}; +use std::collections::{BTreeMap, BTreeSet, BinaryHeap, HashMap, HashSet}; +use std::io; +use std::sync::Arc; + +const QUERY_WINDOW_BUFFER_LIMIT_BYTES: usize = 8 * 1024 * 1024; +const QUERY_ADJACENCY_WINDOW_LIMIT_BYTES: usize = 8 * 1024 * 1024; +const BATCH_WINDOW_BUFFER_LIMIT_BYTES: usize = 64 * 1024 * 1024; +const FILTERED_BATCH_RERANK_MAX_BYTES: usize = 64 * 1024 * 1024; +const FILTERED_BATCH_RERANK_MAX_RANGES: usize = 1024; +const BATCH_QUERY_CHUNK_SIZE: usize = 1024; +const FILTERED_PQ_MAX_QUERY_TILE_SIZE: usize = 4; +const FILTERED_PQ_TILE_TABLE_LIMIT_BYTES: usize = 2 * 1024 * 1024; +const FILTERED_SINGLE_PQ_NODE_CHUNK_SIZE: usize = 1024; +const PARALLEL_EXACT_RERANK_MIN_COMPONENTS: usize = 16 * 1024; +const PARALLEL_SESSION_MAX_QUERIES_PER_WORKER: usize = 4; +const SPARSE_VISITED_MIN_MEMORY_SAVINGS: usize = 2; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct DiskAnnSearchStats { + pub query_count: usize, + pub query_chunks: usize, + pub max_queries_per_chunk: usize, + pub filtered_exhaustive_queries: usize, + pub filtered_graph_queries: usize, + pub filtered_graph_fallbacks: usize, + pub pq_distance_evaluations: usize, + pub pq_code_loads: usize, + pub adjacency_cache_hits: usize, + pub adjacency_cache_misses: usize, + pub adjacency_cache_waits: usize, + pub adjacency_cache_evictions: usize, + pub adjacency_cache_lock_acquisitions: usize, + pub adjacency_cache_lock_wait_nanos: u64, + pub query_adjacency_cache_peak_bytes: usize, + pub query_adjacency_cache_evictions: usize, + pub rerank_candidate_references: usize, + pub rerank_unique_windows: usize, + pub rerank_chunks: usize, + pub raw_vector_cache_hits: usize, + pub raw_vector_cache_misses: usize, + pub raw_vector_cache_evictions: usize, + pub parallel_exact_rerank_chunks: usize, + pub parallel_exact_rerank_references: usize, + pub parallel_session_queries: usize, +} + +impl DiskAnnSearchStats { + fn record_adjacency_cache_lock(&mut self, metrics: CacheLockMetrics) { + self.adjacency_cache_lock_acquisitions = self + .adjacency_cache_lock_acquisitions + .saturating_add(metrics.acquisitions); + self.adjacency_cache_lock_wait_nanos = self + .adjacency_cache_lock_wait_nanos + .saturating_add(metrics.wait_nanos); + } + + fn merge_candidate_generation(&mut self, worker: Self) { + self.filtered_exhaustive_queries = self + .filtered_exhaustive_queries + .saturating_add(worker.filtered_exhaustive_queries); + self.filtered_graph_queries = self + .filtered_graph_queries + .saturating_add(worker.filtered_graph_queries); + self.filtered_graph_fallbacks = self + .filtered_graph_fallbacks + .saturating_add(worker.filtered_graph_fallbacks); + self.pq_distance_evaluations = self + .pq_distance_evaluations + .saturating_add(worker.pq_distance_evaluations); + self.pq_code_loads = self.pq_code_loads.saturating_add(worker.pq_code_loads); + self.adjacency_cache_hits = self + .adjacency_cache_hits + .saturating_add(worker.adjacency_cache_hits); + self.adjacency_cache_misses = self + .adjacency_cache_misses + .saturating_add(worker.adjacency_cache_misses); + self.adjacency_cache_waits = self + .adjacency_cache_waits + .saturating_add(worker.adjacency_cache_waits); + self.adjacency_cache_evictions = self + .adjacency_cache_evictions + .saturating_add(worker.adjacency_cache_evictions); + self.adjacency_cache_lock_acquisitions = self + .adjacency_cache_lock_acquisitions + .saturating_add(worker.adjacency_cache_lock_acquisitions); + self.adjacency_cache_lock_wait_nanos = self + .adjacency_cache_lock_wait_nanos + .saturating_add(worker.adjacency_cache_lock_wait_nanos); + self.query_adjacency_cache_peak_bytes = self + .query_adjacency_cache_peak_bytes + .max(worker.query_adjacency_cache_peak_bytes); + self.query_adjacency_cache_evictions = self + .query_adjacency_cache_evictions + .saturating_add(worker.query_adjacency_cache_evictions); + } + + fn merge_complete_query(&mut self, worker: Self) { + let worker_query_count = worker.query_count; + self.merge_candidate_generation(worker); + self.rerank_candidate_references = self + .rerank_candidate_references + .saturating_add(worker.rerank_candidate_references); + self.rerank_unique_windows = self + .rerank_unique_windows + .saturating_add(worker.rerank_unique_windows); + self.rerank_chunks = self.rerank_chunks.saturating_add(worker.rerank_chunks); + self.raw_vector_cache_hits = self + .raw_vector_cache_hits + .saturating_add(worker.raw_vector_cache_hits); + self.raw_vector_cache_misses = self + .raw_vector_cache_misses + .saturating_add(worker.raw_vector_cache_misses); + self.raw_vector_cache_evictions = self + .raw_vector_cache_evictions + .saturating_add(worker.raw_vector_cache_evictions); + self.parallel_exact_rerank_chunks = self + .parallel_exact_rerank_chunks + .saturating_add(worker.parallel_exact_rerank_chunks); + self.parallel_exact_rerank_references = self + .parallel_exact_rerank_references + .saturating_add(worker.parallel_exact_rerank_references); + self.parallel_session_queries = self + .parallel_session_queries + .saturating_add(worker_query_count.max(1)); + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct ReadWindow { + pub offset: u64, + pub length: usize, +} + +impl ReadWindow { + pub const fn new(offset: u64, length: usize) -> Self { + Self { offset, length } + } +} + +pub(crate) struct ReadWindowPlanner { + plan: ReadPlan, + section: SectionRange, +} + +impl ReadWindowPlanner { + pub const fn new(plan: ReadPlan, section: SectionRange) -> Self { + Self { plan, section } + } + + #[cfg(test)] + pub const fn beam_width(&self) -> usize { + self.plan.graph_beam_width + } + + pub fn plan_logical_pages( + &self, + logical_pages: impl IntoIterator, + ) -> Vec { + let mut windows = BTreeMap::new(); + for logical_page in logical_pages { + if let Some(window) = self.window_for_logical_page(logical_page) { + windows.insert(window.offset, window.length); + } + } + windows + .into_iter() + .map(|(offset, length)| ReadWindow::new(offset, length)) + .collect() + } + + fn window_for_logical_page(&self, logical_page: usize) -> Option { + let window_size = self.plan.window_bytes as u64; + let relative_page = (logical_page as u64).checked_mul(DISKANN_PAGE_SIZE as u64)?; + if relative_page >= self.section.length { + return None; + } + let relative_window = relative_page / window_size * window_size; + let length = window_size.min(self.section.length - relative_window) as usize; + Some(ReadWindow::new( + self.section.offset + relative_window, + length, + )) + } +} + +pub(crate) struct VectorWindowPlanner { + section: SectionRange, + record_size: usize, + records_per_window: usize, +} + +impl VectorWindowPlanner { + fn new(plan: ReadPlan, section: SectionRange, record_size: usize) -> io::Result { + if record_size == 0 { + return Err(invalid_data( + "DiskANN raw-vector record size must be greater than zero", + )); + } + Ok(Self { + section, + record_size, + records_per_window: (plan.window_bytes / record_size).max(1), + }) + } + + fn window_for_node(&self, node: usize) -> Option { + let record_offset = node.checked_mul(self.record_size)?; + if u64::try_from(record_offset).ok()? >= self.section.length { + return None; + } + let first_node = node / self.records_per_window * self.records_per_window; + let relative_offset = first_node.checked_mul(self.record_size)?; + let maximum_length = self.records_per_window.checked_mul(self.record_size)?; + let remaining = usize::try_from(self.section.length - relative_offset as u64).ok()?; + Some(ReadWindow::new( + self.section.offset.checked_add(relative_offset as u64)?, + maximum_length.min(remaining), + )) + } + + fn plan_nodes(&self, nodes: impl IntoIterator) -> Vec { + let mut windows = BTreeMap::new(); + for node in nodes { + if let Some(window) = self.window_for_node(node) { + windows.insert(window.offset, window.length); + } + } + windows + .into_iter() + .map(|(offset, length)| ReadWindow::new(offset, length)) + .collect() + } + + fn record<'a>( + &self, + window: ReadWindow, + payload: &'a [u8], + node: usize, + ) -> io::Result<&'a [u8]> { + let absolute_offset = self + .section + .offset + .checked_add( + u64::try_from( + node.checked_mul(self.record_size) + .ok_or_else(|| invalid_data("DiskANN raw-vector offset overflows"))?, + ) + .map_err(|_| invalid_data("DiskANN raw-vector offset exceeds u64"))?, + ) + .ok_or_else(|| invalid_data("DiskANN raw-vector offset overflows"))?; + let relative_offset = absolute_offset + .checked_sub(window.offset) + .and_then(|offset| usize::try_from(offset).ok()) + .ok_or_else(|| invalid_data("DiskANN raw-vector window starts after its record"))?; + let record_end = relative_offset + .checked_add(self.record_size) + .ok_or_else(|| invalid_data("DiskANN raw-vector record range overflows"))?; + payload + .get(relative_offset..record_end) + .ok_or_else(|| invalid_data("DiskANN raw-vector record is truncated")) + } +} + +#[derive(Debug, Clone, Copy)] +struct SearchCandidate { + node: usize, + distance: f32, +} + +#[derive(Debug, Clone, Copy)] +struct ExactSearchResult { + row_id: i64, + distance: f32, +} + +#[derive(Clone, Copy)] +struct ExactRerankReference<'a> { + query_index: usize, + row_id: i64, + record: &'a [u8], +} + +enum WindowPayload { + Owned(Vec), + Shared(Arc>), +} + +impl WindowPayload { + fn as_slice(&self) -> &[u8] { + match self { + Self::Owned(payload) => payload, + Self::Shared(payload) => payload, + } + } + + fn capacity(&self) -> usize { + match self { + Self::Owned(payload) => payload.capacity(), + Self::Shared(payload) => payload.capacity(), + } + } +} + +impl From> for WindowPayload { + fn from(payload: Vec) -> Self { + Self::Owned(payload) + } +} + +#[derive(Default)] +struct AdjacencyWindowCache { + entries: HashMap, + recency: OffsetLru, + retained_capacity: usize, +} + +impl AdjacencyWindowCache { + fn contains_key(&self, offset: &u64) -> bool { + self.entries.contains_key(offset) + } + + fn get(&self, offset: &u64) -> Option<&WindowPayload> { + self.entries.get(offset) + } + + fn insert(&mut self, offset: u64, payload: WindowPayload) { + let payload_capacity = payload.capacity(); + if let Some(previous) = self.entries.insert(offset, payload) { + self.retained_capacity = self.retained_capacity.saturating_sub(previous.capacity()); + self.recency.remove(offset); + } + self.retained_capacity = self.retained_capacity.saturating_add(payload_capacity); + self.recency.touch(offset); + } + + fn touch_windows(&mut self, windows: &[ReadWindow]) { + for window in windows { + if self.entries.contains_key(&window.offset) { + self.recency.touch(window.offset); + } + } + } + + fn trim(&mut self, window_buffers: &mut WindowBufferPool, capacity_limit: usize) -> usize { + let mut evictions = 0usize; + while self.retained_capacity > capacity_limit { + let Some(offset) = self.recency.pop_oldest() else { + break; + }; + if let Some(payload) = self.entries.remove(&offset) { + self.retained_capacity = self.retained_capacity.saturating_sub(payload.capacity()); + if let WindowPayload::Owned(payload) = payload { + window_buffers.recycle(payload); + } + evictions = evictions.saturating_add(1); + } + } + evictions + } + + fn recycle(&mut self, window_buffers: &mut WindowBufferPool) { + for (_, payload) in self.entries.drain() { + if let WindowPayload::Owned(payload) = payload { + window_buffers.recycle(payload); + } + } + self.recency.clear(); + self.retained_capacity = 0; + } + + #[cfg(test)] + fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + fn retained_capacity(&self) -> usize { + self.retained_capacity + } + + #[cfg(test)] + fn reserve(&mut self, additional: usize) { + self.entries.reserve(additional); + } + + #[cfg(test)] + fn capacity(&self) -> usize { + self.entries.capacity() + } +} + +fn prepare_adjacency_window_cache( + required_windows: &[ReadWindow], + incoming_bytes: usize, + cache: &mut AdjacencyWindowCache, + window_buffers: &mut WindowBufferPool, +) -> usize { + // A window that was available when the read plan was assembled may still be + // needed by this round even when its individual page does not need loading. + // Mark all round inputs as most-recent before making room for missing + // windows, otherwise trimming for a different page can evict one that + // decode is about to consume. + cache.touch_windows(required_windows); + cache.trim( + window_buffers, + QUERY_ADJACENCY_WINDOW_LIMIT_BYTES.saturating_sub(incoming_bytes), + ) +} + +fn share_window_payload(payload: Vec) -> Arc> { + Arc::new(payload) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FilteredCandidateStrategy { + Exhaustive { + target_candidates: usize, + }, + Graph { + target_candidates: usize, + search_list_size: usize, + }, +} + +#[derive(Default)] +pub(crate) struct DiskAnnQueryScratch { + visited: Vec, + sparse_visited: SparseTable<()>, + uses_sparse_visited: bool, + touched_nodes: Vec, + distance_table: Vec, + candidates: Vec, + rerank_candidates: Vec, + rerank_windows: HashSet, + retained_candidates: BinaryHeap, + frontier: BinaryHeap>, + selected_nodes: Vec, + loaded_adjacency_pages: HashSet, + adjacency_windows: AdjacencyWindowCache, + vector_windows: VectorWindowCache, + window_buffers: WindowBufferPool, + neighbor_buffer: Vec, + scored_neighbors: Vec, +} + +#[derive(Default)] +struct VectorWindowCache { + entries: HashMap, + recency: OffsetLru, + retained_capacity: usize, +} + +#[derive(Debug, Default)] +struct VectorWindowLoadStats { + hits: usize, + misses: usize, + evictions: usize, +} + +impl VectorWindowCache { + fn contains_key(&self, offset: &u64) -> bool { + self.entries.contains_key(offset) + } + + fn get(&self, offset: &u64) -> Option<&[u8]> { + self.entries.get(offset).map(WindowPayload::as_slice) + } + + fn insert(&mut self, offset: u64, payload: impl Into) { + let payload = payload.into(); + let payload_capacity = payload.capacity(); + if let Some(previous) = self.entries.insert(offset, payload) { + self.retained_capacity = self.retained_capacity.saturating_sub(previous.capacity()); + self.recency.remove(offset); + } + self.retained_capacity = self.retained_capacity.saturating_add(payload_capacity); + self.recency.touch(offset); + } + + #[cfg(test)] + fn remove(&mut self, offset: u64) -> Option { + let payload = self.entries.remove(&offset)?; + self.retained_capacity = self.retained_capacity.saturating_sub(payload.capacity()); + self.recency.remove(offset); + Some(payload) + } + + #[cfg(test)] + fn touch(&mut self, offset: u64) { + if self.entries.contains_key(&offset) { + self.recency.touch(offset); + } + } + + fn touch_windows(&mut self, windows: &[ReadWindow]) { + for window in windows { + debug_assert!(self.entries.contains_key(&window.offset)); + self.recency.touch(window.offset); + } + } + + fn trim(&mut self, window_buffers: &mut WindowBufferPool, capacity_limit: usize) -> usize { + let mut evictions = 0usize; + while self.retained_capacity > capacity_limit { + let Some(offset) = self.recency.pop_oldest() else { + break; + }; + if let Some(payload) = self.entries.remove(&offset) { + self.retained_capacity = self.retained_capacity.saturating_sub(payload.capacity()); + if let WindowPayload::Owned(payload) = payload { + window_buffers.recycle(payload); + } + evictions = evictions.saturating_add(1); + } + } + evictions + } + + fn recycle(&mut self, window_buffers: &mut WindowBufferPool) { + for (_, payload) in self.entries.drain() { + if let WindowPayload::Owned(payload) = payload { + window_buffers.recycle(payload); + } + } + self.recency.clear(); + self.retained_capacity = 0; + } + + fn len(&self) -> usize { + self.entries.len() + } + + #[cfg(test)] + fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + #[cfg(test)] + fn retained_capacity(&self) -> usize { + self.retained_capacity + } +} + +struct WindowBufferPool { + buffers: Vec>, + retained_capacity: usize, + retained_capacity_limit: usize, +} + +impl Default for WindowBufferPool { + fn default() -> Self { + Self { + buffers: Vec::new(), + retained_capacity: 0, + retained_capacity_limit: QUERY_WINDOW_BUFFER_LIMIT_BYTES, + } + } +} + +impl WindowBufferPool { + #[cfg(test)] + fn with_retained_capacity_limit(retained_capacity_limit: usize) -> Self { + Self { + retained_capacity_limit, + ..Self::default() + } + } + + fn recycle(&mut self, mut buffer: Vec) { + buffer.clear(); + let capacity = buffer.capacity(); + let Some(retained_capacity) = self.retained_capacity.checked_add(capacity) else { + return; + }; + if retained_capacity > self.retained_capacity_limit { + return; + } + self.retained_capacity = retained_capacity; + self.buffers.push(buffer); + } + + fn take(&mut self, len: usize) -> io::Result> { + let best_fit = self + .buffers + .last() + .is_some_and(|buffer| buffer.capacity() == len) + .then(|| self.buffers.len() - 1) + .or_else(|| { + self.buffers + .iter() + .enumerate() + .filter(|(_, buffer)| buffer.capacity() >= len) + .min_by_key(|(_, buffer)| buffer.capacity()) + .map(|(index, _)| index) + }) + .or_else(|| { + self.buffers + .iter() + .enumerate() + .max_by_key(|(_, buffer)| buffer.capacity()) + .map(|(index, _)| index) + }); + let mut buffer = if let Some(index) = best_fit { + let buffer = self.buffers.swap_remove(index); + self.retained_capacity -= buffer.capacity(); + buffer + } else { + Vec::new() + }; + let additional_capacity = len.saturating_sub(buffer.capacity()); + if additional_capacity != 0 && buffer.try_reserve_exact(additional_capacity).is_err() { + self.recycle(buffer); + return Err(invalid_input( + "DiskANN query window buffer allocation failed", + )); + } + buffer.resize(len, 0); + Ok(buffer) + } + + fn set_retained_capacity_limit(&mut self, retained_capacity_limit: usize) { + self.retained_capacity_limit = retained_capacity_limit; + while self.retained_capacity > self.retained_capacity_limit { + let Some(buffer) = self.buffers.pop() else { + self.retained_capacity = 0; + break; + }; + self.retained_capacity -= buffer.capacity(); + } + } +} + +impl DiskAnnQueryScratch { + #[cfg(test)] + fn with_window_buffer_limit(retained_capacity_limit: usize) -> Self { + Self { + window_buffers: WindowBufferPool::with_retained_capacity_limit(retained_capacity_limit), + ..Self::default() + } + } + + fn set_window_buffer_limit(&mut self, retained_capacity_limit: usize) { + self.window_buffers + .set_retained_capacity_limit(retained_capacity_limit); + } + + #[cfg(test)] + fn begin_search(&mut self, vector_count: usize) { + self.begin_graph_search(vector_count, vector_count, 1) + .expect("test-sized DiskANN visited allocation"); + } + + fn begin_graph_search( + &mut self, + vector_count: usize, + search_list_size: usize, + max_degree: usize, + ) -> io::Result<()> { + self.begin_rerank(); + let expected_visited = search_list_size + .saturating_mul(max_degree) + .saturating_add(1) + .min(vector_count); + let dense_bytes = vector_count.div_ceil(8); + let sparse_bytes = + sparse_table_memory_bytes(expected_visited, size_of::<()>()).unwrap_or(usize::MAX); + // Dense bitmap probes are substantially cheaper than open-addressed + // hashing. Prefer them unless sparse storage saves at least 2x memory. + self.uses_sparse_visited = sparse_bytes + .checked_mul(SPARSE_VISITED_MIN_MEMORY_SAVINGS) + .is_some_and(|threshold| threshold < dense_bytes); + if self.uses_sparse_visited { + if expected_visited > self.sparse_visited.entry_capacity() { + self.sparse_visited = SparseTable::try_with_capacity(expected_visited) + .map_err(|_| invalid_input("DiskANN sparse visited allocation failed"))?; + } + } else { + self.visited.resize(vector_count, false); + } + Ok(()) + } + + fn begin_rerank(&mut self) { + if self.uses_sparse_visited { + self.sparse_visited.clear(); + self.touched_nodes.clear(); + } else { + for node in self.touched_nodes.drain(..) { + self.visited[node] = false; + } + } + self.candidates.clear(); + self.rerank_candidates.clear(); + self.rerank_windows.clear(); + self.retained_candidates.clear(); + self.frontier.clear(); + self.selected_nodes.clear(); + self.loaded_adjacency_pages.clear(); + self.recycle_adjacency_windows(); + self.neighbor_buffer.clear(); + self.scored_neighbors.clear(); + } + + fn recycle_adjacency_windows(&mut self) { + self.adjacency_windows.recycle(&mut self.window_buffers); + } + + fn recycle_vector_windows(&mut self) { + self.vector_windows.recycle(&mut self.window_buffers); + } + + fn recycle_window_caches(&mut self) { + self.recycle_adjacency_windows(); + self.recycle_vector_windows(); + } + + fn prepare_distance_table(&mut self, len: usize) -> &mut [f32] { + self.distance_table.resize(len, 0.0); + &mut self.distance_table + } + + fn select_round(&mut self, limit: usize) { + self.selected_nodes.clear(); + while self.selected_nodes.len() < limit { + let Some(Reverse(candidate)) = self.frontier.pop() else { + break; + }; + if self + .retained_candidates + .peek() + .is_some_and(|worst| candidate > *worst) + { + self.frontier.clear(); + break; + } + self.selected_nodes.push(candidate.node); + } + } + + fn insert_graph_candidate( + &mut self, + candidate: SearchCandidate, + limit: usize, + ) -> io::Result<()> { + if limit == 0 { + return Ok(()); + } + let replacing_worst = self.retained_candidates.len() == limit; + if replacing_worst { + let Some(worst) = self.retained_candidates.peek().copied() else { + return Ok(()); + }; + if candidate >= worst { + return Ok(()); + } + } else { + self.retained_candidates + .try_reserve(1) + .map_err(|_| invalid_input("DiskANN graph candidate allocation failed"))?; + } + self.frontier + .try_reserve(1) + .map_err(|_| invalid_input("DiskANN graph frontier allocation failed"))?; + if replacing_worst { + self.retained_candidates.pop(); + } + self.retained_candidates.push(candidate); + self.frontier.push(Reverse(candidate)); + if self.frontier.len() > limit.saturating_mul(2) { + let worst = *self + .retained_candidates + .peek() + .expect("non-empty retained DiskANN candidates"); + self.frontier + .retain(|Reverse(candidate)| *candidate <= worst); + } + Ok(()) + } + + fn finish_graph_candidates(&mut self) { + self.candidates.extend(self.retained_candidates.drain()); + sort_candidates(&mut self.candidates); + } + + #[cfg(test)] + fn is_visited(&self, node: usize) -> bool { + if self.uses_sparse_visited { + self.sparse_visited.get(node as u32).is_some() + } else { + self.visited[node] + } + } + + fn mark_visited(&mut self, node: usize) -> bool { + if self.uses_sparse_visited { + return self.sparse_visited.insert(node as u32, ()).is_none(); + } + if self.visited[node] { + return false; + } + self.visited[node] = true; + self.touched_nodes.push(node); + true + } + + #[cfg(test)] + fn visited_capacity(&self) -> usize { + self.visited.capacity() + } + + #[cfg(test)] + fn uses_sparse_visited(&self) -> bool { + self.uses_sparse_visited + } + + #[cfg(test)] + fn retained_window_capacity(&self) -> usize { + self.window_buffers.retained_capacity + } +} + +fn window_buffer_limit_per_worker(worker_count: usize) -> usize { + QUERY_WINDOW_BUFFER_LIMIT_BYTES.min(BATCH_WINDOW_BUFFER_LIMIT_BYTES / worker_count.max(1)) +} + +fn prepare_vector_window_cache( + windows: &[ReadWindow], + cache: &mut VectorWindowCache, + window_buffers: &mut WindowBufferPool, + capacity_limit: usize, +) -> (bool, usize) { + let retain = windows + .iter() + .try_fold(0usize, |total, window| total.checked_add(window.length)) + .is_some_and(|total| total <= capacity_limit); + if !retain { + let evictions = cache.len(); + cache.recycle(window_buffers); + return (false, evictions); + } + let evictions = cache.trim(window_buffers, capacity_limit); + (true, evictions) +} + +type CandidatePartition = Vec<(usize, Vec)>; +type SessionQueryOutput = (usize, Vec, Vec, DiskAnnSearchStats); + +impl PartialEq for SearchCandidate { + fn eq(&self, other: &Self) -> bool { + self.node == other.node && self.distance.to_bits() == other.distance.to_bits() + } +} + +impl Eq for SearchCandidate {} + +impl PartialOrd for SearchCandidate { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for SearchCandidate { + fn cmp(&self, other: &Self) -> Ordering { + self.distance + .total_cmp(&other.distance) + .then_with(|| self.node.cmp(&other.node)) + } +} + +impl PartialEq for ExactSearchResult { + fn eq(&self, other: &Self) -> bool { + self.row_id == other.row_id && self.distance.to_bits() == other.distance.to_bits() + } +} + +impl Eq for ExactSearchResult {} + +impl PartialOrd for ExactSearchResult { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for ExactSearchResult { + fn cmp(&self, other: &Self) -> Ordering { + self.distance + .total_cmp(&other.distance) + .then_with(|| self.row_id.cmp(&other.row_id)) + } +} + +impl crate::diskann_io::DiskAnnIndexReader { + fn preprocess_queries<'a>(&self, queries: &'a [f32], query_count: usize) -> Cow<'a, [f32]> { + if self.header.metric_type() == MetricType::Cosine { + Cow::Owned(preprocess_vectors( + queries, + query_count, + self.header.dimension as usize, + MetricType::Cosine, + )) + } else { + Cow::Borrowed(queries) + } + } + + fn take_batch_workers( + &mut self, + worker_count: usize, + filtered: bool, + ) -> io::Result>> { + if filtered { + self.ensure_resident()?; + } else { + self.optimize_for_search()?; + } + let mut workers = std::mem::take(&mut self.batch_workers); + workers.truncate(worker_count); + while workers.len() < worker_count { + let worker = if filtered { + self.try_clone_for_filtered_search() + } else { + self.try_clone_for_search() + }; + match worker { + Ok(Some(worker)) => workers.push(worker), + Ok(None) => { + self.batch_workers = workers; + return Ok(None); + } + Err(error) => { + self.batch_workers = workers; + return Err(error); + } + } + } + let window_buffer_limit = window_buffer_limit_per_worker(worker_count); + for worker in &mut workers { + worker.refresh_shared_state_from(self); + worker.limit_raw_vector_cache_bytes(window_buffer_limit); + worker + .query_scratch + .set_window_buffer_limit(window_buffer_limit); + } + Ok(Some(workers)) + } + + pub(crate) fn search_batch( + &mut self, + queries: &[f32], + top_k: usize, + l_search: usize, + ) -> io::Result<(Vec, Vec)> { + let dimension = self.header.dimension as usize; + let query_count = queries.len() / dimension; + self.last_search_stats = DiskAnnSearchStats { + query_count, + ..DiskAnnSearchStats::default() + }; + if top_k == 0 { + return Ok((Vec::new(), Vec::new())); + } + let processed_queries = self.preprocess_queries(queries, query_count); + let queries = processed_queries.as_ref(); + let worker_count = query_count.min(rayon::current_num_threads()); + if worker_count <= 1 { + self.batch_workers.clear(); + if self.header.is_interleaved() { + return self.search_batch_direct_serial(queries, top_k, l_search); + } + return self.search_batch_serial(queries, top_k, l_search); + } + + let Some(mut workers) = self.take_batch_workers(worker_count, false)? else { + if self.header.is_interleaved() { + return self.search_batch_direct_serial(queries, top_k, l_search); + } + return self.search_batch_serial(queries, top_k, l_search); + }; + if self.header.is_interleaved() + || query_count <= worker_count.saturating_mul(PARALLEL_SESSION_MAX_QUERIES_PER_WORKER) + { + let result = + self.search_batch_in_parallel_sessions(queries, top_k, l_search, &mut workers); + self.batch_workers = workers; + return result; + } + let result = (|| { + let mut ids = Vec::with_capacity(query_count.saturating_mul(top_k)); + let mut distances = Vec::with_capacity(query_count.saturating_mul(top_k)); + for query_chunk in queries.chunks(BATCH_QUERY_CHUNK_SIZE * dimension) { + let chunk_query_count = query_chunk.len() / dimension; + self.record_query_chunk(chunk_query_count); + let worker_outputs = workers + .par_iter_mut() + .enumerate() + .map(|(worker_index, worker)| { + worker.last_search_stats = DiskAnnSearchStats::default(); + let mut partition = Vec::new(); + for query_index in (worker_index..chunk_query_count).step_by(worker_count) { + let query = &query_chunk + [query_index * dimension..(query_index + 1) * dimension]; + let candidates = worker + .generate_unfiltered_candidate_nodes(query, top_k, l_search)?; + partition.push((query_index, candidates)); + } + Ok::<_, io::Error>((partition, worker.last_search_stats)) + }) + .collect::>>()?; + let mut partitions = Vec::with_capacity(worker_outputs.len()); + for (partition, worker_stats) in worker_outputs { + self.last_search_stats + .merge_candidate_generation(worker_stats); + partitions.push(partition); + } + let (chunk_ids, chunk_distances) = + self.rerank_candidate_batch_streaming(query_chunk, top_k, partitions)?; + ids.extend(chunk_ids); + distances.extend(chunk_distances); + } + Ok((ids, distances)) + })(); + self.batch_workers = workers; + result + } + + /// Warms resident metadata and the query-dependent adjacency/raw-vector + /// caches with representative queries without changing reported search + /// statistics. + pub fn warmup_queries(&mut self, queries: &[f32], l_search: usize) -> io::Result<()> { + let dimension = self.header.dimension as usize; + if !queries.len().is_multiple_of(dimension) { + return Err(invalid_input(format!( + "warmup query length {} is not divisible by dimension {}", + queries.len(), + dimension + ))); + } + if queries.iter().any(|value| !value.is_finite()) { + return Err(invalid_input("warmup query values must be finite")); + } + self.optimize_for_search()?; + if queries.is_empty() { + return Ok(()); + } + let saved_stats = self.last_search_stats; + // Replay on the parent Reader so both adjacency and raw-vector windows + // are useful to the subsequent single-query path. A batch warm-up may + // otherwise populate only retained worker-local raw-vector caches. + let result = queries + .chunks_exact(dimension) + .try_for_each(|query| self.search(query, 1, l_search).map(|_| ())); + self.last_search_stats = saved_stats; + result + } + + /// Calibrates the automatic search width from representative queries. + /// + /// This is a stability proxy, not a ground-truth recall guarantee: it + /// chooses the first width whose Top-K overlap with the next wider search + /// reaches 98% across the sample. + pub fn calibrate_l_search(&mut self, queries: &[f32], top_k: usize) -> io::Result { + let dimension = self.header.dimension as usize; + if queries.is_empty() || !queries.len().is_multiple_of(dimension) { + return Err(invalid_input( + "calibration queries must contain one or more complete vectors", + )); + } + if top_k == 0 { + return Err(invalid_input("calibration top_k must be greater than 0")); + } + if queries.iter().any(|value| !value.is_finite()) { + return Err(invalid_input("calibration query values must be finite")); + } + self.optimize_for_search()?; + let widths = [ + 100usize.max(top_k), + 200usize.max(top_k), + 400usize.max(top_k), + ]; + let mut results = Vec::with_capacity(widths.len()); + let saved_stats = self.last_search_stats; + for width in widths { + results.push(self.search_batch(queries, top_k, width)?); + } + self.last_search_stats = saved_stats; + let chosen = if topk_result_stability( + &results[0].0, + &results[0].1, + &results[1].0, + &results[1].1, + top_k, + ) >= 0.98 + { + widths[0] + } else if topk_result_stability( + &results[1].0, + &results[1].1, + &results[2].0, + &results[2].1, + top_k, + ) >= 0.98 + { + widths[1] + } else { + widths[2] + }; + self.calibrated_l_search = Some(chosen); + for worker in &mut self.batch_workers { + worker.calibrated_l_search = Some(chosen); + } + Ok(chosen) + } + + pub(crate) fn search_batch_with_roaring_filter( + &mut self, + queries: &[f32], + top_k: usize, + l_search: usize, + roaring_filter_bytes: &[u8], + ) -> io::Result<(Vec, Vec)> { + let filter = decode_roaring_filter(roaring_filter_bytes)?; + let dimension = self.header.dimension as usize; + let query_count = queries.len() / dimension; + self.last_search_stats = DiskAnnSearchStats { + query_count, + ..DiskAnnSearchStats::default() + }; + if top_k == 0 { + return Ok((Vec::new(), Vec::new())); + } + let processed_queries = self.preprocess_queries(queries, query_count); + let queries = processed_queries.as_ref(); + if filter.is_empty() { + return Ok(( + vec![-1; query_count * top_k], + vec![f32::MAX; query_count * top_k], + )); + } + let matching_nodes = self.matching_nodes_for_filter(&filter)?; + if matching_nodes.is_empty() { + return Ok(( + vec![-1; query_count * top_k], + vec![f32::MAX; query_count * top_k], + )); + } + if self.header.is_interleaved() { + let worker_count = query_count.min(rayon::current_num_threads()); + if worker_count <= 1 { + self.batch_workers.clear(); + return self.search_batch_with_matching_nodes_direct_serial( + queries, + top_k, + l_search, + &matching_nodes, + ); + } + let Some(mut workers) = self.take_batch_workers(worker_count, true)? else { + return self.search_batch_with_matching_nodes_direct_serial( + queries, + top_k, + l_search, + &matching_nodes, + ); + }; + let result = self.search_batch_with_matching_nodes_in_parallel_sessions( + queries, + top_k, + l_search, + &matching_nodes, + &mut workers, + ); + self.batch_workers = workers; + return result; + } + let matching_count = usize::try_from(matching_nodes.len()).unwrap_or(usize::MAX); + if let FilteredCandidateStrategy::Exhaustive { target_candidates } = + select_filtered_candidate_strategy( + self.header.vector_count as usize, + matching_count, + top_k, + l_search, + self.header.max_degree as usize, + self.read_plan(), + self.adjacency_fully_preloaded(), + ) + { + return self.search_batch_filtered_exhaustive( + queries, + top_k, + &matching_nodes, + target_candidates, + ); + } + let worker_count = query_count.min(rayon::current_num_threads()); + if worker_count <= 1 { + self.batch_workers.clear(); + return self.search_batch_with_matching_nodes_serial( + queries, + top_k, + l_search, + &matching_nodes, + ); + } + + let Some(mut workers) = self.take_batch_workers(worker_count, true)? else { + return self.search_batch_with_matching_nodes_serial( + queries, + top_k, + l_search, + &matching_nodes, + ); + }; + if query_count <= worker_count.saturating_mul(PARALLEL_SESSION_MAX_QUERIES_PER_WORKER) { + let result = self.search_batch_with_matching_nodes_in_parallel_sessions( + queries, + top_k, + l_search, + &matching_nodes, + &mut workers, + ); + self.batch_workers = workers; + return result; + } + let result = (|| { + let mut ids = Vec::with_capacity(query_count.saturating_mul(top_k)); + let mut distances = Vec::with_capacity(query_count.saturating_mul(top_k)); + for query_chunk in queries.chunks(BATCH_QUERY_CHUNK_SIZE * dimension) { + let chunk_query_count = query_chunk.len() / dimension; + self.record_query_chunk(chunk_query_count); + let worker_outputs = workers + .par_iter_mut() + .enumerate() + .map(|(worker_index, worker)| { + worker.last_search_stats = DiskAnnSearchStats::default(); + let mut partition = Vec::new(); + for query_index in (worker_index..chunk_query_count).step_by(worker_count) { + let query = &query_chunk + [query_index * dimension..(query_index + 1) * dimension]; + let candidates = worker.generate_filtered_candidates( + query, + top_k, + l_search, + &matching_nodes, + )?; + partition.push(( + query_index, + candidates + .into_iter() + .map(|candidate| candidate.node) + .collect(), + )); + } + Ok::<_, io::Error>((partition, worker.last_search_stats)) + }) + .collect::>>()?; + let mut partitions = Vec::with_capacity(worker_outputs.len()); + for (partition, worker_stats) in worker_outputs { + self.last_search_stats + .merge_candidate_generation(worker_stats); + partitions.push(partition); + } + let (chunk_ids, chunk_distances) = + self.rerank_candidate_batch_streaming(query_chunk, top_k, partitions)?; + ids.extend(chunk_ids); + distances.extend(chunk_distances); + } + Ok((ids, distances)) + })(); + self.batch_workers = workers; + result + } + + fn search_batch_in_parallel_sessions( + &mut self, + queries: &[f32], + top_k: usize, + l_search: usize, + workers: &mut [Self], + ) -> io::Result<(Vec, Vec)> { + let dimension = self.header.dimension as usize; + let query_count = queries.len() / dimension; + self.record_query_chunk(query_count); + let worker_count = workers.len(); + let worker_outputs = workers + .par_iter_mut() + .enumerate() + .map(|(worker_index, worker)| { + let mut outputs = Vec::new(); + for query_index in (worker_index..query_count).step_by(worker_count) { + let query = &queries[query_index * dimension..(query_index + 1) * dimension]; + let (ids, distances) = worker.search_preprocessed(query, top_k, l_search)?; + outputs.push((query_index, ids, distances, worker.last_search_stats)); + } + Ok::<_, io::Error>(outputs) + }) + .collect::>>()?; + self.collect_parallel_session_outputs(worker_outputs, query_count, top_k) + } + + fn search_batch_with_matching_nodes_in_parallel_sessions( + &mut self, + queries: &[f32], + top_k: usize, + l_search: usize, + matching_nodes: &RoaringBitmap, + workers: &mut [Self], + ) -> io::Result<(Vec, Vec)> { + let dimension = self.header.dimension as usize; + let query_count = queries.len() / dimension; + self.record_query_chunk(query_count); + let worker_count = workers.len(); + let worker_outputs = workers + .par_iter_mut() + .enumerate() + .map(|(worker_index, worker)| { + let mut outputs = Vec::new(); + for query_index in (worker_index..query_count).step_by(worker_count) { + let query = &queries[query_index * dimension..(query_index + 1) * dimension]; + worker.last_search_stats = DiskAnnSearchStats { + query_count: 1, + ..DiskAnnSearchStats::default() + }; + let (ids, distances) = worker.search_with_matching_nodes( + query, + top_k, + l_search, + matching_nodes, + )?; + outputs.push((query_index, ids, distances, worker.last_search_stats)); + } + Ok::<_, io::Error>(outputs) + }) + .collect::>>()?; + self.collect_parallel_session_outputs(worker_outputs, query_count, top_k) + } + + fn collect_parallel_session_outputs( + &mut self, + worker_outputs: Vec>, + query_count: usize, + top_k: usize, + ) -> io::Result<(Vec, Vec)> { + let mut ordered = (0..query_count) + .map(|_| None) + .collect::, Vec)>>>(); + for (query_index, ids, distances, stats) in worker_outputs.into_iter().flatten() { + let slot = ordered + .get_mut(query_index) + .ok_or_else(|| invalid_data("DiskANN parallel session query index is invalid"))?; + if slot.replace((ids, distances)).is_some() { + return Err(invalid_data( + "DiskANN parallel sessions returned a duplicate query", + )); + } + self.last_search_stats.merge_complete_query(stats); + } + if ordered.iter().any(Option::is_none) { + return Err(invalid_data( + "DiskANN parallel sessions did not return every query", + )); + } + let mut ids = Vec::with_capacity(query_count.saturating_mul(top_k)); + let mut distances = Vec::with_capacity(query_count.saturating_mul(top_k)); + for result in ordered { + let (query_ids, query_distances) = + result.expect("validated DiskANN parallel session output"); + ids.extend(query_ids); + distances.extend(query_distances); + } + Ok((ids, distances)) + } + + fn search_batch_filtered_exhaustive( + &mut self, + queries: &[f32], + top_k: usize, + matching_nodes: &RoaringBitmap, + candidate_limit: usize, + ) -> io::Result<(Vec, Vec)> { + let dimension = self.header.dimension as usize; + let query_count = queries.len() / dimension; + let matching_count = usize::try_from(matching_nodes.len()).unwrap_or(usize::MAX); + let pq_m = self.header.pq_m as usize; + let pq_ksub = 1usize << self.header.pq_bits; + let tile_size = filtered_pq_query_tile_size(pq_m, pq_ksub); + let mut ids = Vec::with_capacity(query_count.saturating_mul(top_k)); + let mut distances = Vec::with_capacity(query_count.saturating_mul(top_k)); + for query_chunk in queries.chunks(BATCH_QUERY_CHUNK_SIZE * dimension) { + let chunk_query_count = query_chunk.len() / dimension; + self.record_query_chunk(chunk_query_count); + let candidate_sets = self.exhaustive_filtered_candidate_nodes_batch( + query_chunk, + matching_nodes, + candidate_limit, + )?; + self.last_search_stats.filtered_exhaustive_queries = self + .last_search_stats + .filtered_exhaustive_queries + .saturating_add(chunk_query_count); + self.last_search_stats.pq_distance_evaluations = self + .last_search_stats + .pq_distance_evaluations + .saturating_add(chunk_query_count.saturating_mul(matching_count)); + self.last_search_stats.pq_code_loads = + self.last_search_stats.pq_code_loads.saturating_add( + chunk_query_count + .div_ceil(tile_size) + .saturating_mul(matching_count), + ); + let partition = candidate_sets.into_iter().enumerate().collect(); + let (chunk_ids, chunk_distances) = + self.rerank_candidate_batch_streaming(query_chunk, top_k, vec![partition])?; + ids.extend(chunk_ids); + distances.extend(chunk_distances); + } + Ok((ids, distances)) + } + + fn exhaustive_filtered_candidate_nodes_batch( + &self, + queries: &[f32], + matching_nodes: &RoaringBitmap, + candidate_limit: usize, + ) -> io::Result>> { + let dimension = self.header.dimension as usize; + let query_count = queries.len() / dimension; + let pq_m = self.header.pq_m as usize; + let pq = self.pq()?; + let pq_ksub = pq.ksub; + let pq_code_size = pq.code_size(); + let pq_codes = self.pq_codes()?; + let metric = self.header.metric_type(); + let tile_size = filtered_pq_query_tile_size(pq_m, pq_ksub); + let tile_count = query_count.div_ceil(tile_size); + let mut tiles = (0..tile_count) + .into_par_iter() + .map(|tile_index| { + let query_start = tile_index * tile_size; + let query_end = (query_start + tile_size).min(query_count); + let tile_query_count = query_end - query_start; + let mut distance_tables = vec![0.0f32; tile_query_count * pq_m * pq_ksub]; + for tile_query_index in 0..tile_query_count { + let query_index = query_start + tile_query_index; + let query = &queries[query_index * dimension..(query_index + 1) * dimension]; + let table_start = tile_query_index * pq_m * pq_ksub; + pq.compute_distance_table( + query, + metric, + &mut distance_tables[table_start..table_start + pq_m * pq_ksub], + ); + } + let mut heaps = (0..tile_query_count) + .map(|_| BinaryHeap::new()) + .collect::>>(); + for heap in &mut heaps { + heap.try_reserve(candidate_limit.min(1024)).map_err(|_| { + invalid_input("DiskANN filtered candidate allocation failed") + })?; + } + for node in matching_nodes.iter() { + let node = node as usize; + let code_start = node + .checked_mul(pq_code_size) + .ok_or_else(|| invalid_data("DiskANN PQ code offset overflows"))?; + let codes = pq_codes + .get(code_start..code_start + pq_code_size) + .ok_or_else(|| invalid_data("DiskANN PQ codes are truncated"))?; + for tile_query_index in 0..tile_query_count { + let table_start = tile_query_index * pq_m * pq_ksub; + push_bounded_candidate( + &mut heaps[tile_query_index], + SearchCandidate { + node, + distance: pq.distance_from_table( + &distance_tables[table_start..table_start + pq_m * pq_ksub], + codes, + ), + }, + candidate_limit, + )?; + } + } + let candidates = heaps + .into_iter() + .map(|heap| { + let mut candidates = heap.into_vec(); + sort_candidates(&mut candidates); + candidates + .into_iter() + .map(|candidate| candidate.node) + .collect::>() + }) + .collect::>(); + Ok::<_, io::Error>((query_start, candidates)) + }) + .collect::>>()?; + tiles.sort_unstable_by_key(|(query_start, _)| *query_start); + Ok(tiles + .into_iter() + .flat_map(|(_, candidates)| candidates) + .collect()) + } + + fn search_batch_direct_serial( + &mut self, + queries: &[f32], + top_k: usize, + l_search: usize, + ) -> io::Result<(Vec, Vec)> { + let dimension = self.header.dimension as usize; + let query_count = queries.len() / dimension; + let mut aggregate = DiskAnnSearchStats { + query_count, + query_chunks: usize::from(query_count != 0), + max_queries_per_chunk: query_count, + ..DiskAnnSearchStats::default() + }; + let mut ids = Vec::with_capacity(query_count.saturating_mul(top_k)); + let mut distances = Vec::with_capacity(query_count.saturating_mul(top_k)); + for query in queries.chunks_exact(dimension) { + let (query_ids, query_distances) = self.search_preprocessed(query, top_k, l_search)?; + aggregate.merge_complete_query(self.last_search_stats); + ids.extend(query_ids); + distances.extend(query_distances); + } + aggregate.parallel_session_queries = 0; + self.last_search_stats = aggregate; + Ok((ids, distances)) + } + + fn search_batch_with_matching_nodes_direct_serial( + &mut self, + queries: &[f32], + top_k: usize, + l_search: usize, + matching_nodes: &RoaringBitmap, + ) -> io::Result<(Vec, Vec)> { + let dimension = self.header.dimension as usize; + let query_count = queries.len() / dimension; + let mut aggregate = DiskAnnSearchStats { + query_count, + query_chunks: usize::from(query_count != 0), + max_queries_per_chunk: query_count, + ..DiskAnnSearchStats::default() + }; + let mut ids = Vec::with_capacity(query_count.saturating_mul(top_k)); + let mut distances = Vec::with_capacity(query_count.saturating_mul(top_k)); + for query in queries.chunks_exact(dimension) { + self.last_search_stats = DiskAnnSearchStats { + query_count: 1, + ..DiskAnnSearchStats::default() + }; + let (query_ids, query_distances) = + self.search_with_matching_nodes(query, top_k, l_search, matching_nodes)?; + aggregate.merge_complete_query(self.last_search_stats); + ids.extend(query_ids); + distances.extend(query_distances); + } + aggregate.parallel_session_queries = 0; + self.last_search_stats = aggregate; + Ok((ids, distances)) + } + + fn search_batch_serial( + &mut self, + queries: &[f32], + top_k: usize, + l_search: usize, + ) -> io::Result<(Vec, Vec)> { + let dimension = self.header.dimension as usize; + let query_count = queries.len() / dimension; + let mut ids = Vec::with_capacity(query_count.saturating_mul(top_k)); + let mut distances = Vec::with_capacity(query_count.saturating_mul(top_k)); + for query_chunk in queries.chunks(BATCH_QUERY_CHUNK_SIZE * dimension) { + let chunk_query_count = query_chunk.len() / dimension; + self.record_query_chunk(chunk_query_count); + let mut partition = Vec::with_capacity(chunk_query_count); + for (query_index, query) in query_chunk.chunks_exact(dimension).enumerate() { + let candidates = + self.generate_unfiltered_candidate_nodes(query, top_k, l_search)?; + partition.push((query_index, candidates)); + } + let (chunk_ids, chunk_distances) = + self.rerank_candidate_batch_streaming(query_chunk, top_k, vec![partition])?; + ids.extend(chunk_ids); + distances.extend(chunk_distances); + } + Ok((ids, distances)) + } + + fn search_batch_with_matching_nodes_serial( + &mut self, + queries: &[f32], + top_k: usize, + l_search: usize, + matching_nodes: &RoaringBitmap, + ) -> io::Result<(Vec, Vec)> { + let dimension = self.header.dimension as usize; + let query_count = queries.len() / dimension; + let mut ids = Vec::with_capacity(query_count.saturating_mul(top_k)); + let mut distances = Vec::with_capacity(query_count.saturating_mul(top_k)); + for query_chunk in queries.chunks(BATCH_QUERY_CHUNK_SIZE * dimension) { + let chunk_query_count = query_chunk.len() / dimension; + self.record_query_chunk(chunk_query_count); + let mut partition = Vec::with_capacity(chunk_query_count); + for (query_index, query) in query_chunk.chunks_exact(dimension).enumerate() { + let candidates = + self.generate_filtered_candidates(query, top_k, l_search, matching_nodes)?; + partition.push(( + query_index, + candidates + .into_iter() + .map(|candidate| candidate.node) + .collect(), + )); + } + let (chunk_ids, chunk_distances) = + self.rerank_candidate_batch_streaming(query_chunk, top_k, vec![partition])?; + ids.extend(chunk_ids); + distances.extend(chunk_distances); + } + Ok((ids, distances)) + } + + fn record_query_chunk(&mut self, query_count: usize) { + self.last_search_stats.query_chunks = self.last_search_stats.query_chunks.saturating_add(1); + self.last_search_stats.max_queries_per_chunk = self + .last_search_stats + .max_queries_per_chunk + .max(query_count); + } + + fn generate_unfiltered_candidate_nodes( + &mut self, + query: &[f32], + top_k: usize, + l_search: usize, + ) -> io::Result> { + self.ensure_resident()?; + let search_list_size = resolve_diskann_l_search(top_k, l_search); + let mut scratch = std::mem::take(&mut self.query_scratch); + let result = (|| { + self.generate_graph_candidates( + query, + search_list_size, + self.read_plan().graph_beam_width, + &mut scratch, + )?; + let rerank_count = search_list_size + .min(top_k.saturating_mul(4).max(64)) + .min(scratch.candidates.len()); + if rerank_count == scratch.candidates.len() { + return Ok(scratch + .candidates + .iter() + .map(|candidate| candidate.node) + .collect()); + } + if self.header.is_interleaved() { + let planner = + ReadWindowPlanner::new(self.read_plan(), self.header.sections.adjacency); + expand_rerank_candidates_within_seed_windows( + &scratch.candidates, + rerank_count, + |node| { + let page = self.adjacency_locator(node)?.page_index as usize; + planner + .window_for_logical_page(page) + .map(|window| window.offset) + .ok_or_else(|| invalid_data("DiskANN adjacency page is out of range")) + }, + &mut scratch.rerank_windows, + &mut scratch.rerank_candidates, + )?; + return Ok(scratch + .rerank_candidates + .iter() + .map(|candidate| candidate.node) + .collect()); + } + let planner = VectorWindowPlanner::new( + self.read_plan(), + self.header.sections.vectors, + self.header.vector_record_size as usize, + )?; + expand_rerank_candidates_within_seed_windows( + &scratch.candidates, + rerank_count, + |node| { + planner + .window_for_node(node) + .map(|window| window.offset) + .ok_or_else(|| invalid_data("DiskANN raw-vector record is out of range")) + }, + &mut scratch.rerank_windows, + &mut scratch.rerank_candidates, + )?; + Ok(scratch + .rerank_candidates + .iter() + .map(|candidate| candidate.node) + .collect()) + })(); + scratch.recycle_adjacency_windows(); + self.query_scratch = scratch; + result + } + + pub fn search( + &mut self, + query: &[f32], + top_k: usize, + l_search: usize, + ) -> io::Result<(Vec, Vec)> { + self.last_search_stats = DiskAnnSearchStats { + query_count: 1, + ..DiskAnnSearchStats::default() + }; + let dimension = self.header.dimension as usize; + if query.len() != dimension { + return Err(invalid_input(format!( + "query dimension mismatch: expected {}, got {}", + dimension, + query.len() + ))); + } + if query.iter().any(|value| !value.is_finite()) { + return Err(invalid_input("query values must be finite")); + } + let processed_query = self.preprocess_queries(query, 1); + self.search_preprocessed(processed_query.as_ref(), top_k, l_search) + } + + fn search_preprocessed( + &mut self, + query: &[f32], + top_k: usize, + l_search: usize, + ) -> io::Result<(Vec, Vec)> { + self.last_search_stats = DiskAnnSearchStats { + query_count: 1, + ..DiskAnnSearchStats::default() + }; + if top_k == 0 { + return Ok((Vec::new(), Vec::new())); + } + self.ensure_resident()?; + let search_list_size = resolve_diskann_l_search(top_k, l_search); + let mut scratch = std::mem::take(&mut self.query_scratch); + let result = (|| { + self.generate_graph_candidates( + query, + search_list_size, + self.read_plan().graph_beam_width, + &mut scratch, + )?; + + let rerank_count = search_list_size + .min(top_k.saturating_mul(4).max(64)) + .min(scratch.candidates.len()); + if rerank_count == scratch.candidates.len() { + if self.header.is_interleaved() { + return self.rerank_interleaved( + query, + &scratch.candidates, + top_k, + &mut scratch.adjacency_windows, + &mut scratch.loaded_adjacency_pages, + &mut scratch.window_buffers, + ); + } + return self.rerank( + query, + &scratch.candidates, + top_k, + &mut scratch.vector_windows, + &mut scratch.window_buffers, + ); + } + if self.header.is_interleaved() { + let planner = + ReadWindowPlanner::new(self.read_plan(), self.header.sections.adjacency); + expand_rerank_candidates_within_seed_windows( + &scratch.candidates, + rerank_count, + |node| { + let page = self.adjacency_locator(node)?.page_index as usize; + planner + .window_for_logical_page(page) + .map(|window| window.offset) + .ok_or_else(|| invalid_data("DiskANN adjacency page is out of range")) + }, + &mut scratch.rerank_windows, + &mut scratch.rerank_candidates, + )?; + return self.rerank_interleaved( + query, + &scratch.rerank_candidates, + top_k, + &mut scratch.adjacency_windows, + &mut scratch.loaded_adjacency_pages, + &mut scratch.window_buffers, + ); + } + let planner = VectorWindowPlanner::new( + self.read_plan(), + self.header.sections.vectors, + self.header.vector_record_size as usize, + )?; + expand_rerank_candidates_within_seed_windows( + &scratch.candidates, + rerank_count, + |node| { + planner + .window_for_node(node) + .map(|window| window.offset) + .ok_or_else(|| invalid_data("DiskANN raw-vector record is out of range")) + }, + &mut scratch.rerank_windows, + &mut scratch.rerank_candidates, + )?; + self.rerank( + query, + &scratch.rerank_candidates, + top_k, + &mut scratch.vector_windows, + &mut scratch.window_buffers, + ) + })(); + scratch.recycle_adjacency_windows(); + self.query_scratch = scratch; + result + } + + fn generate_graph_candidates( + &mut self, + query: &[f32], + search_list_size: usize, + beam_width: usize, + scratch: &mut DiskAnnQueryScratch, + ) -> io::Result<()> { + let vector_count = self.header.vector_count as usize; + let search_list_size = search_list_size.min(vector_count); + scratch.begin_graph_search( + vector_count, + search_list_size, + self.header.max_degree as usize, + )?; + let pq = self.pq()?; + let distance_table_len = pq.m * pq.ksub; + pq.compute_distance_table( + query, + self.header.metric_type(), + scratch.prepare_distance_table(distance_table_len), + ); + + let entry_node = self.header.entry_node as usize; + scratch.mark_visited(entry_node); + scratch.insert_graph_candidate( + SearchCandidate { + node: entry_node, + distance: self.pq_distance(entry_node, &scratch.distance_table)?, + }, + search_list_size, + )?; + let mut expanded_count = 0usize; + while expanded_count < search_list_size { + scratch.select_round(beam_width.min(search_list_size - expanded_count)); + if scratch.selected_nodes.is_empty() { + break; + } + self.load_adjacency_pages( + &scratch.selected_nodes, + &mut scratch.adjacency_windows, + &mut scratch.loaded_adjacency_pages, + &mut scratch.window_buffers, + )?; + expanded_count += scratch.selected_nodes.len(); + for selected_node_index in 0..scratch.selected_nodes.len() { + let node = scratch.selected_nodes[selected_node_index]; + self.decode_adjacency_neighbors( + node, + &scratch.adjacency_windows, + &mut scratch.neighbor_buffer, + )?; + let mut retained_neighbors = 0; + for neighbor_index in 0..scratch.neighbor_buffer.len() { + let neighbor = scratch.neighbor_buffer[neighbor_index] as usize; + if !scratch.mark_visited(neighbor) { + continue; + } + scratch.neighbor_buffer[retained_neighbors] = neighbor as u32; + retained_neighbors += 1; + } + scratch.neighbor_buffer.truncate(retained_neighbors); + score_pq_neighbors( + &scratch.distance_table, + self.pq_codes()?, + self.header.pq_m as usize, + self.header.pq_bits as usize, + &scratch.neighbor_buffer, + &mut scratch.scored_neighbors, + )?; + for scored_index in 0..scratch.scored_neighbors.len() { + let candidate = scratch.scored_neighbors[scored_index]; + scratch.insert_graph_candidate(candidate, search_list_size)?; + } + } + let evictions = scratch.adjacency_windows.trim( + &mut scratch.window_buffers, + QUERY_ADJACENCY_WINDOW_LIMIT_BYTES, + ); + self.last_search_stats.query_adjacency_cache_evictions = self + .last_search_stats + .query_adjacency_cache_evictions + .saturating_add(evictions); + } + scratch.finish_graph_candidates(); + Ok(()) + } + + pub fn search_with_roaring_filter( + &mut self, + query: &[f32], + top_k: usize, + l_search: usize, + roaring_filter_bytes: &[u8], + ) -> io::Result<(Vec, Vec)> { + let dimension = self.header.dimension as usize; + if query.len() != dimension { + return Err(invalid_input(format!( + "query dimension mismatch: expected {}, got {}", + dimension, + query.len() + ))); + } + if query.iter().any(|value| !value.is_finite()) { + return Err(invalid_input("query values must be finite")); + } + let processed_query = self.preprocess_queries(query, 1); + let query = processed_query.as_ref(); + let filter = decode_roaring_filter(roaring_filter_bytes)?; + self.search_with_decoded_roaring_filter(query, top_k, l_search, &filter) + } + + fn search_with_decoded_roaring_filter( + &mut self, + query: &[f32], + top_k: usize, + l_search: usize, + filter: &RoaringTreemap, + ) -> io::Result<(Vec, Vec)> { + self.last_search_stats = DiskAnnSearchStats { + query_count: 1, + ..DiskAnnSearchStats::default() + }; + if top_k == 0 { + return Ok((Vec::new(), Vec::new())); + } + if filter.is_empty() { + return Ok((vec![-1; top_k], vec![f32::MAX; top_k])); + } + let matching_nodes = self.matching_nodes_for_filter(filter)?; + if matching_nodes.is_empty() { + return Ok((vec![-1; top_k], vec![f32::MAX; top_k])); + } + self.search_with_matching_nodes(query, top_k, l_search, &matching_nodes) + } + + fn exhaustive_filtered_candidates( + &mut self, + query: &[f32], + matching_nodes: &RoaringBitmap, + candidate_limit: usize, + ) -> io::Result> { + let matching_count = usize::try_from(matching_nodes.len()).unwrap_or(usize::MAX); + self.last_search_stats.filtered_exhaustive_queries = self + .last_search_stats + .filtered_exhaustive_queries + .saturating_add(1); + self.last_search_stats.pq_distance_evaluations = self + .last_search_stats + .pq_distance_evaluations + .saturating_add(matching_count); + self.last_search_stats.pq_code_loads = self + .last_search_stats + .pq_code_loads + .saturating_add(matching_count); + let mut scratch = std::mem::take(&mut self.query_scratch); + let result = (|| { + scratch.begin_rerank(); + let pq = self.pq()?; + let distance_table_len = pq.m * pq.ksub; + pq.compute_distance_table( + query, + self.header.metric_type(), + scratch.prepare_distance_table(distance_table_len), + ); + let pq_codes = self.pq_codes()?; + let pq_m = self.header.pq_m as usize; + let pq_bits = self.header.pq_bits as usize; + let mut candidates = BinaryHeap::new(); + candidates + .try_reserve(candidate_limit.min(1024)) + .map_err(|_| invalid_input("DiskANN filtered candidate allocation failed"))?; + for node in matching_nodes.iter() { + scratch.neighbor_buffer.push(node); + if scratch.neighbor_buffer.len() == FILTERED_SINGLE_PQ_NODE_CHUNK_SIZE { + score_filtered_candidate_chunk( + &scratch.distance_table, + pq_codes, + pq_m, + pq_bits, + &scratch.neighbor_buffer, + &mut scratch.scored_neighbors, + &mut candidates, + candidate_limit, + )?; + scratch.neighbor_buffer.clear(); + } + } + if !scratch.neighbor_buffer.is_empty() { + score_filtered_candidate_chunk( + &scratch.distance_table, + pq_codes, + pq_m, + pq_bits, + &scratch.neighbor_buffer, + &mut scratch.scored_neighbors, + &mut candidates, + candidate_limit, + )?; + scratch.neighbor_buffer.clear(); + } + let mut candidates = candidates.into_vec(); + sort_candidates(&mut candidates); + Ok(candidates) + })(); + self.query_scratch = scratch; + result + } + + fn matching_nodes_for_filter(&mut self, filter: &RoaringTreemap) -> io::Result { + self.ensure_resident()?; + if use_row_id_order(filter.len(), self.header.vector_count as usize) { + if let Some(row_id_order) = self.ensure_row_id_order()? { + let matching_ranges = + matching_ranges_from_row_id_order(&row_id_order, filter, |node| { + self.row_id(node) + })?; + let mut matching = RoaringBitmap::new(); + for range in matching_ranges { + for &node in &row_id_order[range] { + matching.insert(node); + } + } + return Ok(matching); + } + } + matching_nodes_from_sequential_row_ids(filter, |visitor| self.try_for_each_row_id(visitor)) + } + + fn search_with_matching_nodes( + &mut self, + query: &[f32], + top_k: usize, + l_search: usize, + matching_nodes: &RoaringBitmap, + ) -> io::Result<(Vec, Vec)> { + if top_k == 0 { + return Ok((Vec::new(), Vec::new())); + } + let candidates = + self.generate_filtered_candidates(query, top_k, l_search, matching_nodes)?; + self.rerank_with_query_scratch(query, &candidates, top_k) + } + + fn generate_filtered_candidates( + &mut self, + query: &[f32], + top_k: usize, + l_search: usize, + matching_nodes: &RoaringBitmap, + ) -> io::Result> { + let matching_count = usize::try_from(matching_nodes.len()).unwrap_or(usize::MAX); + let strategy = select_filtered_candidate_strategy( + self.header.vector_count as usize, + matching_count, + top_k, + l_search, + self.header.max_degree as usize, + self.read_plan(), + self.adjacency_fully_preloaded(), + ); + match strategy { + FilteredCandidateStrategy::Exhaustive { target_candidates } => { + self.exhaustive_filtered_candidates(query, matching_nodes, target_candidates) + } + FilteredCandidateStrategy::Graph { + target_candidates, + search_list_size, + } => { + self.last_search_stats.filtered_graph_queries = self + .last_search_stats + .filtered_graph_queries + .saturating_add(1); + let mut scratch = std::mem::take(&mut self.query_scratch); + let graph_candidates = (|| { + self.generate_graph_candidates( + query, + search_list_size, + self.options() + .read_tier + .read_plan() + .filtered_graph_beam_width, + &mut scratch, + )?; + Ok::<_, io::Error>(post_filter_graph_candidates( + &scratch.candidates, + matching_nodes, + target_candidates, + )) + })(); + scratch.recycle_adjacency_windows(); + self.query_scratch = scratch; + if let Some(candidates) = graph_candidates? { + Ok(candidates) + } else { + self.last_search_stats.filtered_graph_fallbacks = self + .last_search_stats + .filtered_graph_fallbacks + .saturating_add(1); + self.exhaustive_filtered_candidates(query, matching_nodes, target_candidates) + } + } + } + } + + fn pq_distance(&self, node: usize, distance_table: &[f32]) -> io::Result { + let pq = self.pq()?; + let code_size = pq.code_size(); + let start = node + .checked_mul(code_size) + .ok_or_else(|| invalid_data("DiskANN PQ code offset overflows"))?; + let end = start + code_size; + let codes = self + .pq_codes()? + .get(start..end) + .ok_or_else(|| invalid_data("DiskANN PQ codes are truncated"))?; + Ok(pq.distance_from_table(distance_table, codes)) + } + + fn rerank( + &mut self, + query: &[f32], + candidates: &[SearchCandidate], + top_k: usize, + window_cache: &mut VectorWindowCache, + window_buffers: &mut WindowBufferPool, + ) -> io::Result<(Vec, Vec)> { + let record_size = self.header.vector_record_size as usize; + let encoding = self.header.raw_vector_encoding(); + let planner = + VectorWindowPlanner::new(self.read_plan(), self.header.sections.vectors, record_size)?; + let windows = planner.plan_nodes(candidates.iter().map(|candidate| candidate.node)); + self.last_search_stats.rerank_candidate_references = self + .last_search_stats + .rerank_candidate_references + .saturating_add(candidates.len()); + self.last_search_stats.rerank_unique_windows = self + .last_search_stats + .rerank_unique_windows + .saturating_add(windows.len()); + self.last_search_stats.rerank_chunks = self + .last_search_stats + .rerank_chunks + .saturating_add(usize::from(!windows.is_empty())); + let raw_vector_cache_bytes = self.options().raw_vector_cache_bytes; + let (retain_vector_windows, preparation_evictions) = prepare_vector_window_cache( + &windows, + window_cache, + window_buffers, + raw_vector_cache_bytes, + ); + let distance_kernel = selected_raw_vector_distance_kernel(query.len()); + let metric = self.header.metric_type(); + self.last_search_stats.raw_vector_cache_evictions = self + .last_search_stats + .raw_vector_cache_evictions + .saturating_add(preparation_evictions); + let cache_load = self.load_vector_windows(&windows, window_cache, window_buffers)?; + self.last_search_stats.raw_vector_cache_hits = self + .last_search_stats + .raw_vector_cache_hits + .saturating_add(cache_load.hits); + self.last_search_stats.raw_vector_cache_misses = self + .last_search_stats + .raw_vector_cache_misses + .saturating_add(cache_load.misses); + self.last_search_stats.raw_vector_cache_evictions = self + .last_search_stats + .raw_vector_cache_evictions + .saturating_add(cache_load.evictions); + let result = (|| { + let mut exact = BinaryHeap::new(); + exact + .try_reserve(top_k.min(candidates.len())) + .map_err(|_| invalid_input("DiskANN exact result allocation failed"))?; + for candidate in candidates { + let window = planner + .window_for_node(candidate.node) + .ok_or_else(|| invalid_data("DiskANN raw-vector record is out of range"))?; + let payload = window_cache + .get(&window.offset) + .ok_or_else(|| invalid_data("DiskANN vector window is not loaded"))?; + let record = planner.record(window, payload, candidate.node)?; + let distance = + raw_vector_distance(query, record, encoding, metric, distance_kernel)?; + let row_id = self.row_id(candidate.node)?; + push_bounded_exact_result( + &mut exact, + ExactSearchResult { row_id, distance }, + top_k, + )?; + } + let exact = exact.into_sorted_vec(); + let mut ids = exact.iter().map(|result| result.row_id).collect::>(); + let mut distances = exact + .iter() + .map(|result| result.distance) + .collect::>(); + ids.resize(top_k, -1); + distances.resize(top_k, f32::MAX); + Ok((ids, distances)) + })(); + if retain_vector_windows && result.is_ok() { + window_cache.touch_windows(&windows); + let evictions = window_cache.trim(window_buffers, raw_vector_cache_bytes); + self.last_search_stats.raw_vector_cache_evictions = self + .last_search_stats + .raw_vector_cache_evictions + .saturating_add(evictions); + } else { + window_cache.recycle(window_buffers); + } + result + } + + fn rerank_interleaved( + &mut self, + query: &[f32], + candidates: &[SearchCandidate], + top_k: usize, + adjacency_windows: &mut AdjacencyWindowCache, + loaded_pages: &mut HashSet, + window_buffers: &mut WindowBufferPool, + ) -> io::Result<(Vec, Vec)> { + let nodes = candidates + .iter() + .map(|candidate| candidate.node) + .collect::>(); + self.load_adjacency_pages(&nodes, adjacency_windows, loaded_pages, window_buffers)?; + let planner = ReadWindowPlanner::new(self.read_plan(), self.header.sections.adjacency); + let windows = nodes + .iter() + .map(|&node| { + let page = self.adjacency_locator(node)?.page_index as usize; + planner + .window_for_logical_page(page) + .map(|window| window.offset) + .ok_or_else(|| invalid_data("DiskANN adjacency page is out of range")) + }) + .collect::>>()?; + self.last_search_stats.rerank_candidate_references = self + .last_search_stats + .rerank_candidate_references + .saturating_add(candidates.len()); + self.last_search_stats.rerank_unique_windows = self + .last_search_stats + .rerank_unique_windows + .saturating_add(windows.len()); + self.last_search_stats.rerank_chunks = self + .last_search_stats + .rerank_chunks + .saturating_add(usize::from(!windows.is_empty())); + let record_size = self.header.vector_record_size as usize; + let encoding = self.header.raw_vector_encoding(); + let distance_kernel = selected_raw_vector_distance_kernel(query.len()); + let metric = self.header.metric_type(); + let mut exact = BinaryHeap::new(); + exact + .try_reserve(top_k.min(candidates.len())) + .map_err(|_| invalid_input("DiskANN exact result allocation failed"))?; + for candidate in candidates { + let locator = self.adjacency_locator(candidate.node)?; + let page_index = locator.page_index as usize; + let window = planner + .window_for_logical_page(page_index) + .ok_or_else(|| invalid_data("DiskANN adjacency page is out of range"))?; + let payload = if let Some(hot) = self.hot_adjacency_window(window.offset, window.length) + { + hot + } else { + adjacency_windows + .get(&window.offset) + .map(WindowPayload::as_slice) + .ok_or_else(|| invalid_data("DiskANN adjacency rerank window is not loaded"))? + }; + let page_offset = self.header.sections.adjacency.offset + + page_index as u64 * DISKANN_PAGE_SIZE as u64 + - window.offset; + let record_offset = (page_offset as usize) + .checked_add(locator.byte_offset as usize) + .and_then(|offset| offset.checked_sub(record_size)) + .ok_or_else(|| invalid_data("DiskANN interleaved vector offset underflows"))?; + let record = payload + .get(record_offset..record_offset + record_size) + .ok_or_else(|| invalid_data("DiskANN interleaved raw vector is truncated"))?; + let distance = raw_vector_distance(query, record, encoding, metric, distance_kernel)?; + push_bounded_exact_result( + &mut exact, + ExactSearchResult { + row_id: self.row_id(candidate.node)?, + distance, + }, + top_k, + )?; + } + let exact = exact.into_sorted_vec(); + let mut ids = exact.iter().map(|result| result.row_id).collect::>(); + let mut distances = exact + .iter() + .map(|result| result.distance) + .collect::>(); + ids.resize(top_k, -1); + distances.resize(top_k, f32::MAX); + Ok((ids, distances)) + } + + fn rerank_with_query_scratch( + &mut self, + query: &[f32], + candidates: &[SearchCandidate], + top_k: usize, + ) -> io::Result<(Vec, Vec)> { + let mut scratch = std::mem::take(&mut self.query_scratch); + let result = { + scratch.begin_rerank(); + if self.header.is_interleaved() { + self.rerank_interleaved( + query, + candidates, + top_k, + &mut scratch.adjacency_windows, + &mut scratch.loaded_adjacency_pages, + &mut scratch.window_buffers, + ) + } else { + self.rerank( + query, + candidates, + top_k, + &mut scratch.vector_windows, + &mut scratch.window_buffers, + ) + } + }; + scratch.recycle_adjacency_windows(); + self.query_scratch = scratch; + result + } + + fn rerank_candidate_batch_streaming( + &mut self, + queries: &[f32], + top_k: usize, + partitions: Vec, + ) -> io::Result<(Vec, Vec)> { + if self.header.is_interleaved() { + return Err(invalid_data( + "DiskANN interleaved rerank must run inside a search session", + )); + } + let dimension = self.header.dimension as usize; + let query_count = queries.len() / dimension; + let distance_kernel = selected_raw_vector_distance_kernel(dimension); + let metric = self.header.metric_type(); + let mut candidate_sets = (0..query_count).map(|_| None).collect::>(); + for (query_index, candidates) in partitions.into_iter().flatten() { + let slot = candidate_sets + .get_mut(query_index) + .ok_or_else(|| invalid_data("DiskANN filtered batch query index is invalid"))?; + if slot.replace(candidates).is_some() { + return Err(invalid_data( + "DiskANN filtered batch contains duplicate query results", + )); + } + } + if candidate_sets.iter().any(Option::is_none) { + return Err(invalid_data( + "DiskANN filtered batch is missing query candidates", + )); + } + + let record_size = self.header.vector_record_size as usize; + let encoding = self.header.raw_vector_encoding(); + let planner = + VectorWindowPlanner::new(self.read_plan(), self.header.sections.vectors, record_size)?; + let mut grouped = HashMap::)>::new(); + for (query_index, candidates) in candidate_sets.into_iter().enumerate() { + for node in candidates.expect("validated DiskANN batch candidates") { + let window = planner + .window_for_node(node) + .ok_or_else(|| invalid_data("DiskANN raw-vector record is out of range"))?; + grouped + .entry(window.offset) + .or_insert_with(|| (window, Vec::new())) + .1 + .push((query_index, node)); + } + } + let mut window_groups = grouped.into_values().collect::>(); + window_groups.sort_unstable_by_key(|(window, _)| window.offset); + let windows = window_groups + .iter() + .map(|(window, _)| *window) + .collect::>(); + let chunks = plan_streaming_window_chunks(&windows); + self.last_search_stats.rerank_candidate_references = self + .last_search_stats + .rerank_candidate_references + .saturating_add( + window_groups + .iter() + .map(|(_, references)| references.len()) + .sum::(), + ); + self.last_search_stats.rerank_unique_windows = self + .last_search_stats + .rerank_unique_windows + .saturating_add(windows.len()); + self.last_search_stats.rerank_chunks = self + .last_search_stats + .rerank_chunks + .saturating_add(chunks.len()); + let mut exact_heaps = (0..query_count) + .map(|_| BinaryHeap::new()) + .collect::>>(); + let raw_vector_cache_bytes = self.options().raw_vector_cache_bytes; + let mut scratch = std::mem::take(&mut self.query_scratch); + scratch.begin_rerank(); + let result = (|| { + for chunk in chunks { + let chunk_windows = &windows[chunk.clone()]; + let cache_load = self.load_vector_windows( + chunk_windows, + &mut scratch.vector_windows, + &mut scratch.window_buffers, + )?; + self.last_search_stats.raw_vector_cache_hits = self + .last_search_stats + .raw_vector_cache_hits + .saturating_add(cache_load.hits); + self.last_search_stats.raw_vector_cache_misses = self + .last_search_stats + .raw_vector_cache_misses + .saturating_add(cache_load.misses); + self.last_search_stats.raw_vector_cache_evictions = self + .last_search_stats + .raw_vector_cache_evictions + .saturating_add(cache_load.evictions); + let chunk_reference_count = window_groups[chunk.clone()] + .iter() + .map(|(_, references)| references.len()) + .sum::(); + let parallel = rayon::current_num_threads() > 1 + && chunk_reference_count.saturating_mul(dimension) + >= PARALLEL_EXACT_RERANK_MIN_COMPONENTS; + if parallel { + self.last_search_stats.parallel_exact_rerank_chunks = self + .last_search_stats + .parallel_exact_rerank_chunks + .saturating_add(1); + self.last_search_stats.parallel_exact_rerank_references = self + .last_search_stats + .parallel_exact_rerank_references + .saturating_add(chunk_reference_count); + let mut rerank_references = Vec::with_capacity(chunk_reference_count); + for (window, references) in &window_groups[chunk.clone()] { + let payload = scratch + .vector_windows + .get(&window.offset) + .ok_or_else(|| invalid_data("DiskANN vector window is not loaded"))?; + for &(query_index, node) in references { + rerank_references.push(ExactRerankReference { + query_index, + row_id: self.row_id(node)?, + record: planner.record(*window, payload, node)?, + }); + } + } + let exact_results = rerank_references + .par_iter() + .map(|reference| { + let query = &queries[reference.query_index * dimension + ..(reference.query_index + 1) * dimension]; + Ok::<_, io::Error>(( + reference.query_index, + ExactSearchResult { + row_id: reference.row_id, + distance: raw_vector_distance( + query, + reference.record, + encoding, + metric, + distance_kernel, + )?, + }, + )) + }) + .collect::>>()?; + for (query_index, exact) in exact_results { + push_bounded_exact_result(&mut exact_heaps[query_index], exact, top_k)?; + } + } else { + for (window, references) in &window_groups[chunk.clone()] { + let payload = scratch + .vector_windows + .get(&window.offset) + .ok_or_else(|| invalid_data("DiskANN vector window is not loaded"))?; + for &(query_index, node) in references { + let query = + &queries[query_index * dimension..(query_index + 1) * dimension]; + push_bounded_exact_result( + &mut exact_heaps[query_index], + ExactSearchResult { + row_id: self.row_id(node)?, + distance: raw_vector_distance( + query, + planner.record(*window, payload, node)?, + encoding, + metric, + distance_kernel, + )?, + }, + top_k, + )?; + } + } + } + scratch.vector_windows.touch_windows(chunk_windows); + let evictions = scratch + .vector_windows + .trim(&mut scratch.window_buffers, raw_vector_cache_bytes); + self.last_search_stats.raw_vector_cache_evictions = self + .last_search_stats + .raw_vector_cache_evictions + .saturating_add(evictions); + } + + let mut ids = Vec::with_capacity(query_count.saturating_mul(top_k)); + let mut distances = Vec::with_capacity(query_count.saturating_mul(top_k)); + for heap in exact_heaps { + let exact = heap.into_sorted_vec(); + ids.extend(exact.iter().map(|result| result.row_id)); + distances.extend(exact.iter().map(|result| result.distance)); + ids.resize(ids.len() + top_k - exact.len(), -1); + distances.resize(distances.len() + top_k - exact.len(), f32::MAX); + } + Ok((ids, distances)) + })(); + if result.is_err() { + scratch.recycle_window_caches(); + } else { + scratch.recycle_adjacency_windows(); + } + self.query_scratch = scratch; + result + } + + fn load_adjacency_pages( + &mut self, + nodes: &[usize], + window_cache: &mut AdjacencyWindowCache, + loaded_pages: &mut HashSet, + window_buffers: &mut WindowBufferPool, + ) -> io::Result<()> { + let planner = ReadWindowPlanner::new(self.read_plan(), self.header.sections.adjacency); + let mut pages = BTreeSet::new(); + let mut required_windows = Vec::with_capacity(nodes.len()); + for &node in nodes { + let page = self.adjacency_locator(node)?.page_index as usize; + let window = planner + .window_for_logical_page(page) + .ok_or_else(|| invalid_data("DiskANN adjacency page is out of range"))?; + let window_is_hot = self + .hot_adjacency_window(window.offset, window.length) + .is_some(); + if !window_is_hot { + required_windows.push(window); + } + let window_is_available = window_is_hot || window_cache.contains_key(&window.offset); + if !loaded_pages.contains(&page) || !window_is_available { + pages.insert(page); + } + } + if pages.is_empty() { + return Ok(()); + } + let windows = planner.plan_logical_pages(pages.iter().copied()); + let cold_windows = windows + .iter() + .copied() + .filter(|window| { + self.hot_adjacency_window(window.offset, window.length) + .is_none() + }) + .collect::>(); + let incoming_bytes = cold_windows + .iter() + .filter(|window| !window_cache.contains_key(&window.offset)) + .fold(0usize, |total, window| total.saturating_add(window.length)); + let preparation_evictions = prepare_adjacency_window_cache( + &required_windows, + incoming_bytes, + window_cache, + window_buffers, + ); + self.last_search_stats.query_adjacency_cache_evictions = self + .last_search_stats + .query_adjacency_cache_evictions + .saturating_add(preparation_evictions); + self.load_adjacency_windows(&cold_windows, window_cache, window_buffers)?; + self.last_search_stats.query_adjacency_cache_peak_bytes = self + .last_search_stats + .query_adjacency_cache_peak_bytes + .max(window_cache.retained_capacity()); + for page_index in pages { + let window = planner + .window_for_logical_page(page_index) + .ok_or_else(|| invalid_data("DiskANN adjacency page is out of range"))?; + let payload = if let Some(hot) = self.hot_adjacency_window(window.offset, window.length) + { + hot + } else { + window_cache + .get(&window.offset) + .map(WindowPayload::as_slice) + .ok_or_else(|| { + invalid_data("DiskANN adjacency validation window is not loaded") + })? + }; + let page_offset = self.header.sections.adjacency.offset + + page_index as u64 * DISKANN_PAGE_SIZE as u64 + - window.offset; + let page_start = page_offset as usize; + let page_end = page_start + DISKANN_PAGE_SIZE as usize; + self.validate_adjacency_page( + page_index, + payload + .get(page_start..page_end) + .ok_or_else(|| invalid_data("DiskANN adjacency page is truncated"))?, + )?; + loaded_pages.insert(page_index); + } + Ok(()) + } + + fn decode_adjacency_neighbors( + &self, + node: usize, + window_cache: &AdjacencyWindowCache, + neighbors: &mut Vec, + ) -> io::Result<()> { + let locator = self.adjacency_locator(node)?; + let page_index = locator.page_index as usize; + let planner = ReadWindowPlanner::new(self.read_plan(), self.header.sections.adjacency); + let window = planner + .window_for_logical_page(page_index) + .ok_or_else(|| invalid_data("DiskANN adjacency page is out of range"))?; + let payload = if let Some(hot) = self.hot_adjacency_window(window.offset, window.length) { + hot + } else { + window_cache + .get(&window.offset) + .map(WindowPayload::as_slice) + .ok_or_else(|| invalid_data("DiskANN adjacency decode window is not loaded"))? + }; + let page_offset = self.header.sections.adjacency.offset + + page_index as u64 * DISKANN_PAGE_SIZE as u64 + - window.offset; + let start = page_offset as usize + locator.byte_offset as usize; + let bytes = payload + .get(start..) + .ok_or_else(|| invalid_data("DiskANN adjacency list is truncated"))?; + decode_adjacency_list(bytes, locator.degree(), locator.encoding(), neighbors)?; + Ok(()) + } + + fn load_adjacency_windows( + &mut self, + windows: &[ReadWindow], + local_cache: &mut AdjacencyWindowCache, + window_buffers: &mut WindowBufferPool, + ) -> io::Result<()> { + let mut pending = windows + .iter() + .copied() + .filter(|window| !local_cache.contains_key(&window.offset)) + .collect::>(); + if pending.is_empty() { + return Ok(()); + } + if self.options().adjacency_cache_bytes == 0 { + self.last_search_stats.adjacency_cache_misses = self + .last_search_stats + .adjacency_cache_misses + .saturating_add(pending.len()); + let mut payloads = Vec::with_capacity(pending.len()); + for window in &pending { + payloads.push(window_buffers.take(window.length)?); + } + let read_result = { + let mut requests = pending + .iter() + .zip(payloads.iter_mut()) + .map(|(window, payload)| ReadRequest::new(window.offset, payload)) + .collect::>(); + self.pread_ranges(&mut requests) + }; + if let Err(error) = read_result { + for payload in payloads { + window_buffers.recycle(payload); + } + return Err(error); + } + for (window, payload) in pending.into_iter().zip(payloads) { + local_cache.insert(window.offset, WindowPayload::Owned(payload)); + } + return Ok(()); + } + + while !pending.is_empty() { + let mut reserved = Vec::new(); + let mut waiting = Vec::new(); + for window in &pending { + let (lookup, lock_metrics) = self + .adjacency_cache()? + .lookup_or_reserve(window.offset, window.length)?; + self.last_search_stats + .record_adjacency_cache_lock(lock_metrics); + match lookup { + SharedWindowCacheLookup::Hit(payload) => { + local_cache.insert(window.offset, WindowPayload::Shared(payload)); + self.last_search_stats.adjacency_cache_hits = self + .last_search_stats + .adjacency_cache_hits + .saturating_add(1); + } + SharedWindowCacheLookup::Reserved => { + reserved.push(*window); + self.last_search_stats.adjacency_cache_misses = self + .last_search_stats + .adjacency_cache_misses + .saturating_add(1); + } + SharedWindowCacheLookup::Loading => { + waiting.push(*window); + self.last_search_stats.adjacency_cache_waits = self + .last_search_stats + .adjacency_cache_waits + .saturating_add(1); + } + } + } + + if !reserved.is_empty() { + let mut payloads = Vec::with_capacity(reserved.len()); + for window in &reserved { + match window_buffers.take(window.length) { + Ok(payload) => payloads.push(payload), + Err(error) => { + let lock_metrics = self.adjacency_cache()?.cancel( + &reserved + .iter() + .map(|window| window.offset) + .collect::>(), + )?; + self.last_search_stats + .record_adjacency_cache_lock(lock_metrics); + for payload in payloads { + window_buffers.recycle(payload); + } + return Err(error); + } + } + } + let read_result = { + let mut requests = reserved + .iter() + .zip(payloads.iter_mut()) + .map(|(window, payload)| ReadRequest::new(window.offset, payload)) + .collect::>(); + self.pread_ranges(&mut requests) + }; + if let Err(error) = read_result { + let lock_metrics = self.adjacency_cache()?.cancel( + &reserved + .iter() + .map(|window| window.offset) + .collect::>(), + )?; + self.last_search_stats + .record_adjacency_cache_lock(lock_metrics); + for payload in payloads { + window_buffers.recycle(payload); + } + return Err(error); + } + for (window, payload) in reserved.into_iter().zip(payloads) { + let payload = share_window_payload(payload); + local_cache.insert(window.offset, WindowPayload::Shared(Arc::clone(&payload))); + let (evictions, lock_metrics) = + self.adjacency_cache()?.publish(window.offset, payload)?; + self.last_search_stats + .record_adjacency_cache_lock(lock_metrics); + self.last_search_stats.adjacency_cache_evictions = self + .last_search_stats + .adjacency_cache_evictions + .saturating_add(evictions); + } + } + + for window in waiting { + let (payload, lock_metrics) = self + .adjacency_cache()? + .wait_for(window.offset, window.length)?; + self.last_search_stats + .record_adjacency_cache_lock(lock_metrics); + if let Some(payload) = payload { + local_cache.insert(window.offset, WindowPayload::Shared(payload)); + self.last_search_stats.adjacency_cache_hits = self + .last_search_stats + .adjacency_cache_hits + .saturating_add(1); + } + } + pending.retain(|window| !local_cache.contains_key(&window.offset)); + } + Ok(()) + } + + fn load_vector_windows( + &mut self, + windows: &[ReadWindow], + cache: &mut VectorWindowCache, + window_buffers: &mut WindowBufferPool, + ) -> io::Result { + let mut pending = windows + .iter() + .copied() + .filter(|window| !cache.contains_key(&window.offset)) + .collect::>(); + let mut stats = VectorWindowLoadStats { + hits: windows.len().saturating_sub(pending.len()), + ..VectorWindowLoadStats::default() + }; + if pending.is_empty() { + return Ok(stats); + } + + if self.options().raw_vector_cache_bytes == 0 { + stats.misses = pending.len(); + let mut payloads = Vec::with_capacity(pending.len()); + for window in &pending { + payloads.push(window_buffers.take(window.length)?); + } + let read_result = { + let mut requests = pending + .iter() + .zip(payloads.iter_mut()) + .map(|(window, payload)| ReadRequest::new(window.offset, payload)) + .collect::>(); + self.pread_ranges(&mut requests) + }; + if let Err(error) = read_result { + for payload in payloads { + window_buffers.recycle(payload); + } + return Err(error); + } + for (window, payload) in pending.into_iter().zip(payloads) { + cache.insert(window.offset, payload); + } + return Ok(stats); + } + + while !pending.is_empty() { + let mut reserved = Vec::new(); + let mut waiting = Vec::new(); + for window in &pending { + match self + .raw_vector_cache()? + .lookup_or_reserve(window.offset, window.length)? + .0 + { + SharedWindowCacheLookup::Hit(payload) => { + cache.insert(window.offset, WindowPayload::Shared(payload)); + stats.hits = stats.hits.saturating_add(1); + } + SharedWindowCacheLookup::Reserved => { + reserved.push(*window); + stats.misses = stats.misses.saturating_add(1); + } + SharedWindowCacheLookup::Loading => waiting.push(*window), + } + } + + if !reserved.is_empty() { + let mut payloads = Vec::with_capacity(reserved.len()); + for window in &reserved { + match window_buffers.take(window.length) { + Ok(payload) => payloads.push(payload), + Err(error) => { + self.raw_vector_cache()?.cancel( + &reserved + .iter() + .map(|window| window.offset) + .collect::>(), + )?; + for payload in payloads { + window_buffers.recycle(payload); + } + return Err(error); + } + } + } + let read_result = { + let mut requests = reserved + .iter() + .zip(payloads.iter_mut()) + .map(|(window, payload)| ReadRequest::new(window.offset, payload)) + .collect::>(); + self.pread_ranges(&mut requests) + }; + if let Err(error) = read_result { + self.raw_vector_cache()?.cancel( + &reserved + .iter() + .map(|window| window.offset) + .collect::>(), + )?; + for payload in payloads { + window_buffers.recycle(payload); + } + return Err(error); + } + for (window, payload) in reserved.into_iter().zip(payloads) { + let payload = share_window_payload(payload); + cache.insert(window.offset, WindowPayload::Shared(Arc::clone(&payload))); + stats.evictions = stats.evictions.saturating_add( + self.raw_vector_cache()?.publish(window.offset, payload)?.0, + ); + } + } + + for window in waiting { + if let Some(payload) = self + .raw_vector_cache()? + .wait_for(window.offset, window.length)? + .0 + { + cache.insert(window.offset, WindowPayload::Shared(payload)); + stats.hits = stats.hits.saturating_add(1); + } + } + pending.retain(|window| !cache.contains_key(&window.offset)); + } + Ok(stats) + } +} + +fn sort_candidates(candidates: &mut [SearchCandidate]) { + candidates.sort_by(|left, right| { + left.distance + .total_cmp(&right.distance) + .then_with(|| left.node.cmp(&right.node)) + }); +} + +fn desired_filtered_candidate_count(matching_count: usize, top_k: usize) -> usize { + matching_count.min(top_k.saturating_mul(4).max(64)) +} + +fn resolve_diskann_l_search(top_k: usize, l_search: usize) -> usize { + let configured = if l_search == 0 { + top_k.saturating_mul(2).max(100) + } else { + l_search + }; + top_k.max(configured) +} + +fn topk_result_stability( + left_ids: &[i64], + left_distances: &[f32], + right_ids: &[i64], + right_distances: &[f32], + top_k: usize, +) -> f32 { + if top_k == 0 + || left_ids.len() != right_ids.len() + || left_ids.len() != left_distances.len() + || right_ids.len() != right_distances.len() + || !left_ids.len().is_multiple_of(top_k) + { + return 0.0; + } + let mut overlap = 0usize; + let mut denominator = 0usize; + for (((left_query_ids, left_query_distances), right_query_ids), right_query_distances) in + left_ids + .chunks_exact(top_k) + .zip(left_distances.chunks_exact(top_k)) + .zip(right_ids.chunks_exact(top_k)) + .zip(right_distances.chunks_exact(top_k)) + { + let mut right_counts = HashMap::::with_capacity(top_k); + for (&row_id, &distance) in right_query_ids.iter().zip(right_query_distances) { + if distance != f32::MAX { + *right_counts.entry(row_id).or_default() += 1; + } + } + for (&row_id, &distance) in left_query_ids.iter().zip(left_query_distances) { + if distance != f32::MAX { + denominator += 1; + if right_counts.get_mut(&row_id).is_some_and(|count| { + if *count == 0 { + false + } else { + *count -= 1; + true + } + }) { + overlap += 1; + } + } + } + } + if denominator == 0 { + 1.0 + } else { + overlap as f32 / denominator as f32 + } +} + +fn filtered_pq_query_tile_size(pq_m: usize, pq_ksub: usize) -> usize { + let table_bytes_per_query = pq_m + .saturating_mul(pq_ksub) + .saturating_mul(size_of::()) + .max(1); + (FILTERED_PQ_TILE_TABLE_LIMIT_BYTES / table_bytes_per_query) + .clamp(1, FILTERED_PQ_MAX_QUERY_TILE_SIZE) +} + +fn expand_rerank_candidates_within_seed_windows( + candidates: &[SearchCandidate], + seed_count: usize, + mut window_for_node: impl FnMut(usize) -> io::Result, + selected_windows: &mut HashSet, + selected: &mut Vec, +) -> io::Result<()> { + selected.clear(); + selected_windows.clear(); + if candidates.is_empty() || seed_count == 0 { + return Ok(()); + } + if seed_count >= candidates.len() { + selected + .try_reserve(candidates.len()) + .map_err(|_| invalid_input("DiskANN rerank candidate allocation failed"))?; + selected.extend_from_slice(candidates); + return Ok(()); + } + selected_windows + .try_reserve(seed_count) + .map_err(|_| invalid_input("DiskANN rerank window allocation failed"))?; + for candidate in candidates.iter().take(seed_count) { + selected_windows.insert(window_for_node(candidate.node)?); + } + selected + .try_reserve(candidates.len().min(seed_count.saturating_mul(2))) + .map_err(|_| invalid_input("DiskANN rerank candidate allocation failed"))?; + for candidate in candidates { + if selected_windows.contains(&window_for_node(candidate.node)?) { + selected.push(*candidate); + } + } + Ok(()) +} + +fn adaptive_filtered_search_list_size( + vector_count: usize, + matching_count: usize, + target_candidates: usize, + configured_l_search: usize, +) -> usize { + if vector_count == 0 || matching_count == 0 || target_candidates == 0 { + return 0; + } + let scaled_target = (target_candidates as u128) + .saturating_mul(vector_count as u128) + .div_ceil(matching_count as u128) + .saturating_mul(2) + .min(vector_count as u128) as usize; + configured_l_search.max(scaled_target).min(vector_count) +} + +#[allow(clippy::too_many_arguments)] +fn select_filtered_candidate_strategy( + vector_count: usize, + matching_count: usize, + top_k: usize, + l_search: usize, + max_degree: usize, + read_plan: ReadPlan, + adjacency_fully_preloaded: bool, +) -> FilteredCandidateStrategy { + let target_candidates = desired_filtered_candidate_count(matching_count, top_k); + let exhaustive = FilteredCandidateStrategy::Exhaustive { target_candidates }; + let configured_l_search = resolve_diskann_l_search(top_k, l_search); + if vector_count == 0 + || matching_count != vector_count + || configured_l_search < 200 + || (read_plan.window_bytes > 16 * 1024 && !adjacency_fully_preloaded) + { + return exhaustive; + } + let search_list_size = adaptive_filtered_search_list_size( + vector_count, + matching_count, + target_candidates, + configured_l_search, + ); + let graph_work = search_list_size + .saturating_mul(max_degree.saturating_add(1)) + .min(vector_count); + if graph_work > matching_count / 2 { + return exhaustive; + } + FilteredCandidateStrategy::Graph { + target_candidates, + search_list_size, + } +} + +fn post_filter_graph_candidates( + graph_candidates: &[SearchCandidate], + matching_nodes: &RoaringBitmap, + target_candidates: usize, +) -> Option> { + let mut filtered = Vec::with_capacity(target_candidates); + for candidate in graph_candidates { + let Ok(node) = u32::try_from(candidate.node) else { + continue; + }; + if matching_nodes.contains(node) { + filtered.push(*candidate); + if filtered.len() == target_candidates { + break; + } + } + } + (filtered.len() == target_candidates).then_some(filtered) +} + +fn plan_streaming_window_chunks(windows: &[ReadWindow]) -> Vec> { + let mut chunks = Vec::new(); + let mut start = 0usize; + while start < windows.len() { + let mut end = start; + let mut payload_bytes = 0usize; + while end < windows.len() && end - start < FILTERED_BATCH_RERANK_MAX_RANGES { + let next_bytes = payload_bytes.saturating_add(windows[end].length); + if end != start && next_bytes > FILTERED_BATCH_RERANK_MAX_BYTES { + break; + } + payload_bytes = next_bytes; + end += 1; + } + chunks.push(start..end); + start = end; + } + chunks +} + +fn use_row_id_order(filter_cardinality: u64, vector_count: usize) -> bool { + filter_cardinality != 0 && filter_cardinality <= (vector_count / 16) as u64 +} + +fn matching_nodes_from_sequential_row_ids( + filter: &RoaringTreemap, + visit_row_ids: impl FnOnce(&mut dyn FnMut(usize, i64) -> io::Result<()>) -> io::Result<()>, +) -> io::Result { + let mut matching = RoaringBitmap::new(); + let mut visitor = |node: usize, row_id: i64| { + if row_id >= 0 && filter.contains(row_id as u64) { + let node = u32::try_from(node) + .map_err(|_| invalid_data("DiskANN internal node ID exceeds u32"))?; + matching.insert(node); + } + Ok(()) + }; + visit_row_ids(&mut visitor)?; + Ok(matching) +} + +fn matching_ranges_from_row_id_order( + order: &[u32], + filter: &RoaringTreemap, + mut row_id_at: impl FnMut(usize) -> io::Result, +) -> io::Result>> { + let mut matches = Vec::new(); + for row_id in filter.iter() { + let Ok(row_id) = i64::try_from(row_id) else { + continue; + }; + let start = + row_id_order_partition_point(order, |node| Ok(row_id_at(node as usize)? < row_id))?; + let end = + row_id_order_partition_point(order, |node| Ok(row_id_at(node as usize)? <= row_id))?; + if start != end { + matches.push(start..end); + } + } + Ok(matches) +} + +fn row_id_order_partition_point( + order: &[u32], + mut predicate: impl FnMut(u32) -> io::Result, +) -> io::Result { + let mut left = 0usize; + let mut right = order.len(); + while left < right { + let middle = left + (right - left) / 2; + if predicate(order[middle])? { + left = middle + 1; + } else { + right = middle; + } + } + Ok(left) +} + +fn push_bounded_candidate( + candidates: &mut BinaryHeap, + candidate: SearchCandidate, + limit: usize, +) -> io::Result<()> { + if limit == 0 { + return Ok(()); + } + if candidates.len() < limit { + candidates + .try_reserve(1) + .map_err(|_| invalid_input("DiskANN filtered candidate allocation failed"))?; + candidates.push(candidate); + return Ok(()); + } + if candidates + .peek() + .is_some_and(|worst| candidate.cmp(worst) == Ordering::Less) + { + candidates.pop(); + candidates.push(candidate); + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn score_filtered_candidate_chunk( + distance_table: &[f32], + pq_codes: &[u8], + pq_m: usize, + pq_bits: usize, + nodes: &[u32], + scored: &mut Vec, + candidates: &mut BinaryHeap, + candidate_limit: usize, +) -> io::Result<()> { + score_pq_neighbors(distance_table, pq_codes, pq_m, pq_bits, nodes, scored)?; + for candidate in scored.iter().copied() { + push_bounded_candidate(candidates, candidate, candidate_limit)?; + } + Ok(()) +} + +fn push_bounded_exact_result( + results: &mut BinaryHeap, + result: ExactSearchResult, + limit: usize, +) -> io::Result<()> { + if limit == 0 { + return Ok(()); + } + if results.len() < limit { + results + .try_reserve(1) + .map_err(|_| invalid_input("DiskANN exact result allocation failed"))?; + results.push(result); + } else if results.peek().is_some_and(|worst| result < *worst) { + results.pop(); + results.push(result); + } + Ok(()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RawVectorDistanceKernel { + Scalar, + #[cfg(all(target_endian = "little", target_arch = "x86_64"))] + Avx2, + #[cfg(all(target_endian = "little", target_arch = "aarch64"))] + Neon, +} + +fn selected_raw_vector_distance_kernel(dimension: usize) -> RawVectorDistanceKernel { + #[cfg(all(target_endian = "little", target_arch = "x86_64"))] + if dimension >= 8 && is_x86_feature_detected!("avx2") { + return RawVectorDistanceKernel::Avx2; + } + #[cfg(all(target_endian = "little", target_arch = "aarch64"))] + if dimension >= 4 { + return RawVectorDistanceKernel::Neon; + } + RawVectorDistanceKernel::Scalar +} + +#[cfg(test)] +fn l2_distance_from_le_bytes(query: &[f32], bytes: &[u8]) -> io::Result { + l2_distance_from_le_bytes_with_kernel( + query, + bytes, + selected_raw_vector_distance_kernel(query.len()), + ) +} + +fn raw_vector_distance( + query: &[f32], + bytes: &[u8], + encoding: DiskAnnRawVectorEncoding, + metric: MetricType, + f32_kernel: RawVectorDistanceKernel, +) -> io::Result { + match encoding { + DiskAnnRawVectorEncoding::F32 if metric == MetricType::L2 => { + l2_distance_from_le_bytes_with_kernel(query, bytes, f32_kernel) + } + DiskAnnRawVectorEncoding::F32 => metric_distance_from_f32_le_bytes(query, bytes, metric), + DiskAnnRawVectorEncoding::F16 if metric == MetricType::L2 => { + l2_distance_from_f16_le_bytes(query, bytes) + } + DiskAnnRawVectorEncoding::F16 => metric_distance_from_f16_le_bytes(query, bytes, metric), + } +} + +fn metric_distance_from_f32_le_bytes( + query: &[f32], + bytes: &[u8], + metric: MetricType, +) -> io::Result { + let expected_len = query + .len() + .checked_mul(size_of::()) + .ok_or_else(|| invalid_data("DiskANN raw vector size overflows"))?; + if bytes.len() != expected_len { + return Err(invalid_data("DiskANN raw vector record has invalid length")); + } + let mut dot = 0.0f32; + let mut query_norm = 0.0f32; + let mut vector_norm = 0.0f32; + for (&query_value, component) in query.iter().zip(bytes.chunks_exact(size_of::())) { + let value = f32::from_le_bytes( + component + .try_into() + .expect("validated four-byte raw vector component"), + ); + if !value.is_finite() { + return Err(invalid_data("DiskANN raw vectors must be finite")); + } + dot += query_value * value; + if metric == MetricType::Cosine { + query_norm += query_value * query_value; + vector_norm += value * value; + } + } + Ok(match metric { + MetricType::InnerProduct => -dot, + MetricType::Cosine if query_norm > 0.0 && vector_norm > 0.0 => { + 1.0 - dot / (query_norm * vector_norm).sqrt() + } + MetricType::Cosine => 1.0, + MetricType::L2 => unreachable!("L2 uses the selected raw-vector kernel"), + }) +} + +fn metric_distance_from_f16_le_bytes( + query: &[f32], + bytes: &[u8], + metric: MetricType, +) -> io::Result { + let expected_len = query + .len() + .checked_mul(size_of::()) + .ok_or_else(|| invalid_data("DiskANN f16 raw-vector size overflows"))?; + if bytes.len() != expected_len { + return Err(invalid_data( + "DiskANN f16 raw-vector record has invalid length", + )); + } + if query.len() > 1024 { + return Err(invalid_data( + "DiskANN f16 raw-vector dimension exceeds the v1 limit", + )); + } + let mut decoded = [0.0f32; 1024]; + for (slot, component) in decoded[..query.len()] + .iter_mut() + .zip(bytes.chunks_exact(size_of::())) + { + let value = half::f16::from_bits(u16::from_le_bytes( + component + .try_into() + .expect("validated two-byte raw-vector component"), + )); + if !value.is_finite() { + return Err(invalid_data("DiskANN raw vectors must be finite")); + } + *slot = value.to_f32(); + } + Ok(fvec_distance(query, &decoded[..query.len()], metric)) +} + +fn l2_distance_from_f16_le_bytes(query: &[f32], bytes: &[u8]) -> io::Result { + let expected_len = query + .len() + .checked_mul(size_of::()) + .ok_or_else(|| invalid_data("DiskANN f16 raw-vector size overflows"))?; + if bytes.len() != expected_len { + return Err(invalid_data( + "DiskANN f16 raw-vector record has invalid length", + )); + } + if query.len() > 1024 { + return Err(invalid_data( + "DiskANN f16 raw-vector dimension exceeds the v1 limit", + )); + } + #[cfg(all(target_endian = "little", target_arch = "aarch64"))] + if query.len() >= 4 { + // SAFETY: AArch64 guarantees NEON. The kernel uses unaligned loads, + // validates every binary16 exponent, and stays inside the checked + // query/record length. + return unsafe { l2_distance_from_f16_le_bytes_neon(query, bytes) }; + } + let mut bits = [0u16; 1024]; + for (slot, component) in bits[..query.len()] + .iter_mut() + .zip(bytes.chunks_exact(size_of::())) + { + *slot = u16::from_le_bytes( + component + .try_into() + .expect("validated two-byte raw-vector component"), + ); + } + let values = bits[..query.len()].reinterpret_cast::(); + if values.iter().any(|value| !value.is_finite()) { + return Err(invalid_data("DiskANN raw vectors must be finite")); + } + let mut decoded = [0.0f32; 1024]; + values.convert_to_f32_slice(&mut decoded[..query.len()]); + Ok(fvec_l2sqr(query, &decoded[..query.len()])) +} + +#[cfg(all(target_endian = "little", target_arch = "aarch64"))] +#[target_feature(enable = "neon")] +unsafe fn l2_distance_from_f16_le_bytes_neon(query: &[f32], bytes: &[u8]) -> io::Result { + use std::arch::aarch64::*; + + let exponent_mask = vdup_n_u16(0x7c00); + let mut invalid = vdup_n_u16(0); + let mut sum = vdupq_n_f32(0.0); + let mut index = 0usize; + while index + 4 <= query.len() { + let bits = unsafe { vld1_u16(bytes.as_ptr().add(index * size_of::()).cast::()) }; + invalid = vorr_u16( + invalid, + vceq_u16(vand_u16(bits, exponent_mask), exponent_mask), + ); + let values = vcvt_f32_f16(vreinterpret_f16_u16(bits)); + let query_values = unsafe { vld1q_f32(query.as_ptr().add(index)) }; + let delta = vsubq_f32(query_values, values); + sum = vmlaq_f32(sum, delta, delta); + index += 4; + } + if vmaxv_u16(invalid) != 0 { + return Err(invalid_data("DiskANN raw vectors must be finite")); + } + + let mut distance = vaddvq_f32(sum); + while index < query.len() { + let start = index * size_of::(); + let value = half::f16::from_bits(u16::from_le_bytes( + bytes[start..start + size_of::()] + .try_into() + .expect("validated two-byte raw-vector component"), + )); + if !value.is_finite() { + return Err(invalid_data("DiskANN raw vectors must be finite")); + } + let delta = query[index] - value.to_f32(); + distance += delta * delta; + index += 1; + } + Ok(distance) +} + +fn l2_distance_from_le_bytes_with_kernel( + query: &[f32], + bytes: &[u8], + kernel: RawVectorDistanceKernel, +) -> io::Result { + let expected_len = query + .len() + .checked_mul(size_of::()) + .ok_or_else(|| invalid_data("DiskANN raw vector size overflows"))?; + if bytes.len() != expected_len { + return Err(invalid_data("DiskANN raw vector record has invalid length")); + } + match kernel { + RawVectorDistanceKernel::Scalar => l2_distance_from_le_bytes_scalar(query, bytes), + #[cfg(all(target_endian = "little", target_arch = "x86_64"))] + RawVectorDistanceKernel::Avx2 => unsafe { l2_distance_from_le_bytes_avx2(query, bytes) }, + #[cfg(all(target_endian = "little", target_arch = "aarch64"))] + RawVectorDistanceKernel::Neon => unsafe { l2_distance_from_le_bytes_neon(query, bytes) }, + } +} + +fn l2_distance_from_le_bytes_scalar(query: &[f32], bytes: &[u8]) -> io::Result { + let mut distance = 0.0f32; + for (&query_value, component) in query.iter().zip(bytes.chunks_exact(4)) { + let value = f32::from_le_bytes( + component + .try_into() + .expect("validated four-byte raw vector component"), + ); + if !value.is_finite() { + return Err(invalid_data("DiskANN raw vectors must be finite")); + } + let delta = query_value - value; + distance += delta * delta; + } + Ok(distance) +} + +#[cfg(all(target_endian = "little", target_arch = "x86_64"))] +#[target_feature(enable = "avx2")] +unsafe fn l2_distance_from_le_bytes_avx2(query: &[f32], bytes: &[u8]) -> io::Result { + use std::arch::x86_64::*; + + let exponent_mask = _mm256_set1_epi32(0x7f80_0000u32 as i32); + let mut invalid = _mm256_setzero_si256(); + let mut sum = _mm256_setzero_ps(); + let mut index = 0usize; + while index + 8 <= query.len() { + let bits = unsafe { + _mm256_loadu_si256( + bytes + .as_ptr() + .add(index * size_of::()) + .cast::<__m256i>(), + ) + }; + let values = _mm256_castsi256_ps(bits); + let query_values = unsafe { _mm256_loadu_ps(query.as_ptr().add(index)) }; + invalid = _mm256_or_si256( + invalid, + _mm256_cmpeq_epi32(_mm256_and_si256(bits, exponent_mask), exponent_mask), + ); + let delta = _mm256_sub_ps(query_values, values); + sum = _mm256_add_ps(sum, _mm256_mul_ps(delta, delta)); + index += 8; + } + if _mm256_movemask_ps(_mm256_castsi256_ps(invalid)) != 0 { + return Err(invalid_data("DiskANN raw vectors must be finite")); + } + + let hi = _mm256_extractf128_ps::<1>(sum); + let lo = _mm256_castps256_ps128(sum); + let sum128 = _mm_add_ps(lo, hi); + let sum64 = _mm_add_ps(sum128, _mm_movehl_ps(sum128, sum128)); + let sum32 = _mm_add_ss(sum64, _mm_shuffle_ps::<1>(sum64, sum64)); + let mut distance = _mm_cvtss_f32(sum32); + while index < query.len() { + let start = index * size_of::(); + let value = f32::from_le_bytes( + bytes[start..start + size_of::()] + .try_into() + .expect("validated four-byte raw vector component"), + ); + if !value.is_finite() { + return Err(invalid_data("DiskANN raw vectors must be finite")); + } + let delta = query[index] - value; + distance += delta * delta; + index += 1; + } + Ok(distance) +} + +#[cfg(all(target_endian = "little", target_arch = "aarch64"))] +#[target_feature(enable = "neon")] +unsafe fn l2_distance_from_le_bytes_neon(query: &[f32], bytes: &[u8]) -> io::Result { + use std::arch::aarch64::*; + + let exponent_mask = vdupq_n_u32(0x7f80_0000); + let mut invalid = vdupq_n_u32(0); + let mut sum = vdupq_n_f32(0.0); + let mut index = 0usize; + while index + 4 <= query.len() { + let bits = unsafe { vld1q_u32(bytes.as_ptr().add(index * size_of::()).cast::()) }; + let values = vreinterpretq_f32_u32(bits); + let query_values = unsafe { vld1q_f32(query.as_ptr().add(index)) }; + invalid = vorrq_u32( + invalid, + vceqq_u32(vandq_u32(bits, exponent_mask), exponent_mask), + ); + let delta = vsubq_f32(query_values, values); + sum = vmlaq_f32(sum, delta, delta); + index += 4; + } + if vmaxvq_u32(invalid) != 0 { + return Err(invalid_data("DiskANN raw vectors must be finite")); + } + + let mut distance = vaddvq_f32(sum); + while index < query.len() { + let start = index * size_of::(); + let value = f32::from_le_bytes( + bytes[start..start + size_of::()] + .try_into() + .expect("validated four-byte raw vector component"), + ); + if !value.is_finite() { + return Err(invalid_data("DiskANN raw vectors must be finite")); + } + let delta = query[index] - value; + distance += delta * delta; + index += 1; + } + Ok(distance) +} + +fn invalid_data(message: impl Into) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message.into()) +} + +fn invalid_input(message: impl Into) -> io::Error { + io::Error::new(io::ErrorKind::InvalidInput, message.into()) +} + +fn score_pq_neighbors( + distance_table: &[f32], + pq_codes: &[u8], + pq_m: usize, + pq_bits: usize, + nodes: &[u32], + scored: &mut Vec, +) -> io::Result { + if pq_m == 0 { + return Err(invalid_data("DiskANN PQ subspace count must be positive")); + } + let (code_size, ksub) = match pq_bits { + 4 => (pq_m.div_ceil(2), 16), + 8 => (pq_m, 256), + _ => return Err(invalid_data("DiskANN PQ bits must be 4 or 8")), + }; + let table_len = pq_m + .checked_mul(ksub) + .ok_or_else(|| invalid_data("DiskANN PQ distance-table length overflows"))?; + if distance_table.len() < table_len { + return Err(invalid_data("DiskANN PQ distance table is truncated")); + } + + scored.clear(); + if scored.capacity() < nodes.len() { + scored.reserve(nodes.len()); + } + let mut node_index = 0; + let mut four_code_batches = 0; + let prefetch_lookahead = pq_prefetch_lookahead(code_size); + while node_index + 4 <= nodes.len() { + for &future_node in nodes + .iter() + .skip(node_index.saturating_add(prefetch_lookahead)) + .take(4) + { + prefetch_pq_code(pq_codes, future_node as usize, code_size); + } + let batch_nodes = [ + nodes[node_index] as usize, + nodes[node_index + 1] as usize, + nodes[node_index + 2] as usize, + nodes[node_index + 3] as usize, + ]; + let mut offsets = [0; 4]; + for index in 0..4 { + offsets[index] = batch_nodes[index] + .checked_mul(code_size) + .ok_or_else(|| invalid_data("DiskANN PQ code offset overflows"))?; + let end = offsets[index] + .checked_add(code_size) + .ok_or_else(|| invalid_data("DiskANN PQ code range overflows"))?; + if end > pq_codes.len() { + return Err(invalid_data("DiskANN PQ codes are truncated")); + } + } + let distances = if pq_bits == 4 { + pq_distance_four_packed_4bit(distance_table, pq_codes, pq_m, offsets) + } else { + pq_distance_four_codes(distance_table, pq_codes, pq_m, ksub, offsets) + }; + for index in 0..4 { + scored.push(SearchCandidate { + node: batch_nodes[index], + distance: distances[index], + }); + } + node_index += 4; + four_code_batches += 1; + } + + for node in &nodes[node_index..] { + let node = *node as usize; + let start = node + .checked_mul(code_size) + .ok_or_else(|| invalid_data("DiskANN PQ code offset overflows"))?; + let end = start + .checked_add(code_size) + .ok_or_else(|| invalid_data("DiskANN PQ code range overflows"))?; + let codes = pq_codes + .get(start..end) + .ok_or_else(|| invalid_data("DiskANN PQ codes are truncated"))?; + scored.push(SearchCandidate { + node, + distance: if pq_bits == 4 { + pq_distance_packed_4bit(distance_table, codes, pq_m) + } else { + pq_distance_from_table(distance_table, codes, pq_m, ksub) + }, + }); + } + Ok(four_code_batches) +} + +#[inline] +fn pq_prefetch_lookahead(code_size: usize) -> usize { + // Aim for roughly 256 bytes of independent PQ-code work between the hint + // and its use, while bounding both tiny-code overhead and large-code delay. + (256 / code_size.max(1)).clamp(4, 16) +} + +#[inline] +fn prefetch_pq_code(pq_codes: &[u8], node: usize, code_size: usize) { + let Some(offset) = node.checked_mul(code_size) else { + return; + }; + let Some(byte) = pq_codes.get(offset) else { + return; + }; + let pointer = byte as *const u8; + #[cfg(target_arch = "x86_64")] + // SAFETY: `pointer` comes from an in-bounds slice element and the intrinsic + // only issues a non-faulting read hint. + unsafe { + std::arch::x86_64::_mm_prefetch(pointer.cast::(), std::arch::x86_64::_MM_HINT_T0); + } + #[cfg(target_arch = "aarch64")] + // SAFETY: `pointer` comes from an in-bounds slice element. `prfm` is a + // non-faulting cache hint and neither dereferences nor mutates Rust memory. + unsafe { + std::arch::asm!( + "prfm pldl1keep, [{address}]", + address = in(reg) pointer, + options(readonly, nostack, preserves_flags) + ); + } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] + let _ = pointer; +} + +#[inline] +fn pq_distance_packed_4bit(distance_table: &[f32], codes: &[u8], pq_m: usize) -> f32 { + let mut distance = 0.0f32; + for sub in 0..pq_m { + let byte = codes[sub / 2]; + let code = if sub.is_multiple_of(2) { + byte & 0x0f + } else { + byte >> 4 + }; + distance += distance_table[sub * 16 + code as usize]; + } + distance +} + +#[inline] +fn pq_distance_four_packed_4bit( + distance_table: &[f32], + codes: &[u8], + pq_m: usize, + offsets: [usize; 4], +) -> [f32; 4] { + let mut distances = [0.0f32; 4]; + for sub in 0..pq_m { + let table_start = sub * 16; + for vector in 0..4 { + let byte = codes[offsets[vector] + sub / 2]; + let code = if sub.is_multiple_of(2) { + byte & 0x0f + } else { + byte >> 4 + }; + distances[vector] += distance_table[table_start + code as usize]; + } + } + distances +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::diskann::{ + DiskAnnBuildParams, DiskAnnIndex, DiskAnnRawVectorEncoding, DiskAnnStorageLayout, + }; + use crate::diskann_io::{write_diskann_index, DiskAnnIndexReader}; + use crate::distance::MetricType; + use crate::index::VectorIndexReaderOptions; + use crate::io::{PosWriter, ReadRequest, SeekRead, SeekReadCapabilities}; + use crate::read_options::DeploymentProfile; + use roaring::RoaringTreemap; + use std::io::Cursor; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering as AtomicOrdering}; + use std::sync::{Arc, Mutex}; + + type ReadRounds = Arc>>>; + + #[test] + fn diskann_compact_vector_windows_pack_complete_records_without_page_padding() { + let section = SectionRange::new(8192, 7 * 3840); + let local = + VectorWindowPlanner::new(DeploymentProfile::LocalStorage.read_plan(), section, 3840) + .unwrap(); + assert_eq!( + local.plan_nodes([6, 1, 0, 6]), + vec![ + ReadWindow::new(8192, 4 * 3840), + ReadWindow::new(8192 + 4 * 3840, 3 * 3840), + ] + ); + + let remote = + VectorWindowPlanner::new(DeploymentProfile::RemoteStorage.read_plan(), section, 3840) + .unwrap(); + assert_eq!( + remote.plan_nodes([6, 0, 6]), + vec![ReadWindow::new(8192, 26880)] + ); + assert_eq!( + remote.window_for_node(6), + Some(ReadWindow::new(8192, 26880)) + ); + assert_eq!(remote.window_for_node(7), None); + } + + #[test] + fn diskann_f16_raw_vector_distance_decodes_little_endian_components() { + let vector = [1.25f32, -2.5, 4.0]; + let bytes = vector + .iter() + .flat_map(|&value| half::f16::from_f32(value).to_bits().to_le_bytes()) + .collect::>(); + let distance = raw_vector_distance( + &[1.0, -2.0, 5.0], + &bytes, + DiskAnnRawVectorEncoding::F16, + MetricType::L2, + RawVectorDistanceKernel::Scalar, + ) + .unwrap(); + assert!((distance - 1.3125).abs() < 1e-6); + } + + #[test] + fn diskann_raw_vector_distance_preserves_ip_and_cosine_score_semantics() { + let vector = [3.0f32, 4.0]; + let f32_bytes = vector + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect::>(); + let f16_bytes = vector + .iter() + .flat_map(|&value| half::f16::from_f32(value).to_bits().to_le_bytes()) + .collect::>(); + + for (bytes, encoding) in [ + (f32_bytes.as_slice(), DiskAnnRawVectorEncoding::F32), + (f16_bytes.as_slice(), DiskAnnRawVectorEncoding::F16), + ] { + assert_eq!( + raw_vector_distance( + &[1.0, -2.0], + bytes, + encoding, + MetricType::InnerProduct, + RawVectorDistanceKernel::Scalar, + ) + .unwrap(), + 5.0 + ); + assert!( + (raw_vector_distance( + &[1.0, 0.0], + bytes, + encoding, + MetricType::Cosine, + RawVectorDistanceKernel::Scalar, + ) + .unwrap() + - 0.4) + .abs() + < 1e-6 + ); + } + } + + #[test] + fn diskann_f16_raw_vector_distance_handles_unaligned_wide_records() { + let query = (0..65).map(|index| index as f32 * 0.25).collect::>(); + let vector = (0..65) + .map(|index| index as f32 * -0.5 + 3.0) + .collect::>(); + let mut storage = vec![0x7f]; + storage.extend( + vector + .iter() + .flat_map(|&value| half::f16::from_f32(value).to_bits().to_le_bytes()), + ); + let bytes = &storage[1..]; + let expected = query + .iter() + .zip(&vector) + .map(|(left, right)| { + let decoded = half::f16::from_f32(*right).to_f32(); + let delta = left - decoded; + delta * delta + }) + .sum::(); + + let distance = l2_distance_from_f16_le_bytes(&query, bytes).unwrap(); + + assert!((distance - expected).abs() <= expected.abs() * 1.0e-5); + let mut non_finite = bytes.to_vec(); + let non_finite_offset = size_of::<[u16; 8]>(); + non_finite[non_finite_offset..non_finite_offset + size_of::()] + .copy_from_slice(&half::f16::INFINITY.to_bits().to_le_bytes()); + assert_eq!( + l2_distance_from_f16_le_bytes(&query, &non_finite) + .expect_err("SIMD F16 distance must reject infinity") + .kind(), + io::ErrorKind::InvalidData + ); + } + + #[test] + fn diskann_compact_f16_roundtrips_dense_vector_records() { + let dimension = 8; + let count = 64; + let data = (0..count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + raw_vector_encoding: DiskAnnRawVectorEncoding::F16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, count).unwrap(); + index.add(&data, &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let header = crate::diskann_io::DiskAnnHeader::decode(&bytes[..256]).unwrap(); + assert_eq!(header.raw_vector_encoding(), DiskAnnRawVectorEncoding::F16); + assert_eq!(header.vector_record_size, (dimension * 2) as u32); + assert_eq!( + header.sections.vectors.length, + (count * dimension * 2) as u64 + ); + + let mut reader = DiskAnnIndexReader::open(Cursor::new(bytes)).unwrap(); + let (result_ids, distances) = reader.search(&data[..dimension], 1, 100).unwrap(); + assert_eq!(result_ids, vec![ids[0]]); + assert_eq!(distances, vec![0.0]); + } + + #[test] + fn diskann_interleaved_layout_searches_and_batches_without_a_vector_section() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + storage_layout: DiskAnnStorageLayout::Interleaved, + raw_vector_encoding: DiskAnnRawVectorEncoding::F16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let header = crate::diskann_io::DiskAnnHeader::decode(&bytes[..256]).unwrap(); + assert!(header.is_interleaved()); + assert_eq!(header.raw_vector_encoding(), DiskAnnRawVectorEncoding::F16); + assert_eq!(header.sections.vectors.length, 0); + assert_eq!(header.file_len, header.sections.vectors.offset); + + let queries = &data[..dimension * 4]; + let mut batch_reader = DiskAnnIndexReader::open(CloneCountingReader { + bytes: Arc::from(bytes.clone()), + clone_count: Arc::new(AtomicUsize::new(0)), + reads: Arc::new(Mutex::new(Vec::new())), + }) + .unwrap(); + let batch = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .unwrap() + .install(|| batch_reader.search_batch(queries, 5, 100)) + .unwrap(); + assert_eq!(batch_reader.last_search_stats().parallel_session_queries, 4); + + let mut single_reader = DiskAnnIndexReader::open(Cursor::new(bytes.clone())).unwrap(); + let mut expected_ids = Vec::new(); + let mut expected_distances = Vec::new(); + for query in queries.chunks_exact(dimension) { + let (query_ids, query_distances) = single_reader.search(query, 5, 100).unwrap(); + expected_ids.extend(query_ids); + expected_distances.extend(query_distances); + } + assert_eq!(batch.0, expected_ids); + assert_eq!(batch.1, expected_distances); + assert_eq!(batch.0[0], ids[0]); + + let mut filter = RoaringTreemap::new(); + filter.extend([ids[0] as u64, ids[1] as u64]); + let mut filter_bytes = Vec::new(); + filter.serialize_into(&mut filter_bytes).unwrap(); + let filtered_batch = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .unwrap() + .install(|| { + batch_reader.search_batch_with_roaring_filter(queries, 3, 100, &filter_bytes) + }) + .unwrap(); + let mut filtered_single = DiskAnnIndexReader::open(Cursor::new(bytes)).unwrap(); + let mut filtered_ids = Vec::new(); + let mut filtered_distances = Vec::new(); + for query in queries.chunks_exact(dimension) { + let (query_ids, query_distances) = filtered_single + .search_with_roaring_filter(query, 3, 100, &filter_bytes) + .unwrap(); + filtered_ids.extend(query_ids); + filtered_distances.extend(query_distances); + } + assert_eq!(filtered_batch, (filtered_ids, filtered_distances)); + } + + #[test] + fn diskann_pq_neighbor_scoring_batches_four_codes_and_matches_scalar_distance() { + let pq_m = 4; + let distance_table = (0..pq_m * 256) + .map(|index| index as f32 * 0.25) + .collect::>(); + let pq_codes = vec![ + 1, 2, 3, 4, // + 5, 6, 7, 8, // + 9, 10, 11, 12, // + 13, 14, 15, 16, // + 17, 18, 19, 20, + ]; + let nodes = [4_u32, 1, 3, 0, 2]; + let mut scored = Vec::new(); + + let four_code_batches = + score_pq_neighbors(&distance_table, &pq_codes, pq_m, 8, &nodes, &mut scored).unwrap(); + + assert_eq!(four_code_batches, 1); + assert_eq!( + scored + .iter() + .map(|candidate| candidate.node) + .collect::>(), + nodes.iter().map(|node| *node as usize).collect::>() + ); + for (candidate, node) in scored.iter().zip(nodes) { + let start = node as usize * pq_m; + let expected = + pq_distance_from_table(&distance_table, &pq_codes[start..start + pq_m], pq_m, 256); + assert_eq!(candidate.distance, expected); + } + + assert!(score_pq_neighbors( + &distance_table, + &pq_codes[..pq_codes.len() - 1], + pq_m, + 8, + &nodes, + &mut scored, + ) + .is_err()); + } + + #[test] + fn diskann_packed_4bit_neighbor_scoring_uses_vector_code_size() { + let pq_m = 4; + let pq_bits = 4; + let distance_table = (0..pq_m * 16) + .map(|index| index as f32 * 0.25) + .collect::>(); + let pq_codes = vec![ + 0x10, 0x32, // node 0: [0, 1, 2, 3] + 0x54, 0x76, // node 1: [4, 5, 6, 7] + 0x98, 0xBA, // node 2: [8, 9, 10, 11] + 0xDC, 0xFE, // node 3: [12, 13, 14, 15] + 0x21, 0x43, // node 4: [1, 2, 3, 4] + ]; + let nodes = [4, 1, 3, 2, 0]; + let mut scored = Vec::new(); + + let batches = score_pq_neighbors( + &distance_table, + &pq_codes, + pq_m, + pq_bits, + &nodes, + &mut scored, + ) + .unwrap(); + + assert_eq!(batches, 1); + for (actual, &node) in scored.iter().zip(&nodes) { + let start = node as usize * 2; + let expected = [0, 1, 2, 3] + .into_iter() + .map(|sub| { + let byte = pq_codes[start + sub / 2]; + let code = if sub.is_multiple_of(2) { + byte & 0x0f + } else { + byte >> 4 + }; + distance_table[sub * 16 + code as usize] + }) + .sum::(); + assert_eq!(actual.node, node as usize); + assert_eq!(actual.distance, expected); + } + + let error = score_pq_neighbors( + &distance_table, + &pq_codes[..pq_codes.len() - 1], + pq_m, + pq_bits, + &nodes, + &mut scored, + ) + .expect_err("a packed code truncated after the last low nibble must fail closed"); + assert!(error.to_string().contains("truncated")); + } + + #[test] + fn diskann_filtered_candidate_selection_stays_bounded_and_deterministic() { + let mut candidates = std::collections::BinaryHeap::new(); + for node in (0..1000).rev() { + push_bounded_candidate( + &mut candidates, + SearchCandidate { + node, + distance: (node / 2) as f32, + }, + 10, + ) + .unwrap(); + assert!(candidates.len() <= 10); + } + let mut selected = candidates.into_vec(); + sort_candidates(&mut selected); + + assert_eq!( + selected + .iter() + .map(|candidate| candidate.node) + .collect::>(), + (0..10).collect::>() + ); + } + + #[test] + fn diskann_filtered_candidate_target_is_bounded_by_matches() { + assert_eq!(desired_filtered_candidate_count(1_000, 10), 64); + assert_eq!(desired_filtered_candidate_count(20, 10), 20); + assert_eq!(desired_filtered_candidate_count(1_000, usize::MAX), 1_000); + } + + #[test] + fn diskann_filtered_pq_tile_reuses_four_queries_without_exceeding_table_budget() { + assert_eq!(filtered_pq_query_tile_size(16, 256), 4); + assert_eq!(filtered_pq_query_tile_size(1024, 256), 2); + assert_eq!(filtered_pq_query_tile_size(1024, 16), 4); + for (pq_m, pq_ksub) in [(16, 256), (1024, 256), (1024, 16)] { + let tile_size = filtered_pq_query_tile_size(pq_m, pq_ksub); + let table_bytes = tile_size * pq_m * pq_ksub * size_of::(); + assert!(table_bytes <= FILTERED_PQ_TILE_TABLE_LIMIT_BYTES); + } + } + + #[test] + fn diskann_rerank_expands_candidates_only_within_seed_windows() { + let planner = ReadWindowPlanner::new( + DeploymentProfile::LocalStorage.read_plan(), + SectionRange::new(4096, 4 * DISKANN_PAGE_SIZE as u64), + ); + let candidates = [ + SearchCandidate { + node: 1, + distance: 1.0, + }, + SearchCandidate { + node: 16, + distance: 2.0, + }, + SearchCandidate { + node: 2, + distance: 3.0, + }, + SearchCandidate { + node: 31, + distance: 4.0, + }, + SearchCandidate { + node: 32, + distance: 5.0, + }, + ]; + let mut selected = Vec::new(); + let mut selected_windows = HashSet::new(); + + expand_rerank_candidates_within_seed_windows( + &candidates, + 2, + |node| { + planner + .window_for_logical_page(node / 16) + .map(|window| window.offset) + .ok_or_else(|| invalid_data("test vector page is out of range")) + }, + &mut selected_windows, + &mut selected, + ) + .unwrap(); + + assert_eq!( + selected + .iter() + .map(|candidate| candidate.node) + .collect::>(), + vec![1, 16, 2, 31, 32] + ); + } + + #[test] + fn diskann_bounded_exact_heap_matches_full_sort_with_row_id_ties() { + let mut results = (0..100) + .map(|row_id| ExactSearchResult { + row_id, + distance: ((row_id * 37) % 11) as f32, + }) + .collect::>(); + let mut expected = results.clone(); + expected.sort(); + expected.truncate(7); + let mut heap = BinaryHeap::new(); + + for result in results.drain(..).rev() { + push_bounded_exact_result(&mut heap, result, 7).unwrap(); + assert!(heap.len() <= 7); + } + + assert_eq!(heap.into_sorted_vec(), expected); + } + + #[test] + fn diskann_adaptive_filter_strategy_gates_selectivity_cost_and_access_pattern() { + let scan = FilteredCandidateStrategy::Exhaustive { + target_candidates: 64, + }; + assert_eq!( + select_filtered_candidate_strategy( + 1_000_000, + 499_999, + 10, + 100, + 8, + DeploymentProfile::LocalStorage.read_plan(), + false, + ), + scan + ); + assert_eq!( + select_filtered_candidate_strategy( + 10_000, + 10_000, + 10, + 100, + 64, + DeploymentProfile::LocalStorage.read_plan(), + false, + ), + scan + ); + assert_eq!( + select_filtered_candidate_strategy( + 1_000_000, + 1_000_000, + 10, + 100, + 64, + DeploymentProfile::LocalStorage.read_plan(), + false, + ), + scan + ); + + let graph = FilteredCandidateStrategy::Graph { + target_candidates: 64, + search_list_size: 200, + }; + assert_eq!( + select_filtered_candidate_strategy( + 1_000_000, + 1_000_000, + 10, + 200, + 64, + DeploymentProfile::LocalStorage.read_plan(), + false, + ), + graph + ); + assert_eq!( + select_filtered_candidate_strategy( + 1_000_000, + 1_000_000, + 10, + 200, + 64, + DeploymentProfile::ObjectStore.read_plan(), + false, + ), + scan + ); + assert_eq!( + select_filtered_candidate_strategy( + 1_000_000, + 1_000_000, + 10, + 200, + 64, + DeploymentProfile::ObjectStore.read_plan(), + true, + ), + graph + ); + } + + #[test] + fn diskann_adaptive_filter_strategy_scales_and_caps_search_list_safely() { + assert_eq!( + adaptive_filtered_search_list_size(1_000_000, 500_000, 64, 100), + 256 + ); + assert_eq!(adaptive_filtered_search_list_size(10, 5, 5, usize::MAX), 10); + assert_eq!( + adaptive_filtered_search_list_size( + usize::MAX, + usize::MAX, + usize::MAX, + resolve_diskann_l_search(usize::MAX, 0), + ), + usize::MAX + ); + } + + #[test] + fn diskann_automatic_l_search_scales_with_top_k_and_preserves_explicit_values() { + assert_eq!(resolve_diskann_l_search(10, 0), 100); + assert_eq!(resolve_diskann_l_search(100, 0), 200); + assert_eq!(resolve_diskann_l_search(usize::MAX, 0), usize::MAX); + assert_eq!(resolve_diskann_l_search(100, 64), 100); + assert_eq!(resolve_diskann_l_search(10, 64), 64); + } + + #[test] + fn diskann_graph_candidates_are_post_filtered_and_require_target_count() { + let graph = vec![ + SearchCandidate { + node: 1, + distance: 1.0, + }, + SearchCandidate { + node: 2, + distance: 2.0, + }, + SearchCandidate { + node: 3, + distance: 3.0, + }, + ]; + let matching = RoaringBitmap::from_iter([2, 3]); + + assert!(post_filter_graph_candidates(&graph, &matching, 3).is_none()); + assert_eq!( + post_filter_graph_candidates(&graph, &matching, 2) + .unwrap() + .iter() + .map(|candidate| candidate.node) + .collect::>(), + vec![2, 3] + ); + } + + #[test] + fn diskann_filtered_batch_rerank_chunks_bound_bytes_and_ranges() { + let range_limited = (0..1025) + .map(|index| ReadWindow::new(index * 4096, 4096)) + .collect::>(); + assert_eq!( + plan_streaming_window_chunks(&range_limited), + vec![0..1024, 1024..1025] + ); + + let byte_limited = vec![ + ReadWindow::new(0, 40 * 1024 * 1024), + ReadWindow::new(40 * 1024 * 1024, 30 * 1024 * 1024), + ]; + assert_eq!( + plan_streaming_window_chunks(&byte_limited), + vec![0..1, 1..2] + ); + } + + #[test] + fn diskann_filtered_lookup_returns_duplicate_rows_and_ignores_oversized_ids() { + let row_ids = [7, -1, 7, 3, 9]; + let order = vec![1, 3, 0, 2, 4]; + let mut filter = RoaringTreemap::new(); + filter.insert(7); + filter.insert(i64::MAX as u64 + 1); + + let ranges = + matching_ranges_from_row_id_order(&order, &filter, |node| Ok(row_ids[node])).unwrap(); + let nodes = ranges + .into_iter() + .flat_map(|range| order[range].iter().copied()) + .collect::>(); + + assert_eq!(nodes, vec![0, 2]); + assert!(!use_row_id_order(0, 64)); + assert!(use_row_id_order(4, 64)); + assert!(!use_row_id_order(5, 64)); + } + + #[test] + fn diskann_filtered_batch_dense_translation_visits_each_row_id_once() { + let row_ids = [-5, 7, 7, 3, 11]; + let mut filter = RoaringTreemap::new(); + filter.insert(7); + filter.insert(i64::MAX as u64 + 1); + let mut visits = 0usize; + + let matching = matching_nodes_from_sequential_row_ids(&filter, |visitor| { + for (node, &row_id) in row_ids.iter().enumerate() { + visits += 1; + visitor(node, row_id)?; + } + Ok(()) + }) + .unwrap(); + + assert_eq!(visits, row_ids.len()); + assert_eq!(matching.iter().collect::>(), vec![1, 2]); + } + + #[test] + fn diskann_bounded_heap_frontier_reuses_allocation_and_retains_total_order() { + let mut scratch = DiskAnnQueryScratch::default(); + scratch.begin_search(1_000); + for node in (0..1_000).rev() { + scratch + .insert_graph_candidate( + SearchCandidate { + node, + distance: (node / 2) as f32, + }, + 10, + ) + .unwrap(); + assert!(scratch.retained_candidates.len() <= 10); + assert!(scratch.frontier.len() <= 20); + } + let frontier_capacity = scratch.frontier.capacity(); + assert!(frontier_capacity <= 40); + + scratch.finish_graph_candidates(); + + assert_eq!( + scratch + .candidates + .iter() + .map(|candidate| candidate.node) + .collect::>(), + (0..10).collect::>() + ); + + scratch.begin_search(1_000); + assert_eq!(scratch.frontier.capacity(), frontier_capacity); + } + + #[test] + fn diskann_bounded_frontier_matches_round_by_round_vector_oracle() { + let discovered_rounds = (0..12) + .map(|round| { + (0..9) + .map(|slot| { + let node = 1 + round * 9 + ((slot * 5 + round) % 9); + SearchCandidate { + node, + distance: ((node * 17) % 23) as f32, + } + }) + .collect::>() + }) + .collect::>(); + for (limit, beam_width) in [(8, 1), (16, 4), (32, 16)] { + let entry = SearchCandidate { + node: 0, + distance: 11.0, + }; + let mut scratch = DiskAnnQueryScratch::default(); + scratch.begin_search(256); + scratch.insert_graph_candidate(entry, limit).unwrap(); + let mut oracle = vec![(entry, false)]; + + for discovered in &discovered_rounds { + scratch.select_round(beam_width); + let oracle_selected = oracle + .iter_mut() + .filter(|(_, expanded)| !*expanded) + .take(beam_width) + .map(|(candidate, expanded)| { + *expanded = true; + candidate.node + }) + .collect::>(); + assert_eq!(scratch.selected_nodes, oracle_selected); + if scratch.selected_nodes.is_empty() { + break; + } + for &candidate in discovered { + scratch.insert_graph_candidate(candidate, limit).unwrap(); + oracle.push((candidate, false)); + } + oracle.sort_by_key(|entry| entry.0); + oracle.truncate(limit); + } + + scratch.finish_graph_candidates(); + let oracle_candidates = oracle + .into_iter() + .map(|(candidate, _)| candidate) + .collect::>(); + assert_eq!(scratch.candidates, oracle_candidates); + } + } + + #[test] + fn diskann_query_scratch_reuses_visited_bitmap_and_clears_touched_nodes() { + let mut scratch = DiskAnnQueryScratch::default(); + scratch.begin_search(1_000_000); + let capacity = scratch.visited_capacity(); + + assert!(scratch.mark_visited(7)); + assert!(!scratch.mark_visited(7)); + assert!(scratch.mark_visited(999_999)); + + scratch.begin_search(1_000_000); + + assert_eq!(scratch.visited_capacity(), capacity); + assert!(!scratch.is_visited(7)); + assert!(!scratch.is_visited(999_999)); + } + + #[test] + fn diskann_adaptive_visited_switches_between_dense_and_sparse_storage() { + let mut scratch = DiskAnnQueryScratch::default(); + scratch.begin_graph_search(1024, 100, 8).unwrap(); + assert!(!scratch.uses_sparse_visited()); + assert!(scratch.mark_visited(7)); + assert!(!scratch.mark_visited(7)); + + scratch.begin_graph_search(1_000_000, 100, 64).unwrap(); + assert!(!scratch.uses_sparse_visited()); + assert!(scratch.mark_visited(999_999)); + assert!(!scratch.mark_visited(999_999)); + + scratch.begin_graph_search(100_000_000, 100, 64).unwrap(); + assert!(scratch.uses_sparse_visited()); + assert!( + std::any::type_name_of_val(&scratch.sparse_visited).contains("SparseTable"), + "DiskANN query hot-path sparse visited state must use the internal table" + ); + assert!(scratch.mark_visited(99_999_999)); + assert!(!scratch.mark_visited(99_999_999)); + + scratch.begin_graph_search(100_000_000, 100, 64).unwrap(); + assert!(scratch.mark_visited(99_999_999)); + assert_eq!(scratch.visited_capacity(), 1_000_000); + } + + #[test] + fn diskann_query_scratch_clears_graph_buffers_and_keeps_vector_working_set() { + let mut scratch = DiskAnnQueryScratch::default(); + scratch.begin_search(1024); + scratch.prepare_distance_table(512).fill(1.0); + scratch.candidates.reserve(128); + scratch.candidates.push(SearchCandidate { + node: 7, + distance: 1.0, + }); + scratch.loaded_adjacency_pages.reserve(16); + scratch.loaded_adjacency_pages.insert(0); + scratch.adjacency_windows.reserve(16); + scratch.adjacency_windows.insert(0, vec![0; 4096].into()); + scratch.vector_windows.insert(4096, vec![0; 64 * 1024]); + let candidate_capacity = scratch.candidates.capacity(); + let page_cache_capacity = scratch.loaded_adjacency_pages.capacity(); + let window_cache_capacity = scratch.adjacency_windows.capacity(); + + scratch.begin_search(1024); + + assert!(scratch.candidates.is_empty()); + assert!(scratch.loaded_adjacency_pages.is_empty()); + assert!(scratch.adjacency_windows.is_empty()); + assert_eq!(scratch.vector_windows.get(&4096).unwrap().len(), 64 * 1024); + assert_eq!(scratch.retained_window_capacity(), 4 * 1024); + assert_eq!(scratch.candidates.capacity(), candidate_capacity); + assert_eq!( + scratch.loaded_adjacency_pages.capacity(), + page_cache_capacity + ); + assert_eq!(scratch.adjacency_windows.capacity(), window_cache_capacity); + assert_eq!(scratch.distance_table.len(), 512); + } + + #[test] + fn diskann_query_scratch_drops_windows_over_its_retained_capacity_limit() { + let mut scratch = DiskAnnQueryScratch::with_window_buffer_limit(4096); + scratch.adjacency_windows.insert(0, vec![0; 8192].into()); + + scratch.begin_search(1); + + assert_eq!(scratch.retained_window_capacity(), 0); + } + + #[test] + fn diskann_query_adjacency_cache_evicts_oldest_window_to_fit_budget() { + let mut cache = AdjacencyWindowCache::default(); + let mut pool = WindowBufferPool::with_retained_capacity_limit(4096); + cache.insert(0, vec![0; 4096].into()); + cache.insert(4096, vec![0; 4096].into()); + + let evictions = cache.trim(&mut pool, 4096); + + assert_eq!(evictions, 1); + assert!(!cache.contains_key(&0)); + assert!(cache.contains_key(&4096)); + assert_eq!(cache.retained_capacity(), 4096); + } + + #[test] + fn diskann_raw_vector_working_set_is_not_retained_over_query_limit() { + let windows = [ + ReadWindow::new(0, QUERY_WINDOW_BUFFER_LIMIT_BYTES), + ReadWindow::new(QUERY_WINDOW_BUFFER_LIMIT_BYTES as u64, 1), + ]; + let mut cache = VectorWindowCache::default(); + cache.insert(0, vec![0; DISKANN_PAGE_SIZE as usize]); + cache.touch(0); + let mut pool = WindowBufferPool::default(); + + let (retain, _) = prepare_vector_window_cache( + &windows, + &mut cache, + &mut pool, + QUERY_WINDOW_BUFFER_LIMIT_BYTES, + ); + + assert!(!retain); + assert!(cache.is_empty()); + assert!(cache.recency.is_empty()); + } + + #[test] + fn diskann_raw_vector_working_set_drops_overallocated_buffers() { + let windows = [ReadWindow::new(0, 1)]; + let mut oversized = Vec::with_capacity(QUERY_WINDOW_BUFFER_LIMIT_BYTES + 1); + oversized.push(0); + let mut cache = VectorWindowCache::default(); + cache.insert(0, oversized); + cache.touch(0); + let mut pool = WindowBufferPool::default(); + + let (retain, _) = prepare_vector_window_cache( + &windows, + &mut cache, + &mut pool, + QUERY_WINDOW_BUFFER_LIMIT_BYTES, + ); + + assert!(retain, "the requested one-byte window is cacheable"); + assert!( + cache.is_empty(), + "an oversized allocation must be replaced before the read" + ); + } + + #[test] + fn diskann_raw_vector_cache_evicts_only_least_recently_used_windows() { + let window_bytes = 4 * 1024 * 1024; + let offsets = [0, window_bytes as u64, (2 * window_bytes) as u64]; + let mut cache = VectorWindowCache::default(); + for offset in offsets { + cache.insert(offset, vec![0; window_bytes]); + cache.touch(offset); + } + let mut pool = WindowBufferPool::default(); + + let evictions = cache.trim(&mut pool, QUERY_WINDOW_BUFFER_LIMIT_BYTES); + + assert_eq!(evictions, 1); + assert!(!cache.contains_key(&offsets[0])); + assert!(cache.contains_key(&offsets[1])); + assert!(cache.contains_key(&offsets[2])); + assert_eq!(cache.retained_capacity(), QUERY_WINDOW_BUFFER_LIMIT_BYTES); + assert_eq!(cache.recency.oldest_offsets(), offsets[1..]); + } + + #[test] + fn diskann_adjacency_read_keeps_cached_windows_required_by_current_round() { + let window_bytes = 1024 * 1024; + let mut cache = AdjacencyWindowCache::default(); + for window in 0..9 { + cache.insert((window * window_bytes) as u64, vec![0; window_bytes].into()); + } + let required = [ReadWindow::new(0, window_bytes)]; + let mut pool = WindowBufferPool::default(); + + let evictions = + prepare_adjacency_window_cache(&required, window_bytes, &mut cache, &mut pool); + + assert_eq!(evictions, 2); + assert!( + cache.contains_key(&0), + "a cached window selected for this graph round must survive preparation trimming" + ); + assert!(!cache.contains_key(&(window_bytes as u64))); + assert!(!cache.contains_key(&((2 * window_bytes) as u64))); + } + + #[test] + fn diskann_vector_window_cache_tracks_capacity_incrementally() { + let mut cache = VectorWindowCache::default(); + let mut first = Vec::with_capacity(64); + first.resize(4, 1); + let mut second = Vec::with_capacity(128); + second.resize(4, 2); + cache.insert(10, first); + cache.insert(20, second); + cache.touch(10); + assert_eq!(cache.retained_capacity(), 192); + + let mut buffers = WindowBufferPool::with_retained_capacity_limit(0); + assert_eq!(cache.trim(&mut buffers, 64), 1); + assert!(cache.contains_key(&10)); + assert!(!cache.contains_key(&20)); + assert_eq!(cache.retained_capacity(), 64); + + cache.remove(10); + assert_eq!(cache.retained_capacity(), 0); + } + + #[test] + fn diskann_raw_vector_cache_uses_constant_time_recency_updates() { + let cache = VectorWindowCache::default(); + + assert!( + std::any::type_name_of_val(&cache.recency).ends_with("OffsetLru"), + "raw-vector cache recency must use linked hash updates instead of scans" + ); + } + + #[test] + fn diskann_window_buffer_pool_reuses_capacity_without_reusing_content() { + let mut pool = WindowBufferPool::with_retained_capacity_limit(8192); + let mut original = vec![7u8; 4096]; + let allocation = original.as_mut_ptr(); + pool.recycle(original); + + let reused = pool.take(4096).unwrap(); + + assert_eq!(reused.as_ptr(), allocation); + assert_eq!(reused.len(), 4096); + assert!(reused.iter().all(|value| *value == 0)); + assert_eq!(pool.retained_capacity, 0); + } + + #[test] + fn diskann_batch_workers_split_the_aggregate_window_buffer_budget() { + assert_eq!(window_buffer_limit_per_worker(1), 8 * 1024 * 1024); + assert_eq!(window_buffer_limit_per_worker(8), 8 * 1024 * 1024); + assert_eq!(window_buffer_limit_per_worker(16), 4 * 1024 * 1024); + } + + #[test] + fn diskann_query_scratch_reuses_round_selection_buffers() { + let mut scratch = DiskAnnQueryScratch::default(); + for (node, distance) in [(7, 2.0), (11, 3.0), (13, 4.0)] { + scratch + .insert_graph_candidate(SearchCandidate { node, distance }, 4) + .unwrap(); + } + + scratch.select_round(2); + assert_eq!(scratch.selected_nodes, vec![7, 11]); + let nodes_capacity = scratch.selected_nodes.capacity(); + + scratch.select_round(1); + + assert_eq!(scratch.selected_nodes, vec![13]); + assert_eq!(scratch.selected_nodes.capacity(), nodes_capacity); + } + + #[test] + fn diskann_raw_vector_distance_decodes_page_bytes_directly() { + let bytes = [1.0f32, 3.0f32] + .into_iter() + .flat_map(f32::to_le_bytes) + .collect::>(); + + let distance = l2_distance_from_le_bytes(&[2.0, 5.0], &bytes).unwrap(); + + assert_eq!(distance, 5.0); + } + + #[test] + fn diskann_raw_vector_distance_uses_available_simd_kernel() { + let query = (0..33).map(|index| index as f32 * 0.25).collect::>(); + let vector = (0..33) + .map(|index| index as f32 * -0.5 + 3.0) + .collect::>(); + let mut storage = vec![0x7f]; + storage.extend(vector.iter().flat_map(|value| value.to_le_bytes())); + let unaligned = &storage[1..]; + let expected = query + .iter() + .zip(&vector) + .map(|(left, right)| { + let delta = left - right; + delta * delta + }) + .sum::(); + + let kernel = selected_raw_vector_distance_kernel(query.len()); + let distance = l2_distance_from_le_bytes(&query, unaligned).unwrap(); + let explicit_distance = + l2_distance_from_le_bytes_with_kernel(&query, unaligned, kernel).unwrap(); + + assert!((distance - expected).abs() <= expected.abs() * 1.0e-5); + assert_eq!(explicit_distance, distance); + #[cfg(all(target_endian = "little", target_arch = "x86_64"))] + assert_eq!( + selected_raw_vector_distance_kernel(query.len()), + if is_x86_feature_detected!("avx2") { + RawVectorDistanceKernel::Avx2 + } else { + RawVectorDistanceKernel::Scalar + } + ); + #[cfg(all(target_endian = "little", target_arch = "aarch64"))] + assert_eq!( + selected_raw_vector_distance_kernel(query.len()), + RawVectorDistanceKernel::Neon + ); + #[cfg(not(all( + target_endian = "little", + any(target_arch = "x86_64", target_arch = "aarch64") + )))] + assert_eq!( + selected_raw_vector_distance_kernel(query.len()), + RawVectorDistanceKernel::Scalar + ); + + let mut non_finite = unaligned.to_vec(); + non_finite[8 * size_of::()..9 * size_of::()] + .copy_from_slice(&f32::INFINITY.to_le_bytes()); + assert_eq!( + l2_distance_from_le_bytes(&query, &non_finite) + .expect_err("SIMD exact distance must reject infinity") + .kind(), + io::ErrorKind::InvalidData + ); + + non_finite[8 * size_of::()..9 * size_of::()] + .copy_from_slice(&f32::NAN.to_le_bytes()); + assert_eq!( + l2_distance_from_le_bytes(&query, &non_finite) + .expect_err("SIMD exact distance must reject NaN") + .kind(), + io::ErrorKind::InvalidData + ); + } + + struct RoundRecordingReader { + inner: Cursor>, + rounds: ReadRounds, + } + + impl SeekRead for RoundRecordingReader { + fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { + self.rounds.lock().unwrap().push( + ranges + .iter() + .map(|range| (range.pos, range.buf.len())) + .collect(), + ); + for range in ranges { + self.inner.set_position(range.pos); + io::Read::read_exact(&mut self.inner, range.buf)?; + } + Ok(()) + } + } + + struct CapabilityRecordingReader { + inner: Cursor>, + rounds: ReadRounds, + capabilities: SeekReadCapabilities, + } + + impl SeekRead for CapabilityRecordingReader { + fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { + self.rounds.lock().unwrap().push( + ranges + .iter() + .map(|range| (range.pos, range.buf.len())) + .collect(), + ); + for range in ranges { + self.inner.set_position(range.pos); + io::Read::read_exact(&mut self.inner, range.buf)?; + } + Ok(()) + } + + fn read_capabilities(&self) -> SeekReadCapabilities { + self.capabilities + } + } + + #[derive(Clone)] + struct CloneCountingReader { + bytes: Arc<[u8]>, + clone_count: Arc, + reads: Arc>>, + } + + impl SeekRead for CloneCountingReader { + fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { + for range in ranges { + self.reads + .lock() + .unwrap() + .push((range.pos, range.buf.len())); + let start = usize::try_from(range.pos) + .map_err(|_| io::Error::other("test read offset exceeds usize"))?; + let end = start + .checked_add(range.buf.len()) + .ok_or_else(|| io::Error::other("test read range overflows"))?; + range.buf.copy_from_slice( + self.bytes + .get(start..end) + .ok_or(io::ErrorKind::UnexpectedEof)?, + ); + } + Ok(()) + } + + fn try_clone_reader(&self) -> io::Result> { + self.clone_count.fetch_add(1, AtomicOrdering::SeqCst); + Ok(Some(self.clone())) + } + } + + #[derive(Clone)] + struct ToggleFailReader { + inner: Cursor>, + fail_reads: Arc, + } + + impl SeekRead for ToggleFailReader { + fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { + if self.fail_reads.load(AtomicOrdering::SeqCst) { + return Err(io::Error::other("injected query read failure")); + } + for range in ranges { + self.inner.set_position(range.pos); + io::Read::read_exact(&mut self.inner, range.buf)?; + } + Ok(()) + } + + fn try_clone_reader(&self) -> io::Result> { + Ok(Some(self.clone())) + } + } + + #[test] + fn diskann_read_tiers_plan_aligned_deduplicated_and_clipped_windows() { + let section = SectionRange::new(139_264, 20 * 4096); + + let local = ReadWindowPlanner::new(DeploymentProfile::LocalStorage.read_plan(), section); + assert_eq!(local.beam_width(), 4); + assert_eq!( + local.plan_logical_pages([2, 0, 2, 0]), + vec![ReadWindow::new(139_264, 16 * 1024)] + ); + + let object_store = + ReadWindowPlanner::new(DeploymentProfile::ObjectStore.read_plan(), section); + assert_eq!(object_store.beam_width(), 16); + assert_eq!( + object_store.plan_logical_pages([19, 1, 16, 0, 15, 19]), + vec![ + ReadWindow::new(139_264, 64 * 1024), + ReadWindow::new(139_264 + 64 * 1024, 16 * 1024), + ] + ); + } + + #[test] + fn diskann_reader_capabilities_refine_windows_and_beam_width() { + let plan = DeploymentProfile::RemoteStorage + .read_plan() + .with_capabilities(SeekReadCapabilities { + estimated_random_read_latency_nanos: 0, + preferred_window_bytes: 16 * 1024, + max_ranges_per_pread: 2, + }); + let planner = ReadWindowPlanner::new(plan, SectionRange::new(128 * 1024, 64 * 1024)); + + assert_eq!(planner.beam_width(), 2); + assert_eq!( + planner.plan_logical_pages([0, 1, 3, 4]), + vec![ + ReadWindow::new(128 * 1024, 16 * 1024), + ReadWindow::new(144 * 1024, 16 * 1024), + ] + ); + } + + #[test] + fn diskann_reader_capabilities_bound_ranges_per_pread_call() { + let dimension = 8; + let count = 32; + let data = (0..count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (0..count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, count).unwrap(); + index.add(&data, &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + + let rounds = Arc::new(Mutex::new(Vec::new())); + let source = CapabilityRecordingReader { + inner: Cursor::new(bytes), + rounds: Arc::clone(&rounds), + capabilities: SeekReadCapabilities { + estimated_random_read_latency_nanos: 0, + preferred_window_bytes: 0, + max_ranges_per_pread: 2, + }, + }; + let mut reader = DiskAnnIndexReader::open(source).unwrap(); + rounds.lock().unwrap().clear(); + let mut buffers = [[0u8; 1]; 5]; + let mut requests = buffers + .iter_mut() + .enumerate() + .map(|(offset, buffer)| ReadRequest::new(offset as u64, buffer)) + .collect::>(); + + reader.pread_ranges(&mut requests).unwrap(); + + assert_eq!( + rounds + .lock() + .unwrap() + .iter() + .map(Vec::len) + .collect::>(), + vec![2, 2, 1] + ); + } + + #[test] + fn diskann_pq_prefetch_lookahead_is_bounded_by_code_work() { + assert_eq!(pq_prefetch_lookahead(1), 16); + assert_eq!(pq_prefetch_lookahead(16), 16); + assert_eq!(pq_prefetch_lookahead(64), 4); + assert_eq!(pq_prefetch_lookahead(1024), 4); + } + + #[test] + fn diskann_representative_query_warmup_preserves_observed_search_stats() { + let dimension = 8; + let count = 64; + let data = (0..count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (0..count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, count).unwrap(); + index.add(&data, &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let mut reader = DiskAnnIndexReader::open(Cursor::new(bytes)).unwrap(); + reader.search(&data[..dimension], 3, 16).unwrap(); + let stats = reader.last_search_stats(); + + reader + .warmup_queries(&data[dimension..5 * dimension], 16) + .unwrap(); + + assert_eq!(reader.last_search_stats(), stats); + let result = reader + .search(&data[dimension..2 * dimension], 1, 16) + .unwrap(); + assert_eq!(result.0.len(), 1); + assert!(result.1[0].is_finite()); + } + + #[test] + fn diskann_window_buffers_are_recycled_after_read_failure() { + let dimension = 8; + let count = 256; + let data = (0..count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (0..count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, count).unwrap(); + index.add(&data, &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let fail_reads = Arc::new(AtomicBool::new(false)); + let mut reader = DiskAnnIndexReader::open(ToggleFailReader { + inner: Cursor::new(bytes), + fail_reads: Arc::clone(&fail_reads), + }) + .unwrap(); + reader.ensure_resident().unwrap(); + fail_reads.store(true, AtomicOrdering::SeqCst); + let planner = ReadWindowPlanner::new( + DeploymentProfile::LocalStorage.read_plan(), + reader.header.sections.vectors, + ); + let window = planner.window_for_logical_page(0).unwrap(); + let mut cache = VectorWindowCache::default(); + let mut pool = WindowBufferPool::with_retained_capacity_limit(window.length); + + let error = reader + .load_vector_windows(&[window], &mut cache, &mut pool) + .expect_err("injected read failure must propagate"); + + assert_eq!(error.kind(), io::ErrorKind::Other); + assert!(cache.is_empty()); + assert_eq!(pool.retained_capacity, window.length); + } + + #[test] + fn diskann_batch_read_failure_clears_query_local_raw_vector_cache() { + let dimension = 128; + let count = 512; + let data = (0..count * dimension) + .map(|offset| ((offset * 31) % 997) as f32 * 0.01) + .collect::>(); + let ids = (0..count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 8, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, count).unwrap(); + index.add(&data, &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let fail_reads = Arc::new(AtomicBool::new(false)); + let mut reader = DiskAnnIndexReader::open(ToggleFailReader { + inner: Cursor::new(bytes), + fail_reads: Arc::clone(&fail_reads), + }) + .unwrap(); + reader.ensure_resident().unwrap(); + reader + .rerank_with_query_scratch( + &data[..dimension], + &[SearchCandidate { + node: 0, + distance: 0.0, + }], + 1, + ) + .unwrap(); + assert!(!reader.query_scratch.vector_windows.is_empty()); + + let second_page_node = + DISKANN_PAGE_SIZE as usize / reader.header.vector_record_size as usize; + fail_reads.store(true, AtomicOrdering::SeqCst); + let error = reader + .rerank_candidate_batch_streaming( + &data[second_page_node * dimension..(second_page_node + 1) * dimension], + 1, + vec![vec![(0, vec![second_page_node])]], + ) + .expect_err("injected batch rerank read failure must propagate"); + + assert_eq!(error.kind(), io::ErrorKind::Other); + assert!(reader.query_scratch.vector_windows.is_empty()); + assert!(reader.query_scratch.vector_windows.recency.is_empty()); + } + + #[test] + fn diskann_graph_search_finds_and_exactly_reranks_query_vector() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let query_index = 37; + let query = &data[query_index * dimension..(query_index + 1) * dimension]; + let mut reader = DiskAnnIndexReader::open(Cursor::new(bytes)).unwrap(); + + let (result_ids, distances) = reader.search(query, 5, 100).unwrap(); + + assert_eq!(result_ids.len(), 5); + assert_eq!(result_ids[0], ids[query_index]); + assert_eq!(distances[0], 0.0); + assert!(distances.windows(2).all(|pair| pair[0] <= pair[1])); + let cached_vector_capacity = reader + .query_scratch + .vector_windows + .entries + .values() + .map(WindowPayload::capacity) + .sum::(); + assert!(cached_vector_capacity > 0); + assert!(cached_vector_capacity <= QUERY_WINDOW_BUFFER_LIMIT_BYTES); + assert!(reader.query_scratch.adjacency_windows.is_empty()); + assert!(reader.query_scratch.retained_window_capacity() <= QUERY_WINDOW_BUFFER_LIMIT_BYTES); + } + + #[test] + fn diskann_4bit_roundtrip_supports_graph_and_filtered_search() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::with_pq_bits( + dimension, + MetricType::L2, + 2, + 4, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let query_index = 37; + let query = &data[query_index * dimension..(query_index + 1) * dimension]; + let mut reader = DiskAnnIndexReader::open(Cursor::new(bytes)).unwrap(); + + reader.ensure_resident().unwrap(); + assert_eq!(reader.header.pq_bits, 4); + assert_eq!(reader.pq().unwrap().ksub, 16); + assert_eq!(reader.pq_codes().unwrap().len(), indexed_count); + + let (result_ids, distances) = reader.search(query, 5, 100).unwrap(); + assert_eq!(result_ids[0], ids[query_index]); + assert_eq!(distances[0], 0.0); + + let mut filter = RoaringTreemap::new(); + filter.extend(ids[32..48].iter().map(|row_id| *row_id as u64)); + let mut filter_bytes = Vec::new(); + filter.serialize_into(&mut filter_bytes).unwrap(); + let (filtered_ids, filtered_distances) = reader + .search_with_roaring_filter(query, 5, 100, &filter_bytes) + .unwrap(); + assert_eq!(filtered_ids[0], ids[query_index]); + assert_eq!(filtered_distances[0], 0.0); + + let mut queries = Vec::new(); + queries.extend_from_slice(query); + queries.extend_from_slice(query); + let (batch_ids, batch_distances) = reader + .search_batch_with_roaring_filter(&queries, 5, 100, &filter_bytes) + .unwrap(); + assert_eq!(batch_ids[0], ids[query_index]); + assert_eq!(batch_ids[5], ids[query_index]); + assert_eq!(batch_distances[0], 0.0); + assert_eq!(batch_distances[5], 0.0); + } + + #[test] + fn diskann_repeated_single_query_reuses_raw_vector_windows() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let header = crate::diskann_io::DiskAnnHeader::decode(&bytes[..256]).unwrap(); + let reads = Arc::new(Mutex::new(Vec::new())); + let mut reader = DiskAnnIndexReader::open(CloneCountingReader { + bytes: Arc::from(bytes), + clone_count: Arc::new(AtomicUsize::new(0)), + reads: Arc::clone(&reads), + }) + .unwrap(); + let query = &data[..dimension]; + + reads.lock().unwrap().clear(); + let first = reader.search(query, 5, 100).unwrap(); + let first_vector_reads = reads + .lock() + .unwrap() + .iter() + .filter(|(offset, _)| { + *offset >= header.sections.vectors.offset + && *offset < header.sections.vectors.offset + header.sections.vectors.length + }) + .count(); + assert!(first_vector_reads > 0); + let first_stats = reader.last_search_stats(); + assert_eq!(first_stats.raw_vector_cache_hits, 0); + assert_eq!( + first_stats.raw_vector_cache_misses, + first_stats.rerank_unique_windows + ); + + reads.lock().unwrap().clear(); + let second = reader.search(query, 5, 100).unwrap(); + let second_vector_reads = reads + .lock() + .unwrap() + .iter() + .filter(|(offset, _)| { + *offset >= header.sections.vectors.offset + && *offset < header.sections.vectors.offset + header.sections.vectors.length + }) + .count(); + + assert_eq!(second, first); + let second_stats = reader.last_search_stats(); + assert_eq!( + second_stats.raw_vector_cache_hits, + second_stats.rerank_unique_windows + ); + assert_eq!(second_stats.raw_vector_cache_misses, 0); + assert_eq!( + second_vector_reads, 0, + "the previous single-query raw-vector working set should be reused" + ); + } + + #[test] + fn diskann_repeated_query_reuses_shared_adjacency_windows() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let header = crate::diskann_io::DiskAnnHeader::decode(&bytes[..256]).unwrap(); + let reads = Arc::new(Mutex::new(Vec::new())); + let mut reader = DiskAnnIndexReader::open_with_options( + CloneCountingReader { + bytes: Arc::from(bytes), + clone_count: Arc::new(AtomicUsize::new(0)), + reads: Arc::clone(&reads), + }, + VectorIndexReaderOptions::with_cache_budgets( + DeploymentProfile::Auto, + 0, + 16 * 1024 * 1024, + 4 * 1024 * 1024 * 1024, + 8 * 1024 * 1024, + ), + ) + .unwrap(); + let query = &data[..dimension]; + + reader.search(query, 5, 100).unwrap(); + reads.lock().unwrap().clear(); + reader.search(query, 5, 100).unwrap(); + let adjacency_reads = reads + .lock() + .unwrap() + .iter() + .filter(|(offset, _)| { + *offset >= header.sections.adjacency.offset + && *offset < header.sections.adjacency.offset + header.sections.adjacency.length + }) + .count(); + + assert_eq!( + adjacency_reads, 0, + "the shared cold-adjacency cache should serve a repeated graph traversal" + ); + let stats = reader.last_search_stats(); + assert!(stats.adjacency_cache_hits > 0); + assert_eq!(stats.adjacency_cache_misses, 0); + } + + #[test] + fn diskann_shared_adjacency_payload_is_zero_copy() { + let mut payload = vec![7u8; DISKANN_PAGE_SIZE as usize]; + let allocation = payload.as_mut_ptr(); + + let shared = share_window_payload(payload); + + assert_eq!(shared.as_ptr(), allocation); + assert_eq!(shared.len(), DISKANN_PAGE_SIZE as usize); + assert!(shared.iter().all(|value| *value == 7)); + } + + #[test] + fn diskann_zero_adjacency_cache_budget_disables_window_reuse() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let header = crate::diskann_io::DiskAnnHeader::decode(&bytes[..256]).unwrap(); + let reads = Arc::new(Mutex::new(Vec::new())); + let mut reader = DiskAnnIndexReader::open_with_options( + CloneCountingReader { + bytes: Arc::from(bytes), + clone_count: Arc::new(AtomicUsize::new(0)), + reads: Arc::clone(&reads), + }, + VectorIndexReaderOptions::with_cache_budgets( + DeploymentProfile::Auto, + 0, + 0, + 4 * 1024 * 1024 * 1024, + 8 * 1024 * 1024, + ), + ) + .unwrap(); + let query = &data[..dimension]; + + reader.search(query, 5, 100).unwrap(); + reads.lock().unwrap().clear(); + reader.search(query, 5, 100).unwrap(); + let adjacency_reads = reads + .lock() + .unwrap() + .iter() + .filter(|(offset, _)| { + *offset >= header.sections.adjacency.offset + && *offset < header.sections.adjacency.offset + header.sections.adjacency.length + }) + .count(); + + assert!(adjacency_reads > 0); + let stats = reader.last_search_stats(); + assert_eq!(stats.adjacency_cache_hits, 0); + assert!(stats.adjacency_cache_misses > 0); + } + + #[test] + fn diskann_zero_raw_vector_cache_budget_disables_window_reuse() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let header = crate::diskann_io::DiskAnnHeader::decode(&bytes[..256]).unwrap(); + let reads = Arc::new(Mutex::new(Vec::new())); + let mut reader = DiskAnnIndexReader::open_with_options( + CloneCountingReader { + bytes: Arc::from(bytes), + clone_count: Arc::new(AtomicUsize::new(0)), + reads: Arc::clone(&reads), + }, + VectorIndexReaderOptions::with_cache_budgets( + DeploymentProfile::Auto, + 16 * 1024 * 1024, + 16 * 1024 * 1024, + 4 * 1024 * 1024 * 1024, + 0, + ), + ) + .unwrap(); + let query = &data[..dimension]; + + reader.search(query, 5, 100).unwrap(); + reads.lock().unwrap().clear(); + reader.search(query, 5, 100).unwrap(); + let second_vector_reads = reads + .lock() + .unwrap() + .iter() + .filter(|(offset, _)| { + *offset >= header.sections.vectors.offset + && *offset < header.sections.vectors.offset + header.sections.vectors.length + }) + .count(); + + assert!(second_vector_reads > 0); + assert!(reader.query_scratch.vector_windows.is_empty()); + } + + #[test] + fn diskann_oversized_rerank_counts_evicted_cached_windows() { + let dimension = 8; + let count = 256; + let data = (0..count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + raw_vector_encoding: DiskAnnRawVectorEncoding::F32, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, count).unwrap(); + index.add(&data, &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let mut reader = DiskAnnIndexReader::open_with_options( + Cursor::new(bytes), + VectorIndexReaderOptions::with_cache_budgets( + DeploymentProfile::Auto, + 16 * 1024 * 1024, + 16 * 1024 * 1024, + 4 * 1024 * 1024 * 1024, + DISKANN_PAGE_SIZE as usize, + ), + ) + .unwrap(); + reader.optimize_for_search().unwrap(); + let second_page_node = + DISKANN_PAGE_SIZE as usize / reader.header.vector_record_size as usize; + let first_page = SearchCandidate { + node: 0, + distance: 0.0, + }; + + reader + .rerank_with_query_scratch(&data[..dimension], &[first_page], 1) + .unwrap(); + assert_eq!(reader.query_scratch.vector_windows.len(), 1); + + reader.last_search_stats = DiskAnnSearchStats::default(); + reader + .rerank_with_query_scratch( + &data[..dimension], + &[ + first_page, + SearchCandidate { + node: second_page_node, + distance: 0.0, + }, + ], + 1, + ) + .unwrap(); + + assert_eq!(reader.last_search_stats().raw_vector_cache_evictions, 2); + assert_eq!(reader.last_search_stats().raw_vector_cache_hits, 1); + assert_eq!(reader.last_search_stats().raw_vector_cache_misses, 1); + } + + #[test] + fn diskann_single_query_cache_reuses_nonconsecutive_hot_vector_window() { + let dimension = 8; + let training_count = 256; + let indexed_count = 256; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + raw_vector_encoding: DiskAnnRawVectorEncoding::F32, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data, &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let header = crate::diskann_io::DiskAnnHeader::decode(&bytes[..256]).unwrap(); + let second_page_node = DISKANN_PAGE_SIZE as usize / header.vector_record_size as usize; + assert!(second_page_node < indexed_count); + let reads = Arc::new(Mutex::new(Vec::new())); + let mut reader = DiskAnnIndexReader::open(CloneCountingReader { + bytes: Arc::from(bytes), + clone_count: Arc::new(AtomicUsize::new(0)), + reads: Arc::clone(&reads), + }) + .unwrap(); + reader.optimize_for_search().unwrap(); + let first_page_candidate = [SearchCandidate { + node: 0, + distance: 0.0, + }]; + let second_page_candidate = [SearchCandidate { + node: second_page_node, + distance: 0.0, + }]; + let vector_read_count = || { + reads + .lock() + .unwrap() + .iter() + .filter(|(offset, _)| { + *offset >= header.sections.vectors.offset + && *offset < header.sections.vectors.offset + header.sections.vectors.length + }) + .count() + }; + + reads.lock().unwrap().clear(); + reader + .rerank_with_query_scratch(&data[..dimension], &first_page_candidate, 1) + .unwrap(); + assert_eq!(vector_read_count(), 1); + + reads.lock().unwrap().clear(); + reader + .rerank_with_query_scratch( + &data[second_page_node * dimension..(second_page_node + 1) * dimension], + &second_page_candidate, + 1, + ) + .unwrap(); + assert_eq!(vector_read_count(), 1); + + reads.lock().unwrap().clear(); + reader + .rerank_with_query_scratch(&data[..dimension], &first_page_candidate, 1) + .unwrap(); + + assert_eq!( + vector_read_count(), + 0, + "a bounded immutable-window cache should retain nonconsecutive hot pages" + ); + } + + #[test] + fn diskann_coalesced_search_batches_graph_beam_and_exact_rerank_reads() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let header = crate::diskann_io::DiskAnnHeader::decode(&bytes[..256]).unwrap(); + let rounds = Arc::new(Mutex::new(Vec::new())); + let recording = RoundRecordingReader { + inner: Cursor::new(bytes), + rounds: Arc::clone(&rounds), + }; + let mut reader = DiskAnnIndexReader::open_with_options( + recording, + VectorIndexReaderOptions::with_cache_budgets( + DeploymentProfile::ObjectStore, + 0, + 16 * 1024 * 1024, + 4 * 1024 * 1024 * 1024, + 8 * 1024 * 1024, + ), + ) + .unwrap(); + + reader.search(&data[..dimension], 5, 100).unwrap(); + + let rounds = rounds.lock().unwrap(); + let adjacency_rounds = rounds + .iter() + .filter(|round| { + round.iter().any(|(offset, _)| { + *offset >= header.sections.adjacency.offset + && *offset + < header.sections.adjacency.offset + header.sections.adjacency.length + }) + }) + .count(); + let vector_rounds = rounds + .iter() + .filter(|round| { + round.iter().any(|(offset, length)| { + *offset >= header.sections.vectors.offset + && *offset < header.sections.vectors.offset + header.sections.vectors.length + && *length > 1 + }) + }) + .count(); + assert!(adjacency_rounds <= 7, "got {adjacency_rounds} graph rounds"); + assert_eq!(vector_rounds, 1, "rerank must use one batched pread"); + } + + #[test] + fn diskann_unfiltered_batch_shared_rerank_reads_overlapping_window_once() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let header = crate::diskann_io::DiskAnnHeader::decode(&bytes[..256]).unwrap(); + let reads = Arc::new(Mutex::new(Vec::new())); + let clone_count = Arc::new(AtomicUsize::new(0)); + let mut reader = DiskAnnIndexReader::open(CloneCountingReader { + bytes: Arc::from(bytes), + clone_count, + reads: Arc::clone(&reads), + }) + .unwrap(); + let query = &data[..dimension]; + let queries = [query, query].concat(); + + let (batch_ids, batch_distances) = rayon::ThreadPoolBuilder::new() + .num_threads(2) + .build() + .unwrap() + .install(|| reader.search_batch(&queries, 5, 100)) + .unwrap(); + + assert_eq!(&batch_ids[..5], &batch_ids[5..]); + assert_eq!(&batch_distances[..5], &batch_distances[5..]); + let vector_reads = reads + .lock() + .unwrap() + .iter() + .filter(|(offset, _)| { + *offset >= header.sections.vectors.offset + && *offset < header.sections.vectors.offset + header.sections.vectors.length + }) + .copied() + .collect::>(); + assert!( + !vector_reads.is_empty(), + "small batches must run complete exact reranks in parallel sessions" + ); + assert_eq!( + vector_reads.len(), + vector_reads.iter().copied().collect::>().len(), + "parallel sessions must share immutable raw-vector windows" + ); + assert_eq!(reader.last_search_stats().parallel_session_queries, 2); + let retained_vector_capacity = reader + .batch_workers + .iter() + .flat_map(|worker| worker.query_scratch.vector_windows.entries.values()) + .map(WindowPayload::capacity) + .sum::(); + assert!(retained_vector_capacity > 0); + assert!( + retained_vector_capacity + <= reader.options().raw_vector_cache_bytes * reader.batch_workers.len() + ); + assert_eq!( + reader + .batch_workers + .iter() + .map(|worker| worker.query_scratch.vector_windows.recency.len()) + .sum::(), + reader + .batch_workers + .iter() + .map(|worker| worker.query_scratch.vector_windows.len()) + .sum::() + ); + assert_eq!(reader.last_search_stats().parallel_exact_rerank_chunks, 0); + } + + #[test] + fn diskann_batch_reuses_worker_readers() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let clone_count = Arc::new(AtomicUsize::new(0)); + let mut reader = DiskAnnIndexReader::open(CloneCountingReader { + bytes: Arc::from(bytes), + clone_count: Arc::clone(&clone_count), + reads: Arc::new(Mutex::new(Vec::new())), + }) + .unwrap(); + let queries = data[..dimension].repeat(4); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .unwrap(); + + pool.install(|| reader.search_batch(&queries, 5, 100)) + .unwrap(); + let first_batch_clones = clone_count.load(AtomicOrdering::SeqCst); + pool.install(|| reader.search_batch(&queries, 5, 100)) + .unwrap(); + + assert_eq!(first_batch_clones, 4); + assert_eq!( + clone_count.load(AtomicOrdering::SeqCst), + first_batch_clones, + "the second batch should reuse retained storage handles" + ); + } + + #[test] + fn diskann_batch_restores_worker_pool_after_read_failure() { + let dimension = 8; + let count = 256; + let data = (0..count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (0..count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, count).unwrap(); + index.add(&data, &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let fail_reads = Arc::new(AtomicBool::new(false)); + let mut reader = DiskAnnIndexReader::open_with_options( + ToggleFailReader { + inner: Cursor::new(bytes), + fail_reads: Arc::clone(&fail_reads), + }, + VectorIndexReaderOptions::with_cache_budgets( + DeploymentProfile::Auto, + 0, + 0, + 4 * 1024 * 1024 * 1024, + 0, + ), + ) + .unwrap(); + let queries = data[..dimension].repeat(4); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .unwrap(); + + pool.install(|| reader.search_batch(&queries, 5, 100)) + .unwrap(); + assert_eq!(reader.batch_workers.len(), 4); + fail_reads.store(true, AtomicOrdering::SeqCst); + pool.install(|| reader.search_batch(&queries, 5, 100)) + .expect_err("injected worker read failure must propagate"); + assert_eq!(reader.batch_workers.len(), 4); + fail_reads.store(false, AtomicOrdering::SeqCst); + pool.install(|| reader.search_batch(&queries, 5, 100)) + .unwrap(); + assert_eq!(reader.batch_workers.len(), 4); + } + + #[test] + fn diskann_repeated_batch_reuses_reader_raw_vector_cache() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let header = crate::diskann_io::DiskAnnHeader::decode(&bytes[..256]).unwrap(); + let reads = Arc::new(Mutex::new(Vec::new())); + let mut reader = DiskAnnIndexReader::open(CloneCountingReader { + bytes: Arc::from(bytes), + clone_count: Arc::new(AtomicUsize::new(0)), + reads: Arc::clone(&reads), + }) + .unwrap(); + let queries = [&data[..dimension], &data[..dimension]].concat(); + + reader.search_batch(&queries, 5, 100).unwrap(); + reads.lock().unwrap().clear(); + reader.search_batch(&queries, 5, 100).unwrap(); + let vector_reads = reads + .lock() + .unwrap() + .iter() + .filter(|(offset, _)| { + *offset >= header.sections.vectors.offset + && *offset < header.sections.vectors.offset + header.sections.vectors.length + }) + .count(); + + assert_eq!(vector_reads, 0); + let stats = reader.last_search_stats(); + assert!(stats.raw_vector_cache_hits > 0); + assert_eq!(stats.raw_vector_cache_misses, 0); + } + + #[test] + fn diskann_parallel_exact_rerank_matches_single_queries() { + let dimension = 128; + let training_count = 512; + let indexed_count = 256; + let data = (0..training_count * dimension) + .map(|offset| { + ((offset * 31) % 997) as f32 * 0.01 + (offset / dimension) as f32 * 0.0001 + }) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 16, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let query_indices = [0, 31, 127, 255]; + let queries = query_indices + .into_iter() + .flat_map(|query_index| { + data[query_index * dimension..(query_index + 1) * dimension] + .iter() + .copied() + }) + .collect::>(); + let mut batch_reader = DiskAnnIndexReader::open(Cursor::new(bytes.clone())).unwrap(); + + let batch = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .unwrap() + .install(|| batch_reader.search_batch(&queries, 10, 100)) + .unwrap(); + let mut single_reader = DiskAnnIndexReader::open(Cursor::new(bytes)).unwrap(); + let mut expected_ids = Vec::new(); + let mut expected_distances = Vec::new(); + for query in queries.chunks_exact(dimension) { + let (query_ids, query_distances) = single_reader.search(query, 10, 100).unwrap(); + expected_ids.extend(query_ids); + expected_distances.extend(query_distances); + } + + assert_eq!(batch.0, expected_ids); + assert_eq!(batch.1, expected_distances); + let stats = batch_reader.last_search_stats(); + assert!(stats.rerank_candidate_references >= 4 * 64); + assert_eq!(stats.parallel_exact_rerank_chunks, 1); + assert_eq!( + stats.parallel_exact_rerank_references, + stats.rerank_candidate_references + ); + } + + #[test] + fn diskann_batch_query_chunks_bound_live_candidates() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let mut reader = DiskAnnIndexReader::open(Cursor::new(bytes)).unwrap(); + let query_count = BATCH_QUERY_CHUNK_SIZE + 1; + let queries = data[..dimension].repeat(query_count); + + let (batch_ids, batch_distances) = reader.search_batch(&queries, 1, 100).unwrap(); + + assert_eq!(batch_ids, vec![ids[0]; query_count]); + assert!(batch_distances.iter().all(|distance| *distance == 0.0)); + let stats = reader.last_search_stats(); + assert_eq!(stats.query_count, query_count); + assert_eq!(stats.query_chunks, 2); + assert_eq!(stats.max_queries_per_chunk, BATCH_QUERY_CHUNK_SIZE); + } + + #[test] + fn diskann_hot_adjacency_does_not_copy_into_query_window_cache() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let mut reader = DiskAnnIndexReader::open_with_options( + Cursor::new(bytes), + VectorIndexReaderOptions::with_cache_budgets( + DeploymentProfile::ObjectStore, + 16 * 1024 * 1024, + 16 * 1024 * 1024, + 4 * 1024 * 1024 * 1024, + 8 * 1024 * 1024, + ), + ) + .unwrap(); + reader.optimize_for_search().unwrap(); + let mut window_cache = AdjacencyWindowCache::default(); + let mut page_cache = HashSet::new(); + let mut window_buffers = WindowBufferPool::default(); + + reader + .load_adjacency_pages( + &[reader.header.entry_node as usize], + &mut window_cache, + &mut page_cache, + &mut window_buffers, + ) + .unwrap(); + + assert!(window_cache.is_empty()); + assert!(!page_cache.is_empty()); + } + + #[test] + fn diskann_filtered_search_skips_graph_and_pads_sparse_results() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let header = crate::diskann_io::DiskAnnHeader::decode(&bytes[..256]).unwrap(); + let rounds = Arc::new(Mutex::new(Vec::new())); + let recording = RoundRecordingReader { + inner: Cursor::new(bytes), + rounds: Arc::clone(&rounds), + }; + let mut reader = DiskAnnIndexReader::open_with_options( + recording, + VectorIndexReaderOptions::with_cache_budgets( + DeploymentProfile::ObjectStore, + 0, + 0, + 4 * 1024 * 1024 * 1024, + 0, + ), + ) + .unwrap(); + let query_index = 37; + let query = &data[query_index * dimension..(query_index + 1) * dimension]; + let mut filter = RoaringTreemap::new(); + filter.insert(ids[query_index] as u64); + let mut filter_bytes = Vec::new(); + filter.serialize_into(&mut filter_bytes).unwrap(); + + let (result_ids, distances) = reader + .search_with_roaring_filter(query, 3, 100, &filter_bytes) + .unwrap(); + + assert_eq!(result_ids, vec![ids[query_index], -1, -1]); + assert_eq!(distances, vec![0.0, f32::MAX, f32::MAX]); + let rounds = rounds.lock().unwrap(); + assert!(!rounds.iter().flatten().any(|(offset, _)| { + *offset >= header.sections.adjacency.offset + && *offset < header.sections.adjacency.offset + header.sections.adjacency.length + })); + assert_eq!( + rounds + .iter() + .filter(|round| round.iter().any(|(offset, length)| { + *offset >= header.sections.vectors.offset + && *offset < header.sections.vectors.offset + header.sections.vectors.length + && *length > 1 + })) + .count(), + 1 + ); + } + + #[test] + fn diskann_broad_random_access_filter_uses_graph_candidate_io() { + let dimension = 8; + let indexed_count = 1024; + let data = (0..indexed_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32 + (offset / dimension) as f32 * 0.01) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 1, + build_search_list_size: 8, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, indexed_count).unwrap(); + index.add(&data, &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let header = crate::diskann_io::DiskAnnHeader::decode(&bytes[..256]).unwrap(); + let rounds = Arc::new(Mutex::new(Vec::new())); + let recording = RoundRecordingReader { + inner: Cursor::new(bytes), + rounds: Arc::clone(&rounds), + }; + let mut reader = DiskAnnIndexReader::open(recording).unwrap(); + let mut filter = RoaringTreemap::new(); + filter.extend(ids.iter().map(|row_id| *row_id as u64)); + let mut filter_bytes = Vec::new(); + filter.serialize_into(&mut filter_bytes).unwrap(); + + let (result_ids, distances) = reader + .search_with_roaring_filter(&data[..dimension], 10, 200, &filter_bytes) + .unwrap(); + + assert_eq!(result_ids[0], ids[0]); + assert_eq!(distances[0], 0.0); + assert!(rounds.lock().unwrap().iter().flatten().any(|(offset, _)| { + *offset >= header.sections.adjacency.offset + && *offset < header.sections.adjacency.offset + header.sections.adjacency.length + })); + } + + #[test] + fn diskann_adaptive_filtered_recall_matrix_stays_within_one_percentage_point() { + let dimension = 8; + let indexed_count = 10_000; + let data = (0..indexed_count) + .flat_map(|node| { + (0..dimension).map(move |component| { + let mut hash = (node as u64) + .wrapping_mul(0x9e37_79b9_7f4a_7c15) + .wrapping_add((component as u64).wrapping_mul(0xbf58_476d_1ce4_e5b9)); + hash ^= hash >> 30; + hash = hash.wrapping_mul(0xbf58_476d_1ce4_e5b9); + hash ^= hash >> 27; + let noise = (hash as u32) as f32 / u32::MAX as f32; + noise + (node / 256) as f32 * 0.25 + }) + }) + .collect::>(); + let ids = (0..indexed_count) + .map(|node| (node / 2) as i64) + .collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 16, + build_search_list_size: 100, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, indexed_count).unwrap(); + index.add(&data, &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let mut reader = DiskAnnIndexReader::open_with_options( + Cursor::new(bytes), + VectorIndexReaderOptions::new(4 * 1024 * 1024 * 1024), + ) + .unwrap(); + reader.ensure_resident().unwrap(); + + for distribution in ["random", "clustered"] { + for selectivity_basis_points in [1usize, 10, 100, 1000, 5000, 10_000] { + let matching_count = indexed_count + .saturating_mul(selectivity_basis_points) + .div_ceil(10_000) + .max(1); + let ordered_nodes = if distribution == "random" { + (0..indexed_count) + .map(|index| (index * 4051) % indexed_count) + .collect::>() + } else { + (0..indexed_count).collect::>() + }; + let matching = RoaringBitmap::from_iter( + ordered_nodes[..matching_count] + .iter() + .map(|node| *node as u32), + ); + let strategy = select_filtered_candidate_strategy( + indexed_count, + matching_count, + 10, + 200, + 16, + DeploymentProfile::Memory.read_plan(), + false, + ); + assert_eq!( + matches!(strategy, FilteredCandidateStrategy::Graph { .. }), + selectivity_basis_points == 10_000, + "unexpected strategy for {distribution} at {selectivity_basis_points} bps" + ); + + let mut hits = 0usize; + let mut total = 0usize; + for &query_node in ordered_nodes[..10.min(matching_count)].iter() { + let query = &data[query_node * dimension..(query_node + 1) * dimension]; + let target = desired_filtered_candidate_count(matching_count, 10); + let baseline_candidates = reader + .exhaustive_filtered_candidates(query, &matching, target) + .unwrap(); + let baseline = reader + .rerank_with_query_scratch(query, &baseline_candidates, 10) + .unwrap() + .0; + let adaptive_candidates = reader + .generate_filtered_candidates(query, 10, 200, &matching) + .unwrap(); + let adaptive = reader + .rerank_with_query_scratch(query, &adaptive_candidates, 10) + .unwrap() + .0; + + let mut expected_counts = HashMap::::new(); + for row_id in baseline.into_iter().filter(|row_id| *row_id >= 0) { + *expected_counts.entry(row_id).or_default() += 1; + total += 1; + } + for row_id in adaptive.into_iter().filter(|row_id| *row_id >= 0) { + if expected_counts.get_mut(&row_id).is_some_and(|count| { + if *count == 0 { + false + } else { + *count -= 1; + true + } + }) { + hits += 1; + } + } + } + let recall = hits as f64 / total as f64; + assert!( + recall + 0.01 + f64::EPSILON >= 1.0, + "{distribution} filter at {selectivity_basis_points} bps has Recall@10 {recall:.4} against exhaustive scan" + ); + } + } + } + + #[test] + fn diskann_zero_k_filtered_search_still_rejects_malformed_filter() { + let header = crate::diskann_io::DiskAnnHeader::for_layout( + 8, + 1, + 0, + 2, + DiskAnnBuildParams { + max_degree: 1, + build_search_list_size: 1, + ..DiskAnnBuildParams::default() + }, + ) + .unwrap(); + let mut bytes = vec![0u8; header.file_len as usize]; + bytes[..crate::diskann_io::DISKANN_HEADER_SIZE].copy_from_slice(&header.encode()); + let mut reader = DiskAnnIndexReader::open(Cursor::new(bytes)).unwrap(); + + let error = reader + .search_with_roaring_filter(&[0.0; 8], 0, 100, &[0xff]) + .expect_err("malformed filters must be rejected even for zero k"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + } + + #[test] + fn diskann_filtered_batch_validates_filter_before_cloning_workers() { + let header = crate::diskann_io::DiskAnnHeader::for_layout( + 8, + 1, + 0, + 2, + DiskAnnBuildParams { + max_degree: 1, + build_search_list_size: 1, + ..DiskAnnBuildParams::default() + }, + ) + .unwrap(); + let mut bytes = vec![0u8; header.file_len as usize]; + bytes[..crate::diskann_io::DISKANN_HEADER_SIZE].copy_from_slice(&header.encode()); + let clone_count = Arc::new(AtomicUsize::new(0)); + let mut reader = DiskAnnIndexReader::open(CloneCountingReader { + bytes: Arc::from(bytes), + clone_count: Arc::clone(&clone_count), + reads: Arc::new(Mutex::new(Vec::new())), + }) + .unwrap(); + + let error = rayon::ThreadPoolBuilder::new() + .num_threads(2) + .build() + .unwrap() + .install(|| reader.search_batch_with_roaring_filter(&[0.0; 16], 1, 100, &[0xff])) + .expect_err("malformed filters must fail before batch fan-out"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(clone_count.load(AtomicOrdering::SeqCst), 0); + } + + #[test] + fn diskann_filtered_batch_shares_lookup_without_preloading_adjacency() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let header = crate::diskann_io::DiskAnnHeader::decode(&bytes[..256]).unwrap(); + let reads = Arc::new(Mutex::new(Vec::new())); + let clone_count = Arc::new(AtomicUsize::new(0)); + let mut reader = DiskAnnIndexReader::open_with_options( + CloneCountingReader { + bytes: Arc::from(bytes), + clone_count: Arc::clone(&clone_count), + reads: Arc::clone(&reads), + }, + VectorIndexReaderOptions::with_cache_budgets( + DeploymentProfile::ObjectStore, + 4096, + 16 * 1024 * 1024, + 4 * 1024 * 1024 * 1024, + 8 * 1024 * 1024, + ), + ) + .unwrap(); + let mut filter = RoaringTreemap::new(); + filter.insert(ids[0] as u64); + let mut filter_bytes = Vec::new(); + filter.serialize_into(&mut filter_bytes).unwrap(); + + rayon::ThreadPoolBuilder::new() + .num_threads(2) + .build() + .unwrap() + .install(|| { + reader + .search_batch_with_roaring_filter(&data[..dimension * 2], 1, 100, &filter_bytes) + .unwrap(); + }); + + assert_eq!( + clone_count.load(AtomicOrdering::SeqCst), + 0, + "resident-PQ batch scan must not clone the storage reader" + ); + assert!(reads + .lock() + .unwrap() + .iter() + .any(|(offset, _)| { *offset == header.sections.row_id_order.offset })); + assert!(!reads.lock().unwrap().iter().any(|(offset, _)| { + *offset >= header.sections.adjacency.offset + && *offset < header.sections.adjacency.offset + header.sections.adjacency.length + })); + assert_eq!( + reads + .lock() + .unwrap() + .iter() + .filter(|(offset, length)| { + *offset >= header.sections.vectors.offset + && *offset < header.sections.vectors.offset + header.sections.vectors.length + && *length > 1 + }) + .count(), + 1, + "the parent reranker must read a shared vector window once" + ); + } + + #[test] + fn diskann_filtered_streaming_batch_matches_single_queries_with_duplicate_row_ids() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let mut ids = (1000..1000 + indexed_count as i64).collect::>(); + ids[0] = 7; + ids[1] = 7; + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let mut filter = RoaringTreemap::new(); + filter.insert(7); + let mut filter_bytes = Vec::new(); + filter.serialize_into(&mut filter_bytes).unwrap(); + + let queries = &data[..dimension * 2]; + let mut batch_reader = DiskAnnIndexReader::open(Cursor::new(bytes.clone())).unwrap(); + let batch = batch_reader + .search_batch_with_roaring_filter(queries, 3, 100, &filter_bytes) + .unwrap(); + let mut single_reader = DiskAnnIndexReader::open(Cursor::new(bytes)).unwrap(); + let mut expected_ids = Vec::new(); + let mut expected_distances = Vec::new(); + for query in queries.chunks_exact(dimension) { + let (ids, distances) = single_reader + .search_with_roaring_filter(query, 3, 100, &filter_bytes) + .unwrap(); + expected_ids.extend(ids); + expected_distances.extend(distances); + } + + assert_eq!(batch.0, expected_ids); + assert_eq!(batch.1, expected_distances); + assert_eq!(&batch.0[..3], &[7, 7, -1]); + assert_eq!(&batch.0[3..], &[7, 7, -1]); + } + + #[test] + fn diskann_search_stats_report_tiled_filtered_batch() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let mut reader = DiskAnnIndexReader::open(CloneCountingReader { + bytes: Arc::from(bytes), + clone_count: Arc::new(AtomicUsize::new(0)), + reads: Arc::new(Mutex::new(Vec::new())), + }) + .unwrap(); + let mut filter = RoaringTreemap::new(); + filter.insert(ids[0] as u64); + let mut filter_bytes = Vec::new(); + filter.serialize_into(&mut filter_bytes).unwrap(); + + rayon::ThreadPoolBuilder::new() + .num_threads(2) + .build() + .unwrap() + .install(|| { + reader + .search_batch_with_roaring_filter(&data[..dimension * 2], 1, 100, &filter_bytes) + .unwrap(); + }); + + let stats = reader.last_search_stats(); + assert_eq!(stats.query_count, 2); + assert_eq!(stats.filtered_exhaustive_queries, 2); + assert_eq!(stats.filtered_graph_queries, 0); + assert_eq!(stats.pq_distance_evaluations, 2); + assert_eq!(stats.pq_code_loads, 1); + assert_eq!(stats.rerank_candidate_references, 2); + assert_eq!(stats.rerank_unique_windows, 1); + } + + #[test] + fn diskann_filtered_pq_batch_kernel_reuses_code_loads() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let mut filter = RoaringTreemap::new(); + filter.extend(ids[..32].iter().map(|row_id| *row_id as u64)); + let mut filter_bytes = Vec::new(); + filter.serialize_into(&mut filter_bytes).unwrap(); + let queries = &data[..dimension * 4]; + let mut batch_reader = DiskAnnIndexReader::open(Cursor::new(bytes.clone())).unwrap(); + + let batch = batch_reader + .search_batch_with_roaring_filter(queries, 3, 100, &filter_bytes) + .unwrap(); + let mut single_reader = DiskAnnIndexReader::open(Cursor::new(bytes)).unwrap(); + let mut expected_ids = Vec::new(); + let mut expected_distances = Vec::new(); + for query in queries.chunks_exact(dimension) { + let (query_ids, query_distances) = single_reader + .search_with_roaring_filter(query, 3, 100, &filter_bytes) + .unwrap(); + expected_ids.extend(query_ids); + expected_distances.extend(query_distances); + } + + assert_eq!(batch.0, expected_ids); + assert_eq!(batch.1, expected_distances); + let stats = batch_reader.last_search_stats(); + assert_eq!(stats.filtered_exhaustive_queries, 4); + assert_eq!(stats.pq_distance_evaluations, 4 * 32); + assert_eq!( + stats.pq_code_loads, 32, + "one four-query tile should load each matching PQ code once" + ); + } + + #[test] + fn diskann_filtered_search_clamps_extreme_l_search_without_panicking() { + let header = crate::diskann_io::DiskAnnHeader::for_layout( + 8, + 1, + 0, + 2, + DiskAnnBuildParams { + max_degree: 1, + build_search_list_size: 1, + ..DiskAnnBuildParams::default() + }, + ) + .unwrap(); + let mut bytes = vec![0u8; header.file_len as usize]; + bytes[..crate::diskann_io::DISKANN_HEADER_SIZE].copy_from_slice(&header.encode()); + let filter = RoaringTreemap::new(); + let mut filter_bytes = Vec::new(); + filter.serialize_into(&mut filter_bytes).unwrap(); + let mut reader = DiskAnnIndexReader::open(Cursor::new(bytes)).unwrap(); + + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + reader.search_with_roaring_filter(&[0.0; 8], 1, usize::MAX, &filter_bytes) + })) + .expect("public io::Result search API must not panic for extreme l_search") + .unwrap(); + + assert_eq!(result.0, vec![-1]); + assert_eq!(result.1, vec![f32::MAX]); + } + + #[test] + fn topk_stability_ignores_padding_and_aggregates_queries() { + let left = [10, 11, -1, 20, 21, 22]; + let right = [11, 10, -1, 20, 99, 22]; + let left_distances = [1.0, 2.0, f32::MAX, 1.0, 2.0, 3.0]; + let right_distances = [2.0, 1.0, f32::MAX, 1.0, 9.0, 3.0]; + + assert_eq!( + topk_result_stability(&left, &left_distances, &right, &right_distances, 3), + 4.0 / 5.0 + ); + assert_eq!(topk_result_stability(&[], &[], &[], &[], 0), 0.0); + assert_eq!( + topk_result_stability(&[1, 2], &[1.0, 2.0], &[1], &[1.0], 2), + 0.0 + ); + } + + #[test] + fn topk_stability_supports_negative_ids_and_duplicate_multiplicity() { + assert_eq!( + topk_result_stability(&[-1, -7], &[1.0, 2.0], &[-1, -8], &[1.0, 2.0], 2), + 0.5 + ); + assert_eq!( + topk_result_stability(&[7, 7], &[1.0, 2.0], &[7, 8], &[1.0, 2.0], 2), + 0.5 + ); + assert_eq!( + topk_result_stability(&[-1, 7], &[f32::MAX, 2.0], &[-1, 7], &[f32::MAX, 2.0], 2,), + 1.0 + ); + } + + #[test] + fn diskann_calibration_selects_and_remembers_smallest_stable_width() { + let dimension = 8; + let indexed_count = 64; + let data = (0..indexed_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32 + (offset / dimension) as f32 * 0.01) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, indexed_count).unwrap(); + index.add(&data, &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let mut reader = DiskAnnIndexReader::open(Cursor::new(bytes)).unwrap(); + let queries = &data[..4 * dimension]; + + assert_eq!(reader.calibrate_l_search(queries, 5).unwrap(), 100); + assert_eq!(reader.calibrated_l_search, Some(100)); + assert_eq!(reader.last_search_stats(), DiskAnnSearchStats::default()); + + let error = reader.calibrate_l_search(&[], 5).unwrap_err(); + assert!(error.to_string().contains("one or more complete vectors")); + } + + #[test] + fn diskann_search_stats_report_actual_filtered_scan() { + let dimension = 8; + let training_count = 256; + let indexed_count = 64; + let data = (0..training_count * dimension) + .map(|offset| ((offset * 31) % 127) as f32) + .collect::>(); + let ids = (1000..1000 + indexed_count as i64).collect::>(); + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 2, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ); + index.train(&data, training_count).unwrap(); + index.add(&data[..indexed_count * dimension], &ids); + let mut bytes = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let mut reader = DiskAnnIndexReader::open(Cursor::new(bytes)).unwrap(); + let mut filter = RoaringTreemap::new(); + filter.insert(ids[0] as u64); + let mut filter_bytes = Vec::new(); + filter.serialize_into(&mut filter_bytes).unwrap(); + + reader + .search_with_roaring_filter(&data[..dimension], 1, 100, &filter_bytes) + .unwrap(); + + let stats = reader.last_search_stats(); + assert_eq!(stats.query_count, 1); + assert_eq!(stats.filtered_exhaustive_queries, 1); + assert_eq!(stats.filtered_graph_queries, 0); + assert_eq!(stats.filtered_graph_fallbacks, 0); + assert_eq!(stats.pq_distance_evaluations, 1); + assert_eq!(stats.rerank_candidate_references, 1); + assert_eq!(stats.rerank_unique_windows, 1); + assert_eq!(stats.rerank_chunks, 1); + assert_eq!( + reader.query_scratch.visited_capacity(), + 0, + "an exhaustive filtered rerank must not allocate graph visited storage" + ); + } +} diff --git a/core/src/distance.rs b/core/src/distance.rs index 0fe983b..99d6cae 100644 --- a/core/src/distance.rs +++ b/core/src/distance.rs @@ -55,6 +55,43 @@ pub fn fvec_l2sqr(a: &[f32], b: &[f32]) -> f32 { fvec_l2sqr_simd(a, b) } +/// Squared L2 distances from one vector to four candidates. +/// +/// Each output uses the same accumulation order as [`fvec_l2sqr`] so callers +/// can batch shared query loads without changing exact pruning decisions. +#[inline] +pub fn fvec_l2sqr_four(a: &[f32], b0: &[f32], b1: &[f32], b2: &[f32], b3: &[f32]) -> [f32; 4] { + for candidate in [b0, b1, b2, b3] { + assert_eq!( + a.len(), + candidate.len(), + "fvec_l2sqr_four inputs must have the same length" + ); + } + fvec_l2sqr_four_simd(a, b0, b1, b2, b3) +} + +/// Returns whether `scale * squared_l2` is strictly greater than `threshold`. +/// +/// L2 components are non-negative, so the architecture-specific kernels can +/// stop after any completed SIMD block that already exceeds the threshold. +#[inline] +pub fn fvec_l2sqr_scaled_exceeds(a: &[f32], b: &[f32], scale: f32, threshold: f32) -> bool { + assert_eq!( + a.len(), + b.len(), + "fvec_l2sqr_scaled_exceeds inputs must have the same length" + ); + assert!( + scale.is_finite() && scale >= 0.0, + "fvec_l2sqr_scaled_exceeds scale must be finite and non-negative" + ); + if threshold < 0.0 { + return true; + } + fvec_l2sqr_scaled_exceeds_simd(a, b, scale, threshold) +} + #[cfg(target_arch = "x86_64")] #[inline] fn fvec_l2sqr_simd(a: &[f32], b: &[f32]) -> f32 { @@ -77,6 +114,50 @@ fn fvec_l2sqr_simd(a: &[f32], b: &[f32]) -> f32 { fvec_l2sqr_scalar(a, b) } +#[cfg(target_arch = "x86_64")] +#[inline] +fn fvec_l2sqr_four_simd(a: &[f32], b0: &[f32], b1: &[f32], b2: &[f32], b3: &[f32]) -> [f32; 4] { + if is_x86_feature_detected!("avx2") && a.len() >= 8 { + unsafe { fvec_l2sqr_four_avx2(a, b0, b1, b2, b3) } + } else { + fvec_l2sqr_four_scalar(a, b0, b1, b2, b3) + } +} + +#[cfg(target_arch = "aarch64")] +#[inline] +fn fvec_l2sqr_four_simd(a: &[f32], b0: &[f32], b1: &[f32], b2: &[f32], b3: &[f32]) -> [f32; 4] { + unsafe { fvec_l2sqr_four_neon(a, b0, b1, b2, b3) } +} + +#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] +#[inline] +fn fvec_l2sqr_four_simd(a: &[f32], b0: &[f32], b1: &[f32], b2: &[f32], b3: &[f32]) -> [f32; 4] { + fvec_l2sqr_four_scalar(a, b0, b1, b2, b3) +} + +#[cfg(target_arch = "x86_64")] +#[inline] +fn fvec_l2sqr_scaled_exceeds_simd(a: &[f32], b: &[f32], scale: f32, threshold: f32) -> bool { + if is_x86_feature_detected!("avx2") && a.len() >= 8 { + unsafe { fvec_l2sqr_scaled_exceeds_avx2(a, b, scale, threshold) } + } else { + fvec_l2sqr_scaled_exceeds_scalar(a, b, scale, threshold) + } +} + +#[cfg(target_arch = "aarch64")] +#[inline] +fn fvec_l2sqr_scaled_exceeds_simd(a: &[f32], b: &[f32], scale: f32, threshold: f32) -> bool { + unsafe { fvec_l2sqr_scaled_exceeds_neon(a, b, scale, threshold) } +} + +#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] +#[inline] +fn fvec_l2sqr_scaled_exceeds_simd(a: &[f32], b: &[f32], scale: f32, threshold: f32) -> bool { + fvec_l2sqr_scaled_exceeds_scalar(a, b, scale, threshold) +} + #[cfg(any( target_arch = "x86_64", not(any(target_arch = "x86_64", target_arch = "aarch64")) @@ -91,6 +172,41 @@ fn fvec_l2sqr_scalar(a: &[f32], b: &[f32]) -> f32 { sum } +#[cfg(any( + target_arch = "x86_64", + not(any(target_arch = "x86_64", target_arch = "aarch64")) +))] +#[inline] +fn fvec_l2sqr_four_scalar(a: &[f32], b0: &[f32], b1: &[f32], b2: &[f32], b3: &[f32]) -> [f32; 4] { + let candidates = [b0, b1, b2, b3]; + let mut sums = [0.0f32; 4]; + for i in 0..a.len() { + let value = a[i]; + for candidate in 0..4 { + let d = value - candidates[candidate][i]; + sums[candidate] += d * d; + } + } + sums +} + +#[cfg(any( + target_arch = "x86_64", + not(any(target_arch = "x86_64", target_arch = "aarch64")) +))] +#[inline] +fn fvec_l2sqr_scaled_exceeds_scalar(a: &[f32], b: &[f32], scale: f32, threshold: f32) -> bool { + let mut sum = 0.0f32; + for i in 0..a.len() { + let d = a[i] - b[i]; + sum += d * d; + if scale * sum > threshold { + return true; + } + } + false +} + #[cfg(target_arch = "x86_64")] #[target_feature(enable = "avx2")] unsafe fn fvec_l2sqr_avx2(a: &[f32], b: &[f32]) -> f32 { @@ -122,6 +238,125 @@ unsafe fn fvec_l2sqr_avx2(a: &[f32], b: &[f32]) -> f32 { result } +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn fvec_l2sqr_four_avx2( + a: &[f32], + b0: &[f32], + b1: &[f32], + b2: &[f32], + b3: &[f32], +) -> [f32; 4] { + use std::arch::x86_64::*; + + let candidates = [b0, b1, b2, b3]; + let mut sums = [_mm256_setzero_ps(); 4]; + let mut i = 0; + while i + 8 <= a.len() { + let va = unsafe { _mm256_loadu_ps(a.as_ptr().add(i)) }; + for candidate in 0..4 { + let vb = unsafe { _mm256_loadu_ps(candidates[candidate].as_ptr().add(i)) }; + let diff = _mm256_sub_ps(va, vb); + sums[candidate] = _mm256_add_ps(sums[candidate], _mm256_mul_ps(diff, diff)); + } + i += 8; + } + + macro_rules! horizontal_sum { + ($sum:expr) => {{ + let hi = _mm256_extractf128_ps::<1>($sum); + let lo = _mm256_castps256_ps128($sum); + let sum128 = _mm_add_ps(lo, hi); + let sum64 = _mm_add_ps(sum128, _mm_movehl_ps(sum128, sum128)); + let sum32 = _mm_add_ss(sum64, _mm_shuffle_ps::<1>(sum64, sum64)); + _mm_cvtss_f32(sum32) + }}; + } + let mut results = [ + horizontal_sum!(sums[0]), + horizontal_sum!(sums[1]), + horizontal_sum!(sums[2]), + horizontal_sum!(sums[3]), + ]; + while i < a.len() { + let value = unsafe { *a.get_unchecked(i) }; + for candidate in 0..4 { + let d = value - unsafe { *candidates[candidate].get_unchecked(i) }; + results[candidate] += d * d; + } + i += 1; + } + results +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn fvec_l2sqr_scaled_exceeds_avx2(a: &[f32], b: &[f32], scale: f32, threshold: f32) -> bool { + use std::arch::x86_64::*; + + let mut total = 0.0f32; + let mut i = 0; + while i + 128 <= a.len() { + let mut sum0 = _mm256_setzero_ps(); + let mut sum1 = _mm256_setzero_ps(); + let mut sum2 = _mm256_setzero_ps(); + let mut sum3 = _mm256_setzero_ps(); + let block_end = i + 128; + while i < block_end { + for (offset, sum) in [ + (0, &mut sum0), + (8, &mut sum1), + (16, &mut sum2), + (24, &mut sum3), + ] { + let va = unsafe { _mm256_loadu_ps(a.as_ptr().add(i + offset)) }; + let vb = unsafe { _mm256_loadu_ps(b.as_ptr().add(i + offset)) }; + let diff = _mm256_sub_ps(va, vb); + *sum = _mm256_add_ps(*sum, _mm256_mul_ps(diff, diff)); + } + i += 32; + } + let sum = _mm256_add_ps(_mm256_add_ps(sum0, sum1), _mm256_add_ps(sum2, sum3)); + let hi = _mm256_extractf128_ps::<1>(sum); + let lo = _mm256_castps256_ps128(sum); + let sum128 = _mm_add_ps(lo, hi); + let sum64 = _mm_add_ps(sum128, _mm_movehl_ps(sum128, sum128)); + let sum32 = _mm_add_ss(sum64, _mm_shuffle_ps::<1>(sum64, sum64)); + total += _mm_cvtss_f32(sum32); + if scale * total > threshold { + return true; + } + } + + let mut tail_sum = _mm256_setzero_ps(); + while i + 8 <= a.len() { + let va = unsafe { _mm256_loadu_ps(a.as_ptr().add(i)) }; + let vb = unsafe { _mm256_loadu_ps(b.as_ptr().add(i)) }; + let diff = _mm256_sub_ps(va, vb); + tail_sum = _mm256_add_ps(tail_sum, _mm256_mul_ps(diff, diff)); + i += 8; + } + let hi = _mm256_extractf128_ps::<1>(tail_sum); + let lo = _mm256_castps256_ps128(tail_sum); + let sum128 = _mm_add_ps(lo, hi); + let sum64 = _mm_add_ps(sum128, _mm_movehl_ps(sum128, sum128)); + let sum32 = _mm_add_ss(sum64, _mm_shuffle_ps::<1>(sum64, sum64)); + total += _mm_cvtss_f32(sum32); + if scale * total > threshold { + return true; + } + + while i < a.len() { + let d = unsafe { *a.get_unchecked(i) - *b.get_unchecked(i) }; + total += d * d; + if scale * total > threshold { + return true; + } + i += 1; + } + false +} + #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] unsafe fn fvec_l2sqr_neon(a: &[f32], b: &[f32]) -> f32 { @@ -154,6 +389,124 @@ unsafe fn fvec_l2sqr_neon(a: &[f32], b: &[f32]) -> f32 { result } +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +unsafe fn fvec_l2sqr_four_neon( + a: &[f32], + b0: &[f32], + b1: &[f32], + b2: &[f32], + b3: &[f32], +) -> [f32; 4] { + use std::arch::aarch64::*; + + let candidates = [b0, b1, b2, b3]; + let mut sums0 = [vdupq_n_f32(0.0); 4]; + let mut sums1 = [vdupq_n_f32(0.0); 4]; + let mut i = 0; + while i + 8 <= a.len() { + let va0 = unsafe { vld1q_f32(a.as_ptr().add(i)) }; + let va1 = unsafe { vld1q_f32(a.as_ptr().add(i + 4)) }; + for candidate in 0..4 { + let vb0 = unsafe { vld1q_f32(candidates[candidate].as_ptr().add(i)) }; + let diff0 = vsubq_f32(va0, vb0); + sums0[candidate] = vmlaq_f32(sums0[candidate], diff0, diff0); + + let vb1 = unsafe { vld1q_f32(candidates[candidate].as_ptr().add(i + 4)) }; + let diff1 = vsubq_f32(va1, vb1); + sums1[candidate] = vmlaq_f32(sums1[candidate], diff1, diff1); + } + i += 8; + } + + let mut results = [ + vaddvq_f32(vaddq_f32(sums0[0], sums1[0])), + vaddvq_f32(vaddq_f32(sums0[1], sums1[1])), + vaddvq_f32(vaddq_f32(sums0[2], sums1[2])), + vaddvq_f32(vaddq_f32(sums0[3], sums1[3])), + ]; + while i < a.len() { + let value = unsafe { *a.get_unchecked(i) }; + for candidate in 0..4 { + let d = value - unsafe { *candidates[candidate].get_unchecked(i) }; + results[candidate] += d * d; + } + i += 1; + } + results +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +unsafe fn fvec_l2sqr_scaled_exceeds_neon(a: &[f32], b: &[f32], scale: f32, threshold: f32) -> bool { + use std::arch::aarch64::*; + + let mut total = 0.0f32; + let mut i = 0; + while i + 128 <= a.len() { + let mut sum0 = vdupq_n_f32(0.0); + let mut sum1 = vdupq_n_f32(0.0); + let mut sum2 = vdupq_n_f32(0.0); + let mut sum3 = vdupq_n_f32(0.0); + let block_end = i + 128; + while i < block_end { + let va0 = unsafe { vld1q_f32(a.as_ptr().add(i)) }; + let vb0 = unsafe { vld1q_f32(b.as_ptr().add(i)) }; + let diff0 = vsubq_f32(va0, vb0); + sum0 = vmlaq_f32(sum0, diff0, diff0); + + let va1 = unsafe { vld1q_f32(a.as_ptr().add(i + 4)) }; + let vb1 = unsafe { vld1q_f32(b.as_ptr().add(i + 4)) }; + let diff1 = vsubq_f32(va1, vb1); + sum1 = vmlaq_f32(sum1, diff1, diff1); + + let va2 = unsafe { vld1q_f32(a.as_ptr().add(i + 8)) }; + let vb2 = unsafe { vld1q_f32(b.as_ptr().add(i + 8)) }; + let diff2 = vsubq_f32(va2, vb2); + sum2 = vmlaq_f32(sum2, diff2, diff2); + + let va3 = unsafe { vld1q_f32(a.as_ptr().add(i + 12)) }; + let vb3 = unsafe { vld1q_f32(b.as_ptr().add(i + 12)) }; + let diff3 = vsubq_f32(va3, vb3); + sum3 = vmlaq_f32(sum3, diff3, diff3); + i += 16; + } + total += vaddvq_f32(vaddq_f32(vaddq_f32(sum0, sum1), vaddq_f32(sum2, sum3))); + if scale * total > threshold { + return true; + } + } + + let mut tail_sum0 = vdupq_n_f32(0.0); + let mut tail_sum1 = vdupq_n_f32(0.0); + while i + 8 <= a.len() { + let va0 = unsafe { vld1q_f32(a.as_ptr().add(i)) }; + let vb0 = unsafe { vld1q_f32(b.as_ptr().add(i)) }; + let diff0 = vsubq_f32(va0, vb0); + tail_sum0 = vmlaq_f32(tail_sum0, diff0, diff0); + + let va1 = unsafe { vld1q_f32(a.as_ptr().add(i + 4)) }; + let vb1 = unsafe { vld1q_f32(b.as_ptr().add(i + 4)) }; + let diff1 = vsubq_f32(va1, vb1); + tail_sum1 = vmlaq_f32(tail_sum1, diff1, diff1); + i += 8; + } + total += vaddvq_f32(vaddq_f32(tail_sum0, tail_sum1)); + if scale * total > threshold { + return true; + } + + while i < a.len() { + let d = unsafe { *a.get_unchecked(i) - *b.get_unchecked(i) }; + total += d * d; + if scale * total > threshold { + return true; + } + i += 1; + } + false +} + /// Squared L2 distance on sub-vectors. pub fn fvec_l2sqr_sub(a: &[f32], a_off: usize, b: &[f32], b_off: usize, len: usize) -> f32 { fvec_l2sqr(&a[a_off..a_off + len], &b[b_off..b_off + len]) @@ -376,20 +729,6 @@ pub fn fvec_distance(query: &[f32], vector: &[f32], metric: MetricType) -> f32 { } } -pub(crate) fn fvec_distance_with_norms( - a: &[f32], - b: &[f32], - metric: MetricType, - a_norm: f32, - b_norm: f32, -) -> f32 { - match metric { - MetricType::L2 => fvec_l2sqr(a, b), - MetricType::InnerProduct => -fvec_inner_product(a, b), - MetricType::Cosine => fvec_cosine_distance_with_norms(a, b, a_norm, b_norm), - } -} - #[derive(Debug, Clone, Copy)] pub(crate) struct QueryDistance<'a> { query: &'a [f32], @@ -838,6 +1177,47 @@ mod tests { assert!((fvec_l2sqr(&a, &b) - 27.0).abs() < 1e-6); } + #[test] + fn test_l2sqr_four_matches_individual_simd_kernels_exactly() { + for len in [0, 1, 7, 8, 17, 67] { + let query = (0..len) + .map(|index| ((index * 29 + 7) % 41) as f32 * 0.125 - 2.0) + .collect::>(); + let candidates = std::array::from_fn::<_, 4, _>(|candidate| { + (0..len) + .map(|index| { + ((index * (candidate + 11) + candidate * 17 + 3) % 53) as f32 * 0.0625 - 1.5 + }) + .collect::>() + }); + + let actual = fvec_l2sqr_four( + &query, + &candidates[0], + &candidates[1], + &candidates[2], + &candidates[3], + ); + let expected = + std::array::from_fn(|candidate| fvec_l2sqr(&query, &candidates[candidate])); + + assert_eq!(actual, expected, "length {len}"); + } + } + + #[test] + fn test_l2sqr_scaled_exceeds_uses_a_strict_threshold_across_multiple_blocks() { + let a = [0.0f32; 129]; + let mut b = [0.0f32; 129]; + b[0] = 2.0; + b[128] = 3.0; + + assert!(!fvec_l2sqr_scaled_exceeds(&a, &b, 2.0, 26.0)); + assert!(fvec_l2sqr_scaled_exceeds(&a, &b, 2.0, 25.0)); + assert!(fvec_l2sqr_scaled_exceeds(&a, &b, 1.0, 3.0)); + assert!(!fvec_l2sqr_scaled_exceeds(&a, &a, 2.0, 0.0)); + } + #[test] #[should_panic(expected = "fvec_l2sqr inputs must have the same length")] fn test_l2sqr_rejects_mismatched_lengths() { diff --git a/core/src/hnsw.rs b/core/src/hnsw.rs deleted file mode 100644 index 0d81fc8..0000000 --- a/core/src/hnsw.rs +++ /dev/null @@ -1,1397 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::distance::{ - fvec_distance, fvec_distance_with_norms, fvec_norm_l2sqr, MetricType, QueryDistance, -}; -use rayon::prelude::*; -use std::cmp::Reverse; -use std::collections::BinaryHeap; -use std::io; -use std::sync::RwLock; - -// Parallel insertion pays off once an IVF list is large enough to amortize -// lock and per-worker visited-set setup. Smaller lists stay on the lean -// sequential path to avoid nested Rayon overhead. -const PARALLEL_BUILD_MIN_N: usize = 5_000; - -#[derive(Debug, Clone, Copy)] -pub struct HnswBuildParams { - pub m: usize, - pub ef_construction: usize, - pub max_level: usize, -} - -impl Default for HnswBuildParams { - fn default() -> Self { - Self { - m: 20, - ef_construction: 150, - max_level: 7, - } - } -} - -impl HnswBuildParams { - pub fn sanitized(self) -> Self { - Self { - m: self.m.max(1), - ef_construction: self.ef_construction.max(1), - max_level: self.max_level.max(1), - } - } -} - -#[derive(Debug, Clone)] -pub struct HnswGraph { - d: usize, - metric: MetricType, - vectors: Vec, - vector_norms: Option>, - levels: Vec, - neighbors: Vec>>, - entry_point: usize, - max_observed_level: usize, - params: HnswBuildParams, -} - -impl HnswGraph { - pub fn build( - vectors: &[f32], - n: usize, - d: usize, - metric: MetricType, - params: HnswBuildParams, - ) -> io::Result { - let expected_len = n.checked_mul(d).ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "n * dimension overflows usize") - })?; - if vectors.len() < expected_len { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "vector data length {} is shorter than n*d {}", - vectors.len(), - expected_len - ), - )); - } - - Self::build_owned(vectors[..expected_len].to_vec(), n, d, metric, params) - } - - pub(crate) fn build_owned( - vectors: Vec, - n: usize, - d: usize, - metric: MetricType, - params: HnswBuildParams, - ) -> io::Result { - let expected_len = n.checked_mul(d).ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "n * dimension overflows usize") - })?; - if vectors.len() != expected_len { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "vector data length {} does not match n*d {}", - vectors.len(), - expected_len - ), - )); - } - - let params = params.sanitized(); - if n >= PARALLEL_BUILD_MIN_N { - return Ok(Self::build_parallel(vectors, n, d, metric, params)); - } - - let vector_norms = vector_norms_for(metric, &vectors, n, d); - let mut graph = HnswGraph { - d, - metric, - vectors, - vector_norms, - levels: Vec::with_capacity(n), - neighbors: Vec::with_capacity(n), - entry_point: 0, - max_observed_level: 0, - params, - }; - - let mut workspace = HnswBuildWorkspace::new(n, params.ef_construction); - for node in 0..n { - graph.insert(node, &mut workspace); - } - Ok(graph) - } - - fn build_parallel( - vectors: Vec, - n: usize, - d: usize, - metric: MetricType, - params: HnswBuildParams, - ) -> Self { - let vector_norms = vector_norms_for(metric, &vectors, n, d); - let levels = parallel_build_levels(n, params); - let max_observed_level = levels.iter().copied().max().unwrap_or(0); - let nodes = levels - .iter() - .map(|&level| RwLock::new(ParallelBuildNode::new(level))) - .collect::>(); - - { - let builder = ParallelHnswBuilder { - d, - metric, - vectors: &vectors, - vector_norms: vector_norms.as_deref(), - levels: &levels, - nodes: &nodes, - params, - entry_point: 0, - max_observed_level, - }; - (1..n).into_par_iter().for_each_init( - || HnswBuildWorkspace::new(n, params.ef_construction), - |workspace, node| builder.insert(node, workspace), - ); - } - - let neighbors = nodes - .into_iter() - .map(|node| { - node.into_inner() - .expect("parallel HNSW builder lock poisoned") - .levels - .into_iter() - .map(|level| level.into_iter().map(|neighbor| neighbor.id).collect()) - .collect() - }) - .collect(); - - Self { - d, - metric, - vectors, - vector_norms, - levels, - neighbors, - entry_point: 0, - max_observed_level, - params, - } - } - - #[allow(clippy::too_many_arguments)] - pub(crate) fn from_parts( - vectors: Vec, - n: usize, - d: usize, - metric: MetricType, - levels: Vec, - neighbors: Vec>>, - entry_point: usize, - max_observed_level: usize, - params: HnswBuildParams, - ) -> io::Result { - let expected_len = n.checked_mul(d).ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "n * dimension overflows usize") - })?; - if vectors.len() != expected_len { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "graph vector length {} does not match n*d {}", - vectors.len(), - expected_len - ), - )); - } - if levels.len() != n || neighbors.len() != n { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "graph level metadata does not match vector count", - )); - } - let vector_norms = vector_norms_for(metric, &vectors, n, d); - if n == 0 { - return Ok(Self { - d, - metric, - vectors, - vector_norms, - levels, - neighbors, - entry_point: 0, - max_observed_level: 0, - params: params.sanitized(), - }); - } - if entry_point >= n { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("graph entry point {} out of range {}", entry_point, n), - )); - } - let observed = levels.iter().copied().max().unwrap_or(0); - if max_observed_level != observed { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "graph max level {} does not match observed {}", - max_observed_level, observed - ), - )); - } - if levels[entry_point] < max_observed_level { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "graph entry point does not reach max observed level", - )); - } - for node in 0..n { - if neighbors[node].len() != levels[node] + 1 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("graph node {} has invalid level adjacency", node), - )); - } - for (level, level_neighbors) in neighbors[node].iter().enumerate() { - for &neighbor in level_neighbors { - if neighbor >= n || levels[neighbor] < level { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "graph edge {} -> {} at level {} is invalid", - node, neighbor, level - ), - )); - } - } - } - } - Ok(Self { - d, - metric, - vectors, - vector_norms, - levels, - neighbors, - entry_point, - max_observed_level, - params: params.sanitized(), - }) - } - - pub fn search(&self, query: &[f32], k: usize, ef: usize) -> Vec<(usize, f32)> { - let mut workspace = HnswSearchWorkspace::new(ef.max(k)); - self.search_with_reusable_workspace(query, k, ef, &mut workspace) - .to_vec() - } - - pub(crate) fn search_with_reusable_workspace<'a>( - &self, - query: &[f32], - k: usize, - ef: usize, - workspace: &'a mut HnswSearchWorkspace, - ) -> &'a [(usize, f32)] { - workspace.output_pairs.clear(); - if self.levels.is_empty() || k == 0 { - return &workspace.output_pairs; - } - let ef = ef.max(k); - workspace.prepare(self.levels.len(), ef); - - let query_distance = QueryDistance::new(query, self.metric); - let mut ep = self.entry_point; - let mut ep_dist = self.distance_to_query(&query_distance, ep); - for level in (1..=self.max_observed_level).rev() { - let (next, dist) = self.greedy_search_query(&query_distance, ep, ep_dist, level); - ep = next; - ep_dist = dist; - } - - let current_mark = workspace.visit_mark; - self.search_layer_query_into(&query_distance, ep, ef, 0, current_mark, workspace); - workspace.visit_mark = advance_visit_mark(&mut workspace.visited, current_mark); - workspace.output_pairs.extend( - workspace - .output - .iter() - .take(k) - .map(|node| (node.id, node.dist)), - ); - &workspace.output_pairs - } - - pub fn len(&self) -> usize { - self.levels.len() - } - - pub fn is_empty(&self) -> bool { - self.levels.is_empty() - } - - pub fn max_degree(&self) -> usize { - self.neighbors - .iter() - .flat_map(|levels| levels.iter().map(Vec::len)) - .max() - .unwrap_or(0) - } - - pub(crate) fn vectors(&self) -> &[f32] { - &self.vectors - } - - pub(crate) fn levels(&self) -> &[usize] { - &self.levels - } - - pub(crate) fn neighbors(&self) -> &[Vec>] { - &self.neighbors - } - - pub(crate) fn entry_point(&self) -> usize { - self.entry_point - } - - pub(crate) fn max_observed_level(&self) -> usize { - self.max_observed_level - } - - fn insert(&mut self, node: usize, workspace: &mut HnswBuildWorkspace) { - let level = random_level(node, self.params.m, self.params.max_level); - self.levels.push(level); - self.neighbors.push(vec![Vec::new(); level + 1]); - - if node == 0 { - self.entry_point = 0; - self.max_observed_level = level; - return; - } - - let mut ep = self.entry_point; - let mut ep_dist = self.distance_between(node, ep); - - for layer in ((level + 1)..=self.max_observed_level).rev() { - let (next, dist) = self.greedy_search_node(node, ep, ep_dist, layer); - ep = next; - ep_dist = dist; - } - - for layer in (0..=level.min(self.max_observed_level)).rev() { - self.search_layer_node_with_workspace(node, ep, layer, workspace); - let next_ep = workspace.output.first().map(|candidate| candidate.id); - let selected = workspace - .select_output_neighbors_by(self.max_neighbors(layer), |candidate, neighbor| { - self.distance_between(candidate, neighbor) - }); - self.connect_selected(node, selected, layer); - if let Some(best) = next_ep { - ep = best; - } - } - - if level > self.max_observed_level { - self.entry_point = node; - self.max_observed_level = level; - } - } - - fn connect_selected(&mut self, node: usize, selected: &[ScoredNode], level: usize) { - let node_neighbors = &mut self.neighbors[node][level]; - node_neighbors.clear(); - node_neighbors.extend(selected.iter().map(|neighbor| neighbor.id)); - - for neighbor in selected { - let neighbor_id = neighbor.id; - if level < self.neighbors[neighbor_id].len() - && !self.neighbors[neighbor_id][level].contains(&node) - { - self.connect_reverse(node, neighbor_id, neighbor.dist, level); - } - } - } - - fn connect_reverse(&mut self, node: usize, neighbor: usize, distance: f32, level: usize) { - let max_neighbors = self.max_neighbors(level); - { - let neighbors = &self.neighbors[neighbor][level]; - if neighbors.len() >= max_neighbors - && !neighbors - .iter() - .any(|&existing| distance < self.distance_between(neighbor, existing)) - { - return; - } - } - - self.neighbors[neighbor][level].push(node); - if self.neighbors[neighbor][level].len() > max_neighbors { - let pruned = self.pruned_neighbors(neighbor, level, max_neighbors); - self.neighbors[neighbor][level] = pruned; - } - } - - fn pruned_neighbors(&self, node: usize, level: usize, max_neighbors: usize) -> Vec { - let neighbors = &self.neighbors[node][level]; - if neighbors.len() <= max_neighbors { - return neighbors.clone(); - } - - let ranked: Vec = neighbors - .iter() - .map(|&id| ScoredNode { - id, - dist: self.distance_between(node, id), - }) - .collect(); - self.select_neighbors(ranked, max_neighbors) - .into_iter() - .map(|neighbor| neighbor.id) - .collect() - } - - fn select_neighbors( - &self, - mut candidates: Vec, - max_neighbors: usize, - ) -> Vec { - candidates.sort_unstable_by(|a, b| a.dist.total_cmp(&b.dist)); - self.select_neighbors_sorted(&candidates, max_neighbors) - } - - fn select_neighbors_sorted( - &self, - candidates: &[ScoredNode], - max_neighbors: usize, - ) -> Vec { - let mut selected = Vec::with_capacity(max_neighbors.min(candidates.len())); - select_neighbors_sorted_into(candidates, max_neighbors, &mut selected, |a, b| { - self.distance_between(a, b) - }); - selected - } - - fn greedy_search_query( - &self, - distance: &QueryDistance<'_>, - mut current: usize, - mut current_dist: f32, - level: usize, - ) -> (usize, f32) { - loop { - let mut best = current; - let mut best_dist = current_dist; - for &neighbor in self.neighbors_at(current, level) { - let dist = self.distance_to_query(distance, neighbor); - if dist < best_dist { - best = neighbor; - best_dist = dist; - } - } - if best == current { - return (current, current_dist); - } - current = best; - current_dist = best_dist; - } - } - - fn greedy_search_node( - &self, - node: usize, - mut current: usize, - mut current_dist: f32, - level: usize, - ) -> (usize, f32) { - loop { - let mut best = current; - let mut best_dist = current_dist; - for &neighbor in self.neighbors_at(current, level) { - let dist = self.distance_between(node, neighbor); - if dist < best_dist { - best = neighbor; - best_dist = dist; - } - } - if best == current { - return (current, current_dist); - } - current = best; - current_dist = best_dist; - } - } - - fn search_layer_query_into( - &self, - distance: &QueryDistance<'_>, - entry: usize, - ef: usize, - level: usize, - visit_mark: usize, - workspace: &mut HnswSearchWorkspace, - ) { - self.search_layer_into( - entry, - ef, - level, - &mut workspace.visited, - visit_mark, - &mut workspace.candidates, - &mut workspace.results, - &mut workspace.output, - |id| self.distance_to_query(distance, id), - ); - } - - fn search_layer_node_with_workspace( - &self, - node: usize, - entry: usize, - level: usize, - workspace: &mut HnswBuildWorkspace, - ) { - let visit_mark = workspace.visit_mark; - self.search_layer_into( - entry, - self.params.ef_construction, - level, - &mut workspace.visited, - visit_mark, - &mut workspace.candidates, - &mut workspace.results, - &mut workspace.output, - |id| self.distance_between(node, id), - ); - workspace.visit_mark = advance_visit_mark(&mut workspace.visited, visit_mark); - } - - #[allow(clippy::too_many_arguments)] - fn search_layer_into( - &self, - entry: usize, - ef: usize, - level: usize, - visited: &mut [usize], - visit_mark: usize, - candidates: &mut BinaryHeap>, - results: &mut BinaryHeap, - output: &mut Vec, - mut distance: impl FnMut(usize) -> f32, - ) { - candidates.clear(); - results.clear(); - output.clear(); - - let entry_dist = distance(entry); - visited[entry] = visit_mark; - - candidates.push(Reverse(HeapNode { - id: entry, - dist: entry_dist, - })); - - results.push(HeapNode { - id: entry, - dist: entry_dist, - }); - - while let Some(Reverse(current)) = candidates.pop() { - let worst = results - .peek() - .map(|node| node.dist) - .unwrap_or(f32::INFINITY); - if current.dist > worst && results.len() >= ef { - break; - } - - for &neighbor in self.neighbors_at(current.id, level) { - if visited[neighbor] == visit_mark { - continue; - } - visited[neighbor] = visit_mark; - let dist = distance(neighbor); - let worst = results - .peek() - .map(|node| node.dist) - .unwrap_or(f32::INFINITY); - if results.len() < ef || dist < worst { - candidates.push(Reverse(HeapNode { id: neighbor, dist })); - results.push(HeapNode { id: neighbor, dist }); - if results.len() > ef { - results.pop(); - } - } - } - } - - output.extend(results.drain().map(|node| ScoredNode { - id: node.id, - dist: node.dist, - })); - output.sort_unstable_by(|a, b| a.dist.total_cmp(&b.dist)); - } - - fn max_neighbors(&self, level: usize) -> usize { - if level == 0 { - self.params.m * 2 - } else { - self.params.m - } - } - - fn neighbors_at(&self, node: usize, level: usize) -> &[usize] { - self.neighbors - .get(node) - .and_then(|levels| levels.get(level)) - .map(Vec::as_slice) - .unwrap_or(&[]) - } - - fn distance_between(&self, a: usize, b: usize) -> f32 { - let va = &self.vectors[a * self.d..(a + 1) * self.d]; - let vb = &self.vectors[b * self.d..(b + 1) * self.d]; - match self.metric { - MetricType::Cosine => fvec_distance_with_norms( - va, - vb, - self.metric, - self.vector_norm(a), - self.vector_norm(b), - ), - _ => fvec_distance(va, vb, self.metric), - } - } - - fn distance_to_query(&self, query_distance: &QueryDistance<'_>, id: usize) -> f32 { - let vector = &self.vectors[id * self.d..(id + 1) * self.d]; - query_distance.distance_to(vector, self.vector_norms.as_ref().map(|norms| norms[id])) - } - - fn vector_norm(&self, id: usize) -> f32 { - self.vector_norms - .as_ref() - .map(|norms| norms[id]) - .unwrap_or_else(|| { - fvec_norm_l2sqr(&self.vectors[id * self.d..(id + 1) * self.d]).sqrt() - }) - } -} - -#[derive(Debug, Clone, Copy)] -struct ScoredNode { - id: usize, - dist: f32, -} - -#[derive(Debug, Clone, Copy, PartialEq)] -struct HeapNode { - id: usize, - dist: f32, -} - -impl Eq for HeapNode {} - -impl PartialOrd for HeapNode { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for HeapNode { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.dist.total_cmp(&other.dist) - } -} - -pub(crate) struct HnswSearchWorkspace { - visited: Vec, - visit_mark: usize, - candidates: BinaryHeap>, - results: BinaryHeap, - output: Vec, - output_pairs: Vec<(usize, f32)>, -} - -impl HnswSearchWorkspace { - pub(crate) fn new(ef: usize) -> Self { - Self { - visited: Vec::new(), - visit_mark: 1, - candidates: BinaryHeap::with_capacity(ef), - results: BinaryHeap::with_capacity(ef), - output: Vec::with_capacity(ef), - output_pairs: Vec::with_capacity(ef), - } - } - - fn prepare(&mut self, graph_len: usize, ef: usize) { - if self.visited.len() < graph_len { - self.visited.resize(graph_len, 0); - } - self.candidates - .reserve(ef.saturating_sub(self.candidates.capacity())); - self.results - .reserve(ef.saturating_sub(self.results.capacity())); - self.output - .reserve(ef.saturating_sub(self.output.capacity())); - self.output_pairs - .reserve(ef.saturating_sub(self.output_pairs.capacity())); - } -} - -struct HnswBuildWorkspace { - visited: Vec, - visit_mark: usize, - candidates: BinaryHeap>, - results: BinaryHeap, - output: Vec, - selected: Vec, -} - -impl HnswBuildWorkspace { - fn new(n: usize, ef_construction: usize) -> Self { - Self { - visited: vec![0; n], - visit_mark: 1, - candidates: BinaryHeap::with_capacity(ef_construction), - results: BinaryHeap::with_capacity(ef_construction), - output: Vec::with_capacity(ef_construction), - selected: Vec::new(), - } - } - - fn select_output_neighbors_by( - &mut self, - max_neighbors: usize, - distance_between: impl FnMut(usize, usize) -> f32, - ) -> &[ScoredNode] { - select_neighbors_sorted_into( - &self.output, - max_neighbors, - &mut self.selected, - distance_between, - ); - &self.selected - } -} - -struct ParallelHnswBuilder<'a> { - d: usize, - metric: MetricType, - vectors: &'a [f32], - vector_norms: Option<&'a [f32]>, - levels: &'a [usize], - nodes: &'a [RwLock], - params: HnswBuildParams, - entry_point: usize, - max_observed_level: usize, -} - -impl ParallelHnswBuilder<'_> { - fn insert(&self, node: usize, workspace: &mut HnswBuildWorkspace) { - let level = self.nodes[node] - .read() - .expect("parallel HNSW builder lock poisoned") - .level(); - let mut ep = self.entry_point; - let mut ep_dist = self.distance_between(node, ep); - - for layer in ((level + 1)..=self.max_observed_level).rev() { - let (next, dist) = self.greedy_search_node(node, ep, ep_dist, layer); - ep = next; - ep_dist = dist; - } - - for layer in (0..=level.min(self.max_observed_level)).rev() { - self.search_layer_node(node, ep, layer, workspace); - let next_ep = workspace.output.first().map(|candidate| candidate.id); - let selected = workspace - .select_output_neighbors_by(self.max_neighbors(layer), |candidate, neighbor| { - self.distance_between(candidate, neighbor) - }); - self.connect_selected(node, selected, layer); - if let Some(best) = next_ep { - ep = best; - } - } - } - - fn connect_selected(&self, node: usize, selected: &[ScoredNode], level: usize) { - { - let mut current = self.nodes[node] - .write() - .expect("parallel HNSW builder lock poisoned"); - let current_neighbors = &mut current.levels[level]; - current_neighbors.clear(); - current_neighbors.extend_from_slice(selected); - } - - for neighbor in selected { - let neighbor_id = neighbor.id; - if self.levels[neighbor_id] >= level { - self.connect_reverse(node, neighbor_id, neighbor.dist, level); - } - } - } - - fn connect_reverse(&self, node: usize, neighbor: usize, distance: f32, level: usize) { - let max_neighbors = self.max_neighbors(level); - { - let neighbor_node = self.nodes[neighbor] - .read() - .expect("parallel HNSW builder lock poisoned"); - let neighbors = &neighbor_node.levels[level]; - if neighbors.iter().any(|existing| existing.id == node) { - return; - } - if neighbors.len() >= max_neighbors - && !neighbors.iter().any(|existing| distance < existing.dist) - { - return; - } - } - - let mut neighbor_node = self.nodes[neighbor] - .write() - .expect("parallel HNSW builder lock poisoned"); - let neighbors = &mut neighbor_node.levels[level]; - if neighbors.iter().any(|existing| existing.id == node) { - return; - } - neighbors.push(ScoredNode { - id: node, - dist: distance, - }); - if neighbors.len() > max_neighbors { - let candidates = std::mem::take(neighbors); - let pruned = self.select_neighbors(candidates, max_neighbors); - *neighbors = pruned; - } - } - - fn greedy_search_node( - &self, - node: usize, - mut current: usize, - mut current_dist: f32, - level: usize, - ) -> (usize, f32) { - loop { - let mut best = current; - let mut best_dist = current_dist; - self.for_each_neighbor(current, level, |neighbor| { - let dist = self.distance_between(node, neighbor); - if dist < best_dist { - best = neighbor; - best_dist = dist; - } - }); - if best == current { - return (current, current_dist); - } - current = best; - current_dist = best_dist; - } - } - - fn search_layer_node( - &self, - node: usize, - entry: usize, - level: usize, - workspace: &mut HnswBuildWorkspace, - ) { - workspace.candidates.clear(); - workspace.results.clear(); - workspace.output.clear(); - - let visit_mark = workspace.visit_mark; - let entry_dist = self.distance_between(node, entry); - workspace.visited[entry] = visit_mark; - workspace.candidates.push(Reverse(HeapNode { - id: entry, - dist: entry_dist, - })); - workspace.results.push(HeapNode { - id: entry, - dist: entry_dist, - }); - - while let Some(Reverse(current)) = workspace.candidates.pop() { - let worst = workspace - .results - .peek() - .map(|node| node.dist) - .unwrap_or(f32::INFINITY); - if current.dist > worst && workspace.results.len() >= self.params.ef_construction { - break; - } - - self.for_each_neighbor(current.id, level, |neighbor| { - if workspace.visited[neighbor] == visit_mark { - return; - } - workspace.visited[neighbor] = visit_mark; - let dist = self.distance_between(node, neighbor); - let worst = workspace - .results - .peek() - .map(|node| node.dist) - .unwrap_or(f32::INFINITY); - if workspace.results.len() < self.params.ef_construction || dist < worst { - workspace - .candidates - .push(Reverse(HeapNode { id: neighbor, dist })); - workspace.results.push(HeapNode { id: neighbor, dist }); - if workspace.results.len() > self.params.ef_construction { - workspace.results.pop(); - } - } - }); - } - - workspace - .output - .extend(workspace.results.drain().map(|node| ScoredNode { - id: node.id, - dist: node.dist, - })); - workspace - .output - .sort_unstable_by(|a, b| a.dist.total_cmp(&b.dist)); - workspace.visit_mark = advance_visit_mark(&mut workspace.visited, visit_mark); - } - - fn select_neighbors( - &self, - mut candidates: Vec, - max_neighbors: usize, - ) -> Vec { - candidates.sort_unstable_by(|a, b| a.dist.total_cmp(&b.dist)); - self.select_neighbors_sorted(&candidates, max_neighbors) - } - - fn select_neighbors_sorted( - &self, - candidates: &[ScoredNode], - max_neighbors: usize, - ) -> Vec { - let mut selected = Vec::with_capacity(max_neighbors.min(candidates.len())); - select_neighbors_sorted_into(candidates, max_neighbors, &mut selected, |a, b| { - self.distance_between(a, b) - }); - selected - } - - fn for_each_neighbor(&self, node: usize, level: usize, mut f: impl FnMut(usize)) { - let node = self.nodes[node] - .read() - .expect("parallel HNSW builder lock poisoned"); - if let Some(neighbors) = node.levels.get(level) { - for neighbor in neighbors { - f(neighbor.id); - } - } - } - - fn max_neighbors(&self, level: usize) -> usize { - if level == 0 { - self.params.m * 2 - } else { - self.params.m - } - } - - fn distance_between(&self, a: usize, b: usize) -> f32 { - let va = &self.vectors[a * self.d..(a + 1) * self.d]; - let vb = &self.vectors[b * self.d..(b + 1) * self.d]; - match self.metric { - MetricType::Cosine => fvec_distance_with_norms( - va, - vb, - self.metric, - self.vector_norm(a), - self.vector_norm(b), - ), - _ => fvec_distance(va, vb, self.metric), - } - } - - fn vector_norm(&self, id: usize) -> f32 { - self.vector_norms.map(|norms| norms[id]).unwrap_or_else(|| { - fvec_norm_l2sqr(&self.vectors[id * self.d..(id + 1) * self.d]).sqrt() - }) - } -} - -struct ParallelBuildNode { - levels: Vec>, -} - -impl ParallelBuildNode { - fn new(level: usize) -> Self { - Self { - levels: vec![Vec::new(); level + 1], - } - } - - fn level(&self) -> usize { - self.levels.len() - 1 - } -} - -fn parallel_build_levels(n: usize, params: HnswBuildParams) -> Vec { - let mut levels: Vec<_> = (0..n) - .map(|node| random_level(node, params.m, params.max_level)) - .collect(); - if let Some(first) = levels.first_mut() { - // LanceDB keeps the fixed entry point reachable from every configured - // layer during parallel build. Mirroring that avoids a serialized - // "promote newest max-level node" phase while preserving high-level - // search quality. - *first = params.max_level - 1; - } - levels -} - -fn select_neighbors_sorted_into( - candidates: &[ScoredNode], - max_neighbors: usize, - selected: &mut Vec, - mut distance_between: impl FnMut(usize, usize) -> f32, -) { - selected.clear(); - if candidates.len() <= max_neighbors { - selected.extend_from_slice(candidates); - return; - } - - selected.reserve(max_neighbors.saturating_sub(selected.len())); - for &candidate in candidates { - if selected.len() >= max_neighbors { - break; - } - let closer_to_selected = selected - .iter() - .any(|neighbor| distance_between(candidate.id, neighbor.id) < candidate.dist); - if !closer_to_selected { - selected.push(candidate); - } - } - for &candidate in candidates { - if selected.len() >= max_neighbors { - break; - } - if !selected.iter().any(|neighbor| neighbor.id == candidate.id) { - selected.push(candidate); - } - } -} - -fn vector_norms_for(metric: MetricType, vectors: &[f32], n: usize, d: usize) -> Option> { - if metric != MetricType::Cosine { - return None; - } - Some( - (0..n) - .map(|id| fvec_norm_l2sqr(&vectors[id * d..(id + 1) * d]).sqrt()) - .collect(), - ) -} - -fn random_level(node: usize, m: usize, max_level: usize) -> usize { - if node == 0 || max_level <= 1 { - // Keep the first insertion deterministic. Later higher-level nodes replace - // the entry point as they appear, while tiny lists naturally stay flat. - return 0; - } - let mut x = splitmix64(node as u64 + 0x9E37_79B9_7F4A_7C15); - let mut level = 0; - let threshold = (u64::MAX / m.max(2) as u64).max(1); - while level + 1 < max_level && x < threshold { - level += 1; - x = splitmix64(x); - } - level -} - -fn advance_visit_mark(visited: &mut [usize], visit_mark: usize) -> usize { - visit_mark.checked_add(1).unwrap_or_else(|| { - visited.fill(0); - 1 - }) -} - -fn splitmix64(mut x: u64) -> u64 { - x = x.wrapping_add(0x9E37_79B9_7F4A_7C15); - let mut z = x; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); - z ^ (z >> 31) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::distance::MetricType; - - #[test] - fn test_hnsw_recalls_query_vector_on_single_partition() { - let d = 4; - let n = 128; - let data: Vec = (0..n) - .flat_map(|i| [i as f32 * 0.01, 1.0, 2.0, 3.0]) - .collect(); - let params = HnswBuildParams { - m: 8, - ef_construction: 32, - max_level: 6, - }; - - let graph = HnswGraph::build(&data, n, d, MetricType::L2, params).unwrap(); - let query_id = 17; - let results = graph.search(&data[query_id * d..(query_id + 1) * d], 5, 32); - - assert_eq!(results[0].0, query_id); - assert_eq!(results[0].1, 0.0); - } - - #[test] - fn test_hnsw_empty_graph_returns_no_results() { - let graph = - HnswGraph::build(&[], 0, 4, MetricType::L2, HnswBuildParams::default()).unwrap(); - - assert!(graph.search(&[0.0, 0.0, 0.0, 0.0], 10, 20).is_empty()); - assert!(graph.is_empty()); - assert_eq!(graph.len(), 0); - assert_eq!(graph.max_degree(), 0); - } - - #[test] - fn test_hnsw_build_rejects_short_vector_input() { - let err = HnswGraph::build( - &[0.0, 1.0, 2.0], - 2, - 2, - MetricType::L2, - HnswBuildParams::default(), - ) - .unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - assert!(err.to_string().contains("shorter than n*d")); - } - - #[test] - fn test_hnsw_respects_neighbor_degree_bound() { - let d = 8; - let n = 512; - let data = generate_clustered_data(n, d, 16); - let params = HnswBuildParams { - m: 12, - ef_construction: 100, - max_level: 6, - }; - - let graph = HnswGraph::build(&data, n, d, MetricType::L2, params).unwrap(); - - assert_eq!(graph.len(), n); - assert!(graph.max_degree() <= params.m * 2); - } - - #[test] - fn test_hnsw_large_partition_recall_tracks_exact_search() { - let d = 16; - let n = 4096; - let nq = 32; - let k = 10; - let data = generate_clustered_data(n, d, 32); - let params = HnswBuildParams { - m: 16, - ef_construction: 200, - max_level: 7, - }; - - let graph = HnswGraph::build(&data, n, d, MetricType::L2, params).unwrap(); - let mut hits = 0usize; - for qi in 0..nq { - let query = &data[qi * d..(qi + 1) * d]; - let expected = exact_topk(&data, n, d, query, k); - let actual = graph.search(query, k, 200); - hits += actual - .iter() - .filter(|(id, _)| expected.contains(id)) - .count(); - } - - let recall = hits as f32 / (nq * k) as f32; - assert!(recall >= 0.95, "recall={}", recall); - } - - #[test] - fn test_hnsw_parallel_build_large_partition_recall_tracks_exact_search() { - let d = 16; - let n = PARALLEL_BUILD_MIN_N + 512; - let nq = 32; - let k = 10; - let data = generate_clustered_data(n, d, 32); - let params = HnswBuildParams { - m: 16, - ef_construction: 200, - max_level: 7, - }; - - let graph = HnswGraph::build(&data, n, d, MetricType::L2, params).unwrap(); - let mut hits = 0usize; - for qi in 0..nq { - let query = &data[qi * d..(qi + 1) * d]; - let expected = exact_topk(&data, n, d, query, k); - let actual = graph.search(query, k, 400); - hits += actual - .iter() - .filter(|(id, _)| expected.contains(id)) - .count(); - } - - let recall = hits as f32 / (nq * k) as f32; - // Parallel graph construction is schedule-dependent; keep the bar high - // enough to catch regressions without making the test flaky. - assert!(recall >= 0.90, "recall={}", recall); - assert!(graph.max_degree() <= params.m * 2); - } - - #[test] - fn test_hnsw_neighbor_selection_backfills_after_diversification() { - let d = 1; - let data = vec![0.0, 1.0, 2.0, 3.0]; - let graph = HnswGraph::build( - &data, - 4, - d, - MetricType::L2, - HnswBuildParams { - m: 2, - ef_construction: 4, - max_level: 1, - }, - ) - .unwrap(); - let candidates = vec![ - ScoredNode { id: 1, dist: 1.0 }, - ScoredNode { id: 2, dist: 2.0 }, - ScoredNode { id: 3, dist: 3.0 }, - ]; - - let selected = graph.select_neighbors(candidates, 3); - - assert_eq!(selected.len(), 3); - } - - #[test] - fn test_hnsw_pruning_keeps_diverse_neighbors() { - let graph = HnswGraph::from_parts( - vec![0.0, 0.0, 1.0, 0.0, 1.1, 0.0, 0.0, 2.0], - 4, - 2, - MetricType::L2, - vec![0, 0, 0, 0], - vec![ - vec![vec![1, 2, 3]], - vec![vec![]], - vec![vec![]], - vec![vec![]], - ], - 0, - 0, - HnswBuildParams::default(), - ) - .unwrap(); - - let selected = graph.pruned_neighbors(0, 0, 2); - - assert_eq!(selected, vec![1, 3]); - } - - #[test] - fn test_hnsw_greedy_search_chooses_best_improving_neighbor() { - let graph = HnswGraph::from_parts( - vec![0.0, 5.0, 2.0], - 3, - 1, - MetricType::L2, - vec![0, 0, 0], - vec![vec![vec![1, 2]], vec![vec![]], vec![vec![]]], - 0, - 0, - HnswBuildParams::default(), - ) - .unwrap(); - - let distance = QueryDistance::new(&[2.0], MetricType::L2); - let (next, dist) = graph.greedy_search_query(&distance, 0, 4.0, 0); - - assert_eq!(next, 2); - assert_eq!(dist, 0.0); - } - - #[test] - fn test_hnsw_cosine_distance_uses_vector_norms() { - let graph = HnswGraph::from_parts( - vec![2.0, 0.0, 4.0, 0.0, 0.0, 3.0], - 3, - 2, - MetricType::Cosine, - vec![0, 0, 0], - vec![vec![vec![]], vec![vec![]], vec![vec![]]], - 0, - 0, - HnswBuildParams::default(), - ) - .unwrap(); - - assert!((graph.distance_between(0, 1) - 0.0).abs() < 1e-6); - assert!((graph.distance_between(0, 2) - 1.0).abs() < 1e-6); - } - - fn exact_topk(data: &[f32], n: usize, d: usize, query: &[f32], k: usize) -> Vec { - let mut distances: Vec<(f32, usize)> = (0..n) - .map(|i| { - let vector = &data[i * d..(i + 1) * d]; - (fvec_distance(query, vector, MetricType::L2), i) - }) - .collect(); - distances.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); - distances[..k].iter().map(|&(_, id)| id).collect() - } - - fn generate_clustered_data(n: usize, d: usize, num_clusters: usize) -> Vec { - let mut data = vec![0.0f32; n * d]; - for i in 0..n { - let cluster = i % num_clusters; - for j in 0..d { - data[i * d + j] = cluster as f32 * 20.0 + j as f32 * 0.01 + i as f32 * 0.0001; - } - } - data - } -} diff --git a/core/src/hnsw_search.rs b/core/src/hnsw_search.rs deleted file mode 100644 index 0bff2ed..0000000 --- a/core/src/hnsw_search.rs +++ /dev/null @@ -1,82 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::hnsw::{HnswGraph, HnswSearchWorkspace}; -use crate::ivfpq::RowIdFilter; -use crate::topk::TopKHeap; - -pub(crate) struct HnswSearchList<'a, P> { - pub(crate) ids: &'a [i64], - pub(crate) graph: Option<&'a HnswGraph>, - pub(crate) payload: P, -} - -pub(crate) fn search_hnsw_lists<'a, P, F>( - query: &[f32], - lists: &[HnswSearchList<'a, P>], - k: usize, - ef_search: usize, - filter: Option<&dyn RowIdFilter>, - mut scan_list: F, -) -> Vec<(f32, i64)> -where - F: FnMut(&HnswSearchList<'a, P>, &mut TopKHeap), -{ - let mut heap = TopKHeap::new(k); - let mut workspace = HnswSearchWorkspace::new(ef_search.max(k)); - let force_scan = filter - .map(|f| count_filtered(lists, f) <= ef_search.max(k)) - .unwrap_or(false); - - for list in lists { - if force_scan { - scan_list(list, &mut heap); - continue; - } - if let Some(graph) = list.graph { - let local_results = graph.search_with_reusable_workspace( - query, - ef_search.max(k), - ef_search.max(k), - &mut workspace, - ); - for &(local_id, dist) in local_results { - let row_id = list.ids[local_id]; - if filter.map(|f| f.contains(row_id)).unwrap_or(true) { - heap.push(dist, row_id); - } - } - } else { - scan_list(list, &mut heap); - } - } - - if filter.is_some() && heap.len() < k && !force_scan { - for list in lists { - scan_list(list, &mut heap); - } - } - - heap.into_sorted() -} - -fn count_filtered

(lists: &[HnswSearchList<'_, P>], filter: &dyn RowIdFilter) -> usize { - lists - .iter() - .map(|list| list.ids.iter().filter(|&&id| filter.contains(id)).count()) - .sum() -} diff --git a/core/src/index.rs b/core/src/index.rs index cc66c1b..c2b4d10 100644 --- a/core/src/index.rs +++ b/core/src/index.rs @@ -15,45 +15,102 @@ // specific language governing permissions and limitations // under the License. +use crate::autotune::{ + default_training_vector_count, diskann_build_preset, infer_diskann_l_search, infer_ivf_nlist, + infer_ivf_nprobe, infer_rq_bits, DiskAnnBuildPreset, TuningObjective, +}; +use crate::diskann::{ + diskann_training_sample_limit, validate_diskann_format_configuration, + validate_diskann_training_budget, DiskAnnBuildDistance, DiskAnnBuildParams, DiskAnnIndex, + DiskAnnRawVectorEncoding, DiskAnnStorageLayout, +}; +use crate::diskann_io::{write_diskann_index, DiskAnnIndexReader, DISKANN_MAGIC}; +pub use crate::diskann_search::DiskAnnSearchStats; use crate::distance::MetricType; -use crate::hnsw::HnswBuildParams; use crate::io::{write_index, IVFPQIndexReader, ReadRequest, SeekRead, SeekWrite, MAGIC}; use crate::ivfflat::IVFFlatIndex; use crate::ivfflat_io::{ search_batch_ivfflat_reader, search_batch_ivfflat_reader_roaring_filter, write_ivfflat_index, IVFFlatIndexReader, IVFFLAT_MAGIC, }; -use crate::ivfhnswflat::IVFHNSWFlatIndex; -use crate::ivfhnswflat_io::{ - search_batch_ivfhnswflat_reader, search_batch_ivfhnswflat_reader_roaring_filter, - write_ivfhnswflat_index, IVFHNSWFlatIndexReader, IVF_HNSW_FLAT_MAGIC, -}; -use crate::ivfhnswsq::IVFHNSWSQIndex; -use crate::ivfhnswsq_io::{ - search_batch_ivfhnswsq_reader, search_batch_ivfhnswsq_reader_roaring_filter, - write_ivfhnswsq_index, IVFHNSWSQIndexReader, IVF_HNSW_SQ_MAGIC, -}; use crate::ivfpq::{ search_batch_reader, search_batch_reader_roaring_filter, search_with_reader, search_with_reader_roaring_filter, IVFPQIndex, }; use crate::ivfrq::IVFRQIndex; use crate::ivfrq_io::{ - search_batch_ivfrq_reader_roaring_filter_with_query_bits, - search_batch_ivfrq_reader_with_query_bits, write_ivfrq_index, IVFRQIndexReader, IVF_RQ_MAGIC, + search_batch_ivfrq_reader, search_batch_ivfrq_reader_roaring_filter, write_ivfrq_index, + IVFRQIndexReader, IVF_RQ_MAGIC, }; -use crate::rq::{is_supported_query_bits, DEFAULT_RQ_QUERY_BITS}; +use crate::ivfsq::IVFSQIndex; +use crate::ivfsq_io::{ + search_batch_ivfsq_reader, search_batch_ivfsq_reader_roaring_filter, write_ivfsq_index, + IVFSQIndexReader, IVF_SQ_MAGIC, +}; +pub use crate::read_options::{DeploymentProfile, VectorIndexReadPlan, VectorIndexReaderOptions}; +use crate::rq::{is_supported_rq_bits, padded_dimension, DEFAULT_RQ_BITS}; +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; +use roaring::RoaringTreemap; use std::collections::{HashMap, HashSet}; -use std::io; +use std::io::{self, Cursor}; + +/// Default ratio between the serialized PQ code and the raw `f32` vector. +/// +/// At 8 bits per subquantizer this resolves to one PQ subquantizer per four +/// dimensions, for example `m=32` at 128 dimensions and `m=240` at 960. +pub const DEFAULT_PQ_CODE_RATIO: f64 = 0.0625; +const PERSISTED_ROW_ID_ESTIMATE_BYTES: usize = 10; + +/// Resolve a concrete PQ subquantizer count from a target code/raw byte ratio. +/// +/// DiskANN uses balanced chunks, so the nearest value in `1..=dimension` is +/// valid. Ties prefer the larger `m` to avoid silently choosing the +/// lower-recall configuration. +pub fn infer_pq_m(dimension: usize, pq_bits: usize, code_ratio: f64) -> io::Result { + validate_positive(dimension, "dimension")?; + validate_pq_code_ratio(pq_bits, code_ratio)?; + + let target_m = dimension as f64 * 32.0 * code_ratio / pq_bits as f64; + Ok(target_m.round().clamp(1.0, dimension as f64) as usize) +} + +fn infer_uniform_pq_m(dimension: usize, pq_bits: usize, code_ratio: f64) -> io::Result { + validate_positive(dimension, "dimension")?; + validate_pq_code_ratio(pq_bits, code_ratio)?; + let target_m = dimension as f64 * 32.0 * code_ratio / pq_bits as f64; + let mut best_m = 1; + let mut best_distance = f64::INFINITY; + let mut consider = |candidate: usize| { + let distance = (candidate as f64 - target_m).abs(); + if distance < best_distance || (distance == best_distance && candidate > best_m) { + best_m = candidate; + best_distance = distance; + } + }; + + let mut divisor = 1; + while divisor <= dimension / divisor { + if dimension.is_multiple_of(divisor) { + consider(divisor); + let paired = dimension / divisor; + if paired != divisor { + consider(paired); + } + } + divisor += 1; + } + Ok(best_m) +} #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[repr(u32)] pub enum IndexType { IvfFlat = 0, IvfPq = 1, - IvfHnswFlat = 2, - IvfHnswSq = 3, IvfRq = 4, + DiskAnn = 5, + IvfSq = 6, } impl IndexType { @@ -61,9 +118,9 @@ impl IndexType { match code { 0 => Some(Self::IvfFlat), 1 => Some(Self::IvfPq), - 2 => Some(Self::IvfHnswFlat), - 3 => Some(Self::IvfHnswSq), 4 => Some(Self::IvfRq), + 5 => Some(Self::DiskAnn), + 6 => Some(Self::IvfSq), _ => None, } } @@ -72,9 +129,9 @@ impl IndexType { match self { Self::IvfFlat => "ivf_flat", Self::IvfPq => "ivf_pq", - Self::IvfHnswFlat => "ivf_hnsw_flat", - Self::IvfHnswSq => "ivf_hnsw_sq", Self::IvfRq => "ivf_rq", + Self::DiskAnn => "diskann", + Self::IvfSq => "ivf_sq", } } } @@ -96,69 +153,80 @@ pub enum VectorIndexConfig { IvfRq { dimension: usize, nlist: usize, + bits: usize, metric: MetricType, }, - IvfHnswFlat { + IvfSq { dimension: usize, nlist: usize, metric: MetricType, - hnsw: HnswBuildParams, }, - IvfHnswSq { + DiskAnn { dimension: usize, - nlist: usize, metric: MetricType, - hnsw: HnswBuildParams, + pq_m: usize, + pq_bits: usize, + build: DiskAnnBuildParams, }, } impl VectorIndexConfig { pub fn from_options(options: &HashMap) -> io::Result { - let mut options = ConfigOptions::new(options)?; - let index_type = parse_index_type_option(&options.required("index.type")?)?; - let dimension = parse_usize_option("dimension", &options.required("dimension")?); - let nlist = parse_usize_option("nlist", &options.required("nlist")?); - let metric = match options.optional("metric") { - Some(metric) => parse_metric_option(&metric)?, - None => MetricType::L2, + let plan = VectorIndexBuildPlan::from_options(options)?; + if plan.objective.max_build_seconds.is_some() { + return Err(invalid_input( + "max-build-seconds requires measured offline calibration through \ + VectorIndexBuildPlan and select_calibrated_candidate", + )); + } + Ok(plan.config) + } + + pub fn resolved(&self) -> ResolvedVectorIndexConfig { + ResolvedVectorIndexConfig::from(self) + } + + pub fn ivf_pq( + dimension: usize, + nlist: usize, + metric: MetricType, + use_opq: bool, + ) -> io::Result { + let config = Self::IvfPq { + dimension, + nlist, + m: infer_uniform_pq_m(dimension, 8, DEFAULT_PQ_CODE_RATIO)?, + metric, + use_opq, }; + validate_config(&config)?; + Ok(config) + } - let config = match index_type { - IndexType::IvfFlat => Self::IvfFlat { - dimension: dimension?, - nlist: nlist?, - metric, - }, - IndexType::IvfPq => Self::IvfPq { - dimension: dimension?, - nlist: nlist?, - m: parse_usize_option("pq.m", &options.required("pq.m")?)?, - metric, - use_opq: match options.optional("use-opq") { - Some(use_opq) => parse_bool_option("use-opq", &use_opq)?, - None => false, - }, - }, - IndexType::IvfRq => Self::IvfRq { - dimension: dimension?, - nlist: nlist?, - metric, - }, - IndexType::IvfHnswFlat => Self::IvfHnswFlat { - dimension: dimension?, - nlist: nlist?, - metric, - hnsw: parse_hnsw_options(&mut options)?, - }, - IndexType::IvfHnswSq => Self::IvfHnswSq { - dimension: dimension?, - nlist: nlist?, - metric, - hnsw: parse_hnsw_options(&mut options)?, - }, + pub fn disk_ann( + dimension: usize, + metric: MetricType, + pq_bits: usize, + build: DiskAnnBuildParams, + ) -> io::Result { + let config = Self::DiskAnn { + dimension, + metric, + pq_m: infer_pq_m(dimension, pq_bits, DEFAULT_PQ_CODE_RATIO)?, + pq_bits, + build, }; + validate_config(&config)?; + Ok(config) + } - options.reject_unknown()?; + pub fn ivf_rq(dimension: usize, nlist: usize, metric: MetricType) -> io::Result { + let config = Self::IvfRq { + dimension, + nlist, + bits: DEFAULT_RQ_BITS, + metric, + }; validate_config(&config)?; Ok(config) } @@ -168,8 +236,8 @@ impl VectorIndexConfig { Self::IvfFlat { .. } => IndexType::IvfFlat, Self::IvfPq { .. } => IndexType::IvfPq, Self::IvfRq { .. } => IndexType::IvfRq, - Self::IvfHnswFlat { .. } => IndexType::IvfHnswFlat, - Self::IvfHnswSq { .. } => IndexType::IvfHnswSq, + Self::IvfSq { .. } => IndexType::IvfSq, + Self::DiskAnn { .. } => IndexType::DiskAnn, } } @@ -178,8 +246,8 @@ impl VectorIndexConfig { Self::IvfFlat { dimension, .. } | Self::IvfPq { dimension, .. } | Self::IvfRq { dimension, .. } - | Self::IvfHnswFlat { dimension, .. } - | Self::IvfHnswSq { dimension, .. } => *dimension, + | Self::IvfSq { dimension, .. } + | Self::DiskAnn { dimension, .. } => *dimension, } } @@ -188,9 +256,269 @@ impl VectorIndexConfig { Self::IvfFlat { nlist, .. } | Self::IvfPq { nlist, .. } | Self::IvfRq { nlist, .. } - | Self::IvfHnswFlat { nlist, .. } - | Self::IvfHnswSq { nlist, .. } => *nlist, + | Self::IvfSq { nlist, .. } => *nlist, + Self::DiskAnn { .. } => 1, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ResolvedVectorIndexConfig { + pub index_type: IndexType, + pub dimension: usize, + pub nlist: usize, + pub metric: MetricType, + pub pq_m: Option, + pub pq_bits: Option, + pub rq_bits: Option, + pub use_opq: bool, + pub diskann_build: Option, +} + +impl From<&VectorIndexConfig> for ResolvedVectorIndexConfig { + fn from(config: &VectorIndexConfig) -> Self { + match config { + VectorIndexConfig::IvfFlat { + dimension, + nlist, + metric, + } + | VectorIndexConfig::IvfSq { + dimension, + nlist, + metric, + } => Self { + index_type: config.index_type(), + dimension: *dimension, + nlist: *nlist, + metric: *metric, + pq_m: None, + pq_bits: None, + rq_bits: None, + use_opq: false, + diskann_build: None, + }, + VectorIndexConfig::IvfPq { + dimension, + nlist, + m, + metric, + use_opq, + } => Self { + index_type: IndexType::IvfPq, + dimension: *dimension, + nlist: *nlist, + metric: *metric, + pq_m: Some(*m), + pq_bits: Some(8), + rq_bits: None, + use_opq: *use_opq, + diskann_build: None, + }, + VectorIndexConfig::IvfRq { + dimension, + nlist, + bits, + metric, + } => Self { + index_type: IndexType::IvfRq, + dimension: *dimension, + nlist: *nlist, + metric: *metric, + pq_m: None, + pq_bits: None, + rq_bits: Some(*bits), + use_opq: false, + diskann_build: None, + }, + VectorIndexConfig::DiskAnn { + dimension, + metric, + pq_m, + pq_bits, + build, + } => Self { + index_type: IndexType::DiskAnn, + dimension: *dimension, + nlist: 1, + metric: *metric, + pq_m: Some(*pq_m), + pq_bits: Some(*pq_bits), + rq_bits: None, + use_opq: false, + diskann_build: Some(*build), + }, + } + } +} + +#[derive(Debug, Clone)] +pub struct VectorIndexBuildPlan { + pub config: VectorIndexConfig, + pub expected_vector_count: Option, + pub objective: TuningObjective, +} + +impl VectorIndexBuildPlan { + pub fn from_options(options: &HashMap) -> io::Result { + let mut options = ConfigOptions::new(options)?; + let index_type = parse_index_type_option(&options.required("index.type")?)?; + let dimension = parse_usize_option("dimension", &options.required("dimension")?)?; + let expected_vector_count = options + .optional("expected-vector-count") + .map(|value| parse_usize_option("expected-vector-count", &value)) + .transpose()?; + if expected_vector_count == Some(0) { + return Err(invalid_input( + "expected-vector-count must be greater than 0", + )); + } + let metric = parse_metric_option(&options.required("metric")?)?; + let target_recall = options + .optional("target-recall") + .map(|value| parse_f32_option("target-recall", &value)) + .transpose()?; + if target_recall.is_some_and(|recall| !recall.is_finite() || !(0.0..=1.0).contains(&recall)) + { + return Err(invalid_input("target-recall must be finite and in [0, 1]")); + } + let max_bytes_per_vector = options + .optional("max-bytes-per-vector") + .map(|value| parse_usize_option("max-bytes-per-vector", &value)) + .transpose()?; + let max_build_seconds = options + .optional("max-build-seconds") + .map(|value| parse_f64_option("max-build-seconds", &value)) + .transpose()?; + if max_build_seconds.is_some_and(|seconds| !seconds.is_finite() || seconds <= 0.0) { + return Err(invalid_input( + "max-build-seconds must be finite and greater than 0", + )); + } + let deployment_profile = options + .optional("deployment-profile") + .map(|value| parse_deployment_profile_option("deployment-profile", &value)) + .transpose()? + .unwrap_or(DeploymentProfile::Auto); + let objective = TuningObjective { + target_recall, + max_bytes_per_vector, + max_build_seconds, + deployment_profile, + }; + + let config = match index_type { + IndexType::IvfFlat => VectorIndexConfig::IvfFlat { + dimension, + nlist: parse_nlist_options(&mut options, expected_vector_count)?, + metric, + }, + IndexType::IvfPq => VectorIndexConfig::IvfPq { + dimension, + nlist: parse_nlist_options(&mut options, expected_vector_count)?, + m: parse_pq_m_options( + &mut options, + dimension, + 8, + true, + max_bytes_per_vector + .map(|bytes| persisted_code_budget(bytes, "IVF-PQ")) + .transpose()?, + )?, + metric, + use_opq: match options.optional("use-opq") { + Some(use_opq) if use_opq.trim() == "auto" => { + target_recall.is_some_and(|recall| recall >= 0.9) + } + Some(use_opq) => parse_bool_option("use-opq", &use_opq)?, + None => target_recall.is_some_and(|recall| recall >= 0.9), + }, + }, + IndexType::IvfRq => { + let explicit_bits = options + .optional("rq.bits") + .map(|value| parse_usize_option("rq.bits", &value)) + .transpose()?; + let bits = match explicit_bits { + Some(bits) => bits, + None => max_bytes_per_vector + .map(|bytes| { + infer_rq_bits(dimension, persisted_code_budget(bytes, "IVF-RQ")?) + }) + .transpose()? + .unwrap_or(DEFAULT_RQ_BITS), + }; + VectorIndexConfig::IvfRq { + dimension, + nlist: parse_nlist_options(&mut options, expected_vector_count)?, + bits, + metric, + } + } + IndexType::IvfSq => VectorIndexConfig::IvfSq { + dimension, + nlist: parse_nlist_options(&mut options, expected_vector_count)?, + metric, + }, + IndexType::DiskAnn => { + let pq_bits = match options.optional("pq.bits") { + Some(value) => parse_usize_option("pq.bits", &value)?, + None if max_bytes_per_vector.is_some_and(|bytes| { + bytes + < dimension + .saturating_mul(4) + .saturating_add(64 * size_of::()) + }) => + { + 4 + } + None => 8, + }; + let build = parse_diskann_options( + &mut options, + dimension, + deployment_profile, + target_recall, + max_bytes_per_vector, + )?; + let max_pq_code_bytes = max_bytes_per_vector + .map(|bytes| { + let non_pq_bytes = + estimate_diskann_row_bytes(dimension, 0, pq_bits, build)?; + bytes.checked_sub(non_pq_bytes).ok_or_else(|| { + invalid_input(format!( + "max-bytes-per-vector {bytes} cannot fit DiskANN raw vectors, \ + graph edges, and row-ID metadata ({non_pq_bytes} bytes before PQ)" + )) + }) + }) + .transpose()?; + VectorIndexConfig::DiskAnn { + dimension, + metric, + pq_m: parse_pq_m_options( + &mut options, + dimension, + pq_bits, + false, + max_pq_code_bytes, + )?, + pq_bits, + build, + } + } + }; + + options.reject_unknown()?; + validate_config(&config)?; + if let Some(max_bytes) = max_bytes_per_vector { + validate_persisted_size_objective(&config, expected_vector_count, max_bytes)?; } + Ok(Self { + config, + expected_vector_count, + objective, + }) } } @@ -250,20 +578,334 @@ impl ConfigOptions { } } -fn parse_hnsw_options(options: &mut ConfigOptions) -> io::Result { - let defaults = HnswBuildParams::default(); - Ok(HnswBuildParams { - m: match options.optional("hnsw.m") { - Some(value) => parse_usize_option("hnsw.m", &value)?, - None => defaults.m, +fn parse_nlist_options( + options: &mut ConfigOptions, + expected_vector_count: Option, +) -> io::Result { + match options.optional("nlist").as_deref().map(str::trim) { + Some("auto") | None => { + let vector_count = expected_vector_count.ok_or_else(|| { + invalid_input( + "automatic nlist requires option 'expected-vector-count'; \ + set nlist explicitly when the final row count is unknown", + ) + })?; + infer_ivf_nlist(vector_count) + } + Some(value) => parse_usize_option("nlist", value), + } +} + +fn parse_deployment_profile_option(name: &str, value: &str) -> io::Result { + match value.trim() { + "auto" => Ok(DeploymentProfile::Auto), + "memory" => Ok(DeploymentProfile::Memory), + "local" | "local_storage" => Ok(DeploymentProfile::LocalStorage), + "remote" | "remote_storage" => Ok(DeploymentProfile::RemoteStorage), + "object_store" => Ok(DeploymentProfile::ObjectStore), + _ => Err(invalid_input(format!( + "option '{name}' must be auto, memory, local_storage, remote_storage, or object_store" + ))), + } +} + +fn parse_pq_m_options( + options: &mut ConfigOptions, + dimension: usize, + pq_bits: usize, + uniform_chunks: bool, + max_code_bytes: Option, +) -> io::Result { + let explicit_m = options + .optional("pq.m") + .map(|value| parse_usize_option("pq.m", &value)) + .transpose()?; + let explicit_code_ratio = options + .optional("pq.code-ratio") + .map(|value| parse_f64_option("pq.code-ratio", &value)) + .transpose()?; + let code_ratio = explicit_code_ratio + .or_else(|| { + max_code_bytes.map(|bytes| { + let raw_bytes = dimension.saturating_mul(size_of::()).max(1); + (bytes as f64 / raw_bytes as f64).min(pq_bits as f64 / 32.0) + }) + }) + .unwrap_or(DEFAULT_PQ_CODE_RATIO); + validate_pq_code_ratio(pq_bits, code_ratio)?; + + match explicit_m { + Some(m) => Ok(m), + None if uniform_chunks => infer_uniform_pq_m(dimension, pq_bits, code_ratio), + None => infer_pq_m(dimension, pq_bits, code_ratio), + } +} + +fn persisted_code_budget(max_bytes_per_vector: usize, index_name: &str) -> io::Result { + max_bytes_per_vector + .checked_sub(PERSISTED_ROW_ID_ESTIMATE_BYTES) + .filter(|&bytes| bytes > 0) + .ok_or_else(|| { + invalid_input(format!( + "max-bytes-per-vector {max_bytes_per_vector} cannot fit the persisted \ + row-ID encoding for {index_name}" + )) + }) +} + +fn estimate_diskann_row_bytes( + dimension: usize, + pq_m: usize, + pq_bits: usize, + build: DiskAnnBuildParams, +) -> io::Result { + let raw_bytes = dimension + .checked_mul(build.raw_vector_encoding.element_size()) + .ok_or_else(|| invalid_input("DiskANN raw row byte estimate overflows usize"))?; + let pq_bytes = pq_m + .checked_mul(pq_bits) + .and_then(|bits| bits.checked_add(7)) + .map(|bits| bits / 8) + .ok_or_else(|| invalid_input("DiskANN PQ row byte estimate overflows usize"))?; + let adjacency_bytes = build + .max_degree + .checked_mul(size_of::()) + .and_then(|bytes| bytes.checked_add(8)) + .ok_or_else(|| invalid_input("DiskANN adjacency row byte estimate overflows usize"))?; + raw_bytes + .checked_add(pq_bytes) + .and_then(|bytes| bytes.checked_add(adjacency_bytes)) + .and_then(|bytes| bytes.checked_add(PERSISTED_ROW_ID_ESTIMATE_BYTES)) + // Row-ID order and block locators are persisted for filtered lookup. + .and_then(|bytes| bytes.checked_add(12)) + .ok_or_else(|| invalid_input("DiskANN persisted row byte estimate overflows usize")) +} + +fn validate_persisted_size_objective( + config: &VectorIndexConfig, + expected_vector_count: Option, + max_bytes_per_vector: usize, +) -> io::Result<()> { + let dimension = config.dimension(); + let nlist = config.nlist(); + let centroid_bytes = nlist + .checked_mul(dimension) + .and_then(|values| values.checked_mul(size_of::())) + .ok_or_else(|| invalid_input("persisted centroid size estimate overflows usize"))?; + let list_metadata_bytes = nlist + .checked_mul(24) + .ok_or_else(|| invalid_input("persisted list metadata estimate overflows usize"))?; + + let (row_bytes, fixed_bytes) = match config { + VectorIndexConfig::IvfFlat { .. } => ( + dimension + .checked_mul(size_of::()) + .and_then(|bytes| bytes.checked_add(PERSISTED_ROW_ID_ESTIMATE_BYTES)) + .ok_or_else(|| invalid_input("IVF-FLAT row byte estimate overflows usize"))?, + centroid_bytes.checked_add(list_metadata_bytes), + ), + VectorIndexConfig::IvfSq { .. } => ( + dimension + .checked_add(PERSISTED_ROW_ID_ESTIMATE_BYTES) + .ok_or_else(|| invalid_input("IVF-SQ row byte estimate overflows usize"))?, + centroid_bytes + .checked_add(list_metadata_bytes) + .and_then(|bytes| { + nlist + .checked_mul(dimension) + .and_then(|values| values.checked_mul(2 * size_of::())) + .and_then(|quantizer_bytes| bytes.checked_add(quantizer_bytes)) + }), + ), + VectorIndexConfig::IvfPq { m, use_opq, .. } => { + let codebook_bytes = dimension + .checked_mul(256) + .and_then(|values| values.checked_mul(size_of::())) + .ok_or_else(|| invalid_input("IVF-PQ codebook estimate overflows usize"))?; + let opq_bytes = if *use_opq { + dimension + .checked_mul(dimension) + .and_then(|values| values.checked_mul(size_of::())) + .ok_or_else(|| invalid_input("OPQ matrix estimate overflows usize"))? + } else { + 0 + }; + ( + m.checked_add(PERSISTED_ROW_ID_ESTIMATE_BYTES) + .ok_or_else(|| invalid_input("IVF-PQ row byte estimate overflows usize"))?, + centroid_bytes + .checked_add(list_metadata_bytes) + .and_then(|bytes| bytes.checked_add(codebook_bytes)) + .and_then(|bytes| bytes.checked_add(opq_bytes)), + ) + } + VectorIndexConfig::IvfRq { bits, .. } => { + let code_bytes = padded_dimension(dimension) + .checked_mul(*bits) + .and_then(|bits| bits.checked_add(7)) + .map(|bits| bits / 8) + .ok_or_else(|| invalid_input("IVF-RQ code estimate overflows usize"))?; + let factor_bytes = if *bits == 1 { 8 } else { 20 }; + ( + code_bytes + .checked_add(factor_bytes) + .and_then(|bytes| bytes.checked_add(PERSISTED_ROW_ID_ESTIMATE_BYTES)) + .ok_or_else(|| invalid_input("IVF-RQ row byte estimate overflows usize"))?, + centroid_bytes.checked_add(list_metadata_bytes), + ) + } + VectorIndexConfig::DiskAnn { + pq_m, + pq_bits, + build, + .. + } => { + let ksub = 1usize << pq_bits; + let codebook_bytes = dimension + .checked_mul(ksub) + .and_then(|values| values.checked_mul(size_of::())) + .ok_or_else(|| invalid_input("DiskANN codebook estimate overflows usize"))?; + ( + estimate_diskann_row_bytes(dimension, *pq_m, *pq_bits, *build)?, + Some(codebook_bytes), + ) + } + }; + let fixed_bytes = fixed_bytes + .ok_or_else(|| invalid_input("persisted fixed-size estimate overflows usize"))?; + let amortized_fixed_bytes = expected_vector_count + .map(|count| fixed_bytes.saturating_add(count - 1) / count) + .unwrap_or(0); + let estimated_bytes = row_bytes + .checked_add(amortized_fixed_bytes) + .ok_or_else(|| invalid_input("persisted per-vector estimate overflows usize"))?; + if estimated_bytes > max_bytes_per_vector { + return Err(invalid_input(format!( + "max-bytes-per-vector {max_bytes_per_vector} cannot be satisfied by {}: \ + estimated persisted size is {estimated_bytes} bytes per vector \ + ({row_bytes} row bytes + {amortized_fixed_bytes} amortized fixed bytes)", + config.index_type().as_str() + ))); + } + Ok(()) +} + +fn validate_pq_code_ratio(pq_bits: usize, code_ratio: f64) -> io::Result<()> { + if !matches!(pq_bits, 4 | 8) { + return Err(invalid_input(format!( + "pq.bits must be 4 or 8, got {pq_bits}" + ))); + } + let max_ratio = pq_bits as f64 / 32.0; + if !code_ratio.is_finite() || code_ratio <= 0.0 || code_ratio > max_ratio { + return Err(invalid_input(format!( + "pq.code-ratio must be finite and in (0, {max_ratio}] for {pq_bits}-bit PQ" + ))); + } + Ok(()) +} + +fn parse_diskann_options( + options: &mut ConfigOptions, + dimension: usize, + deployment_profile: DeploymentProfile, + target_recall: Option, + max_bytes_per_vector: Option, +) -> io::Result { + let defaults = DiskAnnBuildParams::default(); + let preset = match options.optional("diskann.build-preset").as_deref() { + Some("fast_build") => DiskAnnBuildPreset::FastBuild, + Some("balanced") => DiskAnnBuildPreset::Balanced, + Some("high_recall") => DiskAnnBuildPreset::HighRecall, + Some(value) => { + return Err(invalid_input(format!( + "diskann.build-preset must be fast_build, balanced, or high_recall, got '{value}'" + ))) + } + None if target_recall.is_some_and(|recall| recall >= 0.97) => { + DiskAnnBuildPreset::HighRecall + } + None if target_recall.is_some_and(|recall| recall <= 0.85) => DiskAnnBuildPreset::FastBuild, + None => DiskAnnBuildPreset::Balanced, + }; + let seed = match options.optional("diskann.seed") { + Some(value) => parse_u64_option("diskann.seed", &value)?, + None => defaults.seed, + }; + let memory_budget_bytes = match options.optional("diskann.memory-budget-bytes") { + Some(value) => parse_usize_option("diskann.memory-budget-bytes", &value)?, + None => defaults.memory_budget_bytes, + }; + let preset_values = diskann_build_preset( + preset, + dimension, + deployment_profile, + memory_budget_bytes, + seed, + )?; + let max_degree = match options.optional("diskann.max-degree") { + Some(value) => parse_usize_option("diskann.max-degree", &value)?, + None => preset_values.max_degree, + }; + Ok(DiskAnnBuildParams { + max_degree, + build_search_list_size: match options.optional("diskann.build-search-list-size") { + Some(value) => parse_usize_option("diskann.build-search-list-size", &value)?, + None => preset_values.build_search_list_size.max(max_degree), }, - ef_construction: match options.optional("hnsw.ef-construction") { - Some(value) => parse_usize_option("hnsw.ef-construction", &value)?, - None => defaults.ef_construction, + alpha: match options.optional("diskann.alpha") { + Some(value) => parse_f32_option("diskann.alpha", &value)?, + None => preset_values.alpha, + }, + seed, + memory_budget_bytes, + storage_layout: match options.optional("diskann.storage-layout") { + Some(value) => match value.trim() { + "compact" => DiskAnnStorageLayout::Compact, + "interleaved" => DiskAnnStorageLayout::Interleaved, + "auto" => preset_values.storage_layout, + _ => { + return Err(invalid_input( + "diskann.storage-layout must be auto, compact, or interleaved", + )) + } + }, + None => preset_values.storage_layout, + }, + raw_vector_encoding: match options.optional("diskann.raw-vector-encoding") { + Some(value) => match value.trim() { + "f32" => DiskAnnRawVectorEncoding::F32, + "f16" => DiskAnnRawVectorEncoding::F16, + "auto" => preset_values.raw_vector_encoding, + _ => { + return Err(invalid_input( + "diskann.raw-vector-encoding must be auto, f32, or f16", + )) + } + }, + None if max_bytes_per_vector.is_some_and(|bytes| { + bytes + < dimension + .saturating_mul(size_of::()) + .saturating_add(max_degree * size_of::()) + }) => + { + DiskAnnRawVectorEncoding::F16 + } + None => preset_values.raw_vector_encoding, }, - max_level: match options.optional("hnsw.max-level") { - Some(value) => parse_usize_option("hnsw.max-level", &value)?, - None => defaults.max_level, + build_distance: match options.optional("diskann.build-distance") { + Some(value) => { + match value.trim() { + "full_precision" => DiskAnnBuildDistance::FullPrecision, + "product_quantized" => DiskAnnBuildDistance::ProductQuantized, + "auto" => preset_values.build_distance, + _ => return Err(invalid_input( + "diskann.build-distance must be auto, full_precision, or product_quantized", + )), + } + } + None => preset_values.build_distance, }, }) } @@ -273,10 +915,10 @@ fn parse_index_type_option(value: &str) -> io::Result { "ivf_flat" => Ok(IndexType::IvfFlat), "ivf_pq" => Ok(IndexType::IvfPq), "ivf_rq" => Ok(IndexType::IvfRq), - "ivf_hnsw_flat" => Ok(IndexType::IvfHnswFlat), - "ivf_hnsw_sq" => Ok(IndexType::IvfHnswSq), + "ivf_sq" => Ok(IndexType::IvfSq), + "diskann" => Ok(IndexType::DiskAnn), _ => Err(invalid_input(format!( - "unknown index.type '{}'; expected ivf_flat, ivf_pq, ivf_rq, ivf_hnsw_flat, or ivf_hnsw_sq", + "unknown index.type '{}'; expected ivf_flat, ivf_sq, ivf_pq, ivf_rq, or diskann", value ))), } @@ -301,6 +943,27 @@ fn parse_usize_option(name: &str, value: &str) -> io::Result { .map_err(|_| invalid_input(format!("option '{}' must be a positive integer", name))) } +fn parse_u64_option(name: &str, value: &str) -> io::Result { + value + .trim() + .parse::() + .map_err(|_| invalid_input(format!("option '{}' must be a non-negative integer", name))) +} + +fn parse_f32_option(name: &str, value: &str) -> io::Result { + value + .trim() + .parse::() + .map_err(|_| invalid_input(format!("option '{}' must be a number", name))) +} + +fn parse_f64_option(name: &str, value: &str) -> io::Result { + value + .trim() + .parse::() + .map_err(|_| invalid_input(format!("option '{}' must be a number", name))) +} + fn parse_bool_option(name: &str, value: &str) -> io::Result { match value.trim() { "true" => Ok(true), @@ -312,61 +975,89 @@ fn parse_bool_option(name: &str, value: &str) -> io::Result { } } -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u32)] +pub enum SearchWidth { + Auto = 0, + IvfNProbe = 1, + DiskAnnLSearch = 2, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct VectorSearchParams { pub top_k: usize, - pub nprobe: usize, - pub ef_search: usize, - pub query_bits: usize, + pub search_width: SearchWidth, + pub width: usize, } impl VectorSearchParams { pub fn new(top_k: usize, nprobe: usize) -> Self { Self { top_k, - nprobe, - ef_search: 0, - query_bits: DEFAULT_RQ_QUERY_BITS, + search_width: SearchWidth::IvfNProbe, + width: nprobe, } } - pub fn with_ef_search(top_k: usize, nprobe: usize, ef_search: usize) -> Self { + pub fn with_l_search(top_k: usize, l_search: usize) -> Self { Self { top_k, - nprobe, - ef_search, - query_bits: DEFAULT_RQ_QUERY_BITS, + search_width: SearchWidth::DiskAnnLSearch, + width: l_search, } } - pub fn with_query_bits(top_k: usize, nprobe: usize, query_bits: usize) -> Self { + pub fn automatic(top_k: usize) -> Self { Self { top_k, - nprobe, - ef_search: 0, - query_bits, + search_width: SearchWidth::Auto, + width: 0, } } - pub fn with_ef_search_and_query_bits( - top_k: usize, - nprobe: usize, - ef_search: usize, - query_bits: usize, - ) -> Self { - Self { - top_k, - nprobe, - ef_search, - query_bits, + pub fn configured_ivf_nprobe(self) -> Option { + (self.search_width == SearchWidth::IvfNProbe).then_some(self.width) + } + + pub fn configured_diskann_l_search(self) -> Option { + (self.search_width == SearchWidth::DiskAnnLSearch).then_some(self.width) + } + + fn validate(self) -> io::Result<()> { + validate_positive(self.top_k, "top_k") + } + + fn resolve_ivf_nprobe( + self, + nlist: usize, + vector_count: usize, + matching_count: Option, + ) -> io::Result { + match self.search_width { + SearchWidth::Auto => infer_ivf_nprobe(nlist, vector_count, self.top_k, matching_count), + SearchWidth::IvfNProbe if self.width > 0 => Ok(self.width.min(nlist)), + SearchWidth::IvfNProbe => Err(invalid_input("nprobe must be greater than 0")), + SearchWidth::DiskAnnLSearch => Err(invalid_input( + "DiskANN l_search cannot be used with an IVF index", + )), } } - fn hnsw_ef_search(self) -> usize { - if self.ef_search == 0 { - self.top_k.max(32) - } else { - self.ef_search + #[cfg(test)] + fn resolve_diskann_l_search(self) -> io::Result { + self.resolve_diskann_l_search_with(None) + } + + fn resolve_diskann_l_search_with(self, calibrated: Option) -> io::Result { + match self.search_width { + SearchWidth::Auto => Ok(calibrated + .unwrap_or(infer_diskann_l_search(self.top_k)?) + .max(self.top_k)), + SearchWidth::DiskAnnLSearch if self.width > 0 => Ok(self.width.max(self.top_k)), + SearchWidth::DiskAnnLSearch => Err(invalid_input("l_search must be greater than 0")), + SearchWidth::IvfNProbe => Err(invalid_input( + "IVF nprobe cannot be used with a DiskANN index", + )), } } } @@ -379,21 +1070,57 @@ pub struct VectorIndexMetadata { pub metric: MetricType, pub total_vectors: i64, pub pq_m: Option, - pub hnsw: Option, + pub pq_bits: Option, + pub rq_bits: Option, + pub diskann: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct DiskAnnMetadata { + pub max_degree: usize, + pub build_search_list_size: usize, + pub alpha: f32, } pub struct VectorIndexTrainer { writer: VectorIndexWriter, training_data: Vec, training_vector_count: usize, + training_vectors_seen: usize, + training_sample_limit: usize, + training_rng: StdRng, } impl VectorIndexTrainer { pub fn new(config: VectorIndexConfig) -> io::Result { + let training_sample_limit = match &config { + VectorIndexConfig::DiskAnn { + dimension, + metric, + pq_m, + pq_bits, + build, + } => diskann_training_sample_limit( + *dimension, + *metric, + *pq_m, + *pq_bits, + build.memory_budget_bytes, + )?, + _ => default_training_vector_count(usize::MAX, config.nlist())?, + }; + let training_seed = match &config { + VectorIndexConfig::DiskAnn { build, .. } => build.seed, + _ => 1234, + }; + let writer = VectorIndexWriter::from_config(config)?; Ok(Self { - writer: VectorIndexWriter::from_config(config)?, + writer, training_data: Vec::new(), training_vector_count: 0, + training_vectors_seen: 0, + training_sample_limit, + training_rng: StdRng::seed_from_u64(training_seed), }) } @@ -416,14 +1143,27 @@ impl VectorIndexTrainer { pub fn add_training_vectors_mut(&mut self, data: &[f32], n: usize) -> io::Result<&mut Self> { validate_vectors(data, n, self.dimension(), "training data")?; - let training_vector_count = self.training_vector_count.checked_add(n).ok_or_else(|| { + let training_vectors_seen = self.training_vectors_seen.checked_add(n).ok_or_else(|| { io::Error::new( io::ErrorKind::InvalidInput, "training vector count overflows usize", ) })?; - self.training_data.extend_from_slice(data); - self.training_vector_count = training_vector_count; + let dimension = self.writer.dimension(); + for (batch_index, vector) in data.chunks_exact(dimension).enumerate() { + let stream_index = self.training_vectors_seen + batch_index; + if stream_index < self.training_sample_limit { + self.training_data.extend_from_slice(vector); + } else { + let replacement = self.training_rng.gen_range(0..=stream_index); + if replacement < self.training_sample_limit { + let start = replacement * dimension; + self.training_data[start..start + dimension].copy_from_slice(vector); + } + } + } + self.training_vector_count = training_vectors_seen.min(self.training_sample_limit); + self.training_vectors_seen = training_vectors_seen; Ok(self) } @@ -453,10 +1193,10 @@ impl VectorIndexTraining { pub enum VectorIndexWriter { IvfFlat(IVFFlatIndex), + IvfSq(IVFSQIndex), IvfPq(IVFPQIndex), IvfRq(IVFRQIndex), - IvfHnswFlat(IVFHNSWFlatIndex), - IvfHnswSq(IVFHNSWSQIndex), + DiskAnn(DiskAnnIndex), } impl VectorIndexWriter { @@ -472,6 +1212,11 @@ impl VectorIndexWriter { nlist, metric, } => Self::IvfFlat(IVFFlatIndex::new(dimension, nlist, metric)), + VectorIndexConfig::IvfSq { + dimension, + nlist, + metric, + } => Self::IvfSq(IVFSQIndex::new(dimension, nlist, metric)), VectorIndexConfig::IvfPq { dimension, nlist, @@ -482,29 +1227,17 @@ impl VectorIndexWriter { VectorIndexConfig::IvfRq { dimension, nlist, + bits, metric, - } => Self::IvfRq(IVFRQIndex::new(dimension, nlist, metric)), - VectorIndexConfig::IvfHnswFlat { - dimension, - nlist, - metric, - hnsw, - } => Self::IvfHnswFlat(IVFHNSWFlatIndex::new( + } => Self::IvfRq(IVFRQIndex::with_bits(dimension, nlist, bits, metric)), + VectorIndexConfig::DiskAnn { dimension, - nlist, - metric, - hnsw.sanitized(), - )), - VectorIndexConfig::IvfHnswSq { - dimension, - nlist, - metric, - hnsw, - } => Self::IvfHnswSq(IVFHNSWSQIndex::new( - dimension, - nlist, metric, - hnsw.sanitized(), + pq_m, + pq_bits, + build, + } => Self::DiskAnn(DiskAnnIndex::with_pq_bits( + dimension, metric, pq_m, pq_bits, build, )), }) } @@ -512,20 +1245,20 @@ impl VectorIndexWriter { pub fn index_type(&self) -> IndexType { match self { Self::IvfFlat(_) => IndexType::IvfFlat, + Self::IvfSq(_) => IndexType::IvfSq, Self::IvfPq(_) => IndexType::IvfPq, Self::IvfRq(_) => IndexType::IvfRq, - Self::IvfHnswFlat(_) => IndexType::IvfHnswFlat, - Self::IvfHnswSq(_) => IndexType::IvfHnswSq, + Self::DiskAnn(_) => IndexType::DiskAnn, } } pub fn dimension(&self) -> usize { match self { Self::IvfFlat(index) => index.d, + Self::IvfSq(index) => index.d, Self::IvfPq(index) => index.d, Self::IvfRq(index) => index.d, - Self::IvfHnswFlat(index) => index.flat.d, - Self::IvfHnswSq(index) => index.d, + Self::DiskAnn(index) => index.d, } } @@ -533,10 +1266,10 @@ impl VectorIndexWriter { debug_assert_eq!(Some(data.len()), n.checked_mul(self.dimension())); match self { Self::IvfFlat(index) => index.train(data, n), + Self::IvfSq(index) => index.train(data, n), Self::IvfPq(index) => index.train(data, n), Self::IvfRq(index) => index.train(data, n), - Self::IvfHnswFlat(index) => index.train(data, n), - Self::IvfHnswSq(index) => index.train(data, n), + Self::DiskAnn(index) => return index.train(data, n), } Ok(()) } @@ -551,10 +1284,10 @@ impl VectorIndexWriter { } match self { Self::IvfFlat(index) => index.add(data, ids, n), + Self::IvfSq(index) => index.add(data, ids, n), Self::IvfPq(index) => index.add(data, ids, n), Self::IvfRq(index) => index.add(data, ids, n), - Self::IvfHnswFlat(index) => index.add(data, ids, n), - Self::IvfHnswSq(index) => index.add(data, ids, n), + Self::DiskAnn(index) => index.add(data, ids), } Ok(()) } @@ -562,40 +1295,62 @@ impl VectorIndexWriter { pub fn write(&mut self, out: &mut dyn SeekWrite) -> io::Result<()> { match self { Self::IvfFlat(index) => write_ivfflat_index(index, out), + Self::IvfSq(index) => write_ivfsq_index(index, out), Self::IvfPq(index) => write_index(index, out), Self::IvfRq(index) => write_ivfrq_index(index, out), - Self::IvfHnswFlat(index) => { - index.build_graphs()?; - write_ivfhnswflat_index(index, out) - } - Self::IvfHnswSq(index) => { - index.build_graphs()?; - write_ivfhnswsq_index(index, out) - } + Self::DiskAnn(index) => write_diskann_index(index, out), } } } pub enum VectorIndexReader { IvfFlat(IVFFlatIndexReader), + IvfSq(IVFSQIndexReader), IvfPq(IVFPQIndexReader), IvfRq(IVFRQIndexReader), - IvfHnswFlat(IVFHNSWFlatIndexReader), - IvfHnswSq(IVFHNSWSQIndexReader), + DiskAnn(DiskAnnIndexReader), } impl VectorIndexReader { - pub fn open(mut reader: R) -> io::Result { - let mut magic_buf = [0u8; 4]; - reader.pread(&mut [ReadRequest::new(0, &mut magic_buf)])?; - let magic = u32::from_le_bytes(magic_buf); + pub fn open(reader: R) -> io::Result { + Self::open_with_options(reader, VectorIndexReaderOptions::default()) + } + + pub fn open_with_options(mut reader: R, options: VectorIndexReaderOptions) -> io::Result { + let mut header = [0u8; 64]; + if let Err(header_error) = reader.pread(&mut [ReadRequest::new(0, &mut header)]) { + let mut magic_buf = [0u8; 4]; + reader.pread(&mut [ReadRequest::new(0, &mut magic_buf)])?; + let magic = u32::from_le_bytes(magic_buf); + if !matches!( + magic, + IVFFLAT_MAGIC | IVF_SQ_MAGIC | MAGIC | IVF_RQ_MAGIC | DISKANN_MAGIC + ) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("unknown vector index magic: 0x{:08X}", magic), + )); + } + return Err(header_error); + } + let magic = u32::from_le_bytes(header[0..4].try_into().unwrap()); match magic { - IVFFLAT_MAGIC => Ok(Self::IvfFlat(IVFFlatIndexReader::open(reader)?)), - MAGIC => Ok(Self::IvfPq(IVFPQIndexReader::open(reader)?)), - IVF_RQ_MAGIC => Ok(Self::IvfRq(IVFRQIndexReader::open(reader)?)), - IVF_HNSW_FLAT_MAGIC => Ok(Self::IvfHnswFlat(IVFHNSWFlatIndexReader::open(reader)?)), - IVF_HNSW_SQ_MAGIC => Ok(Self::IvfHnswSq(IVFHNSWSQIndexReader::open(reader)?)), + IVFFLAT_MAGIC => Ok(Self::IvfFlat(IVFFlatIndexReader::open_with_header( + reader, header, + )?)), + IVF_SQ_MAGIC => Ok(Self::IvfSq(IVFSQIndexReader::open_with_header( + reader, header, + )?)), + MAGIC => Ok(Self::IvfPq(IVFPQIndexReader::open_with_header( + reader, header, + )?)), + IVF_RQ_MAGIC => Ok(Self::IvfRq(IVFRQIndexReader::open_with_header( + reader, header, + )?)), + DISKANN_MAGIC => Ok(Self::DiskAnn(DiskAnnIndexReader::open_with_options( + reader, options, + )?)), _ => Err(io::Error::new( io::ErrorKind::InvalidData, format!("unknown vector index magic: 0x{:08X}", magic), @@ -606,10 +1361,10 @@ impl VectorIndexReader { pub fn index_type(&self) -> IndexType { match self { Self::IvfFlat(_) => IndexType::IvfFlat, + Self::IvfSq(_) => IndexType::IvfSq, Self::IvfPq(_) => IndexType::IvfPq, Self::IvfRq(_) => IndexType::IvfRq, - Self::IvfHnswFlat(_) => IndexType::IvfHnswFlat, - Self::IvfHnswSq(_) => IndexType::IvfHnswSq, + Self::DiskAnn(_) => IndexType::DiskAnn, } } @@ -622,7 +1377,20 @@ impl VectorIndexReader { metric: reader.metric, total_vectors: reader.total_vectors, pq_m: None, - hnsw: None, + pq_bits: None, + rq_bits: None, + diskann: None, + }, + Self::IvfSq(reader) => VectorIndexMetadata { + index_type: IndexType::IvfSq, + dimension: reader.d, + nlist: reader.nlist, + metric: reader.metric, + total_vectors: reader.total_vectors, + pq_m: None, + pq_bits: Some(8), + rq_bits: None, + diskann: None, }, Self::IvfPq(reader) => VectorIndexMetadata { index_type: IndexType::IvfPq, @@ -631,7 +1399,9 @@ impl VectorIndexReader { metric: reader.metric, total_vectors: reader.total_vectors, pq_m: Some(reader.m), - hnsw: None, + pq_bits: Some(reader.pq.nbits), + rq_bits: None, + diskann: None, }, Self::IvfRq(reader) => VectorIndexMetadata { index_type: IndexType::IvfRq, @@ -640,25 +1410,25 @@ impl VectorIndexReader { metric: reader.metric, total_vectors: reader.total_vectors, pq_m: None, - hnsw: None, + pq_bits: None, + rq_bits: Some(reader.num_bits), + diskann: None, }, - Self::IvfHnswFlat(reader) => VectorIndexMetadata { - index_type: IndexType::IvfHnswFlat, - dimension: reader.d, - nlist: reader.nlist, - metric: reader.metric, - total_vectors: reader.total_vectors, - pq_m: None, - hnsw: Some(reader.hnsw_params), - }, - Self::IvfHnswSq(reader) => VectorIndexMetadata { - index_type: IndexType::IvfHnswSq, - dimension: reader.d, - nlist: reader.nlist, - metric: reader.metric, - total_vectors: reader.total_vectors, - pq_m: None, - hnsw: Some(reader.hnsw_params), + Self::DiskAnn(reader) => VectorIndexMetadata { + index_type: IndexType::DiskAnn, + dimension: reader.header.dimension as usize, + nlist: 1, + metric: MetricType::from_code(reader.header.metric) + .expect("validated DiskANN metric code"), + total_vectors: reader.header.vector_count as i64, + pq_m: Some(reader.header.pq_m as usize), + pq_bits: Some(reader.header.pq_bits as usize), + rq_bits: None, + diskann: Some(DiskAnnMetadata { + max_degree: reader.header.max_degree as usize, + build_search_list_size: reader.header.build_search_list_size as usize, + alpha: reader.header.alpha, + }), }, } } @@ -671,15 +1441,69 @@ impl VectorIndexReader { self.metadata().total_vectors } + pub fn diskann_search_stats(&self) -> Option { + match self { + Self::DiskAnn(reader) => Some(reader.last_search_stats()), + _ => None, + } + } + + pub fn read_plan(&self) -> Option { + match self { + Self::DiskAnn(reader) => Some(reader.vector_read_plan()), + _ => None, + } + } + pub fn optimize_for_search(&mut self) -> io::Result<()> { match self { Self::IvfFlat(reader) => reader.ensure_loaded(), + Self::IvfSq(reader) => reader.optimize_for_search(), Self::IvfPq(reader) => reader.optimize_for_search(), Self::IvfRq(reader) => reader.ensure_loaded(), - Self::IvfHnswFlat(reader) => reader.ensure_loaded(), - // IVF_HNSW_SQ warms SQ scan/fallback structures used by filtered - // searches; normal unfiltered search primarily uses the HNSW graph. - Self::IvfHnswSq(reader) => reader.optimize_for_search(), + Self::DiskAnn(reader) => reader.optimize_for_search(), + } + } + + /// Warm query-dependent caches with representative queries. DiskANN runs + /// the graph and rerank path; other index types perform their normal + /// resident optimization because they do not expose a paged query cache. + pub fn warmup_queries( + &mut self, + queries: &[f32], + query_count: usize, + l_search: usize, + ) -> io::Result<()> { + let expected_len = query_count + .checked_mul(self.dimension()) + .ok_or_else(|| invalid_input("warmup query count * dimension overflows usize"))?; + if queries.len() != expected_len { + return Err(invalid_input(format!( + "warmup queries length {} does not match query count * dimension {}", + queries.len(), + expected_len + ))); + } + validate_finite_values(queries, expected_len, "warmup queries")?; + match self { + Self::DiskAnn(reader) => reader.warmup_queries(queries, l_search), + _ => self.optimize_for_search(), + } + } + + pub fn calibrate_search_width( + &mut self, + queries: &[f32], + query_count: usize, + top_k: usize, + ) -> io::Result { + validate_queries(queries, query_count, self.dimension())?; + validate_positive(top_k, "top_k")?; + match self { + Self::DiskAnn(reader) => reader.calibrate_l_search(queries, top_k), + _ => Err(invalid_input( + "search-width calibration is currently only available for DiskANN", + )), } } @@ -689,18 +1513,67 @@ impl VectorIndexReader { params: VectorSearchParams, ) -> io::Result<(Vec, Vec)> { validate_query(query, self.dimension())?; - validate_query_bits_for_index(self.index_type(), params.query_bits)?; + params.validate()?; match self { - Self::IvfFlat(reader) => reader.search(query, params.top_k, params.nprobe), - Self::IvfPq(reader) => search_with_reader(reader, query, params.top_k, params.nprobe), - Self::IvfRq(reader) => { - reader.search_with_query_bits(query, params.top_k, params.nprobe, params.query_bits) + Self::IvfFlat(reader) => { + let total_vectors = usize::try_from(reader.total_vectors) + .map_err(|_| invalid_input("negative IVF vector count"))?; + let nprobe = params.resolve_ivf_nprobe(reader.nlist, total_vectors, None)?; + progressive_ivf_search( + params, + reader.nlist, + nprobe, + 1, + params.top_k, + total_vectors, + |nprobe| reader.search(query, params.top_k, nprobe), + ) + } + Self::IvfSq(reader) => { + let total_vectors = usize::try_from(reader.total_vectors) + .map_err(|_| invalid_input("negative IVF vector count"))?; + let nprobe = params.resolve_ivf_nprobe(reader.nlist, total_vectors, None)?; + progressive_ivf_search( + params, + reader.nlist, + nprobe, + 1, + params.top_k, + total_vectors, + |nprobe| reader.search(query, params.top_k, nprobe), + ) + } + Self::IvfPq(reader) => { + let total_vectors = usize::try_from(reader.total_vectors) + .map_err(|_| invalid_input("negative IVF vector count"))?; + let nprobe = params.resolve_ivf_nprobe(reader.nlist, total_vectors, None)?; + progressive_ivf_search( + params, + reader.nlist, + nprobe, + 1, + params.top_k, + total_vectors, + |nprobe| search_with_reader(reader, query, params.top_k, nprobe), + ) } - Self::IvfHnswFlat(reader) => { - reader.search(query, params.top_k, params.nprobe, params.hnsw_ef_search()) + Self::IvfRq(reader) => { + let total_vectors = usize::try_from(reader.total_vectors) + .map_err(|_| invalid_input("negative IVF vector count"))?; + let nprobe = params.resolve_ivf_nprobe(reader.nlist, total_vectors, None)?; + progressive_ivf_search( + params, + reader.nlist, + nprobe, + 1, + params.top_k, + total_vectors, + |nprobe| reader.search(query, params.top_k, nprobe), + ) } - Self::IvfHnswSq(reader) => { - reader.search(query, params.top_k, params.nprobe, params.hnsw_ef_search()) + Self::DiskAnn(reader) => { + let l_search = params.resolve_diskann_l_search_with(reader.calibrated_l_search)?; + reader.search(query, params.top_k, l_search) } } } @@ -712,40 +1585,106 @@ impl VectorIndexReader { roaring_filter_bytes: &[u8], ) -> io::Result<(Vec, Vec)> { validate_query(query, self.dimension())?; - validate_query_bits_for_index(self.index_type(), params.query_bits)?; + params.validate()?; + let matching_count = if params.search_width == SearchWidth::Auto { + Some(decode_roaring_filter_cardinality(roaring_filter_bytes)?) + } else { + None + }; match self { - Self::IvfFlat(reader) => reader.search_with_roaring_filter( - query, - params.top_k, - params.nprobe, - roaring_filter_bytes, - ), - Self::IvfPq(reader) => search_with_reader_roaring_filter( - reader, - query, - params.top_k, - params.nprobe, - roaring_filter_bytes, - ), - Self::IvfRq(reader) => reader.search_with_roaring_filter_and_query_bits( - query, - params.top_k, - params.nprobe, - roaring_filter_bytes, - params.query_bits, - ), - Self::IvfHnswFlat(reader) => reader.search_with_roaring_filter( - query, - params.top_k, - params.nprobe, - params.hnsw_ef_search(), - roaring_filter_bytes, - ), - Self::IvfHnswSq(reader) => reader.search_with_roaring_filter( + Self::IvfFlat(reader) => { + let total_vectors = usize::try_from(reader.total_vectors) + .map_err(|_| invalid_input("negative IVF vector count"))?; + let nprobe = + params.resolve_ivf_nprobe(reader.nlist, total_vectors, matching_count)?; + progressive_ivf_search( + params, + reader.nlist, + nprobe, + 1, + params.top_k, + matching_count.unwrap_or(total_vectors), + |nprobe| { + reader.search_with_roaring_filter( + query, + params.top_k, + nprobe, + roaring_filter_bytes, + ) + }, + ) + } + Self::IvfSq(reader) => { + let total_vectors = usize::try_from(reader.total_vectors) + .map_err(|_| invalid_input("negative IVF vector count"))?; + let nprobe = + params.resolve_ivf_nprobe(reader.nlist, total_vectors, matching_count)?; + progressive_ivf_search( + params, + reader.nlist, + nprobe, + 1, + params.top_k, + matching_count.unwrap_or(total_vectors), + |nprobe| { + reader.search_with_roaring_filter( + query, + params.top_k, + nprobe, + roaring_filter_bytes, + ) + }, + ) + } + Self::IvfPq(reader) => { + let total_vectors = usize::try_from(reader.total_vectors) + .map_err(|_| invalid_input("negative IVF vector count"))?; + let nprobe = + params.resolve_ivf_nprobe(reader.nlist, total_vectors, matching_count)?; + progressive_ivf_search( + params, + reader.nlist, + nprobe, + 1, + params.top_k, + matching_count.unwrap_or(total_vectors), + |nprobe| { + search_with_reader_roaring_filter( + reader, + query, + params.top_k, + nprobe, + roaring_filter_bytes, + ) + }, + ) + } + Self::IvfRq(reader) => { + let total_vectors = usize::try_from(reader.total_vectors) + .map_err(|_| invalid_input("negative IVF vector count"))?; + let nprobe = + params.resolve_ivf_nprobe(reader.nlist, total_vectors, matching_count)?; + progressive_ivf_search( + params, + reader.nlist, + nprobe, + 1, + params.top_k, + matching_count.unwrap_or(total_vectors), + |nprobe| { + reader.search_with_roaring_filter( + query, + params.top_k, + nprobe, + roaring_filter_bytes, + ) + }, + ) + } + Self::DiskAnn(reader) => reader.search_with_roaring_filter( query, params.top_k, - params.nprobe, - params.hnsw_ef_search(), + params.resolve_diskann_l_search_with(reader.calibrated_l_search)?, roaring_filter_bytes, ), } @@ -758,41 +1697,94 @@ impl VectorIndexReader { params: VectorSearchParams, ) -> io::Result<(Vec, Vec)> { validate_queries(queries, query_count, self.dimension())?; - validate_query_bits_for_index(self.index_type(), params.query_bits)?; + params.validate()?; match self { - Self::IvfFlat(reader) => search_batch_ivfflat_reader( - reader, - queries, - query_count, - params.top_k, - params.nprobe, - ), + Self::IvfFlat(reader) => { + let total_vectors = usize::try_from(reader.total_vectors) + .map_err(|_| invalid_input("negative IVF vector count"))?; + let nprobe = params.resolve_ivf_nprobe(reader.nlist, total_vectors, None)?; + progressive_ivf_search( + params, + reader.nlist, + nprobe, + query_count, + params.top_k, + total_vectors, + |nprobe| { + search_batch_ivfflat_reader( + reader, + queries, + query_count, + params.top_k, + nprobe, + ) + }, + ) + } + Self::IvfSq(reader) => { + let total_vectors = usize::try_from(reader.total_vectors) + .map_err(|_| invalid_input("negative IVF vector count"))?; + let nprobe = params.resolve_ivf_nprobe(reader.nlist, total_vectors, None)?; + progressive_ivf_search( + params, + reader.nlist, + nprobe, + query_count, + params.top_k, + total_vectors, + |nprobe| { + search_batch_ivfsq_reader( + reader, + queries, + query_count, + params.top_k, + nprobe, + ) + }, + ) + } Self::IvfPq(reader) => { - search_batch_reader(reader, queries, query_count, params.top_k, params.nprobe) + let total_vectors = usize::try_from(reader.total_vectors) + .map_err(|_| invalid_input("negative IVF vector count"))?; + let nprobe = params.resolve_ivf_nprobe(reader.nlist, total_vectors, None)?; + progressive_ivf_search( + params, + reader.nlist, + nprobe, + query_count, + params.top_k, + total_vectors, + |nprobe| { + search_batch_reader(reader, queries, query_count, params.top_k, nprobe) + }, + ) } - Self::IvfRq(reader) => search_batch_ivfrq_reader_with_query_bits( - reader, - queries, - query_count, - params.top_k, - params.nprobe, - params.query_bits, - ), - Self::IvfHnswFlat(reader) => search_batch_ivfhnswflat_reader( - reader, - queries, - query_count, - params.top_k, - params.nprobe, - params.hnsw_ef_search(), - ), - Self::IvfHnswSq(reader) => search_batch_ivfhnswsq_reader( - reader, + Self::IvfRq(reader) => { + let total_vectors = usize::try_from(reader.total_vectors) + .map_err(|_| invalid_input("negative IVF vector count"))?; + let nprobe = params.resolve_ivf_nprobe(reader.nlist, total_vectors, None)?; + progressive_ivf_search( + params, + reader.nlist, + nprobe, + query_count, + params.top_k, + total_vectors, + |nprobe| { + search_batch_ivfrq_reader( + reader, + queries, + query_count, + params.top_k, + nprobe, + ) + }, + ) + } + Self::DiskAnn(reader) => reader.search_batch( queries, - query_count, params.top_k, - params.nprobe, - params.hnsw_ef_search(), + params.resolve_diskann_l_search_with(reader.calibrated_l_search)?, ), } } @@ -805,55 +1797,162 @@ impl VectorIndexReader { roaring_filter_bytes: &[u8], ) -> io::Result<(Vec, Vec)> { validate_queries(queries, query_count, self.dimension())?; - validate_query_bits_for_index(self.index_type(), params.query_bits)?; + params.validate()?; + let matching_count = if params.search_width == SearchWidth::Auto { + Some(decode_roaring_filter_cardinality(roaring_filter_bytes)?) + } else { + None + }; match self { - Self::IvfFlat(reader) => search_batch_ivfflat_reader_roaring_filter( - reader, - queries, - query_count, - params.top_k, - params.nprobe, - roaring_filter_bytes, - ), - Self::IvfPq(reader) => search_batch_reader_roaring_filter( - reader, - queries, - query_count, - params.top_k, - params.nprobe, - roaring_filter_bytes, - ), - Self::IvfRq(reader) => search_batch_ivfrq_reader_roaring_filter_with_query_bits( - reader, - queries, - query_count, - params.top_k, - params.nprobe, - roaring_filter_bytes, - params.query_bits, - ), - Self::IvfHnswFlat(reader) => search_batch_ivfhnswflat_reader_roaring_filter( - reader, - queries, - query_count, - params.top_k, - params.nprobe, - params.hnsw_ef_search(), - roaring_filter_bytes, - ), - Self::IvfHnswSq(reader) => search_batch_ivfhnswsq_reader_roaring_filter( - reader, + Self::IvfFlat(reader) => { + let total_vectors = usize::try_from(reader.total_vectors) + .map_err(|_| invalid_input("negative IVF vector count"))?; + let nprobe = + params.resolve_ivf_nprobe(reader.nlist, total_vectors, matching_count)?; + progressive_ivf_search( + params, + reader.nlist, + nprobe, + query_count, + params.top_k, + matching_count.unwrap_or(total_vectors), + |nprobe| { + search_batch_ivfflat_reader_roaring_filter( + reader, + queries, + query_count, + params.top_k, + nprobe, + roaring_filter_bytes, + ) + }, + ) + } + Self::IvfSq(reader) => { + let total_vectors = usize::try_from(reader.total_vectors) + .map_err(|_| invalid_input("negative IVF vector count"))?; + let nprobe = + params.resolve_ivf_nprobe(reader.nlist, total_vectors, matching_count)?; + progressive_ivf_search( + params, + reader.nlist, + nprobe, + query_count, + params.top_k, + matching_count.unwrap_or(total_vectors), + |nprobe| { + search_batch_ivfsq_reader_roaring_filter( + reader, + queries, + query_count, + params.top_k, + nprobe, + roaring_filter_bytes, + ) + }, + ) + } + Self::IvfPq(reader) => { + let total_vectors = usize::try_from(reader.total_vectors) + .map_err(|_| invalid_input("negative IVF vector count"))?; + let nprobe = + params.resolve_ivf_nprobe(reader.nlist, total_vectors, matching_count)?; + progressive_ivf_search( + params, + reader.nlist, + nprobe, + query_count, + params.top_k, + matching_count.unwrap_or(total_vectors), + |nprobe| { + search_batch_reader_roaring_filter( + reader, + queries, + query_count, + params.top_k, + nprobe, + roaring_filter_bytes, + ) + }, + ) + } + Self::IvfRq(reader) => { + let total_vectors = usize::try_from(reader.total_vectors) + .map_err(|_| invalid_input("negative IVF vector count"))?; + let nprobe = + params.resolve_ivf_nprobe(reader.nlist, total_vectors, matching_count)?; + progressive_ivf_search( + params, + reader.nlist, + nprobe, + query_count, + params.top_k, + matching_count.unwrap_or(total_vectors), + |nprobe| { + search_batch_ivfrq_reader_roaring_filter( + reader, + queries, + query_count, + params.top_k, + nprobe, + roaring_filter_bytes, + ) + }, + ) + } + Self::DiskAnn(reader) => reader.search_batch_with_roaring_filter( queries, - query_count, params.top_k, - params.nprobe, - params.hnsw_ef_search(), + params.resolve_diskann_l_search_with(reader.calibrated_l_search)?, roaring_filter_bytes, ), } } } +fn decode_roaring_filter_cardinality(bytes: &[u8]) -> io::Result { + let filter = RoaringTreemap::deserialize_from(&mut Cursor::new(bytes)).map_err(|error| { + invalid_input(format!("invalid serialized RoaringTreemap filter: {error}")) + })?; + usize::try_from(filter.len()) + .map_err(|_| invalid_input("RoaringTreemap filter cardinality exceeds usize")) +} + +fn progressive_ivf_search( + params: VectorSearchParams, + nlist: usize, + initial_nprobe: usize, + query_count: usize, + top_k: usize, + available_matches: usize, + mut search: impl FnMut(usize) -> io::Result<(Vec, Vec)>, +) -> io::Result<(Vec, Vec)> { + let required_per_query = top_k.min(available_matches); + let mut nprobe = initial_nprobe; + loop { + let result = search(nprobe)?; + if params.search_width != SearchWidth::Auto + || nprobe >= nlist + || required_per_query == 0 + || result + .1 + .chunks_exact(top_k) + .take(query_count) + .all(|distances| { + distances + .iter() + .filter(|&&distance| distance != f32::MAX) + .take(required_per_query) + .count() + >= required_per_query + }) + { + return Ok(result); + } + nprobe = nprobe.saturating_mul(2).min(nlist); + } +} + fn validate_config(config: &VectorIndexConfig) -> io::Result<()> { validate_positive(config.dimension(), "dimension")?; validate_positive(config.nlist(), "nlist")?; @@ -867,43 +1966,35 @@ fn validate_config(config: &VectorIndexConfig) -> io::Result<()> { )); } } - VectorIndexConfig::IvfRq { dimension, .. } if !dimension.is_multiple_of(8) => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("dimension {} must be divisible by 8 for IVF_RQ", dimension), - )); + VectorIndexConfig::IvfRq { bits, .. } if !is_supported_rq_bits(*bits) => { + return Err(invalid_input(format!( + "rq.bits must be in 1..=8, got {bits}" + ))); } - VectorIndexConfig::IvfHnswFlat { hnsw, .. } | VectorIndexConfig::IvfHnswSq { hnsw, .. } => { - validate_hnsw_params(*hnsw)? + VectorIndexConfig::DiskAnn { + dimension, + metric, + pq_m, + pq_bits, + build, + } => { + validate_diskann_config(*dimension, *metric, *pq_m, *pq_bits, *build)?; } _ => {} } Ok(()) } -fn validate_hnsw_params(params: HnswBuildParams) -> io::Result<()> { - validate_positive(params.m, "hnsw m")?; - validate_positive(params.ef_construction, "hnsw ef_construction")?; - validate_positive(params.max_level, "hnsw max_level") -} - -fn validate_query_bits_for_index(index_type: IndexType, query_bits: usize) -> io::Result<()> { - if query_bits == DEFAULT_RQ_QUERY_BITS { - return Ok(()); - } - if index_type != IndexType::IvfRq { - return Err(invalid_input(format!( - "query_bits is only supported by IVF_RQ, but index type is {}", - index_type.as_str() - ))); - } - if !is_supported_query_bits(query_bits) { - return Err(invalid_input(format!( - "invalid IVF_RQ query_bits {}; expected 0, 4, or 8", - query_bits - ))); - } - Ok(()) +fn validate_diskann_config( + dimension: usize, + metric: MetricType, + pq_m: usize, + pq_bits: usize, + build: DiskAnnBuildParams, +) -> io::Result<()> { + validate_diskann_format_configuration(dimension, pq_m, pq_bits, build)?; + validate_positive(build.memory_budget_bytes, "DiskANN memory budget")?; + validate_diskann_training_budget(dimension, metric, pq_m, pq_bits, build.memory_budget_bytes) } fn validate_positive(value: usize, name: &str) -> io::Result<()> { @@ -994,8 +2085,117 @@ fn validate_finite_values(values: &[f32], len: usize, value_name: &str) -> io::R #[cfg(test)] mod tests { use super::*; - use crate::io::PosWriter; + use crate::diskann::DISKANN_MAX_PQ_TRAINING_VECTORS; + use crate::io::{PosWriter, ReadRequest, SeekRead}; + use roaring::RoaringTreemap; use std::io::Cursor; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Condvar, Mutex}; + use std::time::{Duration, Instant}; + + #[derive(Default)] + struct ReadRendezvousState { + armed: bool, + arrivals: usize, + } + + struct CountingReader { + inner: Cursor>, + calls: Arc, + } + + impl SeekRead for CountingReader { + fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { + self.calls.fetch_add(1, Ordering::Relaxed); + self.inner.pread(ranges) + } + } + + #[derive(Default)] + struct ReadRendezvous { + state: Mutex, + ready: Condvar, + } + + impl ReadRendezvous { + fn arm(&self) { + *self.state.lock().unwrap() = ReadRendezvousState { + armed: true, + arrivals: 0, + }; + } + + fn wait_for_peer(&self) -> io::Result<()> { + let mut state = self.state.lock().unwrap(); + if !state.armed { + return Ok(()); + } + state.arrivals += 1; + if state.arrivals >= 2 { + state.armed = false; + self.ready.notify_all(); + return Ok(()); + } + + let deadline = Instant::now() + Duration::from_secs(5); + while state.armed { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + state.armed = false; + self.ready.notify_all(); + return Err(io::Error::other( + "timed out waiting for a concurrent cloned-reader pread", + )); + } + let (next, timeout) = self.ready.wait_timeout(state, remaining).unwrap(); + state = next; + if timeout.timed_out() && state.armed { + state.armed = false; + self.ready.notify_all(); + return Err(io::Error::other( + "timed out waiting for a concurrent cloned-reader pread", + )); + } + } + Ok(()) + } + } + + #[derive(Clone)] + struct ConcurrentCloneReader { + bytes: Arc<[u8]>, + active_reads: Arc, + max_active_reads: Arc, + rendezvous: Arc, + } + + impl SeekRead for ConcurrentCloneReader { + fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { + let active = self.active_reads.fetch_add(1, Ordering::SeqCst) + 1; + self.max_active_reads.fetch_max(active, Ordering::SeqCst); + let result = self.rendezvous.wait_for_peer().and_then(|()| { + ranges.iter_mut().try_for_each(|range| { + let start = usize::try_from(range.pos) + .map_err(|_| io::Error::other("test read offset exceeds usize"))?; + let end = start + .checked_add(range.buf.len()) + .ok_or_else(|| io::Error::other("test read range overflows"))?; + let source = self + .bytes + .get(start..end) + .ok_or_else(|| io::Error::from(io::ErrorKind::UnexpectedEof))?; + range.buf.copy_from_slice(source); + Ok(()) + }) + }); + self.active_reads.fetch_sub(1, Ordering::SeqCst); + result + } + + fn try_clone_reader(&self) -> io::Result> { + Ok(Some(self.clone())) + } + } fn generate_clustered_data(n: usize, d: usize, clusters: usize) -> Vec { let mut data = vec![0.0; n * d]; @@ -1028,12 +2228,47 @@ mod tests { assert_eq!(metadata.dimension, d); assert_eq!(metadata.nlist, nlist); assert_eq!(metadata.total_vectors, n as i64); + match &config { + VectorIndexConfig::IvfPq { m, .. } => { + assert_eq!(metadata.pq_m, Some(*m)); + assert_eq!(metadata.pq_bits, Some(8)); + } + VectorIndexConfig::IvfSq { .. } => { + assert_eq!(metadata.pq_m, None); + assert_eq!(metadata.pq_bits, Some(8)); + } + VectorIndexConfig::DiskAnn { + pq_m, + pq_bits, + build, + .. + } => { + assert_eq!(metadata.pq_m, Some(*pq_m)); + assert_eq!(metadata.pq_bits, Some(*pq_bits)); + let diskann = metadata.diskann.expect("DiskANN metadata"); + assert_eq!(diskann.max_degree, build.max_degree); + assert_eq!(diskann.build_search_list_size, build.build_search_list_size); + assert_eq!(diskann.alpha, build.alpha); + } + _ => { + assert_eq!(metadata.pq_m, None); + assert_eq!(metadata.pq_bits, None); + } + } - let params = VectorSearchParams::with_ef_search(5, nlist, 32); + let params = if config.index_type() == IndexType::DiskAnn { + VectorSearchParams::with_l_search(5, 100) + } else { + VectorSearchParams::new(5, nlist) + }; let (result_ids, result_dists) = reader.search(&data[0..d], params).unwrap(); assert_eq!(result_ids.len(), 5); assert_eq!(result_dists.len(), 5); - assert_eq!(result_ids[0], 0); + if config.index_type() == IndexType::DiskAnn { + assert!(result_ids[0] >= 0); + } else { + assert_eq!(result_ids[0], 0); + } } fn build_reader(config: VectorIndexConfig) -> (VectorIndexReader>>, Vec) { @@ -1073,6 +2308,141 @@ mod tests { VectorIndexWriter::new(training) } + #[test] + fn diskann_search_stats_are_available_through_unified_reader() { + let dimension = 8; + let (mut reader, data) = build_reader(VectorIndexConfig::DiskAnn { + dimension, + metric: MetricType::L2, + pq_m: 2, + pq_bits: 8, + build: DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + }); + + reader + .search( + &data[..dimension], + VectorSearchParams::with_l_search(3, 100), + ) + .unwrap(); + + let stats = reader.diskann_search_stats().expect("DiskANN diagnostics"); + assert_eq!(stats.query_count, 1); + assert!(stats.rerank_candidate_references >= 3); + assert!(stats.rerank_unique_windows >= 1); + } + + #[test] + fn diskann_batch_search_overlaps_graph_reads_and_centralizes_filtered_rerank() { + let dimension = 8; + let count = 256; + let data = generate_clustered_data(count, dimension, 16); + let mut writer = build_writer( + VectorIndexConfig::DiskAnn { + dimension, + metric: MetricType::L2, + pq_m: 2, + pq_bits: 8, + build: DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + }, + &data, + count, + ); + writer + .add_vectors(&(0..count as i64).collect::>(), &data, count) + .unwrap(); + let mut bytes = Vec::new(); + writer.write(&mut PosWriter::new(&mut bytes)).unwrap(); + + let active_reads = Arc::new(AtomicUsize::new(0)); + let max_active_reads = Arc::new(AtomicUsize::new(0)); + let rendezvous = Arc::new(ReadRendezvous::default()); + let source = ConcurrentCloneReader { + bytes: Arc::from(bytes), + active_reads: Arc::clone(&active_reads), + max_active_reads: Arc::clone(&max_active_reads), + rendezvous: Arc::clone(&rendezvous), + }; + let mut reader = VectorIndexReader::open_with_options( + source, + VectorIndexReaderOptions::with_cache_budgets( + DeploymentProfile::Auto, + 0, + 0, + 4 * 1024 * 1024 * 1024, + 8 * 1024 * 1024, + ), + ) + .unwrap(); + reader.optimize_for_search().unwrap(); + let queries = [0, 17, 34, 51] + .into_iter() + .flat_map(|row| data[row * dimension..(row + 1) * dimension].iter().copied()) + .collect::>(); + let params = VectorSearchParams::with_l_search(3, 32); + let expected = queries + .chunks_exact(dimension) + .flat_map(|query| reader.search(query, params).unwrap().0) + .collect::>(); + max_active_reads.store(0, Ordering::SeqCst); + rendezvous.arm(); + + let (actual, _) = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .unwrap() + .install(|| reader.search_batch(&queries, 4, params)) + .unwrap(); + + assert_eq!(actual, expected); + assert!( + max_active_reads.load(Ordering::SeqCst) > 1, + "DiskANN batch search should overlap reads from independent queries" + ); + + let mut filter = RoaringTreemap::new(); + for row_id in 0..count as u64 { + filter.insert(row_id); + } + let mut filter_bytes = Vec::new(); + filter.serialize_into(&mut filter_bytes).unwrap(); + let expected_filtered = queries + .chunks_exact(dimension) + .flat_map(|query| { + reader + .search_with_roaring_filter(query, params, &filter_bytes) + .unwrap() + .0 + }) + .collect::>(); + max_active_reads.store(0, Ordering::SeqCst); + + let (actual_filtered, _) = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .unwrap() + .install(|| reader.search_batch_with_roaring_filter(&queries, 4, params, &filter_bytes)) + .unwrap(); + + assert_eq!(actual_filtered, expected_filtered); + assert!( + max_active_reads.load(Ordering::SeqCst) <= 1, + "filtered candidate workers must leave raw-vector I/O to the parent reranker" + ); + assert!( + reader.diskann_search_stats().unwrap().raw_vector_cache_hits > 0, + "the parent reranker should reuse prewarmed raw-vector windows" + ); + } + fn assert_invalid_input_contains(result: io::Result<()>, expected: &str) { let err = result.expect_err("invalid input should be rejected"); assert_eq!(err.kind(), io::ErrorKind::InvalidInput); @@ -1091,30 +2461,46 @@ mod tests { nlist: 4, metric: MetricType::L2, }); - roundtrip(VectorIndexConfig::IvfPq { - dimension: 16, - nlist: 4, - m: 4, - metric: MetricType::L2, - use_opq: false, - }); + roundtrip(VectorIndexConfig::ivf_pq(16, 4, MetricType::L2, false).unwrap()); roundtrip(VectorIndexConfig::IvfRq { dimension: 8, nlist: 4, + bits: DEFAULT_RQ_BITS, metric: MetricType::L2, }); - roundtrip(VectorIndexConfig::IvfHnswFlat { - dimension: 8, - nlist: 4, - metric: MetricType::L2, - hnsw: HnswBuildParams::default(), - }); - roundtrip(VectorIndexConfig::IvfHnswSq { + roundtrip(VectorIndexConfig::IvfSq { dimension: 8, nlist: 4, metric: MetricType::L2, - hnsw: HnswBuildParams::default(), }); + roundtrip( + VectorIndexConfig::disk_ann( + 8, + MetricType::L2, + 4, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ) + .unwrap(), + ); + for metric in [MetricType::InnerProduct, MetricType::Cosine] { + roundtrip( + VectorIndexConfig::disk_ann( + 8, + metric, + 4, + DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }, + ) + .unwrap(), + ); + } } #[test] @@ -1135,24 +2521,18 @@ mod tests { VectorIndexConfig::IvfRq { dimension: 8, nlist: 4, + bits: DEFAULT_RQ_BITS, metric: MetricType::L2, }, - VectorIndexConfig::IvfHnswFlat { + VectorIndexConfig::IvfSq { dimension: 8, nlist: 4, metric: MetricType::L2, - hnsw: HnswBuildParams::default(), - }, - VectorIndexConfig::IvfHnswSq { - dimension: 8, - nlist: 4, - metric: MetricType::L2, - hnsw: HnswBuildParams::default(), }, ] { let d = config.dimension(); let nlist = config.nlist(); - let params = VectorSearchParams::with_ef_search(5, nlist, 32); + let params = VectorSearchParams::new(5, nlist); let (mut baseline, data) = build_reader(config.clone()); let query = data[0..d].to_vec(); let expected = baseline.search(&query, params).unwrap(); @@ -1174,6 +2554,19 @@ mod tests { } } + #[test] + fn representative_warmup_accepts_an_empty_query_set_for_resident_only_initialization() { + let mut reader = build_ivfflat_reader(); + + reader.warmup_queries(&[], 0, 0).unwrap(); + + let error = reader + .warmup_queries(&[0.0], 0, 0) + .expect_err("empty warmup count must still validate the vector buffer"); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert!(error.to_string().contains("warmup queries length")); + } + #[test] fn unified_reader_rejects_unknown_magic() { let err = match VectorIndexReader::open(Cursor::new(vec![0xFF; 8])) { @@ -1183,6 +2576,36 @@ mod tests { assert!(err.to_string().contains("unknown vector index magic")); } + #[test] + fn unified_ivf_reader_reuses_dispatch_header() { + let d = 2; + let data = vec![0.0, 0.0, 1.0, 1.0]; + let ids = vec![10, 11]; + let mut index = IVFFlatIndex::new(d, 1, MetricType::L2); + index.train(&data, 2); + index.add(&data, &ids, 2); + let mut bytes = Vec::new(); + write_ivfflat_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + + let calls = Arc::new(AtomicUsize::new(0)); + let source = CountingReader { + inner: Cursor::new(bytes), + calls: Arc::clone(&calls), + }; + let mut reader = VectorIndexReader::open(source).unwrap(); + assert_eq!( + calls.load(Ordering::Relaxed), + 1, + "type dispatch and format parsing share one header read" + ); + reader.optimize_for_search().unwrap(); + assert_eq!( + calls.load(Ordering::Relaxed), + 2, + "resident IVF metadata adds only one read round" + ); + } + #[test] fn unified_config_rejects_invalid_pq_m() { let err = match VectorIndexTrainer::new(VectorIndexConfig::IvfPq { @@ -1199,16 +2622,57 @@ mod tests { } #[test] - fn unified_config_rejects_invalid_rq_dimension() { + fn unified_config_accepts_unaligned_rq_dimension_and_rejects_invalid_bits() { + VectorIndexTrainer::new(VectorIndexConfig::IvfRq { + dimension: 10, + nlist: 4, + bits: DEFAULT_RQ_BITS, + metric: MetricType::L2, + }) + .unwrap(); let err = match VectorIndexTrainer::new(VectorIndexConfig::IvfRq { dimension: 10, nlist: 4, + bits: 9, metric: MetricType::L2, }) { Ok(_) => panic!("invalid RQ config should be rejected"), Err(err) => err, }; - assert!(err.to_string().contains("divisible by 8")); + assert!(err.to_string().contains("rq.bits")); + } + + #[test] + fn pq_m_inference_preserves_relative_code_budget_across_dimensions_and_bits() { + assert_eq!(infer_pq_m(128, 8, DEFAULT_PQ_CODE_RATIO).unwrap(), 32); + assert_eq!(infer_pq_m(960, 8, DEFAULT_PQ_CODE_RATIO).unwrap(), 240); + assert_eq!(infer_pq_m(128, 4, DEFAULT_PQ_CODE_RATIO).unwrap(), 64); + assert_eq!(infer_pq_m(960, 4, DEFAULT_PQ_CODE_RATIO).unwrap(), 480); + } + + #[test] + fn pq_m_inference_selects_the_nearest_balanced_subvector_shape() { + assert_eq!(infer_pq_m(10, 8, DEFAULT_PQ_CODE_RATIO).unwrap(), 3); + assert_eq!(infer_pq_m(384, 8, 0.125).unwrap(), 192); + assert_eq!(infer_pq_m(7, 4, DEFAULT_PQ_CODE_RATIO).unwrap(), 4); + } + + #[test] + fn pq_m_inference_rejects_invalid_code_ratios() { + for ratio in [0.0, f64::NAN, f64::INFINITY, 0.2501] { + let error = infer_pq_m(128, 8, ratio).expect_err("invalid ratio should fail"); + assert!(error.to_string().contains("pq.code-ratio")); + } + let error = infer_pq_m(128, 4, 0.1251) + .expect_err("4-bit code ratio cannot exceed its full-width encoding"); + assert!(error.to_string().contains("(0, 0.125]")); + } + + #[test] + fn diskann_l_search_is_independent_of_ivf_nprobe() { + let diskann = VectorSearchParams::with_l_search(10, 200); + assert_eq!(diskann.search_width, SearchWidth::DiskAnnLSearch); + assert_eq!(diskann.width, 200); } fn options(values: &[(&str, &str)]) -> HashMap { @@ -1218,6 +2682,737 @@ mod tests { .collect() } + #[test] + fn diskann_config_index_type_code_and_name() { + let index_type = IndexType::from_code(5).expect("DiskANN index type code should exist"); + assert_eq!(index_type.as_str(), "diskann"); + } + + #[test] + fn ivfsq_uses_a_new_type_code_and_retired_type_codes_stay_reserved() { + assert_eq!(IndexType::from_code(6), Some(IndexType::IvfSq)); + assert_eq!(IndexType::IvfSq.as_str(), "ivf_sq"); + assert_eq!(IndexType::from_code(2), None); + assert_eq!(IndexType::from_code(3), None); + } + + #[test] + fn diskann_config_parses_without_nlist() { + let config = VectorIndexConfig::from_options(&options(&[ + ("index.type", "diskann"), + ("dimension", "128"), + ("metric", "l2"), + ])) + .expect("valid DiskANN options should infer pq.m without nlist"); + + assert_eq!(config.index_type(), IndexType::DiskAnn); + assert_eq!(config.dimension(), 128); + assert_eq!(config.nlist(), 1); + let VectorIndexConfig::DiskAnn { + pq_m, + pq_bits, + build, + .. + } = config + else { + panic!("expected DiskANN config"); + }; + assert_eq!(pq_m, 32); + assert_eq!(pq_bits, 8); + assert_eq!(build.memory_budget_bytes, 8 * 1024 * 1024 * 1024); + } + + #[test] + fn pq_config_parses_code_ratio_and_explicit_m_takes_precedence() { + let auto = VectorIndexConfig::from_options(&options(&[ + ("index.type", "ivf_pq"), + ("dimension", "128"), + ("nlist", "4"), + ("metric", "l2"), + ("pq.code-ratio", "0.125"), + ])) + .expect("IVF-PQ should infer pq.m from a relative code budget"); + let VectorIndexConfig::IvfPq { m, .. } = auto else { + panic!("expected IVF-PQ config"); + }; + assert_eq!(m, 64); + + let explicit = VectorIndexConfig::from_options(&options(&[ + ("index.type", "diskann"), + ("dimension", "128"), + ("metric", "l2"), + ("pq.m", "16"), + ("pq.code-ratio", "0.125"), + ])) + .expect("explicit pq.m should override the code ratio"); + let VectorIndexConfig::DiskAnn { pq_m, .. } = explicit else { + panic!("expected DiskANN config"); + }; + assert_eq!(pq_m, 16); + } + + #[test] + fn ivf_config_resolves_automatic_nlist_from_expected_count() { + let plan = VectorIndexBuildPlan::from_options(&options(&[ + ("index.type", "ivf_sq"), + ("dimension", "100"), + ("expected-vector-count", "1183514"), + ("nlist", "auto"), + ("metric", "cosine"), + ])) + .unwrap(); + assert_eq!(plan.expected_vector_count, Some(1_183_514)); + assert_eq!(plan.config.nlist(), 1024); + assert_eq!(plan.config.resolved().nlist, 1024); + + let missing_count = VectorIndexConfig::from_options(&options(&[ + ("index.type", "ivf_flat"), + ("dimension", "8"), + ("metric", "l2"), + ])) + .unwrap_err(); + assert!(missing_count.to_string().contains("expected-vector-count")); + } + + #[test] + fn capacity_goal_resolves_rq_bits_and_diskann_deployment_layout() { + let rq = VectorIndexConfig::from_options(&options(&[ + ("index.type", "ivf_rq"), + ("dimension", "100"), + ("nlist", "16"), + ("metric", "l2"), + ("max-bytes-per-vector", "88"), + ])) + .unwrap(); + assert_eq!(rq.resolved().rq_bits, Some(3)); + + let diskann = VectorIndexConfig::from_options(&options(&[ + ("index.type", "diskann"), + ("dimension", "128"), + ("metric", "l2"), + ("deployment-profile", "local_storage"), + ("diskann.storage-layout", "auto"), + ])) + .unwrap(); + assert_eq!( + diskann.resolved().diskann_build.unwrap().storage_layout, + DiskAnnStorageLayout::Interleaved + ); + } + + #[test] + fn capacity_goal_rejects_indexes_that_cannot_meet_persisted_size_budget() { + let flat = VectorIndexConfig::from_options(&options(&[ + ("index.type", "ivf_flat"), + ("dimension", "128"), + ("nlist", "4"), + ("metric", "l2"), + ("max-bytes-per-vector", "64"), + ])) + .unwrap_err(); + assert!(flat.to_string().contains("cannot be satisfied")); + + let diskann = VectorIndexConfig::from_options(&options(&[ + ("index.type", "diskann"), + ("dimension", "128"), + ("metric", "l2"), + ("max-bytes-per-vector", "64"), + ])) + .unwrap_err(); + assert!(diskann.to_string().contains("cannot fit DiskANN")); + + let pq = VectorIndexConfig::from_options(&options(&[ + ("index.type", "ivf_pq"), + ("dimension", "128"), + ("nlist", "4"), + ("metric", "l2"), + ("max-bytes-per-vector", "80"), + ])) + .unwrap(); + assert!(pq.resolved().pq_m.unwrap() + PERSISTED_ROW_ID_ESTIMATE_BYTES <= 80); + } + + #[test] + fn build_time_goal_is_never_silently_guessed_from_hardware() { + let values = options(&[ + ("index.type", "ivf_sq"), + ("dimension", "16"), + ("nlist", "4"), + ("metric", "l2"), + ("max-build-seconds", "10"), + ]); + let plan = VectorIndexBuildPlan::from_options(&values).unwrap(); + assert_eq!(plan.objective.max_build_seconds, Some(10.0)); + + let error = VectorIndexConfig::from_options(&values).unwrap_err(); + assert!(error + .to_string() + .contains("requires measured offline calibration")); + } + + #[test] + fn tagged_search_width_rejects_cross_index_parameters() { + assert_eq!( + VectorSearchParams::automatic(10) + .resolve_ivf_nprobe(1024, 1_000_000, None) + .unwrap(), + 64 + ); + assert!(VectorSearchParams::with_l_search(10, 100) + .resolve_ivf_nprobe(1024, 1_000_000, None) + .unwrap_err() + .to_string() + .contains("cannot be used with an IVF")); + assert!(VectorSearchParams::new(10, 64) + .resolve_diskann_l_search() + .unwrap_err() + .to_string() + .contains("cannot be used with a DiskANN")); + } + + #[test] + fn automatic_filtered_search_expands_until_results_are_filled() { + let mut observed = Vec::new(); + let result = progressive_ivf_search( + VectorSearchParams::automatic(2), + 16, + 2, + 1, + 2, + 10, + |nprobe| { + observed.push(nprobe); + if nprobe < 8 { + Ok((vec![7, -1], vec![1.0, f32::MAX])) + } else { + Ok((vec![7, 8], vec![1.0, 2.0])) + } + }, + ) + .unwrap(); + assert_eq!(observed, vec![2, 4, 8]); + assert_eq!(result.0, vec![7, 8]); + } + + #[test] + fn automatic_search_treats_negative_row_ids_as_valid_results() { + let mut observed = Vec::new(); + let result = progressive_ivf_search( + VectorSearchParams::automatic(2), + 16, + 2, + 1, + 2, + 10, + |nprobe| { + observed.push(nprobe); + Ok((vec![-1, -7], vec![1.0, 2.0])) + }, + ) + .unwrap(); + assert_eq!(observed, vec![2]); + assert_eq!(result.0, vec![-1, -7]); + } + + #[test] + fn search_params_reject_zero_top_k_before_index_dispatch() { + assert!(VectorSearchParams::automatic(0) + .validate() + .unwrap_err() + .to_string() + .contains("top_k")); + assert!(VectorSearchParams::new(0, 1).validate().is_err()); + assert!(VectorSearchParams::with_l_search(0, 100) + .validate() + .is_err()); + } + + #[test] + fn pq_config_constructors_use_the_default_relative_budget() { + let ivf = VectorIndexConfig::ivf_pq(128, 4, MetricType::L2, false).unwrap(); + let VectorIndexConfig::IvfPq { m, .. } = ivf else { + panic!("expected IVF-PQ config"); + }; + assert_eq!(m, 32); + + let diskann = + VectorIndexConfig::disk_ann(960, MetricType::L2, 8, DiskAnnBuildParams::default()) + .unwrap(); + let VectorIndexConfig::DiskAnn { pq_m, .. } = diskann else { + panic!("expected DiskANN config"); + }; + assert_eq!(pq_m, 240); + } + + #[test] + fn diskann_config_parses_explicit_build_parameters() { + let config = VectorIndexConfig::from_options(&options(&[ + ("index.type", "diskann"), + ("dimension", "128"), + ("metric", "l2"), + ("pq.m", "16"), + ("pq.bits", "4"), + ("diskann.max-degree", "32"), + ("diskann.build-search-list-size", "64"), + ("diskann.alpha", "1.4"), + ("diskann.seed", "7"), + ("diskann.memory-budget-bytes", "123456"), + ("diskann.storage-layout", "interleaved"), + ("diskann.raw-vector-encoding", "f16"), + ("diskann.build-distance", "full_precision"), + ])) + .expect("explicit DiskANN build parameters should parse"); + + let VectorIndexConfig::DiskAnn { + pq_m, + pq_bits, + build, + .. + } = config + else { + panic!("expected DiskANN config"); + }; + assert_eq!(pq_m, 16); + assert_eq!(pq_bits, 4); + assert_eq!(build.max_degree, 32); + assert_eq!(build.build_search_list_size, 64); + assert_eq!(build.alpha, 1.4); + assert_eq!(build.seed, 7); + assert_eq!(build.memory_budget_bytes, 123456); + assert_eq!(build.storage_layout, DiskAnnStorageLayout::Interleaved); + assert_eq!(build.raw_vector_encoding, DiskAnnRawVectorEncoding::F16); + assert_eq!(build.build_distance, DiskAnnBuildDistance::FullPrecision); + } + + #[test] + fn diskann_build_width_default_follows_explicit_degree() { + let diskann = VectorIndexConfig::from_options(&options(&[ + ("index.type", "diskann"), + ("dimension", "128"), + ("metric", "l2"), + ("diskann.max-degree", "128"), + ])) + .expect("omitted Lbuild should follow an explicit DiskANN degree"); + let VectorIndexConfig::DiskAnn { build, .. } = diskann else { + panic!("expected DiskANN config"); + }; + assert_eq!(build.max_degree, 128); + assert_eq!(build.build_search_list_size, 128); + } + + #[test] + fn explicit_diskann_build_search_width_is_preserved() { + let diskann = VectorIndexConfig::from_options(&options(&[ + ("index.type", "diskann"), + ("dimension", "128"), + ("metric", "l2"), + ("diskann.max-degree", "128"), + ("diskann.build-search-list-size", "256"), + ])) + .expect("explicit Lbuild should take precedence over its automatic default"); + let VectorIndexConfig::DiskAnn { build, .. } = diskann else { + panic!("expected DiskANN config"); + }; + assert_eq!(build.build_search_list_size, 256); + } + + #[test] + fn diskann_config_accepts_all_supported_metrics() { + for (name, expected) in [ + ("l2", MetricType::L2), + ("inner_product", MetricType::InnerProduct), + ("cosine", MetricType::Cosine), + ] { + let config = VectorIndexConfig::from_options(&options(&[ + ("index.type", "diskann"), + ("dimension", "128"), + ("metric", name), + ("pq.m", "16"), + ])) + .expect("DiskANN should accept every public metric"); + let VectorIndexConfig::DiskAnn { metric, .. } = config else { + panic!("expected DiskANN config"); + }; + assert_eq!(metric, expected); + } + } + + fn assert_diskann_metric_roundtrip( + metric: MetricType, + data: &[f32], + ids: &[i64], + queries: &[f32], + expected_ids: &[i64], + expected_distances: &[f32], + ) { + let dimension = 2; + let count = ids.len(); + let config = VectorIndexConfig::DiskAnn { + dimension, + metric, + pq_m: 1, + pq_bits: 4, + build: DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + raw_vector_encoding: DiskAnnRawVectorEncoding::F32, + build_distance: DiskAnnBuildDistance::ProductQuantized, + ..DiskAnnBuildParams::default() + }, + }; + let mut writer = build_writer(config, data, count); + writer.add_vectors(ids, data, count).unwrap(); + let mut bytes = Vec::new(); + writer.write(&mut PosWriter::new(&mut bytes)).unwrap(); + + let mut reader = VectorIndexReader::open(Cursor::new(bytes)).unwrap(); + assert_eq!(reader.metadata().metric, metric); + let params = VectorSearchParams::with_l_search(1, count); + let (single_ids, single_distances) = reader.search(&queries[..dimension], params).unwrap(); + assert_eq!(single_ids[0], expected_ids[0]); + assert!((single_distances[0] - expected_distances[0]).abs() < 1e-5); + + let query_count = queries.len() / dimension; + let (batch_ids, batch_distances) = + reader.search_batch(queries, query_count, params).unwrap(); + assert_eq!(batch_ids, expected_ids); + for (&actual, &expected) in batch_distances.iter().zip(expected_distances) { + assert!((actual - expected).abs() < 1e-5); + } + + let mut filter = RoaringTreemap::new(); + filter.insert(expected_ids[1] as u64); + let mut filter_bytes = Vec::new(); + filter.serialize_into(&mut filter_bytes).unwrap(); + let (filtered_ids, filtered_distances) = reader + .search_with_roaring_filter(&queries[dimension..2 * dimension], params, &filter_bytes) + .unwrap(); + assert_eq!(filtered_ids[0], expected_ids[1]); + assert!((filtered_distances[0] - expected_distances[1]).abs() < 1e-5); + } + + #[test] + fn diskann_inner_product_roundtrips_single_batch_and_filtered_search() { + let ids = (100..116).collect::>(); + let data = vec![ + 10.0, 0.0, // highest IP for [1, 0], deliberately far in L2 + 1.0, 0.0, 0.0, 8.0, // highest IP for [0, 1] + 0.0, 1.0, -1.0, 0.0, 0.0, -1.0, -2.0, 1.0, 1.0, -2.0, -3.0, -1.0, -1.0, -3.0, -4.0, + 0.5, 0.5, -4.0, -5.0, -2.0, -2.0, -5.0, -6.0, -1.0, -2.0, 0.0, + ]; + + assert_diskann_metric_roundtrip( + MetricType::InnerProduct, + &data, + &ids, + &[1.0, 0.0, 0.0, 1.0], + &[100, 102], + &[-10.0, -8.0], + ); + } + + #[test] + fn diskann_cosine_roundtrips_single_batch_and_filtered_search() { + let ids = (200..216).collect::>(); + let data = vec![ + 10.0, 0.0, // cosine winner for [1, 0], deliberately far in L2 + 1.0, 1.0, 0.0, 5.0, // cosine winner for [0, 1] + 1.0, 2.0, -1.0, 0.0, 0.0, -1.0, -2.0, 1.0, 1.0, -2.0, -3.0, -1.0, -1.0, -3.0, -4.0, + 0.5, 0.5, -4.0, -5.0, -2.0, -2.0, -5.0, -6.0, -1.0, -2.0, 0.0, + ]; + + assert_diskann_metric_roundtrip( + MetricType::Cosine, + &data, + &ids, + &[1.0, 0.0, 0.0, 1.0, 0.0, 0.0], + &[200, 202, 200], + &[0.0, 0.0, 1.0], + ); + } + + #[test] + fn diskann_public_metrics_have_approximate_recall_below_full_scan_width() { + let dimension = 16; + let count = 512; + let top_k = 10; + let query_rows = (0..20).map(|index| index * 23 % count).collect::>(); + let data = (0..count * dimension) + .map(|offset| { + let row = offset / dimension; + let column = offset % dimension; + ((row * 31 + column * 17) as f32 * 0.037).sin() + + ((row * 13 + column * 29) as f32 * 0.019).cos() + + (row % 7) as f32 * 0.03 + }) + .collect::>(); + let queries = query_rows + .iter() + .flat_map(|&row| data[row * dimension..(row + 1) * dimension].iter().copied()) + .collect::>(); + + for metric in [MetricType::InnerProduct, MetricType::Cosine] { + let mut writer = build_writer( + VectorIndexConfig::DiskAnn { + dimension, + metric, + pq_m: 4, + pq_bits: 8, + build: DiskAnnBuildParams { + max_degree: 32, + build_search_list_size: 64, + raw_vector_encoding: DiskAnnRawVectorEncoding::F32, + ..DiskAnnBuildParams::default() + }, + }, + &data, + count, + ); + writer + .add_vectors(&(0..count as i64).collect::>(), &data, count) + .unwrap(); + let mut bytes = Vec::new(); + writer.write(&mut PosWriter::new(&mut bytes)).unwrap(); + let mut reader = VectorIndexReader::open(Cursor::new(bytes)).unwrap(); + + let (actual, _) = reader + .search_batch( + &queries, + query_rows.len(), + VectorSearchParams::with_l_search(top_k, 96), + ) + .unwrap(); + let mut overlap = 0usize; + for (query_index, query) in queries.chunks_exact(dimension).enumerate() { + let mut exact = (0..count) + .map(|row| { + ( + crate::distance::fvec_distance( + query, + &data[row * dimension..(row + 1) * dimension], + metric, + ), + row as i64, + ) + }) + .collect::>(); + exact.sort_by(|left, right| { + left.0 + .total_cmp(&right.0) + .then_with(|| left.1.cmp(&right.1)) + }); + let expected = exact + .iter() + .take(top_k) + .map(|entry| entry.1) + .collect::>(); + overlap += actual[query_index * top_k..(query_index + 1) * top_k] + .iter() + .filter(|row_id| expected.contains(row_id)) + .count(); + } + let recall = overlap as f32 / (query_rows.len() * top_k) as f32; + assert!( + recall >= 0.75, + "{metric:?} recall@{top_k} was {recall} with l_search=96 << {count}" + ); + } + } + + #[test] + fn diskann_config_rejects_invalid_shape_and_build_parameters() { + for (extra, expected) in [ + (vec![("dimension", "1025")], "at most 1024"), + (vec![("pq.m", "0")], "pq.m must be greater than 0"), + (vec![("pq.m", "129")], "must not exceed dimension"), + (vec![("pq.bits", "6")], "pq.bits must be 4 or 8"), + ( + vec![("diskann.raw-vector-encoding", "bf16")], + "must be auto, f32, or f16", + ), + ( + vec![ + ("diskann.max-degree", "65"), + ("diskann.build-search-list-size", "64"), + ], + "build search-list size", + ), + (vec![("diskann.alpha", "0.9")], "alpha must be at least 1"), + ( + vec![ + ("diskann.max-degree", "1024"), + ("diskann.build-search-list-size", "1024"), + ], + "adjacency list", + ), + ( + vec![ + ("dimension", "1024"), + ("pq.m", "16"), + ("diskann.raw-vector-encoding", "f32"), + ("diskann.storage-layout", "interleaved"), + ], + "raw vector and maximum adjacency", + ), + ] { + let mut values = vec![ + ("index.type", "diskann"), + ("dimension", "128"), + ("metric", "l2"), + ("pq.m", "16"), + ]; + for replacement in extra { + if let Some(existing) = values.iter_mut().find(|item| item.0 == replacement.0) { + *existing = replacement; + } else { + values.push(replacement); + } + } + + let error = VectorIndexConfig::from_options(&options(&values)) + .expect_err("invalid DiskANN config should be rejected"); + assert!( + error.to_string().contains(expected), + "error '{}' should contain '{}'", + error, + expected + ); + } + } + + #[test] + fn diskann_trainer_uses_configured_dimension() { + let trainer = VectorIndexTrainer::new(VectorIndexConfig::DiskAnn { + dimension: 8, + metric: MetricType::L2, + pq_m: 2, + pq_bits: 8, + build: DiskAnnBuildParams::default(), + }) + .expect("DiskANN trainer should open"); + + assert_eq!(trainer.dimension(), 8); + } + + #[test] + fn diskann_trainer_bounds_and_deterministically_reservoir_samples_training_data() { + let dimension = 2; + let count = DISKANN_MAX_PQ_TRAINING_VECTORS + 257; + let data = (0..count * dimension) + .map(|offset| offset as f32) + .collect::>(); + let config = || VectorIndexConfig::DiskAnn { + dimension, + metric: MetricType::L2, + pq_m: 1, + pq_bits: 8, + build: DiskAnnBuildParams { + seed: 73, + ..DiskAnnBuildParams::default() + }, + }; + + let mut whole = VectorIndexTrainer::new(config()).unwrap(); + whole.add_training_vectors_mut(&data, count).unwrap(); + let mut batched = VectorIndexTrainer::new(config()).unwrap(); + for vectors in data.chunks(137 * dimension) { + batched + .add_training_vectors_mut(vectors, vectors.len() / dimension) + .unwrap(); + } + + assert_eq!(whole.training_vector_count, DISKANN_MAX_PQ_TRAINING_VECTORS); + assert_eq!(whole.training_vectors_seen, count); + assert_eq!( + whole.training_data.len(), + DISKANN_MAX_PQ_TRAINING_VECTORS * dimension + ); + assert_eq!(whole.training_data, batched.training_data); + assert_ne!( + &whole.training_data, + &data[..DISKANN_MAX_PQ_TRAINING_VECTORS * dimension], + "reservoir sampling must give tail vectors a chance to enter the sample" + ); + } + + #[test] + fn diskann_trainer_derives_reservoir_limit_from_memory_budget() { + let dimension = 1024; + let memory_budget_bytes = 128 * 1024 * 1024; + let config = VectorIndexConfig::DiskAnn { + dimension, + metric: MetricType::L2, + pq_m: 256, + pq_bits: 8, + build: DiskAnnBuildParams { + memory_budget_bytes, + ..DiskAnnBuildParams::default() + }, + }; + let trainer = VectorIndexTrainer::new(config).unwrap(); + assert!(trainer.training_sample_limit < DISKANN_MAX_PQ_TRAINING_VECTORS); + assert_eq!( + trainer.training_sample_limit, + diskann_training_sample_limit(dimension, MetricType::L2, 256, 8, memory_budget_bytes,) + .unwrap() + ); + } + + #[test] + fn diskann_trainer_trains_existing_product_quantizer() { + let dimension = 8; + let count = 256; + let data = generate_clustered_data(count, dimension, 16); + let training = VectorIndexTrainer::train( + VectorIndexConfig::DiskAnn { + dimension, + metric: MetricType::L2, + pq_m: 2, + pq_bits: 8, + build: DiskAnnBuildParams::default(), + }, + &data, + count, + ) + .expect("DiskANN PQ training should succeed"); + + assert_eq!(training.index_type(), IndexType::DiskAnn); + assert_eq!(training.dimension(), dimension); + } + + #[test] + fn diskann_writer_accumulates_multiple_vector_batches() { + let dimension = 8; + let training_count = 256; + let training_data = generate_clustered_data(training_count, dimension, 16); + let training = VectorIndexTrainer::train( + VectorIndexConfig::DiskAnn { + dimension, + metric: MetricType::L2, + pq_m: 2, + pq_bits: 8, + build: DiskAnnBuildParams::default(), + }, + &training_data, + training_count, + ) + .unwrap(); + let mut writer = VectorIndexWriter::new(training); + + writer + .add_vectors(&[10, 11], &training_data[..2 * dimension], 2) + .unwrap(); + writer + .add_vectors(&[12], &training_data[2 * dimension..3 * dimension], 1) + .unwrap(); + + let VectorIndexWriter::DiskAnn(index) = writer else { + panic!("expected DiskANN writer"); + }; + assert_eq!(index.ids, vec![10, 11, 12]); + assert_eq!(index.vectors, training_data[..3 * dimension]); + } + #[test] fn config_from_options_parses_all_index_types() { assert_eq!( @@ -1236,7 +3431,7 @@ mod tests { ("index.type", "ivf_pq"), ("dimension", "16"), ("nlist", "4"), - ("pq.m", "4"), + ("metric", "l2"), ("use-opq", "true"), ])) .unwrap() @@ -1259,31 +3454,35 @@ mod tests { VectorIndexConfig::IvfRq { dimension, nlist, + bits, metric, } => { assert_eq!(dimension, 8); assert_eq!(nlist, 4); + assert_eq!(bits, DEFAULT_RQ_BITS); assert_eq!(metric, MetricType::Cosine); } _ => panic!("expected IVF RQ config"), } match VectorIndexConfig::from_options(&options(&[ - ("index.type", "ivf_hnsw_sq"), + ("index.type", "ivf_sq"), ("dimension", "8"), ("nlist", "4"), - ("hnsw.m", "12"), - ("hnsw.ef-construction", "64"), - ("hnsw.max-level", "5"), + ("metric", "inner_product"), ])) .unwrap() { - VectorIndexConfig::IvfHnswSq { hnsw, .. } => { - assert_eq!(hnsw.m, 12); - assert_eq!(hnsw.ef_construction, 64); - assert_eq!(hnsw.max_level, 5); + VectorIndexConfig::IvfSq { + dimension, + nlist, + metric, + } => { + assert_eq!(dimension, 8); + assert_eq!(nlist, 4); + assert_eq!(metric, MetricType::InnerProduct); } - _ => panic!("expected IVF HNSW SQ config"), + _ => panic!("expected IVF-SQ config"), } } @@ -1325,6 +3524,7 @@ mod tests { ("index.type", "ivf_flat"), ("dimension", "8"), ("nlist", "4"), + ("metric", "l2"), ("unused", "value"), ])) .unwrap_err(); diff --git a/core/src/index_io_util.rs b/core/src/index_io_util.rs index e5f2796..6d486e9 100644 --- a/core/src/index_io_util.rs +++ b/core/src/index_io_util.rs @@ -15,11 +15,225 @@ // specific language governing permissions and limitations // under the License. -use crate::distance::MetricType; -use crate::hnsw::{HnswBuildParams, HnswGraph}; -use crate::io::{PreadCursor, SeekRead, SeekWrite}; +use crate::io::{ReadRequest, SeekRead, SeekWrite}; use roaring::RoaringTreemap; use std::io; +use std::mem::size_of; + +pub(crate) const MAX_IVF_BATCH_READ_BYTES: usize = 64 * 1024 * 1024; +const IVF_STREAM_ALLOCATION_SLACK: usize = 64; + +pub(crate) fn ivf_payload_is_oversized(payload_len: usize) -> bool { + payload_len > MAX_IVF_BATCH_READ_BYTES +} + +/// Resolves a row count whose list payload plus the retained decoded IDs stays +/// within the IVF search allocation bound. Blocked layouts can require chunks +/// before the final one to contain a whole number of rows per block. +pub(crate) fn bounded_ivf_stream_chunk_rows( + remaining_rows: usize, + row_bytes: usize, + retained_id_bytes: usize, + row_alignment: usize, +) -> io::Result { + if remaining_rows == 0 { + return Ok(0); + } + if row_bytes == 0 || row_alignment == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "IVF streaming row shape must be greater than zero", + )); + } + let available = MAX_IVF_BATCH_READ_BYTES + .checked_sub(retained_id_bytes) + .and_then(|bytes| bytes.checked_sub(IVF_STREAM_ALLOCATION_SLACK)) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVF decoded row IDs exceed the bounded streaming allocation", + ) + })?; + let mut rows = (available / row_bytes).min(remaining_rows); + if rows < remaining_rows && row_alignment > 1 { + rows -= rows % row_alignment; + } + if rows == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "one IVF streaming block exceeds the bounded allocation", + )); + } + Ok(rows) +} + +/// Reads and validates one delta-varint ID section without retaining the list's +/// potentially much larger vector/code payload. The encoded prefix and decoded +/// IDs are checked together before allocation. +pub(crate) fn read_delta_varint_ids_at( + reader: &mut R, + offset: u64, + count: usize, + id_bytes_len: usize, + format_name: &str, +) -> io::Result> { + let prefix_len = 12usize.checked_add(id_bytes_len).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("{format_name} ID prefix size overflows usize"), + ) + })?; + let decoded_bytes = count.checked_mul(size_of::()).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("{format_name} decoded ID size overflows usize"), + ) + })?; + if prefix_len + .checked_add(decoded_bytes) + .is_none_or(|peak| peak > MAX_IVF_BATCH_READ_BYTES) + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "{format_name} ID section and decoded IDs exceed the bounded streaming allocation" + ), + )); + } + let mut prefix = vec![0u8; prefix_len]; + reader.pread(&mut [ReadRequest::new(offset, &mut prefix)])?; + let base_id = i64::from_le_bytes(prefix[0..8].try_into().unwrap()); + let stored_id_bytes_len = i32::from_le_bytes(prefix[8..12].try_into().unwrap()); + if stored_id_bytes_len < 0 || stored_id_bytes_len as usize != id_bytes_len { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("{format_name} ID length does not match the offset table"), + )); + } + decode_delta_varint_ids(base_id, &prefix[12..], count) +} + +/// Returns the largest non-empty prefix whose aggregate payload and range +/// count fit one IVF read/scan batch. Zero-length entries do not consume a +/// range. A single oversized payload is returned alone. +pub(crate) fn bounded_ivf_payload_batch_end( + payload_lengths: &[usize], + max_ranges_per_pread: usize, +) -> io::Result { + if payload_lengths.is_empty() { + return Ok(0); + } + let max_ranges = match max_ranges_per_pread { + 0 => usize::MAX, + value => value, + }; + let mut bytes = 0usize; + let mut ranges = 0usize; + for (index, &payload_len) in payload_lengths.iter().enumerate() { + let next_ranges = ranges + usize::from(payload_len != 0); + let next_bytes = bytes.checked_add(payload_len).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVF batch payload size overflow", + ) + })?; + if index > 0 && (next_ranges > max_ranges || next_bytes > MAX_IVF_BATCH_READ_BYTES) { + return Ok(index); + } + ranges = next_ranges; + bytes = next_bytes; + } + Ok(payload_lengths.len()) +} + +/// Reads payloads in capability-bounded multi-range batches. +/// +/// A single payload larger than `max_batch_bytes` is still issued alone so +/// callers never stall at the same batch boundary. +pub(crate) fn pread_batched_payloads( + reader: &mut R, + offsets: &[u64], + payloads: &mut [Vec], +) -> io::Result<()> { + let mut buffers = payloads + .iter_mut() + .map(Vec::as_mut_slice) + .collect::>(); + pread_batched_slices(reader, offsets, &mut buffers) +} + +/// Reads caller-owned byte slices with the same bounded IVF multi-range plan. +/// +/// This variant lets typed or aligned payload owners receive bytes directly +/// without first allocating a second `Vec` for every inverted list. +pub(crate) fn pread_batched_slices( + reader: &mut R, + offsets: &[u64], + payloads: &mut [&mut [u8]], +) -> io::Result<()> { + pread_batched_slices_with_limit(reader, offsets, payloads, MAX_IVF_BATCH_READ_BYTES) +} + +#[cfg(test)] +fn pread_batched_payloads_with_limit( + reader: &mut R, + offsets: &[u64], + payloads: &mut [Vec], + max_batch_bytes: usize, +) -> io::Result<()> { + let mut buffers = payloads + .iter_mut() + .map(Vec::as_mut_slice) + .collect::>(); + pread_batched_slices_with_limit(reader, offsets, &mut buffers, max_batch_bytes) +} + +fn pread_batched_slices_with_limit( + reader: &mut R, + offsets: &[u64], + payloads: &mut [&mut [u8]], + max_batch_bytes: usize, +) -> io::Result<()> { + if offsets.len() != payloads.len() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "IVF batch offsets and payloads must have the same length", + )); + } + let max_ranges = match reader.read_capabilities().max_ranges_per_pread { + 0 => usize::MAX, + value => value, + }; + let mut start = 0usize; + while start < payloads.len() { + let mut end = start; + let mut bytes = 0usize; + while end < payloads.len() && end - start < max_ranges { + let next = bytes.checked_add(payloads[end].len()).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVF batch payload size overflow", + ) + })?; + if end > start && next > max_batch_bytes { + break; + } + bytes = next; + end += 1; + } + if end == start { + end += 1; + } + let mut requests = payloads[start..end] + .iter_mut() + .zip(&offsets[start..end]) + .map(|(payload, &offset)| ReadRequest::new(offset, payload)) + .collect::>(); + reader.pread(&mut requests)?; + start = end; + } + Ok(()) +} pub(crate) fn validate_search_inputs( queries: &[f32], @@ -75,12 +289,6 @@ pub(crate) fn validate_reserved_zero(bytes: &[u8], format_name: &str) -> io::Res Ok(()) } -const HNSW_GRAPH_MAGIC: u32 = 0x48574752; // "HWGR" -const HNSW_GRAPH_VERSION: u32 = 1; -const HNSW_GRAPH_FLAG_DELTA_VARINT: u32 = 1 << 0; -const HNSW_GRAPH_REQUIRED_FLAGS: u32 = HNSW_GRAPH_FLAG_DELTA_VARINT; -const HNSW_GRAPH_SUPPORTED_FLAGS: u32 = HNSW_GRAPH_REQUIRED_FLAGS; - pub(crate) fn encode_delta_varint_ids(ids: &[i64]) -> (i64, Vec) { if ids.is_empty() { return (0, Vec::new()); @@ -123,228 +331,6 @@ pub(crate) fn decode_delta_varint_ids(base: i64, buf: &[u8], count: usize) -> io Ok(ids) } -pub(crate) fn encode_graph(graph: Option<&HnswGraph>) -> io::Result> { - let Some(graph) = graph else { - return Ok(Vec::new()); - }; - let mut buf = Vec::new(); - write_u32_fixed(&mut buf, HNSW_GRAPH_MAGIC)?; - write_u32_fixed(&mut buf, HNSW_GRAPH_VERSION)?; - write_u32_fixed(&mut buf, HNSW_GRAPH_REQUIRED_FLAGS)?; - write_u32_varint(&mut buf, graph.len())?; - write_u32_varint(&mut buf, graph.entry_point())?; - write_u32_varint(&mut buf, graph.max_observed_level())?; - for &level in graph.levels() { - write_u32_varint(&mut buf, level)?; - } - for node_levels in graph.neighbors() { - for level_neighbors in node_levels { - write_u32_varint(&mut buf, level_neighbors.len())?; - let mut sorted_neighbors = level_neighbors.clone(); - sorted_neighbors.sort_unstable(); - let mut previous = 0usize; - for neighbor in sorted_neighbors { - let delta = neighbor.checked_sub(previous).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "graph neighbor delta underflow", - ) - })?; - write_u32_varint(&mut buf, delta)?; - previous = neighbor; - } - } - } - Ok(buf) -} - -#[cfg(test)] -pub(crate) fn encode_graph_u32_for_size_estimate(graph: &HnswGraph) -> io::Result> { - let mut buf = Vec::new(); - write_u32_fixed(&mut buf, checked_u32_value(graph.len())?)?; - write_u32_fixed(&mut buf, checked_u32_value(graph.entry_point())?)?; - write_u32_fixed(&mut buf, checked_u32_value(graph.max_observed_level())?)?; - for &level in graph.levels() { - write_u32_fixed(&mut buf, checked_u32_value(level)?)?; - } - for node_levels in graph.neighbors() { - for level_neighbors in node_levels { - write_u32_fixed(&mut buf, checked_u32_value(level_neighbors.len())?)?; - for &neighbor in level_neighbors { - write_u32_fixed(&mut buf, checked_u32_value(neighbor)?)?; - } - } - } - Ok(buf) -} - -pub(crate) fn decode_graph( - bytes: &[u8], - vectors: Vec, - count: usize, - d: usize, - metric: MetricType, - hnsw_params: HnswBuildParams, -) -> io::Result> { - if bytes.is_empty() { - return Ok(None); - } - let mut pos = 0usize; - let graph_magic = read_u32_fixed(bytes, &mut pos)?; - if graph_magic != HNSW_GRAPH_MAGIC { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("Invalid HNSW graph magic: 0x{:08X}", graph_magic), - )); - } - let graph_version = read_u32_fixed(bytes, &mut pos)?; - if graph_version != HNSW_GRAPH_VERSION { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("Unsupported HNSW graph version: {}", graph_version), - )); - } - let graph_flags = read_u32_fixed(bytes, &mut pos)?; - let unknown_graph_flags = graph_flags & !HNSW_GRAPH_SUPPORTED_FLAGS; - if unknown_graph_flags != 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "Unsupported HNSW graph flags: 0x{:08X}", - unknown_graph_flags - ), - )); - } - if graph_flags & HNSW_GRAPH_REQUIRED_FLAGS != HNSW_GRAPH_REQUIRED_FLAGS { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "HNSW graph v1 requires delta-varint adjacency encoding", - )); - } - let graph_count = read_u32_varint(bytes, &mut pos)? as usize; - if graph_count != count { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "graph count {} does not match list count {}", - graph_count, count - ), - )); - } - let entry_point = read_u32_varint(bytes, &mut pos)? as usize; - let max_observed_level = read_u32_varint(bytes, &mut pos)? as usize; - let mut levels = Vec::with_capacity(count); - for node in 0..count { - let level = read_u32_varint(bytes, &mut pos)? as usize; - if level >= hnsw_params.max_level { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "graph node {} level {} exceeds max_level {}", - node, - level, - hnsw_params.max_level - 1 - ), - )); - } - levels.push(level); - } - let mut neighbors = Vec::with_capacity(count); - for (node, &level) in levels.iter().enumerate() { - let mut node_levels = Vec::with_capacity(level + 1); - for graph_level in 0..=level { - let degree = read_u32_varint(bytes, &mut pos)? as usize; - let max_degree = if graph_level == 0 { - hnsw_params.m.saturating_mul(2) - } else { - hnsw_params.m - }; - if degree > max_degree { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "graph node {} degree {} at level {} exceeds max degree {}", - node, degree, graph_level, max_degree - ), - )); - } - let mut level_neighbors = Vec::with_capacity(degree); - let mut previous = 0usize; - for _ in 0..degree { - let delta = read_u32_varint(bytes, &mut pos)? as usize; - let neighbor = previous.checked_add(delta).ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidData, "graph neighbor id overflow") - })?; - if neighbor > u32::MAX as usize { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("graph neighbor id {} is out of range", neighbor), - )); - } - level_neighbors.push(neighbor); - previous = neighbor; - } - node_levels.push(level_neighbors); - } - neighbors.push(node_levels); - } - if pos != bytes.len() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "trailing bytes in HNSW graph section", - )); - } - Ok(Some(HnswGraph::from_parts( - vectors, - count, - d, - metric, - levels, - neighbors, - entry_point, - max_observed_level, - hnsw_params, - )?)) -} - -fn checked_u32_value(value: usize) -> io::Result { - if value > u32::MAX as usize { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("value {} exceeds u32 limit", value), - )); - } - Ok(value as u32) -} - -fn write_u32_fixed(buf: &mut Vec, value: u32) -> io::Result<()> { - buf.extend_from_slice(&value.to_le_bytes()); - Ok(()) -} - -fn read_u32_fixed(bytes: &[u8], pos: &mut usize) -> io::Result { - let end = pos.checked_add(4).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "graph section position overflow", - ) - })?; - if end > bytes.len() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "truncated HNSW graph section", - )); - } - let value = u32::from_le_bytes(bytes[*pos..end].try_into().unwrap()); - *pos = end; - Ok(value) -} - -fn write_u32_varint(buf: &mut Vec, value: usize) -> io::Result<()> { - write_u64_varint(buf, checked_u32_value(value)? as u64); - Ok(()) -} - fn write_u64_varint(buf: &mut Vec, mut value: u64) { while value >= 0x80 { buf.push((value as u8 & 0x7f) | 0x80); @@ -353,32 +339,27 @@ fn write_u64_varint(buf: &mut Vec, mut value: u64) { buf.push(value as u8); } -fn read_u32_varint(bytes: &[u8], pos: &mut usize) -> io::Result { - let value = read_u64_varint(bytes, pos)?; - u32::try_from(value).map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("HNSW graph varint value {} exceeds u32 limit", value), - ) - }) -} - +#[inline] fn read_u64_varint(bytes: &[u8], pos: &mut usize) -> io::Result { - let mut value = 0u64; - let mut shift = 0u32; - for _ in 0..10 { - if *pos >= bytes.len() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "truncated HNSW graph varint", - )); - } - let byte = bytes[*pos]; + let first = bytes.get(*pos).copied().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "truncated delta-varint value") + })?; + *pos += 1; + if first < 0x80 { + return Ok(first as u64); + } + + let mut value = (first & 0x7f) as u64; + let mut shift = 7u32; + for _ in 1..10 { + let byte = bytes.get(*pos).copied().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "truncated delta-varint value") + })?; *pos += 1; if shift == 63 && (byte & 0x7e) != 0 { return Err(io::Error::new( io::ErrorKind::InvalidData, - "HNSW graph varint exceeds u64 limit", + "delta-varint value exceeds u64 limit", )); } value |= ((byte & 0x7f) as u64) << shift; @@ -389,7 +370,7 @@ fn read_u64_varint(bytes: &[u8], pos: &mut usize) -> io::Result { } Err(io::Error::new( io::ErrorKind::InvalidData, - "HNSW graph varint exceeds u64 limit", + "delta-varint value exceeds u64 limit", )) } @@ -410,45 +391,6 @@ pub(crate) fn write_f32_slice(out: &mut dyn SeekWrite, data: &[f32]) -> io::Resu out.write_all(&bytes) } -pub(crate) fn read_u32_le( - reader: &mut PreadCursor<'_, R>, -) -> io::Result { - let mut buf = [0u8; 4]; - reader.read_exact(&mut buf)?; - Ok(u32::from_le_bytes(buf)) -} - -pub(crate) fn read_i32_le( - reader: &mut PreadCursor<'_, R>, -) -> io::Result { - let mut buf = [0u8; 4]; - reader.read_exact(&mut buf)?; - Ok(i32::from_le_bytes(buf)) -} - -pub(crate) fn read_i64_le( - reader: &mut PreadCursor<'_, R>, -) -> io::Result { - let mut buf = [0u8; 8]; - reader.read_exact(&mut buf)?; - Ok(i64::from_le_bytes(buf)) -} - -pub(crate) fn read_f32_vec( - reader: &mut PreadCursor<'_, R>, - count: usize, -) -> io::Result> { - let byte_len = count.checked_mul(4).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "f32 section byte length overflow", - ) - })?; - let mut buf = vec![0u8; byte_len]; - reader.read_exact(&mut buf)?; - bytes_to_f32_vec(&buf) -} - pub(crate) fn bytes_to_f32_vec(bytes: &[u8]) -> io::Result> { if !bytes.len().is_multiple_of(4) { return Err(io::Error::new( @@ -544,3 +486,100 @@ pub(crate) fn decode_roaring_filter(bytes: &[u8]) -> io::Result ) }) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::io::SeekReadCapabilities; + + struct RecordingReader { + bytes: Vec, + max_ranges: usize, + calls: Vec, + } + + impl SeekRead for RecordingReader { + fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { + self.calls.push(ranges.len()); + for range in ranges { + let start = range.pos as usize; + let end = start + range.buf.len(); + range.buf.copy_from_slice(&self.bytes[start..end]); + } + Ok(()) + } + + fn read_capabilities(&self) -> SeekReadCapabilities { + SeekReadCapabilities { + max_ranges_per_pread: self.max_ranges, + ..SeekReadCapabilities::default() + } + } + } + + #[test] + fn batched_pread_honors_range_and_byte_limits_without_dropping_payloads() { + let mut reader = RecordingReader { + bytes: (0..32).collect(), + max_ranges: 2, + calls: Vec::new(), + }; + let offsets = [0, 3, 6, 9, 12]; + let mut payloads = vec![vec![0; 3]; offsets.len()]; + pread_batched_payloads_with_limit(&mut reader, &offsets, &mut payloads, 6).unwrap(); + assert_eq!(reader.calls, vec![2, 2, 1]); + assert_eq!(payloads[0], vec![0, 1, 2]); + assert_eq!(payloads[4], vec![12, 13, 14]); + + let mut byte_limited = RecordingReader { + bytes: (0..32).collect(), + max_ranges: 8, + calls: Vec::new(), + }; + let mut payloads = vec![vec![0; 4], vec![0; 4], vec![0; 4]]; + pread_batched_payloads_with_limit(&mut byte_limited, &[0, 4, 8], &mut payloads, 6).unwrap(); + assert_eq!(byte_limited.calls, vec![1, 1, 1]); + } + + #[test] + fn aggregate_ivf_batch_plan_bounds_allocated_payloads() { + assert_eq!(bounded_ivf_payload_batch_end(&[32, 32, 32], 2).unwrap(), 2); + assert_eq!( + bounded_ivf_payload_batch_end( + &[ + MAX_IVF_BATCH_READ_BYTES / 2, + MAX_IVF_BATCH_READ_BYTES / 2, + 1 + ], + 8, + ) + .unwrap(), + 2 + ); + assert_eq!( + bounded_ivf_payload_batch_end(&[MAX_IVF_BATCH_READ_BYTES + 1, 1], 8).unwrap(), + 1 + ); + assert_eq!(bounded_ivf_payload_batch_end(&[0, 0, 1], 1).unwrap(), 3); + } + + #[test] + fn ivf_stream_chunks_reserve_decoded_ids_and_block_alignment() { + let retained_ids = 8 * 1024 * 1024; + let rows = + bounded_ivf_stream_chunk_rows(usize::MAX, 1024 * size_of::(), retained_ids, 1) + .unwrap(); + assert!(rows * 1024 * size_of::() + retained_ids <= MAX_IVF_BATCH_READ_BYTES); + + let blocked = bounded_ivf_stream_chunk_rows(100_000, 1024, retained_ids, 32).unwrap(); + assert!(blocked.is_multiple_of(32)); + assert!(blocked * 1024 + retained_ids <= MAX_IVF_BATCH_READ_BYTES); + } + + #[test] + fn ivf_stream_rejects_id_state_that_leaves_no_payload_block() { + let error = + bounded_ivf_stream_chunk_rows(1, 4096, MAX_IVF_BATCH_READ_BYTES, 1).unwrap_err(); + assert!(error.to_string().contains("bounded streaming allocation")); + } +} diff --git a/core/src/io.rs b/core/src/io.rs index 60772de..f66eade 100644 --- a/core/src/io.rs +++ b/core/src/io.rs @@ -17,12 +17,16 @@ use crate::distance::MetricType; use crate::index_io_util::{ - decode_delta_varint_ids, encode_delta_varint_ids, validate_reserved_zero, + bounded_ivf_payload_batch_end, bounded_ivf_stream_chunk_rows, bytes_to_f32_vec, + decode_delta_varint_ids, encode_delta_varint_ids, pread_batched_slices, + read_delta_varint_ids_at, validate_reserved_zero, }; use crate::ivfpq::IVFPQIndex; use crate::opq::OPQMatrix; use crate::pq::ProductQuantizer; +use rayon::prelude::*; use std::io; +use std::mem::size_of; pub const MAGIC: u32 = 0x49565051; // "IVPQ" pub const VERSION: u32 = 1; @@ -46,6 +50,25 @@ impl<'a> ReadRequest<'a> { } } +/// Optional immutable-source hints used to refine DiskANN's automatic read +/// plan. Zero means unspecified for every field. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct SeekReadCapabilities { + /// Estimated end-to-end latency of one representative random read. + /// + /// Zero means unknown and lets DiskANN reuse the mandatory header read's + /// elapsed time while opening. + pub estimated_random_read_latency_nanos: u64, + /// Efficient coalesced window size for random reads. + pub preferred_window_bytes: usize, + /// Maximum ranges accepted by one `pread` invocation. + pub max_ranges_per_pread: usize, +} + +/// Positional access to one immutable byte sequence. +/// +/// Implementations must return the same bytes for the lifetime of an opened +/// index Reader. Replacing an index requires opening a new Reader. pub trait SeekRead: Send { /// Positional reads for one or more ranges. /// @@ -53,6 +76,23 @@ pub trait SeekRead: Send { /// them concurrently when the underlying source supports independent /// positional reads. fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()>; + + /// Creates an independent handle to the same immutable byte sequence when supported. + fn try_clone_reader(&self) -> io::Result> + where + Self: Sized, + { + Ok(None) + } + + /// Optionally refines the latency-derived range plan. + /// + /// This describes an immutable reader implementation, not a local-file + /// type. Object stores, remote caches, memory readers, and custom storage + /// adapters can all provide the same hints. + fn read_capabilities(&self) -> SeekReadCapabilities { + SeekReadCapabilities::default() + } } pub(crate) struct PreadCursor<'a, R: SeekRead + ?Sized> { @@ -65,10 +105,6 @@ impl<'a, R: SeekRead + ?Sized> PreadCursor<'a, R> { Self { reader, pos } } - pub(crate) fn seek(&mut self, pos: u64) { - self.pos = pos; - } - pub(crate) fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> { self.reader.pread(&mut [ReadRequest::new(self.pos, buf)])?; self.pos = self @@ -138,24 +174,6 @@ fn write_f32_slice(out: &mut dyn SeekWrite, data: &[f32]) -> io::Result<()> { out.write_all(&bytes) } -fn read_u32_le(reader: &mut PreadCursor<'_, R>) -> io::Result { - let mut buf = [0u8; 4]; - reader.read_exact(&mut buf)?; - Ok(u32::from_le_bytes(buf)) -} - -fn read_i32_le(reader: &mut PreadCursor<'_, R>) -> io::Result { - let mut buf = [0u8; 4]; - reader.read_exact(&mut buf)?; - Ok(i32::from_le_bytes(buf)) -} - -fn read_i64_le(reader: &mut PreadCursor<'_, R>) -> io::Result { - let mut buf = [0u8; 8]; - reader.read_exact(&mut buf)?; - Ok(i64::from_le_bytes(buf)) -} - fn validate_positive_i32(val: i32, field: &str) -> io::Result { if val <= 0 { return Err(io::Error::new( @@ -207,19 +225,6 @@ fn checked_list_bytes(count: usize, bytes_per_entry: usize) -> io::Result }) } -fn read_f32_vec( - reader: &mut PreadCursor<'_, R>, - count: usize, -) -> io::Result> { - let mut buf = vec![0u8; count * 4]; - reader.read_exact(&mut buf)?; - let floats: Vec = buf - .chunks_exact(4) - .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) - .collect(); - Ok(floats) -} - /// Write a complete IVF-PQ index with delta-varint ID encoding. pub fn write_index(index: &IVFPQIndex, out: &mut dyn SeekWrite) -> io::Result<()> { let d = index.d; @@ -228,6 +233,12 @@ pub fn write_index(index: &IVFPQIndex, out: &mut dyn SeekWrite) -> io::Result<() let ksub = index.pq.ksub; let dsub = index.pq.dsub; let code_size = index.pq.code_size(); + if ksub == 16 && !m.is_multiple_of(2) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("4-bit IVF-PQ requires even m, got {}", m), + )); + } let d_i32 = usize_to_i32(d, "dimension")?; let nlist_i32 = usize_to_i32(nlist, "nlist")?; let m_i32 = usize_to_i32(m, "pq m")?; @@ -253,28 +264,35 @@ pub fn write_index(index: &IVFPQIndex, out: &mut dyn SeekWrite) -> io::Result<() })?; // Sort IDs within each list and prepare delta-varint encoded data - let mut sorted_lists: Vec<(Vec, Vec, Vec)> = Vec::with_capacity(nlist); + let mut sorted_lists = Vec::with_capacity(nlist); for i in 0..nlist { let count = index.ids[i].len(); + let expected_code_bytes = checked_list_bytes(count, code_size)?; + if index.codes[i].len() != expected_code_bytes { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("IVFPQ code length mismatch at list {i}"), + )); + } if count == 0 { - sorted_lists.push((Vec::new(), Vec::new(), Vec::new())); + sorted_lists.push(SortedPqListMetadata::default()); continue; } - // Sort by ID, reorder codes accordingly + // The compact codes are reordered lazily, one list at a time, while + // writing. Keeping only the permutation avoids a second index-sized + // code allocation in the writer. let mut indices: Vec = (0..count).collect(); indices.sort_by_key(|&idx| index.ids[i][idx]); let sorted_ids: Vec = indices.iter().map(|&idx| index.ids[i][idx]).collect(); - let code_bytes = checked_list_bytes(count, code_size)?; - let mut sorted_codes = vec![0u8; code_bytes]; - for (new_idx, &old_idx) in indices.iter().enumerate() { - sorted_codes[new_idx * code_size..(new_idx + 1) * code_size] - .copy_from_slice(&index.codes[i][old_idx * code_size..(old_idx + 1) * code_size]); - } - + let base_id = sorted_ids[0]; let (_, id_bytes) = encode_delta_varint_ids(&sorted_ids); - sorted_lists.push((sorted_ids, id_bytes, sorted_codes)); + sorted_lists.push(SortedPqListMetadata { + base_id, + order: indices, + id_bytes, + }); } // Header @@ -322,11 +340,11 @@ pub fn write_index(index: &IVFPQIndex, out: &mut dyn SeekWrite) -> io::Result<() for i in 0..nlist { list_offsets[i] = u64_to_i64(current_offset, "list offset")?; - let count = sorted_lists[i].0.len(); + let count = sorted_lists[i].order.len(); list_counts[i] = usize_to_i32(count, "list count")?; if count > 0 { // base_id(8) + id_bytes_len(4) + id_bytes + codes - let id_bytes_len = sorted_lists[i].1.len(); + let id_bytes_len = sorted_lists[i].id_bytes.len(); list_id_bytes_lens[i] = usize_to_i32(id_bytes_len, "delta ID section")?; let code_bytes = checked_list_bytes(count, code_size)?; let list_bytes = 12usize @@ -351,44 +369,58 @@ pub fn write_index(index: &IVFPQIndex, out: &mut dyn SeekWrite) -> io::Result<() } // Write inverted list data + let mut transposed = Vec::new(); for i in 0..nlist { - let (ref sorted_ids, ref id_bytes, ref sorted_codes) = sorted_lists[i]; - if sorted_ids.is_empty() { + let list = &sorted_lists[i]; + if list.order.is_empty() { continue; } // base_id - write_i64_le(out, sorted_ids[0])?; + write_i64_le(out, list.base_id)?; // id_bytes_len + id_bytes - write_i32_le(out, usize_to_i32(id_bytes.len(), "delta ID section")?)?; - out.write_all(id_bytes)?; - // PQ codes — transpose for cache-friendly SIMD scan - let count = sorted_ids.len(); - if code_size == m { - // 8-bit: transpose from [n][M] to [M][n] - let transposed_len = checked_list_bytes(count, m)?; - let mut transposed = vec![0u8; transposed_len]; - for vec_idx in 0..count { - for sub in 0..m { - transposed[sub * count + vec_idx] = sorted_codes[vec_idx * m + sub]; - } - } - out.write_all(&transposed)?; - } else { - // 4-bit: transpose from [n][M/2] to [M/2][n] - // Each byte at position `pair` in a vector goes to column `pair` - let cs = code_size; - let transposed_len = checked_list_bytes(count, cs)?; - let mut transposed = vec![0u8; transposed_len]; - for vec_idx in 0..count { - for pair in 0..cs { - transposed[pair * count + vec_idx] = sorted_codes[vec_idx * cs + pair]; + write_i32_le(out, usize_to_i32(list.id_bytes.len(), "delta ID section")?)?; + out.write_all(&list.id_bytes)?; + // PQ codes — transpose for cache-friendly SIMD scan. Blocking keeps + // both the row-major source and column-major destination hot while the + // writer applies the row-ID sort permutation. + transpose_sorted_pq_codes(&index.codes[i], &list.order, code_size, &mut transposed); + out.write_all(&transposed)?; + } + + Ok(()) +} + +fn transpose_sorted_pq_codes( + codes: &[u8], + order: &[usize], + code_size: usize, + transposed: &mut Vec, +) { + const TILE: usize = 32; + + let count = order.len(); + transposed.resize(count * code_size, 0); + for row_start in (0..count).step_by(TILE) { + let row_end = (row_start + TILE).min(count); + for column_start in (0..code_size).step_by(TILE) { + let column_end = (column_start + TILE).min(code_size); + for (target_row, &source_row) in order[row_start..row_end].iter().enumerate() { + let target_row = row_start + target_row; + let source = &codes + [source_row * code_size + column_start..source_row * code_size + column_end]; + for (column_offset, &code) in source.iter().enumerate() { + transposed[(column_start + column_offset) * count + target_row] = code; } } - out.write_all(&transposed)?; } } +} - Ok(()) +#[derive(Default)] +struct SortedPqListMetadata { + base_id: i64, + order: Vec, + id_bytes: Vec, } fn usize_to_i32(value: usize, field: &str) -> io::Result { @@ -443,8 +475,6 @@ pub struct IVFPQIndexReader { pub transposed_codes: bool, /// Whether heavy data (centroids, codebooks, offset table) has been loaded loaded: bool, - /// File offset where centroids section starts (for lazy loading) - centroids_offset: u64, /// Whether file has OPQ rotation matrix has_opq: bool, } @@ -453,9 +483,20 @@ impl IVFPQIndexReader { /// Open an index file. Only reads the 64-byte header. /// Centroids, codebooks, and offset table are loaded lazily on first search. pub fn open(mut reader: R) -> io::Result { - let mut cursor = PreadCursor::new(&mut reader, 0); + let mut header = [0u8; HEADER_SIZE]; + reader.pread(&mut [ReadRequest::new(0, &mut header)])?; + Self::open_with_header(reader, header) + } + + pub(crate) fn open_with_header(reader: R, header: [u8; HEADER_SIZE]) -> io::Result { + let read_u32 = + |offset: usize| u32::from_le_bytes(header[offset..offset + 4].try_into().unwrap()); + let read_i32 = + |offset: usize| i32::from_le_bytes(header[offset..offset + 4].try_into().unwrap()); + let read_i64 = + |offset: usize| i64::from_le_bytes(header[offset..offset + 8].try_into().unwrap()); - let magic = read_u32_le(&mut cursor)?; + let magic = read_u32(0); if magic != MAGIC { return Err(io::Error::new( io::ErrorKind::InvalidData, @@ -463,7 +504,7 @@ impl IVFPQIndexReader { )); } - let version = read_u32_le(&mut cursor)?; + let version = read_u32(4); if version != VERSION { return Err(io::Error::new( io::ErrorKind::InvalidData, @@ -471,11 +512,11 @@ impl IVFPQIndexReader { )); } - let d = validate_positive_i32(read_i32_le(&mut cursor)?, "d")? as usize; - let nlist = validate_positive_i32(read_i32_le(&mut cursor)?, "nlist")? as usize; - let m = validate_positive_i32(read_i32_le(&mut cursor)?, "m")? as usize; - let ksub = validate_positive_i32(read_i32_le(&mut cursor)?, "ksub")? as usize; - let dsub = validate_positive_i32(read_i32_le(&mut cursor)?, "dsub")? as usize; + let d = validate_positive_i32(read_i32(8), "d")? as usize; + let nlist = validate_positive_i32(read_i32(12), "nlist")? as usize; + let m = validate_positive_i32(read_i32(16), "m")? as usize; + let ksub = validate_positive_i32(read_i32(20), "ksub")? as usize; + let dsub = validate_positive_i32(read_i32(24), "dsub")? as usize; if ksub != 16 && ksub != 256 { return Err(io::Error::new( @@ -502,19 +543,23 @@ impl IVFPQIndexReader { )); } - let metric_code = read_u32_le(&mut cursor)?; + let metric_code = read_u32(28); let metric = MetricType::from_code(metric_code).ok_or_else(|| { io::Error::new( io::ErrorKind::InvalidData, format!("Unknown metric type: {}", metric_code), ) })?; - let total_vectors = read_i64_le(&mut cursor)?; + let total_vectors = read_i64(32); + if total_vectors < 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "IVFPQ total vector count must be non-negative", + )); + } - let flags = read_u32_le(&mut cursor)?; - let mut reserved = [0u8; 20]; - cursor.read_exact(&mut reserved)?; - validate_reserved_zero(&reserved, "IVFPQ")?; + let flags = read_u32(40); + validate_reserved_zero(&header[44..64], "IVFPQ")?; let unknown_flags = flags & !SUPPORTED_FLAGS; if unknown_flags != 0 { return Err(io::Error::new( @@ -531,12 +576,6 @@ impl IVFPQIndexReader { let by_residual = flags & FLAG_BY_RESIDUAL != 0; let transposed_codes = flags & FLAG_TRANSPOSED_CODES != 0; let has_opq = flags & FLAG_HAS_OPQ != 0; - let centroids_offset = if has_opq { - let opq_elements = checked_section_size(d, d)?; - HEADER_SIZE as u64 + (opq_elements * 4) as u64 - } else { - HEADER_SIZE as u64 - }; Ok(IVFPQIndexReader { reader, @@ -556,6 +595,7 @@ impl IVFPQIndexReader { nbits: ksub.trailing_zeros() as usize, dsub, ksub, + chunk_offsets: (0..=m).map(|chunk| chunk * dsub).collect(), centroids: Vec::new(), centroid_norms_cache: Vec::new(), }, @@ -565,7 +605,6 @@ impl IVFPQIndexReader { precomputed_table: Vec::new(), transposed_codes, loaded: false, - centroids_offset, has_opq, }) } @@ -590,11 +629,43 @@ impl IVFPQIndexReader { .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "m*ksub overflow"))?; let pq_centroids_count = checked_section_size(mk, dsub)?; - // Seek to start of data sections - let mut cursor = PreadCursor::new(&mut self.reader, self.centroids_offset); + let rotation_bytes = if self.has_opq { + rotation_count.checked_mul(4).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "OPQ byte length overflow") + })? + } else { + 0 + }; + let centroid_bytes = centroids_count.checked_mul(4).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "centroid byte length overflow") + })?; + let pq_centroid_bytes = pq_centroids_count.checked_mul(4).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "PQ centroid byte length overflow", + ) + })?; + let offset_table_bytes = nlist.checked_mul(16).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVFPQ offset table byte length overflow", + ) + })?; + let metadata_bytes = rotation_bytes + .checked_add(centroid_bytes) + .and_then(|size| size.checked_add(pq_centroid_bytes)) + .and_then(|size| size.checked_add(offset_table_bytes)) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVFPQ metadata size overflow") + })?; + let mut metadata = vec![0u8; metadata_bytes]; + self.reader + .pread(&mut [ReadRequest::new(HEADER_SIZE as u64, &mut metadata)])?; + let mut position = 0usize; + if self.has_opq { - cursor.seek(HEADER_SIZE as u64); - let rotation = read_f32_vec(&mut cursor, rotation_count)?; + let rotation = bytes_to_f32_vec(&metadata[position..position + rotation_bytes])?; + position += rotation_bytes; self.opq = Some(OPQMatrix { d, m, @@ -607,15 +678,19 @@ impl IVFPQIndexReader { }); } - self.quantizer_centroids = read_f32_vec(&mut cursor, centroids_count)?; + self.quantizer_centroids = + bytes_to_f32_vec(&metadata[position..position + centroid_bytes])?; + position += centroid_bytes; - let pq_centroids = read_f32_vec(&mut cursor, pq_centroids_count)?; + let pq_centroids = bytes_to_f32_vec(&metadata[position..position + pq_centroid_bytes])?; + position += pq_centroid_bytes; self.pq = ProductQuantizer { d, m, nbits: ksub.trailing_zeros() as usize, dsub, ksub, + chunk_offsets: (0..=m).map(|chunk| chunk * dsub).collect(), centroids: pq_centroids, centroid_norms_cache: Vec::new(), }; @@ -624,9 +699,11 @@ impl IVFPQIndexReader { self.list_offsets = vec![0i64; nlist]; self.list_counts = vec![0i32; nlist]; self.list_id_bytes_lens = vec![0i32; nlist]; - for i in 0..nlist { - self.list_offsets[i] = read_i64_le(&mut cursor)?; - let count = read_i32_le(&mut cursor)?; + let offset_table = &metadata[position..]; + let mut actual_total = 0i64; + for (i, entry) in offset_table.chunks_exact(16).enumerate() { + self.list_offsets[i] = i64::from_le_bytes(entry[0..8].try_into().unwrap()); + let count = i32::from_le_bytes(entry[8..12].try_into().unwrap()); if count < 0 { return Err(io::Error::new( io::ErrorKind::InvalidData, @@ -634,7 +711,10 @@ impl IVFPQIndexReader { )); } self.list_counts[i] = count; - let id_bytes_len = read_i32_le(&mut cursor)?; + actual_total = actual_total.checked_add(count as i64).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVFPQ vector count overflow") + })?; + let id_bytes_len = i32::from_le_bytes(entry[12..16].try_into().unwrap()); if id_bytes_len < 0 { return Err(io::Error::new( io::ErrorKind::InvalidData, @@ -643,6 +723,15 @@ impl IVFPQIndexReader { } self.list_id_bytes_lens[i] = id_bytes_len; } + if actual_total != self.total_vectors { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "IVFPQ header vector count {} does not match list total {actual_total}", + self.total_vectors + ), + )); + } self.loaded = true; Ok(()) @@ -660,51 +749,32 @@ impl IVFPQIndexReader { /// Read an inverted list's IDs and PQ codes. /// Calls ensure_loaded() if not yet loaded. pub fn read_inverted_list(&mut self, list_id: usize) -> io::Result<(Vec, Vec)> { - self.ensure_loaded()?; - if list_id >= self.nlist { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("list_id {} out of range (nlist={})", list_id, self.nlist), - )); - } - let count = self.list_counts[list_id] as usize; - if count == 0 { - return Ok((Vec::new(), Vec::new())); - } - - let offset = checked_list_offset(self.list_offsets[list_id], list_id)?; - let code_size = self.pq.code_size(); - let code_bytes = checked_list_bytes(count, code_size)?; - - let id_bytes_len = self.list_id_bytes_lens[list_id]; - if id_bytes_len == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("missing id_bytes_len for non-empty IVFPQ list {}", list_id), - )); - } - let payload_len = 12usize - .checked_add(id_bytes_len as usize) - .and_then(|len| len.checked_add(code_bytes)) - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "inverted list payload size overflow", - ) - })?; - let mut payload = vec![0u8; payload_len]; - self.reader - .pread(&mut [ReadRequest::new(offset, &mut payload)])?; - decode_delta_list_payload(&payload, count, id_bytes_len) + let mut lists = self.read_inverted_list_payloads(&[list_id])?; + let list = lists.pop().expect("one requested list has one result"); + let codes = list.codes().to_vec(); + Ok((list.ids, codes)) } /// Read multiple inverted lists. Lists whose payload length is known from /// metadata are issued through a single batched pread call. pub fn read_inverted_lists(&mut self, list_ids: &[usize]) -> io::Result> { + Ok(self + .read_inverted_list_payloads(list_ids)? + .into_iter() + .map(InvertedListPayload::into_public) + .collect()) + } + + /// Internal zero-copy form used by search. It retains the compact ID + /// prefix and exposes the PQ-code suffix from the original read buffer. + pub(crate) fn read_inverted_list_payloads( + &mut self, + list_ids: &[usize], + ) -> io::Result> { self.ensure_loaded()?; let code_size = self.pq.code_size(); - let mut results: Vec> = + let mut results: Vec> = (0..list_ids.len()).map(|_| None).collect(); let mut metas = Vec::new(); let mut payloads = Vec::new(); @@ -718,11 +788,7 @@ impl IVFPQIndexReader { } let count = self.list_counts[list_id] as usize; if count == 0 { - results[input_idx] = Some(InvertedListData { - list_id, - ids: Vec::new(), - codes: Vec::new(), - }); + results[input_idx] = Some(InvertedListPayload::empty(list_id)); continue; } @@ -752,27 +818,28 @@ impl IVFPQIndexReader { offset, id_bytes_len, }); - payloads.push(vec![0u8; payload_len]); + payloads.push(AlignedCodePayload::new( + payload_len, + 12 + id_bytes_len as usize, + )?); } if !metas.is_empty() { - { - let mut requests: Vec<_> = payloads - .iter_mut() - .zip(metas.iter()) - .map(|(payload, meta)| ReadRequest::new(meta.offset, payload.as_mut_slice())) - .collect(); - self.reader.pread(&mut requests)?; - } + let offsets = metas.iter().map(|meta| meta.offset).collect::>(); + let mut buffers = payloads + .iter_mut() + .map(AlignedCodePayload::read_buf_mut) + .collect::>(); + pread_batched_slices(&mut self.reader, &offsets, &mut buffers)?; + drop(buffers); for (meta, payload) in metas.into_iter().zip(payloads) { - let (ids, codes) = - decode_delta_list_payload(&payload, meta.count, meta.id_bytes_len)?; - results[meta.input_idx] = Some(InvertedListData { - list_id: meta.list_id, - ids, - codes, - }); + results[meta.input_idx] = Some(decode_delta_list_payload( + meta.list_id, + payload, + meta.count, + meta.id_bytes_len, + )?); } } @@ -789,6 +856,120 @@ impl IVFPQIndexReader { .collect() } + pub(crate) fn batch_read_end(&self, list_ids: &[usize]) -> io::Result { + let payload_lengths = list_ids + .iter() + .map(|&list_id| self.list_payload_len(list_id)) + .collect::>>()?; + bounded_ivf_payload_batch_end( + &payload_lengths, + self.reader.read_capabilities().max_ranges_per_pread, + ) + } + + pub(crate) fn list_payload_len(&self, list_id: usize) -> io::Result { + if list_id >= self.nlist { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("list_id {} out of range (nlist={})", list_id, self.nlist), + )); + } + let count = self.list_counts[list_id] as usize; + if count == 0 { + return Ok(0); + } + let code_bytes = checked_list_bytes(count, self.pq.code_size())?; + 12usize + .checked_add(self.list_id_bytes_lens[list_id] as usize) + .and_then(|len| len.checked_add(code_bytes)) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "inverted list payload size overflow", + ) + }) + } + + pub(crate) fn for_each_streamed_list_chunk( + &mut self, + list_id: usize, + mut consume: impl FnMut(&[i64], &[u8]), + ) -> io::Result<()> { + self.ensure_loaded()?; + let count = self.list_counts[list_id] as usize; + let list_offset = checked_list_offset(self.list_offsets[list_id], list_id)?; + let id_bytes_len = self.list_id_bytes_lens[list_id] as usize; + let ids = + read_delta_varint_ids_at(&mut self.reader, list_offset, count, id_bytes_len, "IVFPQ")?; + let code_size = self.pq.code_size(); + let code_offset = list_offset + .checked_add((12usize + id_bytes_len) as u64) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVFPQ code offset overflow") + })?; + let retained_id_bytes = ids.len().checked_mul(size_of::()).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVFPQ decoded ID size overflow") + })?; + let mut row_start = 0usize; + while row_start < count { + let chunk_rows = + bounded_ivf_stream_chunk_rows(count - row_start, code_size, retained_id_bytes, 1)?; + let code_bytes = chunk_rows.checked_mul(code_size).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVFPQ chunk size overflow") + })?; + let mut payload = AlignedCodePayload::new(code_bytes, 0)?; + if self.transposed_codes { + let offsets = (0..code_size) + .map(|column| { + code_offset + .checked_add( + column + .checked_mul(count) + .and_then(|value| value.checked_add(row_start)) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVFPQ transposed chunk offset overflow", + ) + })? as u64, + ) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVFPQ transposed chunk offset overflow", + ) + }) + }) + .collect::>>()?; + let mut buffers = payload + .codes_mut() + .chunks_exact_mut(chunk_rows) + .collect::>(); + pread_batched_slices(&mut self.reader, &offsets, &mut buffers)?; + } else { + let chunk_offset = code_offset + .checked_add(row_start.checked_mul(code_size).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVFPQ row-major chunk offset overflow", + ) + })? as u64) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVFPQ row-major chunk offset overflow", + ) + })?; + self.reader + .pread(&mut [ReadRequest::new(chunk_offset, payload.codes_mut())])?; + } + let row_end = row_start + chunk_rows; + consume(&ids[row_start..row_end], payload.codes()); + row_start = row_end; + } + Ok(()) + } + pub fn search( &mut self, query: &[f32], @@ -823,6 +1004,113 @@ pub struct InvertedListData { pub codes: Vec, } +struct AlignedCodePayload { + storage: Vec, + read_start: usize, + payload_len: usize, + code_start: usize, +} + +impl AlignedCodePayload { + const ALIGNMENT: usize = std::mem::align_of::(); + + fn new(payload_len: usize, code_start: usize) -> io::Result { + if code_start > payload_len { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "IVFPQ code suffix exceeds list payload", + )); + } + let read_start = (Self::ALIGNMENT - code_start % Self::ALIGNMENT) % Self::ALIGNMENT; + let storage_bytes = read_start.checked_add(payload_len).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVFPQ aligned payload overflow") + })?; + Ok(Self { + storage: vec![0; storage_bytes.div_ceil(Self::ALIGNMENT)], + read_start, + payload_len, + code_start, + }) + } + + fn empty() -> Self { + Self { + storage: Vec::new(), + read_start: 0, + payload_len: 0, + code_start: 0, + } + } + + fn storage_bytes_mut(&mut self) -> &mut [u8] { + let byte_len = self.storage.len() * size_of::(); + // SAFETY: `storage` is initialized and exclusively borrowed for the + // returned byte view. + unsafe { std::slice::from_raw_parts_mut(self.storage.as_mut_ptr().cast::(), byte_len) } + } + + fn storage_bytes(&self) -> &[u8] { + let byte_len = self.storage.len() * size_of::(); + // SAFETY: `storage` remains alive and immutably borrowed. + unsafe { std::slice::from_raw_parts(self.storage.as_ptr().cast::(), byte_len) } + } + + fn read_buf_mut(&mut self) -> &mut [u8] { + let start = self.read_start; + let end = start + self.payload_len; + &mut self.storage_bytes_mut()[start..end] + } + + fn read_bytes(&self) -> &[u8] { + &self.storage_bytes()[self.read_start..self.read_start + self.payload_len] + } + + fn codes(&self) -> &[u8] { + let codes = &self.read_bytes()[self.code_start..]; + debug_assert_eq!( + codes.as_ptr().align_offset(Self::ALIGNMENT), + 0, + "IVFPQ search codes must retain SIMD-friendly alignment" + ); + codes + } + + fn codes_mut(&mut self) -> &mut [u8] { + let code_start = self.read_start + self.code_start; + let code_end = self.read_start + self.payload_len; + &mut self.storage_bytes_mut()[code_start..code_end] + } +} + +pub(crate) struct InvertedListPayload { + pub(crate) list_id: usize, + pub(crate) ids: Vec, + payload: AlignedCodePayload, +} + +impl InvertedListPayload { + fn empty(list_id: usize) -> Self { + Self { + list_id, + ids: Vec::new(), + payload: AlignedCodePayload::empty(), + } + } + + pub(crate) fn codes(&self) -> &[u8] { + self.payload.codes() + } + + fn into_public(self) -> InvertedListData { + let codes = self.codes().to_vec(); + InvertedListData { + list_id: self.list_id, + ids: self.ids, + codes, + } + } +} + #[derive(Clone, Copy)] struct BatchedListRead { input_idx: usize, @@ -833,10 +1121,11 @@ struct BatchedListRead { } fn decode_delta_list_payload( - payload: &[u8], + list_id: usize, + payload: AlignedCodePayload, count: usize, id_bytes_len_from_table: i32, -) -> io::Result<(Vec, Vec)> { +) -> io::Result { let id_bytes_len = id_bytes_len_from_table as usize; let header_len = 12usize.checked_add(id_bytes_len).ok_or_else(|| { io::Error::new( @@ -844,14 +1133,15 @@ fn decode_delta_list_payload( "inverted list payload size overflow", ) })?; - if payload.len() < header_len { + let bytes = payload.read_bytes(); + if bytes.len() < header_len { return Err(io::Error::new( io::ErrorKind::UnexpectedEof, "truncated delta inverted list payload", )); } - let base_id = i64::from_le_bytes(payload[0..8].try_into().unwrap()); - let encoded_id_bytes_len = i32::from_le_bytes(payload[8..12].try_into().unwrap()); + let base_id = i64::from_le_bytes(bytes[0..8].try_into().unwrap()); + let encoded_id_bytes_len = i32::from_le_bytes(bytes[8..12].try_into().unwrap()); if encoded_id_bytes_len != id_bytes_len_from_table { return Err(io::Error::new( io::ErrorKind::InvalidData, @@ -861,10 +1151,13 @@ fn decode_delta_list_payload( ), )); } - let id_bytes = &payload[12..header_len]; + let id_bytes = &bytes[12..header_len]; let ids = decode_delta_varint_ids(base_id, id_bytes, count)?; - let codes = payload[header_len..].to_vec(); - Ok((ids, codes)) + Ok(InvertedListPayload { + list_id, + ids, + payload, + }) } #[allow(dead_code)] @@ -881,25 +1174,26 @@ fn compute_precomputed_table( let mut table = vec![0.0f32; table_size]; let pq_norms = pq.compute_centroid_norms(); - - for i in 0..nlist { - let centroid = ¢roids[i * d..(i + 1) * d]; - let tab_base = i * m * ksub; - - for sub in 0..m { - let sub_centroid = ¢roid[sub * dsub..(sub + 1) * dsub]; - let pq_base = sub * ksub * dsub; - - for j in 0..ksub { - let pq_off = pq_base + j * dsub; - let mut ip = 0.0f32; - for dd in 0..dsub { - ip += sub_centroid[dd] * pq.centroids[pq_off + dd]; + table + .par_chunks_mut(m * ksub) + .enumerate() + .for_each(|(i, list_table)| { + let centroid = ¢roids[i * d..(i + 1) * d]; + + for sub in 0..m { + let sub_centroid = ¢roid[sub * dsub..(sub + 1) * dsub]; + let pq_base = sub * ksub * dsub; + + for j in 0..ksub { + let pq_off = pq_base + j * dsub; + let mut ip = 0.0f32; + for dd in 0..dsub { + ip += sub_centroid[dd] * pq.centroids[pq_off + dd]; + } + list_table[sub * ksub + j] = pq_norms[sub * ksub + j] + 2.0 * ip; } - table[tab_base + sub * ksub + j] = pq_norms[sub * ksub + j] + 2.0 * ip; } - } - } + }); table } @@ -944,6 +1238,36 @@ mod tests { } } + #[test] + fn blocked_pq_transpose_matches_reference_across_tile_edges() { + for count in [0usize, 1, 31, 32, 33, 67] { + for code_size in [1usize, 25, 32, 240] { + let codes = (0..count * code_size) + .map(|offset| ((offset * 131 + 17) % 251) as u8) + .collect::>(); + let mut order = (0..count).collect::>(); + if count > 1 { + order.rotate_left(count / 3); + order.reverse(); + } + + let mut reference = Vec::with_capacity(count * code_size); + for column in 0..code_size { + for &source_row in &order { + reference.push(codes[source_row * code_size + column]); + } + } + + let mut transposed = Vec::new(); + transpose_sorted_pq_codes(&codes, &order, code_size, &mut transposed); + assert_eq!( + transposed, reference, + "count={count}, code_size={code_size}" + ); + } + } + } + #[test] fn test_varint_roundtrip() { let ids = [0, 127, 128, 16_383, 1_000_000]; @@ -1005,7 +1329,37 @@ mod tests { } #[test] - fn test_read_inverted_list_uses_pread_after_metadata_loaded() { + fn ivfpq_streamed_list_reader_matches_full_transposed_payload() { + let d = 8; + let m = 2; + let n = 300; + let mut index = IVFPQIndex::new(d, 1, m, MetricType::L2, false); + let mut rng = rand::rngs::StdRng::seed_from_u64(43); + let data = (0..n * d).map(|_| rng.gen::()).collect::>(); + let ids = (0..n as i64).collect::>(); + index.train(&data, n); + index.add(&data, &ids, n); + let mut bytes = Vec::new(); + write_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + + let mut full_reader = IVFPQIndexReader::open(Cursor::new(bytes.clone())).unwrap(); + let expected = full_reader.read_inverted_list(0).unwrap(); + let mut streamed_reader = IVFPQIndexReader::open(Cursor::new(bytes)).unwrap(); + streamed_reader.ensure_loaded().unwrap(); + let mut actual_ids = Vec::new(); + let mut actual_codes = Vec::new(); + streamed_reader + .for_each_streamed_list_chunk(0, |ids, codes| { + actual_ids.extend_from_slice(ids); + actual_codes.extend_from_slice(codes); + }) + .unwrap(); + assert_eq!(actual_ids, expected.0); + assert_eq!(actual_codes, expected.1); + } + + #[test] + fn test_ivfpq_search_payload_uses_one_pread_and_aligned_codes() { let d = 8; let nlist = 2; let m = 2; @@ -1042,10 +1396,20 @@ mod tests { reader.list_id_bytes_lens[non_empty_list] > 0, "v1 files must store id_bytes_len in the offset table" ); - let (read_ids, codes) = reader.read_inverted_list(non_empty_list).unwrap(); + let mut lists = reader + .read_inverted_list_payloads(&[non_empty_list]) + .unwrap(); + let list = lists.pop().unwrap(); + let read_ids = &list.ids; + let codes = list.codes(); assert!(!read_ids.is_empty()); assert!(!codes.is_empty()); + assert_eq!( + codes.as_ptr().align_offset(AlignedCodePayload::ALIGNMENT), + 0, + "the transposed scan should not lose code alignment after the ID prefix" + ); let stats = stats.lock().unwrap(); assert_eq!( @@ -1054,6 +1418,32 @@ mod tests { ); } + #[test] + fn test_ivfpq_open_coalesces_resident_metadata() { + let d = 8; + let nlist = 16; + let m = 2; + let n = 512; + let mut index = IVFPQIndex::new(d, nlist, m, MetricType::L2, false); + let mut rng = rand::rngs::StdRng::seed_from_u64(91); + let data = (0..n * d).map(|_| rng.gen::()).collect::>(); + let ids = (0..n as i64).collect::>(); + index.train(&data, n); + index.add(&data, &ids, n); + + let mut bytes = Vec::new(); + write_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let stats = Arc::new(Mutex::new(ReadStats::default())); + let source = CountingPreadCursor::new(bytes, Arc::clone(&stats)); + let mut reader = IVFPQIndexReader::open(source).unwrap(); + reader.ensure_loaded().unwrap(); + assert_eq!( + stats.lock().unwrap().pread_calls, + 2, + "direct IVF-PQ open should use one header read and one resident-metadata read" + ); + } + #[test] fn test_default_pread_handles_multiple_ranges() { let mut cursor = Cursor::new(vec![0, 1, 2, 3, 4, 5, 6, 7]); @@ -1104,6 +1494,12 @@ mod tests { } } + #[test] + #[should_panic(expected = "4-bit IVF-PQ requires even m")] + fn construction_rejects_odd_4bit_subquantizer_count() { + let _ = IVFPQIndex::with_nbits(15, 4, 5, 4, MetricType::L2, false); + } + #[test] #[ignore] fn test_space_savings() { diff --git a/core/src/ivfflat_io.rs b/core/src/ivfflat_io.rs index 56d860f..1409caa 100644 --- a/core/src/ivfflat_io.rs +++ b/core/src/ivfflat_io.rs @@ -15,14 +15,21 @@ // specific language governing permissions and limitations // under the License. -use crate::distance::{fvec_distance, fvec_normalize, MetricType}; -use crate::index_io_util::validate_reserved_zero; -use crate::io::{PreadCursor, ReadRequest, SeekRead, SeekWrite}; +use crate::distance::{ + fvec_l2sqr, fvec_l2sqr_scaled_exceeds, fvec_normalize, MetricType, QueryDistance, +}; +use crate::index_io_util::{ + bounded_ivf_payload_batch_end, bounded_ivf_stream_chunk_rows, ivf_payload_is_oversized, + pread_batched_slices, read_delta_varint_ids_at, validate_reserved_zero, +}; +use crate::io::{ReadRequest, SeekRead, SeekWrite}; use crate::ivfflat::IVFFlatIndex; use crate::ivfpq::RowIdFilter; use crate::kmeans; +use rayon::prelude::*; use roaring::RoaringTreemap; use std::io; +use std::mem::{align_of, size_of}; pub const IVFFLAT_MAGIC: u32 = 0x4956464C; // "IVFL" pub const IVFFLAT_VERSION: u32 = 1; @@ -31,6 +38,9 @@ pub const IVFFLAT_HEADER_SIZE: usize = 64; const FLAG_DELTA_IDS: u32 = 1 << 0; const REQUIRED_FLAGS: u32 = FLAG_DELTA_IDS; const SUPPORTED_FLAGS: u32 = REQUIRED_FLAGS; +// Raw-vector scan cost scales with both rows and dimension. Below this amount, +// Rayon scheduling and list-local heap merging outweigh the saved CPU time. +const PARALLEL_FLAT_SCAN_MIN_COMPONENTS: usize = 1024 * 1024; pub fn write_ivfflat_index(index: &IVFFlatIndex, out: &mut dyn SeekWrite) -> io::Result<()> { let d = index.d; @@ -48,11 +58,13 @@ pub fn write_ivfflat_index(index: &IVFFlatIndex, out: &mut dyn SeekWrite) -> io: }) })?; - let mut sorted_lists: Vec<(Vec, Vec, Vec)> = Vec::with_capacity(nlist); + // Keep only sort permutations and encoded IDs resident. Materializing every + // sorted raw-vector list at once duplicates the largest part of IVF-FLAT. + let mut sorted_lists: Vec<(Vec, Vec)> = Vec::with_capacity(nlist); for list_id in 0..nlist { let count = index.ids[list_id].len(); if count == 0 { - sorted_lists.push((Vec::new(), Vec::new(), Vec::new())); + sorted_lists.push((Vec::new(), Vec::new())); continue; } @@ -60,12 +72,8 @@ pub fn write_ivfflat_index(index: &IVFFlatIndex, out: &mut dyn SeekWrite) -> io: order.sort_by_key(|&idx| index.ids[list_id][idx]); let sorted_ids: Vec = order.iter().map(|&idx| index.ids[list_id][idx]).collect(); - let mut sorted_vectors = Vec::with_capacity(count * d); - for idx in order { - sorted_vectors.extend_from_slice(&index.vectors[list_id][idx * d..(idx + 1) * d]); - } let (_, id_bytes) = encode_delta_varint_ids(&sorted_ids); - sorted_lists.push((sorted_ids, id_bytes, sorted_vectors)); + sorted_lists.push((order, id_bytes)); } write_u32_le(out, IVFFLAT_MAGIC)?; @@ -135,13 +143,17 @@ pub fn write_ivfflat_index(index: &IVFFlatIndex, out: &mut dyn SeekWrite) -> io: write_i32_le(out, list_id_bytes_lens[list_id])?; } - for (sorted_ids, id_bytes, sorted_vectors) in sorted_lists { - if sorted_ids.is_empty() { + for (list_id, (order, id_bytes)) in sorted_lists.into_iter().enumerate() { + if order.is_empty() { continue; } - write_i64_le(out, sorted_ids[0])?; + write_i64_le(out, index.ids[list_id][order[0]])?; write_i32_le(out, id_bytes.len() as i32)?; out.write_all(&id_bytes)?; + let mut sorted_vectors = Vec::with_capacity(order.len() * d); + for idx in order { + sorted_vectors.extend_from_slice(&index.vectors[list_id][idx * d..(idx + 1) * d]); + } write_f32_slice(out, &sorted_vectors)?; } @@ -162,18 +174,156 @@ pub struct IVFFlatIndexReader { loaded: bool, } +struct FlatListData { + list_id: usize, + ids: Vec, + payload: AlignedFlatPayload, +} + +impl FlatListData { + fn vectors(&self) -> &[f32] { + self.payload.vectors() + } +} + +/// Owns one v1 list payload while exposing its raw-vector suffix as aligned +/// native `f32` values on little-endian hosts. +/// +/// Delta-varint IDs make the vector suffix arbitrarily aligned in the file. +/// Prefixing the read target by at most three bytes lets search scan the +/// original I/O allocation directly instead of parsing and copying every raw +/// vector into a second allocation. +struct AlignedFlatPayload { + storage: Vec, + read_start: usize, + payload_len: usize, + vector_start: usize, + vector_len: usize, + decoded_vectors: Option>, +} + +impl AlignedFlatPayload { + fn new(payload_len: usize, vector_start: usize, vector_len: usize) -> io::Result { + let vector_end = vector_start.checked_add(vector_len).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVF-FLAT vector suffix overflow", + ) + })?; + if vector_end != payload_len || !vector_len.is_multiple_of(size_of::()) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "IVF-FLAT vector suffix has an invalid shape", + )); + } + let alignment = align_of::(); + let read_start = (alignment - vector_start % alignment) % alignment; + let storage_bytes = read_start.checked_add(payload_len).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVF-FLAT aligned payload overflow", + ) + })?; + let storage_len = storage_bytes.div_ceil(size_of::()); + Ok(Self { + storage: vec![0.0; storage_len], + read_start, + payload_len, + vector_start, + vector_len, + decoded_vectors: None, + }) + } + + fn empty() -> Self { + Self { + storage: Vec::new(), + read_start: 0, + payload_len: 0, + vector_start: 0, + vector_len: 0, + decoded_vectors: None, + } + } + + fn read_buf_mut(&mut self) -> &mut [u8] { + let storage_bytes = self.storage.len() * size_of::(); + // SAFETY: `storage` is initialized and exclusively borrowed. Viewing + // its allocation as bytes is valid for the exact initialized extent. + let bytes = unsafe { + std::slice::from_raw_parts_mut(self.storage.as_mut_ptr().cast::(), storage_bytes) + }; + &mut bytes[self.read_start..self.read_start + self.payload_len] + } + + fn read_bytes(&self) -> &[u8] { + let storage_bytes = self.storage.len() * size_of::(); + // SAFETY: `storage` remains alive and immutably borrowed for the + // returned byte slice's lifetime. + let bytes = unsafe { + std::slice::from_raw_parts(self.storage.as_ptr().cast::(), storage_bytes) + }; + &bytes[self.read_start..self.read_start + self.payload_len] + } + + fn prepare_vectors(&mut self) -> io::Result<()> { + #[cfg(target_endian = "big")] + { + self.decoded_vectors = Some(bytes_to_f32_vec(&self.read_bytes()[self.vector_start..])?); + } + Ok(()) + } + + fn vectors(&self) -> &[f32] { + if let Some(decoded) = &self.decoded_vectors { + return decoded; + } + let vector_bytes = &self.read_bytes()[self.vector_start..]; + debug_assert_eq!( + vector_bytes.as_ptr().align_offset(align_of::()), + 0, + "aligned IVF-FLAT payload must expose an aligned vector suffix" + ); + // SAFETY: `new` verifies a whole number of f32 values and chooses + // `read_start` so this suffix is f32-aligned. On little-endian hosts + // the file representation is the native f32 representation. Big-endian + // hosts populate `decoded_vectors` before this method is used. + unsafe { + std::slice::from_raw_parts( + vector_bytes.as_ptr().cast::(), + self.vector_len / size_of::(), + ) + } + } +} + +struct FlatListRead { + input_index: usize, + list_id: usize, + count: usize, + id_bytes_len: usize, + offset: u64, +} + impl IVFFlatIndexReader { pub fn open(mut reader: R) -> io::Result { - let mut cursor = PreadCursor::new(&mut reader, 0); + let mut header = [0u8; IVFFLAT_HEADER_SIZE]; + reader.pread(&mut [ReadRequest::new(0, &mut header)])?; + Self::open_with_header(reader, header) + } - let magic = read_u32_le(&mut cursor)?; + pub(crate) fn open_with_header( + reader: R, + header: [u8; IVFFLAT_HEADER_SIZE], + ) -> io::Result { + let magic = u32::from_le_bytes(header[0..4].try_into().unwrap()); if magic != IVFFLAT_MAGIC { return Err(io::Error::new( io::ErrorKind::InvalidData, format!("Invalid IVFFLAT magic: 0x{:08X}", magic), )); } - let version = read_u32_le(&mut cursor)?; + let version = u32::from_le_bytes(header[4..8].try_into().unwrap()); if version != IVFFLAT_VERSION { return Err(io::Error::new( io::ErrorKind::InvalidData, @@ -181,20 +331,22 @@ impl IVFFlatIndexReader { )); } - let d = validate_positive_i32(read_i32_le(&mut cursor)?, "d")? as usize; - let nlist = validate_positive_i32(read_i32_le(&mut cursor)?, "nlist")? as usize; - let metric_code = read_u32_le(&mut cursor)?; + let d = validate_positive_i32(i32::from_le_bytes(header[8..12].try_into().unwrap()), "d")? + as usize; + let nlist = validate_positive_i32( + i32::from_le_bytes(header[12..16].try_into().unwrap()), + "nlist", + )? as usize; + let metric_code = u32::from_le_bytes(header[16..20].try_into().unwrap()); let metric = MetricType::from_code(metric_code).ok_or_else(|| { io::Error::new( io::ErrorKind::InvalidData, format!("Unknown metric type: {}", metric_code), ) })?; - let total_vectors = read_i64_le(&mut cursor)?; - let flags = read_u32_le(&mut cursor)?; - let mut reserved = [0u8; 32]; - cursor.read_exact(&mut reserved)?; - validate_reserved_zero(&reserved, "IVFFLAT")?; + let total_vectors = i64::from_le_bytes(header[20..28].try_into().unwrap()); + let flags = u32::from_le_bytes(header[28..32].try_into().unwrap()); + validate_reserved_zero(&header[32..64], "IVFFLAT")?; let unknown_flags = flags & !SUPPORTED_FLAGS; if unknown_flags != 0 { return Err(io::Error::new( @@ -229,15 +381,37 @@ impl IVFFlatIndexReader { return Ok(()); } - let mut cursor = PreadCursor::new(&mut self.reader, IVFFLAT_HEADER_SIZE as u64); - self.quantizer_centroids = - read_f32_vec(&mut cursor, checked_section_size(self.nlist, self.d)?)?; + let centroid_count = checked_section_size(self.nlist, self.d)?; + let centroid_bytes = centroid_count.checked_mul(4).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVF-FLAT centroid bytes overflow", + ) + })?; + let table_bytes = self.nlist.checked_mul(16).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVF-FLAT offset table overflow") + })?; + let mut metadata = vec![ + 0u8; + centroid_bytes.checked_add(table_bytes).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVF-FLAT metadata size overflow", + ) + })? + ]; + self.reader + .pread(&mut [ReadRequest::new(IVFFLAT_HEADER_SIZE as u64, &mut metadata)])?; + self.quantizer_centroids = bytes_to_f32_vec(&metadata[..centroid_bytes])?; self.list_offsets = vec![0; self.nlist]; self.list_counts = vec![0; self.nlist]; self.list_id_bytes_lens = vec![0; self.nlist]; + let mut actual_total = 0i64; for list_id in 0..self.nlist { - self.list_offsets[list_id] = read_i64_le(&mut cursor)?; - let count = read_i32_le(&mut cursor)?; + let base = centroid_bytes + list_id * 16; + self.list_offsets[list_id] = + i64::from_le_bytes(metadata[base..base + 8].try_into().unwrap()); + let count = i32::from_le_bytes(metadata[base + 8..base + 12].try_into().unwrap()); if count < 0 { return Err(io::Error::new( io::ErrorKind::InvalidData, @@ -245,22 +419,151 @@ impl IVFFlatIndexReader { )); } self.list_counts[list_id] = count; - let id_bytes_len = read_i32_le(&mut cursor)?; + actual_total = actual_total.checked_add(count as i64).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVF-FLAT vector count overflow") + })?; + let id_bytes_len = + i32::from_le_bytes(metadata[base + 12..base + 16].try_into().unwrap()); if id_bytes_len < 0 { return Err(io::Error::new( io::ErrorKind::InvalidData, format!("negative id_bytes_len {} at list {}", id_bytes_len, list_id), )); } + if count > 0 && id_bytes_len == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("missing delta ID bytes for non-empty IVF-FLAT list {list_id}"), + )); + } self.list_id_bytes_lens[list_id] = id_bytes_len; } + if actual_total != self.total_vectors { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "IVF-FLAT header vector count {} does not match list total {actual_total}", + self.total_vectors + ), + )); + } self.loaded = true; Ok(()) } pub fn read_inverted_list(&mut self, list_id: usize) -> io::Result<(Vec, Vec)> { + let mut lists = self.read_inverted_lists(&[list_id])?; + let list = lists.pop().expect("one requested list has one result"); + let vectors = list.vectors().to_vec(); + Ok((list.ids, vectors)) + } + + fn read_inverted_lists(&mut self, list_ids: &[usize]) -> io::Result> { self.ensure_loaded()?; + if !self.delta_ids { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "IVF-FLAT reader only supports delta IDs", + )); + } + let mut results = (0..list_ids.len()).map(|_| None).collect::>(); + let mut metas = Vec::new(); + let mut payloads = Vec::new(); + for (input_index, &list_id) in list_ids.iter().enumerate() { + if list_id >= self.nlist { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("list_id {} out of range (nlist={})", list_id, self.nlist), + )); + } + let count = self.list_counts[list_id] as usize; + if count == 0 { + results[input_index] = Some(FlatListData { + list_id, + ids: Vec::new(), + payload: AlignedFlatPayload::empty(), + }); + continue; + } + let offset = checked_list_offset(self.list_offsets[list_id], list_id)?; + let vector_bytes = checked_list_bytes(count, self.d * 4)?; + let id_bytes_len = self.list_id_bytes_lens[list_id] as usize; + let payload_len = 12usize + .checked_add(id_bytes_len) + .and_then(|len| len.checked_add(vector_bytes)) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVF-FLAT list payload overflow") + })?; + metas.push(FlatListRead { + input_index, + list_id, + count, + id_bytes_len, + offset, + }); + payloads.push(AlignedFlatPayload::new( + payload_len, + 12 + id_bytes_len, + vector_bytes, + )?); + } + if !metas.is_empty() { + let offsets = metas.iter().map(|meta| meta.offset).collect::>(); + let mut buffers = payloads + .iter_mut() + .map(AlignedFlatPayload::read_buf_mut) + .collect::>(); + pread_batched_slices(&mut self.reader, &offsets, &mut buffers)?; + drop(buffers); + for (meta, mut payload) in metas.into_iter().zip(payloads) { + let bytes = payload.read_bytes(); + let base_id = i64::from_le_bytes(bytes[0..8].try_into().unwrap()); + let encoded_len = i32::from_le_bytes(bytes[8..12].try_into().unwrap()); + if encoded_len < 0 || encoded_len as usize != meta.id_bytes_len { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "IVF-FLAT id_bytes_len mismatch", + )); + } + let ids = decode_delta_varint_ids( + base_id, + &bytes[12..12 + meta.id_bytes_len], + meta.count, + )?; + payload.prepare_vectors()?; + results[meta.input_index] = Some(FlatListData { + list_id: meta.list_id, + ids, + payload, + }); + } + } + results + .into_iter() + .map(|result| { + result.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "missing batched IVF-FLAT list read result", + ) + }) + }) + .collect() + } + + fn batch_read_end(&self, list_ids: &[usize]) -> io::Result { + let payload_lengths = list_ids + .iter() + .map(|&list_id| self.list_payload_len(list_id)) + .collect::>>()?; + bounded_ivf_payload_batch_end( + &payload_lengths, + self.reader.read_capabilities().max_ranges_per_pread, + ) + } + + fn list_payload_len(&self, list_id: usize) -> io::Result { if list_id >= self.nlist { return Err(io::Error::new( io::ErrorKind::InvalidInput, @@ -269,39 +572,77 @@ impl IVFFlatIndexReader { } let count = self.list_counts[list_id] as usize; if count == 0 { - return Ok((Vec::new(), Vec::new())); + return Ok(0); } + 12usize + .checked_add(self.list_id_bytes_lens[list_id] as usize) + .and_then(|len| { + self.d + .checked_mul(size_of::()) + .and_then(|row_bytes| count.checked_mul(row_bytes)) + .and_then(|vector_bytes| len.checked_add(vector_bytes)) + }) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVF-FLAT list payload overflow") + }) + } - let offset = checked_list_offset(self.list_offsets[list_id], list_id)?; - let vector_bytes = checked_list_bytes(count, self.d * 4)?; - if self.delta_ids { - let id_bytes_len = self.list_id_bytes_lens[list_id] as usize; - let payload_len = 12usize - .checked_add(id_bytes_len) - .and_then(|len| len.checked_add(vector_bytes)) + fn for_each_streamed_list_chunk( + &mut self, + list_id: usize, + mut consume: impl FnMut(&[i64], &[f32]), + ) -> io::Result<()> { + self.ensure_loaded()?; + let count = self.list_counts[list_id] as usize; + let list_offset = checked_list_offset(self.list_offsets[list_id], list_id)?; + let id_bytes_len = self.list_id_bytes_lens[list_id] as usize; + let ids = read_delta_varint_ids_at( + &mut self.reader, + list_offset, + count, + id_bytes_len, + "IVF-FLAT", + )?; + let row_bytes = self.d.checked_mul(size_of::()).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVF-FLAT row size overflow") + })?; + let vector_offset = list_offset + .checked_add((12usize + id_bytes_len) as u64) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVF-FLAT vector offset overflow", + ) + })?; + let retained_id_bytes = ids.len().checked_mul(size_of::()).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVF-FLAT decoded ID size overflow", + ) + })?; + let mut row_start = 0usize; + while row_start < count { + let chunk_rows = + bounded_ivf_stream_chunk_rows(count - row_start, row_bytes, retained_id_bytes, 1)?; + let vector_bytes = chunk_rows.checked_mul(row_bytes).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVF-FLAT chunk size overflow") + })?; + let mut payload = AlignedFlatPayload::new(vector_bytes, 0, vector_bytes)?; + let chunk_offset = vector_offset + .checked_add(row_start.checked_mul(row_bytes).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVF-FLAT chunk offset overflow") + })? as u64) .ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidData, "IVF-FLAT list payload overflow") + io::Error::new(io::ErrorKind::InvalidData, "IVF-FLAT chunk offset overflow") })?; - let mut payload = vec![0u8; payload_len]; self.reader - .pread(&mut [ReadRequest::new(offset, &mut payload)])?; - let base_id = i64::from_le_bytes(payload[0..8].try_into().unwrap()); - let encoded_len = i32::from_le_bytes(payload[8..12].try_into().unwrap()); - if encoded_len < 0 || encoded_len as usize != id_bytes_len { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "IVF-FLAT id_bytes_len mismatch", - )); - } - let ids = decode_delta_varint_ids(base_id, &payload[12..12 + id_bytes_len], count)?; - let vectors = bytes_to_f32_vec(&payload[12 + id_bytes_len..])?; - Ok((ids, vectors)) - } else { - Err(io::Error::new( - io::ErrorKind::InvalidData, - "IVF-FLAT reader only supports delta IDs", - )) + .pread(&mut [ReadRequest::new(chunk_offset, payload.read_buf_mut())])?; + payload.prepare_vectors()?; + let row_end = row_start + chunk_rows; + consume(&ids[row_start..row_end], payload.vectors()); + row_start = row_end; } + Ok(()) } pub fn search( @@ -352,26 +693,47 @@ impl IVFFlatIndexReader { let (probe_indices, _) = kmeans::find_topk(&q, &self.quantizer_centroids, self.nlist, self.d, nprobe); let mut heap = ReaderTopKHeap::new(k); - - for list_id in probe_indices { - let (ids, vectors) = self.read_inverted_list(list_id)?; - for (local_idx, &id) in ids.iter().enumerate() { - if let Some(f) = filter { - if !f.contains(id) { - continue; - } + let mut batch_start = 0usize; + while batch_start < probe_indices.len() { + let first_list = probe_indices[batch_start]; + if ivf_payload_is_oversized(self.list_payload_len(first_list)?) { + let metric = self.metric; + let d = self.d; + self.for_each_streamed_list_chunk(first_list, |ids, vectors| { + scan_flat_rows(&q, ids, vectors, d, metric, filter, &mut heap); + })?; + batch_start += 1; + continue; + } + let count = self.batch_read_end(&probe_indices[batch_start..])?.max(1); + let batch_end = (batch_start + count).min(probe_indices.len()); + let lists = self.read_inverted_lists(&probe_indices[batch_start..batch_end])?; + let scan_components = lists + .iter() + .map(|list| list.ids.len()) + .sum::() + .saturating_mul(self.d); + if lists.len() > 1 && scan_components >= PARALLEL_FLAT_SCAN_MIN_COMPONENTS { + let per_list_results = lists + .par_iter() + .map(|list| { + let mut local_heap = ReaderTopKHeap::new(k); + scan_flat_list(&q, list, self.d, self.metric, filter, &mut local_heap); + local_heap.into_sorted() + }) + .collect::>(); + for results in per_list_results { + merge_flat_results(&mut heap, results); + } + } else { + for list in &lists { + scan_flat_list(&q, list, self.d, self.metric, filter, &mut heap); } - let vector = &vectors[local_idx * self.d..(local_idx + 1) * self.d]; - heap.push(fvec_distance(&q, vector, self.metric), id); } + batch_start = batch_end; } - let sorted = heap.into_sorted(); - let mut labels: Vec = sorted.iter().map(|&(_, id)| id).collect(); - let mut distances: Vec = sorted.iter().map(|&(dist, _)| dist).collect(); - labels.resize(k, -1); - distances.resize(k, f32::MAX); - Ok((labels, distances)) + Ok(padded_flat_results(heap, k)) } pub fn search_with_roaring_filter( @@ -408,6 +770,7 @@ pub fn search_batch_ivfflat_reader_filter( ) -> io::Result<(Vec, Vec)> { reader.ensure_loaded()?; let d = reader.d; + let metric = reader.metric; if nq == 0 { return Err(io::Error::new( io::ErrorKind::InvalidInput, @@ -471,30 +834,69 @@ pub fn search_batch_ivfflat_reader_filter( } let mut heaps: Vec = (0..nq).map(|_| ReaderTopKHeap::new(k)).collect(); - for list_id in unique_lists { - let count = reader.list_counts[list_id] as usize; - if count == 0 { + let mut batch_start = 0usize; + while batch_start < unique_lists.len() { + let first_list = unique_lists[batch_start]; + if ivf_payload_is_oversized(reader.list_payload_len(first_list)?) { + let query_indices = &list_to_queries[first_list]; + reader.for_each_streamed_list_chunk(first_list, |ids, vectors| { + for &qi in query_indices { + let query = &processed[qi * d..(qi + 1) * d]; + scan_flat_rows(query, ids, vectors, d, metric, filter, &mut heaps[qi]); + } + })?; + batch_start += 1; continue; } - let (ids, vectors) = reader.read_inverted_list(list_id)?; - for &qi in &list_to_queries[list_id] { - let query = &processed[qi * d..(qi + 1) * d]; - for (local_idx, &id) in ids.iter().enumerate() { - if let Some(f) = filter { - if !f.contains(id) { - continue; - } + let count = reader.batch_read_end(&unique_lists[batch_start..])?.max(1); + let batch_end = (batch_start + count).min(unique_lists.len()); + let loaded_lists = reader.read_inverted_lists(&unique_lists[batch_start..batch_end])?; + let scan_components = loaded_lists + .iter() + .map(|list| { + list.ids + .len() + .saturating_mul(list_to_queries[list.list_id].len()) + }) + .sum::() + .saturating_mul(d); + if loaded_lists.len() > 1 && scan_components >= PARALLEL_FLAT_SCAN_MIN_COMPONENTS { + let per_list_results = loaded_lists + .par_iter() + .map(|list| { + let list_id = list.list_id; + list_to_queries[list_id] + .iter() + .map(|&qi| { + let query = &processed[qi * d..(qi + 1) * d]; + let mut local_heap = ReaderTopKHeap::new(k); + scan_flat_list(query, list, d, reader.metric, filter, &mut local_heap); + (qi, local_heap.into_sorted()) + }) + .collect::>() + }) + .collect::>(); + for list_results in per_list_results { + for (qi, results) in list_results { + merge_flat_results(&mut heaps[qi], results); + } + } + } else { + for list in &loaded_lists { + let list_id = list.list_id; + for &qi in &list_to_queries[list_id] { + let query = &processed[qi * d..(qi + 1) * d]; + scan_flat_list(query, list, d, reader.metric, filter, &mut heaps[qi]); } - let vector = &vectors[local_idx * d..(local_idx + 1) * d]; - heaps[qi].push(fvec_distance(query, vector, reader.metric), id); } } + batch_start = batch_end; } let mut result_ids = vec![-1i64; nq * k]; let mut result_dists = vec![f32::MAX; nq * k]; - for qi in 0..nq { - let sorted = std::mem::replace(&mut heaps[qi], ReaderTopKHeap::new(0)).into_sorted(); + for (qi, heap) in heaps.into_iter().enumerate() { + let sorted = heap.into_sorted(); let base = qi * k; for (i, &(dist, id)) in sorted.iter().enumerate() { result_ids[base + i] = id; @@ -517,9 +919,70 @@ pub fn search_batch_ivfflat_reader_roaring_filter( search_batch_ivfflat_reader_filter(reader, queries, nq, k, nprobe, Some(&filter)) } +fn scan_flat_list( + query: &[f32], + list: &FlatListData, + d: usize, + metric: MetricType, + filter: Option<&dyn RowIdFilter>, + heap: &mut ReaderTopKHeap, +) { + scan_flat_rows(query, &list.ids, list.vectors(), d, metric, filter, heap); +} + +fn scan_flat_rows( + query: &[f32], + ids: &[i64], + vectors: &[f32], + d: usize, + metric: MetricType, + filter: Option<&dyn RowIdFilter>, + heap: &mut ReaderTopKHeap, +) { + // In cosine mode this caches the query norm once per list instead of + // recomputing it for every candidate vector. + let distance_context = QueryDistance::new(query, metric); + for (local_idx, &id) in ids.iter().enumerate() { + if filter.is_some_and(|value| !value.contains(id)) { + continue; + } + let vector = &vectors[local_idx * d..(local_idx + 1) * d]; + let distance = if metric == MetricType::L2 { + if let Some(threshold) = heap.worst_distance() { + if fvec_l2sqr_scaled_exceeds(query, vector, 1.0, threshold) { + continue; + } + } + fvec_l2sqr(query, vector) + } else { + distance_context.distance_to(vector, None) + }; + heap.push(distance, id); + } +} + +fn merge_flat_results(heap: &mut ReaderTopKHeap, results: Vec<(f32, i64)>) { + for (distance, id) in results { + heap.push(distance, id); + } +} + +fn padded_flat_results(heap: ReaderTopKHeap, k: usize) -> (Vec, Vec) { + let sorted = heap.into_sorted(); + let mut labels = sorted.iter().map(|&(_, id)| id).collect::>(); + let mut distances = sorted + .iter() + .map(|&(distance, _)| distance) + .collect::>(); + labels.resize(k, -1); + distances.resize(k, f32::MAX); + (labels, distances) +} + struct ReaderTopKHeap { k: usize, data: Vec<(f32, i64)>, + worst_index: Option, } impl ReaderTopKHeap { @@ -527,27 +990,43 @@ impl ReaderTopKHeap { Self { k, data: Vec::with_capacity(k), + worst_index: None, } } + #[inline] + fn worst_distance(&self) -> Option { + self.worst_index.map(|index| self.data[index].0) + } + + #[inline] fn push(&mut self, dist: f32, id: i64) { if self.k == 0 { return; } if self.data.len() < self.k { self.data.push((dist, id)); + if self.data.len() == self.k { + self.refresh_worst(); + } return; } - if let Some((worst_idx, _)) = self + let worst_index = self + .worst_index + .expect("a full IVF-FLAT top-k heap has a worst entry"); + if dist < self.data[worst_index].0 { + self.data[worst_index] = (dist, id); + self.refresh_worst(); + } + } + + fn refresh_worst(&mut self) { + self.worst_index = self .data .iter() .enumerate() .max_by(|(_, a), (_, b)| a.0.partial_cmp(&b.0).unwrap()) - { - if dist < self.data[worst_idx].0 { - self.data[worst_idx] = (dist, id); - } - } + .map(|(index, _)| index); } fn into_sorted(mut self) -> Vec<(f32, i64)> { @@ -573,24 +1052,6 @@ fn write_f32_slice(out: &mut dyn SeekWrite, data: &[f32]) -> io::Result<()> { out.write_all(&bytes) } -fn read_u32_le(reader: &mut PreadCursor<'_, R>) -> io::Result { - let mut buf = [0u8; 4]; - reader.read_exact(&mut buf)?; - Ok(u32::from_le_bytes(buf)) -} - -fn read_i32_le(reader: &mut PreadCursor<'_, R>) -> io::Result { - let mut buf = [0u8; 4]; - reader.read_exact(&mut buf)?; - Ok(i32::from_le_bytes(buf)) -} - -fn read_i64_le(reader: &mut PreadCursor<'_, R>) -> io::Result { - let mut buf = [0u8; 8]; - reader.read_exact(&mut buf)?; - Ok(i64::from_le_bytes(buf)) -} - fn validate_positive_i32(val: i32, field: &str) -> io::Result { if val <= 0 { return Err(io::Error::new( @@ -727,15 +1188,6 @@ fn checked_list_bytes(count: usize, bytes_per_entry: usize) -> io::Result }) } -fn read_f32_vec( - reader: &mut PreadCursor<'_, R>, - count: usize, -) -> io::Result> { - let mut buf = vec![0u8; count * 4]; - reader.read_exact(&mut buf)?; - bytes_to_f32_vec(&buf) -} - fn bytes_to_f32_vec(bytes: &[u8]) -> io::Result> { if !bytes.len().is_multiple_of(4) { return Err(io::Error::new( @@ -840,9 +1292,83 @@ fn decode_roaring_filter(bytes: &[u8]) -> io::Result { mod tests { use super::*; use crate::distance::MetricType; - use crate::io::PosWriter; + use crate::io::{PosWriter, ReadRequest}; use crate::ivfflat::IVFFlatIndex; use std::io::Cursor; + use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + use std::sync::Arc; + + fn balanced_flat_index(d: usize, nlist: usize, rows_per_list: usize) -> IVFFlatIndex { + let mut index = IVFFlatIndex::new(d, nlist, MetricType::L2); + index.quantizer_centroids = (0..nlist) + .flat_map(|list_id| { + (0..d).map(move |dimension| list_id as f32 * 10.0 + dimension as f32 * 0.01) + }) + .collect(); + for list_id in 0..nlist { + index.ids[list_id] = (0..rows_per_list) + .map(|row| (list_id * rows_per_list + row) as i64) + .collect(); + index.vectors[list_id] = (0..rows_per_list) + .flat_map(|row| { + (0..d).map(move |dimension| { + list_id as f32 * 10.0 + row as f32 * 0.001 + dimension as f32 * 0.01 + }) + }) + .collect(); + } + index + } + + fn serialized_flat_index(index: &IVFFlatIndex) -> Vec { + let mut bytes = Vec::new(); + write_ivfflat_index(index, &mut PosWriter::new(&mut bytes)).unwrap(); + bytes + } + + #[test] + fn ivfflat_streamed_list_reader_matches_full_payload() { + let index = balanced_flat_index(8, 1, 257); + let bytes = serialized_flat_index(&index); + let mut full_reader = IVFFlatIndexReader::open(Cursor::new(bytes.clone())).unwrap(); + let expected = full_reader.read_inverted_list(0).unwrap(); + let mut streamed_reader = IVFFlatIndexReader::open(Cursor::new(bytes)).unwrap(); + let mut actual_ids = Vec::new(); + let mut actual_vectors = Vec::new(); + streamed_reader + .for_each_streamed_list_chunk(0, |ids, vectors| { + actual_ids.extend_from_slice(ids); + actual_vectors.extend_from_slice(vectors); + }) + .unwrap(); + assert_eq!(actual_ids, expected.0); + assert_eq!(actual_vectors, expected.1); + } + + struct ThreadTrackingFilter { + workers: AtomicU64, + } + + impl RowIdFilter for ThreadTrackingFilter { + fn contains(&self, _id: i64) -> bool { + if let Some(worker) = rayon::current_thread_index() { + self.workers.fetch_or(1u64 << worker, Ordering::Relaxed); + } + true + } + } + + struct CountingReader { + inner: Cursor>, + calls: Arc, + } + + impl SeekRead for CountingReader { + fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { + self.calls.fetch_add(1, Ordering::Relaxed); + self.inner.pread(ranges) + } + } #[test] fn test_ivfflat_write_read_search_roundtrip() { @@ -883,6 +1409,65 @@ mod tests { assert_eq!(distances, expected_distances); } + #[test] + fn test_ivfflat_reader_handles_unaligned_vector_suffix_without_copy_contract_change() { + let mut index = IVFFlatIndex::new(3, 1, MetricType::L2); + index.quantizer_centroids = vec![0.0; 3]; + // These deltas occupy 1, 2, and 3 bytes, so the raw-vector suffix + // starts at byte 18 inside the list payload instead of a f32 boundary. + index.ids[0] = vec![1, 130, 16_515]; + index.vectors[0] = vec![0.0, 0.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + let (_, encoded_ids) = encode_delta_varint_ids(&index.ids[0]); + assert_eq!((12 + encoded_ids.len()) % align_of::(), 2); + + let mut reader = + IVFFlatIndexReader::open(Cursor::new(serialized_flat_index(&index))).unwrap(); + let mut direct_lists = reader.read_inverted_lists(&[0]).unwrap(); + let direct = direct_lists.pop().unwrap(); + assert_eq!( + direct.vectors().as_ptr().align_offset(align_of::()), + 0, + "the direct scan buffer must repair the unaligned on-disk suffix" + ); + assert_eq!(direct.vectors(), index.vectors[0]); + + let (ids, vectors) = reader.read_inverted_list(0).unwrap(); + assert_eq!(ids, index.ids[0]); + assert_eq!(vectors, index.vectors[0]); + + let (result_ids, distances) = reader.search(&[1.0, 2.0, 3.0], 1, 1).unwrap(); + assert_eq!(result_ids, vec![130]); + assert_eq!(distances, vec![0.0]); + } + + #[test] + fn test_ivfflat_reader_cosine_matches_in_memory_with_zero_vector() { + let d = 2; + let data = vec![3.0, 4.0, 0.0, 0.0, -3.0, -4.0, 4.0, 3.0]; + let ids = vec![10, 11, 12, 13]; + let query = [3.0, 4.0]; + let mut index = IVFFlatIndex::new(d, 1, MetricType::Cosine); + index.train(&data, ids.len()); + index.add(&data, &ids, ids.len()); + + let mut expected_distances = vec![0.0; ids.len()]; + let mut expected_labels = vec![0; ids.len()]; + index.search( + &query, + 1, + ids.len(), + 1, + &mut expected_distances, + &mut expected_labels, + ); + + let mut reader = + IVFFlatIndexReader::open(Cursor::new(serialized_flat_index(&index))).unwrap(); + let actual = reader.search(&query, ids.len(), 1).unwrap(); + assert_eq!(actual.0, expected_labels); + assert_eq!(actual.1, expected_distances); + } + #[test] fn test_ivfflat_reader_search_with_filter() { use std::collections::HashSet; @@ -978,6 +1563,175 @@ mod tests { } } + #[test] + fn test_ivfflat_large_single_query_scans_lists_in_parallel() { + let d = 16; + let nlist = 8; + let index = balanced_flat_index(d, nlist, 8192); + let query = index.vectors[3][17 * d..18 * d].to_vec(); + let k = 10; + let mut expected_distances = vec![0.0; k]; + let mut expected_labels = vec![0; k]; + index.search( + &query, + 1, + k, + nlist, + &mut expected_distances, + &mut expected_labels, + ); + + let filter = ThreadTrackingFilter { + workers: AtomicU64::new(0), + }; + let mut reader = + IVFFlatIndexReader::open(Cursor::new(serialized_flat_index(&index))).unwrap(); + let actual = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .unwrap() + .install(|| { + reader + .search_with_filter(&query, k, nlist, Some(&filter)) + .unwrap() + }); + + let mut actual_pairs = actual.0.into_iter().zip(actual.1).collect::>(); + let mut expected_pairs = expected_labels + .into_iter() + .zip(expected_distances) + .collect::>(); + actual_pairs.sort_by(|a, b| a.1.total_cmp(&b.1).then_with(|| a.0.cmp(&b.0))); + expected_pairs.sort_by(|a, b| a.1.total_cmp(&b.1).then_with(|| a.0.cmp(&b.0))); + assert_eq!(actual_pairs, expected_pairs); + assert!( + filter.workers.load(Ordering::Relaxed).count_ones() > 1, + "a large single query should scan probed lists on multiple Rayon workers" + ); + } + + #[test] + fn test_ivfflat_batch_scans_lists_in_parallel_without_duplicate_reads() { + let d = 16; + let nlist = 8; + let nq = 8; + let k = 10; + let index = balanced_flat_index(d, nlist, 1024); + let queries = (0..nq) + .flat_map(|query_index| { + let row = query_index * 7; + index.vectors[query_index][row * d..(row + 1) * d].iter() + }) + .copied() + .collect::>(); + let bytes = serialized_flat_index(&index); + let mut expected_ids = Vec::with_capacity(nq * k); + let mut expected_distances = Vec::with_capacity(nq * k); + for query in queries.chunks_exact(d) { + let mut reader = IVFFlatIndexReader::open(Cursor::new(bytes.clone())).unwrap(); + let (ids, distances) = reader.search(query, k, nlist).unwrap(); + expected_ids.extend(ids); + expected_distances.extend(distances); + } + + let calls = Arc::new(AtomicUsize::new(0)); + let source = CountingReader { + inner: Cursor::new(bytes), + calls: Arc::clone(&calls), + }; + let mut reader = IVFFlatIndexReader::open(source).unwrap(); + reader.ensure_loaded().unwrap(); + calls.store(0, Ordering::Relaxed); + let filter = ThreadTrackingFilter { + workers: AtomicU64::new(0), + }; + let actual = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .unwrap() + .install(|| { + search_batch_ivfflat_reader_filter( + &mut reader, + &queries, + nq, + k, + nlist, + Some(&filter), + ) + .unwrap() + }); + + for query_index in 0..nq { + let range = query_index * k..(query_index + 1) * k; + assert!( + actual.1[range.clone()] + .windows(2) + .all(|pair| pair[0] <= pair[1]), + "parallel batch results must remain distance-sorted" + ); + let mut actual_pairs = actual.0[range.clone()] + .iter() + .copied() + .zip(actual.1[range.clone()].iter().copied()) + .collect::>(); + let mut expected_pairs = expected_ids[range.clone()] + .iter() + .copied() + .zip(expected_distances[range].iter().copied()) + .collect::>(); + actual_pairs.sort_by_key(|&(id, _)| id); + expected_pairs.sort_by_key(|&(id, _)| id); + assert_eq!(actual_pairs, expected_pairs); + } + assert!( + filter.workers.load(Ordering::Relaxed).count_ones() > 1, + "a large batch should scan lists on multiple Rayon workers" + ); + assert_eq!( + calls.load(Ordering::Relaxed), + 1, + "parallel scan must not duplicate the multi-range list read" + ); + } + + #[test] + fn test_ivfflat_open_and_metadata_load_use_two_reads() { + let index = balanced_flat_index(16, 32, 4); + let calls = Arc::new(AtomicUsize::new(0)); + let source = CountingReader { + inner: Cursor::new(serialized_flat_index(&index)), + calls: Arc::clone(&calls), + }; + + let mut reader = IVFFlatIndexReader::open(source).unwrap(); + assert_eq!(calls.load(Ordering::Relaxed), 1, "header is one range read"); + reader.ensure_loaded().unwrap(); + assert_eq!( + calls.load(Ordering::Relaxed), + 2, + "centroids and the offset table are one contiguous range read" + ); + reader.ensure_loaded().unwrap(); + assert_eq!( + calls.load(Ordering::Relaxed), + 2, + "resident metadata is not read twice" + ); + } + + #[test] + fn test_ivfflat_metadata_rejects_header_vector_count_mismatch() { + let index = balanced_flat_index(8, 4, 16); + let mut bytes = serialized_flat_index(&index); + bytes[20..28].copy_from_slice(&(index.total_vectors() as i64 + 1).to_le_bytes()); + + let mut reader = IVFFlatIndexReader::open(Cursor::new(bytes)).unwrap(); + let error = reader.ensure_loaded().unwrap_err(); + assert!(error + .to_string() + .contains("header vector count 65 does not match list total 64")); + } + #[test] fn test_ivfflat_batch_reader_search_with_roaring_filter_bytes() { let d = 2; diff --git a/core/src/ivfhnswflat.rs b/core/src/ivfhnswflat.rs deleted file mode 100644 index e534dd6..0000000 --- a/core/src/ivfhnswflat.rs +++ /dev/null @@ -1,329 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::distance::{MetricType, QueryDistance}; -use crate::hnsw::{HnswBuildParams, HnswGraph}; -use crate::hnsw_search::{search_hnsw_lists, HnswSearchList}; -use crate::ivfflat::IVFFlatIndex; -use crate::ivfpq::RowIdFilter; -use crate::kmeans; -use crate::topk::TopKHeap; -use rayon::prelude::*; -use std::io; - -pub struct IVFHNSWFlatIndex { - /// Exposed to match the existing core index structs. Mutating `flat` - /// directly can stale `graphs`; call `build_graphs` again before HNSW search. - pub flat: IVFFlatIndex, - pub graphs: Vec>, - pub hnsw_params: HnswBuildParams, -} - -impl IVFHNSWFlatIndex { - pub fn new(d: usize, nlist: usize, metric: MetricType, hnsw_params: HnswBuildParams) -> Self { - IVFHNSWFlatIndex { - flat: IVFFlatIndex::new(d, nlist, metric), - graphs: vec![None; nlist], - hnsw_params, - } - } - - pub fn train(&mut self, data: &[f32], n: usize) { - self.flat.train(data, n); - } - - pub fn add(&mut self, data: &[f32], ids: &[i64], n: usize) { - self.flat.add(data, ids, n); - self.graphs.fill(None); - } - - pub fn build_graphs(&mut self) -> io::Result<()> { - self.graphs = (0..self.flat.nlist) - .into_par_iter() - .map(|list_id| { - let count = self.flat.ids[list_id].len(); - if count == 0 { - Ok(None) - } else { - HnswGraph::build( - &self.flat.vectors[list_id], - count, - self.flat.d, - self.flat.metric, - self.hnsw_params, - ) - .map(Some) - } - }) - .collect::>>()?; - Ok(()) - } - - #[allow(clippy::too_many_arguments)] - pub fn search( - &self, - queries: &[f32], - nq: usize, - k: usize, - nprobe: usize, - ef_search: usize, - result_distances: &mut [f32], - result_labels: &mut [i64], - ) { - self.search_with_filter( - queries, - nq, - k, - nprobe, - ef_search, - None, - result_distances, - result_labels, - ); - } - - #[allow(clippy::too_many_arguments)] - pub fn search_with_filter( - &self, - queries: &[f32], - nq: usize, - k: usize, - nprobe: usize, - ef_search: usize, - filter: Option<&dyn RowIdFilter>, - result_distances: &mut [f32], - result_labels: &mut [i64], - ) { - let processed_queries = self.flat.preprocess_vectors(queries, nq); - let (all_probe_indices, _) = kmeans::find_topk_batch( - &processed_queries, - nq, - &self.flat.quantizer_centroids, - self.flat.nlist, - self.flat.d, - nprobe, - ); - - for qi in 0..nq { - let query = &processed_queries[qi * self.flat.d..(qi + 1) * self.flat.d]; - let lists: Vec<_> = all_probe_indices[qi] - .iter() - .map(|&list_id| HnswSearchList { - ids: self.flat.ids[list_id].as_slice(), - graph: self.graphs[list_id].as_ref(), - payload: list_id, - }) - .collect(); - let sorted = search_hnsw_lists(query, &lists, k, ef_search, filter, |list, heap| { - self.scan_flat_list(query, list.payload, filter, heap); - }); - let out_base = qi * k; - for (i, &(dist, id)) in sorted.iter().enumerate() { - result_distances[out_base + i] = dist; - result_labels[out_base + i] = id; - } - for i in sorted.len()..k { - result_distances[out_base + i] = f32::MAX; - result_labels[out_base + i] = -1; - } - } - } - - fn scan_flat_list( - &self, - query: &[f32], - list_id: usize, - filter: Option<&dyn RowIdFilter>, - heap: &mut TopKHeap, - ) { - let distance = QueryDistance::new(query, self.flat.metric); - for (local_id, &row_id) in self.flat.ids[list_id].iter().enumerate() { - if let Some(f) = filter { - if !f.contains(row_id) { - continue; - } - } - let vector = - &self.flat.vectors[list_id][local_id * self.flat.d..(local_id + 1) * self.flat.d]; - heap.push(distance.distance_to(vector, None), row_id); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::distance::MetricType; - use crate::hnsw::HnswBuildParams; - - #[test] - fn test_ivfhnswflat_recalls_query_vector() { - let d = 4; - let nlist = 4; - let n = 128; - let data: Vec = (0..n) - .flat_map(|i| { - let cluster = (i % nlist) as f32 * 100.0; - [cluster + i as f32 * 0.01, 1.0, 2.0, 3.0] - }) - .collect(); - let ids: Vec = (0..n as i64).collect(); - - let mut index = IVFHNSWFlatIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, n); - index.add(&data, &ids, n); - index.build_graphs().unwrap(); - - let query_id = 23; - let mut distances = vec![0.0; 5]; - let mut labels = vec![0; 5]; - index.search( - &data[query_id * d..(query_id + 1) * d], - 1, - 5, - nlist, - 32, - &mut distances, - &mut labels, - ); - - assert_eq!(labels[0], ids[query_id]); - assert_eq!(distances[0], 0.0); - } - - #[test] - fn test_ivfhnswflat_without_built_graphs_falls_back_to_flat_scan() { - let d = 4; - let nlist = 4; - let n = 128; - let data: Vec = (0..n) - .flat_map(|i| { - let cluster = (i % nlist) as f32 * 100.0; - [cluster + i as f32 * 0.01, 1.0, 2.0, 3.0] - }) - .collect(); - let ids: Vec = (0..n as i64).collect(); - - let mut index = IVFHNSWFlatIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, n); - index.add(&data, &ids, n); - - let query_id = 23; - let mut distances = vec![0.0; 5]; - let mut labels = vec![0; 5]; - index.search( - &data[query_id * d..(query_id + 1) * d], - 1, - 5, - nlist, - 32, - &mut distances, - &mut labels, - ); - - assert_eq!(labels[0], ids[query_id]); - assert_eq!(distances[0], 0.0); - } - - #[test] - fn test_ivfhnswflat_selective_filter_uses_exact_results() { - use std::collections::HashSet; - - let d = 2; - let nlist = 1; - let n = 64; - let mut data = Vec::with_capacity(n * d); - for i in 0..n { - data.push(i as f32); - data.push(0.0); - } - let ids: Vec = (0..n as i64).collect(); - - let mut index = IVFHNSWFlatIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, n); - index.add(&data, &ids, n); - index.build_graphs().unwrap(); - - let filter: HashSet = [63].into_iter().collect(); - let mut distances = vec![0.0; 1]; - let mut labels = vec![0; 1]; - index.search_with_filter( - &[0.0, 0.0], - 1, - 1, - 1, - 4, - Some(&filter), - &mut distances, - &mut labels, - ); - - assert_eq!(labels[0], 63); - assert_eq!(distances[0], 63.0 * 63.0); - } - - #[test] - fn test_ivfhnswflat_filter_backfills_when_graph_returns_too_few_matches() { - use std::collections::HashSet; - - let d = 2; - let nlist = 1; - let n = 128; - let mut data = Vec::with_capacity(n * d); - for i in 0..n { - data.push(i as f32); - data.push(0.0); - } - let ids: Vec = (0..n as i64).collect(); - - let mut index = IVFHNSWFlatIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, n); - index.add(&data, &ids, n); - index.build_graphs().unwrap(); - - let filter: HashSet = (0..n as i64).filter(|id| id % 2 == 0).collect(); - let mut distances = vec![0.0; 10]; - let mut labels = vec![0; 10]; - index.search_with_filter( - &[127.0, 0.0], - 1, - 10, - 1, - 1, - Some(&filter), - &mut distances, - &mut labels, - ); - - assert_eq!( - labels, - vec![126, 124, 122, 120, 118, 116, 114, 112, 110, 108] - ); - assert!(labels.iter().all(|id| id % 2 == 0)); - } - - #[test] - fn test_topk_heap_keeps_closest_duplicate_id() { - let mut heap = TopKHeap::new(2); - - heap.push(10.0, 7); - heap.push(5.0, 8); - heap.push(1.0, 7); - - assert_eq!(heap.into_sorted(), vec![(1.0, 7), (5.0, 8)]); - } -} diff --git a/core/src/ivfhnswflat_io.rs b/core/src/ivfhnswflat_io.rs deleted file mode 100644 index 679fe1c..0000000 --- a/core/src/ivfhnswflat_io.rs +++ /dev/null @@ -1,1753 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::distance::{preprocess_vectors, MetricType, QueryDistance}; -use crate::hnsw::{HnswBuildParams, HnswGraph, HnswSearchWorkspace}; -use crate::hnsw_search::{search_hnsw_lists, HnswSearchList}; -use crate::index_io_util::{ - bytes_to_f32_vec, checked_list_bytes, checked_list_offset, checked_section_size, - decode_delta_varint_ids, decode_graph, decode_roaring_filter, encode_delta_varint_ids, - encode_graph, read_f32_vec, read_i32_le, read_i64_le, read_u32_le, u64_to_i64, usize_to_i32, - usize_to_i64, validate_positive_i32, validate_reserved_zero, validate_search_inputs, - write_f32_slice, write_i32_le, write_i64_le, write_u32_le, -}; -use crate::io::{PreadCursor, ReadRequest, SeekRead, SeekWrite}; -use crate::ivfhnswflat::IVFHNSWFlatIndex; -use crate::ivfpq::RowIdFilter; -use crate::kmeans; -use crate::topk::TopKHeap; -use std::io; - -pub const IVF_HNSW_FLAT_MAGIC: u32 = 0x4948464C; // "IHFL" -pub const IVF_HNSW_FLAT_VERSION: u32 = 1; -pub const IVF_HNSW_FLAT_HEADER_SIZE: usize = 64; -const FLAG_DELTA_IDS: u32 = 1 << 0; -const FLAG_GRAPH_V1: u32 = 1 << 1; -const REQUIRED_FLAGS: u32 = FLAG_DELTA_IDS | FLAG_GRAPH_V1; -const SUPPORTED_FLAGS: u32 = REQUIRED_FLAGS; -const MAX_COALESCED_READ_GAP_BYTES: u64 = 1 << 20; - -pub fn write_ivfhnswflat_index( - index: &IVFHNSWFlatIndex, - out: &mut dyn SeekWrite, -) -> io::Result<()> { - validate_index_shape(index)?; - let d = index.flat.d; - let nlist = index.flat.nlist; - let total_vectors = index.flat.ids.iter().try_fold(0i64, |sum, ids| { - let count = usize_to_i64(ids.len(), "total vector count")?; - sum.checked_add(count).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "total vector count exceeds i64 length limit", - ) - }) - })?; - let sorted_lists: Vec = (0..nlist) - .map(|list_id| build_sorted_flat_graph_list(index, list_id)) - .collect::>()?; - - write_u32_le(out, IVF_HNSW_FLAT_MAGIC)?; - write_u32_le(out, IVF_HNSW_FLAT_VERSION)?; - write_i32_le(out, usize_to_i32(d, "dimension")?)?; - write_i32_le(out, usize_to_i32(nlist, "nlist")?)?; - write_u32_le(out, index.flat.metric as u32)?; - write_i64_le(out, total_vectors)?; - let params = index.hnsw_params.sanitized(); - write_i32_le(out, usize_to_i32(params.m, "hnsw m")?)?; - write_i32_le( - out, - usize_to_i32(params.ef_construction, "hnsw ef_construction")?, - )?; - write_i32_le(out, usize_to_i32(params.max_level, "hnsw max_level")?)?; - write_u32_le(out, REQUIRED_FLAGS)?; - out.write_all(&[0u8; 20])?; - - write_f32_slice(out, &index.flat.quantizer_centroids)?; - - let offset_table_size = nlist.checked_mul(24).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "IVF-HNSW-FLAT offset table size overflow", - ) - })?; - let data_start = out - .pos() - .checked_add(offset_table_size as u64) - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "IVF-HNSW-FLAT data start offset overflow", - ) - })?; - let mut list_offsets = vec![0i64; nlist]; - let mut list_counts = vec![0i32; nlist]; - let mut list_graph_bytes_lens = vec![0i32; nlist]; - let mut list_payload_bytes_lens = vec![0i64; nlist]; - let mut current_offset = data_start; - - for list_id in 0..nlist { - list_offsets[list_id] = u64_to_i64(current_offset, "list offset")?; - let count = sorted_lists[list_id].ids.len(); - list_counts[list_id] = usize_to_i32(count, "list count")?; - list_graph_bytes_lens[list_id] = - usize_to_i32(sorted_lists[list_id].graph_bytes.len(), "graph bytes")?; - if count > 0 { - let payload_len = list_payload_len( - count, - d, - sorted_lists[list_id].id_bytes.len(), - sorted_lists[list_id].graph_bytes.len(), - )?; - list_payload_bytes_lens[list_id] = usize_to_i64(payload_len, "list payload bytes")?; - current_offset = current_offset - .checked_add(payload_len as u64) - .ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "IVF-HNSW-FLAT offset overflow") - })?; - } - } - - for list_id in 0..nlist { - write_i64_le(out, list_offsets[list_id])?; - write_i32_le(out, list_counts[list_id])?; - write_i32_le(out, list_graph_bytes_lens[list_id])?; - write_i64_le(out, list_payload_bytes_lens[list_id])?; - } - - for list_id in 0..nlist { - let list = &sorted_lists[list_id]; - if list.ids.is_empty() { - continue; - } - write_i64_le(out, list.ids[0])?; - write_i32_le(out, usize_to_i32(list.id_bytes.len(), "delta ID section")?)?; - out.write_all(&list.id_bytes)?; - write_f32_slice(out, &list.vectors)?; - out.write_all(&list.graph_bytes)?; - } - - Ok(()) -} - -pub struct IVFHNSWFlatIndexReader { - reader: R, - pub d: usize, - pub nlist: usize, - pub metric: MetricType, - pub total_vectors: i64, - pub hnsw_params: HnswBuildParams, - pub quantizer_centroids: Vec, - pub list_offsets: Vec, - pub list_counts: Vec, - pub list_graph_bytes_lens: Vec, - pub list_payload_bytes_lens: Vec, - loaded: bool, -} - -impl IVFHNSWFlatIndexReader { - pub fn open(mut reader: R) -> io::Result { - let mut cursor = PreadCursor::new(&mut reader, 0); - - let magic = read_u32_le(&mut cursor)?; - if magic != IVF_HNSW_FLAT_MAGIC { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("Invalid IVF_HNSW_FLAT magic: 0x{:08X}", magic), - )); - } - let version = read_u32_le(&mut cursor)?; - if version != IVF_HNSW_FLAT_VERSION { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("Unsupported IVF_HNSW_FLAT version: {}", version), - )); - } - - let d = validate_positive_i32(read_i32_le(&mut cursor)?, "d")? as usize; - let nlist = validate_positive_i32(read_i32_le(&mut cursor)?, "nlist")? as usize; - let metric_code = read_u32_le(&mut cursor)?; - let metric = MetricType::from_code(metric_code).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("Unknown metric type: {}", metric_code), - ) - })?; - let total_vectors = read_i64_le(&mut cursor)?; - let hnsw_params = HnswBuildParams { - m: validate_positive_i32(read_i32_le(&mut cursor)?, "hnsw m")? as usize, - ef_construction: validate_positive_i32( - read_i32_le(&mut cursor)?, - "hnsw ef_construction", - )? as usize, - max_level: validate_positive_i32(read_i32_le(&mut cursor)?, "hnsw max_level")? as usize, - } - .sanitized(); - let flags = read_u32_le(&mut cursor)?; - let mut reserved = [0u8; 20]; - cursor.read_exact(&mut reserved)?; - validate_reserved_zero(&reserved, "IVF_HNSW_FLAT")?; - let unknown_flags = flags & !SUPPORTED_FLAGS; - if unknown_flags != 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("Unsupported IVF_HNSW_FLAT flags: 0x{:08X}", unknown_flags), - )); - } - if flags & REQUIRED_FLAGS != REQUIRED_FLAGS { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "IVF_HNSW_FLAT v1 requires delta-varint IDs and graph v1", - )); - } - - Ok(Self { - reader, - d, - nlist, - metric, - total_vectors, - hnsw_params, - quantizer_centroids: Vec::new(), - list_offsets: Vec::new(), - list_counts: Vec::new(), - list_graph_bytes_lens: Vec::new(), - list_payload_bytes_lens: Vec::new(), - loaded: false, - }) - } - - pub fn ensure_loaded(&mut self) -> io::Result<()> { - if self.loaded { - return Ok(()); - } - - let mut cursor = PreadCursor::new(&mut self.reader, IVF_HNSW_FLAT_HEADER_SIZE as u64); - self.quantizer_centroids = - read_f32_vec(&mut cursor, checked_section_size(self.nlist, self.d)?)?; - self.list_offsets = vec![0; self.nlist]; - self.list_counts = vec![0; self.nlist]; - self.list_graph_bytes_lens = vec![0; self.nlist]; - self.list_payload_bytes_lens = vec![0; self.nlist]; - for list_id in 0..self.nlist { - self.list_offsets[list_id] = read_i64_le(&mut cursor)?; - let count = read_i32_le(&mut cursor)?; - if count < 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("negative list count {} at list {}", count, list_id), - )); - } - self.list_counts[list_id] = count; - let graph_bytes_len = read_i32_le(&mut cursor)?; - if graph_bytes_len < 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "negative graph_bytes_len {} at list {}", - graph_bytes_len, list_id - ), - )); - } - self.list_graph_bytes_lens[list_id] = graph_bytes_len; - let payload_bytes_len = read_i64_le(&mut cursor)?; - if payload_bytes_len < 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "negative payload_bytes_len {} at list {}", - payload_bytes_len, list_id - ), - )); - } - self.list_payload_bytes_lens[list_id] = payload_bytes_len; - } - - self.loaded = true; - Ok(()) - } - - pub fn read_inverted_list( - &mut self, - list_id: usize, - ) -> io::Result<(Vec, Vec, Option)> { - let Some(list) = self.read_graph_list(list_id)? else { - return Ok((Vec::new(), Vec::new(), None)); - }; - let vectors = list.graph.vectors().to_vec(); - Ok((list.ids, vectors, Some(list.graph))) - } - - fn read_graph_list(&mut self, list_id: usize) -> io::Result> { - self.ensure_loaded()?; - let Some(meta) = self.list_payload_meta(list_id)? else { - return Ok(None); - }; - let mut payload = vec![0u8; meta.payload_len]; - self.reader - .pread(&mut [ReadRequest::new(meta.offset, &mut payload)])?; - - self.decode_graph_list_payload(meta, &payload).map(Some) - } - - fn read_graph_lists_coalesced( - &mut self, - list_ids: &[usize], - ) -> io::Result> { - self.ensure_loaded()?; - let mut metas = Vec::new(); - for &list_id in list_ids { - if let Some(meta) = self.list_payload_meta(list_id)? { - metas.push(meta); - } - } - if metas.is_empty() { - return Ok(Vec::new()); - } - - metas.sort_by_key(|meta| meta.offset); - let mut loaded = Vec::with_capacity(metas.len()); - let mut range_start = metas[0].offset; - let mut range_end = metas[0].end_offset()?; - let mut range_payload_bytes = metas[0].payload_len; - let mut range_metas = vec![metas[0]]; - for &meta in metas.iter().skip(1) { - let meta_end = meta.end_offset()?; - let gap = meta.offset.checked_sub(range_end).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "IVF-HNSW-FLAT list payload offsets overlap", - ) - })?; - if should_coalesce_gap( - gap, - range_start, - meta_end, - range_payload_bytes, - meta.payload_len, - ) { - range_end = meta_end; - range_payload_bytes = range_payload_bytes - .checked_add(meta.payload_len) - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "coalesced IVF-HNSW-FLAT requested payload bytes overflow", - ) - })?; - range_metas.push(meta); - } else { - self.read_coalesced_graph_list_range( - range_start, - range_end, - &range_metas, - &mut loaded, - )?; - range_start = meta.offset; - range_end = meta_end; - range_payload_bytes = meta.payload_len; - range_metas.clear(); - range_metas.push(meta); - } - } - self.read_coalesced_graph_list_range(range_start, range_end, &range_metas, &mut loaded)?; - - loaded.sort_by_key(|(list_id, _)| { - list_ids - .iter() - .position(|&requested_id| requested_id == *list_id) - .unwrap_or(usize::MAX) - }); - Ok(loaded) - } - - fn read_coalesced_graph_list_range( - &mut self, - range_start: u64, - range_end: u64, - metas: &[ListPayloadMeta], - loaded: &mut Vec<(usize, GraphList)>, - ) -> io::Result<()> { - let byte_len = usize::try_from(range_end.checked_sub(range_start).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "coalesced IVF-HNSW-FLAT read range is invalid", - ) - })?) - .map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidData, - "coalesced IVF-HNSW-FLAT read range exceeds usize", - ) - })?; - let mut payload = vec![0u8; byte_len]; - self.reader - .pread(&mut [ReadRequest::new(range_start, &mut payload)])?; - - for &meta in metas { - let start = usize::try_from(meta.offset - range_start).map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidData, - "coalesced IVF-HNSW-FLAT payload offset exceeds usize", - ) - })?; - let end = start.checked_add(meta.payload_len).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "coalesced IVF-HNSW-FLAT payload slice overflows", - ) - })?; - loaded.push(( - meta.list_id, - self.decode_graph_list_payload(meta, &payload[start..end])?, - )); - } - Ok(()) - } - - fn list_payload_meta(&self, list_id: usize) -> io::Result> { - if list_id >= self.nlist { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("list_id {} out of range (nlist={})", list_id, self.nlist), - )); - } - let count = self.list_counts[list_id] as usize; - if count == 0 { - return Ok(None); - } - - let offset = checked_list_offset(self.list_offsets[list_id], list_id)?; - let graph_bytes_len = self.list_graph_bytes_lens[list_id] as usize; - if graph_bytes_len == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("list {} is missing HNSW graph", list_id), - )); - } - let payload_len = self.list_payload_bytes_lens[list_id] as usize; - if payload_len == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("list {} is missing payload length", list_id), - )); - } - let minimum_payload_len = 12usize.checked_add(graph_bytes_len).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "IVF-HNSW-FLAT minimum payload length overflow", - ) - })?; - if payload_len < minimum_payload_len { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "list {} payload length {} is shorter than expected {}", - list_id, payload_len, minimum_payload_len - ), - )); - } - Ok(Some(ListPayloadMeta { - list_id, - offset, - count, - payload_len, - })) - } - - fn decode_graph_list_payload( - &self, - meta: ListPayloadMeta, - payload: &[u8], - ) -> io::Result { - if payload.len() != meta.payload_len { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "list {} payload length {} does not match expected {}", - meta.list_id, - payload.len(), - meta.payload_len - ), - )); - } - let base_header_len = 12usize; - if payload.len() < base_header_len { - return Err(io::Error::new( - io::ErrorKind::UnexpectedEof, - format!("list {} has truncated ID header", meta.list_id), - )); - } - let base_id = i64::from_le_bytes(payload[0..8].try_into().unwrap()); - let id_bytes_len = i32::from_le_bytes(payload[8..12].try_into().unwrap()); - if id_bytes_len < 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "negative id_bytes_len {} at list {}", - id_bytes_len, meta.list_id - ), - )); - } - let id_bytes_len = id_bytes_len as usize; - let ids_end = base_header_len.checked_add(id_bytes_len).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "IVF-HNSW-FLAT ID payload length overflow", - ) - })?; - let vector_bytes_len = checked_list_bytes( - meta.count, - self.d.checked_mul(4).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "IVF-HNSW-FLAT bytes per vector overflow", - ) - })?, - )?; - let vectors_end = ids_end.checked_add(vector_bytes_len).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "IVF-HNSW-FLAT vector payload length overflow", - ) - })?; - if vectors_end > payload.len() { - return Err(io::Error::new( - io::ErrorKind::UnexpectedEof, - format!("list {} has truncated vector payload", meta.list_id), - )); - } - let ids = decode_delta_varint_ids(base_id, &payload[base_header_len..ids_end], meta.count)?; - let vectors = bytes_to_f32_vec(&payload[ids_end..vectors_end])?; - let graph = decode_graph( - &payload[vectors_end..], - vectors.clone(), - meta.count, - self.d, - self.metric, - self.hnsw_params, - )?; - let graph = graph.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("list {} is missing HNSW graph", meta.list_id), - ) - })?; - Ok(GraphList { ids, graph }) - } - - pub fn search( - &mut self, - query: &[f32], - k: usize, - nprobe: usize, - ef_search: usize, - ) -> io::Result<(Vec, Vec)> { - self.search_with_filter(query, k, nprobe, ef_search, None) - } - - pub fn search_with_filter( - &mut self, - query: &[f32], - k: usize, - nprobe: usize, - ef_search: usize, - filter: Option<&dyn RowIdFilter>, - ) -> io::Result<(Vec, Vec)> { - self.ensure_loaded()?; - if query.len() != self.d { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "query length {} does not match index dimension {}", - query.len(), - self.d - ), - )); - } - if k == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "k must be greater than 0", - )); - } - if nprobe == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "nprobe must be greater than 0", - )); - } - - let q = preprocess_vectors(query, 1, self.d, self.metric); - let (probe_indices, _) = - kmeans::find_topk(&q, &self.quantizer_centroids, self.nlist, self.d, nprobe); - let mut loaded_lists = Vec::with_capacity(probe_indices.len()); - for (_, list) in self.read_graph_lists_coalesced(&probe_indices)? { - loaded_lists.push(list); - } - let search_lists: Vec<_> = loaded_lists - .iter() - .map(|list| HnswSearchList { - ids: list.ids.as_slice(), - graph: Some(&list.graph), - payload: list, - }) - .collect(); - let sorted = search_hnsw_lists(&q, &search_lists, k, ef_search, filter, |list, heap| { - let list = list.payload; - scan_flat_list( - &q, - &list.ids, - list.graph.vectors(), - self.d, - self.metric, - filter, - heap, - ); - }); - let mut labels: Vec = sorted.iter().map(|&(_, id)| id).collect(); - let mut distances: Vec = sorted.iter().map(|&(dist, _)| dist).collect(); - labels.resize(k, -1); - distances.resize(k, f32::MAX); - Ok((labels, distances)) - } - - pub fn search_with_roaring_filter( - &mut self, - query: &[f32], - k: usize, - nprobe: usize, - ef_search: usize, - roaring_filter_bytes: &[u8], - ) -> io::Result<(Vec, Vec)> { - let filter = decode_roaring_filter(roaring_filter_bytes)?; - self.search_with_filter(query, k, nprobe, ef_search, Some(&filter)) - } -} - -pub fn search_batch_ivfhnswflat_reader( - reader: &mut IVFHNSWFlatIndexReader, - queries: &[f32], - nq: usize, - k: usize, - nprobe: usize, - ef_search: usize, -) -> io::Result<(Vec, Vec)> { - search_batch_ivfhnswflat_reader_filter(reader, queries, nq, k, nprobe, ef_search, None) -} - -pub fn search_batch_ivfhnswflat_reader_filter( - reader: &mut IVFHNSWFlatIndexReader, - queries: &[f32], - nq: usize, - k: usize, - nprobe: usize, - ef_search: usize, - filter: Option<&dyn RowIdFilter>, -) -> io::Result<(Vec, Vec)> { - reader.ensure_loaded()?; - validate_search_inputs(queries, nq, reader.d, k, nprobe)?; - - let processed = preprocess_vectors(queries, nq, reader.d, reader.metric); - let (all_probe_indices, _) = kmeans::find_topk_batch( - &processed, - nq, - &reader.quantizer_centroids, - reader.nlist, - reader.d, - nprobe, - ); - - let mut list_to_queries = vec![Vec::new(); reader.nlist]; - let mut unique_lists = Vec::new(); - for (qi, probe_indices) in all_probe_indices.iter().enumerate() { - for &list_id in probe_indices { - if list_to_queries[list_id].is_empty() { - unique_lists.push(list_id); - } - list_to_queries[list_id].push(qi); - } - } - - let mut heaps: Vec = (0..nq).map(|_| TopKHeap::new(k)).collect(); - let mut search_workspace = HnswSearchWorkspace::new(ef_search.max(k)); - let mut query_filtered_counts = vec![0usize; nq]; - let mut loaded_lists = Vec::with_capacity(unique_lists.len()); - for (list_id, list) in reader.read_graph_lists_coalesced(&unique_lists)? { - if let Some(f) = filter { - let filtered = list.ids.iter().filter(|&&id| f.contains(id)).count(); - for &qi in &list_to_queries[list_id] { - query_filtered_counts[qi] = query_filtered_counts[qi] - .checked_add(filtered) - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "filtered vector count overflows usize", - ) - })?; - } - } - loaded_lists.push(LoadedBatchList { - query_ids: std::mem::take(&mut list_to_queries[list_id]), - ids: list.ids, - graph: list.graph, - }); - } - - for list in &loaded_lists { - for &qi in &list.query_ids { - let query = &processed[qi * reader.d..(qi + 1) * reader.d]; - let force_flat_scan = filter - .map(|_| query_filtered_counts[qi] <= ef_search.max(k)) - .unwrap_or(false); - if force_flat_scan { - scan_flat_list( - query, - &list.ids, - list.graph.vectors(), - reader.d, - reader.metric, - filter, - &mut heaps[qi], - ); - } else { - let local_results = list.graph.search_with_reusable_workspace( - query, - ef_search.max(k), - ef_search.max(k), - &mut search_workspace, - ); - for &(local_id, dist) in local_results { - let row_id = list.ids[local_id]; - if filter.map(|f| f.contains(row_id)).unwrap_or(true) { - heaps[qi].push(dist, row_id); - } - } - } - } - } - if filter.is_some() { - for list in &loaded_lists { - for &qi in &list.query_ids { - if heaps[qi].len() >= k { - continue; - } - let query = &processed[qi * reader.d..(qi + 1) * reader.d]; - scan_flat_list( - query, - &list.ids, - list.graph.vectors(), - reader.d, - reader.metric, - filter, - &mut heaps[qi], - ); - } - } - } - - let mut result_ids = vec![-1i64; nq * k]; - let mut result_dists = vec![f32::MAX; nq * k]; - for qi in 0..nq { - let sorted = std::mem::replace(&mut heaps[qi], TopKHeap::new(0)).into_sorted(); - let base = qi * k; - for (i, &(dist, id)) in sorted.iter().enumerate() { - result_ids[base + i] = id; - result_dists[base + i] = dist; - } - } - - Ok((result_ids, result_dists)) -} - -pub fn search_batch_ivfhnswflat_reader_roaring_filter( - reader: &mut IVFHNSWFlatIndexReader, - queries: &[f32], - nq: usize, - k: usize, - nprobe: usize, - ef_search: usize, - roaring_filter_bytes: &[u8], -) -> io::Result<(Vec, Vec)> { - let filter = decode_roaring_filter(roaring_filter_bytes)?; - search_batch_ivfhnswflat_reader_filter(reader, queries, nq, k, nprobe, ef_search, Some(&filter)) -} - -struct GraphList { - ids: Vec, - graph: HnswGraph, -} - -#[derive(Clone, Copy)] -struct ListPayloadMeta { - list_id: usize, - offset: u64, - count: usize, - payload_len: usize, -} - -impl ListPayloadMeta { - fn end_offset(self) -> io::Result { - self.offset - .checked_add(self.payload_len as u64) - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "IVF-HNSW-FLAT list payload offset overflows u64", - ) - }) - } -} - -fn should_coalesce_gap( - gap: u64, - range_start: u64, - next_range_end: u64, - current_payload_bytes: usize, - next_payload_bytes: usize, -) -> bool { - if gap > MAX_COALESCED_READ_GAP_BYTES { - return false; - } - let Some(requested_bytes) = current_payload_bytes.checked_add(next_payload_bytes) else { - return false; - }; - let Some(range_bytes) = next_range_end.checked_sub(range_start) else { - return false; - }; - range_bytes <= requested_bytes.saturating_mul(2) as u64 -} - -struct LoadedBatchList { - query_ids: Vec, - ids: Vec, - graph: HnswGraph, -} - -fn scan_flat_list( - query: &[f32], - ids: &[i64], - vectors: &[f32], - d: usize, - metric: MetricType, - filter: Option<&dyn RowIdFilter>, - heap: &mut TopKHeap, -) { - let distance = QueryDistance::new(query, metric); - for (local_id, &row_id) in ids.iter().enumerate() { - if filter.map(|f| !f.contains(row_id)).unwrap_or(false) { - continue; - } - let vector = &vectors[local_id * d..(local_id + 1) * d]; - heap.push(distance.distance_to(vector, None), row_id); - } -} - -fn validate_index_shape(index: &IVFHNSWFlatIndex) -> io::Result<()> { - if index.flat.d == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "dimension must be greater than 0", - )); - } - if index.flat.nlist == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "nlist must be greater than 0", - )); - } - if index.flat.ids.len() != index.flat.nlist - || index.flat.vectors.len() != index.flat.nlist - || index.graphs.len() != index.flat.nlist - { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "IVF-HNSW-FLAT list storage does not match nlist", - )); - } - let centroid_len = checked_section_size(index.flat.nlist, index.flat.d)?; - if index.flat.quantizer_centroids.len() != centroid_len { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "centroid length {} does not match nlist*d {}", - index.flat.quantizer_centroids.len(), - centroid_len - ), - )); - } - for list_id in 0..index.flat.nlist { - let count = index.flat.ids[list_id].len(); - let expected_vector_len = count.checked_mul(index.flat.d).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "IVF-HNSW-FLAT vector length overflow", - ) - })?; - if index.flat.vectors[list_id].len() != expected_vector_len { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "list {} vector length {} does not match ids*d {}", - list_id, - index.flat.vectors[list_id].len(), - expected_vector_len - ), - )); - } - match &index.graphs[list_id] { - Some(graph) - if graph.len() == count - && graph.vectors() == index.flat.vectors[list_id].as_slice() => {} - Some(_) => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("list {} graph does not match vector storage", list_id), - )); - } - None if count == 0 => {} - None => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("list {} is missing HNSW graph", list_id), - )); - } - } - } - Ok(()) -} - -struct SortedFlatGraphList { - ids: Vec, - id_bytes: Vec, - vectors: Vec, - graph_bytes: Vec, -} - -fn build_sorted_flat_graph_list( - index: &IVFHNSWFlatIndex, - list_id: usize, -) -> io::Result { - let count = index.flat.ids[list_id].len(); - if count == 0 { - return Ok(SortedFlatGraphList { - ids: Vec::new(), - id_bytes: Vec::new(), - vectors: Vec::new(), - graph_bytes: Vec::new(), - }); - } - - let mut order: Vec = (0..count).collect(); - order.sort_by_key(|&idx| index.flat.ids[list_id][idx]); - - let ids: Vec = order - .iter() - .map(|&idx| index.flat.ids[list_id][idx]) - .collect(); - let (_, id_bytes) = encode_delta_varint_ids(&ids); - - let mut vectors = Vec::with_capacity(count * index.flat.d); - for &idx in &order { - vectors.extend_from_slice( - &index.flat.vectors[list_id][idx * index.flat.d..(idx + 1) * index.flat.d], - ); - } - let old_to_new = old_to_new_order(&order); - let source_graph = index.graphs[list_id].as_ref().ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - format!("list {} is missing HNSW graph", list_id), - ) - })?; - let graph = reorder_graph( - source_graph, - &order, - &old_to_new, - vectors.clone(), - index.flat.d, - index.flat.metric, - index.hnsw_params, - )?; - let graph_bytes = encode_graph(Some(&graph))?; - - Ok(SortedFlatGraphList { - ids, - id_bytes, - vectors, - graph_bytes, - }) -} - -fn old_to_new_order(order: &[usize]) -> Vec { - let mut old_to_new = vec![0; order.len()]; - for (new_idx, &old_idx) in order.iter().enumerate() { - old_to_new[old_idx] = new_idx; - } - old_to_new -} - -fn reorder_graph( - graph: &HnswGraph, - order: &[usize], - old_to_new: &[usize], - vectors: Vec, - d: usize, - metric: MetricType, - hnsw_params: HnswBuildParams, -) -> io::Result { - let levels: Vec = order - .iter() - .map(|&old_idx| graph.levels()[old_idx]) - .collect(); - let neighbors: Vec>> = order - .iter() - .map(|&old_idx| { - graph.neighbors()[old_idx] - .iter() - .map(|level_neighbors| { - level_neighbors - .iter() - .map(|&neighbor| old_to_new[neighbor]) - .collect() - }) - .collect() - }) - .collect(); - HnswGraph::from_parts( - vectors, - order.len(), - d, - metric, - levels, - neighbors, - old_to_new[graph.entry_point()], - graph.max_observed_level(), - hnsw_params, - ) -} - -fn list_payload_len( - count: usize, - d: usize, - id_bytes_len: usize, - graph_bytes_len: usize, -) -> io::Result { - let id_bytes = 12usize.checked_add(id_bytes_len).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "IVF-HNSW-FLAT ID payload length overflow", - ) - })?; - let vector_bytes = checked_list_bytes( - count, - d.checked_mul(4).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "IVF-HNSW-FLAT bytes per vector overflow", - ) - })?, - )?; - id_bytes - .checked_add(vector_bytes) - .and_then(|len| len.checked_add(graph_bytes_len)) - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "IVF-HNSW-FLAT list payload overflow", - ) - }) -} - -#[cfg(test)] -mod tests { - use super::REQUIRED_FLAGS; - use crate::distance::MetricType; - use crate::hnsw::{HnswBuildParams, HnswGraph}; - use crate::index_io_util::{decode_graph, encode_graph, encode_graph_u32_for_size_estimate}; - use crate::io::{PosWriter, ReadRequest, SeekRead}; - use crate::ivfhnswflat::IVFHNSWFlatIndex; - use crate::ivfhnswflat_io::{ - search_batch_ivfhnswflat_reader, search_batch_ivfhnswflat_reader_roaring_filter, - write_ivfhnswflat_index, IVFHNSWFlatIndexReader, IVF_HNSW_FLAT_HEADER_SIZE, - IVF_HNSW_FLAT_MAGIC, IVF_HNSW_FLAT_VERSION, - }; - use roaring::RoaringTreemap; - use std::io; - use std::io::Cursor; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::Arc; - - #[test] - fn test_ivfhnswflat_reader_rejects_missing_required_flags() { - let mut buf = vec![0u8; IVF_HNSW_FLAT_HEADER_SIZE]; - buf[0..4].copy_from_slice(&IVF_HNSW_FLAT_MAGIC.to_le_bytes()); - buf[4..8].copy_from_slice(&IVF_HNSW_FLAT_VERSION.to_le_bytes()); - buf[8..12].copy_from_slice(&2i32.to_le_bytes()); - buf[12..16].copy_from_slice(&1i32.to_le_bytes()); - buf[16..20].copy_from_slice(&(MetricType::L2 as u32).to_le_bytes()); - buf[20..28].copy_from_slice(&0i64.to_le_bytes()); - buf[28..32].copy_from_slice(&2i32.to_le_bytes()); - buf[32..36].copy_from_slice(&8i32.to_le_bytes()); - buf[36..40].copy_from_slice(&3i32.to_le_bytes()); - buf[40..44].copy_from_slice(&0u32.to_le_bytes()); - - let err = match IVFHNSWFlatIndexReader::open(Cursor::new(buf)) { - Ok(_) => panic!("missing required flags should be rejected"), - Err(err) => err, - }; - assert!(err - .to_string() - .contains("requires delta-varint IDs and graph v1")); - } - - #[test] - fn test_ivfhnswflat_reader_rejects_unknown_flags() { - let mut buf = vec![0u8; IVF_HNSW_FLAT_HEADER_SIZE]; - buf[0..4].copy_from_slice(&IVF_HNSW_FLAT_MAGIC.to_le_bytes()); - buf[4..8].copy_from_slice(&IVF_HNSW_FLAT_VERSION.to_le_bytes()); - buf[8..12].copy_from_slice(&2i32.to_le_bytes()); - buf[12..16].copy_from_slice(&1i32.to_le_bytes()); - buf[16..20].copy_from_slice(&(MetricType::L2 as u32).to_le_bytes()); - buf[20..28].copy_from_slice(&0i64.to_le_bytes()); - buf[28..32].copy_from_slice(&2i32.to_le_bytes()); - buf[32..36].copy_from_slice(&8i32.to_le_bytes()); - buf[36..40].copy_from_slice(&3i32.to_le_bytes()); - buf[40..44].copy_from_slice(&(REQUIRED_FLAGS | (1 << 31)).to_le_bytes()); - - let err = match IVFHNSWFlatIndexReader::open(Cursor::new(buf)) { - Ok(_) => panic!("unknown flags should be rejected"), - Err(err) => err, - }; - assert!(err.to_string().contains("Unsupported IVF_HNSW_FLAT flags")); - } - - #[test] - fn test_ivfhnswflat_reader_rejects_nonzero_reserved_bytes() { - let mut buf = vec![0u8; IVF_HNSW_FLAT_HEADER_SIZE]; - buf[0..4].copy_from_slice(&IVF_HNSW_FLAT_MAGIC.to_le_bytes()); - buf[4..8].copy_from_slice(&IVF_HNSW_FLAT_VERSION.to_le_bytes()); - buf[8..12].copy_from_slice(&2i32.to_le_bytes()); - buf[12..16].copy_from_slice(&1i32.to_le_bytes()); - buf[16..20].copy_from_slice(&(MetricType::L2 as u32).to_le_bytes()); - buf[20..28].copy_from_slice(&0i64.to_le_bytes()); - buf[28..32].copy_from_slice(&2i32.to_le_bytes()); - buf[32..36].copy_from_slice(&8i32.to_le_bytes()); - buf[36..40].copy_from_slice(&3i32.to_le_bytes()); - buf[40..44].copy_from_slice(&REQUIRED_FLAGS.to_le_bytes()); - buf[44] = 1; - - let err = match IVFHNSWFlatIndexReader::open(Cursor::new(buf)) { - Ok(_) => panic!("non-zero reserved bytes should be rejected"), - Err(err) => err, - }; - assert!(err.to_string().contains("reserved bytes must be zero")); - } - - #[test] - fn test_ivfhnswflat_write_read_search_roundtrip() { - let d = 4; - let nlist = 4; - let n = 128; - let data: Vec = (0..n) - .flat_map(|i| { - let cluster = (i % nlist) as f32 * 100.0; - [cluster + i as f32 * 0.01, 1.0, 2.0, 3.0] - }) - .collect(); - let ids: Vec = (1000..1000 + n as i64).collect(); - - let mut index = IVFHNSWFlatIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, n); - index.add(&data, &ids, n); - index.build_graphs().unwrap(); - - let query = &data[7 * d..8 * d]; - let mut expected_distances = vec![0.0; 5]; - let mut expected_labels = vec![0; 5]; - index.search( - query, - 1, - 5, - nlist, - 32, - &mut expected_distances, - &mut expected_labels, - ); - - let mut buf = Vec::new(); - let mut writer = PosWriter::new(&mut buf); - write_ivfhnswflat_index(&index, &mut writer).unwrap(); - - let mut reader = IVFHNSWFlatIndexReader::open(Cursor::new(buf)).unwrap(); - let (labels, distances) = reader.search(query, 5, nlist, 32).unwrap(); - - assert_eq!(labels, expected_labels); - assert_eq!(distances, expected_distances); - } - - #[test] - fn test_ivfhnswflat_write_read_search_roundtrip_cosine() { - let d = 3; - let nlist = 2; - let data = vec![1.0, 0.0, 0.0, 0.9, 0.1, 0.0, 0.0, 1.0, 0.0, 0.0, 0.9, 0.1]; - let ids = vec![10, 11, 12, 13]; - - let mut index = - IVFHNSWFlatIndex::new(d, nlist, MetricType::Cosine, HnswBuildParams::default()); - index.train(&data, 4); - index.add(&data, &ids, 4); - index.build_graphs().unwrap(); - - let query = [9.0, 1.0, 0.0]; - let mut expected_distances = vec![0.0; 2]; - let mut expected_labels = vec![0; 2]; - index.search( - &query, - 1, - 2, - nlist, - 8, - &mut expected_distances, - &mut expected_labels, - ); - - let mut buf = Vec::new(); - let mut writer = PosWriter::new(&mut buf); - write_ivfhnswflat_index(&index, &mut writer).unwrap(); - - let mut reader = IVFHNSWFlatIndexReader::open(Cursor::new(buf)).unwrap(); - let (labels, distances) = reader.search(&query, 2, nlist, 8).unwrap(); - - assert_eq!(labels, expected_labels); - assert_eq!(distances, expected_distances); - } - - #[test] - fn test_ivfhnswflat_reader_filter_backfills_exact_results() { - use std::collections::HashSet; - - let d = 2; - let nlist = 1; - let n = 128; - let mut data = Vec::with_capacity(n * d); - for i in 0..n { - data.push(i as f32); - data.push(0.0); - } - let ids: Vec = (0..n as i64).collect(); - - let mut index = IVFHNSWFlatIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, n); - index.add(&data, &ids, n); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - let mut writer = PosWriter::new(&mut buf); - write_ivfhnswflat_index(&index, &mut writer).unwrap(); - - let filter: HashSet = (0..n as i64).filter(|id| id % 2 == 0).collect(); - let mut reader = IVFHNSWFlatIndexReader::open(Cursor::new(buf)).unwrap(); - let (labels, _) = reader - .search_with_filter(&[127.0, 0.0], 10, 1, 1, Some(&filter)) - .unwrap(); - - assert_eq!( - labels, - vec![126, 124, 122, 120, 118, 116, 114, 112, 110, 108] - ); - } - - #[test] - fn test_ivfhnswflat_batch_reader_matches_single_reader_search() { - let d = 4; - let nlist = 4; - let n = 128; - let data: Vec = (0..n) - .flat_map(|i| { - let cluster = (i % nlist) as f32 * 100.0; - [cluster + i as f32 * 0.01, 1.0, 2.0, 3.0] - }) - .collect(); - let ids: Vec = (1000..1000 + n as i64).collect(); - - let mut index = IVFHNSWFlatIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, n); - index.add(&data, &ids, n); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - let mut writer = PosWriter::new(&mut buf); - write_ivfhnswflat_index(&index, &mut writer).unwrap(); - - let queries = [&data[7 * d..8 * d], &data[63 * d..64 * d]].concat(); - let k = 5; - let nprobe = 3; - let ef_search = 32; - let mut batch_reader = IVFHNSWFlatIndexReader::open(Cursor::new(buf.clone())).unwrap(); - let (batch_labels, batch_distances) = - search_batch_ivfhnswflat_reader(&mut batch_reader, &queries, 2, k, nprobe, ef_search) - .unwrap(); - - for qi in 0..2 { - let mut single_reader = IVFHNSWFlatIndexReader::open(Cursor::new(buf.clone())).unwrap(); - let query = &queries[qi * d..(qi + 1) * d]; - let (single_labels, single_distances) = - single_reader.search(query, k, nprobe, ef_search).unwrap(); - assert_eq!(&batch_labels[qi * k..(qi + 1) * k], single_labels); - assert_eq!(&batch_distances[qi * k..(qi + 1) * k], single_distances); - } - } - - #[test] - fn test_ivfhnswflat_batch_reader_coalesces_contiguous_list_reads() { - let d = 4; - let nlist = 4; - let n = 128; - let data: Vec = (0..n) - .flat_map(|i| { - let cluster = (i % nlist) as f32 * 100.0; - [cluster + i as f32 * 0.01, 1.0, 2.0, 3.0] - }) - .collect(); - let ids: Vec = (1000..1000 + n as i64).collect(); - - let mut index = IVFHNSWFlatIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, n); - index.add(&data, &ids, n); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - write_ivfhnswflat_index(&index, &mut PosWriter::new(&mut buf)).unwrap(); - - let pread_count = Arc::new(AtomicUsize::new(0)); - let cursor = CountingPreadCursor::new(buf, Arc::clone(&pread_count)); - let mut reader = IVFHNSWFlatIndexReader::open(cursor).unwrap(); - reader.ensure_loaded().unwrap(); - pread_count.store(0, Ordering::SeqCst); - let queries = data[0..d].to_vec(); - - search_batch_ivfhnswflat_reader(&mut reader, &queries, 1, 5, nlist, 32).unwrap(); - - assert_eq!(pread_count.load(Ordering::SeqCst), 1); - } - - #[test] - fn test_ivfhnswflat_reader_coalesces_small_gap_between_requested_lists() { - let d = 4; - let nlist = 4; - let n = 128; - let data: Vec = (0..n) - .flat_map(|i| { - let cluster = (i % nlist) as f32 * 100.0; - [cluster + i as f32 * 0.01, 1.0, 2.0, 3.0] - }) - .collect(); - let ids: Vec = (1000..1000 + n as i64).collect(); - - let mut index = IVFHNSWFlatIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, n); - index.add(&data, &ids, n); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - write_ivfhnswflat_index(&index, &mut PosWriter::new(&mut buf)).unwrap(); - - let pread_count = Arc::new(AtomicUsize::new(0)); - let cursor = CountingPreadCursor::new(buf, Arc::clone(&pread_count)); - let mut reader = IVFHNSWFlatIndexReader::open(cursor).unwrap(); - reader.ensure_loaded().unwrap(); - assert!(reader.list_counts[..3].iter().all(|&count| count > 0)); - pread_count.store(0, Ordering::SeqCst); - - let lists = reader.read_graph_lists_coalesced(&[0, 2]).unwrap(); - - assert_eq!(lists.len(), 2); - assert_eq!(pread_count.load(Ordering::SeqCst), 1); - } - - #[test] - fn test_ivfhnswflat_batch_reader_search_with_roaring_filter_bytes() { - let d = 2; - let nlist = 1; - let data = vec![0.0, 0.0, 0.1, 0.0, 10.0, 10.0]; - let ids = vec![10, 11, 12]; - - let mut index = IVFHNSWFlatIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, 3); - index.add(&data, &ids, 3); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - let mut writer = PosWriter::new(&mut buf); - write_ivfhnswflat_index(&index, &mut writer).unwrap(); - - let mut allowed = RoaringTreemap::new(); - allowed.insert(12); - let mut filter_bytes = Vec::new(); - allowed.serialize_into(&mut filter_bytes).unwrap(); - - let mut reader = IVFHNSWFlatIndexReader::open(Cursor::new(buf)).unwrap(); - let queries = vec![0.0, 0.0, 10.0, 10.0]; - let (labels, distances) = search_batch_ivfhnswflat_reader_roaring_filter( - &mut reader, - &queries, - 2, - 2, - 1, - 4, - &filter_bytes, - ) - .unwrap(); - - assert_eq!(labels, vec![12, -1, 12, -1]); - assert_eq!(distances, vec![200.0, f32::MAX, 0.0, f32::MAX]); - } - - #[test] - fn test_ivfhnswflat_batch_reader_validates_inputs() { - let d = 2; - let nlist = 1; - let data = vec![0.0, 0.0, 1.0, 0.0]; - let ids = vec![10, 11]; - - let mut index = IVFHNSWFlatIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, 2); - index.add(&data, &ids, 2); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - let mut writer = PosWriter::new(&mut buf); - write_ivfhnswflat_index(&index, &mut writer).unwrap(); - - let mut reader = IVFHNSWFlatIndexReader::open(Cursor::new(buf.clone())).unwrap(); - assert!(search_batch_ivfhnswflat_reader(&mut reader, &[], 0, 1, 1, 4).is_err()); - - let mut reader = IVFHNSWFlatIndexReader::open(Cursor::new(buf.clone())).unwrap(); - assert!(search_batch_ivfhnswflat_reader(&mut reader, &[0.0], 1, 1, 1, 4).is_err()); - - let mut reader = IVFHNSWFlatIndexReader::open(Cursor::new(buf.clone())).unwrap(); - assert!(search_batch_ivfhnswflat_reader(&mut reader, &[0.0, 0.0], 1, 0, 1, 4).is_err()); - - let mut reader = IVFHNSWFlatIndexReader::open(Cursor::new(buf)).unwrap(); - assert!(search_batch_ivfhnswflat_reader(&mut reader, &[0.0, 0.0], 1, 1, 0, 4).is_err()); - } - - #[test] - fn test_ivfhnswflat_reader_filter_reads_probed_list_once() { - use std::collections::HashSet; - - let d = 2; - let nlist = 1; - let data = vec![0.0, 0.0, 1.0, 0.0, 2.0, 0.0, 3.0, 0.0]; - let ids = vec![10, 11, 12, 13]; - - let mut index = IVFHNSWFlatIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, 4); - index.add(&data, &ids, 4); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - let mut writer = PosWriter::new(&mut buf); - write_ivfhnswflat_index(&index, &mut writer).unwrap(); - - let pread_count = Arc::new(AtomicUsize::new(0)); - let cursor = CountingPreadCursor::new(buf, Arc::clone(&pread_count)); - let filter: HashSet = [10, 12].into_iter().collect(); - let mut reader = IVFHNSWFlatIndexReader::open(cursor).unwrap(); - reader.ensure_loaded().unwrap(); - pread_count.store(0, Ordering::SeqCst); - - reader - .search_with_filter(&[0.0, 0.0], 2, 1, 1, Some(&filter)) - .unwrap(); - - assert_eq!(pread_count.load(Ordering::SeqCst), 1); - } - - #[test] - fn test_ivfhnswflat_reader_search_coalesces_contiguous_list_reads() { - let d = 4; - let nlist = 4; - let n = 128; - let data: Vec = (0..n) - .flat_map(|i| { - let cluster = (i % nlist) as f32 * 100.0; - [cluster + i as f32 * 0.01, 1.0, 2.0, 3.0] - }) - .collect(); - let ids: Vec = (1000..1000 + n as i64).collect(); - - let mut index = IVFHNSWFlatIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, n); - index.add(&data, &ids, n); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - write_ivfhnswflat_index(&index, &mut PosWriter::new(&mut buf)).unwrap(); - - let pread_count = Arc::new(AtomicUsize::new(0)); - let cursor = CountingPreadCursor::new(buf, Arc::clone(&pread_count)); - let mut reader = IVFHNSWFlatIndexReader::open(cursor).unwrap(); - reader.ensure_loaded().unwrap(); - pread_count.store(0, Ordering::SeqCst); - - reader.search(&data[0..d], 5, nlist, 32).unwrap(); - - assert_eq!(pread_count.load(Ordering::SeqCst), 1); - } - - #[test] - fn test_ivfhnswflat_reader_rejects_truncated_graph_section() { - let d = 2; - let nlist = 1; - let data = vec![0.0, 0.0, 1.0, 0.0, 2.0, 0.0, 3.0, 0.0]; - let ids = vec![10, 11, 12, 13]; - - let mut index = IVFHNSWFlatIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, 4); - index.add(&data, &ids, 4); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - let mut writer = PosWriter::new(&mut buf); - write_ivfhnswflat_index(&index, &mut writer).unwrap(); - buf.pop(); - - let mut reader = IVFHNSWFlatIndexReader::open(Cursor::new(buf)).unwrap(); - let err = reader.search(&[0.0, 0.0], 2, 1, 4).unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::UnexpectedEof); - } - - #[test] - fn test_ivfhnswflat_reader_rejects_missing_graph_section() { - let d = 2; - let nlist = 1; - let data = vec![0.0, 0.0, 1.0, 0.0, 2.0, 0.0, 3.0, 0.0]; - let ids = vec![10, 11, 12, 13]; - - let mut index = IVFHNSWFlatIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, 4); - index.add(&data, &ids, 4); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - let mut writer = PosWriter::new(&mut buf); - write_ivfhnswflat_index(&index, &mut writer).unwrap(); - let graph_len_offset = IVF_HNSW_FLAT_HEADER_SIZE + nlist * d * 4 + 8 + 4; - buf[graph_len_offset..graph_len_offset + 4].copy_from_slice(&0i32.to_le_bytes()); - - let mut reader = IVFHNSWFlatIndexReader::open(Cursor::new(buf)).unwrap(); - let err = reader.search(&[0.0, 0.0], 2, 1, 4).unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::InvalidData); - assert!(err.to_string().contains("missing HNSW graph")); - } - - #[test] - fn test_ivfhnswflat_graph_delta_varint_reduces_graph_bytes() { - let d = 2; - let n = 128; - let data: Vec = (0..n).flat_map(|i| [i as f32, 0.0]).collect(); - let params = HnswBuildParams { - m: 8, - ef_construction: 32, - max_level: 4, - }; - let graph = HnswGraph::build(&data, n, d, MetricType::L2, params).unwrap(); - - let fixed = encode_graph_u32_for_size_estimate(&graph).unwrap(); - let compressed = encode_graph(Some(&graph)).unwrap(); - - assert!(compressed.len() < fixed.len()); - assert!(compressed.len() * 2 < fixed.len()); - } - - #[test] - #[ignore] - fn print_ivfhnswflat_graph_delta_varint_size_report() { - let d = 8; - for n in [128usize, 1_024, 4_096] { - let data: Vec = (0..n) - .flat_map(|i| { - (0..d).map(move |j| { - let bucket = (i % 64) as f32; - bucket * 0.01 + i as f32 * 0.0001 + j as f32 * 0.001 - }) - }) - .collect(); - let params = HnswBuildParams { - m: 16, - ef_construction: 64, - max_level: 5, - }; - let graph = HnswGraph::build(&data, n, d, MetricType::L2, params).unwrap(); - let fixed = encode_graph_u32_for_size_estimate(&graph).unwrap(); - let compressed = encode_graph(Some(&graph)).unwrap(); - println!( - "n={n}, fixed_u32={} bytes, delta_varint={} bytes, saved={:.1}%", - fixed.len(), - compressed.len(), - 100.0 - (compressed.len() as f64 * 100.0 / fixed.len() as f64) - ); - } - } - - #[test] - fn test_ivfhnswflat_decoder_rejects_level_above_hnsw_max_before_allocation() { - let params = HnswBuildParams { - m: 2, - ef_construction: 8, - max_level: 3, - }; - let mut graph_bytes = Vec::new(); - append_graph_header(&mut graph_bytes); - append_u32_varint(&mut graph_bytes, 1); - append_u32_varint(&mut graph_bytes, 0); - append_u32_varint(&mut graph_bytes, 0); - append_u32_varint(&mut graph_bytes, params.max_level as u32 + 1); - - let err = - decode_graph(&graph_bytes, vec![0.0, 0.0], 1, 2, MetricType::L2, params).unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::InvalidData); - assert!(err.to_string().contains("level")); - } - - #[test] - fn test_ivfhnswflat_decoder_rejects_degree_above_hnsw_bound_before_allocation() { - let params = HnswBuildParams { - m: 2, - ef_construction: 8, - max_level: 3, - }; - let mut graph_bytes = Vec::new(); - append_graph_header(&mut graph_bytes); - append_u32_varint(&mut graph_bytes, 1); - append_u32_varint(&mut graph_bytes, 0); - append_u32_varint(&mut graph_bytes, 0); - append_u32_varint(&mut graph_bytes, 0); - append_u32_varint(&mut graph_bytes, (params.m * 2) as u32 + 1); - - let err = - decode_graph(&graph_bytes, vec![0.0, 0.0], 1, 2, MetricType::L2, params).unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::InvalidData); - assert!(err.to_string().contains("degree")); - } - - #[test] - fn test_ivfhnswflat_decoder_rejects_truncated_graph_varint() { - let params = HnswBuildParams { - m: 2, - ef_construction: 8, - max_level: 3, - }; - let mut graph_bytes = Vec::new(); - append_graph_header(&mut graph_bytes); - graph_bytes.push(0x81); - - let err = - decode_graph(&graph_bytes, vec![0.0, 0.0], 1, 2, MetricType::L2, params).unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::InvalidData); - assert!(err.to_string().contains("truncated HNSW graph varint")); - } - - #[test] - fn test_ivfhnswflat_decoder_rejects_oversized_graph_varint() { - let params = HnswBuildParams { - m: 2, - ef_construction: 8, - max_level: 3, - }; - let mut graph_bytes = Vec::new(); - append_graph_header(&mut graph_bytes); - graph_bytes.extend_from_slice(&[0xff; 10]); - - let err = - decode_graph(&graph_bytes, vec![0.0, 0.0], 1, 2, MetricType::L2, params).unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::InvalidData); - assert!(err.to_string().contains("varint exceeds u64 limit")); - } - - #[test] - fn test_ivfhnswflat_writer_requires_built_graphs() { - let d = 2; - let nlist = 1; - let data = vec![0.0, 0.0, 1.0, 0.0]; - let ids = vec![10, 11]; - - let mut index = IVFHNSWFlatIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, 2); - index.add(&data, &ids, 2); - - let mut buf = Vec::new(); - let mut writer = PosWriter::new(&mut buf); - let err = write_ivfhnswflat_index(&index, &mut writer).unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - assert!(err.to_string().contains("missing HNSW graph")); - } - - struct CountingPreadCursor { - data: Vec, - pread_count: Arc, - } - - impl CountingPreadCursor { - fn new(data: Vec, pread_count: Arc) -> Self { - Self { data, pread_count } - } - } - - impl SeekRead for CountingPreadCursor { - fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { - for range in ranges { - self.pread_count.fetch_add(1, Ordering::SeqCst); - let pos = range.pos as usize; - let end = pos.checked_add(range.buf.len()).ok_or_else(|| { - io::Error::new(io::ErrorKind::UnexpectedEof, "cursor position overflow") - })?; - if end > self.data.len() { - return Err(io::Error::new( - io::ErrorKind::UnexpectedEof, - "failed to fill whole buffer", - )); - } - range.buf.copy_from_slice(&self.data[pos..end]); - } - Ok(()) - } - } - - fn append_u32_varint(buf: &mut Vec, mut value: u32) { - while value >= 0x80 { - buf.push((value as u8 & 0x7f) | 0x80); - value >>= 7; - } - buf.push(value as u8); - } - - fn append_graph_header(buf: &mut Vec) { - buf.extend_from_slice(&0x48574752u32.to_le_bytes()); - buf.extend_from_slice(&1u32.to_le_bytes()); - buf.extend_from_slice(&1u32.to_le_bytes()); - } -} diff --git a/core/src/ivfhnswsq_io.rs b/core/src/ivfhnswsq_io.rs deleted file mode 100644 index c7416d6..0000000 --- a/core/src/ivfhnswsq_io.rs +++ /dev/null @@ -1,1696 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::distance::{preprocess_vectors, MetricType}; -use crate::hnsw::{HnswBuildParams, HnswGraph, HnswSearchWorkspace}; -use crate::hnsw_search::{search_hnsw_lists, HnswSearchList}; -use crate::index_io_util::{ - checked_list_bytes, checked_list_offset, checked_section_size, decode_delta_varint_ids, - decode_graph, decode_roaring_filter, encode_delta_varint_ids, encode_graph, read_f32_vec, - read_i32_le, read_i64_le, read_u32_le, u64_to_i64, usize_to_i32, usize_to_i64, - validate_positive_i32, validate_reserved_zero, validate_search_inputs, write_f32_slice, - write_i32_le, write_i64_le, write_u32_le, -}; -use crate::io::{PreadCursor, ReadRequest, SeekRead, SeekWrite}; -use crate::ivfhnswsq::IVFHNSWSQIndex; -use crate::ivfpq::RowIdFilter; -use crate::kmeans; -use crate::sq::{ScalarQuantizer, ScalarQuantizerDecodeLut}; -use crate::topk::TopKHeap; -use std::io; -use std::sync::Arc; - -pub const IVF_HNSW_SQ_MAGIC: u32 = 0x49485351; // "IHSQ" -pub const IVF_HNSW_SQ_VERSION: u32 = 1; -pub const IVF_HNSW_SQ_HEADER_SIZE: usize = 64; -const FLAG_DELTA_IDS: u32 = 1 << 0; -const FLAG_GRAPH_V1: u32 = 1 << 1; -const REQUIRED_FLAGS: u32 = FLAG_DELTA_IDS | FLAG_GRAPH_V1; -const SUPPORTED_FLAGS: u32 = REQUIRED_FLAGS; -const MAX_COALESCED_READ_GAP_BYTES: u64 = 1 << 20; - -pub fn write_ivfhnswsq_index(index: &IVFHNSWSQIndex, out: &mut dyn SeekWrite) -> io::Result<()> { - validate_index_shape(index)?; - let total_vectors = index.ids.iter().try_fold(0i64, |sum, ids| { - let count = usize_to_i64(ids.len(), "total vector count")?; - sum.checked_add(count).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "total vector count exceeds i64 length limit", - ) - }) - })?; - let sorted_lists: Vec = (0..index.nlist) - .map(|list_id| build_sorted_sq_graph_list(index, list_id)) - .collect::>()?; - - write_u32_le(out, IVF_HNSW_SQ_MAGIC)?; - write_u32_le(out, IVF_HNSW_SQ_VERSION)?; - write_i32_le(out, usize_to_i32(index.d, "dimension")?)?; - write_i32_le(out, usize_to_i32(index.nlist, "nlist")?)?; - write_u32_le(out, index.metric as u32)?; - write_i64_le(out, total_vectors)?; - let params = index.hnsw_params.sanitized(); - write_i32_le(out, usize_to_i32(params.m, "hnsw m")?)?; - write_i32_le( - out, - usize_to_i32(params.ef_construction, "hnsw ef_construction")?, - )?; - write_i32_le(out, usize_to_i32(params.max_level, "hnsw max_level")?)?; - let (sq_min, sq_max) = sq_global_bounds(&index.sq.mins, &index.sq.maxs); - out.write_all(&sq_min.to_le_bytes())?; - out.write_all(&sq_max.to_le_bytes())?; - write_u32_le(out, REQUIRED_FLAGS)?; - out.write_all(&[0u8; 12])?; - - write_f32_slice(out, &index.sq.mins)?; - write_f32_slice(out, &index.sq.maxs)?; - for sq in &index.list_sqs { - write_f32_slice(out, &sq.mins)?; - write_f32_slice(out, &sq.maxs)?; - } - write_f32_slice(out, &index.quantizer_centroids)?; - - let offset_table_size = index.nlist.checked_mul(24).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "IVF-HNSW-SQ offset table size overflow", - ) - })?; - let data_start = out - .pos() - .checked_add(offset_table_size as u64) - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "IVF-HNSW-SQ data start offset overflow", - ) - })?; - let mut list_offsets = vec![0i64; index.nlist]; - let mut list_counts = vec![0i32; index.nlist]; - let mut list_graph_bytes_lens = vec![0i32; index.nlist]; - let mut list_payload_bytes_lens = vec![0i64; index.nlist]; - let mut current_offset = data_start; - - for list_id in 0..index.nlist { - list_offsets[list_id] = u64_to_i64(current_offset, "list offset")?; - let count = sorted_lists[list_id].ids.len(); - list_counts[list_id] = usize_to_i32(count, "list count")?; - list_graph_bytes_lens[list_id] = - usize_to_i32(sorted_lists[list_id].graph_bytes.len(), "graph bytes")?; - if count > 0 { - let payload_len = list_payload_len( - count, - index.sq.code_size(), - sorted_lists[list_id].id_bytes.len(), - sorted_lists[list_id].graph_bytes.len(), - )?; - list_payload_bytes_lens[list_id] = usize_to_i64(payload_len, "list payload bytes")?; - current_offset = current_offset - .checked_add(payload_len as u64) - .ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "IVF-HNSW-SQ offset overflow") - })?; - } - } - - for list_id in 0..index.nlist { - write_i64_le(out, list_offsets[list_id])?; - write_i32_le(out, list_counts[list_id])?; - write_i32_le(out, list_graph_bytes_lens[list_id])?; - write_i64_le(out, list_payload_bytes_lens[list_id])?; - } - - for list_id in 0..index.nlist { - let list = &sorted_lists[list_id]; - if list.ids.is_empty() { - continue; - } - write_i64_le(out, list.ids[0])?; - write_i32_le(out, usize_to_i32(list.id_bytes.len(), "delta ID section")?)?; - out.write_all(&list.id_bytes)?; - out.write_all(&list.codes)?; - out.write_all(&list.graph_bytes)?; - } - - Ok(()) -} - -pub struct IVFHNSWSQIndexReader { - reader: R, - pub d: usize, - pub nlist: usize, - pub metric: MetricType, - pub total_vectors: i64, - pub hnsw_params: HnswBuildParams, - pub sq: ScalarQuantizer, - pub list_sqs: Vec, - pub quantizer_centroids: Vec, - pub list_offsets: Vec, - pub list_counts: Vec, - pub list_graph_bytes_lens: Vec, - pub list_payload_bytes_lens: Vec, - sq_decode_luts: Vec>, - loaded: bool, -} - -impl IVFHNSWSQIndexReader { - pub fn open(mut reader: R) -> io::Result { - let mut cursor = PreadCursor::new(&mut reader, 0); - - let magic = read_u32_le(&mut cursor)?; - if magic != IVF_HNSW_SQ_MAGIC { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("Invalid IVF_HNSW_SQ magic: 0x{:08X}", magic), - )); - } - let version = read_u32_le(&mut cursor)?; - if version != IVF_HNSW_SQ_VERSION { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("Unsupported IVF_HNSW_SQ version: {}", version), - )); - } - - let d = validate_positive_i32(read_i32_le(&mut cursor)?, "d")? as usize; - let nlist = validate_positive_i32(read_i32_le(&mut cursor)?, "nlist")? as usize; - let metric_code = read_u32_le(&mut cursor)?; - let metric = MetricType::from_code(metric_code).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("Unknown metric type: {}", metric_code), - ) - })?; - let total_vectors = read_i64_le(&mut cursor)?; - let hnsw_params = HnswBuildParams { - m: validate_positive_i32(read_i32_le(&mut cursor)?, "hnsw m")? as usize, - ef_construction: validate_positive_i32( - read_i32_le(&mut cursor)?, - "hnsw ef_construction", - )? as usize, - max_level: validate_positive_i32(read_i32_le(&mut cursor)?, "hnsw max_level")? as usize, - } - .sanitized(); - let sq_min_summary = read_f32_le(&mut cursor)?; - let sq_max_summary = read_f32_le(&mut cursor)?; - let flags = read_u32_le(&mut cursor)?; - let mut reserved = [0u8; 12]; - cursor.read_exact(&mut reserved)?; - validate_reserved_zero(&reserved, "IVF_HNSW_SQ")?; - let unknown_flags = flags & !SUPPORTED_FLAGS; - if unknown_flags != 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("Unsupported IVF_HNSW_SQ flags: 0x{:08X}", unknown_flags), - )); - } - if flags & REQUIRED_FLAGS != REQUIRED_FLAGS { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "IVF_HNSW_SQ v1 requires delta-varint IDs and graph v1", - )); - } - - let mins = read_f32_vec(&mut cursor, d)?; - let maxs = read_f32_vec(&mut cursor, d)?; - validate_sq_bounds(d, &mins, &maxs)?; - let (sq_min, sq_max) = sq_global_bounds(&mins, &maxs); - if sq_min.to_bits() != sq_min_summary.to_bits() - || sq_max.to_bits() != sq_max_summary.to_bits() - { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "SQ bounds summary does not match global SQ bounds", - )); - } - let sq = ScalarQuantizer::with_dimension_bounds(d, mins, maxs); - let mut list_sqs = Vec::with_capacity(nlist); - for _ in 0..nlist { - let mins = read_f32_vec(&mut cursor, d)?; - let maxs = read_f32_vec(&mut cursor, d)?; - validate_sq_bounds(d, &mins, &maxs)?; - list_sqs.push(ScalarQuantizer::with_dimension_bounds(d, mins, maxs)); - } - - Ok(Self { - reader, - d, - nlist, - metric, - total_vectors, - hnsw_params, - sq, - list_sqs, - quantizer_centroids: Vec::new(), - list_offsets: Vec::new(), - list_counts: Vec::new(), - list_graph_bytes_lens: Vec::new(), - list_payload_bytes_lens: Vec::new(), - sq_decode_luts: Vec::new(), - loaded: false, - }) - } - - pub fn ensure_loaded(&mut self) -> io::Result<()> { - if self.loaded { - return Ok(()); - } - - let quantizer_centroids_offset = - IVF_HNSW_SQ_HEADER_SIZE as u64 + (self.d as u64) * 8 * (self.nlist as u64 + 1); - let mut cursor = PreadCursor::new(&mut self.reader, quantizer_centroids_offset); - self.quantizer_centroids = - read_f32_vec(&mut cursor, checked_section_size(self.nlist, self.d)?)?; - self.list_offsets = vec![0; self.nlist]; - self.list_counts = vec![0; self.nlist]; - self.list_graph_bytes_lens = vec![0; self.nlist]; - self.list_payload_bytes_lens = vec![0; self.nlist]; - for list_id in 0..self.nlist { - self.list_offsets[list_id] = read_i64_le(&mut cursor)?; - let count = read_i32_le(&mut cursor)?; - if count < 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("negative list count {} at list {}", count, list_id), - )); - } - self.list_counts[list_id] = count; - let graph_bytes_len = read_i32_le(&mut cursor)?; - if graph_bytes_len < 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "negative graph_bytes_len {} at list {}", - graph_bytes_len, list_id - ), - )); - } - self.list_graph_bytes_lens[list_id] = graph_bytes_len; - let payload_bytes_len = read_i64_le(&mut cursor)?; - if payload_bytes_len < 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "negative payload_bytes_len {} at list {}", - payload_bytes_len, list_id - ), - )); - } - self.list_payload_bytes_lens[list_id] = payload_bytes_len; - } - - self.loaded = true; - Ok(()) - } - - pub fn optimize_for_search(&mut self) -> io::Result<()> { - self.ensure_loaded()?; - if self.sq_decode_luts.len() != self.nlist { - // These LUTs only help when search falls back to scanning SQ codes, - // for example filtered searches with a small candidate set. The - // normal unfiltered path searches decoded vectors in the HNSW graph. - self.sq_decode_luts = self - .list_sqs - .iter() - .map(|sq| Arc::new(sq.build_decode_lut())) - .collect(); - } - Ok(()) - } - - pub fn read_inverted_list( - &mut self, - list_id: usize, - ) -> io::Result<(Vec, Vec, Option)> { - let Some(list) = self.read_graph_list(list_id)? else { - return Ok((Vec::new(), Vec::new(), None)); - }; - Ok((list.ids, list.codes, Some(list.graph))) - } - - fn read_graph_list(&mut self, list_id: usize) -> io::Result> { - self.ensure_loaded()?; - let Some(meta) = self.list_payload_meta(list_id)? else { - return Ok(None); - }; - let mut payload = vec![0u8; meta.payload_len]; - self.reader - .pread(&mut [ReadRequest::new(meta.offset, &mut payload)])?; - - self.decode_graph_list_payload(meta, &payload).map(Some) - } - - fn read_graph_lists_coalesced( - &mut self, - list_ids: &[usize], - ) -> io::Result> { - self.ensure_loaded()?; - let mut metas = Vec::new(); - for &list_id in list_ids { - if let Some(meta) = self.list_payload_meta(list_id)? { - metas.push(meta); - } - } - if metas.is_empty() { - return Ok(Vec::new()); - } - - metas.sort_by_key(|meta| meta.offset); - let mut loaded = Vec::with_capacity(metas.len()); - let mut range_start = metas[0].offset; - let mut range_end = metas[0].end_offset()?; - let mut range_payload_bytes = metas[0].payload_len; - let mut range_metas = vec![metas[0]]; - for &meta in metas.iter().skip(1) { - let meta_end = meta.end_offset()?; - let gap = meta.offset.checked_sub(range_end).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "IVF-HNSW-SQ list payload offsets overlap", - ) - })?; - if should_coalesce_gap( - gap, - range_start, - meta_end, - range_payload_bytes, - meta.payload_len, - ) { - range_end = meta_end; - range_payload_bytes = range_payload_bytes - .checked_add(meta.payload_len) - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "coalesced IVF-HNSW-SQ requested payload bytes overflow", - ) - })?; - range_metas.push(meta); - } else { - self.read_coalesced_graph_list_range( - range_start, - range_end, - &range_metas, - &mut loaded, - )?; - range_start = meta.offset; - range_end = meta_end; - range_payload_bytes = meta.payload_len; - range_metas.clear(); - range_metas.push(meta); - } - } - self.read_coalesced_graph_list_range(range_start, range_end, &range_metas, &mut loaded)?; - - loaded.sort_by_key(|(list_id, _)| { - list_ids - .iter() - .position(|&requested_id| requested_id == *list_id) - .unwrap_or(usize::MAX) - }); - Ok(loaded) - } - - fn read_coalesced_graph_list_range( - &mut self, - range_start: u64, - range_end: u64, - metas: &[ListPayloadMeta], - loaded: &mut Vec<(usize, GraphList)>, - ) -> io::Result<()> { - let byte_len = usize::try_from(range_end.checked_sub(range_start).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "coalesced IVF-HNSW-SQ read range is invalid", - ) - })?) - .map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidData, - "coalesced IVF-HNSW-SQ read range exceeds usize", - ) - })?; - let mut payload = vec![0u8; byte_len]; - self.reader - .pread(&mut [ReadRequest::new(range_start, &mut payload)])?; - - for &meta in metas { - let start = usize::try_from(meta.offset - range_start).map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidData, - "coalesced IVF-HNSW-SQ payload offset exceeds usize", - ) - })?; - let end = start.checked_add(meta.payload_len).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "coalesced IVF-HNSW-SQ payload slice overflows", - ) - })?; - loaded.push(( - meta.list_id, - self.decode_graph_list_payload(meta, &payload[start..end])?, - )); - } - Ok(()) - } - - fn list_payload_meta(&self, list_id: usize) -> io::Result> { - if list_id >= self.nlist { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("list_id {} out of range (nlist={})", list_id, self.nlist), - )); - } - let count = self.list_counts[list_id] as usize; - if count == 0 { - return Ok(None); - } - - let offset = checked_list_offset(self.list_offsets[list_id], list_id)?; - let graph_bytes_len = self.list_graph_bytes_lens[list_id] as usize; - if graph_bytes_len == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("list {} is missing HNSW graph", list_id), - )); - } - let payload_len = self.list_payload_bytes_lens[list_id] as usize; - if payload_len == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("list {} is missing payload length", list_id), - )); - } - let minimum_payload_len = 12usize.checked_add(graph_bytes_len).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "IVF-HNSW-SQ minimum payload length overflow", - ) - })?; - if payload_len < minimum_payload_len { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "list {} payload length {} is shorter than expected {}", - list_id, payload_len, minimum_payload_len - ), - )); - } - Ok(Some(ListPayloadMeta { - list_id, - offset, - count, - payload_len, - })) - } - - fn decode_graph_list_payload( - &self, - meta: ListPayloadMeta, - payload: &[u8], - ) -> io::Result { - if payload.len() != meta.payload_len { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "list {} payload length {} does not match expected {}", - meta.list_id, - payload.len(), - meta.payload_len - ), - )); - } - let base_header_len = 12usize; - if payload.len() < base_header_len { - return Err(io::Error::new( - io::ErrorKind::UnexpectedEof, - format!("list {} has truncated ID header", meta.list_id), - )); - } - let base_id = i64::from_le_bytes(payload[0..8].try_into().unwrap()); - let id_bytes_len = i32::from_le_bytes(payload[8..12].try_into().unwrap()); - if id_bytes_len < 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "negative id_bytes_len {} at list {}", - id_bytes_len, meta.list_id - ), - )); - } - let id_bytes_len = id_bytes_len as usize; - let ids_end = base_header_len.checked_add(id_bytes_len).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "IVF-HNSW-SQ ID payload length overflow", - ) - })?; - let code_size = self.sq.code_size(); - let codes_bytes_len = checked_list_bytes(meta.count, code_size)?; - let codes_end = ids_end.checked_add(codes_bytes_len).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "IVF-HNSW-SQ codes payload length overflow", - ) - })?; - if codes_end > payload.len() { - return Err(io::Error::new( - io::ErrorKind::UnexpectedEof, - format!("list {} has truncated SQ codes payload", meta.list_id), - )); - } - let ids = decode_delta_varint_ids(base_id, &payload[base_header_len..ids_end], meta.count)?; - let codes = payload[ids_end..codes_end].to_vec(); - let mut vectors = vec![0.0f32; meta.count * self.d]; - let centroid = self.list_centroid(meta.list_id).to_vec(); - self.list_sq(meta.list_id).decode_batch_with_offset( - &codes, - meta.count, - ¢roid, - &mut vectors, - ); - let graph = decode_graph( - &payload[codes_end..], - vectors, - meta.count, - self.d, - self.metric, - self.hnsw_params, - )?; - let graph = graph.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("list {} is missing HNSW graph", meta.list_id), - ) - })?; - Ok(GraphList { - ids, - codes, - graph, - centroid: Some(centroid), - sq: self.list_sq(meta.list_id).clone(), - sq_decode_lut: self.list_sq_decode_lut(meta.list_id).map(Arc::clone), - }) - } - - fn list_centroid(&self, list_id: usize) -> &[f32] { - &self.quantizer_centroids[list_id * self.d..(list_id + 1) * self.d] - } - - fn list_sq(&self, list_id: usize) -> &ScalarQuantizer { - self.list_sqs.get(list_id).unwrap_or(&self.sq) - } - - fn list_sq_decode_lut(&self, list_id: usize) -> Option<&Arc> { - self.sq_decode_luts.get(list_id) - } - - pub fn search( - &mut self, - query: &[f32], - k: usize, - nprobe: usize, - ef_search: usize, - ) -> io::Result<(Vec, Vec)> { - self.search_with_filter(query, k, nprobe, ef_search, None) - } - - pub fn search_with_filter( - &mut self, - query: &[f32], - k: usize, - nprobe: usize, - ef_search: usize, - filter: Option<&dyn RowIdFilter>, - ) -> io::Result<(Vec, Vec)> { - self.ensure_loaded()?; - validate_search_inputs(query, 1, self.d, k, nprobe)?; - - let q = preprocess_vectors(query, 1, self.d, self.metric); - let (probe_indices, _) = - kmeans::find_topk(&q, &self.quantizer_centroids, self.nlist, self.d, nprobe); - let mut loaded_lists = Vec::with_capacity(probe_indices.len()); - for (_, list) in self.read_graph_lists_coalesced(&probe_indices)? { - loaded_lists.push(list); - } - let search_lists: Vec<_> = loaded_lists - .iter() - .map(|list| HnswSearchList { - ids: list.ids.as_slice(), - graph: Some(&list.graph), - payload: list, - }) - .collect(); - let sorted = search_hnsw_lists(&q, &search_lists, k, ef_search, filter, |list, heap| { - let list = list.payload; - scan_sq_list( - &q, - &list.ids, - &list.codes, - list.centroid.as_deref(), - &list.sq, - list.sq_decode_lut.as_deref(), - self.metric, - filter, - heap, - ); - }); - let mut labels: Vec = sorted.iter().map(|&(_, id)| id).collect(); - let mut distances: Vec = sorted.iter().map(|&(dist, _)| dist).collect(); - labels.resize(k, -1); - distances.resize(k, f32::MAX); - Ok((labels, distances)) - } - - pub fn search_with_roaring_filter( - &mut self, - query: &[f32], - k: usize, - nprobe: usize, - ef_search: usize, - roaring_filter_bytes: &[u8], - ) -> io::Result<(Vec, Vec)> { - let filter = decode_roaring_filter(roaring_filter_bytes)?; - self.search_with_filter(query, k, nprobe, ef_search, Some(&filter)) - } -} - -pub fn search_batch_ivfhnswsq_reader( - reader: &mut IVFHNSWSQIndexReader, - queries: &[f32], - nq: usize, - k: usize, - nprobe: usize, - ef_search: usize, -) -> io::Result<(Vec, Vec)> { - search_batch_ivfhnswsq_reader_filter(reader, queries, nq, k, nprobe, ef_search, None) -} - -pub fn search_batch_ivfhnswsq_reader_filter( - reader: &mut IVFHNSWSQIndexReader, - queries: &[f32], - nq: usize, - k: usize, - nprobe: usize, - ef_search: usize, - filter: Option<&dyn RowIdFilter>, -) -> io::Result<(Vec, Vec)> { - reader.ensure_loaded()?; - validate_search_inputs(queries, nq, reader.d, k, nprobe)?; - - let processed = preprocess_vectors(queries, nq, reader.d, reader.metric); - let (all_probe_indices, _) = kmeans::find_topk_batch( - &processed, - nq, - &reader.quantizer_centroids, - reader.nlist, - reader.d, - nprobe, - ); - - let mut list_to_queries = vec![Vec::new(); reader.nlist]; - let mut unique_lists = Vec::new(); - for (qi, probe_indices) in all_probe_indices.iter().enumerate() { - for &list_id in probe_indices { - if list_to_queries[list_id].is_empty() { - unique_lists.push(list_id); - } - list_to_queries[list_id].push(qi); - } - } - - let mut heaps: Vec = (0..nq).map(|_| TopKHeap::new(k)).collect(); - let mut search_workspace = HnswSearchWorkspace::new(ef_search.max(k)); - let mut query_filtered_counts = vec![0usize; nq]; - let mut loaded_lists = Vec::with_capacity(unique_lists.len()); - for (list_id, list) in reader.read_graph_lists_coalesced(&unique_lists)? { - if let Some(f) = filter { - let filtered = list.ids.iter().filter(|&&id| f.contains(id)).count(); - for &qi in &list_to_queries[list_id] { - query_filtered_counts[qi] = query_filtered_counts[qi] - .checked_add(filtered) - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "filtered vector count overflows usize", - ) - })?; - } - } - loaded_lists.push(LoadedBatchList { - query_ids: std::mem::take(&mut list_to_queries[list_id]), - ids: list.ids, - codes: list.codes, - graph: list.graph, - centroid: list.centroid, - sq: list.sq, - sq_decode_lut: list.sq_decode_lut, - }); - } - - for list in &loaded_lists { - for &qi in &list.query_ids { - let query = &processed[qi * reader.d..(qi + 1) * reader.d]; - let force_sq_scan = filter - .map(|_| query_filtered_counts[qi] <= ef_search.max(k)) - .unwrap_or(false); - if force_sq_scan { - scan_sq_list( - query, - &list.ids, - &list.codes, - list.centroid.as_deref(), - &list.sq, - list.sq_decode_lut.as_deref(), - reader.metric, - filter, - &mut heaps[qi], - ); - } else { - let local_results = list.graph.search_with_reusable_workspace( - query, - ef_search.max(k), - ef_search.max(k), - &mut search_workspace, - ); - for &(local_id, dist) in local_results { - let row_id = list.ids[local_id]; - if filter.map(|f| f.contains(row_id)).unwrap_or(true) { - heaps[qi].push(dist, row_id); - } - } - } - } - } - if filter.is_some() { - for list in &loaded_lists { - for &qi in &list.query_ids { - if heaps[qi].len() >= k { - continue; - } - if query_filtered_counts[qi] <= ef_search.max(k) { - continue; - } - let query = &processed[qi * reader.d..(qi + 1) * reader.d]; - scan_sq_list( - query, - &list.ids, - &list.codes, - list.centroid.as_deref(), - &list.sq, - list.sq_decode_lut.as_deref(), - reader.metric, - filter, - &mut heaps[qi], - ); - } - } - } - - let mut result_ids = vec![-1i64; nq * k]; - let mut result_dists = vec![f32::MAX; nq * k]; - for qi in 0..nq { - let sorted = std::mem::replace(&mut heaps[qi], TopKHeap::new(0)).into_sorted(); - let base = qi * k; - for (i, &(dist, id)) in sorted.iter().enumerate() { - result_ids[base + i] = id; - result_dists[base + i] = dist; - } - } - - Ok((result_ids, result_dists)) -} - -pub fn search_batch_ivfhnswsq_reader_roaring_filter( - reader: &mut IVFHNSWSQIndexReader, - queries: &[f32], - nq: usize, - k: usize, - nprobe: usize, - ef_search: usize, - roaring_filter_bytes: &[u8], -) -> io::Result<(Vec, Vec)> { - let filter = decode_roaring_filter(roaring_filter_bytes)?; - search_batch_ivfhnswsq_reader_filter(reader, queries, nq, k, nprobe, ef_search, Some(&filter)) -} - -struct GraphList { - ids: Vec, - codes: Vec, - graph: HnswGraph, - centroid: Option>, - sq: ScalarQuantizer, - sq_decode_lut: Option>, -} - -#[derive(Clone, Copy)] -struct ListPayloadMeta { - list_id: usize, - offset: u64, - count: usize, - payload_len: usize, -} - -impl ListPayloadMeta { - fn end_offset(self) -> io::Result { - self.offset - .checked_add(self.payload_len as u64) - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "IVF-HNSW-SQ list payload offset overflows u64", - ) - }) - } -} - -fn should_coalesce_gap( - gap: u64, - range_start: u64, - next_range_end: u64, - current_payload_bytes: usize, - next_payload_bytes: usize, -) -> bool { - if gap > MAX_COALESCED_READ_GAP_BYTES { - return false; - } - let Some(requested_bytes) = current_payload_bytes.checked_add(next_payload_bytes) else { - return false; - }; - let Some(range_bytes) = next_range_end.checked_sub(range_start) else { - return false; - }; - range_bytes <= requested_bytes.saturating_mul(2) as u64 -} - -struct LoadedBatchList { - query_ids: Vec, - ids: Vec, - codes: Vec, - graph: HnswGraph, - centroid: Option>, - sq: ScalarQuantizer, - sq_decode_lut: Option>, -} - -fn scan_sq_list( - query: &[f32], - ids: &[i64], - codes: &[u8], - centroid: Option<&[f32]>, - sq: &ScalarQuantizer, - sq_decode_lut: Option<&ScalarQuantizerDecodeLut>, - metric: MetricType, - filter: Option<&dyn RowIdFilter>, - heap: &mut TopKHeap, -) { - let context = sq.distance_context(query, metric); - let code_size = sq.code_size(); - for (local_id, &row_id) in ids.iter().enumerate() { - if filter.map(|f| !f.contains(row_id)).unwrap_or(false) { - continue; - } - let code = &codes[local_id * code_size..(local_id + 1) * code_size]; - let dist = match (centroid, sq_decode_lut) { - (Some(centroid), Some(lut)) => sq - .distance_to_code_with_lut_offset_with_context(query, code, centroid, lut, context), - (Some(centroid), None) => { - sq.distance_to_code_with_offset_with_context(query, code, centroid, context) - } - (None, Some(lut)) => { - sq.distance_to_code_with_lut_with_context(query, code, lut, context) - } - (None, None) => sq.distance_to_code_with_context(query, code, context), - }; - heap.push(dist, row_id); - } -} - -fn validate_index_shape(index: &IVFHNSWSQIndex) -> io::Result<()> { - if index.d == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "dimension must be greater than 0", - )); - } - if index.nlist == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "nlist must be greater than 0", - )); - } - if index.sq.d != index.d { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "SQ dimension does not match index dimension", - )); - } - validate_sq_bounds(index.d, &index.sq.mins, &index.sq.maxs)?; - if index.list_sqs.len() != index.nlist { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "SQ list bounds count does not match nlist", - )); - } - for (list_id, sq) in index.list_sqs.iter().enumerate() { - if sq.d != index.d { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "SQ dimension for list {} does not match index dimension", - list_id - ), - )); - } - validate_sq_bounds(index.d, &sq.mins, &sq.maxs)?; - } - let centroid_len = checked_section_size(index.nlist, index.d)?; - if index.quantizer_centroids.len() != centroid_len { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "quantizer centroid length {} does not match nlist*d {}", - index.quantizer_centroids.len(), - centroid_len - ), - )); - } - if index.ids.len() != index.nlist - || index.codes.len() != index.nlist - || index.graphs.len() != index.nlist - { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "inverted list count does not match nlist", - )); - } - for list_id in 0..index.nlist { - let count = index.ids[list_id].len(); - let expected_codes_len = checked_list_bytes(count, index.sq.code_size())?; - if index.codes[list_id].len() != expected_codes_len { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "list {} SQ code length {} does not match count*d {}", - list_id, - index.codes[list_id].len(), - expected_codes_len - ), - )); - } - match &index.graphs[list_id] { - Some(graph) if count > 0 => { - let decoded = index.decode_list_vectors(list_id, count); - if graph.len() != count || graph.vectors() != decoded.as_slice() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("list {} graph does not match SQ code storage", list_id), - )); - } - } - Some(_) => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("list {} has graph for an empty list", list_id), - )); - } - None if count == 0 => {} - None => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "list {} is missing HNSW graph; call build_graphs first", - list_id - ), - )); - } - } - } - Ok(()) -} - -fn validate_sq_bounds(d: usize, mins: &[f32], maxs: &[f32]) -> io::Result<()> { - if mins.len() != d || maxs.len() != d { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "SQ bounds length mismatch: d={}, mins={}, maxs={}", - d, - mins.len(), - maxs.len() - ), - )); - } - for (dim, (&min, &max)) in mins.iter().zip(maxs.iter()).enumerate() { - if !min.is_finite() || !max.is_finite() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("SQ bounds at dimension {} must be finite", dim), - )); - } - } - Ok(()) -} - -fn read_f32_le(reader: &mut PreadCursor<'_, R>) -> io::Result { - let mut buf = [0u8; 4]; - reader.read_exact(&mut buf)?; - Ok(f32::from_le_bytes(buf)) -} - -fn sq_global_bounds(mins: &[f32], maxs: &[f32]) -> (f32, f32) { - let min = mins.iter().copied().fold(f32::INFINITY, f32::min); - let max = maxs.iter().copied().fold(f32::NEG_INFINITY, f32::max); - if min.is_finite() && max.is_finite() { - (min, max) - } else { - (0.0, 0.0) - } -} - -struct SortedSqGraphList { - ids: Vec, - id_bytes: Vec, - codes: Vec, - graph_bytes: Vec, -} - -fn build_sorted_sq_graph_list( - index: &IVFHNSWSQIndex, - list_id: usize, -) -> io::Result { - let count = index.ids[list_id].len(); - if count == 0 { - return Ok(SortedSqGraphList { - ids: Vec::new(), - id_bytes: Vec::new(), - codes: Vec::new(), - graph_bytes: Vec::new(), - }); - } - - let code_size = index.sq.code_size(); - let mut order: Vec = (0..count).collect(); - order.sort_by_key(|&idx| index.ids[list_id][idx]); - - let ids: Vec = order.iter().map(|&idx| index.ids[list_id][idx]).collect(); - let (_, id_bytes) = encode_delta_varint_ids(&ids); - - let mut codes = vec![0u8; checked_list_bytes(count, code_size)?]; - for (new_idx, &old_idx) in order.iter().enumerate() { - codes[new_idx * code_size..(new_idx + 1) * code_size] - .copy_from_slice(&index.codes[list_id][old_idx * code_size..(old_idx + 1) * code_size]); - } - - let mut vectors = vec![0.0f32; count * index.d]; - let centroid = &index.quantizer_centroids[list_id * index.d..(list_id + 1) * index.d]; - index - .list_sq(list_id) - .decode_batch_with_offset(&codes, count, centroid, &mut vectors); - let old_to_new = old_to_new_order(&order); - let source_graph = index.graphs[list_id].as_ref().ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - format!("list {} is missing HNSW graph", list_id), - ) - })?; - let graph = reorder_graph( - source_graph, - &order, - &old_to_new, - vectors, - index.d, - index.metric, - index.hnsw_params, - )?; - let graph_bytes = encode_graph(Some(&graph))?; - - Ok(SortedSqGraphList { - ids, - id_bytes, - codes, - graph_bytes, - }) -} - -fn old_to_new_order(order: &[usize]) -> Vec { - let mut old_to_new = vec![0; order.len()]; - for (new_idx, &old_idx) in order.iter().enumerate() { - old_to_new[old_idx] = new_idx; - } - old_to_new -} - -fn reorder_graph( - graph: &HnswGraph, - order: &[usize], - old_to_new: &[usize], - vectors: Vec, - d: usize, - metric: MetricType, - hnsw_params: HnswBuildParams, -) -> io::Result { - let levels: Vec = order - .iter() - .map(|&old_idx| graph.levels()[old_idx]) - .collect(); - let neighbors: Vec>> = order - .iter() - .map(|&old_idx| { - graph.neighbors()[old_idx] - .iter() - .map(|level_neighbors| { - level_neighbors - .iter() - .map(|&neighbor| old_to_new[neighbor]) - .collect() - }) - .collect() - }) - .collect(); - HnswGraph::from_parts( - vectors, - order.len(), - d, - metric, - levels, - neighbors, - old_to_new[graph.entry_point()], - graph.max_observed_level(), - hnsw_params, - ) -} - -fn list_payload_len( - count: usize, - code_size: usize, - id_bytes_len: usize, - graph_bytes_len: usize, -) -> io::Result { - let id_bytes = 12usize.checked_add(id_bytes_len).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "IVF-HNSW-SQ ID payload length overflow", - ) - })?; - let code_bytes = checked_list_bytes(count, code_size)?; - id_bytes - .checked_add(code_bytes) - .and_then(|len| len.checked_add(graph_bytes_len)) - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "IVF-HNSW-SQ list payload length overflow", - ) - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::hnsw::HnswBuildParams; - use crate::io::PosWriter; - use crate::io::{ReadRequest, SeekRead}; - use roaring::RoaringTreemap; - use std::io::Cursor; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::Arc; - - #[test] - fn test_ivfhnswsq_reader_rejects_missing_required_flags() { - let mut buf = vec![0u8; IVF_HNSW_SQ_HEADER_SIZE + 16]; - buf[0..4].copy_from_slice(&IVF_HNSW_SQ_MAGIC.to_le_bytes()); - buf[4..8].copy_from_slice(&IVF_HNSW_SQ_VERSION.to_le_bytes()); - buf[8..12].copy_from_slice(&2i32.to_le_bytes()); - buf[12..16].copy_from_slice(&1i32.to_le_bytes()); - buf[16..20].copy_from_slice(&(MetricType::L2 as u32).to_le_bytes()); - buf[20..28].copy_from_slice(&0i64.to_le_bytes()); - buf[28..32].copy_from_slice(&2i32.to_le_bytes()); - buf[32..36].copy_from_slice(&8i32.to_le_bytes()); - buf[36..40].copy_from_slice(&3i32.to_le_bytes()); - buf[40..44].copy_from_slice(&0.0f32.to_le_bytes()); - buf[44..48].copy_from_slice(&0.0f32.to_le_bytes()); - buf[48..52].copy_from_slice(&0u32.to_le_bytes()); - - let err = match IVFHNSWSQIndexReader::open(Cursor::new(buf)) { - Ok(_) => panic!("missing required flags should be rejected"), - Err(err) => err, - }; - assert!(err - .to_string() - .contains("requires delta-varint IDs and graph v1")); - } - - #[test] - fn test_ivfhnswsq_reader_rejects_unknown_flags() { - let mut buf = vec![0u8; IVF_HNSW_SQ_HEADER_SIZE + 16]; - buf[0..4].copy_from_slice(&IVF_HNSW_SQ_MAGIC.to_le_bytes()); - buf[4..8].copy_from_slice(&IVF_HNSW_SQ_VERSION.to_le_bytes()); - buf[8..12].copy_from_slice(&2i32.to_le_bytes()); - buf[12..16].copy_from_slice(&1i32.to_le_bytes()); - buf[16..20].copy_from_slice(&(MetricType::L2 as u32).to_le_bytes()); - buf[20..28].copy_from_slice(&0i64.to_le_bytes()); - buf[28..32].copy_from_slice(&2i32.to_le_bytes()); - buf[32..36].copy_from_slice(&8i32.to_le_bytes()); - buf[36..40].copy_from_slice(&3i32.to_le_bytes()); - buf[40..44].copy_from_slice(&0.0f32.to_le_bytes()); - buf[44..48].copy_from_slice(&0.0f32.to_le_bytes()); - buf[48..52].copy_from_slice(&(REQUIRED_FLAGS | (1 << 31)).to_le_bytes()); - - let err = match IVFHNSWSQIndexReader::open(Cursor::new(buf)) { - Ok(_) => panic!("unknown flags should be rejected"), - Err(err) => err, - }; - assert!(err.to_string().contains("Unsupported IVF_HNSW_SQ flags")); - } - - #[test] - fn test_ivfhnswsq_reader_rejects_nonzero_reserved_bytes() { - let mut buf = vec![0u8; IVF_HNSW_SQ_HEADER_SIZE + 16]; - buf[0..4].copy_from_slice(&IVF_HNSW_SQ_MAGIC.to_le_bytes()); - buf[4..8].copy_from_slice(&IVF_HNSW_SQ_VERSION.to_le_bytes()); - buf[8..12].copy_from_slice(&2i32.to_le_bytes()); - buf[12..16].copy_from_slice(&1i32.to_le_bytes()); - buf[16..20].copy_from_slice(&(MetricType::L2 as u32).to_le_bytes()); - buf[20..28].copy_from_slice(&0i64.to_le_bytes()); - buf[28..32].copy_from_slice(&2i32.to_le_bytes()); - buf[32..36].copy_from_slice(&8i32.to_le_bytes()); - buf[36..40].copy_from_slice(&3i32.to_le_bytes()); - buf[40..44].copy_from_slice(&0.0f32.to_le_bytes()); - buf[44..48].copy_from_slice(&0.0f32.to_le_bytes()); - buf[48..52].copy_from_slice(&REQUIRED_FLAGS.to_le_bytes()); - buf[52] = 1; - - let err = match IVFHNSWSQIndexReader::open(Cursor::new(buf)) { - Ok(_) => panic!("non-zero reserved bytes should be rejected"), - Err(err) => err, - }; - assert!(err.to_string().contains("reserved bytes must be zero")); - } - - #[test] - fn test_ivfhnswsq_write_read_search_roundtrip() { - let d = 4; - let nlist = 4; - let n = 128; - let data: Vec = (0..n) - .flat_map(|i| { - let cluster = (i % nlist) as f32 * 100.0; - [ - cluster + i as f32 * 2.0, - 10.0 + i as f32, - 20.0 + i as f32, - 30.0 + i as f32, - ] - }) - .collect(); - let ids: Vec = (10_000..10_000 + n as i64).collect(); - - let mut index = IVFHNSWSQIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, n); - index.add(&data, &ids, n); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - let mut writer = PosWriter::new(&mut buf); - write_ivfhnswsq_index(&index, &mut writer).unwrap(); - - let mut reader = IVFHNSWSQIndexReader::open(Cursor::new(buf)).unwrap(); - let query_id = 23; - let (labels, distances) = reader - .search(&data[query_id * d..(query_id + 1) * d], 5, nlist, 32) - .unwrap(); - - assert_eq!(labels[0], ids[query_id]); - assert!(distances[0].is_finite()); - } - - #[test] - fn test_ivfhnswsq_write_read_preserves_sq_dimension_bounds() { - let d = 2; - let nlist = 1; - let data = vec![0.0, -100.0, 1.0, 100.0]; - let ids = vec![10, 11]; - let mut index = IVFHNSWSQIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, 2); - index.add(&data, &ids, 2); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - write_ivfhnswsq_index(&index, &mut PosWriter::new(&mut buf)).unwrap(); - assert_eq!( - u32::from_le_bytes(buf[4..8].try_into().unwrap()), - IVF_HNSW_SQ_VERSION - ); - - let reader = IVFHNSWSQIndexReader::open(Cursor::new(buf)).unwrap(); - - assert_eq!(reader.sq.mins, index.sq.mins); - assert_eq!(reader.sq.maxs, index.sq.maxs); - assert_eq!(reader.sq.min, index.sq.min); - assert_eq!(reader.sq.max, index.sq.max); - assert_eq!(reader.list_sqs.len(), nlist); - assert_eq!(reader.list_sqs[0].mins, index.list_sqs[0].mins); - assert_eq!(reader.list_sqs[0].maxs, index.list_sqs[0].maxs); - } - - #[test] - fn test_ivfhnswsq_reader_rejects_mismatched_sq_bounds_summary() { - let d = 2; - let nlist = 1; - let data = vec![0.0, -100.0, 1.0, 100.0]; - let ids = vec![10, 11]; - let mut index = IVFHNSWSQIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, 2); - index.add(&data, &ids, 2); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - write_ivfhnswsq_index(&index, &mut PosWriter::new(&mut buf)).unwrap(); - buf[40..44].copy_from_slice(&123.0f32.to_le_bytes()); - - let err = match IVFHNSWSQIndexReader::open(Cursor::new(buf)) { - Ok(_) => panic!("mismatched SQ bounds summary should be rejected"), - Err(err) => err, - }; - assert_eq!(err.kind(), io::ErrorKind::InvalidData); - assert!(err.to_string().contains("SQ bounds summary")); - } - - #[test] - fn test_ivfhnswsq_reader_search_with_roaring_filter() { - let d = 2; - let nlist = 1; - let data = vec![0.0, 0.0, 0.1, 0.0, 10.0, 10.0]; - let ids = vec![10, 11, 12]; - let mut index = IVFHNSWSQIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, 3); - index.add(&data, &ids, 3); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - write_ivfhnswsq_index(&index, &mut PosWriter::new(&mut buf)).unwrap(); - let mut reader = IVFHNSWSQIndexReader::open(Cursor::new(buf)).unwrap(); - - let mut filter = RoaringTreemap::new(); - filter.insert(12); - let mut filter_bytes = Vec::new(); - filter.serialize_into(&mut filter_bytes).unwrap(); - - let (labels, _) = reader - .search_with_roaring_filter(&[0.0, 0.0], 2, nlist, 8, &filter_bytes) - .unwrap(); - - assert_eq!(labels, vec![12, -1]); - } - - #[test] - fn test_ivfhnswsq_reader_optimized_filter_search_matches_unoptimized() { - let d = 4; - let nlist = 4; - let n = 128; - let data: Vec = (0..n) - .flat_map(|i| { - let cluster = (i % nlist) as f32 * 100.0; - [cluster + i as f32 * 0.01, 1.0, 2.0, 3.0] - }) - .collect(); - let ids: Vec = (0..n as i64).collect(); - - let mut index = IVFHNSWSQIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, n); - index.add(&data, &ids, n); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - write_ivfhnswsq_index(&index, &mut PosWriter::new(&mut buf)).unwrap(); - - let mut filter = RoaringTreemap::new(); - filter.insert(0); - filter.insert(64); - let mut filter_bytes = Vec::new(); - filter.serialize_into(&mut filter_bytes).unwrap(); - - let mut baseline = IVFHNSWSQIndexReader::open(Cursor::new(buf.clone())).unwrap(); - let expected = baseline - .search_with_roaring_filter(&data[0..d], 3, nlist, 64, &filter_bytes) - .unwrap(); - - let mut optimized = IVFHNSWSQIndexReader::open(Cursor::new(buf)).unwrap(); - optimized.optimize_for_search().unwrap(); - let actual = optimized - .search_with_roaring_filter(&data[0..d], 3, nlist, 64, &filter_bytes) - .unwrap(); - - assert_eq!(actual, expected); - } - - #[test] - fn test_ivfhnswsq_reader_search_coalesces_contiguous_list_reads() { - let d = 4; - let nlist = 4; - let n = 128; - let data: Vec = (0..n) - .flat_map(|i| { - let cluster = (i % nlist) as f32 * 100.0; - [cluster + i as f32 * 0.01, 1.0, 2.0, 3.0] - }) - .collect(); - let ids: Vec = (0..n as i64).collect(); - - let mut index = IVFHNSWSQIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, n); - index.add(&data, &ids, n); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - write_ivfhnswsq_index(&index, &mut PosWriter::new(&mut buf)).unwrap(); - - let pread_count = Arc::new(AtomicUsize::new(0)); - let cursor = CountingPreadCursor::new(buf, Arc::clone(&pread_count)); - let mut reader = IVFHNSWSQIndexReader::open(cursor).unwrap(); - reader.ensure_loaded().unwrap(); - pread_count.store(0, Ordering::SeqCst); - - reader.search(&data[0..d], 5, nlist, 32).unwrap(); - - assert_eq!(pread_count.load(Ordering::SeqCst), 1); - } - - #[test] - fn test_ivfhnswsq_write_read_search_roundtrip_cosine() { - let d = 3; - let nlist = 2; - let data = vec![1.0, 0.0, 0.0, 0.9, 0.1, 0.0, 0.0, 1.0, 0.0, 0.0, 0.9, 0.1]; - let ids = vec![10, 11, 12, 13]; - let mut index = - IVFHNSWSQIndex::new(d, nlist, MetricType::Cosine, HnswBuildParams::default()); - index.train(&data, 4); - index.add(&data, &ids, 4); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - write_ivfhnswsq_index(&index, &mut PosWriter::new(&mut buf)).unwrap(); - - let mut reader = IVFHNSWSQIndexReader::open(Cursor::new(buf)).unwrap(); - let (labels, distances) = reader.search(&[1.0, 0.0, 0.0], 2, nlist, 16).unwrap(); - - assert_eq!(labels[0], 10); - assert!(distances[0].is_finite()); - } - - #[test] - fn test_ivfhnswsq_batch_matches_single_search() { - let d = 4; - let nlist = 4; - let n = 128; - let data: Vec = (0..n) - .flat_map(|i| { - let cluster = (i % nlist) as f32 * 100.0; - [cluster + i as f32 * 0.01, 1.0, 2.0, 3.0] - }) - .collect(); - let ids: Vec = (0..n as i64).collect(); - - let mut index = IVFHNSWSQIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, n); - index.add(&data, &ids, n); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - write_ivfhnswsq_index(&index, &mut PosWriter::new(&mut buf)).unwrap(); - let queries = [&data[7 * d..8 * d], &data[23 * d..24 * d]].concat(); - - let mut batch_reader = IVFHNSWSQIndexReader::open(Cursor::new(buf.clone())).unwrap(); - let (batch_labels, batch_distances) = - search_batch_ivfhnswsq_reader(&mut batch_reader, &queries, 2, 3, nlist, 32).unwrap(); - - for qi in 0..2 { - let mut single_reader = IVFHNSWSQIndexReader::open(Cursor::new(buf.clone())).unwrap(); - let (single_labels, single_distances) = single_reader - .search(&queries[qi * d..(qi + 1) * d], 3, nlist, 32) - .unwrap(); - assert_eq!( - &batch_labels[qi * 3..(qi + 1) * 3], - single_labels.as_slice() - ); - assert_eq!( - &batch_distances[qi * 3..(qi + 1) * 3], - single_distances.as_slice() - ); - } - } - - #[test] - fn test_ivfhnswsq_batch_reader_coalesces_contiguous_list_reads() { - let d = 4; - let nlist = 4; - let n = 128; - let data: Vec = (0..n) - .flat_map(|i| { - let cluster = (i % nlist) as f32 * 100.0; - [cluster + i as f32 * 0.01, 1.0, 2.0, 3.0] - }) - .collect(); - let ids: Vec = (0..n as i64).collect(); - - let mut index = IVFHNSWSQIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, n); - index.add(&data, &ids, n); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - write_ivfhnswsq_index(&index, &mut PosWriter::new(&mut buf)).unwrap(); - - let pread_count = Arc::new(AtomicUsize::new(0)); - let cursor = CountingPreadCursor::new(buf, Arc::clone(&pread_count)); - let mut reader = IVFHNSWSQIndexReader::open(cursor).unwrap(); - reader.ensure_loaded().unwrap(); - pread_count.store(0, Ordering::SeqCst); - let queries = data[0..d].to_vec(); - - search_batch_ivfhnswsq_reader(&mut reader, &queries, 1, 5, nlist, 32).unwrap(); - - assert_eq!(pread_count.load(Ordering::SeqCst), 1); - } - - #[test] - fn test_ivfhnswsq_reader_coalesces_small_gap_between_requested_lists() { - let d = 4; - let nlist = 4; - let n = 128; - let data: Vec = (0..n) - .flat_map(|i| { - let cluster = (i % nlist) as f32 * 100.0; - [cluster + i as f32 * 0.01, 1.0, 2.0, 3.0] - }) - .collect(); - let ids: Vec = (0..n as i64).collect(); - - let mut index = IVFHNSWSQIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, n); - index.add(&data, &ids, n); - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - write_ivfhnswsq_index(&index, &mut PosWriter::new(&mut buf)).unwrap(); - - let pread_count = Arc::new(AtomicUsize::new(0)); - let cursor = CountingPreadCursor::new(buf, Arc::clone(&pread_count)); - let mut reader = IVFHNSWSQIndexReader::open(cursor).unwrap(); - reader.ensure_loaded().unwrap(); - assert!(reader.list_counts[..3].iter().all(|&count| count > 0)); - pread_count.store(0, Ordering::SeqCst); - - let lists = reader.read_graph_lists_coalesced(&[0, 2]).unwrap(); - - assert_eq!(lists.len(), 2); - assert_eq!(pread_count.load(Ordering::SeqCst), 1); - } - - #[test] - fn test_ivfhnswsq_write_requires_graphs() { - let mut index = IVFHNSWSQIndex::new(2, 1, MetricType::L2, HnswBuildParams::default()); - let data = vec![0.0, 0.0]; - index.train(&data, 1); - index.add(&data, &[1], 1); - - let mut buf = Vec::new(); - let err = write_ivfhnswsq_index(&index, &mut PosWriter::new(&mut buf)).unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - assert!(err.to_string().contains("build_graphs")); - } - - #[test] - fn test_ivfhnswsq_writer_rejects_stale_graph() { - let mut index = IVFHNSWSQIndex::new(2, 1, MetricType::L2, HnswBuildParams::default()); - let data = vec![0.0, 0.0, 1.0, 0.0]; - index.train(&data, 2); - index.add(&data, &[10, 11], 2); - index.build_graphs().unwrap(); - index.graphs[0] = Some( - HnswGraph::build( - &[10.0, 10.0, 11.0, 11.0], - 2, - 2, - MetricType::L2, - HnswBuildParams::default(), - ) - .unwrap(), - ); - - let mut buf = Vec::new(); - let err = write_ivfhnswsq_index(&index, &mut PosWriter::new(&mut buf)).unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - assert!(err.to_string().contains("graph does not match")); - } - - struct CountingPreadCursor { - data: Vec, - pread_count: Arc, - } - - impl CountingPreadCursor { - fn new(data: Vec, pread_count: Arc) -> Self { - Self { data, pread_count } - } - } - - impl SeekRead for CountingPreadCursor { - fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { - for range in ranges { - self.pread_count.fetch_add(1, Ordering::SeqCst); - let pos = range.pos as usize; - let end = pos.checked_add(range.buf.len()).ok_or_else(|| { - io::Error::new(io::ErrorKind::UnexpectedEof, "cursor position overflow") - })?; - if end > self.data.len() { - return Err(io::Error::new( - io::ErrorKind::UnexpectedEof, - "failed to fill whole buffer", - )); - } - range.buf.copy_from_slice(&self.data[pos..end]); - } - Ok(()) - } - } -} diff --git a/core/src/ivfpq.rs b/core/src/ivfpq.rs index 4e663f2..7237d93 100644 --- a/core/src/ivfpq.rs +++ b/core/src/ivfpq.rs @@ -19,13 +19,15 @@ use crate::distance::{ fvec_inner_product, fvec_madd, fvec_normalize, pq_distance_four_codes, pq_distance_from_table, MetricType, }; -use crate::io::{IVFPQIndexReader, SeekRead}; +use crate::index_io_util::ivf_payload_is_oversized; +use crate::io::{IVFPQIndexReader, InvertedListPayload, SeekRead}; use crate::kmeans::{self, KMeansConfig}; use crate::opq::OPQMatrix; use crate::pq::ProductQuantizer; use rayon::prelude::*; use roaring::RoaringTreemap; -use std::collections::{HashMap, HashSet}; +use std::borrow::Cow; +use std::collections::HashSet; use std::io; pub trait RowIdFilter: Sync { @@ -87,6 +89,10 @@ impl IVFPQIndex { metric: MetricType, use_opq: bool, ) -> Self { + assert!( + nbits != 4 || m.is_multiple_of(2), + "4-bit IVF-PQ requires even m, got {m}" + ); let by_residual = metric == MetricType::L2; IVFPQIndex { d, @@ -137,6 +143,7 @@ impl IVFPQIndex { nbits: trained.pq.nbits, dsub: trained.pq.dsub, ksub: trained.pq.ksub, + chunk_offsets: trained.pq.chunk_offsets.clone(), centroids: trained.pq.centroids.clone(), centroid_norms_cache: trained.pq.centroid_norms_cache.clone(), }, @@ -215,14 +222,11 @@ impl IVFPQIndex { fn add_batch(&mut self, data: &[f32], ids: &[i64], n: usize) { let d = self.d; - // Step 1: Preprocess (normalize + OPQ rotate) + // L2/IP without OPQ borrows the caller's batch instead of copying it. let processed = self.preprocess_queries(data, n); - - // Step 2: Batch assign to coarse centroids (uses sgemm) let assignments = kmeans::find_nearest_batch(&processed, n, &self.quantizer_centroids, self.nlist, d); - // Step 3: Batch compute residuals (parallel) let to_encode = if self.by_residual { let mut residuals = vec![0.0f32; n * d]; residuals @@ -237,24 +241,21 @@ impl IVFPQIndex { res, ); }); - residuals + Cow::Owned(residuals) } else { processed }; - // Step 4: Batch PQ encode (parallel) - let cs = self.pq.code_size(); - let mut codes = vec![0u8; n * cs]; + let code_size = self.pq.code_size(); + let mut codes = vec![0u8; n * code_size]; self.pq.encode_batch(&to_encode, n, &mut codes); - // Step 5: Distribute to inverted lists for i in 0..n { let list_id = assignments[i]; self.ids[list_id].push(ids[i]); - self.codes[list_id].extend_from_slice(&codes[i * cs..(i + 1) * cs]); + self.codes[list_id].extend_from_slice(&codes[i * code_size..(i + 1) * code_size]); } - // Invalidate stale precomputed structures (must rebuild after all adds) if !self.fastscan_codes.is_empty() { self.fastscan_codes.clear(); } @@ -300,24 +301,25 @@ impl IVFPQIndex { let pq_norms = self.pq.compute_centroid_norms(); let mut table = vec![0.0f32; nlist * m * ksub]; - for i in 0..nlist { - let centroid = &self.quantizer_centroids[i * d..(i + 1) * d]; - let tab_base = i * m * ksub; - - for sub in 0..m { - let sub_centroid = ¢roid[sub * self.pq.dsub..(sub + 1) * self.pq.dsub]; - let pq_base = sub * ksub * self.pq.dsub; - - for j in 0..ksub { - let pq_off = pq_base + j * self.pq.dsub; - let ip = fvec_inner_product( - sub_centroid, - &self.pq.centroids[pq_off..pq_off + self.pq.dsub], - ); - table[tab_base + sub * ksub + j] = pq_norms[sub * ksub + j] + 2.0 * ip; + table + .par_chunks_mut(m * ksub) + .enumerate() + .for_each(|(i, list_table)| { + let centroid = &self.quantizer_centroids[i * d..(i + 1) * d]; + for sub in 0..m { + let sub_centroid = ¢roid[sub * self.pq.dsub..(sub + 1) * self.pq.dsub]; + let pq_base = sub * ksub * self.pq.dsub; + + for j in 0..ksub { + let pq_off = pq_base + j * self.pq.dsub; + let ip = fvec_inner_product( + sub_centroid, + &self.pq.centroids[pq_off..pq_off + self.pq.dsub], + ); + list_table[sub * ksub + j] = pq_norms[sub * ksub + j] + 2.0 * ip; + } } - } - } + }); self.precomputed_table = table; } } @@ -478,20 +480,23 @@ impl IVFPQIndex { } } - fn preprocess_queries(&self, queries: &[f32], nq: usize) -> Vec { + fn preprocess_queries<'a>(&self, queries: &'a [f32], nq: usize) -> Cow<'a, [f32]> { let d = self.d; - let mut processed = queries[..nq * d].to_vec(); - - if self.metric == MetricType::Cosine { - for i in 0..nq { - fvec_normalize(&mut processed[i * d..(i + 1) * d]); + let processed = match self.metric { + MetricType::Cosine => { + let mut normalized = queries[..nq * d].to_vec(); + for vector in normalized.chunks_exact_mut(d) { + fvec_normalize(vector); + } + Cow::Owned(normalized) } - } + MetricType::L2 | MetricType::InnerProduct => Cow::Borrowed(&queries[..nq * d]), + }; if let Some(ref opq) = self.opq { let mut rotated = vec![0.0f32; nq * d]; opq.apply_batch(&processed, &mut rotated, nq); - return rotated; + return Cow::Owned(rotated); } processed @@ -842,7 +847,8 @@ fn scan_codes_4bit_transposed( /// Scan transposed (column-major) codes: layout is [M][n]. /// The distance table sub-slice stays in L1 cache for the entire inner loop. -fn scan_codes_transposed( +#[allow(clippy::too_many_arguments)] +fn scan_codes_transposed_with_scratch( sim_table: &[f32], codes: &[u8], ids: &[i64], @@ -852,13 +858,47 @@ fn scan_codes_transposed( dis0: f32, filter: Option<&dyn RowIdFilter>, heap: &mut TopKHeap, + dists: &mut Vec, ) { - let mut dists = vec![dis0; count]; - for sub in 0..m { - let tab_base = sub * ksub; + debug_assert!(m > 0); + dists.resize(count, 0.0); + let table = &sim_table[..ksub]; + let column = &codes[..count]; + let mut row = 0usize; + while row + 8 <= count { + dists[row] = dis0 + table[column[row] as usize]; + dists[row + 1] = dis0 + table[column[row + 1] as usize]; + dists[row + 2] = dis0 + table[column[row + 2] as usize]; + dists[row + 3] = dis0 + table[column[row + 3] as usize]; + dists[row + 4] = dis0 + table[column[row + 4] as usize]; + dists[row + 5] = dis0 + table[column[row + 5] as usize]; + dists[row + 6] = dis0 + table[column[row + 6] as usize]; + dists[row + 7] = dis0 + table[column[row + 7] as usize]; + row += 8; + } + while row < count { + dists[row] = dis0 + table[column[row] as usize]; + row += 1; + } + for sub in 1..m { + let table = &sim_table[sub * ksub..(sub + 1) * ksub]; let col_base = sub * count; - for i in 0..count { - dists[i] += sim_table[tab_base + codes[col_base + i] as usize]; + let column = &codes[col_base..col_base + count]; + let mut row = 0usize; + while row + 8 <= count { + dists[row] += table[column[row] as usize]; + dists[row + 1] += table[column[row + 1] as usize]; + dists[row + 2] += table[column[row + 2] as usize]; + dists[row + 3] += table[column[row + 3] as usize]; + dists[row + 4] += table[column[row + 4] as usize]; + dists[row + 5] += table[column[row + 5] as usize]; + dists[row + 6] += table[column[row + 6] as usize]; + dists[row + 7] += table[column[row + 7] as usize]; + row += 8; + } + while row < count { + dists[row] += table[column[row] as usize]; + row += 1; } } @@ -923,14 +963,6 @@ fn scan_codes_batched( } } -struct PreReadList { - list_id: usize, - count: usize, - dis0: f32, - ids: Vec, - codes: Vec, -} - struct ReaderSearchContext<'a> { q: &'a [f32], ip_table: &'a [f32], @@ -947,6 +979,12 @@ struct ReaderSearchContext<'a> { precomputed_table: &'a [f32], } +#[derive(Default)] +struct ReaderScanScratch { + sim_table: Vec, + distances: Vec, +} + /// Search using a lazy reader (reads inverted lists on demand). pub fn search_with_reader( reader: &mut IVFPQIndexReader, @@ -1034,57 +1072,92 @@ pub fn search_with_reader_filter( }; lists_to_read.push((list_id, count, dis0)); } + lists_to_read.sort_unstable_by_key(|&(list_id, _, _)| reader.list_offsets[list_id]); - let read_list_ids: Vec = lists_to_read + let read_list_ids = lists_to_read .iter() .map(|&(list_id, _, _)| list_id) - .collect(); - let read_lists = reader.read_inverted_lists(&read_list_ids)?; - let mut list_data: Vec = Vec::with_capacity(read_lists.len()); - for ((list_id, count, dis0), read_list) in lists_to_read.into_iter().zip(read_lists) { - if list_id != read_list.list_id { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "batched inverted list read returned lists out of order", - )); - } - list_data.push(PreReadList { - list_id, - count, - dis0, - ids: read_list.ids, - codes: read_list.codes, - }); - } + .collect::>(); + let mut batch_start = 0usize; + while batch_start < read_list_ids.len() { + let first_list = read_list_ids[batch_start]; + if ivf_payload_is_oversized(reader.list_payload_len(first_list)?) { + let (_, _, dis0) = lists_to_read[batch_start]; + let sim_table = reader_sim_table(reader, first_list, &q, &ip_table, use_precomputed); + let pq_nbits = reader.pq.nbits; + let transposed_codes = reader.transposed_codes; + let mut scratch = ReaderScanScratch::default(); + reader.for_each_streamed_list_chunk(first_list, |ids, codes| { + scan_reader_codes( + &sim_table, + codes, + ids, + m, + ksub, + pq_nbits, + transposed_codes, + dis0, + filter, + &mut scratch.distances, + &mut heap, + ); + })?; + batch_start += 1; + continue; + } + let count = reader.batch_read_end(&read_list_ids[batch_start..])?.max(1); + let batch_end = (batch_start + count).min(read_list_ids.len()); + let read_lists = + reader.read_inverted_list_payloads(&read_list_ids[batch_start..batch_end])?; + let mut list_data = Vec::with_capacity(read_lists.len()); + for (&(list_id, expected_count, dis0), read_list) in + lists_to_read[batch_start..batch_end].iter().zip(read_lists) + { + if list_id != read_list.list_id { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "batched inverted list read returned lists out of order", + )); + } + if expected_count != read_list.ids.len() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "batched inverted list read returned an unexpected row count", + )); + } + list_data.push((read_list, dis0)); + } - let ctx = ReaderSearchContext { - q: &q, - ip_table: &ip_table, - use_precomputed, - filter, - d, - m, - ksub, - metric, - by_residual, - transposed_codes: reader.transposed_codes, - pq: &reader.pq, - quantizer_centroids: &reader.quantizer_centroids, - precomputed_table: &reader.precomputed_table, - }; - let per_list_results: Vec> = list_data - .par_iter() - .map(|entry| { - let mut local_heap = TopKHeap::new(k); - scan_reader_list(entry, &ctx, &mut local_heap); - local_heap.into_sorted() - }) - .collect(); + let ctx = ReaderSearchContext { + q: &q, + ip_table: &ip_table, + use_precomputed, + filter, + d, + m, + ksub, + metric, + by_residual, + transposed_codes: reader.transposed_codes, + pq: &reader.pq, + quantizer_centroids: &reader.quantizer_centroids, + precomputed_table: &reader.precomputed_table, + }; + let per_list_results = list_data + .par_iter() + .map_init(ReaderScanScratch::default, |scratch, (entry, dis0)| { + let mut local_heap = TopKHeap::new(k); + scan_reader_list(entry, *dis0, &ctx, scratch, &mut local_heap); + local_heap.into_sorted() + }) + .collect::>(); - for results in per_list_results { - for (dist, id) in results { - heap.push(dist, id); + for results in per_list_results { + for (dist, id) in results { + heap.push(dist, id); + } } + batch_start = batch_end; } let sorted = heap.into_sorted(); @@ -1106,83 +1179,110 @@ pub fn search_with_reader_roaring_filter( search_with_reader_filter(reader, query, k, nprobe, Some(&filter)) } -fn scan_reader_list(entry: &PreReadList, ctx: &ReaderSearchContext<'_>, heap: &mut TopKHeap) { +fn scan_reader_list( + entry: &InvertedListPayload, + dis0: f32, + ctx: &ReaderSearchContext<'_>, + scratch: &mut ReaderScanScratch, + heap: &mut TopKHeap, +) { + fill_reader_sim_table(entry.list_id, ctx, &mut scratch.sim_table); + scan_reader_codes( + &scratch.sim_table, + entry.codes(), + &entry.ids, + ctx.m, + ctx.ksub, + ctx.pq.nbits, + ctx.transposed_codes, + dis0, + ctx.filter, + &mut scratch.distances, + heap, + ); +} + +fn fill_reader_sim_table(list_id: usize, ctx: &ReaderSearchContext<'_>, sim_table: &mut Vec) { let d = ctx.d; let m = ctx.m; let ksub = ctx.ksub; - let metric = ctx.metric; - let mut sim_table = vec![0.0f32; m * ksub]; - + sim_table.resize(m * ksub, 0.0); if ctx.use_precomputed { - let tab_base = entry.list_id * m * ksub; + let tab_base = list_id * m * ksub; fvec_madd( &ctx.precomputed_table[tab_base..tab_base + m * ksub], ctx.ip_table, -2.0, - &mut sim_table, + sim_table, ); } else if ctx.by_residual { let mut residual_query = vec![0.0f32; d]; fvec_madd( ctx.q, - &ctx.quantizer_centroids[entry.list_id * d..(entry.list_id + 1) * d], + &ctx.quantizer_centroids[list_id * d..(list_id + 1) * d], -1.0, &mut residual_query, ); ctx.pq - .compute_distance_table(&residual_query, metric, &mut sim_table); + .compute_distance_table(&residual_query, ctx.metric, sim_table); } else { - ctx.pq.compute_distance_table(ctx.q, metric, &mut sim_table); + ctx.pq.compute_distance_table(ctx.q, ctx.metric, sim_table); } +} - let is_4bit = ctx.pq.nbits == 4; - if is_4bit && ctx.transposed_codes { - scan_codes_4bit_transposed( - &sim_table, - &entry.codes, - &entry.ids, - entry.count, - m, - entry.dis0, - ctx.filter, - heap, - ); +fn reader_sim_table( + reader: &IVFPQIndexReader, + list_id: usize, + query: &[f32], + ip_table: &[f32], + use_precomputed: bool, +) -> Vec { + let ctx = ReaderSearchContext { + q: query, + ip_table, + use_precomputed, + filter: None, + d: reader.d, + m: reader.m, + ksub: reader.ksub, + metric: reader.metric, + by_residual: reader.by_residual, + transposed_codes: reader.transposed_codes, + pq: &reader.pq, + quantizer_centroids: &reader.quantizer_centroids, + precomputed_table: &reader.precomputed_table, + }; + let mut sim_table = Vec::new(); + fill_reader_sim_table(list_id, &ctx, &mut sim_table); + sim_table +} + +#[allow(clippy::too_many_arguments)] +fn scan_reader_codes( + sim_table: &[f32], + codes: &[u8], + ids: &[i64], + m: usize, + ksub: usize, + pq_nbits: usize, + transposed_codes: bool, + dis0: f32, + filter: Option<&dyn RowIdFilter>, + distances: &mut Vec, + heap: &mut TopKHeap, +) { + let is_4bit = pq_nbits == 4; + let count = ids.len(); + if is_4bit && transposed_codes { + scan_codes_4bit_transposed(sim_table, codes, ids, count, m, dis0, filter, heap); } else if is_4bit { - scan_codes_4bit( - &sim_table, - &entry.codes, - &entry.ids, - entry.count, - m, - ksub, - entry.dis0, - ctx.filter, - heap, - ); - } else if ctx.transposed_codes { - scan_codes_transposed( - &sim_table, - &entry.codes, - &entry.ids, - entry.count, - m, - ksub, - entry.dis0, - ctx.filter, - heap, + scan_codes_4bit(sim_table, codes, ids, count, m, ksub, dis0, filter, heap); + } else if transposed_codes { + scan_codes_transposed_with_scratch( + sim_table, codes, ids, count, m, ksub, dis0, filter, heap, distances, ); } else { - scan_codes_batched( - &sim_table, - &entry.codes, - &entry.ids, - entry.count, - m, - ksub, - entry.dis0, - ctx.filter, - heap, - ); + scan_codes_batched(sim_table, codes, ids, count, m, ksub, dis0, filter, heap); } } @@ -1272,26 +1372,26 @@ pub fn search_batch_reader_filter( nprobe, ); - // Step 3: Group (query_idx, probe_rank) pairs by probed list_id only. - let mut list_to_queries: HashMap> = HashMap::new(); + // Step 3: Read every probed list once. Queries share the decoded list + // payloads, then scan independently in parallel. + let mut seen = vec![false; reader.nlist]; let mut unique_lists = Vec::new(); - for qi in 0..nq { - for (rank, &list_id) in all_probe_indices[qi].iter().enumerate() { - let coarse_dist = all_coarse_dists[qi][rank]; - let entry = list_to_queries.entry(list_id).or_insert_with(|| { + for probe_indices in &all_probe_indices { + for &list_id in probe_indices { + if !seen[list_id] && reader.list_counts[list_id] > 0 { + seen[list_id] = true; unique_lists.push(list_id); - Vec::new() - }); - entry.push((qi, coarse_dist)); + } } } + unique_lists.sort_unstable_by_key(|&list_id| reader.list_offsets[list_id]); - // Step 4: For each unique list that has queries, read once and scan for all let use_precomputed = metric == MetricType::L2 && by_residual && !reader.precomputed_table.is_empty(); let all_ip_tables: Vec> = if use_precomputed { (0..nq) + .into_par_iter() .map(|qi| { let mut t = vec![0.0f32; m * ksub]; reader @@ -1304,56 +1404,124 @@ pub fn search_batch_reader_filter( Vec::new() }; - let mut heaps: Vec = (0..nq).map(|_| TopKHeap::new(k)).collect(); - - let non_empty_lists: Vec = unique_lists - .into_iter() - .filter(|&list_id| reader.list_counts[list_id] > 0) - .collect(); - let read_lists = reader.read_inverted_lists(&non_empty_lists)?; + let mut heaps = (0..nq).map(|_| TopKHeap::new(k)).collect::>(); + let mut batch_start = 0usize; + while batch_start < unique_lists.len() { + let first_list = unique_lists[batch_start]; + if ivf_payload_is_oversized(reader.list_payload_len(first_list)?) { + let query_tables = (0..nq) + .filter_map(|query_index| { + all_probe_indices[query_index] + .iter() + .position(|&list_id| list_id == first_list) + .map(|probe_rank| { + let query = &processed[query_index * d..(query_index + 1) * d]; + let sim_table = reader_sim_table( + reader, + first_list, + query, + if use_precomputed { + &all_ip_tables[query_index] + } else { + &[] + }, + use_precomputed, + ); + let dis0 = if use_precomputed { + all_coarse_dists[query_index][probe_rank] + } else { + 0.0 + }; + (query_index, dis0, sim_table) + }) + }) + .collect::>(); + let pq_nbits = reader.pq.nbits; + let transposed_codes = reader.transposed_codes; + // The loop is sequential across queries. Reuse one chunk-sized + // distance buffer instead of retaining one per query. + let mut distances = Vec::new(); + reader.for_each_streamed_list_chunk(first_list, |ids, codes| { + for (query_index, dis0, sim_table) in &query_tables { + scan_reader_codes( + sim_table, + codes, + ids, + m, + ksub, + pq_nbits, + transposed_codes, + *dis0, + filter, + &mut distances, + &mut heaps[*query_index], + ); + } + })?; + batch_start += 1; + continue; + } + let count = reader.batch_read_end(&unique_lists[batch_start..])?.max(1); + let batch_end = (batch_start + count).min(unique_lists.len()); + let loaded_lists = + reader.read_inverted_list_payloads(&unique_lists[batch_start..batch_end])?; + let mut list_positions = vec![usize::MAX; reader.nlist]; + for (position, list) in loaded_lists.iter().enumerate() { + list_positions[list.list_id] = position; + } - for read_list in read_lists { - let count = read_list.ids.len(); - let mut entry = PreReadList { - list_id: read_list.list_id, - count, - dis0: 0.0, - ids: read_list.ids, - codes: read_list.codes, - }; + let rows = (0..nq) + .into_par_iter() + .map(|qi| { + let query = &processed[qi * d..(qi + 1) * d]; + let ctx = ReaderSearchContext { + q: query, + ip_table: if use_precomputed { + &all_ip_tables[qi] + } else { + &[] + }, + use_precomputed, + filter, + d, + m, + ksub, + metric, + by_residual, + transposed_codes: reader.transposed_codes, + pq: &reader.pq, + quantizer_centroids: &reader.quantizer_centroids, + precomputed_table: &reader.precomputed_table, + }; + let mut heap = TopKHeap::new(k); + let mut scratch = ReaderScanScratch::default(); + for (probe_rank, &list_id) in all_probe_indices[qi].iter().enumerate() { + let position = list_positions[list_id]; + if position == usize::MAX { + continue; + } + let dis0 = if use_precomputed { + all_coarse_dists[qi][probe_rank] + } else { + 0.0 + }; + scan_reader_list(&loaded_lists[position], dis0, &ctx, &mut scratch, &mut heap); + } + heap.into_sorted() + }) + .collect::>(); + for (qi, row) in rows.into_iter().enumerate() { + for (distance, row_id) in row { + heaps[qi].push(distance, row_id); + } + } + batch_start = batch_end; + } - for &(qi, coarse_dist) in &list_to_queries[&entry.list_id] { - let query = &processed[qi * d..(qi + 1) * d]; - let dis0 = if use_precomputed { coarse_dist } else { 0.0 }; - let ctx = ReaderSearchContext { - q: query, - ip_table: if use_precomputed { - &all_ip_tables[qi] - } else { - &[] - }, - use_precomputed, - filter, - d, - m, - ksub, - metric, - by_residual, - transposed_codes: reader.transposed_codes, - pq: &reader.pq, - quantizer_centroids: &reader.quantizer_centroids, - precomputed_table: &reader.precomputed_table, - }; - entry.dis0 = dis0; - scan_reader_list(&entry, &ctx, &mut heaps[qi]); - } - } - - // Collect results let mut result_ids = vec![-1i64; nq * k]; let mut result_dists = vec![f32::MAX; nq * k]; - for qi in 0..nq { - let sorted = std::mem::replace(&mut heaps[qi], TopKHeap::new(0)).into_sorted(); + for (qi, heap) in heaps.into_iter().enumerate() { + let sorted = heap.into_sorted(); let base = qi * k; for (i, &(dist, id)) in sorted.iter().enumerate() { result_ids[base + i] = id; @@ -1486,6 +1654,7 @@ mod tests { pread_batches: usize, max_ranges_per_batch: usize, max_pread_len: usize, + last_positions: Vec, } struct NonConcurrentPreadCursor { @@ -1508,6 +1677,7 @@ mod tests { let mut stats = self.stats.lock().unwrap(); stats.pread_batches += 1; stats.max_ranges_per_batch = stats.max_ranges_per_batch.max(ranges.len()); + stats.last_positions = ranges.iter().map(|range| range.pos).collect(); } for range in ranges { { @@ -1908,6 +2078,57 @@ mod tests { } } + #[test] + fn ivfpq_only_allocates_preprocessed_vectors_when_required() { + let data = vec![3.0, 4.0, 1.0, 2.0]; + let l2 = IVFPQIndex::new(2, 1, 1, MetricType::L2, false); + assert!(matches!(l2.preprocess_queries(&data, 2), Cow::Borrowed(_))); + + let cosine = IVFPQIndex::new(2, 1, 1, MetricType::Cosine, false); + assert!(matches!(cosine.preprocess_queries(&data, 2), Cow::Owned(_))); + } + + #[test] + fn transposed_scan_matches_scalar_distance_table() { + let count = 37; + let m = 7; + let ksub = 256; + let dis0 = 3.25; + let ids = (1_000..1_000 + count as i64).collect::>(); + let codes = (0..m * count) + .map(|index| ((index * 73 + 19) % ksub) as u8) + .collect::>(); + let table = (0..m * ksub) + .map(|index| (index % 101) as f32 * 0.03125) + .collect::>(); + + let mut heap = TopKHeap::new(count); + let mut scratch = Vec::new(); + scan_codes_transposed_with_scratch( + &table, + &codes, + &ids, + count, + m, + ksub, + dis0, + None, + &mut heap, + &mut scratch, + ); + + let mut expected = (0..count) + .map(|row| { + let distance = (0..m).fold(dis0, |distance, sub| { + distance + table[sub * ksub + codes[sub * count + row] as usize] + }); + (distance, ids[row]) + }) + .collect::>(); + expected.sort_by(|left, right| left.0.total_cmp(&right.0)); + assert_eq!(heap.into_sorted(), expected); + } + #[test] fn test_precomputed_table_matches_normal_search() { let d = 16; @@ -2149,6 +2370,13 @@ mod tests { stats.max_ranges_per_batch > 1, "multiple probed IVF-PQ lists should share one batched pread" ); + assert!( + stats + .last_positions + .windows(2) + .all(|positions| positions[0] <= positions[1]), + "fallback readers should receive inverted-list ranges in physical order" + ); } #[test] @@ -2523,4 +2751,10 @@ mod tests { let err = search_batch_reader(&mut reader, queries, nq, k, 0).unwrap_err(); assert_eq!(err.kind(), io::ErrorKind::InvalidInput); } + + #[test] + #[should_panic(expected = "4-bit IVF-PQ requires even m")] + fn ivfpq_rejects_odd_4bit_subquantizer_count_at_construction() { + let _ = IVFPQIndex::with_nbits(12, 4, 3, 4, MetricType::L2, false); + } } diff --git a/core/src/ivfrq.rs b/core/src/ivfrq.rs index 0051498..a2b8b3f 100644 --- a/core/src/ivfrq.rs +++ b/core/src/ivfrq.rs @@ -6,7 +6,7 @@ // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an @@ -15,58 +15,81 @@ // specific language governing permissions and limitations // under the License. -use crate::distance::{fvec_madd, fvec_norm_l2sqr, preprocess_vectors, MetricType}; +use crate::distance::{fvec_madd, preprocess_vectors, MetricType}; use crate::ivfpq::RowIdFilter; use crate::kmeans::{self, KMeansConfig}; use crate::rq::{ - RQCodeFactors, RQRotation, RaBitQuantizer, DEFAULT_RQ_QUERY_BITS, DEFAULT_RQ_ROTATION_ROUNDS, - DEFAULT_RQ_ROTATION_SEED, RQ_BYTE_LUT_MIN_LIST_SIZE, + RQEncodeScratch, RQQueryContext, RQRotation, RQVectorFactors, RaBitQuantizer, DEFAULT_RQ_BITS, + DEFAULT_RQ_ROTATION_ROUNDS, DEFAULT_RQ_ROTATION_SEED, }; use crate::topk::TopKHeap; +use rayon::prelude::*; +use std::borrow::Cow; pub struct IVFRQIndex { pub d: usize, + pub padded_d: usize, pub nlist: usize, + pub bits: usize, pub metric: MetricType, pub quantizer_centroids: Vec, + pub rotated_centroids: Vec, pub rotation_seed: u64, pub rotation_rounds: u32, pub ids: Vec>, pub codes: Vec>, - pub factors: Vec>, + pub factors: Vec>, quantizer: RaBitQuantizer, rotation: RQRotation, } impl IVFRQIndex { pub fn new(d: usize, nlist: usize, metric: MetricType) -> Self { - Self::with_rotation( + Self::with_options( d, nlist, + DEFAULT_RQ_BITS, metric, DEFAULT_RQ_ROTATION_SEED, DEFAULT_RQ_ROTATION_ROUNDS, ) } - pub fn with_rotation( + pub fn with_bits(d: usize, nlist: usize, bits: usize, metric: MetricType) -> Self { + Self::with_options( + d, + nlist, + bits, + metric, + DEFAULT_RQ_ROTATION_SEED, + DEFAULT_RQ_ROTATION_ROUNDS, + ) + } + + pub fn with_options( d: usize, nlist: usize, + bits: usize, metric: MetricType, rotation_seed: u64, rotation_rounds: u32, ) -> Self { + let quantizer = RaBitQuantizer::new(d, bits); + let padded_d = quantizer.padded_dimension(); Self { d, + padded_d, nlist, + bits, metric, quantizer_centroids: Vec::new(), + rotated_centroids: Vec::new(), rotation_seed, rotation_rounds, ids: vec![Vec::new(); nlist], codes: vec![Vec::new(); nlist], factors: vec![Vec::new(); nlist], - quantizer: RaBitQuantizer::new(d), + quantizer, rotation: RQRotation::new(d, rotation_seed, rotation_rounds), } } @@ -75,6 +98,16 @@ impl IVFRQIndex { let processed = self.preprocess_vectors(data, n); self.quantizer_centroids = kmeans::kmeans_train(&KMeansConfig::default(), &processed, n, self.d, self.nlist); + self.rotated_centroids = vec![0.0; self.nlist * self.padded_d]; + let mut scratch = vec![0.0; self.padded_d]; + for list_id in 0..self.nlist { + let centroid = &self.quantizer_centroids[list_id * self.d..(list_id + 1) * self.d]; + self.rotation.rotate( + centroid, + &mut self.rotated_centroids[list_id * self.padded_d..(list_id + 1) * self.padded_d], + &mut scratch, + ); + } } pub fn add(&mut self, data: &[f32], ids: &[i64], n: usize) { @@ -86,20 +119,71 @@ impl IVFRQIndex { self.nlist, self.d, ); - let code_size = self.code_size(); - let mut residual = vec![0.0f32; self.d]; - let mut code = vec![0u8; code_size]; + let mut list_rows = vec![Vec::new(); self.nlist]; + for (row, list_id) in list_ids.into_iter().enumerate() { + list_rows[list_id].push(row); + } - for i in 0..n { - let list_id = list_ids[i]; - let vector = &processed[i * self.d..(i + 1) * self.d]; - self.write_rotated_residual(vector, list_id, &mut residual); - let factors = self - .quantizer - .encode(&residual, fvec_norm_l2sqr(vector), &mut code); - self.ids[list_id].push(ids[i]); - self.codes[list_id].extend_from_slice(&code); - self.factors[list_id].push(factors); + let d = self.d; + let padded_d = self.padded_d; + let metric = self.metric; + let centroids = &self.quantizer_centroids; + let rotated_centroids = &self.rotated_centroids; + let quantizer = &self.quantizer; + let rotation = &self.rotation; + let output_ids = &mut self.ids; + let output_codes = &mut self.codes; + let output_factors = &mut self.factors; + if n > 1_000 && self.nlist > 1 { + output_ids + .par_iter_mut() + .zip(output_codes.par_iter_mut()) + .zip(output_factors.par_iter_mut()) + .zip(list_rows.into_par_iter()) + .enumerate() + .for_each( + |(list_id, (((list_ids, list_codes), list_factors), rows))| { + append_encoded_rows( + &processed, + ids, + &rows, + d, + padded_d, + ¢roids[list_id * d..(list_id + 1) * d], + &rotated_centroids[list_id * padded_d..(list_id + 1) * padded_d], + metric, + quantizer, + rotation, + list_ids, + list_codes, + list_factors, + ); + }, + ); + } else { + for (list_id, (((list_ids, list_codes), list_factors), rows)) in output_ids + .iter_mut() + .zip(output_codes.iter_mut()) + .zip(output_factors.iter_mut()) + .zip(list_rows) + .enumerate() + { + append_encoded_rows( + &processed, + ids, + &rows, + d, + padded_d, + ¢roids[list_id * d..(list_id + 1) * d], + &rotated_centroids[list_id * padded_d..(list_id + 1) * padded_d], + metric, + quantizer, + rotation, + list_ids, + list_codes, + list_factors, + ); + } } } @@ -111,34 +195,16 @@ impl IVFRQIndex { self.quantizer.code_size() } - pub fn search( - &self, - queries: &[f32], - nq: usize, - k: usize, - nprobe: usize, - result_distances: &mut [f32], - result_labels: &mut [i64], - ) { - self.search_with_filter( - queries, - nq, - k, - nprobe, - None, - DEFAULT_RQ_QUERY_BITS, - result_distances, - result_labels, - ); + pub fn plane_size(&self) -> usize { + self.quantizer.plane_size() } - pub fn search_with_query_bits( + pub fn search( &self, queries: &[f32], nq: usize, k: usize, nprobe: usize, - query_bits: usize, result_distances: &mut [f32], result_labels: &mut [i64], ) { @@ -148,7 +214,6 @@ impl IVFRQIndex { k, nprobe, None, - query_bits, result_distances, result_labels, ); @@ -161,7 +226,6 @@ impl IVFRQIndex { k: usize, nprobe: usize, filter: Option<&dyn RowIdFilter>, - query_bits: usize, result_distances: &mut [f32], result_labels: &mut [i64], ) { @@ -174,12 +238,17 @@ impl IVFRQIndex { self.d, nprobe, ); + let mut rotated_query = vec![0.0; self.padded_d]; + let mut rotation_scratch = vec![0.0; self.padded_d]; for qi in 0..nq { let query = &processed_queries[qi * self.d..(qi + 1) * self.d]; + self.rotation + .rotate(query, &mut rotated_query, &mut rotation_scratch); + let query_context = self.quantizer.prepare_query(rotated_query.clone()); let mut heap = TopKHeap::new(k); for &list_id in &all_probe_indices[qi] { - self.scan_list(query, list_id, filter, query_bits, &mut heap); + self.scan_list(&query_context, list_id, filter, &mut heap); } let sorted = heap.into_sorted(); @@ -195,54 +264,109 @@ impl IVFRQIndex { } } - pub(crate) fn preprocess_vectors(&self, data: &[f32], n: usize) -> Vec { - preprocess_vectors(data, n, self.d, self.metric) - } - - pub(crate) fn list_centroid(&self, list_id: usize) -> &[f32] { - &self.quantizer_centroids[list_id * self.d..(list_id + 1) * self.d] + pub(crate) fn preprocess_vectors<'a>(&self, data: &'a [f32], n: usize) -> Cow<'a, [f32]> { + match self.metric { + MetricType::Cosine => { + Cow::Owned(preprocess_vectors(data, n, self.d, MetricType::Cosine)) + } + MetricType::L2 | MetricType::InnerProduct => Cow::Borrowed(&data[..n * self.d]), + } } - pub(crate) fn rotated_query_residual(&self, query: &[f32], list_id: usize) -> Vec { - let mut residual = vec![0.0f32; self.d]; - self.write_rotated_residual(query, list_id, &mut residual); - residual + pub(crate) fn rotated_centroid(&self, list_id: usize) -> &[f32] { + &self.rotated_centroids[list_id * self.padded_d..(list_id + 1) * self.padded_d] } fn scan_list( &self, - query: &[f32], + query_context: &RQQueryContext, list_id: usize, filter: Option<&dyn RowIdFilter>, - query_bits: usize, heap: &mut TopKHeap, ) { - let rotated_query_residual = self.rotated_query_residual(query, list_id); - let distance_context = self.quantizer.prepare_distance_context_with_query_bits( - rotated_query_residual, - query, - self.ids[list_id].len() >= RQ_BYTE_LUT_MIN_LIST_SIZE, - query_bits, - ); + let query_terms = + self.quantizer + .query_terms(query_context, self.rotated_centroid(list_id), self.metric); let code_size = self.code_size(); for (local_idx, &id) in self.ids[list_id].iter().enumerate() { if filter.map(|f| !f.contains(id)).unwrap_or(false) { continue; } let code = &self.codes[list_id][local_idx * code_size..(local_idx + 1) * code_size]; - let dist = self.quantizer.distance_to_code_prepared( - &distance_context, - code, - self.factors[list_id][local_idx], - self.metric, + let factors = self.factors[list_id][local_idx]; + if self.bits == 1 { + let distance = self.quantizer.estimate( + self.quantizer.coarse_inner_product(query_context, code), + factors.coarse, + query_terms, + ); + if heap.should_consider(distance) { + heap.push(distance, id); + } + continue; + } + + let coarse = self.quantizer.estimate( + self.quantizer.coarse_inner_product(query_context, code), + factors.coarse, + query_terms, ); - heap.push(dist, id); + let lower = self + .quantizer + .lower_bound(coarse, factors.coarse, query_terms); + if heap.should_consider(lower) { + let distance = self.quantizer.estimate( + self.quantizer.full_inner_product(query_context, code), + factors.full, + query_terms, + ); + if heap.should_consider(distance) { + heap.push(distance, id); + } + } } } +} - fn write_rotated_residual(&self, vector: &[f32], list_id: usize, out: &mut [f32]) { - fvec_madd(vector, self.list_centroid(list_id), -1.0, out); - self.rotation.apply(out); +#[allow(clippy::too_many_arguments)] +fn append_encoded_rows( + data: &[f32], + input_ids: &[i64], + rows: &[usize], + d: usize, + padded_d: usize, + centroid: &[f32], + rotated_centroid: &[f32], + metric: MetricType, + quantizer: &RaBitQuantizer, + rotation: &RQRotation, + output_ids: &mut Vec, + output_codes: &mut Vec, + output_factors: &mut Vec, +) { + let code_size = quantizer.code_size(); + output_ids.reserve(rows.len()); + output_codes.reserve(rows.len().saturating_mul(code_size)); + output_factors.reserve(rows.len()); + let mut residual = vec![0.0f32; d]; + let mut rotated_residual = vec![0.0f32; padded_d]; + let mut rotation_scratch = vec![0.0f32; padded_d]; + let mut code = vec![0u8; code_size]; + let mut encode_scratch = RQEncodeScratch::new(padded_d); + for &row in rows { + let vector = &data[row * d..(row + 1) * d]; + fvec_madd(vector, centroid, -1.0, &mut residual); + rotation.rotate(&residual, &mut rotated_residual, &mut rotation_scratch); + let factors = quantizer.encode_with_scratch( + &rotated_residual, + rotated_centroid, + metric, + &mut code, + &mut encode_scratch, + ); + output_ids.push(input_ids[row]); + output_codes.extend_from_slice(&code); + output_factors.push(factors); } } @@ -251,19 +375,31 @@ mod tests { use super::*; #[test] - fn ivfrq_recalls_query_vector() { - let d = 8; + fn ivfrq_only_allocates_preprocessed_vectors_for_cosine() { + let data = vec![3.0, 4.0, 1.0, 2.0]; + let l2 = IVFRQIndex::new(2, 1, MetricType::L2); + let ip = IVFRQIndex::new(2, 1, MetricType::InnerProduct); + let cosine = IVFRQIndex::new(2, 1, MetricType::Cosine); + + assert!(matches!(l2.preprocess_vectors(&data, 2), Cow::Borrowed(_))); + assert!(matches!(ip.preprocess_vectors(&data, 2), Cow::Borrowed(_))); + assert!(matches!(cosine.preprocess_vectors(&data, 2), Cow::Owned(_))); + } + + #[test] + fn ivfrq_four_bit_recalls_query_vector_without_dimension_alignment_requirement() { + let d = 13; let nlist = 4; let n = 128; let data: Vec = (0..n) .flat_map(|i| { let cluster = (i % nlist) as f32 * 100.0; - [cluster + i as f32 * 0.01, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0] + (0..d).map(move |dim| cluster + i as f32 * 0.01 + dim as f32) }) .collect(); let ids: Vec = (1000..1000 + n as i64).collect(); - let mut index = IVFRQIndex::new(d, nlist, MetricType::L2); + let mut index = IVFRQIndex::with_bits(d, nlist, 4, MetricType::L2); index.train(&data, n); index.add(&data, &ids, n); @@ -279,36 +415,64 @@ mod tests { ); assert_eq!(labels[0], ids[7]); - assert!(distances[0] <= 1e-4); + assert!(distances[0] <= 1e-3); + assert_eq!(index.padded_d, 64); } #[test] fn ivfrq_inner_product_recalls_query_vector() { - let d = 8; - let nlist = 1; - let n = 8; + let d = 64; + let n = d; let mut data = vec![0.0f32; n * d]; for i in 0..n { data[i * d + i] = 1.0; } let ids: Vec = (1000..1000 + n as i64).collect(); - let mut index = IVFRQIndex::new(d, nlist, MetricType::InnerProduct); + let mut index = IVFRQIndex::with_bits(d, 1, 4, MetricType::InnerProduct); index.train(&data, n); index.add(&data, &ids, n); - let query_id = 7; + let query_id = 37; let mut distances = vec![0.0; 5]; let mut labels = vec![0; 5]; index.search( &data[query_id * d..(query_id + 1) * d], 1, 5, - nlist, + 1, &mut distances, &mut labels, ); assert_eq!(labels[0], ids[query_id]); } + + #[test] + fn ivfrq_parallel_add_matches_incremental_serial_add() { + let d = 65; + let nlist = 8; + let n = 2_048; + let data = (0..n) + .flat_map(|row| { + (0..d).map(move |dimension| { + (row % nlist) as f32 * 100.0 + row as f32 * 0.003 + dimension as f32 * 0.07 + }) + }) + .collect::>(); + let ids = (10_000..10_000 + n as i64).collect::>(); + let mut parallel = IVFRQIndex::with_bits(d, nlist, 4, MetricType::L2); + let mut serial = IVFRQIndex::with_bits(d, nlist, 4, MetricType::L2); + parallel.train(&data, n); + serial.train(&data, n); + + parallel.add(&data, &ids, n); + for (data_chunk, id_chunk) in data.chunks_exact(512 * d).zip(ids.chunks_exact(512)) { + serial.add(data_chunk, id_chunk, 512); + } + + assert_eq!(parallel.ids, serial.ids); + assert_eq!(parallel.codes, serial.codes); + assert_eq!(parallel.factors, serial.factors); + } } diff --git a/core/src/ivfrq_io.rs b/core/src/ivfrq_io.rs index c4da6fa..f23dcc5 100644 --- a/core/src/ivfrq_io.rs +++ b/core/src/ivfrq_io.rs @@ -6,7 +6,7 @@ // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an @@ -15,19 +15,21 @@ // specific language governing permissions and limitations // under the License. -use crate::distance::{fvec_madd, preprocess_vectors, MetricType}; +use crate::distance::{preprocess_vectors, MetricType}; use crate::index_io_util::{ - decode_delta_varint_ids, encode_delta_varint_ids, validate_search_inputs, + decode_delta_varint_ids, encode_delta_varint_ids, pread_batched_payloads, + validate_search_inputs, }; use crate::io::{PreadCursor, ReadRequest, SeekRead, SeekWrite}; use crate::ivfpq::RowIdFilter; use crate::ivfrq::IVFRQIndex; use crate::kmeans; use crate::rq::{ - is_supported_query_bits, RQCodeFactors, RQDistanceContext, RQRotation, RaBitQuantizer, - DEFAULT_RQ_QUERY_BITS, RQ_BYTE_LUT_MIN_LIST_SIZE, + is_supported_rq_bits, padded_dimension, RQCodeFactors, RQQueryContext, RQRotation, + RQVectorFactors, RaBitQuantizer, DEFAULT_RQ_ROTATION_ROUNDS, RQ_SCAN_BLOCK_SIZE, }; use crate::topk::TopKHeap; +use rayon::prelude::*; use roaring::RoaringTreemap; use std::io; @@ -35,177 +37,175 @@ pub const IVF_RQ_MAGIC: u32 = 0x49565251; // "IVRQ" pub const IVF_RQ_VERSION: u32 = 1; pub const IVF_RQ_HEADER_SIZE: usize = 64; +pub const IVF_RQ_ROTATION_TYPE_BLOCK_FHT: u32 = 2; +pub const IVF_RQ_FACTOR_LAYOUT_COMPACT_V1: u32 = 3; + const FLAG_DELTA_IDS: u32 = 1 << 0; -const REQUIRED_FLAGS: u32 = FLAG_DELTA_IDS; +const FLAG_BLOCK_TRANSPOSED_CODES: u32 = 1 << 1; +const FLAG_BLOCK_SOA_FACTORS: u32 = 1 << 2; +const REQUIRED_FLAGS: u32 = FLAG_DELTA_IDS | FLAG_BLOCK_TRANSPOSED_CODES | FLAG_BLOCK_SOA_FACTORS; const SUPPORTED_FLAGS: u32 = REQUIRED_FLAGS; -const FACTOR_BYTES: usize = 12; -const MAX_RQ_BATCH_READ_BYTES: usize = 16 * 1024 * 1024; - -pub const IVF_RQ_NUM_BITS_ONE: u32 = 1; -pub const IVF_RQ_ROTATION_TYPE_KAC: u32 = 1; -pub const IVF_RQ_FACTOR_LAYOUT_RABITQ_V1: u32 = 1; - -const FORMAT_FLAG_EX_CODES_PRESENT: u32 = 1 << 0; -const FORMAT_FLAG_ERROR_FACTOR_PRESENT: u32 = 1 << 1; -const SUPPORTED_FORMAT_FLAGS: u32 = FORMAT_FLAG_EX_CODES_PRESENT | FORMAT_FLAG_ERROR_FACTOR_PRESENT; -const CURRENT_FORMAT_FLAGS: u32 = 0; +const FACTOR_BYTES: usize = 4; +const MAX_RQ_BATCH_READ_BYTES: usize = 64 * 1024 * 1024; +const PARALLEL_RQ_SCAN_MIN_CANDIDATES: usize = 8 * 1024; -struct SortedRQList { - ids: Vec, +struct RQListWritePlan { + order: Vec, + base_id: i64, id_bytes: Vec, - codes: Vec, - factors: Vec, } pub fn write_ivfrq_index(index: &IVFRQIndex, out: &mut dyn SeekWrite) -> io::Result<()> { validate_index_shape(index)?; - let d = index.d; - let nlist = index.nlist; - let code_size = index.code_size(); let total_vectors = index.ids.iter().try_fold(0i64, |sum, ids| { - let count = usize_to_i64(ids.len(), "total vector count")?; - sum.checked_add(count).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "total vector count exceeds i64 length limit", - ) - }) + sum.checked_add(usize_to_i64(ids.len(), "total vector count")?) + .ok_or_else(|| invalid_input("total vector count exceeds i64")) })?; - - let mut sorted_lists = Vec::with_capacity(nlist); - for list_id in 0..nlist { - let count = index.ids[list_id].len(); - if count == 0 { - sorted_lists.push(SortedRQList { - ids: Vec::new(), - id_bytes: Vec::new(), - codes: Vec::new(), - factors: Vec::new(), - }); - continue; - } - - let mut order: Vec = (0..count).collect(); - order.sort_by_key(|&idx| index.ids[list_id][idx]); - - let sorted_ids: Vec = order.iter().map(|&idx| index.ids[list_id][idx]).collect(); - let mut sorted_codes = Vec::with_capacity(count * code_size); - let mut sorted_factors = Vec::with_capacity(count); - for idx in order { - sorted_codes - .extend_from_slice(&index.codes[list_id][idx * code_size..(idx + 1) * code_size]); - sorted_factors.push(index.factors[list_id][idx]); - } - let (_, id_bytes) = encode_delta_varint_ids(&sorted_ids); - sorted_lists.push(SortedRQList { - ids: sorted_ids, - id_bytes, - codes: sorted_codes, - factors: sorted_factors, - }); - } + let write_plans = plan_sorted_lists(index); write_u32_le(out, IVF_RQ_MAGIC)?; write_u32_le(out, IVF_RQ_VERSION)?; - write_i32_le(out, usize_to_i32(d, "dimension")?)?; - write_i32_le(out, usize_to_i32(nlist, "nlist")?)?; + write_i32_le(out, usize_to_i32(index.d, "dimension")?)?; + write_i32_le(out, usize_to_i32(index.padded_d, "padded dimension")?)?; + write_i32_le(out, usize_to_i32(index.nlist, "nlist")?)?; write_u32_le(out, index.metric as u32)?; write_u32_le(out, REQUIRED_FLAGS)?; + write_u32_le(out, usize_to_u32(index.bits, "RQ bits")?)?; write_i64_le(out, total_vectors)?; write_u64_le(out, index.rotation_seed)?; write_u32_le(out, index.rotation_rounds)?; - write_i32_le(out, usize_to_i32(code_size, "code size")?)?; - write_u32_le(out, IVF_RQ_NUM_BITS_ONE)?; - write_u32_le(out, IVF_RQ_ROTATION_TYPE_KAC)?; - write_u32_le(out, IVF_RQ_FACTOR_LAYOUT_RABITQ_V1)?; - write_u32_le(out, CURRENT_FORMAT_FLAGS)?; + write_i32_le(out, usize_to_i32(index.plane_size(), "plane size")?)?; + write_u32_le(out, IVF_RQ_ROTATION_TYPE_BLOCK_FHT)?; + write_u32_le(out, IVF_RQ_FACTOR_LAYOUT_COMPACT_V1)?; write_f32_slice(out, &index.quantizer_centroids)?; - let offset_table_size = nlist.checked_mul(16).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "IVF-RQ offset table size overflow", - ) - })?; - let data_start = out + let offset_table_bytes = index + .nlist + .checked_mul(16) + .ok_or_else(|| invalid_input("IVF-RQ offset table size overflow"))?; + let mut current_offset = out .pos() - .checked_add(offset_table_size as u64) - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "IVF-RQ data start offset overflow", - ) - })?; - let mut list_offsets = vec![0i64; nlist]; - let mut list_counts = vec![0i32; nlist]; - let mut list_id_bytes_lens = vec![0i32; nlist]; - let mut current_offset = data_start; - - for list_id in 0..nlist { - list_offsets[list_id] = u64_to_i64(current_offset, "list offset")?; - let count = sorted_lists[list_id].ids.len(); - list_counts[list_id] = usize_to_i32(count, "list count")?; - if count > 0 { - let id_bytes_len = sorted_lists[list_id].id_bytes.len(); - list_id_bytes_lens[list_id] = usize_to_i32(id_bytes_len, "delta ID section")?; - let code_bytes = checked_list_bytes(count, code_size)?; - let factor_bytes = checked_list_bytes(count, FACTOR_BYTES)?; - let list_bytes = 12usize - .checked_add(id_bytes_len) - .and_then(|len| len.checked_add(code_bytes)) - .and_then(|len| len.checked_add(factor_bytes)) - .ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "IVF-RQ list size overflow") - })?; + .checked_add(offset_table_bytes as u64) + .ok_or_else(|| invalid_input("IVF-RQ data offset overflow"))?; + let mut offsets = Vec::with_capacity(index.nlist); + + for plan in &write_plans { + offsets.push(( + u64_to_i64(current_offset, "list offset")?, + usize_to_i32(plan.order.len(), "list count")?, + usize_to_i32(plan.id_bytes.len(), "delta ID bytes")?, + )); + if !plan.order.is_empty() { + let code_bytes = checked_list_bytes(plan.order.len(), index.code_size())?; + let factor_values = + checked_list_bytes(plan.order.len(), index_factors_fields(index.bits))?; + let factor_bytes = checked_list_bytes(factor_values, FACTOR_BYTES)?; + let list_bytes = 16usize + .checked_add(plan.id_bytes.len()) + .and_then(|size| size.checked_add(code_bytes)) + .and_then(|size| size.checked_add(factor_bytes)) + .ok_or_else(|| invalid_input("IVF-RQ list size overflow"))?; current_offset = current_offset .checked_add(list_bytes as u64) - .ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "IVF-RQ offset overflow") - })?; + .ok_or_else(|| invalid_input("IVF-RQ list offset overflow"))?; } } - for list_id in 0..nlist { - write_i64_le(out, list_offsets[list_id])?; - write_i32_le(out, list_counts[list_id])?; - write_i32_le(out, list_id_bytes_lens[list_id])?; + for (offset, count, id_bytes) in offsets { + write_i64_le(out, offset)?; + write_i32_le(out, count)?; + write_i32_le(out, id_bytes)?; } - for sorted_list in sorted_lists { - if sorted_list.ids.is_empty() { + for (list_id, plan) in write_plans.into_iter().enumerate() { + if plan.order.is_empty() { continue; } - write_i64_le(out, sorted_list.ids[0])?; - write_i32_le(out, sorted_list.id_bytes.len() as i32)?; - out.write_all(&sorted_list.id_bytes)?; - out.write_all(&sorted_list.codes)?; - write_factors(out, &sorted_list.factors)?; + let (blocked_codes, blocked_factors) = block_list(index, list_id, &plan.order); + write_i64_le(out, plan.base_id)?; + write_i32_le(out, usize_to_i32(plan.id_bytes.len(), "delta ID bytes")?)?; + write_i32_le( + out, + usize_to_i32(blocked_codes.len(), "blocked code bytes")?, + )?; + out.write_all(&plan.id_bytes)?; + out.write_all(&blocked_codes)?; + write_f32_slice(out, &blocked_factors)?; } - Ok(()) } +fn plan_sorted_lists(index: &IVFRQIndex) -> Vec { + (0..index.nlist) + .map(|list_id| { + let count = index.ids[list_id].len(); + let mut order: Vec = (0..count).collect(); + order.sort_by_key(|&position| index.ids[list_id][position]); + let sorted_ids = order + .iter() + .map(|&position| index.ids[list_id][position]) + .collect::>(); + let (base_id, id_bytes) = encode_delta_varint_ids(&sorted_ids); + RQListWritePlan { + order, + base_id, + id_bytes, + } + }) + .collect() +} + +fn block_list(index: &IVFRQIndex, list_id: usize, order: &[usize]) -> (Vec, Vec) { + let code_size = index.code_size(); + let plane_size = index.plane_size(); + let factor_fields = index_factors_fields(index.bits); + let mut blocked_codes = Vec::with_capacity(order.len() * code_size); + let mut blocked_factors = Vec::with_capacity(order.len() * factor_fields); + + for block in order.chunks(RQ_SCAN_BLOCK_SIZE) { + for plane in 0..index.bits { + for byte_idx in 0..plane_size { + for &position in block { + blocked_codes.push( + index.codes[list_id][position * code_size + plane * plane_size + byte_idx], + ); + } + } + } + for field in 0..factor_fields { + for &position in block { + blocked_factors.push(factor_value( + index.factors[list_id][position], + field, + index.bits, + )); + } + } + } + (blocked_codes, blocked_factors) +} + pub struct IVFRQIndexReader { reader: R, pub d: usize, + pub padded_d: usize, pub nlist: usize, pub metric: MetricType, pub total_vectors: i64, pub rotation_seed: u64, pub rotation_rounds: u32, - pub code_size: usize, - pub num_bits: u32, + pub plane_size: usize, + pub num_bits: usize, pub rotation_type: u32, pub factor_layout: u32, - pub format_flags: u32, pub quantizer_centroids: Vec, + pub rotated_centroids: Vec, pub list_offsets: Vec, pub list_counts: Vec, pub list_id_bytes_lens: Vec, quantizer: RaBitQuantizer, rotation: RQRotation, - delta_ids: bool, loaded: bool, } @@ -215,96 +215,110 @@ struct RQListPayloadMeta { count: usize, offset: u64, id_bytes_len: usize, + code_bytes_len: usize, + factor_values_len: usize, payload_len: usize, } impl IVFRQIndexReader { pub fn open(mut reader: R) -> io::Result { - let mut cursor = PreadCursor::new(&mut reader, 0); + let mut header = [0u8; IVF_RQ_HEADER_SIZE]; + reader.pread(&mut [ReadRequest::new(0, &mut header)])?; + Self::open_with_header(reader, header) + } + + pub(crate) fn open_with_header( + reader: R, + header: [u8; IVF_RQ_HEADER_SIZE], + ) -> io::Result { + let mut header_reader = std::io::Cursor::new(header); + let mut cursor = PreadCursor::new(&mut header_reader, 0); let magic = read_u32_le(&mut cursor)?; if magic != IVF_RQ_MAGIC { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("Invalid IVF-RQ magic: 0x{:08X}", magic), - )); + return Err(invalid_data(format!("invalid IVF-RQ magic: 0x{magic:08X}"))); } let version = read_u32_le(&mut cursor)?; if version != IVF_RQ_VERSION { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("Unsupported IVF-RQ version: {}", version), - )); + return Err(invalid_data(format!( + "unsupported IVF-RQ version: {version}" + ))); + } + let d = validate_positive_i32(read_i32_le(&mut cursor)?, "dimension")? as usize; + let padded_d = + validate_positive_i32(read_i32_le(&mut cursor)?, "padded_dimension")? as usize; + if padded_d != padded_dimension(d) { + return Err(invalid_data(format!( + "IVF-RQ padded_dimension {padded_d} does not match dimension-derived {}", + padded_dimension(d) + ))); } - let d = validate_positive_i32(read_i32_le(&mut cursor)?, "d")? as usize; let nlist = validate_positive_i32(read_i32_le(&mut cursor)?, "nlist")? as usize; let metric_code = read_u32_le(&mut cursor)?; - let metric = MetricType::from_code(metric_code).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("Unknown metric type: {}", metric_code), - ) - })?; + let metric = MetricType::from_code(metric_code) + .ok_or_else(|| invalid_data(format!("unknown metric type: {metric_code}")))?; let flags = read_u32_le(&mut cursor)?; + let num_bits = read_u32_le(&mut cursor)? as usize; let total_vectors = read_i64_le(&mut cursor)?; - if total_vectors < 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("negative total vector count: {}", total_vectors), - )); - } let rotation_seed = read_u64_le(&mut cursor)?; let rotation_rounds = read_u32_le(&mut cursor)?; - let code_size = validate_positive_i32(read_i32_le(&mut cursor)?, "code_size")? as usize; - let num_bits = read_u32_le(&mut cursor)?; + let plane_size = validate_positive_i32(read_i32_le(&mut cursor)?, "plane_size")? as usize; let rotation_type = read_u32_le(&mut cursor)?; let factor_layout = read_u32_le(&mut cursor)?; - let format_flags = read_u32_le(&mut cursor)?; - let expected_code_size = d.div_ceil(8); - if code_size != expected_code_size { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "IVF-RQ code_size {} does not match dimension-derived size {}", - code_size, expected_code_size - ), - )); + + if total_vectors < 0 { + return Err(invalid_data("negative IVF-RQ vector count")); + } + if plane_size != padded_d / 8 { + return Err(invalid_data(format!( + "IVF-RQ plane size {plane_size} does not match padded dimension {padded_d}" + ))); + } + if !is_supported_rq_bits(num_bits) { + return Err(invalid_data(format!( + "unsupported IVF-RQ bits {num_bits}; expected 1..=8" + ))); + } + if rotation_rounds != DEFAULT_RQ_ROTATION_ROUNDS { + return Err(invalid_data(format!( + "unsupported IVF-RQ rotation rounds {rotation_rounds}; expected {DEFAULT_RQ_ROTATION_ROUNDS}" + ))); } - validate_rq_format(num_bits, rotation_type, factor_layout, format_flags)?; - - let unknown_flags = flags & !SUPPORTED_FLAGS; - if unknown_flags != 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("Unsupported IVF-RQ flags: 0x{:08X}", unknown_flags), - )); + if rotation_type != IVF_RQ_ROTATION_TYPE_BLOCK_FHT { + return Err(invalid_data(format!( + "unsupported IVF-RQ rotation type {rotation_type}" + ))); } - if flags & REQUIRED_FLAGS != REQUIRED_FLAGS { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "IVF-RQ v1 requires delta IDs", - )); + if factor_layout != IVF_RQ_FACTOR_LAYOUT_COMPACT_V1 { + return Err(invalid_data(format!( + "unsupported IVF-RQ factor layout {factor_layout}" + ))); + } + if flags & !SUPPORTED_FLAGS != 0 || flags & REQUIRED_FLAGS != REQUIRED_FLAGS { + return Err(invalid_data(format!( + "unsupported IVF-RQ flags 0x{flags:08X}" + ))); } Ok(Self { reader, d, + padded_d, nlist, metric, total_vectors, rotation_seed, rotation_rounds, - code_size, + plane_size, num_bits, rotation_type, factor_layout, - format_flags, quantizer_centroids: Vec::new(), + rotated_centroids: Vec::new(), list_offsets: Vec::new(), list_counts: Vec::new(), list_id_bytes_lens: Vec::new(), - quantizer: RaBitQuantizer::new(d), + quantizer: RaBitQuantizer::new(d, num_bits), rotation: RQRotation::new(d, rotation_seed, rotation_rounds), - delta_ids: true, loaded: false, }) } @@ -313,30 +327,47 @@ impl IVFRQIndexReader { if self.loaded { return Ok(()); } - let mut cursor = PreadCursor::new(&mut self.reader, IVF_RQ_HEADER_SIZE as u64); - self.quantizer_centroids = - read_f32_vec(&mut cursor, checked_section_size(self.nlist, self.d)?)?; + let centroid_values = checked_section_size(self.nlist, self.d)?; + let centroid_bytes = checked_list_bytes(centroid_values, FACTOR_BYTES)?; + let offset_table_bytes = checked_list_bytes(self.nlist, 16)?; + let metadata_bytes = centroid_bytes + .checked_add(offset_table_bytes) + .ok_or_else(|| invalid_data("IVF-RQ resident metadata size overflow"))?; + let mut metadata = vec![0u8; metadata_bytes]; + self.reader + .pread(&mut [ReadRequest::new(IVF_RQ_HEADER_SIZE as u64, &mut metadata)])?; + + self.quantizer_centroids = bytes_to_f32_vec(&metadata[..centroid_bytes])?; self.list_offsets = vec![0; self.nlist]; self.list_counts = vec![0; self.nlist]; self.list_id_bytes_lens = vec![0; self.nlist]; + for (list_id, entry) in metadata[centroid_bytes..].chunks_exact(16).enumerate() { + self.list_offsets[list_id] = i64::from_le_bytes(entry[0..8].try_into().unwrap()); + self.list_counts[list_id] = validate_non_negative_i32( + i32::from_le_bytes(entry[8..12].try_into().unwrap()), + "list count", + )?; + self.list_id_bytes_lens[list_id] = validate_non_negative_i32( + i32::from_le_bytes(entry[12..16].try_into().unwrap()), + "delta ID bytes", + )?; + } + let actual_total: i64 = self.list_counts.iter().map(|&count| count as i64).sum(); + if actual_total != self.total_vectors { + return Err(invalid_data(format!( + "IVF-RQ list counts total {actual_total} does not match header {}", + self.total_vectors + ))); + } + + self.rotated_centroids = vec![0.0; self.nlist * self.padded_d]; + let mut scratch = vec![0.0; self.padded_d]; for list_id in 0..self.nlist { - self.list_offsets[list_id] = read_i64_le(&mut cursor)?; - let count = read_i32_le(&mut cursor)?; - if count < 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("negative list count {} at list {}", count, list_id), - )); - } - self.list_counts[list_id] = count; - let id_bytes_len = read_i32_le(&mut cursor)?; - if id_bytes_len < 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("negative id_bytes_len {} at list {}", id_bytes_len, list_id), - )); - } - self.list_id_bytes_lens[list_id] = id_bytes_len; + self.rotation.rotate( + &self.quantizer_centroids[list_id * self.d..(list_id + 1) * self.d], + &mut self.rotated_centroids[list_id * self.padded_d..(list_id + 1) * self.padded_d], + &mut scratch, + ); } self.loaded = true; Ok(()) @@ -345,124 +376,95 @@ impl IVFRQIndexReader { pub fn read_inverted_list(&mut self, list_id: usize) -> io::Result { self.ensure_loaded()?; let Some(meta) = self.list_payload_meta(list_id)? else { - return Ok(RQReadList { - list_id, - ids: Vec::new(), - codes: Vec::new(), - factors: Vec::new(), - }); + return Ok(RQReadList::empty(list_id)); }; let mut payload = vec![0u8; meta.payload_len]; self.reader .pread(&mut [ReadRequest::new(meta.offset, &mut payload)])?; - self.decode_inverted_list_payload(meta, &payload) + self.decode_inverted_list_payload(meta, payload) } fn read_inverted_lists(&mut self, list_ids: &[usize]) -> io::Result> { self.ensure_loaded()?; - let mut results: Vec> = (0..list_ids.len()).map(|_| None).collect(); let mut metas = Vec::new(); let mut payloads = Vec::new(); - for (input_index, &list_id) in list_ids.iter().enumerate() { - let Some(meta) = self.list_payload_meta(list_id)? else { - results[input_index] = Some(RQReadList { - list_id, - ids: Vec::new(), - codes: Vec::new(), - factors: Vec::new(), - }); - continue; - }; - metas.push((input_index, meta)); - payloads.push(vec![0u8; meta.payload_len]); + if let Some(meta) = self.list_payload_meta(list_id)? { + metas.push((input_index, meta)); + payloads.push(vec![0; meta.payload_len]); + } else { + results[input_index] = Some(RQReadList::empty(list_id)); + } } - if !metas.is_empty() { - { - let mut requests: Vec<_> = payloads - .iter_mut() - .zip(metas.iter()) - .map(|(payload, (_, meta))| { - ReadRequest::new(meta.offset, payload.as_mut_slice()) - }) - .collect(); - self.reader.pread(&mut requests)?; - } - + let offsets = metas + .iter() + .map(|(_, meta)| meta.offset) + .collect::>(); + pread_batched_payloads(&mut self.reader, &offsets, &mut payloads)?; for ((input_index, meta), payload) in metas.into_iter().zip(payloads) { - results[input_index] = Some(self.decode_inverted_list_payload(meta, &payload)?); + results[input_index] = Some(self.decode_inverted_list_payload(meta, payload)?); } } - results .into_iter() - .map(|result| { - result.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "missing IVF-RQ inverted list read result", - ) - }) - }) + .map(|result| result.ok_or_else(|| invalid_data("missing IVF-RQ list result"))) .collect() } fn batch_read_end(&self, list_ids: &[usize]) -> io::Result { let mut payload_bytes = 0usize; + let mut request_count = 0usize; + let max_ranges = match self.reader.read_capabilities().max_ranges_per_pread { + 0 => usize::MAX, + value => value, + }; for (index, &list_id) in list_ids.iter().enumerate() { let Some(meta) = self.list_payload_meta(list_id)? else { continue; }; - let next_payload_bytes = - payload_bytes.checked_add(meta.payload_len).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "IVF-RQ batched payload size overflow", - ) - })?; - if index > 0 && next_payload_bytes > MAX_RQ_BATCH_READ_BYTES { + if request_count >= max_ranges { return Ok(index); } - payload_bytes = next_payload_bytes; + let next = payload_bytes + .checked_add(meta.payload_len) + .ok_or_else(|| invalid_data("IVF-RQ batch payload overflow"))?; + if index > 0 && next > MAX_RQ_BATCH_READ_BYTES { + return Ok(index); + } + payload_bytes = next; + request_count += 1; } Ok(list_ids.len()) } fn list_payload_meta(&self, list_id: usize) -> io::Result> { if list_id >= self.nlist { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("list_id {} out of range (nlist={})", list_id, self.nlist), - )); + return Err(invalid_input(format!( + "list ID {list_id} out of range for nlist={}", + self.nlist + ))); } let count = self.list_counts[list_id] as usize; if count == 0 { return Ok(None); } - if !self.delta_ids { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "IVF-RQ reader only supports delta IDs", - )); - } - let offset = checked_list_offset(self.list_offsets[list_id], list_id)?; let id_bytes_len = self.list_id_bytes_lens[list_id] as usize; - let code_bytes = checked_list_bytes(count, self.code_size)?; - let factor_bytes = checked_list_bytes(count, FACTOR_BYTES)?; - let payload_len = 12usize + let code_bytes_len = checked_list_bytes(count, self.quantizer.code_size())?; + let factor_values_len = checked_list_bytes(count, self.quantizer.factor_fields())?; + let payload_len = 16usize .checked_add(id_bytes_len) - .and_then(|len| len.checked_add(code_bytes)) - .and_then(|len| len.checked_add(factor_bytes)) - .ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidData, "IVF-RQ list payload overflow") - })?; + .and_then(|size| size.checked_add(code_bytes_len)) + .and_then(|size| size.checked_add(factor_values_len * FACTOR_BYTES)) + .ok_or_else(|| invalid_data("IVF-RQ list payload overflow"))?; Ok(Some(RQListPayloadMeta { list_id, count, - offset, + offset: checked_list_offset(self.list_offsets[list_id], list_id)?, id_bytes_len, + code_bytes_len, + factor_values_len, payload_len, })) } @@ -470,37 +472,39 @@ impl IVFRQIndexReader { fn decode_inverted_list_payload( &self, meta: RQListPayloadMeta, - payload: &[u8], + mut payload: Vec, ) -> io::Result { if payload.len() != meta.payload_len { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "IVF-RQ list {} payload length {} does not match expected {}", - meta.list_id, - payload.len(), - meta.payload_len - ), - )); + return Err(invalid_data(format!( + "IVF-RQ list {} payload length mismatch", + meta.list_id + ))); } let base_id = i64::from_le_bytes(payload[0..8].try_into().unwrap()); - let encoded_len = i32::from_le_bytes(payload[8..12].try_into().unwrap()); - if encoded_len < 0 || encoded_len as usize != meta.id_bytes_len { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "IVF-RQ id_bytes_len mismatch", - )); + let id_bytes_len = i32::from_le_bytes(payload[8..12].try_into().unwrap()); + let code_bytes_len = i32::from_le_bytes(payload[12..16].try_into().unwrap()); + if id_bytes_len < 0 || id_bytes_len as usize != meta.id_bytes_len { + return Err(invalid_data("IVF-RQ delta ID length mismatch")); + } + if code_bytes_len < 0 || code_bytes_len as usize != meta.code_bytes_len { + return Err(invalid_data("IVF-RQ code length mismatch")); } let ids = - decode_delta_varint_ids(base_id, &payload[12..12 + meta.id_bytes_len], meta.count)?; - let code_start = 12 + meta.id_bytes_len; - let code_bytes = checked_list_bytes(meta.count, self.code_size)?; - let factor_start = code_start + code_bytes; + decode_delta_varint_ids(base_id, &payload[16..16 + meta.id_bytes_len], meta.count)?; + let code_start = 16 + meta.id_bytes_len; + let factor_start = code_start + meta.code_bytes_len; + let factors = bytes_to_f32_vec(&payload[factor_start..])?; + if factors.len() != meta.factor_values_len { + return Err(invalid_data("IVF-RQ factor count mismatch")); + } + payload.truncate(factor_start); Ok(RQReadList { list_id: meta.list_id, ids, - codes: payload[code_start..factor_start].to_vec(), - factors: bytes_to_factors(&payload[factor_start..])?, + blocked_code_start: code_start, + blocked_code_end: factor_start, + payload, + blocked_factors: factors, }) } @@ -510,17 +514,7 @@ impl IVFRQIndexReader { k: usize, nprobe: usize, ) -> io::Result<(Vec, Vec)> { - self.search_with_query_bits(query, k, nprobe, DEFAULT_RQ_QUERY_BITS) - } - - pub fn search_with_query_bits( - &mut self, - query: &[f32], - k: usize, - nprobe: usize, - query_bits: usize, - ) -> io::Result<(Vec, Vec)> { - self.search_with_filter(query, k, nprobe, None, query_bits) + self.search_with_filter(query, k, nprobe, None) } pub fn search_with_filter( @@ -529,11 +523,8 @@ impl IVFRQIndexReader { k: usize, nprobe: usize, filter: Option<&dyn RowIdFilter>, - query_bits: usize, ) -> io::Result<(Vec, Vec)> { - search_batch_ivfrq_reader_filter_with_query_bits( - self, query, 1, k, nprobe, filter, query_bits, - ) + search_batch_ivfrq_reader_filter(self, query, 1, k, nprobe, filter) } pub fn search_with_roaring_filter( @@ -544,84 +535,56 @@ impl IVFRQIndexReader { roaring_filter_bytes: &[u8], ) -> io::Result<(Vec, Vec)> { let filter = decode_roaring_filter(roaring_filter_bytes)?; - self.search_with_filter(query, k, nprobe, Some(&filter), DEFAULT_RQ_QUERY_BITS) - } - - pub fn search_with_roaring_filter_and_query_bits( - &mut self, - query: &[f32], - k: usize, - nprobe: usize, - roaring_filter_bytes: &[u8], - query_bits: usize, - ) -> io::Result<(Vec, Vec)> { - let filter = decode_roaring_filter(roaring_filter_bytes)?; - self.search_with_filter(query, k, nprobe, Some(&filter), query_bits) + self.search_with_filter(query, k, nprobe, Some(&filter)) } } pub struct RQReadList { pub list_id: usize, pub ids: Vec, - pub codes: Vec, - pub factors: Vec, + payload: Vec, + blocked_code_start: usize, + blocked_code_end: usize, + pub blocked_factors: Vec, } -pub fn search_batch_ivfrq_reader( - reader: &mut IVFRQIndexReader, - queries: &[f32], - nq: usize, - k: usize, - nprobe: usize, -) -> io::Result<(Vec, Vec)> { - search_batch_ivfrq_reader_with_query_bits(reader, queries, nq, k, nprobe, DEFAULT_RQ_QUERY_BITS) -} +impl RQReadList { + fn blocked_codes(&self) -> &[u8] { + &self.payload[self.blocked_code_start..self.blocked_code_end] + } -pub fn search_batch_ivfrq_reader_with_query_bits( - reader: &mut IVFRQIndexReader, - queries: &[f32], - nq: usize, - k: usize, - nprobe: usize, - query_bits: usize, -) -> io::Result<(Vec, Vec)> { - search_batch_ivfrq_reader_filter_with_query_bits( - reader, queries, nq, k, nprobe, None, query_bits, - ) + fn empty(list_id: usize) -> Self { + Self { + list_id, + ids: Vec::new(), + payload: Vec::new(), + blocked_code_start: 0, + blocked_code_end: 0, + blocked_factors: Vec::new(), + } + } } -pub fn search_batch_ivfrq_reader_filter( +pub fn search_batch_ivfrq_reader( reader: &mut IVFRQIndexReader, queries: &[f32], nq: usize, k: usize, nprobe: usize, - filter: Option<&dyn RowIdFilter>, ) -> io::Result<(Vec, Vec)> { - search_batch_ivfrq_reader_filter_with_query_bits( - reader, - queries, - nq, - k, - nprobe, - filter, - DEFAULT_RQ_QUERY_BITS, - ) + search_batch_ivfrq_reader_filter(reader, queries, nq, k, nprobe, None) } -pub fn search_batch_ivfrq_reader_filter_with_query_bits( +pub fn search_batch_ivfrq_reader_filter( reader: &mut IVFRQIndexReader, queries: &[f32], nq: usize, k: usize, nprobe: usize, filter: Option<&dyn RowIdFilter>, - query_bits: usize, ) -> io::Result<(Vec, Vec)> { reader.ensure_loaded()?; validate_search_inputs(queries, nq, reader.d, k, nprobe)?; - validate_query_bits(query_bits)?; - let processed = preprocess_vectors(queries, nq, reader.d, reader.metric); let (all_probe_indices, _) = kmeans::find_topk_batch( &processed, @@ -632,64 +595,105 @@ pub fn search_batch_ivfrq_reader_filter_with_query_bits( nprobe, ); - let mut list_to_queries = vec![Vec::new(); reader.nlist]; + let mut query_contexts = Vec::with_capacity(nq); + let mut rotated = vec![0.0; reader.padded_d]; + let mut scratch = vec![0.0; reader.padded_d]; + for query in processed.chunks_exact(reader.d) { + reader.rotation.rotate(query, &mut rotated, &mut scratch); + query_contexts.push(reader.quantizer.prepare_query(rotated.clone())); + } + + let mut seen_lists = vec![false; reader.nlist]; let mut unique_lists = Vec::new(); - for (qi, probe_indices) in all_probe_indices.iter().enumerate() { - for &list_id in probe_indices { - if list_to_queries[list_id].is_empty() { + for probes in &all_probe_indices { + for &list_id in probes { + if !seen_lists[list_id] { + seen_lists[list_id] = true; unique_lists.push(list_id); } - list_to_queries[list_id].push(qi); } } let mut heaps: Vec = (0..nq).map(|_| TopKHeap::new(k)).collect(); let mut batch_start = 0; while batch_start < unique_lists.len() { - let batch_end = batch_start + reader.batch_read_end(&unique_lists[batch_start..])?; - for read_list in reader.read_inverted_lists(&unique_lists[batch_start..batch_end])? { - let list_id = read_list.list_id; - let count = read_list.ids.len(); - if count == 0 { - continue; - } - let centroid = - &reader.quantizer_centroids[list_id * reader.d..(list_id + 1) * reader.d]; - for &qi in &list_to_queries[list_id] { - let query = &processed[qi * reader.d..(qi + 1) * reader.d]; - let rotated_query_residual = - rotated_residual(query, centroid, reader.d, &reader.rotation); - let distance_context = reader.quantizer.prepare_distance_context_with_query_bits( - rotated_query_residual, - query, - count >= RQ_BYTE_LUT_MIN_LIST_SIZE, - query_bits, - ); - scan_read_list( - &read_list, - &reader.quantizer, - reader.code_size, - reader.metric, - &distance_context, - filter, - &mut heaps[qi], - ); + let count = reader.batch_read_end(&unique_lists[batch_start..])?; + let batch_end = batch_start + count; + let loaded_lists = reader.read_inverted_lists(&unique_lists[batch_start..batch_end])?; + let mut list_positions = vec![usize::MAX; reader.nlist]; + for (position, list) in loaded_lists.iter().enumerate() { + list_positions[list.list_id] = position; + } + let quantizer = &reader.quantizer; + let metric = reader.metric; + let rotated_centroids = &reader.rotated_centroids; + let padded_d = reader.padded_d; + let candidate_count = loaded_lists + .iter() + .map(|list| list.ids.len()) + .sum::(); + if nq == 1 && candidate_count >= PARALLEL_RQ_SCAN_MIN_CANDIDATES { + let per_list_results = loaded_lists + .par_iter() + .map(|list| { + let mut heap = TopKHeap::new(k); + let list_id = list.list_id; + let rotated_centroid = + &rotated_centroids[list_id * padded_d..(list_id + 1) * padded_d]; + scan_blocked_list( + list, + quantizer, + metric, + &query_contexts[0], + rotated_centroid, + filter, + &mut heap, + ); + heap.into_sorted() + }) + .collect::>(); + for results in per_list_results { + for (distance, row_id) in results { + heaps[0].push(distance, row_id); + } } + } else { + heaps + .par_iter_mut() + .enumerate() + .for_each(|(query_index, heap)| { + for &list_id in &all_probe_indices[query_index] { + let position = list_positions[list_id]; + if position == usize::MAX { + continue; + } + let rotated_centroid = + &rotated_centroids[list_id * padded_d..(list_id + 1) * padded_d]; + scan_blocked_list( + &loaded_lists[position], + quantizer, + metric, + &query_contexts[query_index], + rotated_centroid, + filter, + heap, + ); + } + }); } batch_start = batch_end; } - let mut result_ids = vec![-1i64; nq * k]; - let mut result_dists = vec![f32::MAX; nq * k]; - for qi in 0..nq { - let sorted = std::mem::replace(&mut heaps[qi], TopKHeap::new(0)).into_sorted(); - let base = qi * k; - for (i, &(dist, id)) in sorted.iter().enumerate() { - result_ids[base + i] = id; - result_dists[base + i] = dist; + let mut result_ids = vec![-1; nq * k]; + let mut result_distances = vec![f32::MAX; nq * k]; + for query_index in 0..nq { + let sorted = std::mem::replace(&mut heaps[query_index], TopKHeap::new(0)).into_sorted(); + for (rank, &(distance, id)) in sorted.iter().enumerate() { + result_ids[query_index * k + rank] = id; + result_distances[query_index * k + rank] = distance; } } - Ok((result_ids, result_dists)) + Ok((result_ids, result_distances)) } pub fn search_batch_ivfrq_reader_roaring_filter( @@ -699,411 +703,372 @@ pub fn search_batch_ivfrq_reader_roaring_filter( k: usize, nprobe: usize, roaring_filter_bytes: &[u8], -) -> io::Result<(Vec, Vec)> { - search_batch_ivfrq_reader_roaring_filter_with_query_bits( - reader, - queries, - nq, - k, - nprobe, - roaring_filter_bytes, - DEFAULT_RQ_QUERY_BITS, - ) -} - -pub fn search_batch_ivfrq_reader_roaring_filter_with_query_bits( - reader: &mut IVFRQIndexReader, - queries: &[f32], - nq: usize, - k: usize, - nprobe: usize, - roaring_filter_bytes: &[u8], - query_bits: usize, ) -> io::Result<(Vec, Vec)> { let filter = decode_roaring_filter(roaring_filter_bytes)?; - search_batch_ivfrq_reader_filter_with_query_bits( - reader, - queries, - nq, - k, - nprobe, - Some(&filter), - query_bits, - ) + search_batch_ivfrq_reader_filter(reader, queries, nq, k, nprobe, Some(&filter)) } -fn scan_read_list( - read_list: &RQReadList, +fn scan_blocked_list( + list: &RQReadList, quantizer: &RaBitQuantizer, - code_size: usize, metric: MetricType, - distance_context: &RQDistanceContext, + query: &RQQueryContext, + rotated_centroid: &[f32], filter: Option<&dyn RowIdFilter>, heap: &mut TopKHeap, ) { - for (local_idx, &id) in read_list.ids.iter().enumerate() { - if filter.map(|f| !f.contains(id)).unwrap_or(false) { + let blocked_codes = list.blocked_codes(); + let query_terms = quantizer.query_terms(query, rotated_centroid, metric); + let plane_size = quantizer.plane_size(); + let bits = quantizer.bits(); + let code_size = quantizer.code_size(); + let factor_fields = quantizer.factor_fields(); + let query_sum = quantizer.query_sum(query); + let center = ((1usize << bits) - 1) as f32 * 0.5; + + let mut block_start = 0usize; + while block_start < list.ids.len() { + let lanes = (list.ids.len() - block_start).min(RQ_SCAN_BLOCK_SIZE); + let code_block_start = block_start * code_size; + let factor_block_start = block_start * factor_fields; + let mut allowed = [true; RQ_SCAN_BLOCK_SIZE]; + let mut coarse_unsigned = [0.0f32; RQ_SCAN_BLOCK_SIZE]; + if let Some(filter) = filter { + for lane in 0..lanes { + allowed[lane] = filter.contains(list.ids[block_start + lane]); + } + for byte_idx in 0..plane_size { + let byte_start = code_block_start + byte_idx * lanes; + for lane in 0..lanes { + if allowed[lane] { + coarse_unsigned[lane] += quantizer.byte_subset_sum( + query, + byte_idx, + blocked_codes[byte_start + lane], + ); + } + } + } + } else { + for byte_idx in 0..plane_size { + let byte_start = code_block_start + byte_idx * lanes; + for lane in 0..lanes { + coarse_unsigned[lane] += quantizer.byte_subset_sum( + query, + byte_idx, + blocked_codes[byte_start + lane], + ); + } + } + } + + let mut refine = [false; RQ_SCAN_BLOCK_SIZE]; + if bits == 1 { + for lane in 0..lanes { + if !allowed[lane] { + continue; + } + let factors = read_block_factor(list, factor_block_start, lanes, 0, lane, false); + let distance = quantizer.estimate( + coarse_unsigned[lane] - 0.5 * query_sum, + factors, + query_terms, + ); + if heap.should_consider(distance) { + heap.push(distance, list.ids[block_start + lane]); + } + } + block_start += lanes; continue; } - let code = &read_list.codes[local_idx * code_size..(local_idx + 1) * code_size]; - let dist = quantizer.distance_to_code_prepared( - distance_context, - code, - read_list.factors[local_idx], - metric, - ); - heap.push(dist, id); + + for lane in 0..lanes { + if !allowed[lane] { + continue; + } + let factors = read_block_factor(list, factor_block_start, lanes, 0, lane, true); + let estimate = quantizer.estimate( + coarse_unsigned[lane] - 0.5 * query_sum, + factors, + query_terms, + ); + refine[lane] = + heap.should_consider(quantizer.lower_bound(estimate, factors, query_terms)); + } + + let mut full_unsigned = [0.0f32; RQ_SCAN_BLOCK_SIZE]; + let sign_weight = (1usize << (bits - 1)) as f32; + for lane in 0..lanes { + full_unsigned[lane] = coarse_unsigned[lane] * sign_weight; + } + for plane in 1..bits { + let weight = (1usize << (bits - 1 - plane)) as f32; + let plane_start = code_block_start + plane * plane_size * lanes; + for byte_idx in 0..plane_size { + let byte_start = plane_start + byte_idx * lanes; + for lane in 0..lanes { + if refine[lane] { + full_unsigned[lane] += weight + * quantizer.byte_subset_sum( + query, + byte_idx, + blocked_codes[byte_start + lane], + ); + } + } + } + } + for lane in 0..lanes { + if !refine[lane] { + continue; + } + let factors = read_block_factor(list, factor_block_start, lanes, 3, lane, false); + let distance = quantizer.estimate( + full_unsigned[lane] - center * query_sum, + factors, + query_terms, + ); + if heap.should_consider(distance) { + heap.push(distance, list.ids[block_start + lane]); + } + } + block_start += lanes; } } -fn rotated_residual(query: &[f32], centroid: &[f32], d: usize, rotation: &RQRotation) -> Vec { - let mut residual = vec![0.0f32; d]; - fvec_madd(query, centroid, -1.0, &mut residual); - rotation.apply(&mut residual); - residual +fn read_block_factor( + list: &RQReadList, + block_start: usize, + lanes: usize, + first_field: usize, + lane: usize, + has_error: bool, +) -> RQCodeFactors { + RQCodeFactors { + f_add: list.blocked_factors[block_start + first_field * lanes + lane], + f_rescale: list.blocked_factors[block_start + (first_field + 1) * lanes + lane], + f_error: if has_error { + list.blocked_factors[block_start + (first_field + 2) * lanes + lane] + } else { + 0.0 + }, + } } -fn validate_query_bits(query_bits: usize) -> io::Result<()> { - if is_supported_query_bits(query_bits) { - Ok(()) - } else { - Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "invalid IVF-RQ query_bits {}; expected 0, 4, or 8", - query_bits - ), - )) +fn factor_value(factors: RQVectorFactors, field: usize, bits: usize) -> f32 { + match (bits, field) { + (_, 0) => factors.coarse.f_add, + (_, 1) => factors.coarse.f_rescale, + (2.., 2) => factors.coarse.f_error, + (2.., 3) => factors.full.f_add, + (2.., 4) => factors.full.f_rescale, + _ => unreachable!("factor field is validated by the caller"), } } -fn validate_rq_format( - num_bits: u32, - rotation_type: u32, - factor_layout: u32, - format_flags: u32, -) -> io::Result<()> { - if !matches!(num_bits, 1 | 2 | 4 | 8) { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "unsupported IVF-RQ num_bits {}; expected 1, 2, 4, or 8", - num_bits - ), - )); - } - if rotation_type != IVF_RQ_ROTATION_TYPE_KAC { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "unsupported IVF-RQ rotation_type {}; expected {}", - rotation_type, IVF_RQ_ROTATION_TYPE_KAC - ), - )); - } - if factor_layout != IVF_RQ_FACTOR_LAYOUT_RABITQ_V1 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "unsupported IVF-RQ factor_layout {}; expected {}", - factor_layout, IVF_RQ_FACTOR_LAYOUT_RABITQ_V1 - ), - )); - } - let unknown_format_flags = format_flags & !SUPPORTED_FORMAT_FLAGS; - if unknown_format_flags != 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "unsupported IVF-RQ format_flags: 0x{:08X}", - unknown_format_flags - ), - )); - } - if num_bits != IVF_RQ_NUM_BITS_ONE || format_flags != CURRENT_FORMAT_FLAGS { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "IVF-RQ reader currently supports only num_bits=1 without optional sections; got num_bits={}, format_flags=0x{:08X}", - num_bits, format_flags - ), - )); +fn index_factors_fields(bits: usize) -> usize { + if bits == 1 { + 2 + } else { + 5 } - Ok(()) } fn validate_index_shape(index: &IVFRQIndex) -> io::Result<()> { - if index.d == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "dimension must be greater than 0", - )); + if index.d == 0 || index.nlist == 0 { + return Err(invalid_input("IVF-RQ dimension and nlist must be positive")); } - if index.nlist == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "nlist must be greater than 0", - )); + if !is_supported_rq_bits(index.bits) { + return Err(invalid_input("IVF-RQ bits must be in 1..=8")); + } + if index.padded_d != padded_dimension(index.d) { + return Err(invalid_input("IVF-RQ padded dimension mismatch")); + } + if index.rotation_rounds != DEFAULT_RQ_ROTATION_ROUNDS { + return Err(invalid_input(format!( + "IVF-RQ rotation rounds must be {DEFAULT_RQ_ROTATION_ROUNDS}" + ))); + } + if index.quantizer_centroids.len() != checked_section_size(index.nlist, index.d)? + || index.rotated_centroids.len() != checked_section_size(index.nlist, index.padded_d)? + { + return Err(invalid_input("IVF-RQ centroid storage shape mismatch")); } if index.ids.len() != index.nlist || index.codes.len() != index.nlist || index.factors.len() != index.nlist { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "IVF-RQ list storage does not match nlist", - )); - } - let centroid_len = checked_section_size(index.nlist, index.d)?; - if index.quantizer_centroids.len() != centroid_len { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "centroid length {} does not match nlist*d {}", - index.quantizer_centroids.len(), - centroid_len - ), - )); + return Err(invalid_input("IVF-RQ list storage does not match nlist")); } - let code_size = index.code_size(); for list_id in 0..index.nlist { let count = index.ids[list_id].len(); - let expected_code_len = checked_list_bytes(count, code_size)?; - if index.codes[list_id].len() != expected_code_len { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "list {} code length {} does not match ids*code_size {}", - list_id, - index.codes[list_id].len(), - expected_code_len - ), - )); + if index.codes[list_id].len() != checked_list_bytes(count, index.code_size())? { + return Err(invalid_input(format!( + "IVF-RQ list {list_id} code length mismatch" + ))); } if index.factors[list_id].len() != count { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "list {} factor count {} does not match ids {}", - list_id, - index.factors[list_id].len(), - count - ), - )); + return Err(invalid_input(format!( + "IVF-RQ list {list_id} factor count mismatch" + ))); } } Ok(()) } -fn write_factors(out: &mut dyn SeekWrite, factors: &[RQCodeFactors]) -> io::Result<()> { - let mut bytes = Vec::with_capacity(factors.len() * FACTOR_BYTES); - for factor in factors { - bytes.extend_from_slice(&factor.residual_norm_sqr.to_le_bytes()); - bytes.extend_from_slice(&factor.vector_norm_sqr.to_le_bytes()); - bytes.extend_from_slice(&factor.dp_multiplier.to_le_bytes()); - } - out.write_all(&bytes) -} - -fn bytes_to_factors(bytes: &[u8]) -> io::Result> { - if !bytes.len().is_multiple_of(FACTOR_BYTES) { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "IVF-RQ factor section is not 12-byte aligned", - )); - } - Ok(bytes - .chunks_exact(FACTOR_BYTES) - .map(|chunk| RQCodeFactors { - residual_norm_sqr: f32::from_le_bytes(chunk[0..4].try_into().unwrap()), - vector_norm_sqr: f32::from_le_bytes(chunk[4..8].try_into().unwrap()), - dp_multiplier: f32::from_le_bytes(chunk[8..12].try_into().unwrap()), - }) - .collect()) +fn write_u32_le(out: &mut dyn SeekWrite, value: u32) -> io::Result<()> { + out.write_all(&value.to_le_bytes()) } -fn write_u32_le(out: &mut dyn SeekWrite, v: u32) -> io::Result<()> { - out.write_all(&v.to_le_bytes()) +fn write_i32_le(out: &mut dyn SeekWrite, value: i32) -> io::Result<()> { + out.write_all(&value.to_le_bytes()) } -fn write_i32_le(out: &mut dyn SeekWrite, v: i32) -> io::Result<()> { - out.write_all(&v.to_le_bytes()) +fn write_i64_le(out: &mut dyn SeekWrite, value: i64) -> io::Result<()> { + out.write_all(&value.to_le_bytes()) } -fn write_i64_le(out: &mut dyn SeekWrite, v: i64) -> io::Result<()> { - out.write_all(&v.to_le_bytes()) +fn write_u64_le(out: &mut dyn SeekWrite, value: u64) -> io::Result<()> { + out.write_all(&value.to_le_bytes()) } -fn write_u64_le(out: &mut dyn SeekWrite, v: u64) -> io::Result<()> { - out.write_all(&v.to_le_bytes()) -} - -fn write_f32_slice(out: &mut dyn SeekWrite, data: &[f32]) -> io::Result<()> { - let bytes: Vec = data.iter().flat_map(|f| f.to_le_bytes()).collect(); +fn write_f32_slice(out: &mut dyn SeekWrite, values: &[f32]) -> io::Result<()> { + let mut bytes = Vec::with_capacity(values.len() * 4); + for value in values { + bytes.extend_from_slice(&value.to_le_bytes()); + } out.write_all(&bytes) } fn read_u32_le(reader: &mut PreadCursor<'_, R>) -> io::Result { - let mut buf = [0u8; 4]; - reader.read_exact(&mut buf)?; - Ok(u32::from_le_bytes(buf)) + let mut bytes = [0; 4]; + reader.read_exact(&mut bytes)?; + Ok(u32::from_le_bytes(bytes)) } fn read_i32_le(reader: &mut PreadCursor<'_, R>) -> io::Result { - let mut buf = [0u8; 4]; - reader.read_exact(&mut buf)?; - Ok(i32::from_le_bytes(buf)) + let mut bytes = [0; 4]; + reader.read_exact(&mut bytes)?; + Ok(i32::from_le_bytes(bytes)) } fn read_i64_le(reader: &mut PreadCursor<'_, R>) -> io::Result { - let mut buf = [0u8; 8]; - reader.read_exact(&mut buf)?; - Ok(i64::from_le_bytes(buf)) + let mut bytes = [0; 8]; + reader.read_exact(&mut bytes)?; + Ok(i64::from_le_bytes(bytes)) } fn read_u64_le(reader: &mut PreadCursor<'_, R>) -> io::Result { - let mut buf = [0u8; 8]; - reader.read_exact(&mut buf)?; - Ok(u64::from_le_bytes(buf)) + let mut bytes = [0; 8]; + reader.read_exact(&mut bytes)?; + Ok(u64::from_le_bytes(bytes)) } -fn validate_positive_i32(val: i32, field: &str) -> io::Result { - if val <= 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("invalid header field {}: {} (must be positive)", field, val), - )); +fn bytes_to_f32_vec(bytes: &[u8]) -> io::Result> { + if !bytes.len().is_multiple_of(4) { + return Err(invalid_data("IVF-RQ f32 section is not aligned")); } - Ok(val) + Ok(bytes + .chunks_exact(4) + .map(|chunk| f32::from_le_bytes(chunk.try_into().unwrap())) + .collect()) } -fn usize_to_i32(value: usize, field: &str) -> io::Result { - if value > i32::MAX as usize { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("{} exceeds i32 length limit: {}", field, value), - )); +fn validate_positive_i32(value: i32, field: &str) -> io::Result { + if value <= 0 { + Err(invalid_data(format!( + "invalid IVF-RQ {field}: {value}; expected positive" + ))) + } else { + Ok(value) } - Ok(value as i32) } -fn usize_to_i64(value: usize, field: &str) -> io::Result { - if value > i64::MAX as usize { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("{} exceeds i64 length limit: {}", field, value), - )); +fn validate_non_negative_i32(value: i32, field: &str) -> io::Result { + if value < 0 { + Err(invalid_data(format!( + "invalid IVF-RQ {field}: {value}; expected non-negative" + ))) + } else { + Ok(value) } - Ok(value as i64) } -fn u64_to_i64(value: u64, field: &str) -> io::Result { - if value > i64::MAX as u64 { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("{} exceeds i64 offset limit: {}", field, value), - )); - } - Ok(value as i64) +fn usize_to_i32(value: usize, field: &str) -> io::Result { + i32::try_from(value).map_err(|_| invalid_input(format!("{field} exceeds i32"))) } -const MAX_SECTION_ELEMENTS: usize = 1 << 30; +fn usize_to_u32(value: usize, field: &str) -> io::Result { + u32::try_from(value).map_err(|_| invalid_input(format!("{field} exceeds u32"))) +} -fn checked_section_size(a: usize, b: usize) -> io::Result { - let result = a.checked_mul(b).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "section size overflow in IVF-RQ header", - ) - })?; - if result > MAX_SECTION_ELEMENTS { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "section size {} exceeds maximum {}", - result, MAX_SECTION_ELEMENTS - ), - )); - } - Ok(result) +fn usize_to_i64(value: usize, field: &str) -> io::Result { + i64::try_from(value).map_err(|_| invalid_input(format!("{field} exceeds i64"))) } -fn checked_list_offset(offset: i64, list_id: usize) -> io::Result { - if offset < 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("negative list offset {} at list {}", offset, list_id), - )); - } - Ok(offset as u64) +fn u64_to_i64(value: u64, field: &str) -> io::Result { + i64::try_from(value).map_err(|_| invalid_input(format!("{field} exceeds i64"))) +} + +fn checked_section_size(left: usize, right: usize) -> io::Result { + left.checked_mul(right) + .filter(|&size| size <= 1 << 30) + .ok_or_else(|| invalid_data("IVF-RQ section size overflow")) } fn checked_list_bytes(count: usize, bytes_per_entry: usize) -> io::Result { count .checked_mul(bytes_per_entry) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "IVF-RQ list byte size overflow")) + .ok_or_else(|| invalid_data("IVF-RQ list byte size overflow")) } -fn read_f32_vec( - reader: &mut PreadCursor<'_, R>, - count: usize, -) -> io::Result> { - let mut buf = vec![0u8; count * 4]; - reader.read_exact(&mut buf)?; - bytes_to_f32_vec(&buf) +fn checked_list_offset(offset: i64, list_id: usize) -> io::Result { + u64::try_from(offset).map_err(|_| { + invalid_data(format!( + "negative IVF-RQ list offset {offset} for list {list_id}" + )) + }) } -fn bytes_to_f32_vec(bytes: &[u8]) -> io::Result> { - if !bytes.len().is_multiple_of(4) { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "f32 byte section is not 4-byte aligned", - )); - } - Ok(bytes - .chunks_exact(4) - .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) - .collect()) +fn decode_roaring_filter(bytes: &[u8]) -> io::Result { + RoaringTreemap::deserialize_from(bytes) + .map_err(|error| invalid_input(format!("invalid RoaringTreemap filter: {error}"))) } -fn decode_roaring_filter(bytes: &[u8]) -> io::Result { - RoaringTreemap::deserialize_from(bytes).map_err(|e| { - io::Error::new( - io::ErrorKind::InvalidInput, - format!("invalid RoaringTreemap filter: {}", e), - ) - }) +fn invalid_input(message: impl Into) -> io::Error { + io::Error::new(io::ErrorKind::InvalidInput, message.into()) +} + +fn invalid_data(message: impl Into) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message.into()) } #[cfg(test)] mod tests { use super::*; use crate::io::{PosWriter, ReadRequest, SeekRead}; + use crate::ivfpq::RowIdFilter; + use std::collections::HashSet; use std::io::{Cursor, Read, Seek, SeekFrom}; + use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; #[derive(Default)] struct ReaderStats { + calls: usize, max_ranges_per_batch: usize, } - struct CountingPreadCursor { + struct CountingReader { inner: Cursor>, stats: Arc>, + max_ranges_per_pread: usize, } - impl CountingPreadCursor { - fn new(data: Vec, stats: Arc>) -> Self { - Self { - inner: Cursor::new(data), - stats, - } - } - } - - impl SeekRead for CountingPreadCursor { + impl SeekRead for CountingReader { fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { let mut stats = self.stats.lock().unwrap(); + stats.calls += 1; stats.max_ranges_per_batch = stats.max_ranges_per_batch.max(ranges.len()); drop(stats); for range in ranges { @@ -1112,193 +1077,325 @@ mod tests { } Ok(()) } + + fn read_capabilities(&self) -> crate::io::SeekReadCapabilities { + crate::io::SeekReadCapabilities { + max_ranges_per_pread: self.max_ranges_per_pread, + ..crate::io::SeekReadCapabilities::default() + } + } } #[test] - fn ivfrq_write_read_search_roundtrip() { - let d = 8; + fn ivfrq_four_bit_roundtrip_uses_blocked_layout() { + let d = 13; let nlist = 4; let n = 128; let data: Vec = (0..n) .flat_map(|i| { - let cluster = (i % nlist) as f32 * 100.0; - [cluster + i as f32 * 0.01, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0] + let cluster = (i % nlist) as f32 * 50.0; + (0..d).map(move |dim| cluster + i as f32 * 0.01 + dim as f32) }) .collect(); let ids: Vec = (1000..1000 + n as i64).collect(); - let mut index = IVFRQIndex::new(d, nlist, MetricType::L2); + let mut index = IVFRQIndex::with_bits(d, nlist, 4, MetricType::L2); index.train(&data, n); index.add(&data, &ids, n); - let mut buf = Vec::new(); - let mut writer = PosWriter::new(&mut buf); - write_ivfrq_index(&index, &mut writer).unwrap(); - - let mut reader = IVFRQIndexReader::open(Cursor::new(buf)).unwrap(); - let (labels, distances) = reader.search(&data[7 * d..8 * d], 5, nlist).unwrap(); + let mut bytes = Vec::new(); + write_ivfrq_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let mut reader = IVFRQIndexReader::open(Cursor::new(bytes)).unwrap(); + let (labels, distances) = reader.search(&data[37 * d..38 * d], 5, nlist).unwrap(); - assert_eq!(labels[0], ids[7]); - assert!(distances[0] <= 1e-4); + assert_eq!(reader.num_bits, 4); + assert_eq!(reader.padded_d, 64); + assert_eq!(labels[0], ids[37]); + assert!(distances[0].abs() <= 1e-3); } #[test] - fn ivfrq_reader_search_with_query_bits() { - let d = 16; - let nlist = 4; - let n = 128; - let data: Vec = (0..n) + fn ivfrq_batch_reader_submits_multiple_lists_together() { + let d = 64; + let nlist = 3; + let data: Vec = (0..96) .flat_map(|i| { - let cluster = (i % nlist) as f32 * 25.0; - (0..d).map(move |j| cluster + i as f32 * 0.02 + j as f32 * 0.125) + let cluster = (i % nlist) as f32 * 100.0; + (0..d).map(move |dim| cluster + dim as f32 * 0.01) }) .collect(); - let ids: Vec = (1000..1000 + n as i64).collect(); - let mut index = IVFRQIndex::new(d, nlist, MetricType::L2); + let ids: Vec = (0..96).collect(); + let mut index = IVFRQIndex::with_bits(d, nlist, 4, MetricType::L2); + index.train(&data, 96); + index.add(&data, &ids, 96); + let mut bytes = Vec::new(); + write_ivfrq_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + + let stats = Arc::new(Mutex::new(ReaderStats::default())); + let stream = CountingReader { + inner: Cursor::new(bytes), + stats: Arc::clone(&stats), + max_ranges_per_pread: 0, + }; + let mut reader = IVFRQIndexReader::open(stream).unwrap(); + search_batch_ivfrq_reader(&mut reader, &data[..d], 1, 1, nlist).unwrap(); + + assert_eq!(stats.lock().unwrap().max_ranges_per_batch, nlist); + } + + #[test] + fn ivfrq_unfiltered_scan_matches_all_rows_filter() { + let d = 64; + let nlist = 4; + let n = 256; + let nq = 8; + let data = (0..n) + .flat_map(|row| { + let cluster = (row % nlist) as f32 * 40.0; + (0..d).map(move |dimension| cluster + row as f32 * 0.003 + dimension as f32 * 0.01) + }) + .collect::>(); + let ids = (0..n as i64).collect::>(); + let mut index = IVFRQIndex::with_bits(d, nlist, 4, MetricType::L2); index.train(&data, n); index.add(&data, &ids, n); + let mut bytes = Vec::new(); + write_ivfrq_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); - let mut buf = Vec::new(); - let mut writer = PosWriter::new(&mut buf); - write_ivfrq_index(&index, &mut writer).unwrap(); - - let query = &data[37 * d..38 * d]; - for query_bits in [0, 4, 8] { - let mut reader = IVFRQIndexReader::open(Cursor::new(buf.clone())).unwrap(); - let (labels, distances) = reader - .search_with_query_bits(query, 5, nlist, query_bits) + let mut unfiltered_reader = IVFRQIndexReader::open(Cursor::new(bytes.clone())).unwrap(); + let unfiltered = + search_batch_ivfrq_reader(&mut unfiltered_reader, &data[..nq * d], nq, 10, nlist) .unwrap(); - assert_eq!(labels[0], ids[37], "query_bits={}", query_bits); - if query_bits == 0 { - assert!(distances[0] <= 1e-3); - } else { - assert!(distances[0].is_finite()); - } - } + let all_rows = ids.iter().copied().collect::>(); + let mut filtered_reader = IVFRQIndexReader::open(Cursor::new(bytes)).unwrap(); + let filtered = search_batch_ivfrq_reader_filter( + &mut filtered_reader, + &data[..nq * d], + nq, + 10, + nlist, + Some(&all_rows), + ) + .unwrap(); + + assert_eq!(unfiltered, filtered); } #[test] - fn ivfrq_batch_reader_reads_lists_in_one_pread_batch() { - let mut index = IVFRQIndex::new(8, 3, MetricType::L2); - index.quantizer_centroids = vec![0.0; index.nlist * index.d]; - index.ids = vec![vec![10], vec![20], vec![30]]; - index.codes = vec![vec![0b0000_0001], vec![0b0000_0010], vec![0b0000_0100]]; - index.factors = vec![ - vec![RQCodeFactors::zero()], - vec![RQCodeFactors::zero()], - vec![RQCodeFactors::zero()], - ]; - + fn ivfrq_resident_metadata_load_uses_one_read_round() { + let d = 64; + let nlist = 8; + let data = (0..256) + .flat_map(|i| { + let cluster = (i % nlist) as f32 * 100.0; + (0..d).map(move |dim| cluster + dim as f32 * 0.01) + }) + .collect::>(); + let ids = (0..256).collect::>(); + let mut index = IVFRQIndex::with_bits(d, nlist, 4, MetricType::L2); + index.train(&data, 256); + index.add(&data, &ids, 256); let mut bytes = Vec::new(); write_ivfrq_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let stats = Arc::new(Mutex::new(ReaderStats::default())); - let stream = CountingPreadCursor::new(bytes, Arc::clone(&stats)); + let stream = CountingReader { + inner: Cursor::new(bytes), + stats: Arc::clone(&stats), + max_ranges_per_pread: 0, + }; let mut reader = IVFRQIndexReader::open(stream).unwrap(); + *stats.lock().unwrap() = ReaderStats::default(); + reader.ensure_loaded().unwrap(); - let (ids, _) = search_batch_ivfrq_reader(&mut reader, &[0.0; 8], 1, 1, 3).unwrap(); + assert_eq!(stats.lock().unwrap().calls, 1); + } + + struct ThreadTrackingFilter { + workers: AtomicU64, + } - assert!(ids[0] >= 0); - assert_eq!(stats.lock().unwrap().max_ranges_per_batch, 3); + impl RowIdFilter for ThreadTrackingFilter { + fn contains(&self, _id: i64) -> bool { + if let Some(worker) = rayon::current_thread_index() { + self.workers.fetch_or(1u64 << worker, Ordering::Relaxed); + } + true + } } #[test] - fn ivfrq_reader_rejects_invalid_query_bits() { - let d = 8; - let nlist = 1; - let n = 8; + fn ivfrq_batch_scans_queries_in_parallel_without_duplicate_reads() { + let d = 64; + let nlist = 8; + let nq = 64; + let n = 4096; let data: Vec = (0..n) - .flat_map(|i| (0..d).map(move |j| i as f32 + j as f32 * 0.25)) + .flat_map(|i| { + let cluster = (i % nlist) as f32 * 50.0; + (0..d).map(move |dim| cluster + i as f32 * 0.001 + dim as f32 * 0.01) + }) .collect(); - let ids: Vec = (0..n as i64).collect(); - let mut index = IVFRQIndex::new(d, nlist, MetricType::L2); + let ids = (0..n as i64).collect::>(); + let mut index = IVFRQIndex::with_bits(d, nlist, 4, MetricType::L2); index.train(&data, n); index.add(&data, &ids, n); + let mut bytes = Vec::new(); + write_ivfrq_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); - let mut buf = Vec::new(); - let mut writer = PosWriter::new(&mut buf); - write_ivfrq_index(&index, &mut writer).unwrap(); + let stats = Arc::new(Mutex::new(ReaderStats::default())); + let stream = CountingReader { + inner: Cursor::new(bytes), + stats: Arc::clone(&stats), + max_ranges_per_pread: 0, + }; + let mut reader = IVFRQIndexReader::open(stream).unwrap(); + let queries = data[..nq * d].to_vec(); + let filter = ThreadTrackingFilter { + workers: AtomicU64::new(0), + }; + rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .unwrap() + .install(|| { + search_batch_ivfrq_reader_filter( + &mut reader, + &queries, + nq, + 10, + nlist, + Some(&filter), + ) + .unwrap(); + }); - let mut reader = IVFRQIndexReader::open(Cursor::new(buf)).unwrap(); - let err = reader - .search_with_query_bits(&data[0..d], 1, nlist, 7) - .unwrap_err(); - assert!(err.to_string().contains("query_bits")); + assert!( + filter.workers.load(Ordering::Relaxed).count_ones() > 1, + "batch scan should use more than one Rayon worker" + ); + assert_eq!(stats.lock().unwrap().max_ranges_per_batch, nlist); } #[test] - fn ivfrq_header_records_current_rq_format_fields() { - let index = tiny_ivfrq_index(); + fn ivfrq_batch_respects_reader_range_capability() { + let d = 64; + let nlist = 5; + let data = (0..160) + .flat_map(|i| { + let cluster = (i % nlist) as f32 * 100.0; + (0..d).map(move |dim| cluster + dim as f32 * 0.01) + }) + .collect::>(); + let ids = (0..160).collect::>(); + let mut index = IVFRQIndex::with_bits(d, nlist, 4, MetricType::L2); + index.train(&data, 160); + index.add(&data, &ids, 160); + let mut bytes = Vec::new(); + write_ivfrq_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + + let stats = Arc::new(Mutex::new(ReaderStats::default())); + let stream = CountingReader { + inner: Cursor::new(bytes), + stats: Arc::clone(&stats), + max_ranges_per_pread: 2, + }; + let mut reader = IVFRQIndexReader::open(stream).unwrap(); + reader.search(&data[..d], 1, nlist).unwrap(); - let mut buf = Vec::new(); - let mut writer = PosWriter::new(&mut buf); - write_ivfrq_index(&index, &mut writer).unwrap(); + assert_eq!(stats.lock().unwrap().max_ranges_per_batch, 2); + } + #[test] + fn ivfrq_header_rejects_the_unreleased_one_bit_layout() { + let mut old_header = vec![0u8; IVF_RQ_HEADER_SIZE]; + old_header[0..4].copy_from_slice(&IVF_RQ_MAGIC.to_le_bytes()); + old_header[4..8].copy_from_slice(&IVF_RQ_VERSION.to_le_bytes()); + old_header[8..12].copy_from_slice(&8i32.to_le_bytes()); + old_header[12..16].copy_from_slice(&1i32.to_le_bytes()); + + let error = match IVFRQIndexReader::open(Cursor::new(old_header)) { + Ok(_) => panic!("old IVF-RQ layout must be rejected"), + Err(error) => error, + }; + assert!(error.to_string().contains("padded_dimension")); + } + + #[test] + fn ivfrq_header_records_data_bits_and_new_layout() { + let d = 64; + let data = vec![0.0; d * 2]; + let mut index = IVFRQIndex::with_bits(d, 1, 5, MetricType::L2); + index.train(&data, 2); + index.add(&data, &[7, 9], 2); + let mut bytes = Vec::new(); + write_ivfrq_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + + assert_eq!(u32::from_le_bytes(bytes[28..32].try_into().unwrap()), 5); assert_eq!( - u32::from_le_bytes(buf[48..52].try_into().unwrap()), - IVF_RQ_NUM_BITS_ONE - ); - assert_eq!( - u32::from_le_bytes(buf[52..56].try_into().unwrap()), - IVF_RQ_ROTATION_TYPE_KAC + u32::from_le_bytes(bytes[56..60].try_into().unwrap()), + IVF_RQ_ROTATION_TYPE_BLOCK_FHT ); assert_eq!( - u32::from_le_bytes(buf[56..60].try_into().unwrap()), - IVF_RQ_FACTOR_LAYOUT_RABITQ_V1 + u32::from_le_bytes(bytes[60..64].try_into().unwrap()), + IVF_RQ_FACTOR_LAYOUT_COMPACT_V1 ); - assert_eq!(u32::from_le_bytes(buf[60..64].try_into().unwrap()), 0); - - let reader = IVFRQIndexReader::open(Cursor::new(buf)).unwrap(); - assert_eq!(reader.num_bits, IVF_RQ_NUM_BITS_ONE); - assert_eq!(reader.rotation_type, IVF_RQ_ROTATION_TYPE_KAC); - assert_eq!(reader.factor_layout, IVF_RQ_FACTOR_LAYOUT_RABITQ_V1); - assert_eq!(reader.format_flags, 0); } #[test] - fn ivfrq_reader_rejects_reserved_future_format_fields() { - let index = tiny_ivfrq_index(); - - let mut buf = Vec::new(); - let mut writer = PosWriter::new(&mut buf); - write_ivfrq_index(&index, &mut writer).unwrap(); - - let mut future_num_bits = buf.clone(); - future_num_bits[48..52].copy_from_slice(&2u32.to_le_bytes()); - let err = open_err(future_num_bits); - assert!(err.to_string().contains("num_bits=1")); - - let mut unknown_rotation = buf.clone(); - unknown_rotation[52..56].copy_from_slice(&99u32.to_le_bytes()); - let err = open_err(unknown_rotation); - assert!(err.to_string().contains("rotation_type")); - - let mut unknown_layout = buf.clone(); - unknown_layout[56..60].copy_from_slice(&99u32.to_le_bytes()); - let err = open_err(unknown_layout); - assert!(err.to_string().contains("factor_layout")); - - let mut optional_sections = buf; - optional_sections[60..64].copy_from_slice(&FORMAT_FLAG_EX_CODES_PRESENT.to_le_bytes()); - let err = open_err(optional_sections); - assert!(err.to_string().contains("optional sections")); - } + fn ivfrq_four_bit_storage_omits_the_unused_full_error_factor() { + let d = 64; + let count = 32; + let data = (0..count * d) + .map(|offset| ((offset * 17) % 101) as f32) + .collect::>(); + let ids = (0..count as i64).collect::>(); + let mut index = IVFRQIndex::with_bits(d, 1, 4, MetricType::L2); + index.train(&data, count); + index.add(&data, &ids, count); - fn open_err(buf: Vec) -> io::Error { - match IVFRQIndexReader::open(Cursor::new(buf)) { - Ok(_) => panic!("expected IVF-RQ open to fail"), - Err(err) => err, - } + let mut bytes = Vec::new(); + write_ivfrq_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + + let offset_entry = IVF_RQ_HEADER_SIZE + d * size_of::(); + let list_offset = + i64::from_le_bytes(bytes[offset_entry..offset_entry + 8].try_into().unwrap()) as usize; + let id_bytes = i32::from_le_bytes( + bytes[offset_entry + 12..offset_entry + 16] + .try_into() + .unwrap(), + ) as usize; + let expected_factor_bytes = count * 5 * size_of::(); + let expected_list_bytes = 16 + id_bytes + count * index.code_size() + expected_factor_bytes; + + assert_eq!(bytes.len() - list_offset, expected_list_bytes); } - fn tiny_ivfrq_index() -> IVFRQIndex { - let d = 8; - let nlist = 2; - let data: Vec = vec![ - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, - ]; - let ids = vec![7, 42]; - let mut index = IVFRQIndex::new(d, nlist, MetricType::L2); - index.train(&data, 2); - index.add(&data, &ids, 2); - index + #[test] + fn ivfrq_one_bit_storage_omits_the_unused_coarse_error_factor() { + let d = 64; + let count = 32; + let data = (0..count * d) + .map(|offset| ((offset * 19) % 103) as f32) + .collect::>(); + let ids = (0..count as i64).collect::>(); + let mut index = IVFRQIndex::with_bits(d, 1, 1, MetricType::L2); + index.train(&data, count); + index.add(&data, &ids, count); + + let mut bytes = Vec::new(); + write_ivfrq_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + + let offset_entry = IVF_RQ_HEADER_SIZE + d * size_of::(); + let list_offset = + i64::from_le_bytes(bytes[offset_entry..offset_entry + 8].try_into().unwrap()) as usize; + let id_bytes = i32::from_le_bytes( + bytes[offset_entry + 12..offset_entry + 16] + .try_into() + .unwrap(), + ) as usize; + let expected_factor_bytes = count * 2 * size_of::(); + let expected_list_bytes = 16 + id_bytes + count * index.code_size() + expected_factor_bytes; + + assert_eq!(bytes.len() - list_offset, expected_list_bytes); } } diff --git a/core/src/ivfhnswsq.rs b/core/src/ivfsq.rs similarity index 54% rename from core/src/ivfhnswsq.rs rename to core/src/ivfsq.rs index a9f5292..6d16191 100644 --- a/core/src/ivfhnswsq.rs +++ b/core/src/ivfsq.rs @@ -15,17 +15,17 @@ // specific language governing permissions and limitations // under the License. +//! IVF with per-list, per-dimension 8-bit residual scalar quantization. + use crate::distance::{fvec_madd, preprocess_vectors, MetricType}; -use crate::hnsw::{HnswBuildParams, HnswGraph}; -use crate::hnsw_search::{search_hnsw_lists, HnswSearchList}; use crate::ivfpq::RowIdFilter; use crate::kmeans::{self, KMeansConfig}; use crate::sq::ScalarQuantizer; use crate::topk::TopKHeap; use rayon::prelude::*; -use std::io; +use std::borrow::Cow; -pub struct IVFHNSWSQIndex { +pub struct IVFSQIndex { pub d: usize, pub nlist: usize, pub metric: MetricType, @@ -34,12 +34,10 @@ pub struct IVFHNSWSQIndex { pub list_sqs: Vec, pub ids: Vec>, pub codes: Vec>, - pub graphs: Vec>, - pub hnsw_params: HnswBuildParams, } -impl IVFHNSWSQIndex { - pub fn new(d: usize, nlist: usize, metric: MetricType, hnsw_params: HnswBuildParams) -> Self { +impl IVFSQIndex { + pub fn new(d: usize, nlist: usize, metric: MetricType) -> Self { Self { d, nlist, @@ -49,8 +47,6 @@ impl IVFHNSWSQIndex { list_sqs: vec![ScalarQuantizer::new(d); nlist], ids: vec![Vec::new(); nlist], codes: vec![Vec::new(); nlist], - graphs: vec![None; nlist], - hnsw_params, } } @@ -65,40 +61,66 @@ impl IVFHNSWSQIndex { pub fn add(&mut self, data: &[f32], ids: &[i64], n: usize) { let processed = self.preprocess_vectors(data, n); - let code_size = self.code_size(); - let (list_ids, residuals) = self.assign_residuals(&processed, n); + let list_ids = kmeans::find_nearest_batch( + &processed, + n, + &self.quantizer_centroids, + self.nlist, + self.d, + ); + let mut list_rows = vec![Vec::new(); self.nlist]; + for (row, list_id) in list_ids.into_iter().enumerate() { + list_rows[list_id].push(row); + } - let mut code = vec![0u8; code_size]; - for i in 0..n { - let list_id = list_ids[i]; - self.list_sqs[list_id].encode(&residuals[i * self.d..(i + 1) * self.d], &mut code); - self.ids[list_id].push(ids[i]); - self.codes[list_id].extend_from_slice(&code); + let d = self.d; + let centroids = &self.quantizer_centroids; + let list_sqs = &self.list_sqs; + let output_ids = &mut self.ids; + let output_codes = &mut self.codes; + if n > 1_000 && self.nlist > 1 { + output_ids + .par_iter_mut() + .zip(output_codes.par_iter_mut()) + .zip(list_rows.into_par_iter()) + .enumerate() + .for_each(|(list_id, ((list_ids, list_codes), rows))| { + append_encoded_rows( + &processed, + ids, + &rows, + d, + ¢roids[list_id * d..(list_id + 1) * d], + &list_sqs[list_id], + list_ids, + list_codes, + ); + }); + } else { + for (list_id, ((list_ids, list_codes), rows)) in output_ids + .iter_mut() + .zip(output_codes.iter_mut()) + .zip(list_rows) + .enumerate() + { + append_encoded_rows( + &processed, + ids, + &rows, + d, + ¢roids[list_id * d..(list_id + 1) * d], + &list_sqs[list_id], + list_ids, + list_codes, + ); + } } - self.graphs.fill(None); } pub fn total_vectors(&self) -> usize { self.ids.iter().map(Vec::len).sum() } - pub fn build_graphs(&mut self) -> io::Result<()> { - self.graphs = (0..self.nlist) - .into_par_iter() - .map(|list_id| { - let count = self.ids[list_id].len(); - if count == 0 { - Ok(None) - } else { - let vectors = self.decode_list_vectors(list_id, count); - HnswGraph::build_owned(vectors, count, self.d, self.metric, self.hnsw_params) - .map(Some) - } - }) - .collect::>>()?; - Ok(()) - } - #[allow(clippy::too_many_arguments)] pub fn search( &self, @@ -106,7 +128,6 @@ impl IVFHNSWSQIndex { nq: usize, k: usize, nprobe: usize, - ef_search: usize, result_distances: &mut [f32], result_labels: &mut [i64], ) { @@ -115,7 +136,6 @@ impl IVFHNSWSQIndex { nq, k, nprobe, - ef_search, None, result_distances, result_labels, @@ -129,7 +149,6 @@ impl IVFHNSWSQIndex { nq: usize, k: usize, nprobe: usize, - ef_search: usize, filter: Option<&dyn RowIdFilter>, result_distances: &mut [f32], result_labels: &mut [i64], @@ -146,17 +165,11 @@ impl IVFHNSWSQIndex { for qi in 0..nq { let query = &processed_queries[qi * self.d..(qi + 1) * self.d]; - let lists: Vec<_> = all_probe_indices[qi] - .iter() - .map(|&list_id| HnswSearchList { - ids: self.ids[list_id].as_slice(), - graph: self.graphs[list_id].as_ref(), - payload: list_id, - }) - .collect(); - let sorted = search_hnsw_lists(query, &lists, k, ef_search, filter, |list, heap| { - self.scan_sq_list(query, list.payload, filter, heap); - }); + let mut heap = TopKHeap::new(k); + for &list_id in &all_probe_indices[qi] { + self.scan_list(query, list_id, filter, &mut heap); + } + let sorted = heap.into_sorted(); let out_base = qi * k; for (i, &(dist, id)) in sorted.iter().enumerate() { result_distances[out_base + i] = dist; @@ -169,45 +182,39 @@ impl IVFHNSWSQIndex { } } - pub(crate) fn preprocess_vectors(&self, data: &[f32], n: usize) -> Vec { - preprocess_vectors(data, n, self.d, self.metric) + pub(crate) fn preprocess_vectors<'a>(&self, data: &'a [f32], n: usize) -> Cow<'a, [f32]> { + match self.metric { + MetricType::Cosine => { + Cow::Owned(preprocess_vectors(data, n, self.d, MetricType::Cosine)) + } + MetricType::L2 | MetricType::InnerProduct => Cow::Borrowed(&data[..n * self.d]), + } } - fn scan_sq_list( + fn scan_list( &self, query: &[f32], list_id: usize, filter: Option<&dyn RowIdFilter>, heap: &mut TopKHeap, ) { - let context = self.sq.distance_context(query, self.metric); let sq = self.list_sq(list_id); - let code_size = self.code_size(); + let context = sq.distance_context(query, self.metric); let centroid = self.list_centroid(list_id); + let code_size = self.code_size(); for (local_id, &row_id) in self.ids[list_id].iter().enumerate() { if filter.map(|f| !f.contains(row_id)).unwrap_or(false) { continue; } let code = &self.codes[list_id][local_id * code_size..(local_id + 1) * code_size]; - heap.push( - sq.distance_to_code_with_offset_with_context(query, code, centroid, context), - row_id, - ); + let distance = + sq.distance_to_code_with_offset_with_context(query, code, centroid, context); + if heap.should_consider(distance) { + heap.push(distance, row_id); + } } } - pub(crate) fn decode_list_vectors(&self, list_id: usize, count: usize) -> Vec { - let mut vectors = vec![0.0f32; count * self.d]; - let centroid = self.list_centroid(list_id); - self.list_sq(list_id).decode_batch_with_offset( - &self.codes[list_id], - count, - centroid, - &mut vectors, - ); - vectors - } - fn assign_residuals(&self, processed: &[f32], n: usize) -> (Vec, Vec) { let list_ids = kmeans::find_nearest_batch(processed, n, &self.quantizer_centroids, self.nlist, self.d); @@ -240,11 +247,10 @@ impl IVFHNSWSQIndex { } fn write_residual(&self, vector: &[f32], list_id: usize, out: &mut [f32]) { - let centroid = self.list_centroid(list_id); - fvec_madd(vector, centroid, -1.0, out); + fvec_madd(vector, self.list_centroid(list_id), -1.0, out); } - fn list_centroid(&self, list_id: usize) -> &[f32] { + pub(crate) fn list_centroid(&self, list_id: usize) -> &[f32] { &self.quantizer_centroids[list_id * self.d..(list_id + 1) * self.d] } @@ -257,14 +263,37 @@ impl IVFHNSWSQIndex { } } +#[allow(clippy::too_many_arguments)] +fn append_encoded_rows( + data: &[f32], + input_ids: &[i64], + rows: &[usize], + d: usize, + centroid: &[f32], + sq: &ScalarQuantizer, + output_ids: &mut Vec, + output_codes: &mut Vec, +) { + output_ids.reserve(rows.len()); + output_codes.reserve(rows.len().saturating_mul(d)); + let mut residual = vec![0.0f32; d]; + let mut code = vec![0u8; d]; + for &row in rows { + let vector = &data[row * d..(row + 1) * d]; + fvec_madd(vector, centroid, -1.0, &mut residual); + sq.encode(&residual, &mut code); + output_ids.push(input_ids[row]); + output_codes.extend_from_slice(&code); + } +} + #[cfg(test)] mod tests { use super::*; - use crate::hnsw::HnswBuildParams; use std::collections::HashSet; #[test] - fn test_ivfhnswsq_recalls_query_vector() { + fn ivfsq_full_scan_recalls_added_vector() { let d = 4; let nlist = 4; let n = 128; @@ -279,22 +308,19 @@ mod tests { ] }) .collect(); - let ids: Vec = (1000..1000 + n as i64).collect(); - - let mut index = IVFHNSWSQIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); + let ids: Vec = (10_000..10_000 + n as i64).collect(); + let mut index = IVFSQIndex::new(d, nlist, MetricType::L2); index.train(&data, n); index.add(&data, &ids, n); - index.build_graphs().unwrap(); + let mut labels = vec![-1; 5]; + let mut distances = vec![f32::MAX; 5]; let query_id = 23; - let mut distances = vec![0.0; 5]; - let mut labels = vec![0; 5]; index.search( &data[query_id * d..(query_id + 1) * d], 1, 5, nlist, - 32, &mut distances, &mut labels, ); @@ -304,48 +330,77 @@ mod tests { } #[test] - fn test_ivfhnswsq_without_built_graphs_falls_back_to_sq_scan() { - let d = 2; - let nlist = 1; - let data = vec![0.0, 0.0, 0.1, 0.0, 10.0, 10.0]; - let ids = vec![10, 11, 12]; - let mut index = IVFHNSWSQIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); - index.train(&data, 3); - index.add(&data, &ids, 3); - - let mut distances = vec![0.0; 2]; - let mut labels = vec![0; 2]; - index.search(&[0.0, 0.0], 1, 2, nlist, 8, &mut distances, &mut labels); - - assert_eq!(labels[0], 10); - } - - #[test] - fn test_ivfhnswsq_search_with_filter() { - let d = 2; - let nlist = 1; + fn ivfsq_filter_is_applied_during_scan() { let data = vec![0.0, 0.0, 0.1, 0.0, 10.0, 10.0]; let ids = vec![10, 11, 12]; - let mut index = IVFHNSWSQIndex::new(d, nlist, MetricType::L2, HnswBuildParams::default()); + let mut index = IVFSQIndex::new(2, 1, MetricType::L2); index.train(&data, 3); index.add(&data, &ids, 3); - index.build_graphs().unwrap(); + let allowed = HashSet::from([12]); - let filter: HashSet = [12].into_iter().collect(); - let mut distances = vec![0.0; 2]; - let mut labels = vec![0; 2]; + let mut labels = vec![-1; 2]; + let mut distances = vec![f32::MAX; 2]; index.search_with_filter( &[0.0, 0.0], 1, 2, - nlist, - 8, - Some(&filter), + 1, + Some(&allowed), &mut distances, &mut labels, ); - assert_eq!(labels[0], 12); - assert_eq!(labels[1], -1); + assert_eq!(labels, vec![12, -1]); + } + + #[test] + fn ivfsq_preprocessing_borrows_non_cosine_input() { + let data = vec![3.0, 4.0, 1.0, 2.0]; + let l2 = IVFSQIndex::new(2, 1, MetricType::L2); + let ip = IVFSQIndex::new(2, 1, MetricType::InnerProduct); + let cosine = IVFSQIndex::new(2, 1, MetricType::Cosine); + + assert!(matches!(l2.preprocess_vectors(&data, 2), Cow::Borrowed(_))); + assert!(matches!(ip.preprocess_vectors(&data, 2), Cow::Borrowed(_))); + assert!(matches!(cosine.preprocess_vectors(&data, 2), Cow::Owned(_))); + } + + #[test] + fn ivfsq_parallel_incremental_add_keeps_ids_aligned_with_codes() { + let d = 8; + let nlist = 8; + let n = 2_048; + let data = (0..n) + .flat_map(|row| { + (0..d).map(move |dimension| { + (row % nlist) as f32 * 100.0 + row as f32 * 0.003 + dimension as f32 * 0.07 + }) + }) + .collect::>(); + let ids = (10_000..10_000 + n as i64).collect::>(); + let mut index = IVFSQIndex::new(d, nlist, MetricType::L2); + index.train(&data, n); + index.add(&data[..1_024 * d], &ids[..1_024], 1_024); + index.add(&data[1_024 * d..], &ids[1_024..], 1_024); + + let mut residual = vec![0.0f32; d]; + let mut expected_code = vec![0u8; d]; + for list_id in 0..nlist { + for (position, &row_id) in index.ids[list_id].iter().enumerate() { + let row = (row_id - 10_000) as usize; + fvec_madd( + &data[row * d..(row + 1) * d], + index.list_centroid(list_id), + -1.0, + &mut residual, + ); + index.list_sq(list_id).encode(&residual, &mut expected_code); + assert_eq!( + &index.codes[list_id][position * d..(position + 1) * d], + expected_code, + "row ID {row_id} was paired with another row's code" + ); + } + } } } diff --git a/core/src/ivfsq_io.rs b/core/src/ivfsq_io.rs new file mode 100644 index 0000000..541abee --- /dev/null +++ b/core/src/ivfsq_io.rs @@ -0,0 +1,1312 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Stable v1 storage and positional-I/O search for IVF-SQ8. + +use crate::distance::{preprocess_vectors, MetricType}; +use crate::index_io_util::{ + bounded_ivf_payload_batch_end, bounded_ivf_stream_chunk_rows, bytes_to_f32_vec, + checked_list_bytes, checked_list_offset, checked_section_size, decode_delta_varint_ids, + decode_roaring_filter, encode_delta_varint_ids, ivf_payload_is_oversized, + pread_batched_payloads, read_delta_varint_ids_at, u64_to_i64, usize_to_i32, usize_to_i64, + validate_positive_i32, validate_reserved_zero, validate_search_inputs, write_f32_slice, + write_i32_le, write_i64_le, write_u32_le, +}; +use crate::io::{ReadRequest, SeekRead, SeekWrite}; +use crate::ivfpq::RowIdFilter; +use crate::ivfsq::IVFSQIndex; +use crate::kmeans; +use crate::sq::ScalarQuantizer; +use crate::topk::TopKHeap; +use rayon::prelude::*; +use std::io; +use std::mem::size_of; + +pub const IVF_SQ_MAGIC: u32 = 0x49565351; // "IVSQ" +pub const IVF_SQ_VERSION: u32 = 1; +pub const IVF_SQ_HEADER_SIZE: usize = 64; +pub const IVF_SQ_BITS: u32 = 8; +const FLAG_DELTA_IDS: u32 = 1 << 0; +const FLAG_BLOCKED_CODES: u32 = 1 << 1; +const REQUIRED_FLAGS: u32 = FLAG_DELTA_IDS | FLAG_BLOCKED_CODES; +const SUPPORTED_FLAGS: u32 = REQUIRED_FLAGS; +pub(crate) const IVF_SQ_SCAN_BLOCK_SIZE: usize = 32; + +pub fn write_ivfsq_index(index: &IVFSQIndex, out: &mut dyn SeekWrite) -> io::Result<()> { + validate_index_shape(index)?; + let total_vectors = index.ids.iter().try_fold(0i64, |sum, ids| { + let count = usize_to_i64(ids.len(), "total vector count")?; + sum.checked_add(count).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "total vector count exceeds i64 length limit", + ) + }) + })?; + let sorted_lists = (0..index.nlist) + .map(|list_id| build_sorted_sq_list_metadata(index, list_id)) + .collect::>>()?; + + write_u32_le(out, IVF_SQ_MAGIC)?; + write_u32_le(out, IVF_SQ_VERSION)?; + write_i32_le(out, usize_to_i32(index.d, "dimension")?)?; + write_i32_le(out, usize_to_i32(index.nlist, "nlist")?)?; + write_u32_le(out, index.metric as u32)?; + write_i64_le(out, total_vectors)?; + write_u32_le(out, IVF_SQ_BITS)?; + write_u32_le(out, REQUIRED_FLAGS)?; + let (sq_min, sq_max) = sq_global_bounds(&index.sq.mins, &index.sq.maxs); + out.write_all(&sq_min.to_le_bytes())?; + out.write_all(&sq_max.to_le_bytes())?; + out.write_all(&[0u8; 20])?; + + write_f32_slice(out, &index.sq.mins)?; + write_f32_slice(out, &index.sq.maxs)?; + for sq in &index.list_sqs { + write_f32_slice(out, &sq.mins)?; + write_f32_slice(out, &sq.maxs)?; + } + write_f32_slice(out, &index.quantizer_centroids)?; + + let offset_table_size = index.nlist.checked_mul(16).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "IVF-SQ offset table size overflow", + ) + })?; + let data_start = out + .pos() + .checked_add(offset_table_size as u64) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "IVF-SQ data offset overflow") + })?; + let mut list_offsets = vec![0i64; index.nlist]; + let mut list_counts = vec![0i32; index.nlist]; + let mut list_id_bytes_lens = vec![0i32; index.nlist]; + let mut current_offset = data_start; + + for (list_id, list) in sorted_lists.iter().enumerate() { + list_offsets[list_id] = u64_to_i64(current_offset, "list offset")?; + list_counts[list_id] = usize_to_i32(list.order.len(), "list count")?; + if !list.order.is_empty() { + list_id_bytes_lens[list_id] = usize_to_i32(list.id_bytes.len(), "delta ID section")?; + current_offset = current_offset + .checked_add(list_payload_len( + list.order.len(), + index.code_size(), + list.id_bytes.len(), + )? as u64) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "IVF-SQ list offset overflow") + })?; + } + } + + for list_id in 0..index.nlist { + write_i64_le(out, list_offsets[list_id])?; + write_i32_le(out, list_counts[list_id])?; + write_i32_le(out, list_id_bytes_lens[list_id])?; + } + for (list_id, list) in sorted_lists.iter().enumerate() { + if list.order.is_empty() { + continue; + } + let codes = block_sorted_sq_codes( + &index.codes[list_id], + &list.order, + index.d, + IVF_SQ_SCAN_BLOCK_SIZE, + ); + out.write_all(&codes)?; + write_i64_le(out, list.base_id)?; + write_i32_le(out, usize_to_i32(list.id_bytes.len(), "delta ID section")?)?; + out.write_all(&list.id_bytes)?; + } + Ok(()) +} + +pub struct IVFSQIndexReader { + reader: R, + pub d: usize, + pub nlist: usize, + pub metric: MetricType, + pub total_vectors: i64, + pub sq: ScalarQuantizer, + pub list_sqs: Vec, + pub quantizer_centroids: Vec, + pub list_offsets: Vec, + pub list_counts: Vec, + pub list_id_bytes_lens: Vec, + loaded: bool, +} + +impl IVFSQIndexReader { + pub fn open(mut reader: R) -> io::Result { + let mut header = [0u8; IVF_SQ_HEADER_SIZE]; + reader.pread(&mut [ReadRequest::new(0, &mut header)])?; + Self::open_with_header(reader, header) + } + + pub(crate) fn open_with_header( + mut reader: R, + header: [u8; IVF_SQ_HEADER_SIZE], + ) -> io::Result { + let read_u32 = + |offset: usize| u32::from_le_bytes(header[offset..offset + 4].try_into().unwrap()); + let read_i32 = + |offset: usize| i32::from_le_bytes(header[offset..offset + 4].try_into().unwrap()); + let read_i64 = + |offset: usize| i64::from_le_bytes(header[offset..offset + 8].try_into().unwrap()); + let read_f32 = + |offset: usize| f32::from_le_bytes(header[offset..offset + 4].try_into().unwrap()); + + let magic = read_u32(0); + if magic != IVF_SQ_MAGIC { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Invalid IVF-SQ magic: 0x{magic:08X}"), + )); + } + let version = read_u32(4); + if version != IVF_SQ_VERSION { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Unsupported IVF-SQ version: {version}"), + )); + } + let d = validate_positive_i32(read_i32(8), "d")? as usize; + let nlist = validate_positive_i32(read_i32(12), "nlist")? as usize; + let metric_code = read_u32(16); + let metric = MetricType::from_code(metric_code).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("Unknown metric type: {metric_code}"), + ) + })?; + let total_vectors = read_i64(20); + if total_vectors < 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "IVF-SQ total vector count must be non-negative", + )); + } + let bits = read_u32(28); + if bits != IVF_SQ_BITS { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Unsupported IVF-SQ bit width: {bits}"), + )); + } + let flags = read_u32(32); + let sq_min_summary = read_f32(36); + let sq_max_summary = read_f32(40); + validate_reserved_zero(&header[44..64], "IVF-SQ")?; + let unknown_flags = flags & !SUPPORTED_FLAGS; + if unknown_flags != 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Unsupported IVF-SQ flags: 0x{unknown_flags:08X}"), + )); + } + if flags & REQUIRED_FLAGS != REQUIRED_FLAGS { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "IVF-SQ v1 requires delta-varint IDs and 32-row blocked codes", + )); + } + + let bounds_values = checked_section_size(nlist + 1, d)? + .checked_mul(2) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVF-SQ bounds size overflow") + })?; + let bounds_bytes = bounds_values.checked_mul(4).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVF-SQ bounds byte length overflow", + ) + })?; + let centroid_values = checked_section_size(nlist, d)?; + let centroid_bytes = centroid_values.checked_mul(4).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVF-SQ centroid byte length overflow", + ) + })?; + let offset_table_bytes = nlist.checked_mul(16).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVF-SQ offset table byte length overflow", + ) + })?; + let metadata_bytes = bounds_bytes + .checked_add(centroid_bytes) + .and_then(|size| size.checked_add(offset_table_bytes)) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVF-SQ metadata size overflow") + })?; + let mut metadata = vec![0u8; metadata_bytes]; + reader.pread(&mut [ReadRequest::new(IVF_SQ_HEADER_SIZE as u64, &mut metadata)])?; + let (sq, list_sqs, mut position) = { + let mut position = 0usize; + let mut next_f32_section = |count: usize| -> io::Result> { + let byte_len = count.checked_mul(4).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVF-SQ f32 size overflow") + })?; + let end = position.checked_add(byte_len).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVF-SQ metadata offset overflow", + ) + })?; + let values = bytes_to_f32_vec(&metadata[position..end])?; + position = end; + Ok(values) + }; + + let mins = next_f32_section(d)?; + let maxs = next_f32_section(d)?; + validate_sq_bounds(d, &mins, &maxs)?; + let (sq_min, sq_max) = sq_global_bounds(&mins, &maxs); + if sq_min.to_bits() != sq_min_summary.to_bits() + || sq_max.to_bits() != sq_max_summary.to_bits() + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "IVF-SQ bounds summary does not match global SQ bounds", + )); + } + let sq = ScalarQuantizer::with_dimension_bounds(d, mins, maxs); + let mut list_sqs = Vec::with_capacity(nlist); + for _ in 0..nlist { + let mins = next_f32_section(d)?; + let maxs = next_f32_section(d)?; + validate_sq_bounds(d, &mins, &maxs)?; + list_sqs.push(ScalarQuantizer::with_dimension_bounds(d, mins, maxs)); + } + (sq, list_sqs, position) + }; + + let quantizer_centroids = bytes_to_f32_vec(&metadata[position..position + centroid_bytes])?; + position += centroid_bytes; + let offset_table = &metadata[position..]; + let mut list_offsets = vec![0; nlist]; + let mut list_counts = vec![0; nlist]; + let mut list_id_bytes_lens = vec![0; nlist]; + let mut actual_total = 0i64; + for (list_id, entry) in offset_table.chunks_exact(16).enumerate() { + list_offsets[list_id] = i64::from_le_bytes(entry[0..8].try_into().unwrap()); + let count = i32::from_le_bytes(entry[8..12].try_into().unwrap()); + let id_bytes_len = i32::from_le_bytes(entry[12..16].try_into().unwrap()); + if count < 0 || id_bytes_len < 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("negative IVF-SQ list metadata at list {list_id}"), + )); + } + if count > 0 && id_bytes_len == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("missing delta ID bytes for non-empty IVF-SQ list {list_id}"), + )); + } + actual_total = actual_total.checked_add(count as i64).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVF-SQ vector count overflow") + })?; + list_counts[list_id] = count; + list_id_bytes_lens[list_id] = id_bytes_len; + } + if actual_total != total_vectors { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "IVF-SQ header vector count {total_vectors} does not match list total {actual_total}" + ), + )); + } + + Ok(Self { + reader, + d, + nlist, + metric, + total_vectors, + sq, + list_sqs, + quantizer_centroids, + list_offsets, + list_counts, + list_id_bytes_lens, + loaded: true, + }) + } + + pub fn ensure_loaded(&mut self) -> io::Result<()> { + debug_assert!(self.loaded); + Ok(()) + } + + pub fn optimize_for_search(&mut self) -> io::Result<()> { + self.ensure_loaded() + } + + pub fn read_inverted_list(&mut self, list_id: usize) -> io::Result<(Vec, Vec)> { + let mut lists = self.read_inverted_lists(&[list_id])?; + let list = lists.pop().expect("one requested list has one result"); + Ok((list.ids, list.codes)) + } + + pub fn read_inverted_lists(&mut self, list_ids: &[usize]) -> io::Result> { + self.ensure_loaded()?; + let mut results = (0..list_ids.len()).map(|_| None).collect::>(); + let mut metas = Vec::new(); + let mut payloads = Vec::new(); + for (input_index, &list_id) in list_ids.iter().enumerate() { + if list_id >= self.nlist { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("list_id {list_id} out of range (nlist={})", self.nlist), + )); + } + let count = self.list_counts[list_id] as usize; + if count == 0 { + results[input_index] = Some(SqListData { + list_id, + ids: Vec::new(), + codes: Vec::new(), + }); + continue; + } + let id_bytes_len = self.list_id_bytes_lens[list_id] as usize; + let payload_len = list_payload_len(count, self.d, id_bytes_len)?; + metas.push(BatchedListRead { + input_index, + list_id, + count, + id_bytes_len, + offset: checked_list_offset(self.list_offsets[list_id], list_id)?, + }); + payloads.push(vec![0u8; payload_len]); + } + + if !metas.is_empty() { + let offsets = metas.iter().map(|meta| meta.offset).collect::>(); + pread_batched_payloads(&mut self.reader, &offsets, &mut payloads)?; + for (meta, payload) in metas.into_iter().zip(payloads) { + let (ids, codes) = + decode_list_payload(payload, meta.count, meta.id_bytes_len, self.d)?; + results[meta.input_index] = Some(SqListData { + list_id: meta.list_id, + ids, + codes, + }); + } + } + results + .into_iter() + .map(|result| { + result.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "missing batched IVF-SQ list read result", + ) + }) + }) + .collect() + } + + fn batch_read_end(&self, list_ids: &[usize]) -> io::Result { + let payload_lengths = list_ids + .iter() + .map(|&list_id| self.list_payload_len(list_id)) + .collect::>>()?; + bounded_ivf_payload_batch_end( + &payload_lengths, + self.reader.read_capabilities().max_ranges_per_pread, + ) + } + + fn list_payload_len(&self, list_id: usize) -> io::Result { + if list_id >= self.nlist { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("list_id {list_id} out of range (nlist={})", self.nlist), + )); + } + let count = self.list_counts[list_id] as usize; + if count == 0 { + Ok(0) + } else { + list_payload_len(count, self.d, self.list_id_bytes_lens[list_id] as usize) + } + } + + fn for_each_streamed_list_chunk( + &mut self, + list_id: usize, + mut consume: impl FnMut(&[i64], &[u8]), + ) -> io::Result<()> { + self.ensure_loaded()?; + let count = self.list_counts[list_id] as usize; + let list_offset = checked_list_offset(self.list_offsets[list_id], list_id)?; + let code_bytes = checked_list_bytes(count, self.d)?; + let id_offset = list_offset.checked_add(code_bytes as u64).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVF-SQ ID offset overflow") + })?; + let ids = read_delta_varint_ids_at( + &mut self.reader, + id_offset, + count, + self.list_id_bytes_lens[list_id] as usize, + "IVF-SQ", + )?; + let retained_id_bytes = ids.len().checked_mul(size_of::()).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVF-SQ decoded ID size overflow", + ) + })?; + let mut row_start = 0usize; + while row_start < count { + let chunk_rows = bounded_ivf_stream_chunk_rows( + count - row_start, + self.d, + retained_id_bytes, + IVF_SQ_SCAN_BLOCK_SIZE, + )?; + let chunk_bytes = chunk_rows.checked_mul(self.d).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVF-SQ chunk size overflow") + })?; + let chunk_offset = list_offset + .checked_add(row_start.checked_mul(self.d).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVF-SQ chunk offset overflow") + })? as u64) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "IVF-SQ chunk offset overflow") + })?; + let mut codes = vec![0u8; chunk_bytes]; + self.reader + .pread(&mut [ReadRequest::new(chunk_offset, &mut codes)])?; + let row_end = row_start + chunk_rows; + consume(&ids[row_start..row_end], &codes); + row_start = row_end; + } + Ok(()) + } + + pub fn search( + &mut self, + query: &[f32], + k: usize, + nprobe: usize, + ) -> io::Result<(Vec, Vec)> { + self.search_with_filter(query, k, nprobe, None) + } + + pub fn search_with_filter( + &mut self, + query: &[f32], + k: usize, + nprobe: usize, + filter: Option<&dyn RowIdFilter>, + ) -> io::Result<(Vec, Vec)> { + self.ensure_loaded()?; + validate_search_inputs(query, 1, self.d, k, nprobe)?; + let query = preprocess_vectors(query, 1, self.d, self.metric); + let (probe_indices, _) = kmeans::find_topk( + &query, + &self.quantizer_centroids, + self.nlist, + self.d, + nprobe, + ); + let mut heap = TopKHeap::new(k); + let d = self.d; + let metric = self.metric; + let mut batch_start = 0usize; + while batch_start < probe_indices.len() { + let first_list = probe_indices[batch_start]; + if ivf_payload_is_oversized(self.list_payload_len(first_list)?) { + let centroid = + self.quantizer_centroids[first_list * d..(first_list + 1) * d].to_vec(); + let sq = self.list_sqs.get(first_list).unwrap_or(&self.sq).clone(); + let mut scratch = SqScanScratch::default(); + self.for_each_streamed_list_chunk(first_list, |ids, codes| { + scan_sq_rows( + &query, + ids, + codes, + ¢roid, + &sq, + metric, + filter, + &mut scratch, + &mut heap, + ); + })?; + batch_start += 1; + continue; + } + let count = self.batch_read_end(&probe_indices[batch_start..])?.max(1); + let batch_end = (batch_start + count).min(probe_indices.len()); + let lists = self.read_inverted_lists(&probe_indices[batch_start..batch_end])?; + let centroids = &self.quantizer_centroids; + let list_sqs = &self.list_sqs; + let global_sq = &self.sq; + let candidate_count = lists.iter().map(|list| list.ids.len()).sum::(); + if candidate_count >= PARALLEL_SQ_SCAN_MIN_CANDIDATES { + let per_list_results = lists + .par_iter() + .map_init(SqScanScratch::default, |scratch, list| { + let mut local_heap = TopKHeap::new(k); + let list_id = list.list_id; + scan_sq_list( + &query, + list, + ¢roids[list_id * d..(list_id + 1) * d], + list_sqs.get(list_id).unwrap_or(global_sq), + metric, + filter, + scratch, + &mut local_heap, + ); + local_heap.into_sorted() + }) + .collect::>(); + for results in per_list_results { + for (distance, row_id) in results { + heap.push(distance, row_id); + } + } + } else { + let mut scratch = SqScanScratch::default(); + for list in &lists { + let list_id = list.list_id; + scan_sq_list( + &query, + list, + ¢roids[list_id * d..(list_id + 1) * d], + list_sqs.get(list_id).unwrap_or(global_sq), + metric, + filter, + &mut scratch, + &mut heap, + ); + } + } + batch_start = batch_end; + } + Ok(padded_results(heap, k)) + } + + pub fn search_with_roaring_filter( + &mut self, + query: &[f32], + k: usize, + nprobe: usize, + roaring_filter_bytes: &[u8], + ) -> io::Result<(Vec, Vec)> { + let filter = decode_roaring_filter(roaring_filter_bytes)?; + self.search_with_filter(query, k, nprobe, Some(&filter)) + } +} + +pub fn search_batch_ivfsq_reader( + reader: &mut IVFSQIndexReader, + queries: &[f32], + nq: usize, + k: usize, + nprobe: usize, +) -> io::Result<(Vec, Vec)> { + search_batch_ivfsq_reader_filter(reader, queries, nq, k, nprobe, None) +} + +pub fn search_batch_ivfsq_reader_filter( + reader: &mut IVFSQIndexReader, + queries: &[f32], + nq: usize, + k: usize, + nprobe: usize, + filter: Option<&dyn RowIdFilter>, +) -> io::Result<(Vec, Vec)> { + reader.ensure_loaded()?; + validate_search_inputs(queries, nq, reader.d, k, nprobe)?; + let processed = preprocess_vectors(queries, nq, reader.d, reader.metric); + let (all_probe_indices, _) = kmeans::find_topk_batch( + &processed, + nq, + &reader.quantizer_centroids, + reader.nlist, + reader.d, + nprobe, + ); + let mut seen = vec![false; reader.nlist]; + let mut unique_lists = Vec::new(); + for list_ids in &all_probe_indices { + for &list_id in list_ids { + if !seen[list_id] { + seen[list_id] = true; + unique_lists.push(list_id); + } + } + } + let mut list_to_queries = vec![Vec::new(); reader.nlist]; + for (query_index, list_ids) in all_probe_indices.iter().enumerate() { + for &list_id in list_ids { + list_to_queries[list_id].push(query_index); + } + } + let d = reader.d; + let metric = reader.metric; + let mut heaps = (0..nq).map(|_| TopKHeap::new(k)).collect::>(); + // Oversized-list chunks are scanned query-by-query, so one reusable + // distance buffer is sufficient regardless of the batch width. + let mut stream_scratch = SqScanScratch::default(); + let mut batch_start = 0usize; + while batch_start < unique_lists.len() { + let first_list = unique_lists[batch_start]; + if ivf_payload_is_oversized(reader.list_payload_len(first_list)?) { + let query_indices = &list_to_queries[first_list]; + let centroid = + reader.quantizer_centroids[first_list * d..(first_list + 1) * d].to_vec(); + let sq = reader + .list_sqs + .get(first_list) + .unwrap_or(&reader.sq) + .clone(); + reader.for_each_streamed_list_chunk(first_list, |ids, codes| { + for &query_index in query_indices { + let query = &processed[query_index * d..(query_index + 1) * d]; + scan_sq_rows( + query, + ids, + codes, + ¢roid, + &sq, + metric, + filter, + &mut stream_scratch, + &mut heaps[query_index], + ); + } + })?; + batch_start += 1; + continue; + } + let count = reader.batch_read_end(&unique_lists[batch_start..])?.max(1); + let batch_end = (batch_start + count).min(unique_lists.len()); + let loaded_lists = reader.read_inverted_lists(&unique_lists[batch_start..batch_end])?; + let centroids = &reader.quantizer_centroids; + let list_sqs = &reader.list_sqs; + let global_sq = &reader.sq; + let per_list_results = loaded_lists + .par_iter() + .map_init(SqScanScratch::default, |scratch, list| { + let list_id = list.list_id; + list_to_queries[list_id] + .iter() + .map(|&query_index| { + let query = &processed[query_index * d..(query_index + 1) * d]; + let mut heap = TopKHeap::new(k); + scan_sq_list( + query, + list, + ¢roids[list_id * d..(list_id + 1) * d], + list_sqs.get(list_id).unwrap_or(global_sq), + metric, + filter, + scratch, + &mut heap, + ); + (query_index, heap.into_sorted()) + }) + .collect::>() + }) + .collect::>(); + for list_results in per_list_results { + for (query_index, results) in list_results { + for (distance, row_id) in results { + heaps[query_index].push(distance, row_id); + } + } + } + batch_start = batch_end; + } + + let mut result_ids = Vec::with_capacity(nq * k); + let mut result_distances = Vec::with_capacity(nq * k); + for heap in heaps { + let (ids, distances) = padded_results(heap, k); + result_ids.extend(ids); + result_distances.extend(distances); + } + Ok((result_ids, result_distances)) +} + +pub fn search_batch_ivfsq_reader_roaring_filter( + reader: &mut IVFSQIndexReader, + queries: &[f32], + nq: usize, + k: usize, + nprobe: usize, + roaring_filter_bytes: &[u8], +) -> io::Result<(Vec, Vec)> { + let filter = decode_roaring_filter(roaring_filter_bytes)?; + search_batch_ivfsq_reader_filter(reader, queries, nq, k, nprobe, Some(&filter)) +} + +pub struct SqListData { + pub list_id: usize, + pub ids: Vec, + pub codes: Vec, +} + +#[derive(Clone, Copy)] +struct BatchedListRead { + input_index: usize, + list_id: usize, + count: usize, + id_bytes_len: usize, + offset: u64, +} + +#[derive(Default)] +struct SqScanScratch { + parameters: Vec, + distances: Vec, +} + +// Below this point Rayon task setup and per-list heap merging dominate the +// blocked SQ arithmetic. Production-sized lists usually cross the threshold; +// small indexes stay on the lower-overhead sequential path. +const PARALLEL_SQ_SCAN_MIN_CANDIDATES: usize = 8 * 1024; + +fn scan_sq_list( + query: &[f32], + list: &SqListData, + centroid: &[f32], + sq: &ScalarQuantizer, + metric: MetricType, + filter: Option<&dyn RowIdFilter>, + scratch: &mut SqScanScratch, + heap: &mut TopKHeap, +) { + scan_sq_rows( + query, + &list.ids, + &list.codes, + centroid, + sq, + metric, + filter, + scratch, + heap, + ); +} + +fn scan_sq_rows( + query: &[f32], + ids: &[i64], + codes: &[u8], + centroid: &[f32], + sq: &ScalarQuantizer, + metric: MetricType, + filter: Option<&dyn RowIdFilter>, + scratch: &mut SqScanScratch, + heap: &mut TopKHeap, +) { + sq.distances_to_blocked_codes_with_offset( + query, + codes, + ids.len(), + centroid, + metric, + IVF_SQ_SCAN_BLOCK_SIZE, + &mut scratch.parameters, + &mut scratch.distances, + ); + for (&row_id, &distance) in ids.iter().zip(&scratch.distances) { + if filter.map(|f| !f.contains(row_id)).unwrap_or(false) { + continue; + } + if heap.should_consider(distance) { + heap.push(distance, row_id); + } + } +} + +fn padded_results(heap: TopKHeap, k: usize) -> (Vec, Vec) { + let sorted = heap.into_sorted(); + let mut ids = sorted.iter().map(|&(_, id)| id).collect::>(); + let mut distances = sorted + .iter() + .map(|&(distance, _)| distance) + .collect::>(); + ids.resize(k, -1); + distances.resize(k, f32::MAX); + (ids, distances) +} + +fn validate_index_shape(index: &IVFSQIndex) -> io::Result<()> { + if index.d == 0 || index.nlist == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "IVF-SQ dimension and nlist must be greater than zero", + )); + } + validate_sq_bounds(index.d, &index.sq.mins, &index.sq.maxs)?; + if index.list_sqs.len() != index.nlist + || index.ids.len() != index.nlist + || index.codes.len() != index.nlist + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "IVF-SQ inverted-list state does not match nlist", + )); + } + if index.quantizer_centroids.len() != checked_section_size(index.nlist, index.d)? { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "IVF-SQ centroid storage does not match nlist * dimension", + )); + } + for list_id in 0..index.nlist { + validate_sq_bounds( + index.d, + &index.list_sqs[list_id].mins, + &index.list_sqs[list_id].maxs, + )?; + let expected = checked_list_bytes(index.ids[list_id].len(), index.code_size())?; + if index.codes[list_id].len() != expected { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("IVF-SQ code length mismatch at list {list_id}"), + )); + } + } + Ok(()) +} + +fn validate_sq_bounds(d: usize, mins: &[f32], maxs: &[f32]) -> io::Result<()> { + if mins.len() != d || maxs.len() != d { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "IVF-SQ bounds length does not match dimension", + )); + } + for (dimension, (&min, &max)) in mins.iter().zip(maxs).enumerate() { + if !min.is_finite() || !max.is_finite() || min > max { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("invalid IVF-SQ bounds at dimension {dimension}"), + )); + } + } + Ok(()) +} + +fn list_payload_len(count: usize, code_size: usize, id_bytes_len: usize) -> io::Result { + 12usize + .checked_add(id_bytes_len) + .and_then(|value| value.checked_add(count.checked_mul(code_size)?)) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "IVF-SQ list size overflow")) +} + +fn decode_list_payload( + mut payload: Vec, + count: usize, + id_bytes_len: usize, + code_size: usize, +) -> io::Result<(Vec, Vec)> { + let expected = list_payload_len(count, code_size, id_bytes_len)?; + if payload.len() != expected { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "IVF-SQ list payload length mismatch", + )); + } + let code_bytes = checked_list_bytes(count, code_size)?; + let id_header_end = code_bytes.checked_add(12).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "IVF-SQ list ID header offset overflow", + ) + })?; + let base_id = i64::from_le_bytes(payload[code_bytes..code_bytes + 8].try_into().unwrap()); + let stored_id_bytes_len = + i32::from_le_bytes(payload[code_bytes + 8..id_header_end].try_into().unwrap()); + if stored_id_bytes_len < 0 || stored_id_bytes_len as usize != id_bytes_len { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "IVF-SQ list ID length does not match offset table", + )); + } + let ids = decode_delta_varint_ids(base_id, &payload[id_header_end..], count)?; + payload.truncate(code_bytes); + Ok((ids, payload)) +} + +fn build_sorted_sq_list_metadata( + index: &IVFSQIndex, + list_id: usize, +) -> io::Result { + let count = index.ids[list_id].len(); + if count == 0 { + return Ok(SortedSqListMetadata::default()); + } + let mut order = (0..count).collect::>(); + order.sort_by_key(|&position| index.ids[list_id][position]); + let ids = order + .iter() + .map(|&position| index.ids[list_id][position]) + .collect::>(); + let base_id = ids[0]; + let (_, id_bytes) = encode_delta_varint_ids(&ids); + Ok(SortedSqListMetadata { + base_id, + order, + id_bytes, + }) +} + +fn block_sorted_sq_codes( + row_major: &[u8], + order: &[usize], + d: usize, + block_size: usize, +) -> Vec { + debug_assert_eq!(row_major.len(), order.len() * d); + let mut blocked = Vec::with_capacity(row_major.len()); + for block_start in (0..order.len()).step_by(block_size) { + let block_len = (order.len() - block_start).min(block_size); + for dimension in 0..d { + for lane in 0..block_len { + let source_row = order[block_start + lane]; + blocked.push(row_major[source_row * d + dimension]); + } + } + } + blocked +} + +#[derive(Default)] +struct SortedSqListMetadata { + base_id: i64, + order: Vec, + id_bytes: Vec, +} + +fn sq_global_bounds(mins: &[f32], maxs: &[f32]) -> (f32, f32) { + let min = mins.iter().copied().fold(f32::INFINITY, f32::min); + let max = maxs.iter().copied().fold(f32::NEG_INFINITY, f32::max); + if min.is_finite() && max.is_finite() { + (min, max) + } else { + (0.0, 0.0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::io::PosWriter; + use crate::io::ReadRequest; + use roaring::RoaringTreemap; + use std::io::Cursor; + use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + use std::sync::Arc; + + fn build_index(d: usize, nlist: usize, n: usize) -> (IVFSQIndex, Vec, Vec) { + let data = (0..n) + .flat_map(|i| { + (0..d).map(move |dimension| { + (i % nlist) as f32 * 100.0 + i as f32 * 0.01 + dimension as f32 * 0.1 + }) + }) + .collect::>(); + let ids = (10_000..10_000 + n as i64).collect::>(); + let mut index = IVFSQIndex::new(d, nlist, MetricType::L2); + index.train(&data, n); + index.add(&data, &ids, n); + (index, data, ids) + } + + fn serialized_index(index: &IVFSQIndex) -> Vec { + let mut bytes = Vec::new(); + write_ivfsq_index(index, &mut PosWriter::new(&mut bytes)).unwrap(); + bytes + } + + #[test] + fn ivfsq_streamed_list_reader_matches_full_payload() { + let (index, _, _) = build_index(8, 1, 257); + let bytes = serialized_index(&index); + let mut full_reader = IVFSQIndexReader::open(Cursor::new(bytes.clone())).unwrap(); + let expected = full_reader.read_inverted_list(0).unwrap(); + let mut streamed_reader = IVFSQIndexReader::open(Cursor::new(bytes)).unwrap(); + let mut actual_ids = Vec::new(); + let mut actual_codes = Vec::new(); + streamed_reader + .for_each_streamed_list_chunk(0, |ids, codes| { + actual_ids.extend_from_slice(ids); + actual_codes.extend_from_slice(codes); + }) + .unwrap(); + assert_eq!(actual_ids, expected.0); + assert_eq!(actual_codes, expected.1); + } + + #[test] + fn ivfsq_write_read_search_roundtrip() { + let (index, data, ids) = build_index(8, 4, 256); + let mut reader = IVFSQIndexReader::open(Cursor::new(serialized_index(&index))).unwrap(); + let query_index = 23; + let (labels, distances) = reader + .search(&data[query_index * 8..(query_index + 1) * 8], 5, 4) + .unwrap(); + assert_eq!(labels[0], ids[query_index]); + assert!(distances[0].is_finite()); + } + + #[test] + fn ivfsq_batch_matches_individual_search() { + let (index, data, _) = build_index(8, 4, 256); + let bytes = serialized_index(&index); + let queries = [&data[0..8], &data[80..88]].concat(); + let mut batch_reader = IVFSQIndexReader::open(Cursor::new(bytes.clone())).unwrap(); + let batch = search_batch_ivfsq_reader(&mut batch_reader, &queries, 2, 5, 4).unwrap(); + let mut single_reader = IVFSQIndexReader::open(Cursor::new(bytes)).unwrap(); + let first = single_reader.search(&queries[0..8], 5, 4).unwrap(); + let second = single_reader.search(&queries[8..16], 5, 4).unwrap(); + assert_eq!(batch.0, [first.0, second.0].concat()); + assert_eq!(batch.1, [first.1, second.1].concat()); + } + + #[test] + fn ivfsq_large_batch_scans_queries_in_parallel_without_duplicate_reads() { + let d = 16; + let nlist = 8; + let nq = 8; + let k = 10; + let (index, data, _) = build_index(d, nlist, 8_192); + let bytes = serialized_index(&index); + let queries = (0..nq) + .flat_map(|query_index| { + let row = query_index * 127; + data[row * d..(row + 1) * d].iter() + }) + .copied() + .collect::>(); + let mut expected = Vec::with_capacity(nq); + for query in queries.chunks_exact(d) { + let mut reader = IVFSQIndexReader::open(Cursor::new(bytes.clone())).unwrap(); + expected.push(reader.search(query, k, nlist).unwrap()); + } + + let calls = Arc::new(AtomicUsize::new(0)); + let source = CountingReader { + inner: Cursor::new(bytes), + calls: Arc::clone(&calls), + }; + let mut reader = IVFSQIndexReader::open(source).unwrap(); + calls.store(0, Ordering::Relaxed); + let filter = ThreadTrackingFilter { + workers: AtomicU64::new(0), + }; + let actual = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .unwrap() + .install(|| { + search_batch_ivfsq_reader_filter(&mut reader, &queries, nq, k, nlist, Some(&filter)) + .unwrap() + }); + + for query_index in 0..nq { + let actual_ids = &actual.0[query_index * k..(query_index + 1) * k]; + let actual_distances = &actual.1[query_index * k..(query_index + 1) * k]; + let (expected_ids, expected_distances) = &expected[query_index]; + let mut actual_pairs = actual_ids + .iter() + .zip(actual_distances) + .map(|(&row_id, &distance)| (row_id, distance.to_bits())) + .collect::>(); + let mut expected_pairs = expected_ids + .iter() + .zip(expected_distances) + .map(|(&row_id, &distance)| (row_id, distance.to_bits())) + .collect::>(); + actual_pairs.sort_unstable(); + expected_pairs.sort_unstable(); + assert_eq!(actual_pairs, expected_pairs); + assert!( + actual_distances.windows(2).all(|pair| pair[0] <= pair[1]), + "parallel batch results must remain distance-sorted" + ); + } + assert!( + filter.workers.load(Ordering::Relaxed).count_ones() > 1, + "a large batch should scan queries on multiple Rayon workers" + ); + assert_eq!( + calls.load(Ordering::Relaxed), + 1, + "parallel scanning must not duplicate the multi-range list read" + ); + } + + #[test] + fn ivfsq_search_supports_roaring_filter() { + let (index, data, ids) = build_index(4, 1, 64); + let mut reader = IVFSQIndexReader::open(Cursor::new(serialized_index(&index))).unwrap(); + let mut filter = RoaringTreemap::new(); + filter.insert(ids[10] as u64); + let mut filter_bytes = Vec::new(); + filter.serialize_into(&mut filter_bytes).unwrap(); + let (labels, _) = reader + .search_with_roaring_filter(&data[0..4], 2, 1, &filter_bytes) + .unwrap(); + assert_eq!(labels, vec![ids[10], -1]); + } + + #[test] + fn ivfsq_selected_lists_share_one_multi_range_pread() { + let (index, data, _) = build_index(8, 8, 512); + let calls = Arc::new(AtomicUsize::new(0)); + let source = CountingReader { + inner: Cursor::new(serialized_index(&index)), + calls: Arc::clone(&calls), + }; + let mut reader = IVFSQIndexReader::open(source).unwrap(); + reader.ensure_loaded().unwrap(); + calls.store(0, Ordering::SeqCst); + reader.search(&data[0..8], 5, 8).unwrap(); + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + + #[test] + fn ivfsq_open_coalesces_resident_metadata() { + let (index, _, _) = build_index(8, 32, 512); + let calls = Arc::new(AtomicUsize::new(0)); + let source = CountingReader { + inner: Cursor::new(serialized_index(&index)), + calls: Arc::clone(&calls), + }; + let mut reader = IVFSQIndexReader::open(source).unwrap(); + reader.optimize_for_search().unwrap(); + assert_eq!( + calls.load(Ordering::SeqCst), + 2, + "direct IVF-SQ open should use one header read and one resident-metadata read" + ); + } + + #[test] + fn ivfsq_blocked_reader_supports_all_metrics() { + for metric in [MetricType::L2, MetricType::InnerProduct, MetricType::Cosine] { + let d = 8; + let nlist = 4; + let n = 256; + let data = (0..n * d) + .map(|index| 1.0 + (index % 37) as f32 * 0.01) + .collect::>(); + let ids = (0..n as i64).collect::>(); + let mut index = IVFSQIndex::new(d, nlist, metric); + index.train(&data, n); + index.add(&data, &ids, n); + + let mut expected_distances = vec![0.0; 10]; + let mut expected_ids = vec![0; 10]; + index.search( + &data[0..d], + 1, + 10, + nlist, + &mut expected_distances, + &mut expected_ids, + ); + let mut reader = IVFSQIndexReader::open(Cursor::new(serialized_index(&index))).unwrap(); + let (actual_ids, actual_distances) = reader.search(&data[0..d], 10, nlist).unwrap(); + // The fixture repeats vectors every 37 rows. SIMD accumulation may + // order those exactly tied row IDs differently, but must preserve + // the same top-k membership and distance ordering. + let mut actual_members = actual_ids.clone(); + let mut expected_members = expected_ids.clone(); + actual_members.sort_unstable(); + expected_members.sort_unstable(); + assert_eq!(actual_members, expected_members, "metric={metric:?}"); + for (actual, expected) in actual_distances.iter().zip(expected_distances) { + assert!((actual - expected).abs() < 1e-3, "metric={metric:?}"); + } + } + } + + #[test] + fn ivfsq_reader_validates_bits_flags_and_reserved_bytes() { + let (index, _, _) = build_index(2, 1, 16); + let bytes = serialized_index(&index); + for (offset, value, expected) in [ + (28, 4u32, "bit width"), + (32, 0u32, "requires delta-varint"), + (32, REQUIRED_FLAGS | (1 << 31), "Unsupported IVF-SQ flags"), + ] { + let mut corrupted = bytes.clone(); + corrupted[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + let error = match IVFSQIndexReader::open(Cursor::new(corrupted)) { + Ok(_) => panic!("corrupt header should fail"), + Err(error) => error, + }; + assert!(error.to_string().contains(expected)); + } + let mut corrupted = bytes; + corrupted[44] = 1; + let error = match IVFSQIndexReader::open(Cursor::new(corrupted)) { + Ok(_) => panic!("reserved header bytes should be validated"), + Err(error) => error, + }; + assert!(error.to_string().contains("reserved bytes")); + } + + struct CountingReader { + inner: Cursor>, + calls: Arc, + } + + struct ThreadTrackingFilter { + workers: AtomicU64, + } + + impl RowIdFilter for ThreadTrackingFilter { + fn contains(&self, _id: i64) -> bool { + if let Some(worker) = rayon::current_thread_index() { + self.workers.fetch_or(1u64 << worker, Ordering::Relaxed); + } + true + } + } + + impl SeekRead for CountingReader { + fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { + self.calls.fetch_add(1, Ordering::SeqCst); + self.inner.pread(ranges) + } + + fn try_clone_reader(&self) -> io::Result> { + Ok(None) + } + } +} diff --git a/core/src/lib.rs b/core/src/lib.rs index d332b8a..da0c188 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -18,27 +18,30 @@ #![allow(clippy::needless_range_loop)] #![allow(clippy::too_many_arguments)] +pub mod autotune; pub mod blas; +pub mod diskann; +pub mod diskann_io; +pub(crate) mod diskann_search; pub mod distance; pub mod fastscan; -pub mod hnsw; -pub(crate) mod hnsw_search; pub mod index; pub(crate) mod index_io_util; pub mod io; pub mod ivfflat; pub mod ivfflat_io; -pub mod ivfhnswflat; -pub mod ivfhnswflat_io; -pub mod ivfhnswsq; -pub mod ivfhnswsq_io; pub mod ivfpq; pub mod ivfrq; pub mod ivfrq_io; +pub mod ivfsq; +pub mod ivfsq_io; pub mod kmeans; pub mod opq; pub mod pq; +pub mod read_options; pub mod rq; pub mod shuffler; +pub(crate) mod sparse_table; pub mod sq; pub mod topk; +pub mod vamana; diff --git a/core/src/pq.rs b/core/src/pq.rs index 5c7e0d5..0d44d9f 100644 --- a/core/src/pq.rs +++ b/core/src/pq.rs @@ -24,17 +24,23 @@ use rayon::prelude::*; /// Product Quantizer aligned with Faiss's ProductQuantizer. /// -/// Splits D-dimensional vectors into M sub-vectors of dimension dsub = D/M, -/// and independently quantizes each sub-vector with ksub centroids. +/// Splits D-dimensional vectors into M contiguous chunks and independently +/// quantizes each chunk with `ksub` centroids. Uniform chunks remain the +/// default for IVF-PQ compatibility; DiskANN may use near-equal chunks when +/// `D` is not divisible by `M`. /// -/// Centroid layout: flat [M * ksub * dsub], row-major. -/// centroids[m][j][d] is at index: m * ksub * dsub + j * dsub + d +/// Centroids are chunk-major. Chunk `m` starts at +/// `chunk_offsets[m] * ksub`, and each of its `ksub` centroids contains +/// `chunk_offsets[m + 1] - chunk_offsets[m]` contiguous components. pub struct ProductQuantizer { pub d: usize, pub m: usize, pub nbits: usize, + /// Uniform chunk width for legacy formats, or the largest chunk width for + /// a balanced non-uniform layout. pub dsub: usize, pub ksub: usize, + pub chunk_offsets: Vec, pub centroids: Vec, /// Pre-computed squared norms of each centroid: [M * ksub]. /// Avoids recomputing per query for L2 distance table. @@ -53,19 +59,63 @@ impl ProductQuantizer { d, m ); + let dsub = d / m; + let chunk_offsets = (0..=m).map(|chunk| chunk * dsub).collect(); + Self::with_validated_chunk_offsets(d, nbits, chunk_offsets) + } + + /// Create a quantizer whose chunks differ in width by at most one. + /// + /// The first `d % m` chunks contain one additional component. This is the + /// DiskANN layout and supports every `1 <= m <= d`. + pub fn with_nbits_balanced(d: usize, m: usize, nbits: usize) -> Self { + assert!(d > 0, "dimension must be greater than zero"); + assert!(m > 0 && m <= d, "m must be in 1..=dimension"); + let base = d / m; + let remainder = d % m; + let mut chunk_offsets = Vec::with_capacity(m + 1); + chunk_offsets.push(0); + let mut offset = 0usize; + for chunk in 0..m { + offset += base + usize::from(chunk < remainder); + chunk_offsets.push(offset); + } + Self::with_validated_chunk_offsets(d, nbits, chunk_offsets) + } + + /// Restore a persisted chunk plan after the caller has decoded it. + pub fn try_with_chunk_offsets( + d: usize, + nbits: usize, + chunk_offsets: Vec, + ) -> Result { + if d == 0 || !matches!(nbits, 4 | 8) || chunk_offsets.len() < 2 { + return Err("invalid PQ shape"); + } + if chunk_offsets[0] != 0 || chunk_offsets.last().copied() != Some(d) { + return Err("invalid PQ chunk bounds"); + } + if chunk_offsets + .windows(2) + .any(|bounds| bounds[0] >= bounds[1]) + { + return Err("PQ chunk offsets must be strictly increasing"); + } + Ok(Self::with_validated_chunk_offsets(d, nbits, chunk_offsets)) + } + + fn with_validated_chunk_offsets(d: usize, nbits: usize, chunk_offsets: Vec) -> Self { assert!( nbits == 4 || nbits == 8, "nbits must be 4 or 8, got {}", nbits ); - if nbits == 4 { - assert!( - m.is_multiple_of(2), - "m must be even for 4-bit PQ, got {}", - m - ); - } - let dsub = d / m; + let m = chunk_offsets.len() - 1; + let dsub = chunk_offsets + .windows(2) + .map(|bounds| bounds[1] - bounds[0]) + .max() + .expect("validated non-empty PQ chunk plan"); let ksub = 1 << nbits; ProductQuantizer { d, @@ -73,11 +123,38 @@ impl ProductQuantizer { nbits, dsub, ksub, + chunk_offsets, centroids: Vec::new(), centroid_norms_cache: Vec::new(), } } + #[inline] + pub fn chunk_range(&self, sub: usize) -> std::ops::Range { + self.chunk_offsets[sub]..self.chunk_offsets[sub + 1] + } + + #[inline] + pub fn chunk_dim(&self, sub: usize) -> usize { + self.chunk_offsets[sub + 1] - self.chunk_offsets[sub] + } + + #[inline] + pub fn centroid_chunk_base(&self, sub: usize) -> usize { + self.chunk_offsets[sub] * self.ksub + } + + pub fn has_valid_layout(&self) -> bool { + Self::try_with_chunk_offsets(self.d, self.nbits, self.chunk_offsets.clone()).is_ok_and( + |layout| { + layout.m == self.m + && layout.dsub == self.dsub + && layout.ksub == self.ksub + && self.centroids.len() == self.d * self.ksub + }, + ) + } + /// Train the codebooks from training data. /// data: flat [n * d], n training vectors. pub fn train(&mut self, data: &[f32], n: usize) { @@ -96,6 +173,23 @@ impl ProductQuantizer { n: usize, km_config: &KMeansConfig, hot_start: bool, + ) { + self.train_hot_start_with_parallelism( + data, + n, + km_config, + hot_start, + rayon::current_num_threads().max(1), + ); + } + + pub(crate) fn train_hot_start_with_parallelism( + &mut self, + data: &[f32], + n: usize, + km_config: &KMeansConfig, + hot_start: bool, + max_parallelism: usize, ) { let prev_centroids = if hot_start && !self.centroids.is_empty() { Some(self.centroids.clone()) @@ -105,55 +199,87 @@ impl ProductQuantizer { let m = self.m; let d = self.d; - let dsub = self.dsub; let ksub = self.ksub; + let chunk_offsets = &self.chunk_offsets; + + let train_subquantizers = || { + (0..m) + .into_par_iter() + .map(|sub| { + let start = chunk_offsets[sub]; + let stop = chunk_offsets[sub + 1]; + let chunk_dim = stop - start; + + let mut sub_data = vec![0.0f32; n * chunk_dim]; + for i in 0..n { + sub_data[i * chunk_dim..(i + 1) * chunk_dim] + .copy_from_slice(&data[i * d + start..i * d + stop]); + } - // Train all M sub-quantizers in parallel - let sub_results: Vec> = (0..m) - .into_par_iter() - .map(|sub| { - let offset = sub * dsub; - - let mut sub_data = vec![0.0f32; n * dsub]; - for i in 0..n { - sub_data[i * dsub..(i + 1) * dsub] - .copy_from_slice(&data[i * d + offset..i * d + offset + dsub]); - } - - let init: Option> = prev_centroids.as_ref().map(|pc| { - let src = sub * ksub * dsub; - pc[src..src + ksub * dsub].to_vec() - }); - - kmeans::kmeans_train_with_init(km_config, &sub_data, n, dsub, ksub, init.as_deref()) - }) - .collect(); + let init: Option> = prev_centroids.as_ref().map(|pc| { + let src = start * ksub; + pc[src..src + ksub * chunk_dim].to_vec() + }); + + kmeans::kmeans_train_with_init( + km_config, + &sub_data, + n, + chunk_dim, + ksub, + init.as_deref(), + ) + }) + .collect::>>() + }; + let current_parallelism = rayon::current_num_threads().max(1); + let sub_results = if max_parallelism >= current_parallelism { + train_subquantizers() + } else { + rayon::ThreadPoolBuilder::new() + .num_threads(max_parallelism.max(1)) + .build() + .expect("bounded PQ training thread pool should be constructible") + .install(train_subquantizers) + }; - self.centroids = vec![0.0f32; m * ksub * dsub]; + self.centroids = vec![0.0f32; d * ksub]; for (sub, sub_centroids) in sub_results.into_iter().enumerate() { - let dst_offset = sub * ksub * dsub; - self.centroids[dst_offset..dst_offset + ksub * dsub].copy_from_slice(&sub_centroids); + let chunk_dim = self.chunk_dim(sub); + let dst_offset = self.centroid_chunk_base(sub); + self.centroids[dst_offset..dst_offset + ksub * chunk_dim] + .copy_from_slice(&sub_centroids); } self.rebuild_norms_cache(); } /// Rebuild the centroid norms cache. Called after training or loading centroids. pub fn rebuild_norms_cache(&mut self) { - self.centroid_norms_cache = vec![0.0f32; self.m * self.ksub]; + self.try_rebuild_norms_cache() + .expect("PQ centroid norms allocation failed"); + } + + pub fn try_rebuild_norms_cache(&mut self) -> Result<(), std::collections::TryReserveError> { + let mut norms = Vec::new(); + norms.try_reserve_exact(self.m * self.ksub)?; + norms.resize(self.m * self.ksub, 0.0f32); for sub in 0..self.m { - let c_base = sub * self.ksub * self.dsub; + let chunk_dim = self.chunk_dim(sub); + let c_base = self.centroid_chunk_base(sub); for j in 0..self.ksub { - let c_off = c_base + j * self.dsub; - self.centroid_norms_cache[sub * self.ksub + j] = - fvec_norm_l2sqr(&self.centroids[c_off..c_off + self.dsub]); + let c_off = c_base + j * chunk_dim; + norms[sub * self.ksub + j] = + fvec_norm_l2sqr(&self.centroids[c_off..c_off + chunk_dim]); } } + self.centroid_norms_cache = norms; + Ok(()) } /// Bytes per encoded vector. pub fn code_size(&self) -> usize { if self.nbits == 4 { - self.m / 2 + self.m.div_ceil(2) } else { self.m } @@ -161,7 +287,8 @@ impl ProductQuantizer { /// Encode a single vector into PQ codes. /// For nbits=8: codes has length M (one byte per sub-quantizer). - /// For nbits=4: codes has length M/2 (two nibbles per byte). + /// For nbits=4: codes has length ceil(M/2). If M is odd, the final high + /// nibble is canonical zero padding. pub fn encode(&self, x: &[f32], codes: &mut [u8]) { let mut distances = vec![0.0f32; self.ksub]; self.encode_with_distances(x, codes, &mut distances); @@ -184,15 +311,19 @@ impl ProductQuantizer { } fn encode_4bit(&self, x: &[f32], codes: &mut [u8], distances: &mut [f32]) { - for pair in 0..self.m / 2 { + for pair in 0..self.m.div_ceil(2) { let sub_lo = pair * 2; let sub_hi = pair * 2 + 1; self.compute_sub_l2_distances(x, sub_lo, distances); let best_lo = argmin_code(&distances[..self.ksub]); - self.compute_sub_l2_distances(x, sub_hi, distances); - let best_hi = argmin_code(&distances[..self.ksub]); + let best_hi = if sub_hi < self.m { + self.compute_sub_l2_distances(x, sub_hi, distances); + argmin_code(&distances[..self.ksub]) + } else { + 0 + }; // Pack: low nibble + high nibble codes[pair] = best_lo | (best_hi << 4); @@ -200,26 +331,27 @@ impl ProductQuantizer { } fn compute_sub_l2_distances(&self, x: &[f32], sub: usize, distances: &mut [f32]) { - let x_off = sub * self.dsub; - let c_base = sub * self.ksub * self.dsub; - let query_sub = &x[x_off..x_off + self.dsub]; - let centroids = &self.centroids[c_base..c_base + self.ksub * self.dsub]; - - if self.dsub >= 4 && self.ksub >= 8 { - fvec_ip_batch(query_sub, centroids, self.dsub, self.ksub, distances); + let range = self.chunk_range(sub); + let chunk_dim = range.len(); + let c_base = self.centroid_chunk_base(sub); + let query_sub = &x[range]; + let centroids = &self.centroids[c_base..c_base + self.ksub * chunk_dim]; + + if chunk_dim >= 4 && self.ksub >= 8 { + fvec_ip_batch(query_sub, centroids, chunk_dim, self.ksub, distances); let q_norm = fvec_norm_l2sqr(query_sub); let norms_base = sub * self.ksub; for j in 0..self.ksub { let c_norm = if !self.centroid_norms_cache.is_empty() { self.centroid_norms_cache[norms_base + j] } else { - let c_off = j * self.dsub; - fvec_norm_l2sqr(¢roids[c_off..c_off + self.dsub]) + let c_off = j * chunk_dim; + fvec_norm_l2sqr(¢roids[c_off..c_off + chunk_dim]) }; distances[j] = (q_norm + c_norm - 2.0 * distances[j]).max(0.0); } } else { - fvec_l2sqr_batch(query_sub, centroids, self.dsub, self.ksub, distances); + fvec_l2sqr_batch(query_sub, centroids, chunk_dim, self.ksub, distances); } } @@ -240,75 +372,69 @@ impl ProductQuantizer { /// Decode PQ codes back to an approximate vector. pub fn decode(&self, codes: &[u8], x: &mut [f32]) { - if self.nbits == 4 { - for pair in 0..self.m / 2 { - let byte = codes[pair]; - let code_lo = (byte & 0x0F) as usize; - let code_hi = ((byte >> 4) & 0x0F) as usize; - - let sub_lo = pair * 2; - let sub_hi = pair * 2 + 1; - - let c_off_lo = sub_lo * self.ksub * self.dsub + code_lo * self.dsub; - let x_off_lo = sub_lo * self.dsub; - x[x_off_lo..x_off_lo + self.dsub] - .copy_from_slice(&self.centroids[c_off_lo..c_off_lo + self.dsub]); - - let c_off_hi = sub_hi * self.ksub * self.dsub + code_hi * self.dsub; - let x_off_hi = sub_hi * self.dsub; - x[x_off_hi..x_off_hi + self.dsub] - .copy_from_slice(&self.centroids[c_off_hi..c_off_hi + self.dsub]); - } - } else { - for sub in 0..self.m { - let c_off = sub * self.ksub * self.dsub + (codes[sub] as usize) * self.dsub; - let x_off = sub * self.dsub; - x[x_off..x_off + self.dsub] - .copy_from_slice(&self.centroids[c_off..c_off + self.dsub]); - } + for sub in 0..self.m { + let code = if self.nbits == 4 { + let byte = codes[sub / 2]; + if sub.is_multiple_of(2) { + byte & 0x0f + } else { + byte >> 4 + } + } else { + codes[sub] + } as usize; + let range = self.chunk_range(sub); + let chunk_dim = range.len(); + let c_off = self.centroid_chunk_base(sub) + code * chunk_dim; + x[range].copy_from_slice(&self.centroids[c_off..c_off + chunk_dim]); } } /// Precompute the distance table from a query to all PQ centroids. - /// Uses sgemm for dsub >= 4 (L2: ||q-c||²=||q||²+||c||²-2q·cᵀ). + /// Uses sgemm for chunks of at least four components + /// (L2: ||q-c||²=||q||²+||c||²-2q·cᵀ). pub fn compute_distance_table(&self, query: &[f32], metric: MetricType, table: &mut [f32]) { - if self.dsub >= 4 { - self.compute_distance_table_sgemm(query, metric, table); - } else { - self.compute_distance_table_loop(query, metric, table); - } - } - - fn compute_distance_table_sgemm(&self, query: &[f32], metric: MetricType, table: &mut [f32]) { for sub in 0..self.m { - let q_off = sub * self.dsub; - let c_base = sub * self.ksub * self.dsub; + let range = self.chunk_range(sub); + let chunk_dim = range.len(); + let c_base = self.centroid_chunk_base(sub); let t_base = sub * self.ksub; - - // Inner product: ip[ksub] = query_sub[1×dsub] · centroids_sub[ksub×dsub]ᵀ - sgemm_a_bt( - 1, - self.ksub, - self.dsub, - 1.0, - &query[q_off..q_off + self.dsub], - &self.centroids[c_base..c_base + self.ksub * self.dsub], - 0.0, - &mut table[t_base..t_base + self.ksub], - ); + let query_chunk = &query[range]; + let centroids = &self.centroids[c_base..c_base + self.ksub * chunk_dim]; + + if chunk_dim >= 4 { + sgemm_a_bt( + 1, + self.ksub, + chunk_dim, + 1.0, + query_chunk, + centroids, + 0.0, + &mut table[t_base..t_base + self.ksub], + ); + } else { + fvec_ip_batch( + query_chunk, + centroids, + chunk_dim, + self.ksub, + &mut table[t_base..t_base + self.ksub], + ); + } match metric { MetricType::L2 | MetricType::Cosine => { // ||q-c||² = ||q||² + ||c||² - 2·q·c // Use pre-cached centroid norms (avoids recomputing per query) - let q_norm = fvec_norm_l2sqr(&query[q_off..q_off + self.dsub]); + let q_norm = fvec_norm_l2sqr(query_chunk); let norms_base = sub * self.ksub; for j in 0..self.ksub { let c_norm = if !self.centroid_norms_cache.is_empty() { self.centroid_norms_cache[norms_base + j] } else { - let c_off = c_base + j * self.dsub; - fvec_norm_l2sqr(&self.centroids[c_off..c_off + self.dsub]) + let c_off = c_base + j * chunk_dim; + fvec_norm_l2sqr(&self.centroids[c_off..c_off + chunk_dim]) }; table[t_base + j] = (q_norm + c_norm - 2.0 * table[t_base + j]).max(0.0); } @@ -322,49 +448,18 @@ impl ProductQuantizer { } } - fn compute_distance_table_loop(&self, query: &[f32], metric: MetricType, table: &mut [f32]) { - for sub in 0..self.m { - let q_off = sub * self.dsub; - let c_base = sub * self.ksub * self.dsub; - let t_base = sub * self.ksub; - - match metric { - MetricType::L2 | MetricType::Cosine => { - fvec_l2sqr_batch( - &query[q_off..q_off + self.dsub], - &self.centroids[c_base..c_base + self.ksub * self.dsub], - self.dsub, - self.ksub, - &mut table[t_base..t_base + self.ksub], - ); - } - MetricType::InnerProduct => { - fvec_ip_batch( - &query[q_off..q_off + self.dsub], - &self.centroids[c_base..c_base + self.ksub * self.dsub], - self.dsub, - self.ksub, - &mut table[t_base..t_base + self.ksub], - ); - for j in 0..self.ksub { - table[t_base + j] = -table[t_base + j]; - } - } - } - } - } - /// Compute inner product table: ip_table[m * ksub + j] = . pub fn compute_inner_product_table(&self, query: &[f32], table: &mut [f32]) { for sub in 0..self.m { - let q_off = sub * self.dsub; - let c_base = sub * self.ksub * self.dsub; + let range = self.chunk_range(sub); + let chunk_dim = range.len(); + let c_base = self.centroid_chunk_base(sub); let t_base = sub * self.ksub; fvec_ip_batch( - &query[q_off..q_off + self.dsub], - &self.centroids[c_base..c_base + self.ksub * self.dsub], - self.dsub, + &query[range], + &self.centroids[c_base..c_base + self.ksub * chunk_dim], + chunk_dim, self.ksub, &mut table[t_base..t_base + self.ksub], ); @@ -385,16 +480,14 @@ impl ProductQuantizer { #[inline] fn distance_from_table_4bit(&self, table: &[f32], codes: &[u8]) -> f32 { let mut dist = 0.0f32; - for pair in 0..self.m / 2 { - let byte = codes[pair]; - let code_lo = (byte & 0x0F) as usize; - let code_hi = ((byte >> 4) & 0x0F) as usize; - - let sub_lo = pair * 2; - let sub_hi = pair * 2 + 1; - - dist += table[sub_lo * self.ksub + code_lo]; - dist += table[sub_hi * self.ksub + code_hi]; + for sub in 0..self.m { + let byte = codes[sub / 2]; + let code = if sub.is_multiple_of(2) { + byte & 0x0f + } else { + byte >> 4 + }; + dist += table[sub * self.ksub + code as usize]; } dist } @@ -407,11 +500,12 @@ impl ProductQuantizer { } let mut norms = vec![0.0f32; self.m * self.ksub]; for sub in 0..self.m { - let c_base = sub * self.ksub * self.dsub; + let chunk_dim = self.chunk_dim(sub); + let c_base = self.centroid_chunk_base(sub); for j in 0..self.ksub { - let c_off = c_base + j * self.dsub; + let c_off = c_base + j * chunk_dim; norms[sub * self.ksub + j] = - fvec_norm_l2sqr(&self.centroids[c_off..c_off + self.dsub]); + fvec_norm_l2sqr(&self.centroids[c_off..c_off + chunk_dim]); } } norms @@ -543,6 +637,59 @@ mod tests { assert!(codes.iter().any(|&b| b != 0)); } + #[test] + fn test_balanced_chunks_encode_decode_and_distance_table() { + let d = 7; + let m = 3; + let n = 100; + let mut rng = StdRng::seed_from_u64(73); + let data: Vec = (0..n * d).map(|_| rng.gen::()).collect(); + let mut pq = ProductQuantizer::with_nbits_balanced(d, m, 8); + + assert_eq!(pq.chunk_offsets, vec![0, 3, 5, 7]); + assert_eq!(pq.dsub, 3); + assert_eq!(pq.code_size(), 3); + + pq.train(&data, n); + assert!(pq.has_valid_layout()); + let query = &data[d..2 * d]; + let mut codes = vec![0u8; pq.code_size()]; + pq.encode(query, &mut codes); + let mut decoded = vec![0.0f32; d]; + pq.decode(&codes, &mut decoded); + let mut table = vec![0.0f32; m * pq.ksub]; + pq.compute_distance_table(query, MetricType::L2, &mut table); + + let table_distance = pq.distance_from_table(&table, &codes); + let decoded_distance = fvec_l2sqr_sub(query, 0, &decoded, 0, d); + assert!((table_distance - decoded_distance).abs() < 1e-4); + } + + #[test] + fn test_odd_4bit_chunk_count_uses_canonical_padding_nibble() { + let d = 7; + let m = 3; + let n = 100; + let mut rng = StdRng::seed_from_u64(74); + let data: Vec = (0..n * d).map(|_| rng.gen::()).collect(); + let mut pq = ProductQuantizer::with_nbits_balanced(d, m, 4); + pq.train(&data, n); + + assert_eq!(pq.chunk_offsets, vec![0, 3, 5, 7]); + assert_eq!(pq.code_size(), 2); + let mut codes = vec![0xff; pq.code_size()]; + pq.encode(&data[..d], &mut codes); + assert_eq!(codes[1] & 0xf0, 0); + + let mut decoded = vec![0.0f32; d]; + pq.decode(&codes, &mut decoded); + let mut table = vec![0.0f32; m * pq.ksub]; + pq.compute_distance_table(&data[..d], MetricType::L2, &mut table); + let table_distance = pq.distance_from_table(&table, &codes); + let decoded_distance = fvec_l2sqr_sub(&data[..d], 0, &decoded, 0, d); + assert!((table_distance - decoded_distance).abs() < 1e-4); + } + #[test] fn test_sgemm_distance_table_clamps_negative_self_distance() { let d = 128; diff --git a/core/src/read_options.rs b/core/src/read_options.rs new file mode 100644 index 0000000..4bcdaa3 --- /dev/null +++ b/core/src/read_options.rs @@ -0,0 +1,339 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +/// Build-time serving intent used to choose DiskANN's persisted layout. +/// +/// Reader I/O policy is always derived from input capabilities and measured +/// latency; this enum is not a Reader option. +pub enum DeploymentProfile { + #[default] + Auto = 0, + Memory = 1, + LocalStorage = 2, + RemoteStorage = 3, + ObjectStore = 4, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct ReadPlan { + pub window_bytes: usize, + pub graph_beam_width: usize, + pub filtered_graph_beam_width: usize, +} + +impl DeploymentProfile { + pub(crate) const fn read_plan(self) -> ReadPlan { + match self { + Self::Auto | Self::LocalStorage => ReadPlan { + window_bytes: 16 * 1024, + graph_beam_width: 4, + filtered_graph_beam_width: 4, + }, + Self::Memory => ReadPlan { + window_bytes: 4 * 1024, + graph_beam_width: 16, + filtered_graph_beam_width: 4, + }, + Self::RemoteStorage => ReadPlan { + window_bytes: 32 * 1024, + graph_beam_width: 16, + filtered_graph_beam_width: 4, + }, + Self::ObjectStore => ReadPlan { + window_bytes: 64 * 1024, + graph_beam_width: 16, + filtered_graph_beam_width: 4, + }, + } + } +} + +impl ReadPlan { + pub(crate) fn with_capabilities( + mut self, + capabilities: crate::io::SeekReadCapabilities, + ) -> Self { + if capabilities.preferred_window_bytes > 0 { + self.window_bytes = capabilities.preferred_window_bytes; + } + // Bound pathological adapter hints while keeping enough room for one + // complete DiskANN page. The planner clips the final window to the + // section, so a non-page-sized capability hint must not split a page + // across two windows. + self.window_bytes = self.window_bytes.clamp(4 * 1024, 1024 * 1024); + self.window_bytes = self.window_bytes.div_ceil(4 * 1024) * (4 * 1024); + self.window_bytes = self.window_bytes.min(1024 * 1024); + if capabilities.max_ranges_per_pread > 0 { + self.graph_beam_width = self + .graph_beam_width + .min(capabilities.max_ranges_per_pread) + .max(1); + self.filtered_graph_beam_width = self + .filtered_graph_beam_width + .min(capabilities.max_ranges_per_pread) + .max(1); + } + self + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct VectorIndexReaderOptions { + /// Total bytes available to required resident state and retained caches. + pub memory_budget_bytes: usize, + cache_overrides: Option, +} + +/// The concrete DiskANN read policy derived from input capabilities, observed +/// latency, index layout, and the Reader memory budget. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct VectorIndexReadPlan { + pub random_read_latency_nanos: u64, + pub window_bytes: usize, + pub max_ranges_per_read: usize, + pub graph_beam_width: usize, + pub filtered_graph_beam_width: usize, + pub adjacency_preload_bytes: usize, + pub adjacency_cache_bytes: usize, + pub raw_vector_cache_bytes: usize, + pub memory_budget_bytes: usize, +} + +impl Default for VectorIndexReaderOptions { + fn default() -> Self { + Self { + memory_budget_bytes: 4 * 1024 * 1024 * 1024, + cache_overrides: None, + } + } +} + +impl VectorIndexReaderOptions { + pub fn new(memory_budget_bytes: usize) -> Self { + Self { + memory_budget_bytes, + cache_overrides: None, + } + } + + pub(crate) fn resolve_cache_budgets( + self, + read_tier: DeploymentProfile, + resident_steady_bytes: usize, + adjacency_section_bytes: usize, + raw_vector_section_bytes: usize, + ) -> ResolvedVectorIndexReaderOptions { + if let Some(mut overrides) = self.cache_overrides { + overrides.read_tier = read_tier; + return overrides; + } + let available = self + .memory_budget_bytes + .saturating_sub(resident_steady_bytes); + let preload_cap = match read_tier { + DeploymentProfile::Auto + | DeploymentProfile::Memory + | DeploymentProfile::LocalStorage => 16 * 1024 * 1024, + DeploymentProfile::RemoteStorage => 32 * 1024 * 1024, + DeploymentProfile::ObjectStore => 64 * 1024 * 1024, + }; + let adjacency_preload_bytes = (available / 2) + .min(preload_cap) + .min(adjacency_section_bytes); + let after_preload = available.saturating_sub(adjacency_preload_bytes); + let per_cache_cap = match read_tier { + DeploymentProfile::RemoteStorage | DeploymentProfile::ObjectStore => usize::MAX, + DeploymentProfile::Auto + | DeploymentProfile::Memory + | DeploymentProfile::LocalStorage => 64 * 1024 * 1024, + }; + let cold_adjacency_bytes = adjacency_section_bytes.saturating_sub(adjacency_preload_bytes); + let mut adjacency_cache_bytes = (after_preload / 2) + .min(per_cache_cap) + .min(cold_adjacency_bytes); + let mut raw_vector_cache_bytes = after_preload + .saturating_sub(adjacency_cache_bytes) + .min(per_cache_cap) + .min(raw_vector_section_bytes); + let mut unassigned = after_preload + .saturating_sub(adjacency_cache_bytes) + .saturating_sub(raw_vector_cache_bytes); + let adjacency_extra = unassigned + .min(per_cache_cap.saturating_sub(adjacency_cache_bytes)) + .min(cold_adjacency_bytes.saturating_sub(adjacency_cache_bytes)); + adjacency_cache_bytes = adjacency_cache_bytes.saturating_add(adjacency_extra); + unassigned = unassigned.saturating_sub(adjacency_extra); + let vector_extra = unassigned + .min(per_cache_cap.saturating_sub(raw_vector_cache_bytes)) + .min(raw_vector_section_bytes.saturating_sub(raw_vector_cache_bytes)); + raw_vector_cache_bytes = raw_vector_cache_bytes.saturating_add(vector_extra); + ResolvedVectorIndexReaderOptions { + read_tier, + adjacency_preload_bytes, + adjacency_cache_bytes, + max_resident_bytes: self.memory_budget_bytes, + raw_vector_cache_bytes, + } + } + + #[cfg(test)] + pub(crate) fn with_cache_budgets( + read_tier: DeploymentProfile, + adjacency_preload_bytes: usize, + adjacency_cache_bytes: usize, + max_resident_bytes: usize, + raw_vector_cache_bytes: usize, + ) -> Self { + Self { + memory_budget_bytes: max_resident_bytes, + cache_overrides: Some(ResolvedVectorIndexReaderOptions { + read_tier, + adjacency_preload_bytes, + adjacency_cache_bytes, + max_resident_bytes, + raw_vector_cache_bytes, + }), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct ResolvedVectorIndexReaderOptions { + pub read_tier: DeploymentProfile, + pub adjacency_preload_bytes: usize, + pub adjacency_cache_bytes: usize, + pub max_resident_bytes: usize, + pub raw_vector_cache_bytes: usize, +} + +#[cfg(test)] +mod tests { + use crate::io::SeekReadCapabilities; + + use super::{DeploymentProfile, VectorIndexReaderOptions}; + + #[test] + fn deployment_profiles_have_distinct_read_plans() { + let options = VectorIndexReaderOptions::default(); + assert_eq!(options.memory_budget_bytes, 4 * 1024 * 1024 * 1024); + + let resolved = options.resolve_cache_budgets( + DeploymentProfile::Auto, + 1024, + 16 * 1024 * 1024, + 8 * 1024 * 1024, + ); + assert!(resolved.adjacency_preload_bytes > 0); + assert_eq!( + resolved.adjacency_preload_bytes + resolved.adjacency_cache_bytes, + 16 * 1024 * 1024 + ); + assert!(resolved.raw_vector_cache_bytes > 0); + assert!( + 1024 + resolved.adjacency_preload_bytes + + resolved.adjacency_cache_bytes + + resolved.raw_vector_cache_bytes + <= options.memory_budget_bytes + ); + + let memory = DeploymentProfile::Memory.read_plan(); + let local = DeploymentProfile::LocalStorage.read_plan(); + let remote = DeploymentProfile::RemoteStorage.read_plan(); + let object_store = DeploymentProfile::ObjectStore.read_plan(); + assert_eq!( + ( + memory.window_bytes, + memory.graph_beam_width, + memory.filtered_graph_beam_width + ), + (4096, 16, 4) + ); + assert_eq!( + ( + local.window_bytes, + local.graph_beam_width, + local.filtered_graph_beam_width + ), + (16 * 1024, 4, 4) + ); + assert_eq!( + ( + remote.window_bytes, + remote.graph_beam_width, + remote.filtered_graph_beam_width + ), + (32 * 1024, 16, 4) + ); + assert_eq!( + ( + object_store.window_bytes, + object_store.graph_beam_width, + object_store.filtered_graph_beam_width + ), + (64 * 1024, 16, 4) + ); + } + + #[test] + fn remote_cache_budgets_use_available_memory_beyond_the_local_cache_caps() { + const MIB: usize = 1024 * 1024; + let resolved = VectorIndexReaderOptions::new(4 * 1024 * MIB).resolve_cache_budgets( + DeploymentProfile::RemoteStorage, + 256 * MIB, + 512 * MIB, + 2 * 1024 * MIB, + ); + + assert_eq!(resolved.adjacency_preload_bytes, 32 * MIB); + assert_eq!( + resolved.adjacency_cache_bytes, + 512 * MIB - resolved.adjacency_preload_bytes + ); + assert_eq!(resolved.raw_vector_cache_bytes, 2 * 1024 * MIB); + assert!( + 256 * MIB + + resolved.adjacency_preload_bytes + + resolved.adjacency_cache_bytes + + resolved.raw_vector_cache_bytes + <= resolved.max_resident_bytes + ); + } + + #[test] + fn capability_windows_are_bounded_and_keep_complete_diskann_pages() { + let plan = DeploymentProfile::LocalStorage + .read_plan() + .with_capabilities(SeekReadCapabilities { + estimated_random_read_latency_nanos: 0, + preferred_window_bytes: 10_000, + max_ranges_per_pread: 2, + }); + assert_eq!(plan.window_bytes, 12 * 1024); + assert_eq!(plan.graph_beam_width, 2); + assert_eq!(plan.filtered_graph_beam_width, 2); + + let bounded = DeploymentProfile::RemoteStorage + .read_plan() + .with_capabilities(SeekReadCapabilities { + estimated_random_read_latency_nanos: 0, + preferred_window_bytes: usize::MAX, + max_ranges_per_pread: 0, + }); + assert_eq!(bounded.window_bytes, 1024 * 1024); + } +} diff --git a/core/src/rq.rs b/core/src/rq.rs index c248c70..3377586 100644 --- a/core/src/rq.rs +++ b/core/src/rq.rs @@ -6,7 +6,7 @@ // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an @@ -15,26 +15,44 @@ // specific language governing permissions and limitations // under the License. -use crate::distance::{fvec_norm_l2sqr, MetricType}; +use crate::distance::MetricType; +pub const DEFAULT_RQ_BITS: usize = 4; pub const DEFAULT_RQ_ROTATION_SEED: u64 = 0x9E37_79B9_7F4A_7C15; -pub const DEFAULT_RQ_ROTATION_ROUNDS: u32 = 3; -pub const RQ_BYTE_LUT_MIN_LIST_SIZE: usize = 64; -pub const DEFAULT_RQ_QUERY_BITS: usize = 0; +pub const DEFAULT_RQ_ROTATION_ROUNDS: u32 = 4; +pub const RQ_ROTATION_BLOCK_SIZE: usize = 64; +pub const RQ_SCAN_BLOCK_SIZE: usize = 32; + +const QUANTIZATION_REFINEMENT_ROUNDS: usize = 3; #[derive(Debug, Clone, Copy, PartialEq)] pub struct RQCodeFactors { - pub residual_norm_sqr: f32, - pub vector_norm_sqr: f32, - pub dp_multiplier: f32, + pub f_add: f32, + pub f_rescale: f32, + pub f_error: f32, } impl RQCodeFactors { pub fn zero() -> Self { Self { - residual_norm_sqr: 0.0, - vector_norm_sqr: 0.0, - dp_multiplier: 0.0, + f_add: 0.0, + f_rescale: 0.0, + f_error: 0.0, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct RQVectorFactors { + pub coarse: RQCodeFactors, + pub full: RQCodeFactors, +} + +impl RQVectorFactors { + pub fn zero() -> Self { + Self { + coarse: RQCodeFactors::zero(), + full: RQCodeFactors::zero(), } } } @@ -42,50 +60,53 @@ impl RQCodeFactors { #[derive(Debug, Clone)] pub struct RQRotation { d: usize, + padded_d: usize, seed: u64, rounds: u32, - ops: Vec, -} - -#[derive(Debug, Clone, Copy)] -struct KacOp { - i: usize, - j: usize, - cos: f32, - sin: f32, + sign_masks: Vec>, + permutations: Vec>, } impl RQRotation { pub fn new(d: usize, seed: u64, rounds: u32) -> Self { + let padded_d = padded_dimension(d); let mut rng = SplitMix64::new(seed ^ (d as u64).rotate_left(17)); - let mut ops = Vec::new(); - if d >= 2 { - for _ in 0..rounds { - let mut order: Vec = (0..d).collect(); - for i in (1..d).rev() { - let j = rng.next_usize(i + 1); - order.swap(i, j); - } - for pair in order.chunks_exact(2) { - let angle = (rng.next_f32() * 2.0 - 1.0) * std::f32::consts::PI; - let (sin, cos) = angle.sin_cos(); - ops.push(KacOp { - i: pair[0], - j: pair[1], - cos, - sin, - }); - } + let mut sign_masks = Vec::with_capacity(rounds as usize); + let mut permutations = Vec::with_capacity(rounds as usize); + + for _ in 0..rounds { + let mut signs = vec![0u8; padded_d.div_ceil(8)]; + for value in &mut signs { + *value = rng.next_u64() as u8; + } + sign_masks.push(signs); + + let mut permutation: Vec = (0..padded_d).collect(); + for i in (1..padded_d).rev() { + let j = rng.next_usize(i + 1); + permutation.swap(i, j); } + permutations.push(permutation); } + Self { d, + padded_d, seed, rounds, - ops, + sign_masks, + permutations, } } + pub fn dimension(&self) -> usize { + self.d + } + + pub fn padded_dimension(&self) -> usize { + self.padded_d + } + pub fn seed(&self) -> u64 { self.seed } @@ -94,13 +115,33 @@ impl RQRotation { self.rounds } - pub fn apply(&self, values: &mut [f32]) { - debug_assert_eq!(values.len(), self.d); - for op in &self.ops { - let x = values[op.i]; - let y = values[op.j]; - values[op.i] = op.cos * x - op.sin * y; - values[op.j] = op.sin * x + op.cos * y; + pub fn rotate(&self, input: &[f32], output: &mut [f32], scratch: &mut [f32]) { + debug_assert_eq!(input.len(), self.d); + debug_assert_eq!(output.len(), self.padded_d); + debug_assert_eq!(scratch.len(), self.padded_d); + + output.fill(0.0); + output[..self.d].copy_from_slice(input); + self.apply_in_place(output, scratch); + } + + pub fn apply_in_place(&self, values: &mut [f32], scratch: &mut [f32]) { + debug_assert_eq!(values.len(), self.padded_d); + debug_assert_eq!(scratch.len(), self.padded_d); + + for (signs, permutation) in self.sign_masks.iter().zip(&self.permutations) { + for (dim, value) in values.iter_mut().enumerate() { + if signs[dim / 8] & (1u8 << (dim % 8)) != 0 { + *value = -*value; + } + } + for block in values.chunks_exact_mut(RQ_ROTATION_BLOCK_SIZE) { + hadamard_64(block); + } + for (source, &destination) in permutation.iter().enumerate() { + scratch[destination] = values[source]; + } + values.copy_from_slice(scratch); } } } @@ -108,412 +149,388 @@ impl RQRotation { #[derive(Debug, Clone)] pub struct RaBitQuantizer { d: usize, - inv_sqrt_d: f32, + padded_d: usize, + bits: usize, + plane_size: usize, } -#[derive(Debug, Clone)] -pub struct RQDistanceContext { - d: usize, - code_size: usize, - rotated_query_residual: Vec, - query_residual_norm_sqr: f32, - query_norm_sqr: f32, - byte_signed_sums: Option>, - quantized_query: Option, +#[derive(Debug)] +pub struct RQEncodeScratch { + levels: Vec, + centered: Vec, + coarse_centered: Vec, +} + +impl RQEncodeScratch { + pub fn new(padded_d: usize) -> Self { + Self { + levels: vec![0; padded_d], + centered: vec![0.0; padded_d], + coarse_centered: vec![0.0; padded_d], + } + } } #[derive(Debug, Clone)] -struct RQQuantizedQuery { - scale: f32, - sign_bits: Vec, - magnitude_bit_planes: Vec>, +pub struct RQQueryContext { + rotated_query: Vec, + sum: f32, + byte_subset_sums: Vec, +} + +#[derive(Debug, Clone, Copy)] +pub struct RQQueryTerms { + pub g_add: f32, + pub g_error: f32, } impl RaBitQuantizer { - pub fn new(d: usize) -> Self { - let inv_sqrt_d = if d == 0 { 1.0 } else { 1.0 / (d as f32).sqrt() }; - Self { d, inv_sqrt_d } + pub fn new(d: usize, bits: usize) -> Self { + assert!(is_supported_rq_bits(bits), "RQ bits must be in 1..=8"); + let padded_d = padded_dimension(d); + Self { + d, + padded_d, + bits, + plane_size: padded_d / 8, + } } - pub fn code_size(&self) -> usize { - self.d.div_ceil(8) + pub fn dimension(&self) -> usize { + self.d } - pub fn encode( - &self, - rotated_residual: &[f32], - vector_norm_sqr: f32, - code: &mut [u8], - ) -> RQCodeFactors { - debug_assert_eq!(rotated_residual.len(), self.d); - debug_assert!(code.len() >= self.code_size()); - code[..self.code_size()].fill(0); - - let (residual_norm_sqr, abs_sum) = fvec_norm_l2sqr_abs_sum(rotated_residual); - for (byte_idx, chunk) in rotated_residual.chunks(8).enumerate() { - let mut byte = 0u8; - for (bit, &value) in chunk.iter().enumerate() { - if value > 0.0 { - byte |= 1u8 << bit; - } - } - code[byte_idx] = byte; - } - - let dp_multiplier = if abs_sum > f32::EPSILON { - residual_norm_sqr / (abs_sum * self.inv_sqrt_d) - } else { - 0.0 - }; - RQCodeFactors { - residual_norm_sqr, - vector_norm_sqr, - dp_multiplier, - } + pub fn padded_dimension(&self) -> usize { + self.padded_d } - pub fn distance_to_code( - &self, - rotated_query_residual: &[f32], - query: &[f32], - code: &[u8], - factors: RQCodeFactors, - metric: MetricType, - ) -> f32 { - debug_assert_eq!(rotated_query_residual.len(), self.d); - debug_assert_eq!(query.len(), self.d); - - let context = self.prepare_distance_context(rotated_query_residual.to_vec(), query, false); - self.distance_to_code_prepared(&context, code, factors, metric) + pub fn bits(&self) -> usize { + self.bits } - pub fn prepare_distance_context( - &self, - rotated_query_residual: Vec, - query: &[f32], - build_byte_lut: bool, - ) -> RQDistanceContext { - self.prepare_distance_context_with_query_bits( - rotated_query_residual, - query, - build_byte_lut, - DEFAULT_RQ_QUERY_BITS, - ) + pub fn plane_size(&self) -> usize { + self.plane_size } - pub fn prepare_distance_context_with_query_bits( - &self, - rotated_query_residual: Vec, - query: &[f32], - build_byte_lut: bool, - query_bits: usize, - ) -> RQDistanceContext { - debug_assert_eq!(rotated_query_residual.len(), self.d); - debug_assert_eq!(query.len(), self.d); - assert!( - is_supported_query_bits(query_bits), - "unsupported IVF-RQ query_bits {}; expected 0, 4, or 8", - query_bits - ); + pub fn code_size(&self) -> usize { + self.plane_size * self.bits + } - let query_residual_norm_sqr = fvec_norm_l2sqr(&rotated_query_residual); - let query_norm_sqr = fvec_norm_l2sqr(query); - let quantized_query = if query_bits == DEFAULT_RQ_QUERY_BITS { - None - } else { - Some(self.quantize_query(&rotated_query_residual, query_bits)) - }; - let byte_signed_sums = if quantized_query.is_none() && build_byte_lut { - Some(self.build_byte_signed_sums(&rotated_query_residual)) + pub fn factor_fields(&self) -> usize { + if self.bits == 1 { + 2 } else { - None - }; - - RQDistanceContext { - d: self.d, - code_size: self.code_size(), - rotated_query_residual, - query_residual_norm_sqr, - query_norm_sqr, - byte_signed_sums, - quantized_query, + 5 } } - pub fn distance_to_code_prepared( + pub fn encode( &self, - context: &RQDistanceContext, - code: &[u8], - factors: RQCodeFactors, + rotated_residual: &[f32], + rotated_centroid: &[f32], metric: MetricType, - ) -> f32 { - debug_assert_eq!(context.d, self.d); - debug_assert!(code.len() >= context.code_size); - - let signed_query_sum = self.signed_query_sum(context, code); - let approx_ip = factors.dp_multiplier * signed_query_sum * self.inv_sqrt_d; - let approx_l2 = (factors.residual_norm_sqr + context.query_residual_norm_sqr - - 2.0 * approx_ip) - .max(0.0); - - match metric { - MetricType::L2 => approx_l2, - MetricType::Cosine => 0.5 * approx_l2, - MetricType::InnerProduct => { - let base = factors.residual_norm_sqr - factors.vector_norm_sqr; - let pre_dist = base + context.query_residual_norm_sqr - 2.0 * approx_ip; - 0.5 * (pre_dist - context.query_norm_sqr) - } - } - } - - fn signed_query_sum(&self, context: &RQDistanceContext, code: &[u8]) -> f32 { - if let Some(quantized_query) = &context.quantized_query { - return quantized_query.signed_query_sum(code, context.code_size); - } - - if let Some(byte_signed_sums) = &context.byte_signed_sums { - let mut sum = 0.0f32; - for byte_idx in 0..context.code_size { - sum += byte_signed_sums[byte_idx * 256 + code[byte_idx] as usize]; - } - return sum; - } - - let mut sum = 0.0f32; - for (dim, &value) in context.rotated_query_residual.iter().enumerate() { - sum += if get_bit(code, dim) { value } else { -value }; - } - sum + code: &mut [u8], + ) -> RQVectorFactors { + let mut scratch = RQEncodeScratch::new(self.padded_d); + self.encode_with_scratch( + rotated_residual, + rotated_centroid, + metric, + code, + &mut scratch, + ) } - fn quantize_query( + pub fn encode_with_scratch( &self, - rotated_query_residual: &[f32], - query_bits: usize, - ) -> RQQuantizedQuery { - let magnitude_bits = query_bits - 1; - let max_level = (1usize << magnitude_bits) - 1; - let code_size = self.code_size(); - let max_abs = rotated_query_residual - .iter() - .map(|value| value.abs()) - .fold(0.0f32, f32::max); - let scale = if max_abs > f32::EPSILON { - max_abs / max_level as f32 - } else { - 0.0 - }; - let mut sign_bits = vec![0u8; code_size]; - let mut magnitude_bit_planes = vec![vec![0u8; code_size]; magnitude_bits]; - - if scale == 0.0 { - return RQQuantizedQuery { - scale, - sign_bits, - magnitude_bit_planes, - }; - } + rotated_residual: &[f32], + rotated_centroid: &[f32], + metric: MetricType, + code: &mut [u8], + scratch: &mut RQEncodeScratch, + ) -> RQVectorFactors { + debug_assert_eq!(rotated_residual.len(), self.padded_d); + debug_assert_eq!(rotated_centroid.len(), self.padded_d); + debug_assert!(code.len() >= self.code_size()); + debug_assert_eq!(scratch.levels.len(), self.padded_d); + code[..self.code_size()].fill(0); - for (dim, &value) in rotated_query_residual.iter().enumerate() { - if value >= 0.0 { - sign_bits[dim / 8] |= 1u8 << (dim % 8); - } - let level = (value.abs() / scale).round().clamp(0.0, max_level as f32) as usize; - for (bit, plane) in magnitude_bit_planes.iter_mut().enumerate() { - if (level >> bit) & 1 != 0 { - plane[dim / 8] |= 1u8 << (dim % 8); + quantize_centered_levels( + rotated_residual, + self.bits, + &mut scratch.levels, + &mut scratch.centered, + ); + for (dim, &level) in scratch.levels.iter().enumerate() { + for stored_plane in 0..self.bits { + let source_bit = self.bits - 1 - stored_plane; + if level & (1u8 << source_bit) != 0 { + code[stored_plane * self.plane_size + dim / 8] |= 1u8 << (dim % 8); } } + scratch.coarse_centered[dim] = if level & (1u8 << (self.bits - 1)) != 0 { + 0.5 + } else { + -0.5 + }; } - RQQuantizedQuery { - scale, - sign_bits, - magnitude_bit_planes, - } + let coarse = compute_factors( + rotated_residual, + rotated_centroid, + &scratch.coarse_centered, + metric, + ); + let full = if self.bits == 1 { + coarse + } else { + compute_factors( + rotated_residual, + rotated_centroid, + &scratch.centered, + metric, + ) + }; + RQVectorFactors { coarse, full } } - fn build_byte_signed_sums(&self, rotated_query_residual: &[f32]) -> Vec { - let code_size = self.code_size(); - let mut byte_signed_sums = vec![0.0f32; code_size * 256]; - for byte_idx in 0..code_size { + pub fn prepare_query(&self, rotated_query: Vec) -> RQQueryContext { + debug_assert_eq!(rotated_query.len(), self.padded_d); + let sum = rotated_query.iter().sum(); + let mut byte_subset_sums = vec![0.0f32; self.plane_size * 256]; + for byte_idx in 0..self.plane_size { let dim_base = byte_idx * 8; - let dim_end = (dim_base + 8).min(self.d); - let lut = &mut byte_signed_sums[byte_idx * 256..(byte_idx + 1) * 256]; - lut[0] = -rotated_query_residual[dim_base..dim_end] - .iter() - .sum::(); + let lut = &mut byte_subset_sums[byte_idx * 256..(byte_idx + 1) * 256]; for pattern in 1..256usize { let bit = pattern.trailing_zeros() as usize; let previous = pattern & (pattern - 1); - let value = if dim_base + bit < dim_end { - rotated_query_residual[dim_base + bit] - } else { - 0.0 - }; - lut[pattern] = lut[previous] + 2.0 * value; + lut[pattern] = lut[previous] + rotated_query[dim_base + bit]; } } - byte_signed_sums + RQQueryContext { + rotated_query, + sum, + byte_subset_sums, + } } -} -impl RQQuantizedQuery { - fn signed_query_sum(&self, code: &[u8], code_size: usize) -> f32 { - if self.scale == 0.0 { - return 0.0; + pub fn query_terms( + &self, + context: &RQQueryContext, + rotated_centroid: &[f32], + metric: MetricType, + ) -> RQQueryTerms { + debug_assert_eq!(rotated_centroid.len(), self.padded_d); + let mut residual_norm_sqr = 0.0f32; + let mut query_centroid_ip = 0.0f32; + for (&query, ¢roid) in context.rotated_query.iter().zip(rotated_centroid) { + let residual = query - centroid; + residual_norm_sqr += residual * residual; + query_centroid_ip += query * centroid; } + match metric { + MetricType::L2 => RQQueryTerms { + g_add: residual_norm_sqr, + g_error: residual_norm_sqr.sqrt(), + }, + MetricType::Cosine => RQQueryTerms { + g_add: 0.5 * residual_norm_sqr, + g_error: residual_norm_sqr.sqrt(), + }, + MetricType::InnerProduct => RQQueryTerms { + g_add: -query_centroid_ip, + g_error: residual_norm_sqr.sqrt(), + }, + } + } - let mut signed_level_sum = 0i64; - for (bit, plane) in self.magnitude_bit_planes.iter().enumerate() { - let weight = 1i64 << bit; - let mut plane_sum = 0i64; - let mut offset = 0usize; - - while offset + 8 <= code_size { - let selected = u64::from_le_bytes(plane[offset..offset + 8].try_into().unwrap()); - if selected != 0 { - let code_bits = - u64::from_le_bytes(code[offset..offset + 8].try_into().unwrap()); - let sign_bits = - u64::from_le_bytes(self.sign_bits[offset..offset + 8].try_into().unwrap()); - let same_sign = !(code_bits ^ sign_bits) & selected; - plane_sum += 2 * same_sign.count_ones() as i64 - selected.count_ones() as i64; - } - offset += 8; - } + pub fn unsigned_plane_inner_product(&self, context: &RQQueryContext, plane_code: &[u8]) -> f32 { + debug_assert!(plane_code.len() >= self.plane_size); + let mut result = 0.0f32; + for byte_idx in 0..self.plane_size { + result += context.byte_subset_sums[byte_idx * 256 + plane_code[byte_idx] as usize]; + } + result + } - while offset < code_size { - let selected = plane[offset]; - if selected != 0 { - let same_sign = !(code[offset] ^ self.sign_bits[offset]) & selected; - plane_sum += 2 * same_sign.count_ones() as i64 - selected.count_ones() as i64; - } - offset += 1; - } + pub(crate) fn byte_subset_sum( + &self, + context: &RQQueryContext, + byte_idx: usize, + pattern: u8, + ) -> f32 { + context.byte_subset_sums[byte_idx * 256 + pattern as usize] + } - signed_level_sum += weight * plane_sum; - } + pub(crate) fn query_sum(&self, context: &RQQueryContext) -> f32 { + context.sum + } - self.scale * signed_level_sum as f32 + pub fn coarse_inner_product(&self, context: &RQQueryContext, code: &[u8]) -> f32 { + self.unsigned_plane_inner_product(context, code) - 0.5 * context.sum } -} -#[inline] -pub fn is_supported_query_bits(query_bits: usize) -> bool { - matches!(query_bits, 0 | 4 | 8) -} + pub fn full_inner_product(&self, context: &RQQueryContext, code: &[u8]) -> f32 { + debug_assert!(code.len() >= self.code_size()); + let mut unsigned = 0.0f32; + for stored_plane in 0..self.bits { + let weight = (1usize << (self.bits - 1 - stored_plane)) as f32; + let start = stored_plane * self.plane_size; + unsigned += weight + * self.unsigned_plane_inner_product(context, &code[start..start + self.plane_size]); + } + let center = ((1usize << self.bits) - 1) as f32 * 0.5; + unsigned - center * context.sum + } -fn get_bit(code: &[u8], dim: usize) -> bool { - code[dim / 8] & (1u8 << (dim % 8)) != 0 -} + pub fn estimate( + &self, + inner_product: f32, + factors: RQCodeFactors, + query_terms: RQQueryTerms, + ) -> f32 { + factors.f_add + query_terms.g_add + factors.f_rescale * inner_product + } -#[cfg(target_arch = "x86_64")] -#[inline] -fn fvec_norm_l2sqr_abs_sum(values: &[f32]) -> (f32, f32) { - if is_x86_feature_detected!("avx2") && values.len() >= 8 { - unsafe { fvec_norm_l2sqr_abs_sum_avx2(values) } - } else { - fvec_norm_l2sqr_abs_sum_scalar(values) + pub fn lower_bound( + &self, + estimate: f32, + factors: RQCodeFactors, + query_terms: RQQueryTerms, + ) -> f32 { + estimate - factors.f_error * query_terms.g_error } } -#[cfg(target_arch = "aarch64")] -#[inline] -fn fvec_norm_l2sqr_abs_sum(values: &[f32]) -> (f32, f32) { - unsafe { fvec_norm_l2sqr_abs_sum_neon(values) } +pub fn padded_dimension(d: usize) -> usize { + d.max(1).div_ceil(RQ_ROTATION_BLOCK_SIZE) * RQ_ROTATION_BLOCK_SIZE } -#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] #[inline] -fn fvec_norm_l2sqr_abs_sum(values: &[f32]) -> (f32, f32) { - fvec_norm_l2sqr_abs_sum_scalar(values) +pub fn is_supported_rq_bits(bits: usize) -> bool { + (1..=8).contains(&bits) } -#[cfg(any( - target_arch = "x86_64", - not(any(target_arch = "x86_64", target_arch = "aarch64")) -))] -#[inline] -fn fvec_norm_l2sqr_abs_sum_scalar(values: &[f32]) -> (f32, f32) { - let mut norm = 0.0f32; - let mut abs_sum = 0.0f32; - for &value in values { - norm += value * value; - abs_sum += value.abs(); - } - (norm, abs_sum) +fn quantize_centered_levels( + residual: &[f32], + bits: usize, + levels: &mut [u8], + centered: &mut [f32], +) { + let max_code = (1usize << bits) - 1; + let center = max_code as f32 * 0.5; + let max_abs = residual + .iter() + .map(|value| value.abs()) + .fold(0.0f32, f32::max); + if max_abs <= f32::EPSILON { + levels.fill(0); + centered.fill(-center); + return; + } + + let mut scale = 2.0 * max_abs / max_code as f32; + for _ in 0..QUANTIZATION_REFINEMENT_ROUNDS { + let mut dot = 0.0f32; + let mut norm_sqr = 0.0f32; + for (dim, &value) in residual.iter().enumerate() { + let level = (value / scale + center).round().clamp(0.0, max_code as f32) as u8; + let centered_level = level as f32 - center; + levels[dim] = level; + centered[dim] = centered_level; + dot += value * centered_level; + norm_sqr += centered_level * centered_level; + } + if dot <= f32::EPSILON || norm_sqr <= f32::EPSILON { + break; + } + scale = dot / norm_sqr; + } } -#[cfg(target_arch = "x86_64")] -#[target_feature(enable = "avx2")] -unsafe fn fvec_norm_l2sqr_abs_sum_avx2(values: &[f32]) -> (f32, f32) { - use std::arch::x86_64::*; - - let n = values.len(); - let abs_mask = _mm256_castsi256_ps(_mm256_set1_epi32(0x7fff_ffff)); - let mut norm_sum = _mm256_setzero_ps(); - let mut abs_sum_vec = _mm256_setzero_ps(); - let mut i = 0; - while i + 8 <= n { - let value = unsafe { _mm256_loadu_ps(values.as_ptr().add(i)) }; - norm_sum = _mm256_add_ps(norm_sum, _mm256_mul_ps(value, value)); - abs_sum_vec = _mm256_add_ps(abs_sum_vec, _mm256_and_ps(value, abs_mask)); - i += 8; - } - - let norm_hi = _mm256_extractf128_ps::<1>(norm_sum); - let norm_lo = _mm256_castps256_ps128(norm_sum); - let norm_128 = _mm_add_ps(norm_lo, norm_hi); - let norm_64 = _mm_add_ps(norm_128, _mm_movehl_ps(norm_128, norm_128)); - let norm_32 = _mm_add_ss(norm_64, _mm_shuffle_ps::<1>(norm_64, norm_64)); - let mut norm = _mm_cvtss_f32(norm_32); - - let abs_hi = _mm256_extractf128_ps::<1>(abs_sum_vec); - let abs_lo = _mm256_castps256_ps128(abs_sum_vec); - let abs_128 = _mm_add_ps(abs_lo, abs_hi); - let abs_64 = _mm_add_ps(abs_128, _mm_movehl_ps(abs_128, abs_128)); - let abs_32 = _mm_add_ss(abs_64, _mm_shuffle_ps::<1>(abs_64, abs_64)); - let mut abs_sum = _mm_cvtss_f32(abs_32); - - while i < n { - let value = unsafe { *values.get_unchecked(i) }; - norm += value * value; - abs_sum += value.abs(); - i += 1; - } - (norm, abs_sum) +fn compute_factors( + residual: &[f32], + centroid: &[f32], + centered_code: &[f32], + metric: MetricType, +) -> RQCodeFactors { + let mut residual_norm_sqr = 0.0f32; + let mut residual_code_ip = 0.0f32; + let mut centroid_code_ip = 0.0f32; + let mut residual_centroid_ip = 0.0f32; + for ((&value, ¢er), &code) in residual.iter().zip(centroid).zip(centered_code) { + residual_norm_sqr += value * value; + residual_code_ip += value * code; + centroid_code_ip += center * code; + residual_centroid_ip += value * center; + } + if residual_norm_sqr <= f32::EPSILON || residual_code_ip.abs() <= f32::EPSILON { + return match metric { + MetricType::L2 => RQCodeFactors { + f_add: residual_norm_sqr, + f_rescale: 0.0, + f_error: 2.0 * residual_norm_sqr.sqrt(), + }, + MetricType::Cosine => RQCodeFactors { + f_add: 0.5 * residual_norm_sqr, + f_rescale: 0.0, + f_error: residual_norm_sqr.sqrt(), + }, + MetricType::InnerProduct => RQCodeFactors { + f_add: -residual_centroid_ip, + f_rescale: 0.0, + f_error: residual_norm_sqr.sqrt(), + }, + }; + } + + let rescale = residual_norm_sqr / residual_code_ip; + let mut reconstruction_error_sqr = 0.0f32; + for (&value, &code) in residual.iter().zip(centered_code) { + let error = value - rescale * code; + reconstruction_error_sqr += error * error; + } + let reconstruction_error = reconstruction_error_sqr.sqrt(); + + match metric { + MetricType::L2 => RQCodeFactors { + f_add: residual_norm_sqr + 2.0 * rescale * centroid_code_ip, + f_rescale: -2.0 * rescale, + f_error: 2.0 * reconstruction_error, + }, + MetricType::Cosine => RQCodeFactors { + f_add: 0.5 * residual_norm_sqr + rescale * centroid_code_ip, + f_rescale: -rescale, + f_error: reconstruction_error, + }, + MetricType::InnerProduct => RQCodeFactors { + f_add: -residual_centroid_ip + rescale * centroid_code_ip, + f_rescale: -rescale, + f_error: reconstruction_error, + }, + } } -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "neon")] -unsafe fn fvec_norm_l2sqr_abs_sum_neon(values: &[f32]) -> (f32, f32) { - use std::arch::aarch64::*; - - let n = values.len(); - let mut norm_sum = vdupq_n_f32(0.0); - let mut abs_sum_vec = vdupq_n_f32(0.0); - let mut i = 0; - while i + 4 <= n { - let value = unsafe { vld1q_f32(values.as_ptr().add(i)) }; - norm_sum = vmlaq_f32(norm_sum, value, value); - abs_sum_vec = vaddq_f32(abs_sum_vec, vabsq_f32(value)); - i += 4; - } - - let mut norm = vaddvq_f32(norm_sum); - let mut abs_sum = vaddvq_f32(abs_sum_vec); - while i < n { - let value = unsafe { *values.get_unchecked(i) }; - norm += value * value; - abs_sum += value.abs(); - i += 1; - } - (norm, abs_sum) +fn hadamard_64(values: &mut [f32]) { + debug_assert_eq!(values.len(), RQ_ROTATION_BLOCK_SIZE); + let mut width = 1; + while width < RQ_ROTATION_BLOCK_SIZE { + for base in (0..RQ_ROTATION_BLOCK_SIZE).step_by(width * 2) { + for offset in 0..width { + let left = values[base + offset]; + let right = values[base + offset + width]; + values[base + offset] = left + right; + values[base + offset + width] = left - right; + } + } + width *= 2; + } + for value in values { + *value *= 0.125; + } } -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] struct SplitMix64 { state: u64, } @@ -534,11 +551,6 @@ impl SplitMix64 { fn next_usize(&mut self, upper: usize) -> usize { (self.next_u64() % upper as u64) as usize } - - fn next_f32(&mut self) -> f32 { - let mantissa = (self.next_u64() >> 40) as u32; - mantissa as f32 / ((1u32 << 24) as f32) - } } #[cfg(test)] @@ -546,168 +558,114 @@ mod tests { use super::*; #[test] - fn rabit_quantizer_estimates_self_distance_as_zero() { - let d = 8; - let rotation = RQRotation::new(d, DEFAULT_RQ_ROTATION_SEED, DEFAULT_RQ_ROTATION_ROUNDS); - let quantizer = RaBitQuantizer::new(d); - let centroid = vec![1.0; d]; - let vector = vec![2.0, 1.5, 0.75, 3.0, 1.25, -1.0, 4.0, 2.5]; - let mut residual: Vec = vector - .iter() - .zip(centroid.iter()) - .map(|(&x, &c)| x - c) - .collect(); - rotation.apply(&mut residual); - - let mut code = vec![0u8; quantizer.code_size()]; - let factors = quantizer.encode(&residual, fvec_norm_l2sqr(&vector), &mut code); - let dist = quantizer.distance_to_code(&residual, &vector, &code, factors, MetricType::L2); + fn rotation_is_deterministic_and_preserves_norm_with_padding() { + let d = 70; + let rotation = RQRotation::new(d, 17, DEFAULT_RQ_ROTATION_ROUNDS); + let input: Vec = (0..d).map(|i| i as f32 * 0.25 - 3.0).collect(); + let mut first = vec![0.0; rotation.padded_dimension()]; + let mut second = vec![0.0; rotation.padded_dimension()]; + let mut scratch = vec![0.0; rotation.padded_dimension()]; + rotation.rotate(&input, &mut first, &mut scratch); + rotation.rotate(&input, &mut second, &mut scratch); + + let before: f32 = input.iter().map(|value| value * value).sum(); + let after: f32 = first.iter().map(|value| value * value).sum(); + assert_eq!(first, second); + assert!((before - after).abs() <= before * 1e-5); + } + #[test] + fn rotation_spreads_a_basis_vector_across_fht_blocks() { + let d = 128; + let rotation = RQRotation::new(d, 91, 2); + let mut input = vec![0.0; d]; + input[7] = 1.0; + let mut output = vec![0.0; rotation.padded_dimension()]; + let mut scratch = vec![0.0; rotation.padded_dimension()]; + rotation.rotate(&input, &mut output, &mut scratch); + + let non_zero = output.iter().filter(|value| value.abs() > 1e-7).count(); assert!( - dist <= 1e-5, - "self distance should be close to zero: {dist}" + non_zero > RQ_ROTATION_BLOCK_SIZE, + "two rounds must spread beyond one 64-dimension block, got {non_zero}" ); } #[test] - fn distance_context_byte_lut_matches_scalar_path() { - let d = 16; - let quantizer = RaBitQuantizer::new(d); - let rotated_residual: Vec = (0..d).map(|i| i as f32 * 0.25 - 1.5).collect(); - let query: Vec = (0..d).map(|i| (i as f32 + 1.0) * 0.125).collect(); - let rotated_query_residual: Vec = (0..d).map(|i| (i as f32 - 3.0) * 0.2).collect(); - - let mut code = vec![0u8; quantizer.code_size()]; - let factors = quantizer.encode(&rotated_residual, fvec_norm_l2sqr(&query), &mut code); - code[0] = 0b1010_0101; - code[1] = 0b0101_1010; - - let scalar_context = - quantizer.prepare_distance_context(rotated_query_residual.clone(), &query, false); - let lut_context = quantizer.prepare_distance_context(rotated_query_residual, &query, true); - - for metric in [MetricType::L2, MetricType::Cosine, MetricType::InnerProduct] { - let scalar = - quantizer.distance_to_code_prepared(&scalar_context, &code, factors, metric); - let lut = quantizer.distance_to_code_prepared(&lut_context, &code, factors, metric); - assert!( - (scalar - lut).abs() < 1e-5, - "metric {:?}: scalar {} != lut {}", - metric, - scalar, - lut - ); - } - } + fn four_bit_estimator_is_exact_for_the_encoded_vector() { + let d = 64; + let quantizer = RaBitQuantizer::new(d, 4); + let residual: Vec = (0..d) + .map(|i| ((i * 17 % 31) as f32 - 15.0) * 0.13) + .collect(); + let centroid: Vec = (0..d).map(|i| (i % 7) as f32 * 0.02).collect(); + let query: Vec = residual + .iter() + .zip(¢roid) + .map(|(&residual, ¢roid)| residual + centroid) + .collect(); + let mut code = vec![0; quantizer.code_size()]; + let factors = quantizer.encode(&residual, ¢roid, MetricType::L2, &mut code); + let context = quantizer.prepare_query(query.clone()); + let terms = quantizer.query_terms(&context, ¢roid, MetricType::L2); + let estimate = quantizer.estimate( + quantizer.full_inner_product(&context, &code), + factors.full, + terms, + ); - #[test] - fn byte_lut_matches_scalar_signed_sum_for_every_pattern() { - let d = 13; - let quantizer = RaBitQuantizer::new(d); - let residual: Vec = (0..d).map(|i| i as f32 * 0.37 - 2.1).collect(); - let lut = quantizer.build_byte_signed_sums(&residual); - let mut code = vec![0u8; quantizer.code_size()]; - - for first_byte in 0..=u8::MAX { - for second_byte in 0..=u8::MAX { - code[0] = first_byte; - code[1] = second_byte; - let scalar: f32 = residual - .iter() - .enumerate() - .map(|(dim, &value)| if get_bit(&code, dim) { value } else { -value }) - .sum(); - let actual = lut[first_byte as usize] + lut[256 + second_byte as usize]; - assert!( - (actual - scalar).abs() < 1e-5, - "code {first_byte:#010b} {second_byte:#010b}: {actual} != {scalar}" - ); - } - } + assert!(estimate.abs() < 1e-4, "self distance was {estimate}"); } #[test] - fn quantized_query_bit_planes_match_scalar_quantization() { - let d = 24; - let quantizer = RaBitQuantizer::new(d); - let rotated_query_residual: Vec = (0..d).map(|i| (i as f32 - 11.0) * 0.17).collect(); - let query: Vec = (0..d).map(|i| (i as f32 + 1.0) * 0.03125).collect(); - let mut code = vec![0u8; quantizer.code_size()]; - for (byte_idx, byte) in code.iter_mut().enumerate() { - *byte = if byte_idx % 2 == 0 { - 0b1010_1100 - } else { - 0b0101_0011 - }; - } - - for query_bits in [4, 8] { - let context = quantizer.prepare_distance_context_with_query_bits( - rotated_query_residual.clone(), - &query, - true, - query_bits, - ); - let quantized_query = context.quantized_query.as_ref().unwrap(); - let actual = quantized_query.signed_query_sum(&code, quantizer.code_size()); - let expected = - scalar_quantized_signed_query_sum(&rotated_query_residual, &code, query_bits); - - assert!( - (actual - expected).abs() < 1e-5, - "query_bits {}: {} != {}", - query_bits, - actual, - expected - ); - assert!(context.byte_signed_sums.is_none()); + fn more_stored_bits_reduce_reconstruction_error() { + let d = 64; + let residual: Vec = (0..d) + .map(|i| ((i * 29 % 53) as f32 - 26.0) * 0.11) + .collect(); + let centroid = vec![0.0; d]; + let mut errors = Vec::new(); + for bits in [1, 2, 4, 8] { + let quantizer = RaBitQuantizer::new(d, bits); + let mut code = vec![0; quantizer.code_size()]; + let factors = quantizer.encode(&residual, ¢roid, MetricType::L2, &mut code); + errors.push(factors.full.f_error); } - } - - #[test] - fn norm_l2sqr_abs_sum_helper_matches_expected_values() { - let values = [-3.0f32, 4.0, 0.5, -0.25, 8.0, -2.0, 1.25, -6.0, 7.0]; - let (norm, abs_sum) = fvec_norm_l2sqr_abs_sum(&values); - let expected_norm: f32 = values.iter().map(|value| value * value).sum(); - let expected_abs_sum: f32 = values.iter().map(|value| value.abs()).sum(); - assert!((norm - expected_norm).abs() < 1e-5); - assert!((abs_sum - expected_abs_sum).abs() < 1e-5); + assert!(errors.windows(2).all(|pair| pair[1] <= pair[0] + 1e-5)); + assert!(errors[3] < errors[0] * 0.1); } #[test] - fn rotation_preserves_l2_norm() { - let d = 16; - let rotation = RQRotation::new(d, 11, 3); - let mut vector: Vec = (0..d).map(|i| i as f32 - 3.0).collect(); - let before = fvec_norm_l2sqr(&vector); - rotation.apply(&mut vector); - let after = fvec_norm_l2sqr(&vector); - assert!((before - after).abs() < 1e-4); - } - - fn scalar_quantized_signed_query_sum( - rotated_query_residual: &[f32], - code: &[u8], - query_bits: usize, - ) -> f32 { - let magnitude_bits = query_bits - 1; - let max_level = (1usize << magnitude_bits) - 1; - let max_abs = rotated_query_residual + fn lower_bound_contains_exact_distance() { + let d = 64; + let quantizer = RaBitQuantizer::new(d, 4); + let residual: Vec = (0..d) + .map(|i| ((i * 11 % 37) as f32 - 18.0) * 0.09) + .collect(); + let query: Vec = (0..d) + .map(|i| ((i * 7 % 41) as f32 - 20.0) * 0.08) + .collect(); + let centroid = vec![0.0; d]; + let mut code = vec![0; quantizer.code_size()]; + let factors = quantizer.encode(&residual, ¢roid, MetricType::L2, &mut code); + let context = quantizer.prepare_query(query.clone()); + let terms = quantizer.query_terms(&context, ¢roid, MetricType::L2); + let coarse = quantizer.estimate( + quantizer.coarse_inner_product(&context, &code), + factors.coarse, + terms, + ); + let lower = quantizer.lower_bound(coarse, factors.coarse, terms); + let exact: f32 = residual .iter() - .map(|value| value.abs()) - .fold(0.0f32, f32::max); - if max_abs <= f32::EPSILON { - return 0.0; - } - let scale = max_abs / max_level as f32; - let mut sum = 0i64; - for (dim, &value) in rotated_query_residual.iter().enumerate() { - let code_sign = if get_bit(code, dim) { 1i64 } else { -1i64 }; - let query_sign = if value >= 0.0 { 1i64 } else { -1i64 }; - let level = (value.abs() / scale).round().clamp(0.0, max_level as f32) as i64; - sum += code_sign * query_sign * level; - } - scale * sum as f32 + .zip(query) + .map(|(&left, right)| { + let delta = left - right; + delta * delta + }) + .sum(); + + assert!(lower <= exact + 1e-4, "lower={lower}, exact={exact}"); } } diff --git a/core/src/sparse_table.rs b/core/src/sparse_table.rs new file mode 100644 index 0000000..11072d8 --- /dev/null +++ b/core/src/sparse_table.rs @@ -0,0 +1,207 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const EMPTY_KEY: u32 = u32::MAX; +const MIN_CAPACITY: usize = 8; + +pub(crate) fn estimated_memory_bytes(expected_entries: usize, value_size: usize) -> Option { + let capacity = expected_entries + .checked_mul(2)? + .max(MIN_CAPACITY) + .checked_next_power_of_two()?; + capacity + .checked_mul(std::mem::size_of::().checked_add(value_size)?)? + .checked_add(expected_entries.checked_mul(std::mem::size_of::())?) +} + +pub(crate) struct SparseTable { + keys: Vec, + values: Vec, + touched_slots: Vec, +} + +impl SparseTable { + pub(crate) fn with_capacity(expected_entries: usize) -> Self { + Self::try_with_capacity(expected_entries).expect("SparseTable allocation failed") + } + + pub(crate) fn try_with_capacity(expected_entries: usize) -> Result { + let capacity = expected_entries + .checked_mul(2) + .ok_or(())? + .max(MIN_CAPACITY) + .checked_next_power_of_two() + .ok_or(())?; + let mut keys = Vec::new(); + keys.try_reserve_exact(capacity).map_err(|_| ())?; + keys.resize(capacity, EMPTY_KEY); + let mut values = Vec::new(); + values.try_reserve_exact(capacity).map_err(|_| ())?; + values.resize(capacity, V::default()); + let mut touched_slots = Vec::new(); + touched_slots + .try_reserve_exact(expected_entries) + .map_err(|_| ())?; + Ok(Self { + keys, + values, + touched_slots, + }) + } + + pub(crate) fn get(&self, key: u32) -> Option<&V> { + assert_ne!(key, EMPTY_KEY, "u32::MAX is reserved by SparseTable"); + let slot = self.find_slot(key)?; + (self.keys[slot] == key).then(|| &self.values[slot]) + } + + pub(crate) fn insert(&mut self, key: u32, value: V) -> Option { + assert_ne!(key, EMPTY_KEY, "u32::MAX is reserved by SparseTable"); + let slot = self.find_slot(key).expect("SparseTable must have a slot"); + if self.keys[slot] == key { + return Some(std::mem::replace(&mut self.values[slot], value)); + } + if self.touched_slots.len() + 1 > self.keys.len() * 7 / 10 { + self.grow(); + } + let slot = self + .find_slot(key) + .expect("grown SparseTable must have a slot"); + self.keys[slot] = key; + self.values[slot] = value; + self.touched_slots.push(slot as u32); + None + } + + #[cfg(test)] + pub(crate) fn len(&self) -> usize { + self.touched_slots.len() + } + + pub(crate) fn clear(&mut self) { + for slot in self.touched_slots.drain(..) { + self.keys[slot as usize] = EMPTY_KEY; + } + } + + pub(crate) fn entry_capacity(&self) -> usize { + self.keys.len() * 7 / 10 + } + + #[cfg(test)] + fn capacity(&self) -> usize { + self.keys.len() + } + + fn find_slot(&self, key: u32) -> Option { + let mask = self.keys.len() - 1; + let mut slot = mix_u32(key) as usize & mask; + for _ in 0..self.keys.len() { + let stored = self.keys[slot]; + if stored == EMPTY_KEY || stored == key { + return Some(slot); + } + slot = (slot + 1) & mask; + } + None + } + + fn grow(&mut self) { + let capacity = self + .keys + .len() + .checked_mul(2) + .expect("SparseTable capacity overflow"); + let old_keys = std::mem::replace(&mut self.keys, vec![EMPTY_KEY; capacity]); + let old_values = std::mem::replace(&mut self.values, vec![V::default(); capacity]); + let old_touched = std::mem::replace( + &mut self.touched_slots, + Vec::with_capacity(capacity.saturating_mul(7) / 10), + ); + for old_slot in old_touched { + let old_slot = old_slot as usize; + let key = old_keys[old_slot]; + let slot = self + .find_slot(key) + .expect("empty grown SparseTable must have a slot"); + self.keys[slot] = key; + self.values[slot] = old_values[old_slot]; + self.touched_slots.push(slot as u32); + } + } +} + +impl Default for SparseTable { + fn default() -> Self { + Self::with_capacity(0) + } +} + +fn mix_u32(mut value: u32) -> u32 { + value ^= value >> 16; + value = value.wrapping_mul(0x7FEB_352D); + value ^= value >> 15; + value = value.wrapping_mul(0x846C_A68B); + value ^ (value >> 16) +} + +#[cfg(test)] +mod tests { + use super::SparseTable; + + #[test] + fn sparse_table_inserts_and_updates_values() { + let mut table = SparseTable::with_capacity(2); + + assert_eq!(table.insert(7, 1u8), None); + assert_eq!(table.get(7), Some(&1)); + assert_eq!(table.insert(7, 3), Some(1)); + assert_eq!(table.get(7), Some(&3)); + assert_eq!(table.len(), 1); + } + + #[test] + fn sparse_table_clear_reuses_allocated_storage() { + let mut table = SparseTable::with_capacity(4); + for key in 0..4 { + table.insert(key, key); + } + let capacity = table.capacity(); + + table.clear(); + + assert_eq!(table.len(), 0); + assert_eq!(table.get(2), None); + assert_eq!(table.capacity(), capacity); + assert_eq!(table.insert(11, 9), None); + assert_eq!(table.get(11), Some(&9)); + } + + #[test] + fn sparse_table_grows_past_the_initial_estimate() { + let mut table = SparseTable::with_capacity(1); + + for key in 0..1_000 { + assert_eq!(table.insert(key, key + 1), None); + } + + assert_eq!(table.len(), 1_000); + for key in 0..1_000 { + assert_eq!(table.get(key), Some(&(key + 1))); + } + } +} diff --git a/core/src/sq.rs b/core/src/sq.rs index 112f887..0785055 100644 --- a/core/src/sq.rs +++ b/core/src/sq.rs @@ -201,6 +201,85 @@ impl ScalarQuantizer { ScalarQuantizerDecodeLut { d: self.d, values } } + pub(crate) fn distances_to_blocked_codes_with_offset( + &self, + query: &[f32], + codes: &[u8], + count: usize, + offset: &[f32], + metric: MetricType, + block_size: usize, + parameters: &mut Vec, + distances: &mut Vec, + ) { + debug_assert!(query.len() >= self.d); + debug_assert!(offset.len() >= self.d); + debug_assert_eq!(codes.len(), count * self.d); + debug_assert!(block_size > 0); + + parameters.resize(self.d * 2, 0.0); + distances.resize(count, 0.0); + let (primary, scales) = parameters.split_at_mut(self.d); + + if metric == MetricType::L2 { + for dimension in 0..self.d { + primary[dimension] = query[dimension] - offset[dimension] - self.mins[dimension]; + scales[dimension] = (self.maxs[dimension] - self.mins[dimension]) * (1.0 / 255.0); + } + blocked_sq_l2(primary, scales, codes, count, self.d, block_size, distances); + return; + } + + for dimension in 0..self.d { + primary[dimension] = offset[dimension] + self.mins[dimension]; + scales[dimension] = (self.maxs[dimension] - self.mins[dimension]) * (1.0 / 255.0); + } + let query_norm = if metric == MetricType::Cosine { + fvec_norm_l2sqr(&query[..self.d]).sqrt() + } else { + 0.0 + }; + let mut code_offset = 0usize; + for block_start in (0..count).step_by(block_size) { + let block_len = (count - block_start).min(block_size); + let block_distances = &mut distances[block_start..block_start + block_len]; + block_distances.fill(0.0); + let mut norms = (metric == MetricType::Cosine).then(|| vec![0.0f32; block_len]); + for dimension in 0..self.d { + let column = &codes[code_offset + dimension * block_len + ..code_offset + (dimension + 1) * block_len]; + let base = primary[dimension]; + let scale = scales[dimension]; + for lane in 0..block_len { + let decoded = base + column[lane] as f32 * scale; + block_distances[lane] += query[dimension] * decoded; + if let Some(norms) = norms.as_mut() { + norms[lane] += decoded * decoded; + } + } + } + match metric { + MetricType::InnerProduct => { + for distance in block_distances { + *distance = -*distance; + } + } + MetricType::Cosine => { + for (distance, norm) in block_distances.iter_mut().zip(norms.unwrap()) { + let denominator = query_norm * norm.sqrt(); + *distance = if denominator > 0.0 { + 1.0 - *distance / denominator + } else { + 1.0 + }; + } + } + MetricType::L2 => unreachable!(), + } + code_offset += block_len * self.d; + } + } + fn distance_to_code_impl( &self, query: &[f32], @@ -213,22 +292,21 @@ impl ScalarQuantizer { debug_assert!(code.len() >= self.d); match context.metric { - MetricType::L2 => { - let mut sum = 0.0f32; - for i in 0..self.d { - let diff = - query[i] - self.decode_value_with_offset(code[i], i, offset, use_offset); - sum += diff * diff; - } - sum - } - MetricType::InnerProduct => { - let mut dot = 0.0f32; - for i in 0..self.d { - dot += query[i] * self.decode_value_with_offset(code[i], i, offset, use_offset); - } - -dot + MetricType::L2 if use_offset => { + sq_l2_with_offset_simd(query, code, &self.mins, &self.maxs, offset, self.d) } + MetricType::L2 => (0..self.d) + .map(|i| { + let diff = query[i] - self.decode_value(code[i], i); + diff * diff + }) + .sum(), + MetricType::InnerProduct if use_offset => -sq_inner_product_with_offset_simd( + query, code, &self.mins, &self.maxs, offset, self.d, + ), + MetricType::InnerProduct => -(0..self.d) + .map(|i| query[i] * self.decode_value(code[i], i)) + .sum::(), MetricType::Cosine => { let mut dot = 0.0f32; let mut vector_norm = 0.0f32; @@ -341,6 +419,339 @@ impl ScalarQuantizer { } } +fn blocked_sq_l2( + biases: &[f32], + scales: &[f32], + codes: &[u8], + count: usize, + d: usize, + block_size: usize, + distances: &mut [f32], +) { + #[cfg(target_arch = "x86_64")] + if block_size == 32 && is_x86_feature_detected!("avx2") { + unsafe { + return blocked_sq_l2_avx2(biases, scales, codes, count, d, distances); + } + } + #[cfg(target_arch = "aarch64")] + if block_size == 32 { + unsafe { + return blocked_sq_l2_neon(biases, scales, codes, count, d, distances); + } + } + blocked_sq_l2_scalar(biases, scales, codes, count, d, block_size, distances); +} + +fn blocked_sq_l2_scalar( + biases: &[f32], + scales: &[f32], + codes: &[u8], + count: usize, + d: usize, + block_size: usize, + distances: &mut [f32], +) { + let mut code_offset = 0usize; + for block_start in (0..count).step_by(block_size) { + let block_len = (count - block_start).min(block_size); + let block_distances = &mut distances[block_start..block_start + block_len]; + block_distances.fill(0.0); + for dimension in 0..d { + let column = &codes + [code_offset + dimension * block_len..code_offset + (dimension + 1) * block_len]; + let bias = biases[dimension]; + let scale = scales[dimension]; + for lane in 0..block_len { + let difference = bias - column[lane] as f32 * scale; + block_distances[lane] += difference * difference; + } + } + code_offset += block_len * d; + } +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +unsafe fn blocked_sq_l2_neon( + biases: &[f32], + scales: &[f32], + codes: &[u8], + count: usize, + d: usize, + distances: &mut [f32], +) { + use std::arch::aarch64::*; + + let full_count = count / 32 * 32; + for block_start in (0..full_count).step_by(32) { + let code_base = block_start * d; + let mut accumulators = [vdupq_n_f32(0.0); 8]; + for dimension in 0..d { + let column = codes.as_ptr().add(code_base + dimension * 32); + let bias = vdupq_n_f32(biases[dimension]); + let scale = vdupq_n_f32(scales[dimension]); + for chunk in 0..4 { + let code_u16 = vmovl_u8(vld1_u8(column.add(chunk * 8))); + let code_low = vcvtq_f32_u32(vmovl_u16(vget_low_u16(code_u16))); + let code_high = vcvtq_f32_u32(vmovl_u16(vget_high_u16(code_u16))); + let difference_low = vfmsq_f32(bias, code_low, scale); + let difference_high = vfmsq_f32(bias, code_high, scale); + accumulators[chunk * 2] = + vfmaq_f32(accumulators[chunk * 2], difference_low, difference_low); + accumulators[chunk * 2 + 1] = vfmaq_f32( + accumulators[chunk * 2 + 1], + difference_high, + difference_high, + ); + } + } + for (chunk, accumulator) in accumulators.into_iter().enumerate() { + vst1q_f32( + distances.as_mut_ptr().add(block_start + chunk * 4), + accumulator, + ); + } + } + if full_count < count { + blocked_sq_l2_scalar( + biases, + scales, + &codes[full_count * d..], + count - full_count, + d, + 32, + &mut distances[full_count..], + ); + } +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn blocked_sq_l2_avx2( + biases: &[f32], + scales: &[f32], + codes: &[u8], + count: usize, + d: usize, + distances: &mut [f32], +) { + use std::arch::x86_64::*; + + let full_count = count / 32 * 32; + for block_start in (0..full_count).step_by(32) { + let code_base = block_start * d; + let mut accumulators = [_mm256_setzero_ps(); 4]; + for dimension in 0..d { + let column = codes.as_ptr().add(code_base + dimension * 32); + let bias = _mm256_set1_ps(biases[dimension]); + let scale = _mm256_set1_ps(scales[dimension]); + for (chunk, accumulator) in accumulators.iter_mut().enumerate() { + let bytes = _mm_loadl_epi64(column.add(chunk * 8).cast()); + let code = _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(bytes)); + let difference = _mm256_sub_ps(bias, _mm256_mul_ps(code, scale)); + *accumulator = _mm256_add_ps(*accumulator, _mm256_mul_ps(difference, difference)); + } + } + for (chunk, accumulator) in accumulators.into_iter().enumerate() { + _mm256_storeu_ps( + distances.as_mut_ptr().add(block_start + chunk * 8), + accumulator, + ); + } + } + if full_count < count { + blocked_sq_l2_scalar( + biases, + scales, + &codes[full_count * d..], + count - full_count, + d, + 32, + &mut distances[full_count..], + ); + } +} + +#[inline] +fn sq_l2_with_offset_simd( + query: &[f32], + codes: &[u8], + mins: &[f32], + maxs: &[f32], + offset: &[f32], + d: usize, +) -> f32 { + #[cfg(target_arch = "x86_64")] + { + if is_x86_feature_detected!("avx2") && d >= 8 { + return unsafe { + sq_distance_with_offset_avx2::(query, codes, mins, maxs, offset, d) + }; + } + } + #[cfg(target_arch = "aarch64")] + { + if d >= 8 { + return unsafe { + sq_distance_with_offset_neon::(query, codes, mins, maxs, offset, d) + }; + } + } + sq_distance_with_offset_scalar::(query, codes, mins, maxs, offset, d) +} + +#[inline] +fn sq_inner_product_with_offset_simd( + query: &[f32], + codes: &[u8], + mins: &[f32], + maxs: &[f32], + offset: &[f32], + d: usize, +) -> f32 { + #[cfg(target_arch = "x86_64")] + { + if is_x86_feature_detected!("avx2") && d >= 8 { + return unsafe { + sq_distance_with_offset_avx2::(query, codes, mins, maxs, offset, d) + }; + } + } + #[cfg(target_arch = "aarch64")] + { + if d >= 8 { + return unsafe { + sq_distance_with_offset_neon::(query, codes, mins, maxs, offset, d) + }; + } + } + sq_distance_with_offset_scalar::(query, codes, mins, maxs, offset, d) +} + +#[inline] +fn sq_distance_with_offset_scalar( + query: &[f32], + codes: &[u8], + mins: &[f32], + maxs: &[f32], + offset: &[f32], + d: usize, +) -> f32 { + let mut sum = 0.0; + for dimension in 0..d { + let decoded = + offset[dimension] + decode_value(codes[dimension], mins[dimension], maxs[dimension]); + if L2 { + let difference = query[dimension] - decoded; + sum += difference * difference; + } else { + sum += query[dimension] * decoded; + } + } + sum +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn sq_distance_with_offset_avx2( + query: &[f32], + codes: &[u8], + mins: &[f32], + maxs: &[f32], + offset: &[f32], + d: usize, +) -> f32 { + use std::arch::x86_64::*; + + let inverse_255 = _mm256_set1_ps(1.0 / 255.0); + let mut accumulator = _mm256_setzero_ps(); + let mut dimension = 0; + while dimension + 8 <= d { + let bytes = _mm_loadl_epi64(codes.as_ptr().add(dimension).cast()); + let code_f32 = _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(bytes)); + let min = _mm256_loadu_ps(mins.as_ptr().add(dimension)); + let max = _mm256_loadu_ps(maxs.as_ptr().add(dimension)); + let base = _mm256_add_ps(min, _mm256_loadu_ps(offset.as_ptr().add(dimension))); + let scale = _mm256_mul_ps(_mm256_sub_ps(max, min), inverse_255); + let decoded = _mm256_add_ps(base, _mm256_mul_ps(code_f32, scale)); + let query_values = _mm256_loadu_ps(query.as_ptr().add(dimension)); + let contribution = if L2 { + let difference = _mm256_sub_ps(query_values, decoded); + _mm256_mul_ps(difference, difference) + } else { + _mm256_mul_ps(query_values, decoded) + }; + accumulator = _mm256_add_ps(accumulator, contribution); + dimension += 8; + } + let mut lanes = [0.0f32; 8]; + _mm256_storeu_ps(lanes.as_mut_ptr(), accumulator); + let mut sum = lanes.into_iter().sum::(); + sum += sq_distance_with_offset_scalar::( + &query[dimension..], + &codes[dimension..], + &mins[dimension..], + &maxs[dimension..], + &offset[dimension..], + d - dimension, + ); + sum +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +unsafe fn sq_distance_with_offset_neon( + query: &[f32], + codes: &[u8], + mins: &[f32], + maxs: &[f32], + offset: &[f32], + d: usize, +) -> f32 { + use std::arch::aarch64::*; + + let inverse_255 = vdupq_n_f32(1.0 / 255.0); + let mut accumulator = vdupq_n_f32(0.0); + let mut dimension = 0; + while dimension + 8 <= d { + let code_u16 = vmovl_u8(vld1_u8(codes.as_ptr().add(dimension))); + for half in 0..2 { + let code_u32 = if half == 0 { + vmovl_u16(vget_low_u16(code_u16)) + } else { + vmovl_u16(vget_high_u16(code_u16)) + }; + let base_dimension = dimension + half * 4; + let code_f32 = vcvtq_f32_u32(code_u32); + let min = vld1q_f32(mins.as_ptr().add(base_dimension)); + let max = vld1q_f32(maxs.as_ptr().add(base_dimension)); + let base = vaddq_f32(min, vld1q_f32(offset.as_ptr().add(base_dimension))); + let scale = vmulq_f32(vsubq_f32(max, min), inverse_255); + let decoded = vfmaq_f32(base, code_f32, scale); + let query_values = vld1q_f32(query.as_ptr().add(base_dimension)); + let contribution = if L2 { + let difference = vsubq_f32(query_values, decoded); + vmulq_f32(difference, difference) + } else { + vmulq_f32(query_values, decoded) + }; + accumulator = vaddq_f32(accumulator, contribution); + } + dimension += 8; + } + let mut sum = vaddvq_f32(accumulator); + sum += sq_distance_with_offset_scalar::( + &query[dimension..], + &codes[dimension..], + &mins[dimension..], + &maxs[dimension..], + &offset[dimension..], + d - dimension, + ); + sum +} + fn update_bounds_batch(data: &[f32], n: usize, d: usize, mins: &mut [f32], maxs: &mut [f32]) { update_bounds_batch_simd(data, n, d, mins, maxs); } @@ -989,4 +1400,41 @@ mod tests { assert!((decoded[2] - 10.0).abs() < 1e-6); assert!((decoded[3] - 21.0).abs() < 1e-6); } + + #[test] + fn scalar_quantizer_simd_distance_matches_scalar_tail_path() { + let d = 37; + let query = (0..d) + .map(|dimension| dimension as f32 * 0.13 - 1.0) + .collect::>(); + let codes = (0..d) + .map(|dimension| ((dimension * 29 + 7) % 256) as u8) + .collect::>(); + let mins = (0..d) + .map(|dimension| -3.0 + dimension as f32 * 0.01) + .collect::>(); + let maxs = (0..d) + .map(|dimension| 4.0 + dimension as f32 * 0.02) + .collect::>(); + let offset = (0..d) + .map(|dimension| dimension as f32 * -0.03) + .collect::>(); + + for l2 in [true, false] { + let expected = if l2 { + sq_distance_with_offset_scalar::(&query, &codes, &mins, &maxs, &offset, d) + } else { + sq_distance_with_offset_scalar::(&query, &codes, &mins, &maxs, &offset, d) + }; + let actual = if l2 { + sq_l2_with_offset_simd(&query, &codes, &mins, &maxs, &offset, d) + } else { + sq_inner_product_with_offset_simd(&query, &codes, &mins, &maxs, &offset, d) + }; + assert!( + (actual - expected).abs() <= expected.abs().max(1.0) * 2e-6, + "SIMD={actual}, scalar={expected}" + ); + } + } } diff --git a/core/src/topk.rs b/core/src/topk.rs index 6407d7f..f883ef7 100644 --- a/core/src/topk.rs +++ b/core/src/topk.rs @@ -58,15 +58,15 @@ impl TopKHeap { } } + pub(crate) fn should_consider(&self, lower_bound: f32) -> bool { + self.k > 0 && (self.data.len() < self.k || lower_bound < self.data[0].0) + } + pub(crate) fn into_sorted(mut self) -> Vec<(f32, i64)> { self.data.sort_by(|a, b| a.0.total_cmp(&b.0)); self.data } - pub(crate) fn len(&self) -> usize { - self.data.len() - } - fn sift_up(&mut self, mut index: usize) { while index > 0 { let parent = (index - 1) / 2; diff --git a/core/src/vamana.rs b/core/src/vamana.rs new file mode 100644 index 0000000..d711e89 --- /dev/null +++ b/core/src/vamana.rs @@ -0,0 +1,2870 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::diskann::DiskAnnBuildParams; +use crate::distance::{ + fvec_distance, fvec_l2sqr, fvec_l2sqr_four, fvec_l2sqr_scaled_exceeds, MetricType, +}; +use crate::kmeans::{self, KMeansConfig}; +use crate::pq::ProductQuantizer; +use crate::sparse_table::{estimated_memory_bytes as sparse_table_memory_bytes, SparseTable}; +use rand::rngs::StdRng; +use rand::seq::SliceRandom; +use rand::{Rng, SeedableRng}; +use rayon::prelude::*; +use std::cmp::{Ordering, Reverse}; +use std::collections::{BinaryHeap, VecDeque}; +use std::io; +use std::ops::Index; +use std::sync::{Mutex, RwLock}; +use std::time::{Duration, Instant}; + +const PARALLEL_ADJACENCY_NODES_PER_SHARD: usize = 256; +const PARALLEL_BUILD_BATCH_NODES: usize = 128; +const CONNECTIVITY_SOURCE_SAMPLE_SIZE: usize = 64; +const SPARSE_BUILD_VISITED_MIN_MEMORY_SAVINGS: usize = 16; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ScoredNode { + pub id: u32, + pub distance: f32, +} + +impl Eq for ScoredNode {} + +impl PartialOrd for ScoredNode { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for ScoredNode { + fn cmp(&self, other: &Self) -> Ordering { + scored_node_order(self, other) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VamanaGraph { + pub entry_node: u32, + pub(crate) adjacency: CompactAdjacency, +} + +pub(crate) struct VamanaMemoryEstimate { + pub(crate) build_peak_bytes: usize, + pub(crate) remap_peak_bytes: usize, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +struct ConnectivityRepairStats { + full_reachability_traversals: usize, + source_distance_evaluations: usize, + edges_added: usize, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(crate) struct VamanaBuildStats { + pub(crate) initialization: Duration, + pub(crate) pass_one: Duration, + pub(crate) pass_two: Duration, + pub(crate) connectivity_repair: Duration, +} + +pub(crate) fn estimate_vamana_memory_bytes( + node_count: usize, + max_degree: usize, + search_list_size: usize, + workers: usize, +) -> Option { + let edge_bytes = max_degree.checked_mul(size_of::())?; + let builder_edges = node_count.checked_mul(edge_bytes)?; + let builder_degrees = node_count.checked_mul(size_of::())?; + let builder_shards = node_count + .div_ceil(PARALLEL_ADJACENCY_NODES_PER_SHARD) + .checked_mul(size_of::>())?; + let builder_graph = builder_edges + .checked_add(builder_degrees)? + .checked_add(builder_shards)?; + let build_order = node_count.checked_mul(size_of::())?; + let expected_visited = search_list_size + .checked_mul(max_degree)? + .checked_add(1)? + .min(node_count); + let dense_worker_states = node_count + .checked_mul(size_of::())? + .checked_add(expected_visited.checked_mul(size_of::())?)?; + let sparse_worker_states = sparse_table_memory_bytes(expected_visited, size_of::())?; + let worker_states = if sparse_worker_states + .checked_mul(SPARSE_BUILD_VISITED_MIN_MEMORY_SAVINGS) + .is_some_and(|threshold| threshold < dense_worker_states) + { + sparse_worker_states + } else { + dense_worker_states + }; + let worker_candidates = search_list_size + .checked_mul(3)? + .checked_mul(size_of::())?; + let prune_candidates = search_list_size.checked_add(max_degree)?; + let worker_prune = + prune_candidates.checked_mul(size_of::().checked_add(size_of::())?)?; + let worker_candidate_ids = search_list_size.checked_mul(size_of::())?; + let worker_neighbors = max_degree.checked_mul(2 * size_of::())?; + let worker_scratch = workers.max(1).checked_mul( + worker_states + .checked_add(worker_candidates)? + .checked_add(worker_prune)? + .checked_add(worker_candidate_ids)? + .checked_add(worker_neighbors)?, + )?; + let reverse_edge_batch = PARALLEL_BUILD_BATCH_NODES + .checked_mul(max_degree)? + .checked_mul(size_of::<(u32, u32)>() + size_of::())?; + let build_peak_bytes = builder_graph + .checked_add(build_order)? + .checked_add(worker_scratch)? + .checked_add(reverse_edge_batch)?; + + let final_shards = node_count + .div_ceil(PARALLEL_ADJACENCY_NODES_PER_SHARD) + .checked_mul(size_of::())?; + let compact_graph = builder_edges + .checked_add(builder_degrees)? + .checked_add(final_shards)?; + let permutations = node_count.checked_mul(2 * size_of::())?; + let permutation_visited = node_count.checked_mul(size_of::())?; + let permutation_scratch = edge_bytes.checked_add(size_of::())?; + let remap_peak_bytes = compact_graph + .checked_add(permutations)? + .checked_add(permutation_visited)? + .checked_add(permutation_scratch)?; + Some(VamanaMemoryEstimate { + build_peak_bytes, + remap_peak_bytes, + }) +} + +pub(crate) fn estimate_sharded_vamana_memory_bytes( + node_count: usize, + dimension: usize, + max_degree: usize, + shard_count: usize, + pq_code_size: usize, +) -> Option { + if node_count == 0 || shard_count < 2 { + return None; + } + let edge_bytes = max_degree.checked_mul(size_of::())?; + let compact_graph = node_count.checked_mul(edge_bytes.checked_add(size_of::())?)?; + let assignments = node_count.checked_mul(2 * size_of::())?; + let local_count = overlapping_shard_capacity(node_count, shard_count)?; + let memberships = shard_count + .checked_mul(local_count)? + .checked_mul(size_of::())? + .checked_add(shard_count.checked_mul(size_of::>())?)?; + let centroids = shard_count + .checked_mul(dimension)? + .checked_mul(size_of::())?; + let kmeans_train_count = node_count.min(shard_count.checked_mul(256)?); + let kmeans_training_copy = kmeans_train_count + .checked_mul(dimension)? + .checked_mul(size_of::())?; + let kmeans_assignments = kmeans_train_count.checked_mul(size_of::())?; + let kmeans_initialization = kmeans_train_count.checked_mul(size_of::())?; + let kmeans_score_matrix = kmeans_train_count + .checked_mul(shard_count)? + .min(4 * 1024 * 1024) + .checked_mul(size_of::())?; + let kmeans_centroid_scratch = centroids.checked_mul(3)?; + let kmeans_peak = [ + kmeans_training_copy, + kmeans_assignments, + kmeans_initialization, + kmeans_score_matrix, + kmeans_centroid_scratch, + ] + .into_iter() + .try_fold(0usize, |total, value| total.checked_add(value))?; + let local_vectors = local_count + .checked_mul(dimension)? + .checked_mul(size_of::())?; + let local_ids = local_count.checked_mul(size_of::())?; + let local_pq_codes = local_count.checked_mul(pq_code_size)?; + // Sequential local construction briefly holds nested and compact + // adjacency plus its order/visited vectors. + let local_graph = local_count.checked_mul( + edge_bytes + .checked_mul(2)? + .checked_add(size_of::>())? + .checked_add(size_of::())? + .checked_add(2 * size_of::())?, + )?; + let build_peak = [ + compact_graph, + assignments, + memberships, + centroids, + kmeans_peak, + local_vectors, + local_ids, + local_pq_codes, + local_graph, + ] + .into_iter() + .try_fold(0usize, |total, value| total.checked_add(value))?; + let remap_peak = estimate_vamana_memory_bytes(node_count, max_degree, max_degree.max(1), 1)? + .remap_peak_bytes; + Some(build_peak.max(remap_peak)) +} + +impl VamanaGraph { + pub fn from_adjacency(entry_node: u32, adjacency: Vec>) -> Self { + let max_degree = adjacency.iter().map(Vec::len).max().unwrap_or(0); + Self { + entry_node, + adjacency: CompactAdjacency::from_nested(adjacency, max_degree), + } + } + + pub fn build( + vectors: &[f32], + count: usize, + dimension: usize, + params: DiskAnnBuildParams, + ) -> io::Result { + Self::build_with_stats(vectors, count, dimension, MetricType::L2, params) + .map(|(graph, _)| graph) + } + + pub(crate) fn build_with_stats( + vectors: &[f32], + count: usize, + dimension: usize, + metric: MetricType, + params: DiskAnnBuildParams, + ) -> io::Result<(Self, VamanaBuildStats)> { + Self::build_with_search_distance( + vectors, + count, + dimension, + metric, + params, + BuildSearchDistance::FullPrecision { + vectors, + dimension, + metric, + }, + ) + } + + pub(crate) fn build_with_pq_stats( + vectors: &[f32], + pq: &ProductQuantizer, + pq_codes: &[u8], + count: usize, + dimension: usize, + metric: MetricType, + params: DiskAnnBuildParams, + ) -> io::Result<(Self, VamanaBuildStats)> { + validate_build_inputs(vectors, count, dimension, params)?; + if pq.d != dimension + || !matches!(pq.nbits, 4 | 8) + || !pq.has_valid_layout() + || pq_codes.len() != count.saturating_mul(pq.code_size()) + { + return Err(invalid_input( + "Vamana PQ-guided build received an invalid codebook or code buffer", + )); + } + let distance_started = Instant::now(); + let distance = PqBuildDistance::new(pq, pq_codes, count, metric)?; + let distance_initialization = distance_started.elapsed(); + let (graph, mut stats) = Self::build_with_search_distance( + vectors, + count, + dimension, + metric, + params, + BuildSearchDistance::ProductQuantized(distance), + )?; + stats.initialization = stats.initialization.saturating_add(distance_initialization); + Ok((graph, stats)) + } + + pub(crate) fn build_sharded_with_stats( + vectors: &[f32], + count: usize, + dimension: usize, + metric: MetricType, + params: DiskAnnBuildParams, + shard_count: usize, + ) -> io::Result<(Self, VamanaBuildStats)> { + Self::build_sharded_with_optional_pq( + vectors, + None, + count, + dimension, + metric, + params, + shard_count, + ) + } + + pub(crate) fn build_sharded_with_pq_stats( + vectors: &[f32], + pq: &ProductQuantizer, + pq_codes: &[u8], + count: usize, + dimension: usize, + metric: MetricType, + params: DiskAnnBuildParams, + shard_count: usize, + ) -> io::Result<(Self, VamanaBuildStats)> { + Self::build_sharded_with_optional_pq( + vectors, + Some((pq, pq_codes)), + count, + dimension, + metric, + params, + shard_count, + ) + } + + #[allow(clippy::too_many_arguments)] + fn build_sharded_with_optional_pq( + vectors: &[f32], + pq_build: Option<(&ProductQuantizer, &[u8])>, + count: usize, + dimension: usize, + metric: MetricType, + params: DiskAnnBuildParams, + shard_count: usize, + ) -> io::Result<(Self, VamanaBuildStats)> { + validate_build_inputs(vectors, count, dimension, params)?; + if shard_count < 2 || shard_count > count { + return Err(invalid_input( + "Vamana shard count must be between 2 and the vector count", + )); + } + if let Some((pq, pq_codes)) = pq_build { + if pq.d != dimension + || !matches!(pq.nbits, 4 | 8) + || !pq.has_valid_layout() + || pq_codes.len() != count.saturating_mul(pq.code_size()) + { + return Err(invalid_input( + "sharded Vamana PQ-guided build received an invalid codebook or code buffer", + )); + } + } + let initialization_started = Instant::now(); + let cluster_config = KMeansConfig { + niter: 8, + nredo: 1, + max_points_per_centroid: 256, + seed: params.seed, + balance_factor: 0.1, + }; + let centroids = + kmeans::kmeans_train(&cluster_config, vectors, count, dimension, shard_count); + let mut assignments = (0..count) + .into_par_iter() + .map(|node| { + nearest_two_centroids( + &vectors[node * dimension..(node + 1) * dimension], + ¢roids, + shard_count, + dimension, + ) + }) + .collect::>(); + let membership_capacity = overlapping_shard_capacity(count, shard_count) + .ok_or_else(|| invalid_input("Vamana overlapping-shard capacity overflows"))?; + let mut memberships = (0..shard_count) + .map(|_| Vec::with_capacity(membership_capacity)) + .collect::>(); + for (node, [first, second]) in assignments.iter().copied().enumerate() { + memberships[first].push(node as u32); + if second != first { + memberships[second].push(node as u32); + } + } + rebalance_overlapping_shards( + vectors, + dimension, + ¢roids, + &mut assignments, + &mut memberships, + )?; + let initialization = initialization_started.elapsed(); + + let entry_node = centroid_entry(vectors, count, dimension, metric) as u32; + let degree = params.max_degree.min(count.saturating_sub(1)); + let mut graph = Self { + entry_node, + adjacency: CompactAdjacency::empty(count, degree), + }; + let mut mapped = Vec::with_capacity(degree); + let mut pass_one = Duration::ZERO; + let mut pass_two = Duration::ZERO; + for (shard, members) in memberships.iter().enumerate() { + if members.len() < 2 { + continue; + } + let local_started = Instant::now(); + let mut local_vectors = Vec::new(); + local_vectors + .try_reserve_exact(members.len().saturating_mul(dimension)) + .map_err(|_| invalid_input("Vamana shard vector allocation failed"))?; + for &node in members { + let node = node as usize; + local_vectors.extend_from_slice(&vectors[node * dimension..(node + 1) * dimension]); + } + let local_pq_codes = pq_build.map(|(pq, pq_codes)| { + let code_size = pq.code_size(); + let mut local_codes = Vec::with_capacity(members.len() * code_size); + for &node in members { + let node = node as usize; + local_codes + .extend_from_slice(&pq_codes[node * code_size..(node + 1) * code_size]); + } + local_codes + }); + let local_degree = params.max_degree.min(members.len() - 1); + let local_params = DiskAnnBuildParams { + max_degree: local_degree, + build_search_list_size: params + .build_search_list_size + .min(members.len()) + .max(local_degree), + seed: derived_seed(params.seed, shard as u64), + ..params + }; + let local_graph = if let Some((pq, _)) = pq_build { + Self::build_with_pq_stats( + &local_vectors, + pq, + local_pq_codes + .as_deref() + .expect("PQ-guided shard has local codes"), + members.len(), + dimension, + metric, + local_params, + )? + .0 + } else { + Self::build_sequential_with_metric( + &local_vectors, + members.len(), + dimension, + metric, + local_params, + )? + }; + pass_one = pass_one.saturating_add(local_started.elapsed()); + let merge_started = Instant::now(); + for (local_node, &global_node) in members.iter().enumerate() { + mapped.clear(); + mapped.extend( + local_graph.adjacency[local_node] + .iter() + .map(|&neighbor| members[neighbor as usize]), + ); + let selected = robust_prune_candidates( + vectors, + dimension, + global_node as usize, + &mapped, + &graph.adjacency[global_node as usize], + count, + degree, + params.alpha, + metric, + ); + graph.adjacency.replace(global_node as usize, &selected); + } + pass_two = pass_two.saturating_add(merge_started.elapsed()); + } + let connectivity_started = Instant::now(); + graph.repair_connectivity(vectors, dimension, degree, metric)?; + let connectivity_repair = connectivity_started.elapsed(); + graph.validate(degree)?; + if !graph.is_fully_reachable() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "sharded Vamana graph is not fully reachable from its entry node", + )); + } + Ok(( + graph, + VamanaBuildStats { + initialization, + pass_one, + pass_two, + connectivity_repair, + }, + )) + } + + fn build_with_search_distance( + vectors: &[f32], + count: usize, + dimension: usize, + metric: MetricType, + params: DiskAnnBuildParams, + search_distance: BuildSearchDistance<'_>, + ) -> io::Result<(Self, VamanaBuildStats)> { + validate_build_inputs(vectors, count, dimension, params)?; + let entry_node = centroid_entry(vectors, count, dimension, metric) as u32; + let degree = params.max_degree.min(count.saturating_sub(1)); + let initialization_started = Instant::now(); + let adjacency = ParallelAdjacency::new_random(count, degree, params.seed); + let initialization = initialization_started.elapsed(); + let mut rng = StdRng::seed_from_u64(derived_seed(params.seed, u64::MAX)); + let builder = ParallelVamanaBuilder { + vectors, + dimension, + metric, + entry_node, + adjacency, + search_distance, + }; + let pass_one_started = Instant::now(); + builder.run_pass(params, 1.0, &mut rng); + let pass_one = pass_one_started.elapsed(); + let pass_two_started = Instant::now(); + builder.run_pass(params, params.alpha, &mut rng); + let mut graph = builder.finish()?; + let pass_two = pass_two_started.elapsed(); + let connectivity_started = Instant::now(); + graph.repair_connectivity(vectors, dimension, params.max_degree, metric)?; + let connectivity_repair = connectivity_started.elapsed(); + graph.validate(params.max_degree)?; + if !graph.is_fully_reachable() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "parallel Vamana graph is not fully reachable from its entry node", + )); + } + Ok(( + graph, + VamanaBuildStats { + initialization, + pass_one, + pass_two, + connectivity_repair, + }, + )) + } + + pub fn build_sequential( + vectors: &[f32], + count: usize, + dimension: usize, + params: DiskAnnBuildParams, + ) -> io::Result { + Self::build_sequential_with_metric(vectors, count, dimension, MetricType::L2, params) + } + + fn build_sequential_with_metric( + vectors: &[f32], + count: usize, + dimension: usize, + metric: MetricType, + params: DiskAnnBuildParams, + ) -> io::Result { + validate_build_inputs(vectors, count, dimension, params)?; + let entry_node = centroid_entry(vectors, count, dimension, metric) as u32; + let mut rng = StdRng::seed_from_u64(params.seed); + let degree = params.max_degree.min(count.saturating_sub(1)); + let adjacency = (0..count) + .map(|node| random_neighbors(&mut rng, count, node, degree)) + .collect::>(); + let mut graph = Self { + entry_node, + adjacency: CompactAdjacency::from_nested(adjacency, degree), + }; + + graph.run_sequential_pass(vectors, dimension, metric, params, 1.0, &mut rng); + graph.run_sequential_pass(vectors, dimension, metric, params, params.alpha, &mut rng); + graph.repair_connectivity(vectors, dimension, params.max_degree, metric)?; + graph.validate(params.max_degree)?; + if !graph.is_fully_reachable() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Vamana graph is not fully reachable from its entry node", + )); + } + Ok(graph) + } + + fn run_sequential_pass( + &mut self, + vectors: &[f32], + dimension: usize, + metric: MetricType, + params: DiskAnnBuildParams, + alpha: f32, + rng: &mut StdRng, + ) { + let mut order = (0..self.adjacency.len()).collect::>(); + order.shuffle(rng); + for node in order { + let query_start = node * dimension; + let candidates = self.greedy_search_with_metric( + vectors, + dimension, + metric, + &vectors[query_start..query_start + dimension], + params.build_search_list_size, + ); + let candidate_ids = candidates + .into_iter() + .map(|candidate| candidate.id) + .collect::>(); + let selected = self.robust_prune_with_metric( + vectors, + dimension, + metric, + node, + &candidate_ids, + params.max_degree, + alpha, + ); + self.adjacency.replace(node, &selected); + for neighbor in selected { + self.insert_reverse_edge( + vectors, + dimension, + metric, + neighbor as usize, + node as u32, + params.max_degree, + alpha, + ); + } + } + } + + fn insert_reverse_edge( + &mut self, + vectors: &[f32], + dimension: usize, + metric: MetricType, + node: usize, + neighbor: u32, + max_degree: usize, + alpha: f32, + ) { + if self.adjacency[node].contains(&neighbor) { + return; + } + if self.adjacency[node].len() < max_degree { + self.adjacency.push(node, neighbor); + return; + } + let mut candidates = self.adjacency[node].to_vec(); + candidates.push(neighbor); + let selected = self.robust_prune_with_metric( + vectors, + dimension, + metric, + node, + &candidates, + max_degree, + alpha, + ); + self.adjacency.replace(node, &selected); + } + + pub fn greedy_search( + &self, + vectors: &[f32], + dimension: usize, + query: &[f32], + search_list_size: usize, + ) -> Vec { + self.greedy_search_with_metric(vectors, dimension, MetricType::L2, query, search_list_size) + } + + fn greedy_search_with_metric( + &self, + vectors: &[f32], + dimension: usize, + metric: MetricType, + query: &[f32], + search_list_size: usize, + ) -> Vec { + if search_list_size == 0 || self.adjacency.is_empty() { + return Vec::new(); + } + + let mut visited = vec![false; self.adjacency.len()]; + let mut expanded = vec![false; self.adjacency.len()]; + let entry = self.entry_node as usize; + visited[entry] = true; + let mut results = vec![ScoredNode { + id: self.entry_node, + distance: node_distance(vectors, dimension, entry, query, metric), + }]; + + loop { + results.sort_by(scored_node_order); + results.truncate(search_list_size); + let Some(current) = results + .iter() + .find(|node| !expanded[node.id as usize]) + .copied() + else { + break; + }; + expanded[current.id as usize] = true; + + for &neighbor in &self.adjacency[current.id as usize] { + let neighbor = neighbor as usize; + if neighbor >= self.adjacency.len() || visited[neighbor] { + continue; + } + visited[neighbor] = true; + results.push(ScoredNode { + id: neighbor as u32, + distance: node_distance(vectors, dimension, neighbor, query, metric), + }); + } + } + + results.sort_by(scored_node_order); + results.truncate(search_list_size); + results + } + + #[allow(clippy::too_many_arguments)] + pub fn robust_prune( + &self, + vectors: &[f32], + dimension: usize, + node: usize, + candidates: &[u32], + max_degree: usize, + alpha: f32, + ) -> Vec { + self.robust_prune_with_metric( + vectors, + dimension, + MetricType::L2, + node, + candidates, + max_degree, + alpha, + ) + } + + #[allow(clippy::too_many_arguments)] + fn robust_prune_with_metric( + &self, + vectors: &[f32], + dimension: usize, + metric: MetricType, + node: usize, + candidates: &[u32], + max_degree: usize, + alpha: f32, + ) -> Vec { + robust_prune_candidates( + vectors, + dimension, + node, + candidates, + &self.adjacency[node], + self.adjacency.len(), + max_degree, + alpha, + metric, + ) + } + + pub fn is_fully_reachable(&self) -> bool { + if self.adjacency.is_empty() || self.entry_node as usize >= self.adjacency.len() { + return false; + } + let mut visited = vec![false; self.adjacency.len()]; + let mut queue = VecDeque::from([self.entry_node as usize]); + visited[self.entry_node as usize] = true; + while let Some(node) = queue.pop_front() { + for &neighbor in &self.adjacency[node] { + let neighbor = neighbor as usize; + if neighbor < visited.len() && !visited[neighbor] { + visited[neighbor] = true; + queue.push_back(neighbor); + } + } + } + visited.into_iter().all(|value| value) + } + + fn repair_connectivity( + &mut self, + vectors: &[f32], + dimension: usize, + max_degree: usize, + metric: MetricType, + ) -> io::Result<()> { + self.repair_connectivity_with_stats(vectors, dimension, max_degree, metric) + .map(|_| ()) + } + + fn repair_connectivity_with_stats( + &mut self, + vectors: &[f32], + dimension: usize, + max_degree: usize, + metric: MetricType, + ) -> io::Result { + let (mut visited, mut parent) = self.reachability_tree(); + let mut stats = ConnectivityRepairStats { + full_reachability_traversals: 1, + ..ConnectivityRepairStats::default() + }; + let mut eligible_sources = visited + .iter() + .enumerate() + .filter_map(|(node, &reachable)| { + (reachable && self.is_repair_source(node, max_degree, &parent)).then_some(node) + }) + .collect::>(); + let mut queue = VecDeque::new(); + let mut newly_reachable = Vec::new(); + let mut target_cursor = 0usize; + + loop { + while target_cursor < visited.len() && visited[target_cursor] { + target_cursor += 1; + } + if target_cursor == visited.len() { + return Ok(stats); + } + let target = target_cursor; + let target_vector = &vectors[target * dimension..target * dimension + dimension]; + let (source_index, source) = self.select_repair_source( + vectors, + dimension, + target, + target_vector, + max_degree, + &parent, + &eligible_sources, + &mut stats, + metric, + )?; + eligible_sources.swap_remove(source_index); + + if self.adjacency[source].len() == max_degree { + let removable = self.adjacency[source] + .iter() + .enumerate() + .filter(|(_, neighbor)| parent[**neighbor as usize] != Some(source)) + .map(|(slot, &neighbor)| { + ( + slot, + node_distance( + vectors, + dimension, + neighbor as usize, + &vectors[source * dimension..source * dimension + dimension], + metric, + ), + ) + }) + .max_by(|left, right| { + left.1 + .total_cmp(&right.1) + .then_with(|| left.0.cmp(&right.0)) + }) + .map(|(slot, _)| slot) + .expect("eligible full source has a non-tree edge"); + self.adjacency.swap_remove(source, removable); + } + self.adjacency.push(source, target as u32); + stats.edges_added += 1; + + visited[target] = true; + parent[target] = Some(source); + queue.push_back(target); + newly_reachable.clear(); + newly_reachable.push(target); + while let Some(node) = queue.pop_front() { + for &neighbor in &self.adjacency[node] { + let neighbor = neighbor as usize; + if !visited[neighbor] { + visited[neighbor] = true; + parent[neighbor] = Some(node); + queue.push_back(neighbor); + newly_reachable.push(neighbor); + } + } + } + if self.is_repair_source(source, max_degree, &parent) { + eligible_sources.push(source); + } + for &node in &newly_reachable { + if self.is_repair_source(node, max_degree, &parent) { + eligible_sources.push(node); + } + } + } + } + + #[allow(clippy::too_many_arguments)] + fn select_repair_source( + &self, + vectors: &[f32], + dimension: usize, + target: usize, + target_vector: &[f32], + max_degree: usize, + parent: &[Option], + eligible_sources: &[usize], + stats: &mut ConnectivityRepairStats, + metric: MetricType, + ) -> io::Result<(usize, usize)> { + if eligible_sources.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Vamana connectivity repair found no replaceable edge", + )); + } + let sample_size = CONNECTIVITY_SOURCE_SAMPLE_SIZE.min(eligible_sources.len()); + let start = + derived_seed(target as u64, stats.edges_added as u64) as usize % eligible_sources.len(); + let mut best: Option<(usize, usize, f32)> = None; + for offset in 0..sample_size { + let index = (start + offset) % eligible_sources.len(); + let source = eligible_sources[index]; + if !self.is_repair_source(source, max_degree, parent) { + continue; + } + stats.source_distance_evaluations += 1; + let distance = node_distance(vectors, dimension, source, target_vector, metric); + let candidate = (index, source, distance); + if best.is_none_or(|(_, best_source, best_distance)| { + distance + .total_cmp(&best_distance) + .then_with(|| source.cmp(&best_source)) + .is_lt() + }) { + best = Some(candidate); + } + } + if let Some((index, source, _)) = best { + return Ok((index, source)); + } + eligible_sources + .iter() + .enumerate() + .find_map(|(index, &source)| { + self.is_repair_source(source, max_degree, parent) + .then_some((index, source)) + }) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "Vamana connectivity repair found no replaceable edge", + ) + }) + } + + fn is_repair_source(&self, source: usize, max_degree: usize, parent: &[Option]) -> bool { + self.adjacency[source].len() < max_degree + || self.adjacency[source] + .iter() + .any(|neighbor| parent[*neighbor as usize] != Some(source)) + } + + fn reachability_tree(&self) -> (Vec, Vec>) { + let mut visited = vec![false; self.adjacency.len()]; + let mut parent = vec![None; self.adjacency.len()]; + let entry = self.entry_node as usize; + let mut queue = VecDeque::from([entry]); + visited[entry] = true; + while let Some(node) = queue.pop_front() { + for &neighbor in &self.adjacency[node] { + let neighbor = neighbor as usize; + if !visited[neighbor] { + visited[neighbor] = true; + parent[neighbor] = Some(node); + queue.push_back(neighbor); + } + } + } + (visited, parent) + } + + fn validate(&self, max_degree: usize) -> io::Result<()> { + for (node, neighbors) in self.adjacency.iter().enumerate() { + if neighbors.len() > max_degree { + return Err(invalid_input(format!( + "Vamana node {} degree {} exceeds maximum {}", + node, + neighbors.len(), + max_degree + ))); + } + let mut sorted = neighbors.to_vec(); + sorted.sort_unstable(); + if sorted.windows(2).any(|pair| pair[0] == pair[1]) { + return Err(invalid_input(format!( + "Vamana node {} contains duplicate neighbors", + node + ))); + } + if sorted.iter().any(|&neighbor| { + neighbor as usize >= self.adjacency.len() || neighbor as usize == node + }) { + return Err(invalid_input(format!( + "Vamana node {} contains invalid neighbor", + node + ))); + } + } + Ok(()) + } +} + +fn overlapping_shard_capacity(node_count: usize, shard_count: usize) -> Option { + if shard_count < 2 { + return None; + } + if shard_count == 2 { + return Some(node_count); + } + // For k > 2, ceil(2N / (k - 1)) leaves enough aggregate capacity even + // when one shard cannot accept a node's second, distinct membership. This + // lets deterministic overflow repair preserve exactly two memberships + // without an unbounded degenerate cluster. + node_count + .checked_mul(2)? + .checked_add(shard_count - 2) + .map(|value| value / (shard_count - 1)) +} + +fn rebalance_overlapping_shards( + vectors: &[f32], + dimension: usize, + centroids: &[f32], + assignments: &mut [[usize; 2]], + memberships: &mut [Vec], +) -> io::Result<()> { + let shard_count = memberships.len(); + let capacity = overlapping_shard_capacity(assignments.len(), shard_count) + .ok_or_else(|| invalid_input("Vamana overlapping-shard capacity overflows"))?; + for shard in 0..shard_count { + while memberships[shard].len() > capacity { + let node = memberships[shard] + .pop() + .expect("an overflowing Vamana shard is non-empty") as usize; + let slot = if assignments[node][0] == shard { + 0 + } else { + debug_assert_eq!(assignments[node][1], shard); + 1 + }; + let other = assignments[node][1 - slot]; + let vector = &vectors[node * dimension..(node + 1) * dimension]; + let mut replacement = None; + let mut replacement_distance = f32::INFINITY; + for candidate in 0..shard_count { + if candidate == other || memberships[candidate].len() >= capacity { + continue; + } + let start = candidate * dimension; + let distance = fvec_l2sqr(vector, ¢roids[start..start + dimension]); + if distance < replacement_distance + || (distance == replacement_distance + && replacement.is_none_or(|current| candidate < current)) + { + replacement = Some(candidate); + replacement_distance = distance; + } + } + let replacement = replacement.ok_or_else(|| { + invalid_input("Vamana overlapping shards cannot be capacity-balanced") + })?; + assignments[node][slot] = replacement; + memberships[replacement].push(node as u32); + } + } + debug_assert!(memberships.iter().all(|members| members.len() <= capacity)); + Ok(()) +} + +struct ParallelVamanaBuilder<'a> { + vectors: &'a [f32], + dimension: usize, + metric: MetricType, + entry_node: u32, + adjacency: ParallelAdjacency, + search_distance: BuildSearchDistance<'a>, +} + +enum BuildSearchDistance<'a> { + FullPrecision { + vectors: &'a [f32], + dimension: usize, + metric: MetricType, + }, + ProductQuantized(PqBuildDistance<'a>), +} + +impl BuildSearchDistance<'_> { + #[inline] + fn between(&self, left: usize, right: usize) -> f32 { + match self { + Self::FullPrecision { + vectors, + dimension, + metric, + } => distance_between(vectors, *dimension, left, right, *metric), + Self::ProductQuantized(distance) => distance.between(left, right), + } + } +} + +struct PqBuildDistance<'a> { + codes: &'a [u8], + code_size: usize, + m: usize, + ksub: usize, + nbits: usize, + centroid_distances: Vec, +} + +impl<'a> PqBuildDistance<'a> { + fn new( + pq: &ProductQuantizer, + codes: &'a [u8], + count: usize, + metric: MetricType, + ) -> io::Result { + let table_len = pq + .m + .checked_mul(pq.ksub) + .and_then(|value| value.checked_mul(pq.ksub)) + .ok_or_else(|| invalid_input("Vamana PQ build-distance table size overflows usize"))?; + let mut centroid_distances = Vec::new(); + centroid_distances + .try_reserve_exact(table_len) + .map_err(|_| invalid_input("Vamana PQ build-distance table allocation failed"))?; + centroid_distances.resize(table_len, 0.0); + for sub in 0..pq.m { + let chunk_dim = pq.chunk_dim(sub); + let sub_base = pq.centroid_chunk_base(sub); + let table_base = sub * pq.ksub * pq.ksub; + for left in 0..pq.ksub { + let left_start = sub_base + left * chunk_dim; + for right in left..pq.ksub { + let right_start = sub_base + right * chunk_dim; + let distance = fvec_distance( + &pq.centroids[left_start..left_start + chunk_dim], + &pq.centroids[right_start..right_start + chunk_dim], + metric, + ); + centroid_distances[table_base + left * pq.ksub + right] = distance; + centroid_distances[table_base + right * pq.ksub + left] = distance; + } + } + } + let code_size = pq.code_size(); + if codes.len() != count.saturating_mul(code_size) { + return Err(invalid_input( + "Vamana PQ code buffer does not match vector count", + )); + } + Ok(Self { + codes, + code_size, + m: pq.m, + ksub: pq.ksub, + nbits: pq.nbits, + centroid_distances, + }) + } + + #[inline] + fn code(&self, node: usize, sub: usize) -> usize { + let codes = &self.codes[node * self.code_size..(node + 1) * self.code_size]; + if self.nbits == 4 { + let byte = codes[sub / 2]; + usize::from(if sub.is_multiple_of(2) { + byte & 0x0f + } else { + byte >> 4 + }) + } else { + usize::from(codes[sub]) + } + } + + #[inline] + fn between(&self, left: usize, right: usize) -> f32 { + let mut distance = 0.0; + for sub in 0..self.m { + let left_code = self.code(left, sub); + let right_code = self.code(right, sub); + distance += self.centroid_distances + [sub * self.ksub * self.ksub + left_code * self.ksub + right_code]; + } + distance + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct AdjacencyShard { + start_node: usize, + node_count: usize, + slots: Vec, + degrees: Vec, +} + +impl AdjacencyShard { + fn neighbors(&self, node: usize, max_degree: usize) -> &[u32] { + let local_node = node - self.start_node; + let start = local_node * max_degree; + &self.slots[start..start + self.degrees[local_node] as usize] + } + + fn neighbors_mut(&mut self, node: usize, max_degree: usize) -> &mut [u32] { + let local_node = node - self.start_node; + let start = local_node * max_degree; + let degree = self.degrees[local_node] as usize; + &mut self.slots[start..start + degree] + } + + fn replace(&mut self, node: usize, max_degree: usize, neighbors: &[u32]) { + debug_assert!(neighbors.len() <= max_degree); + let local_node = node - self.start_node; + let start = local_node * max_degree; + self.slots[start..start + neighbors.len()].copy_from_slice(neighbors); + self.degrees[local_node] = neighbors.len() as u16; + } + + fn push(&mut self, node: usize, max_degree: usize, neighbor: u32) { + let local_node = node - self.start_node; + let degree = self.degrees[local_node] as usize; + assert!(degree < max_degree, "compact adjacency node is full"); + self.slots[local_node * max_degree + degree] = neighbor; + self.degrees[local_node] += 1; + } + + fn swap_remove(&mut self, node: usize, max_degree: usize, slot: usize) { + let local_node = node - self.start_node; + let degree = self.degrees[local_node] as usize; + assert!(slot < degree, "compact adjacency removal slot is invalid"); + let start = local_node * max_degree; + self.slots[start + slot] = self.slots[start + degree - 1]; + self.degrees[local_node] -= 1; + } + + fn swap_node_with_buffer( + &mut self, + node: usize, + max_degree: usize, + slots: &mut [u32], + degree: &mut u16, + ) { + let local_node = node - self.start_node; + let start = local_node * max_degree; + for (stored, buffered) in self.slots[start..start + max_degree].iter_mut().zip(slots) { + std::mem::swap(stored, buffered); + } + std::mem::swap(&mut self.degrees[local_node], degree); + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CompactAdjacency { + shards: Vec, + node_count: usize, + max_degree: usize, +} + +impl CompactAdjacency { + fn empty(node_count: usize, max_degree: usize) -> Self { + let mut shards = + Vec::with_capacity(node_count.div_ceil(PARALLEL_ADJACENCY_NODES_PER_SHARD)); + for start_node in (0..node_count).step_by(PARALLEL_ADJACENCY_NODES_PER_SHARD) { + let shard_node_count = PARALLEL_ADJACENCY_NODES_PER_SHARD.min(node_count - start_node); + shards.push(AdjacencyShard { + start_node, + node_count: shard_node_count, + slots: vec![0; shard_node_count * max_degree], + degrees: vec![0; shard_node_count], + }); + } + Self { + shards, + node_count, + max_degree, + } + } + + fn from_nested(adjacency: Vec>, max_degree: usize) -> Self { + let node_count = adjacency.len(); + let mut shards = + Vec::with_capacity(node_count.div_ceil(PARALLEL_ADJACENCY_NODES_PER_SHARD)); + for start_node in (0..node_count).step_by(PARALLEL_ADJACENCY_NODES_PER_SHARD) { + let shard_node_count = PARALLEL_ADJACENCY_NODES_PER_SHARD.min(node_count - start_node); + let mut slots = vec![0; shard_node_count * max_degree]; + let mut degrees = vec![0; shard_node_count]; + for local_node in 0..shard_node_count { + let neighbors = &adjacency[start_node + local_node]; + assert!(neighbors.len() <= max_degree); + let slot_start = local_node * max_degree; + slots[slot_start..slot_start + neighbors.len()].copy_from_slice(neighbors); + degrees[local_node] = u16::try_from(neighbors.len()) + .expect("Vamana degree exceeds compact adjacency metadata"); + } + shards.push(AdjacencyShard { + start_node, + node_count: shard_node_count, + slots, + degrees, + }); + } + Self { + shards, + node_count, + max_degree, + } + } + + pub(crate) fn len(&self) -> usize { + self.node_count + } + + fn is_empty(&self) -> bool { + self.node_count == 0 + } + + pub(crate) fn neighbors(&self, node: usize) -> &[u32] { + self.shards[ParallelAdjacency::shard_index(node)].neighbors(node, self.max_degree) + } + + pub(crate) fn iter(&self) -> impl ExactSizeIterator { + (0..self.node_count).map(|node| self.neighbors(node)) + } + + fn replace(&mut self, node: usize, neighbors: &[u32]) { + self.shards[ParallelAdjacency::shard_index(node)].replace(node, self.max_degree, neighbors); + } + + fn push(&mut self, node: usize, neighbor: u32) { + self.shards[ParallelAdjacency::shard_index(node)].push(node, self.max_degree, neighbor); + } + + fn swap_remove(&mut self, node: usize, slot: usize) { + self.shards[ParallelAdjacency::shard_index(node)].swap_remove(node, self.max_degree, slot); + } + + pub(crate) fn permute_and_map_neighbors(&mut self, old_to_new: &[u32]) { + assert_eq!(old_to_new.len(), self.node_count); + let mut visited = vec![false; self.node_count]; + let mut slots = vec![0; self.max_degree]; + for start in 0..self.node_count { + if visited[start] { + continue; + } + slots.copy_from_slice(self.node_slots(start)); + let mut degree = self.node_degree(start); + let mut current = start; + loop { + visited[current] = true; + let destination = old_to_new[current] as usize; + assert!(destination < self.node_count); + self.swap_node_with_buffer(destination, &mut slots, &mut degree); + current = destination; + if current == start { + break; + } + assert!( + !visited[current], + "DiskANN locality mapping is not a permutation" + ); + } + } + for node in 0..self.node_count { + let neighbors = self.neighbors_mut(node); + for neighbor in neighbors.iter_mut() { + *neighbor = old_to_new[*neighbor as usize]; + } + neighbors.sort_unstable(); + } + } + + fn node_slots(&self, node: usize) -> &[u32] { + let shard = &self.shards[ParallelAdjacency::shard_index(node)]; + let local_node = node - shard.start_node; + let start = local_node * self.max_degree; + &shard.slots[start..start + self.max_degree] + } + + fn node_degree(&self, node: usize) -> u16 { + let shard = &self.shards[ParallelAdjacency::shard_index(node)]; + shard.degrees[node - shard.start_node] + } + + fn neighbors_mut(&mut self, node: usize) -> &mut [u32] { + self.shards[ParallelAdjacency::shard_index(node)].neighbors_mut(node, self.max_degree) + } + + fn swap_node_with_buffer(&mut self, node: usize, slots: &mut [u32], degree: &mut u16) { + self.shards[ParallelAdjacency::shard_index(node)].swap_node_with_buffer( + node, + self.max_degree, + slots, + degree, + ); + } +} + +impl Index for CompactAdjacency { + type Output = [u32]; + + fn index(&self, index: usize) -> &Self::Output { + self.neighbors(index) + } +} + +struct ParallelAdjacency { + shards: Vec>, + node_count: usize, + max_degree: usize, +} + +impl ParallelAdjacency { + fn new_random(node_count: usize, max_degree: usize, seed: u64) -> Self { + assert!(max_degree <= node_count.saturating_sub(1)); + let shard_count = node_count.div_ceil(PARALLEL_ADJACENCY_NODES_PER_SHARD); + let shards = (0..shard_count) + .into_par_iter() + .map(|shard_index| { + let start_node = shard_index * PARALLEL_ADJACENCY_NODES_PER_SHARD; + let shard_node_count = + PARALLEL_ADJACENCY_NODES_PER_SHARD.min(node_count - start_node); + let mut slots = vec![0; shard_node_count * max_degree]; + let degrees = vec![max_degree as u16; shard_node_count]; + let mut rng = StdRng::seed_from_u64(derived_seed(seed, shard_index as u64)); + let mut swaps = SparseTable::with_capacity(max_degree); + for local_node in 0..shard_node_count { + let node = start_node + local_node; + let slot_start = local_node * max_degree; + sample_random_neighbors_into( + &mut rng, + node_count, + node, + &mut swaps, + &mut slots[slot_start..slot_start + max_degree], + ); + } + RwLock::new(AdjacencyShard { + start_node, + node_count: shard_node_count, + slots, + degrees, + }) + }) + .collect(); + Self { + shards, + node_count, + max_degree, + } + } + + #[cfg(test)] + fn new( + node_count: usize, + max_degree: usize, + mut neighbors_for: impl FnMut(usize) -> Vec, + ) -> Self { + let mut shards = + Vec::with_capacity(node_count.div_ceil(PARALLEL_ADJACENCY_NODES_PER_SHARD)); + for start_node in (0..node_count).step_by(PARALLEL_ADJACENCY_NODES_PER_SHARD) { + let shard_node_count = PARALLEL_ADJACENCY_NODES_PER_SHARD.min(node_count - start_node); + let mut slots = vec![0; shard_node_count * max_degree]; + let mut degrees = vec![0; shard_node_count]; + for local_node in 0..shard_node_count { + let neighbors = neighbors_for(start_node + local_node); + assert!( + neighbors.len() <= max_degree, + "parallel Vamana initializer exceeds maximum degree" + ); + let slot_start = local_node * max_degree; + slots[slot_start..slot_start + neighbors.len()].copy_from_slice(&neighbors); + degrees[local_node] = neighbors.len() as u16; + } + shards.push(RwLock::new(AdjacencyShard { + start_node, + node_count: shard_node_count, + slots, + degrees, + })); + } + Self { + shards, + node_count, + max_degree, + } + } + + fn node_count(&self) -> usize { + self.node_count + } + + fn shard_index(node: usize) -> usize { + node / PARALLEL_ADJACENCY_NODES_PER_SHARD + } + + fn copy_neighbors(&self, node: usize, target: &mut Vec) { + let shard = self.shards[Self::shard_index(node)] + .read() + .expect("parallel Vamana adjacency shard lock poisoned"); + target.extend_from_slice(shard.neighbors(node, self.max_degree)); + } + + fn replace(&self, node: usize, neighbors: &[u32]) { + let mut shard = self.shards[Self::shard_index(node)] + .write() + .expect("parallel Vamana adjacency shard lock poisoned"); + shard.replace(node, self.max_degree, neighbors); + } + + fn update_from_buffer( + &self, + node: usize, + replacement: &mut Vec, + update: impl FnOnce(&[u32], &mut Vec), + ) { + let mut shard = self.shards[Self::shard_index(node)] + .write() + .expect("parallel Vamana adjacency shard lock poisoned"); + update(shard.neighbors(node, self.max_degree), replacement); + shard.replace(node, self.max_degree, replacement); + } + + fn into_adjacency(self) -> io::Result { + let mut shards = Vec::with_capacity(self.shards.len()); + for shard in self.shards { + let shard = shard.into_inner().map_err(|_| { + io::Error::other("parallel Vamana adjacency shard lock poisoned during finish") + })?; + shards.push(shard); + } + Ok(CompactAdjacency { + shards, + node_count: self.node_count, + max_degree: self.max_degree, + }) + } +} + +enum BuildVisitStates { + Dense { + states: Vec, + touched_nodes: Vec, + }, + Sparse(SparseTable), +} + +struct GreedySearchScratch { + visit_states: BuildVisitStates, + results: BinaryHeap, + frontier: BinaryHeap>, + neighbor_buffer: Vec, + candidate_ids: Vec, + prune_unique: Vec, + prune_pool: Vec, + prune_selected: Vec, + #[cfg(test)] + peak_retained: usize, + #[cfg(test)] + peak_frontier: usize, +} + +impl GreedySearchScratch { + fn new(node_count: usize, max_degree: usize, search_list_size: usize) -> Self { + let expected_visited = search_list_size + .saturating_mul(max_degree) + .saturating_add(1) + .min(node_count); + let dense_bytes = node_count + .saturating_mul(size_of::()) + .saturating_add(expected_visited.saturating_mul(size_of::())); + let sparse_bytes = + sparse_table_memory_bytes(expected_visited, size_of::()).unwrap_or(usize::MAX); + let visit_states = if sparse_bytes + .checked_mul(SPARSE_BUILD_VISITED_MIN_MEMORY_SAVINGS) + .is_some_and(|threshold| threshold < dense_bytes) + { + BuildVisitStates::Sparse(SparseTable::with_capacity(expected_visited)) + } else { + BuildVisitStates::Dense { + states: vec![0; node_count], + touched_nodes: Vec::with_capacity(expected_visited), + } + }; + Self { + visit_states, + results: BinaryHeap::new(), + frontier: BinaryHeap::new(), + neighbor_buffer: Vec::with_capacity(max_degree), + candidate_ids: Vec::with_capacity(search_list_size), + prune_unique: Vec::with_capacity(search_list_size.saturating_add(max_degree)), + prune_pool: Vec::with_capacity(search_list_size.saturating_add(max_degree)), + prune_selected: Vec::with_capacity(max_degree), + #[cfg(test)] + peak_retained: 0, + #[cfg(test)] + peak_frontier: 0, + } + } + + fn begin_search(&mut self) { + match &mut self.visit_states { + BuildVisitStates::Dense { + states, + touched_nodes, + } => { + for node in touched_nodes.drain(..) { + states[node as usize] = 0; + } + } + BuildVisitStates::Sparse(states) => states.clear(), + } + self.results.clear(); + self.frontier.clear(); + self.neighbor_buffer.clear(); + #[cfg(test)] + { + self.peak_retained = 0; + self.peak_frontier = 0; + } + } + + fn is_visited(&self, node: usize) -> bool { + match &self.visit_states { + BuildVisitStates::Dense { states, .. } => states[node] != 0, + BuildVisitStates::Sparse(states) => states.get(node as u32).is_some(), + } + } + + fn mark_visited(&mut self, node: usize) { + match &mut self.visit_states { + BuildVisitStates::Dense { + states, + touched_nodes, + } => { + if states[node] == 0 { + touched_nodes.push(node as u32); + } + states[node] = 1; + } + BuildVisitStates::Sparse(states) => { + states.insert(node as u32, 1); + } + } + } + + #[cfg(test)] + fn is_expanded(&self, node: usize) -> bool { + match &self.visit_states { + BuildVisitStates::Dense { states, .. } => states[node] == 3, + BuildVisitStates::Sparse(states) => states.get(node as u32) == Some(&3), + } + } + + fn is_retained_unexpanded(&self, node: usize) -> bool { + match &self.visit_states { + BuildVisitStates::Dense { states, .. } => states[node] == 2, + BuildVisitStates::Sparse(states) => states.get(node as u32) == Some(&2), + } + } + + fn mark_expanded(&mut self, node: usize) { + match &mut self.visit_states { + BuildVisitStates::Dense { states, .. } => states[node] = 3, + BuildVisitStates::Sparse(states) => { + states.insert(node as u32, 3); + } + } + } + + fn mark_retained(&mut self, node: usize) { + match &mut self.visit_states { + BuildVisitStates::Dense { states, .. } => states[node] = 2, + BuildVisitStates::Sparse(states) => { + states.insert(node as u32, 2); + } + } + } + + #[cfg(test)] + fn uses_sparse_states(&self) -> bool { + matches!(self.visit_states, BuildVisitStates::Sparse(_)) + } + + fn insert_candidate(&mut self, candidate: ScoredNode, search_list_size: usize) { + self.mark_visited(candidate.id as usize); + if search_list_size == 0 { + return; + } + if self.results.len() == search_list_size + && self.results.peek().is_some_and(|worst| candidate >= *worst) + { + return; + } + if self.results.len() == search_list_size { + let evicted = self + .results + .pop() + .expect("full result heap has a worst node"); + self.mark_visited(evicted.id as usize); + } + self.results.push(candidate); + self.frontier.push(Reverse(candidate)); + self.mark_retained(candidate.id as usize); + if self.frontier.len() > search_list_size.saturating_mul(2) { + let visit_states = &self.visit_states; + self.frontier + .retain(|Reverse(candidate)| match visit_states { + BuildVisitStates::Dense { states, .. } => states[candidate.id as usize] == 2, + BuildVisitStates::Sparse(states) => states.get(candidate.id) == Some(&2), + }); + } + #[cfg(test)] + { + self.peak_retained = self.peak_retained.max(self.results.len()); + self.peak_frontier = self.peak_frontier.max(self.frontier.len()); + } + } + + fn pop_nearest_unexpanded(&mut self) -> Option { + while let Some(Reverse(candidate)) = self.frontier.pop() { + if self.is_retained_unexpanded(candidate.id as usize) { + return Some(candidate); + } + } + None + } + + #[cfg(test)] + fn peak_retained_len(&self) -> usize { + self.peak_retained + } + + #[cfg(test)] + fn peak_frontier_len(&self) -> usize { + self.peak_frontier + } +} + +impl ParallelVamanaBuilder<'_> { + fn run_pass(&self, params: DiskAnnBuildParams, alpha: f32, rng: &mut StdRng) { + let mut order = (0..self.adjacency.node_count()).collect::>(); + order.shuffle(rng); + let worker_count = rayon::current_num_threads().max(1); + let scratches = (0..worker_count) + .map(|_| { + Mutex::new(GreedySearchScratch::new( + self.adjacency.node_count(), + params.max_degree, + params.build_search_list_size, + )) + }) + .collect::>(); + let mut reverse_edges = + Vec::with_capacity(PARALLEL_BUILD_BATCH_NODES.saturating_mul(params.max_degree)); + for batch in order.chunks(PARALLEL_BUILD_BATCH_NODES) { + // Every node in a batch searches and prunes against the same graph + // snapshot. Results are committed in shuffled order below, making + // the persisted graph independent of Rayon scheduling and worker + // count without serializing the expensive distance work. + let mut selected_by_node = (0..batch.len()) + .map(|_| Vec::with_capacity(params.max_degree)) + .collect::>(); + batch + .par_iter() + .zip(selected_by_node.par_iter_mut()) + .for_each(|(&node, selected)| { + let worker = rayon::current_thread_index().unwrap_or(0) % worker_count; + let mut scratch = scratches[worker] + .lock() + .expect("parallel Vamana scratch lock poisoned"); + self.greedy_search(node, params.build_search_list_size, &mut scratch); + let mut candidate_ids = std::mem::take(&mut scratch.candidate_ids); + candidate_ids.clear(); + candidate_ids.extend(scratch.results.iter().map(|candidate| candidate.id)); + let mut unique = std::mem::take(&mut scratch.prune_unique); + let mut pool = std::mem::take(&mut scratch.prune_pool); + self.adjacency + .copy_neighbors(node, &mut scratch.neighbor_buffer); + robust_prune_candidates_into( + self.vectors, + self.dimension, + node, + &candidate_ids, + &scratch.neighbor_buffer, + self.adjacency.node_count(), + params.max_degree, + alpha, + self.metric, + &mut unique, + &mut pool, + selected, + ); + scratch.neighbor_buffer.clear(); + scratch.candidate_ids = candidate_ids; + scratch.prune_unique = unique; + scratch.prune_pool = pool; + }); + + reverse_edges.clear(); + for (&node, selected) in batch.iter().zip(&selected_by_node) { + self.adjacency.replace(node, selected); + reverse_edges.extend(selected.iter().map(|&neighbor| (neighbor, node as u32))); + } + let reverse_edge_groups = group_reverse_edges(&mut reverse_edges); + reverse_edge_groups.par_iter().for_each(|group| { + let worker = rayon::current_thread_index().unwrap_or(0) % worker_count; + let mut scratch = scratches[worker] + .lock() + .expect("parallel Vamana scratch lock poisoned"); + self.insert_reverse_edges( + reverse_edges[group.start].0 as usize, + &reverse_edges[group.clone()], + params.max_degree, + alpha, + &mut scratch, + ); + }); + } + } + + fn greedy_search( + &self, + query_node: usize, + search_list_size: usize, + scratch: &mut GreedySearchScratch, + ) { + scratch.begin_search(); + let entry = self.entry_node as usize; + scratch.insert_candidate( + ScoredNode { + id: self.entry_node, + distance: self.search_distance.between(entry, query_node), + }, + search_list_size, + ); + + while let Some(current) = scratch.pop_nearest_unexpanded() { + scratch.mark_expanded(current.id as usize); + self.adjacency + .copy_neighbors(current.id as usize, &mut scratch.neighbor_buffer); + for slot in 0..scratch.neighbor_buffer.len() { + let neighbor = scratch.neighbor_buffer[slot]; + let neighbor = neighbor as usize; + if neighbor >= self.adjacency.node_count() || scratch.is_visited(neighbor) { + continue; + } + scratch.insert_candidate( + ScoredNode { + id: neighbor as u32, + distance: self.search_distance.between(neighbor, query_node), + }, + search_list_size, + ); + } + scratch.neighbor_buffer.clear(); + } + } + + fn insert_reverse_edges( + &self, + node: usize, + reverse_edges: &[(u32, u32)], + max_degree: usize, + alpha: f32, + scratch: &mut GreedySearchScratch, + ) { + let mut incoming = std::mem::take(&mut scratch.candidate_ids); + incoming.clear(); + incoming.extend(reverse_edges.iter().map(|&(_, source)| source)); + let mut selected = std::mem::take(&mut scratch.prune_selected); + let mut unique = std::mem::take(&mut scratch.prune_unique); + let mut pool = std::mem::take(&mut scratch.prune_pool); + self.adjacency + .update_from_buffer(node, &mut selected, |neighbors, selected| { + let missing = incoming + .iter() + .filter(|neighbor| !neighbors.contains(neighbor)) + .count(); + if missing == 0 { + selected.clear(); + selected.extend_from_slice(neighbors); + return; + } + if neighbors.len().saturating_add(missing) <= max_degree { + selected.clear(); + selected.extend_from_slice(neighbors); + selected.extend( + incoming + .iter() + .copied() + .filter(|neighbor| !neighbors.contains(neighbor)), + ); + return; + } + robust_prune_candidates_into( + self.vectors, + self.dimension, + node, + &incoming, + neighbors, + self.adjacency.node_count(), + max_degree, + alpha, + self.metric, + &mut unique, + &mut pool, + selected, + ); + }); + scratch.candidate_ids = incoming; + scratch.prune_selected = selected; + scratch.prune_unique = unique; + scratch.prune_pool = pool; + } + + fn finish(self) -> io::Result { + let adjacency = self.adjacency.into_adjacency()?; + Ok(VamanaGraph { + entry_node: self.entry_node, + adjacency, + }) + } +} + +#[allow(clippy::too_many_arguments)] +fn robust_prune_candidates( + vectors: &[f32], + dimension: usize, + node: usize, + candidates: &[u32], + current_neighbors: &[u32], + node_count: usize, + max_degree: usize, + alpha: f32, + metric: MetricType, +) -> Vec { + let mut unique = Vec::new(); + let mut pool = Vec::new(); + let mut selected = Vec::new(); + robust_prune_candidates_into( + vectors, + dimension, + node, + candidates, + current_neighbors, + node_count, + max_degree, + alpha, + metric, + &mut unique, + &mut pool, + &mut selected, + ); + selected +} + +#[allow(clippy::too_many_arguments)] +fn robust_prune_candidates_into( + vectors: &[f32], + dimension: usize, + node: usize, + candidates: &[u32], + current_neighbors: &[u32], + node_count: usize, + max_degree: usize, + alpha: f32, + metric: MetricType, + unique: &mut Vec, + pool: &mut Vec, + selected: &mut Vec, +) { + unique.clear(); + unique.extend( + candidates + .iter() + .chain(current_neighbors.iter()) + .copied() + .filter(|candidate| *candidate as usize != node && (*candidate as usize) < node_count), + ); + unique.sort_unstable(); + unique.dedup(); + pool.clear(); + pool.extend(unique.iter().copied().map(|candidate| ScoredNode { + id: candidate, + distance: distance_between(vectors, dimension, node, candidate as usize, metric), + })); + pool.sort_unstable_by(scored_node_order); + + selected.clear(); + let mut next = 0usize; + while next < pool.len() && selected.len() < max_degree { + let pivot = pool[next]; + selected.push(pivot.id); + next += 1; + let mut retained = next; + if metric == MetricType::L2 && dimension < 256 { + let mut candidate_index = next; + while candidate_index + 4 <= pool.len() { + let candidates = [ + pool[candidate_index], + pool[candidate_index + 1], + pool[candidate_index + 2], + pool[candidate_index + 3], + ]; + let distances = distance_between_four( + vectors, + dimension, + pivot.id as usize, + [ + candidates[0].id as usize, + candidates[1].id as usize, + candidates[2].id as usize, + candidates[3].id as usize, + ], + ); + for (candidate, distance) in candidates.into_iter().zip(distances) { + if retain_after_prune(metric, distance, candidate.distance, alpha) { + pool[retained] = candidate; + retained += 1; + } + } + candidate_index += 4; + } + for candidate_index in candidate_index..pool.len() { + let candidate = pool[candidate_index]; + let distance = distance_between( + vectors, + dimension, + pivot.id as usize, + candidate.id as usize, + metric, + ); + if retain_after_prune(metric, distance, candidate.distance, alpha) { + pool[retained] = candidate; + retained += 1; + } + } + } else if metric == MetricType::L2 { + for candidate_index in next..pool.len() { + let candidate = pool[candidate_index]; + if distance_between_exceeds( + vectors, + dimension, + pivot.id as usize, + candidate.id as usize, + alpha, + candidate.distance, + ) { + pool[retained] = candidate; + retained += 1; + } + } + } else { + for candidate_index in next..pool.len() { + let candidate = pool[candidate_index]; + let distance = distance_between( + vectors, + dimension, + pivot.id as usize, + candidate.id as usize, + metric, + ); + if retain_after_prune(metric, distance, candidate.distance, alpha) { + pool[retained] = candidate; + retained += 1; + } + } + } + pool.truncate(retained); + } +} + +#[inline] +fn retain_after_prune( + metric: MetricType, + selected_to_candidate: f32, + source_to_candidate: f32, + alpha: f32, +) -> bool { + match metric { + MetricType::InnerProduct => selected_to_candidate >= alpha * source_to_candidate, + MetricType::L2 | MetricType::Cosine => alpha * selected_to_candidate > source_to_candidate, + } +} + +fn group_reverse_edges(reverse_edges: &mut Vec<(u32, u32)>) -> Vec> { + reverse_edges.sort_unstable(); + reverse_edges.dedup(); + let mut groups = Vec::new(); + let mut start = 0usize; + while start < reverse_edges.len() { + let target = reverse_edges[start].0; + let mut end = start + 1; + while end < reverse_edges.len() && reverse_edges[end].0 == target { + end += 1; + } + groups.push(start..end); + start = end; + } + groups +} + +fn validate_build_inputs( + vectors: &[f32], + count: usize, + dimension: usize, + params: DiskAnnBuildParams, +) -> io::Result<()> { + if count == 0 || dimension == 0 || params.max_degree == 0 { + return Err(invalid_input( + "Vamana count, dimension, and maximum degree must be greater than zero", + )); + } + if count > u32::MAX as usize { + return Err(invalid_input( + "Vamana vector count exceeds u32 node ID limit", + )); + } + if params.max_degree > u16::MAX as usize { + return Err(invalid_input( + "Vamana maximum degree exceeds u16 adjacency degree limit", + )); + } + let expected = count + .checked_mul(dimension) + .ok_or_else(|| invalid_input("Vamana vector shape overflows usize"))?; + if vectors.len() != expected { + return Err(invalid_input(format!( + "Vamana vector length {} does not match {}", + vectors.len(), + expected + ))); + } + Ok(()) +} + +fn centroid_entry(vectors: &[f32], count: usize, dimension: usize, metric: MetricType) -> usize { + let mut centroid = vec![0.0f32; dimension]; + for vector in vectors.chunks_exact(dimension) { + for (sum, &value) in centroid.iter_mut().zip(vector) { + *sum += value; + } + } + for value in &mut centroid { + *value /= count as f32; + } + (0..count) + .min_by(|&left, &right| { + node_distance(vectors, dimension, left, ¢roid, metric) + .total_cmp(&node_distance(vectors, dimension, right, ¢roid, metric)) + .then_with(|| left.cmp(&right)) + }) + .expect("validated non-empty Vamana data") +} + +fn nearest_two_centroids( + vector: &[f32], + centroids: &[f32], + centroid_count: usize, + dimension: usize, +) -> [usize; 2] { + debug_assert!(centroid_count > 0); + let mut first = (0usize, f32::MAX); + let mut second = (0usize, f32::MAX); + for centroid in 0..centroid_count { + let start = centroid * dimension; + let distance = fvec_l2sqr(vector, ¢roids[start..start + dimension]); + let candidate = (centroid, distance); + if distance + .total_cmp(&first.1) + .then_with(|| centroid.cmp(&first.0)) + .is_lt() + { + second = first; + first = candidate; + } else if distance + .total_cmp(&second.1) + .then_with(|| centroid.cmp(&second.0)) + .is_lt() + { + second = candidate; + } + } + if centroid_count == 1 { + second = first; + } + [first.0, second.0] +} + +fn random_neighbors(rng: &mut StdRng, count: usize, node: usize, degree: usize) -> Vec { + if degree == count.saturating_sub(1) { + let mut neighbors = (0..count) + .filter(|&candidate| candidate != node) + .map(|candidate| candidate as u32) + .collect::>(); + neighbors.shuffle(rng); + return neighbors; + } + + let mut neighbors = Vec::with_capacity(degree); + while neighbors.len() < degree { + let candidate = rng.gen_range(0..count); + if candidate != node && !neighbors.contains(&(candidate as u32)) { + neighbors.push(candidate as u32); + } + } + neighbors +} + +fn sample_random_neighbors_into( + rng: &mut StdRng, + count: usize, + node: usize, + swaps: &mut SparseTable, + neighbors: &mut [u32], +) { + debug_assert!(node < count); + debug_assert!(neighbors.len() <= count.saturating_sub(1)); + swaps.clear(); + let population = count - 1; + for (index, neighbor) in neighbors.iter_mut().enumerate() { + let selected_index = rng.gen_range(index..population); + let selected = swaps + .get(selected_index as u32) + .copied() + .unwrap_or(selected_index as u32); + let replacement = swaps.get(index as u32).copied().unwrap_or(index as u32); + if selected_index != index { + swaps.insert(selected_index as u32, replacement); + } + *neighbor = if selected as usize >= node { + selected + 1 + } else { + selected + }; + } +} + +fn derived_seed(seed: u64, stream: u64) -> u64 { + let mut value = seed ^ stream.wrapping_mul(0x9E37_79B9_7F4A_7C15); + value = (value ^ (value >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + value = (value ^ (value >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + value ^ (value >> 31) +} + +fn invalid_input(message: impl Into) -> io::Error { + io::Error::new(io::ErrorKind::InvalidInput, message.into()) +} + +fn node_distance( + vectors: &[f32], + dimension: usize, + node: usize, + query: &[f32], + metric: MetricType, +) -> f32 { + let start = node * dimension; + fvec_distance(&vectors[start..start + dimension], query, metric) +} + +fn distance_between( + vectors: &[f32], + dimension: usize, + left: usize, + right: usize, + metric: MetricType, +) -> f32 { + let left_start = left * dimension; + let right_start = right * dimension; + fvec_distance( + &vectors[left_start..left_start + dimension], + &vectors[right_start..right_start + dimension], + metric, + ) +} + +#[inline] +fn distance_between_four( + vectors: &[f32], + dimension: usize, + left: usize, + rights: [usize; 4], +) -> [f32; 4] { + let left_start = left * dimension; + let right_starts = rights.map(|right| right * dimension); + fvec_l2sqr_four( + &vectors[left_start..left_start + dimension], + &vectors[right_starts[0]..right_starts[0] + dimension], + &vectors[right_starts[1]..right_starts[1] + dimension], + &vectors[right_starts[2]..right_starts[2] + dimension], + &vectors[right_starts[3]..right_starts[3] + dimension], + ) +} + +#[inline] +fn distance_between_exceeds( + vectors: &[f32], + dimension: usize, + left: usize, + right: usize, + scale: f32, + threshold: f32, +) -> bool { + let left_start = left * dimension; + let right_start = right * dimension; + let left = &vectors[left_start..left_start + dimension]; + let right = &vectors[right_start..right_start + dimension]; + fvec_l2sqr_scaled_exceeds(left, right, scale, threshold) +} + +fn scored_node_order(left: &ScoredNode, right: &ScoredNode) -> std::cmp::Ordering { + left.distance + .total_cmp(&right.distance) + .then_with(|| left.id.cmp(&right.id)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::diskann::DiskAnnBuildParams; + use crate::pq::ProductQuantizer; + + #[test] + fn vamana_pq_build_distance_matches_decoded_centroid_distance_for_4bit_codes() { + let mut pq = ProductQuantizer::with_nbits(4, 2, 4); + pq.centroids = (0..pq.d * pq.ksub) + .map(|index| index as f32 * 0.125) + .collect(); + let codes = [0x21, 0x43]; + let distance = PqBuildDistance::new(&pq, &codes, 2, MetricType::L2).unwrap(); + let mut left = vec![0.0; pq.d]; + let mut right = vec![0.0; pq.d]; + pq.decode(&codes[..1], &mut left); + pq.decode(&codes[1..], &mut right); + + assert_eq!(distance.between(0, 1), fvec_l2sqr(&left, &right)); + assert_eq!(distance.between(1, 0), distance.between(0, 1)); + assert_eq!(distance.between(0, 0), 0.0); + } + + #[test] + fn vamana_pq_build_distance_and_pruning_follow_inner_product_semantics() { + let mut pq = ProductQuantizer::with_nbits(2, 1, 4); + pq.centroids = (0..pq.d * pq.ksub) + .map(|index| index as f32 * 0.25 - 1.0) + .collect(); + let codes = [0x01, 0x03]; + let distance = PqBuildDistance::new(&pq, &codes, 2, MetricType::InnerProduct).unwrap(); + let mut left = vec![0.0; pq.d]; + let mut right = vec![0.0; pq.d]; + pq.decode(&codes[..1], &mut left); + pq.decode(&codes[1..], &mut right); + + assert_eq!( + distance.between(0, 1), + fvec_distance(&left, &right, MetricType::InnerProduct) + ); + assert!(retain_after_prune( + MetricType::InnerProduct, + -1.0, + -2.0, + 1.0 + )); + assert!(!retain_after_prune( + MetricType::InnerProduct, + -4.0, + -3.0, + 1.0 + )); + } + + #[test] + fn vamana_pq_guided_build_keeps_exact_pruned_graph_bounded_and_reachable() { + let dimension = 8; + let count = 128; + let vectors = (0..count * dimension) + .map(|offset| ((offset * 37) % 101) as f32) + .collect::>(); + let mut pq = ProductQuantizer::with_nbits(dimension, 2, 4); + pq.train(&vectors, count); + let mut codes = vec![0; count * pq.code_size()]; + pq.encode_batch(&vectors, count, &mut codes); + let params = DiskAnnBuildParams { + max_degree: 12, + build_search_list_size: 32, + ..DiskAnnBuildParams::default() + }; + + let (graph, _) = VamanaGraph::build_with_pq_stats( + &vectors, + &pq, + &codes, + count, + dimension, + MetricType::L2, + params, + ) + .unwrap(); + + assert!(graph.is_fully_reachable()); + assert!(graph + .adjacency + .iter() + .all(|neighbors| neighbors.len() <= params.max_degree)); + } + + #[test] + fn vamana_greedy_search_returns_nearest_unique_nodes() { + let graph = + VamanaGraph::from_adjacency(0, vec![vec![1, 2, 2], vec![3], vec![4], vec![], vec![]]); + let vectors = [0.0, 1.0, 2.0, 3.0, 4.0]; + + let result = graph.greedy_search(&vectors, 1, &[3.1], 3); + + assert_eq!( + result.iter().map(|node| node.id).collect::>(), + vec![3, 4, 2] + ); + } + + #[test] + fn vamana_prune_removes_occluded_and_duplicate_neighbors() { + let graph = VamanaGraph::from_adjacency(0, vec![vec![], vec![], vec![], vec![]]); + let vectors = [0.0, 1.0, 1.1, -1.0]; + + let selected = graph.robust_prune(&vectors, 1, 0, &[1, 2, 3, 3, 0], 2, 1.2); + + assert_eq!(selected, vec![1, 3]); + } + + #[test] + fn vamana_build_sequential_is_deterministic_bounded_and_reachable() { + let dimension = 2; + let count = 64; + let vectors = (0..count) + .flat_map(|node| [(node % 8) as f32, (node / 8) as f32]) + .collect::>(); + let params = DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }; + + let first = VamanaGraph::build_sequential(&vectors, count, dimension, params).unwrap(); + let second = VamanaGraph::build_sequential(&vectors, count, dimension, params).unwrap(); + + assert_eq!(first, second); + assert!(first.adjacency.iter().all(|neighbors| neighbors.len() <= 8)); + assert!(first.is_fully_reachable()); + } + + #[test] + fn vamana_build_parallel_preserves_graph_invariants() { + let dimension = 4; + let count = 128; + let vectors = (0..count * dimension) + .map(|offset| ((offset * 37) % 101) as f32) + .collect::>(); + let params = DiskAnnBuildParams { + max_degree: 12, + build_search_list_size: 32, + ..DiskAnnBuildParams::default() + }; + + let graph = VamanaGraph::build(&vectors, count, dimension, params).unwrap(); + + assert!(graph + .adjacency + .iter() + .all(|neighbors| neighbors.len() <= 12)); + assert!(graph.is_fully_reachable()); + } + + #[test] + fn vamana_parallel_build_is_deterministic_across_worker_counts() { + let dimension = 4; + let count = 160; + let vectors = (0..count * dimension) + .map(|offset| ((offset * 37) % 101) as f32) + .collect::>(); + let params = DiskAnnBuildParams { + max_degree: 12, + build_search_list_size: 32, + seed: 77, + ..DiskAnnBuildParams::default() + }; + let build = |threads| { + rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .unwrap() + .install(|| VamanaGraph::build(&vectors, count, dimension, params).unwrap()) + }; + + assert_eq!(build(1), build(4)); + assert_eq!(build(4), build(4)); + } + + #[test] + fn vamana_overlapping_shard_build_is_bounded_and_reachable() { + let count = 128; + let dimension = 8; + let vectors = (0..count * dimension) + .map(|offset| { + let cluster = (offset / dimension) % 8; + cluster as f32 * 10.0 + (offset % dimension) as f32 * 0.01 + }) + .collect::>(); + let params = DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + seed: 91, + ..DiskAnnBuildParams::default() + }; + + let (graph, stats) = VamanaGraph::build_sharded_with_stats( + &vectors, + count, + dimension, + MetricType::L2, + params, + 4, + ) + .unwrap(); + let mut pq = ProductQuantizer::with_nbits_balanced(dimension, 2, 4); + pq.train(&vectors, count); + let mut pq_codes = vec![0; count * pq.code_size()]; + pq.encode_batch(&vectors, count, &mut pq_codes); + let (pq_graph, _) = VamanaGraph::build_sharded_with_pq_stats( + &vectors, + &pq, + &pq_codes, + count, + dimension, + MetricType::L2, + DiskAnnBuildParams { + build_distance: crate::diskann::DiskAnnBuildDistance::ProductQuantized, + ..params + }, + 4, + ) + .unwrap(); + + assert!(graph.is_fully_reachable()); + assert!(pq_graph.is_fully_reachable()); + assert_ne!( + graph, pq_graph, + "PQ-guided sharded construction must not silently use the full-precision path" + ); + assert!(stats.initialization > Duration::ZERO); + for (node, neighbors) in graph.adjacency.iter().enumerate() { + assert!(neighbors.len() <= params.max_degree); + assert!(!neighbors.contains(&(node as u32))); + let mut unique = neighbors.to_vec(); + unique.sort_unstable(); + unique.dedup(); + assert_eq!(unique.len(), neighbors.len()); + } + } + + #[test] + fn overlapping_shards_capacity_balance_degenerate_assignments() { + let count = 128; + let dimension = 4; + let shard_count = 8; + let vectors = vec![0.0; count * dimension]; + let centroids = vec![0.0; shard_count * dimension]; + let mut assignments = vec![[0, 1]; count]; + let mut memberships = (0..shard_count).map(|_| Vec::new()).collect::>(); + memberships[0].extend(0..count as u32); + memberships[1].extend(0..count as u32); + + rebalance_overlapping_shards( + &vectors, + dimension, + ¢roids, + &mut assignments, + &mut memberships, + ) + .unwrap(); + + let capacity = overlapping_shard_capacity(count, shard_count).unwrap(); + assert!(memberships.iter().all(|members| members.len() <= capacity)); + assert!(assignments.iter().all(|pair| pair[0] != pair[1])); + let mut seen = vec![0usize; count]; + for members in memberships { + for node in members { + seen[node as usize] += 1; + } + } + assert!(seen.into_iter().all(|memberships| memberships == 2)); + } + + #[test] + fn vamana_parallel_build_rejects_degree_above_contiguous_storage_limit() { + let error = VamanaGraph::build( + &[0.0, 1.0], + 2, + 1, + DiskAnnBuildParams { + max_degree: u16::MAX as usize + 1, + build_search_list_size: u16::MAX as usize + 1, + ..DiskAnnBuildParams::default() + }, + ) + .expect_err("parallel adjacency degree is stored as u16"); + + assert!(error.to_string().contains("u16")); + } + + #[test] + fn vamana_memory_estimate_includes_candidate_and_batched_reverse_edge_buffers() { + let search_list_size = 100; + let max_degree = 64; + let estimate = estimate_vamana_memory_bytes(0, max_degree, search_list_size, 1).unwrap(); + let expected_worker_bytes = search_list_size * 3 * size_of::() + + (search_list_size + max_degree) * (size_of::() + size_of::()) + + max_degree * 2 * size_of::() + + search_list_size * size_of::() + + 8 * max_degree * size_of::<(u32, u32)>(); + + assert!(estimate.build_peak_bytes >= expected_worker_bytes); + } + + #[test] + fn vamana_build_parallel_repairs_disconnected_clustered_graph() { + let count = 512; + let dimension = 8; + let mut vectors = vec![0.0f32; count * dimension]; + for node in 0..count { + let cluster = node / 128; + let local = node % 128; + for dim in 0..dimension { + vectors[node * dimension + dim] = + cluster as f32 * 20.0 + dim as f32 * 0.01 + (local % 16) as f32 * 0.001; + } + } + let params = DiskAnnBuildParams { + max_degree: 8, + build_search_list_size: 16, + ..DiskAnnBuildParams::default() + }; + + let graph = VamanaGraph::build(&vectors, count, dimension, params).unwrap(); + + assert!( + std::any::type_name_of_val(&graph.adjacency).contains("CompactAdjacency"), + "finished Vamana graph must retain compact adjacency storage" + ); + assert!(graph.is_fully_reachable()); + assert!(graph + .adjacency + .iter() + .all(|neighbors| neighbors.len() <= params.max_degree)); + } + + #[test] + fn vamana_connectivity_repair_reuses_reachability_and_bounds_source_search() { + let count = 1_024; + let vectors = (0..count).map(|node| node as f32).collect::>(); + let adjacency = (0..count) + .map(|node| vec![(node ^ 1) as u32]) + .collect::>(); + let mut graph = VamanaGraph::from_adjacency(0, adjacency); + + let stats = graph + .repair_connectivity_with_stats(&vectors, 1, 1, MetricType::L2) + .unwrap(); + + assert!(graph.is_fully_reachable()); + assert_eq!(stats.full_reachability_traversals, 1); + assert_eq!(stats.edges_added, count / 2 - 1); + assert!( + stats.source_distance_evaluations <= stats.edges_added * 64, + "bounded repair evaluated {} sources for {} edges", + stats.source_distance_evaluations, + stats.edges_added + ); + } + + #[test] + fn vamana_parallel_adjacency_stores_nodes_contiguously_and_isolates_updates() { + let adjacency = ParallelAdjacency::new(4, 3, |node| { + vec![((node + 1) % 4) as u32, ((node + 2) % 4) as u32] + }); + let mut neighbors = Vec::new(); + adjacency.copy_neighbors(2, &mut neighbors); + assert_eq!(neighbors, vec![3, 0]); + + adjacency.replace(2, &[1]); + neighbors.clear(); + adjacency.copy_neighbors(2, &mut neighbors); + assert_eq!(neighbors, vec![1]); + + neighbors.clear(); + adjacency.copy_neighbors(1, &mut neighbors); + assert_eq!(neighbors, vec![2, 3]); + } + + #[test] + fn vamana_reverse_edges_are_deduplicated_and_grouped_by_target() { + let mut reverse_edges = vec![(7, 3), (2, 9), (7, 1), (2, 9), (7, 3), (2, 4)]; + + let groups = group_reverse_edges(&mut reverse_edges); + + assert_eq!(reverse_edges, vec![(2, 4), (2, 9), (7, 1), (7, 3)]); + assert_eq!(groups, vec![0..2, 2..4]); + assert!(groups.iter().all(|group| reverse_edges[group.clone()] + .iter() + .all(|edge| edge.0 == reverse_edges[group.start].0))); + } + + #[test] + fn vamana_parallel_random_initialization_is_bounded_unique_and_deterministic() { + let first = ParallelAdjacency::new_random(1_024, 64, 91); + let second = ParallelAdjacency::new_random(1_024, 64, 91); + let mut first_neighbors = Vec::new(); + let mut second_neighbors = Vec::new(); + + for node in 0..1_024 { + first_neighbors.clear(); + second_neighbors.clear(); + first.copy_neighbors(node, &mut first_neighbors); + second.copy_neighbors(node, &mut second_neighbors); + assert_eq!(first_neighbors, second_neighbors); + assert_eq!(first_neighbors.len(), 64); + assert!(!first_neighbors.contains(&(node as u32))); + first_neighbors.sort_unstable(); + first_neighbors.dedup(); + assert_eq!(first_neighbors.len(), 64); + } + } + + #[test] + fn vamana_greedy_scratch_uses_reusable_dense_bytes_for_million_node_graphs() { + let mut scratch = GreedySearchScratch::new(1_000_000, 64, 100); + + assert!(!scratch.uses_sparse_states()); + scratch.begin_search(); + scratch.mark_visited(999_999); + scratch.mark_retained(999_999); + assert!(scratch.is_visited(999_999)); + assert!(scratch.is_retained_unexpanded(999_999)); + + scratch.begin_search(); + + assert!(!scratch.is_visited(999_999)); + let BuildVisitStates::Dense { + states, + touched_nodes, + } = &scratch.visit_states + else { + panic!("million-node graph should use byte-dense visit states"); + }; + assert_eq!(states.len(), 1_000_000); + assert!(touched_nodes.is_empty()); + } + + #[test] + fn vamana_greedy_scratch_uses_sparse_states_for_large_graphs() { + let mut scratch = GreedySearchScratch::new(100_000_000, 64, 100); + + assert!(scratch.uses_sparse_states()); + let BuildVisitStates::Sparse(states) = &scratch.visit_states else { + panic!("large graph should use sparse visit states"); + }; + assert!( + std::any::type_name_of_val(states).contains("SparseTable"), + "Vamana hot-path sparse states must use the internal open-addressed table" + ); + scratch.begin_search(); + scratch.mark_visited(99_999_999); + scratch.mark_expanded(99_999_999); + assert!(scratch.is_visited(99_999_999)); + assert!(scratch.is_expanded(99_999_999)); + + scratch.begin_search(); + + assert!(!scratch.is_visited(99_999_999)); + assert!(!scratch.is_expanded(99_999_999)); + assert!(scratch.neighbor_buffer.capacity() >= 64); + } + + #[test] + fn vamana_parallel_greedy_search_keeps_worker_frontiers_bounded() { + let count = 256; + let vectors = (0..count).map(|node| node as f32).collect::>(); + let adjacency = ParallelAdjacency::new(count, 32, |node| { + (1..=32) + .map(|step| ((node + step) % count) as u32) + .collect::>() + }); + let builder = ParallelVamanaBuilder { + vectors: &vectors, + dimension: 1, + metric: MetricType::L2, + entry_node: 0, + adjacency, + search_distance: BuildSearchDistance::FullPrecision { + vectors: &vectors, + dimension: 1, + metric: MetricType::L2, + }, + }; + let mut scratch = GreedySearchScratch::new(count, 32, 16); + + builder.greedy_search(127, 16, &mut scratch); + + assert_eq!(scratch.results.len(), 16); + assert!(scratch.peak_retained_len() <= 16); + assert!(scratch.peak_frontier_len() <= 32); + } + + #[test] + fn vamana_parallel_greedy_search_uses_heap_frontiers() { + let scratch = GreedySearchScratch::new(1024, 64, 100); + + assert!( + std::any::type_name_of_val(&scratch.frontier).contains("BinaryHeap"), + "parallel Vamana frontier must avoid sorted Vec insertion and removal" + ); + } +} diff --git a/core/tests/ann_bench_support.rs b/core/tests/ann_bench_support.rs new file mode 100644 index 0000000..0085e09 --- /dev/null +++ b/core/tests/ann_bench_support.rs @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#[path = "../benches/support/ann_bench_support.rs"] +mod ann_bench_support; + +use ann_bench_support::{ + add_fixed_round_latency, default_training_vector_count, inspect_public_dataset, + parse_storage_case_names, resolve_shape_value, should_isolate_indexes, PublicDatasetShape, +}; +use std::fs::{self, File}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +static NEXT_TEST_DIRECTORY: AtomicU64 = AtomicU64::new(0); + +struct TestDirectory(PathBuf); + +impl TestDirectory { + fn new() -> Self { + let suffix = NEXT_TEST_DIRECTORY.fetch_add(1, Ordering::Relaxed); + let path = std::env::temp_dir().join(format!( + "paimon-ann-bench-support-{}-{suffix}", + std::process::id() + )); + fs::create_dir(&path).unwrap(); + Self(path) + } +} + +impl Drop for TestDirectory { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn write_i32_records(path: &Path, rows: usize, width: usize) { + let mut file = File::create(path).unwrap(); + for row in 0..rows { + file.write_all(&(width as i32).to_le_bytes()).unwrap(); + for column in 0..width { + file.write_all(&((row * width + column) as i32).to_le_bytes()) + .unwrap(); + } + } +} + +#[test] +fn public_dataset_shape_is_inferred_without_loading_vector_payloads() { + let directory = TestDirectory::new(); + let base = directory.0.join("base.fvecs"); + let queries = directory.0.join("query.fvecs"); + let truth = directory.0.join("truth.ivecs"); + write_i32_records(&base, 10, 8); + write_i32_records(&queries, 3, 8); + write_i32_records(&truth, 3, 5); + + assert_eq!( + inspect_public_dataset(&base, &queries, &truth).unwrap(), + PublicDatasetShape { + vector_count: 10, + query_count: 3, + dimension: 8, + ground_truth_width: 5, + } + ); +} + +#[test] +fn public_dataset_shape_rejects_query_and_ground_truth_mismatches() { + let directory = TestDirectory::new(); + let base = directory.0.join("base.fvecs"); + let queries = directory.0.join("query.fvecs"); + let truth = directory.0.join("truth.ivecs"); + write_i32_records(&base, 10, 8); + write_i32_records(&queries, 3, 4); + write_i32_records(&truth, 2, 5); + + let dimension_error = inspect_public_dataset(&base, &queries, &truth).unwrap_err(); + assert!(dimension_error + .to_string() + .contains("base/query fvec dimensions differ")); + + write_i32_records(&queries, 3, 8); + let count_error = inspect_public_dataset(&base, &queries, &truth).unwrap_err(); + assert!(count_error + .to_string() + .contains("query/ground-truth counts differ")); +} + +#[test] +fn explicit_public_shape_is_an_assertion_and_generated_shape_keeps_defaults() { + assert_eq!( + resolve_shape_value("ANN_D", None, Some(128), 64).unwrap(), + 128 + ); + assert_eq!( + resolve_shape_value("ANN_D", Some(128), Some(128), 64).unwrap(), + 128 + ); + assert_eq!(resolve_shape_value("ANN_D", None, None, 64).unwrap(), 64); + + let error = resolve_shape_value("ANN_D", Some(960), Some(128), 64).unwrap_err(); + assert!(error + .to_string() + .contains("ANN_D=960 does not match public dataset shape 128")); +} + +#[test] +fn training_sample_default_scales_with_nlist_and_is_capped_by_the_dataset() { + assert_eq!( + default_training_vector_count(1_000_000, 1_024).unwrap(), + 65_536 + ); + assert_eq!( + default_training_vector_count(1_000_000, 4_096).unwrap(), + 262_144 + ); + assert_eq!( + default_training_vector_count(10_000, 1_024).unwrap(), + 10_000 + ); + assert!(default_training_vector_count(usize::MAX, usize::MAX).is_err()); +} + +#[test] +fn only_multi_index_public_parent_runs_require_process_isolation() { + assert!(should_isolate_indexes(true, 6, false, false)); + assert!(!should_isolate_indexes(false, 6, false, false)); + assert!(!should_isolate_indexes(true, 1, false, false)); + assert!(!should_isolate_indexes(true, 6, true, false)); + assert!(!should_isolate_indexes(true, 6, false, true)); +} + +#[test] +fn storage_case_selection_defaults_to_all_and_validates_subsets() { + assert_eq!( + parse_storage_case_names(None).unwrap(), + [ + "local_ssd_warm_cache", + "remote_cache_2ms", + "object_store_20ms" + ] + ); + assert_eq!( + parse_storage_case_names(Some("object-store-20ms,remote_cache_2ms,object_store_20ms")) + .unwrap(), + ["object_store_20ms", "remote_cache_2ms"] + ); + assert!(parse_storage_case_names(Some("")).is_err()); + assert!(parse_storage_case_names(Some("unknown")).is_err()); +} + +#[test] +fn fixed_round_latency_adds_the_idealized_dependency_cost() { + assert_eq!( + add_fixed_round_latency(Duration::from_millis(7), 8, Duration::from_millis(20)), + Duration::from_millis(167) + ); + assert_eq!( + add_fixed_round_latency(Duration::MAX, usize::MAX, Duration::MAX), + Duration::MAX + ); +} diff --git a/core/tests/fixtures/diskann_compact_multipage_v1.hex b/core/tests/fixtures/diskann_compact_multipage_v1.hex new file mode 100644 index 0000000..7819ec1 --- /dev/null +++ b/core/tests/fixtures/diskann_compact_multipage_v1.hex @@ -0,0 +1,1793 @@ +44 41 4e 4e 01 00 00 00 00 01 00 00 1f 00 00 00 10 00 00 00 00 00 00 00 01 02 00 00 00 00 00 00 +00 00 00 00 20 00 00 00 40 00 00 00 9a 99 99 3f 07 00 00 00 00 00 00 00 04 00 00 00 08 00 00 00 +00 10 00 00 04 00 00 00 03 00 00 00 02 00 00 00 20 00 00 00 07 00 00 00 20 e0 00 00 00 00 00 00 +00 10 00 00 00 00 00 00 34 40 00 00 00 00 00 00 34 50 00 00 00 00 00 00 22 03 00 00 00 00 00 00 +56 53 00 00 00 00 00 00 04 08 00 00 00 00 00 00 5a 5b 00 00 00 00 00 00 04 08 00 00 00 00 00 00 +5e 63 00 00 00 00 00 00 0c 09 00 00 00 00 00 00 00 70 00 00 00 00 00 00 00 30 00 00 00 00 00 00 +00 a0 00 00 00 00 00 00 20 40 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +44 50 51 31 01 00 00 00 10 00 00 00 04 00 00 00 08 00 00 00 00 01 00 00 05 00 00 00 00 00 00 00 +00 00 00 00 04 00 00 00 08 00 00 00 0c 00 00 00 10 00 00 00 00 00 00 00 00 00 80 3f 00 00 00 40 +00 00 40 40 00 00 80 40 00 00 a0 40 00 00 c0 40 00 00 e0 40 00 00 00 41 00 00 10 41 00 00 20 41 +00 00 30 41 00 00 40 41 00 00 50 41 00 00 60 41 00 00 70 41 00 00 80 41 00 00 88 41 00 00 90 41 +00 00 98 41 00 00 a0 41 00 00 a8 41 00 00 b0 41 00 00 b8 41 00 00 c0 41 00 00 c8 41 00 00 d0 41 +00 00 d8 41 00 00 e0 41 00 00 e8 41 00 00 f0 41 00 00 f8 41 00 00 00 42 00 00 04 42 00 00 08 42 +00 00 0c 42 00 00 10 42 00 00 14 42 00 00 18 42 00 00 1c 42 00 00 20 42 00 00 24 42 00 00 28 42 +00 00 2c 42 00 00 30 42 00 00 34 42 00 00 38 42 00 00 3c 42 00 00 40 42 00 00 44 42 00 00 48 42 +00 00 4c 42 00 00 50 42 00 00 54 42 00 00 58 42 00 00 5c 42 00 00 60 42 00 00 64 42 00 00 68 42 +00 00 6c 42 00 00 70 42 00 00 74 42 00 00 78 42 00 00 7c 42 00 00 80 42 00 00 82 42 00 00 84 42 +00 00 86 42 00 00 88 42 00 00 8a 42 00 00 8c 42 00 00 8e 42 00 00 90 42 00 00 92 42 00 00 94 42 +00 00 96 42 00 00 98 42 00 00 9a 42 00 00 9c 42 00 00 9e 42 00 00 a0 42 00 00 a2 42 00 00 a4 42 +00 00 a6 42 00 00 a8 42 00 00 aa 42 00 00 ac 42 00 00 ae 42 00 00 b0 42 00 00 b2 42 00 00 b4 42 +00 00 b6 42 00 00 b8 42 00 00 ba 42 00 00 bc 42 00 00 be 42 00 00 c0 42 00 00 c2 42 00 00 c4 42 +00 00 c6 42 00 00 c8 42 00 00 ca 42 00 00 cc 42 00 00 ce 42 00 00 d0 42 00 00 d2 42 00 00 d4 42 +00 00 d6 42 00 00 d8 42 00 00 da 42 00 00 dc 42 00 00 de 42 00 00 e0 42 00 00 e2 42 00 00 e4 42 +00 00 e6 42 00 00 e8 42 00 00 ea 42 00 00 ec 42 00 00 ee 42 00 00 f0 42 00 00 f2 42 00 00 f4 42 +00 00 f6 42 00 00 f8 42 00 00 fa 42 00 00 fc 42 00 00 fe 42 00 00 00 43 00 00 01 43 00 00 02 43 +00 00 03 43 00 00 04 43 00 00 05 43 00 00 06 43 00 00 07 43 00 00 08 43 00 00 09 43 00 00 0a 43 +00 00 0b 43 00 00 0c 43 00 00 0d 43 00 00 0e 43 00 00 0f 43 00 00 10 43 00 00 11 43 00 00 12 43 +00 00 13 43 00 00 14 43 00 00 15 43 00 00 16 43 00 00 17 43 00 00 18 43 00 00 19 43 00 00 1a 43 +00 00 1b 43 00 00 1c 43 00 00 1d 43 00 00 1e 43 00 00 1f 43 00 00 20 43 00 00 21 43 00 00 22 43 +00 00 23 43 00 00 24 43 00 00 25 43 00 00 26 43 00 00 27 43 00 00 28 43 00 00 29 43 00 00 2a 43 +00 00 2b 43 00 00 2c 43 00 00 2d 43 00 00 2e 43 00 00 2f 43 00 00 30 43 00 00 31 43 00 00 32 43 +00 00 33 43 00 00 34 43 00 00 35 43 00 00 36 43 00 00 37 43 00 00 38 43 00 00 39 43 00 00 3a 43 +00 00 3b 43 00 00 3c 43 00 00 3d 43 00 00 3e 43 00 00 3f 43 00 00 40 43 00 00 41 43 00 00 42 43 +00 00 43 43 00 00 44 43 00 00 45 43 00 00 46 43 00 00 47 43 00 00 48 43 00 00 49 43 00 00 4a 43 +00 00 4b 43 00 00 4c 43 00 00 4d 43 00 00 4e 43 00 00 4f 43 00 00 50 43 00 00 51 43 00 00 52 43 +00 00 53 43 00 00 54 43 00 00 55 43 00 00 56 43 00 00 57 43 00 00 58 43 00 00 59 43 00 00 5a 43 +00 00 5b 43 00 00 5c 43 00 00 5d 43 00 00 5e 43 00 00 5f 43 00 00 60 43 00 00 61 43 00 00 62 43 +00 00 63 43 00 00 64 43 00 00 65 43 00 00 66 43 00 00 67 43 00 00 68 43 00 00 69 43 00 00 6a 43 +00 00 6b 43 00 00 6c 43 00 00 6d 43 00 00 6e 43 00 00 6f 43 00 00 70 43 00 00 71 43 00 00 72 43 +00 00 73 43 00 00 74 43 00 00 75 43 00 00 76 43 00 00 77 43 00 00 78 43 00 00 79 43 00 00 7a 43 +00 00 7b 43 00 00 7c 43 00 00 7d 43 00 00 7e 43 00 00 7f 43 6f 12 83 3a c5 20 80 3f 62 10 00 40 +62 10 40 40 31 08 80 40 31 08 a0 40 31 08 c0 40 31 08 e0 40 19 04 00 41 19 04 10 41 19 04 20 41 +19 04 30 41 19 04 40 41 19 04 50 41 19 04 60 41 19 04 70 41 0c 02 80 41 0c 02 88 41 0c 02 90 41 +0c 02 98 41 0c 02 a0 41 0c 02 a8 41 0c 02 b0 41 0c 02 b8 41 0c 02 c0 41 0c 02 c8 41 0c 02 d0 41 +0c 02 d8 41 0c 02 e0 41 0c 02 e8 41 0c 02 f0 41 0c 02 f8 41 06 01 00 42 06 01 04 42 06 01 08 42 +06 01 0c 42 06 01 10 42 06 01 14 42 06 01 18 42 06 01 1c 42 06 01 20 42 06 01 24 42 06 01 28 42 +06 01 2c 42 06 01 30 42 06 01 34 42 06 01 38 42 06 01 3c 42 06 01 40 42 06 01 44 42 06 01 48 42 +06 01 4c 42 06 01 50 42 06 01 54 42 06 01 58 42 06 01 5c 42 06 01 60 42 06 01 64 42 06 01 68 42 +06 01 6c 42 06 01 70 42 06 01 74 42 06 01 78 42 06 01 7c 42 83 00 80 42 83 00 82 42 83 00 84 42 +83 00 86 42 83 00 88 42 83 00 8a 42 83 00 8c 42 83 00 8e 42 83 00 90 42 83 00 92 42 83 00 94 42 +83 00 96 42 83 00 98 42 83 00 9a 42 83 00 9c 42 83 00 9e 42 83 00 a0 42 83 00 a2 42 83 00 a4 42 +83 00 a6 42 83 00 a8 42 83 00 aa 42 83 00 ac 42 83 00 ae 42 83 00 b0 42 83 00 b2 42 83 00 b4 42 +83 00 b6 42 83 00 b8 42 83 00 ba 42 83 00 bc 42 83 00 be 42 83 00 c0 42 83 00 c2 42 83 00 c4 42 +83 00 c6 42 83 00 c8 42 83 00 ca 42 83 00 cc 42 83 00 ce 42 83 00 d0 42 83 00 d2 42 83 00 d4 42 +83 00 d6 42 83 00 d8 42 83 00 da 42 83 00 dc 42 83 00 de 42 83 00 e0 42 83 00 e2 42 83 00 e4 42 +83 00 e6 42 83 00 e8 42 83 00 ea 42 83 00 ec 42 83 00 ee 42 83 00 f0 42 83 00 f2 42 83 00 f4 42 +83 00 f6 42 83 00 f8 42 83 00 fa 42 83 00 fc 42 83 00 fe 42 42 00 00 43 42 00 01 43 42 00 02 43 +42 00 03 43 42 00 04 43 42 00 05 43 42 00 06 43 42 00 07 43 42 00 08 43 42 00 09 43 42 00 0a 43 +42 00 0b 43 42 00 0c 43 42 00 0d 43 42 00 0e 43 42 00 0f 43 42 00 10 43 42 00 11 43 42 00 12 43 +42 00 13 43 42 00 14 43 42 00 15 43 42 00 16 43 42 00 17 43 42 00 18 43 42 00 19 43 42 00 1a 43 +42 00 1b 43 42 00 1c 43 42 00 1d 43 42 00 1e 43 42 00 1f 43 42 00 20 43 42 00 21 43 42 00 22 43 +42 00 23 43 42 00 24 43 42 00 25 43 42 00 26 43 42 00 27 43 42 00 28 43 42 00 29 43 42 00 2a 43 +42 00 2b 43 42 00 2c 43 42 00 2d 43 42 00 2e 43 42 00 2f 43 42 00 30 43 42 00 31 43 42 00 32 43 +42 00 33 43 42 00 34 43 42 00 35 43 42 00 36 43 42 00 37 43 42 00 38 43 42 00 39 43 42 00 3a 43 +42 00 3b 43 42 00 3c 43 42 00 3d 43 42 00 3e 43 42 00 3f 43 42 00 40 43 42 00 41 43 42 00 42 43 +42 00 43 43 42 00 44 43 42 00 45 43 42 00 46 43 42 00 47 43 42 00 48 43 42 00 49 43 42 00 4a 43 +42 00 4b 43 42 00 4c 43 42 00 4d 43 42 00 4e 43 42 00 4f 43 42 00 50 43 42 00 51 43 42 00 52 43 +42 00 53 43 42 00 54 43 42 00 55 43 42 00 56 43 42 00 57 43 42 00 58 43 42 00 59 43 42 00 5a 43 +42 00 5b 43 42 00 5c 43 42 00 5d 43 42 00 5e 43 42 00 5f 43 42 00 60 43 42 00 61 43 42 00 62 43 +42 00 63 43 42 00 64 43 42 00 65 43 42 00 66 43 42 00 67 43 42 00 68 43 42 00 69 43 42 00 6a 43 +42 00 6b 43 42 00 6c 43 42 00 6d 43 42 00 6e 43 42 00 6f 43 42 00 70 43 42 00 71 43 42 00 72 43 +42 00 73 43 42 00 74 43 42 00 75 43 42 00 76 43 42 00 77 43 42 00 78 43 42 00 79 43 42 00 7a 43 +42 00 7b 43 42 00 7c 43 42 00 7d 43 42 00 7e 43 42 00 7f 43 6f 12 03 3b 89 41 80 3f c5 20 00 40 +c5 20 40 40 62 10 80 40 62 10 a0 40 62 10 c0 40 62 10 e0 40 31 08 00 41 31 08 10 41 31 08 20 41 +31 08 30 41 31 08 40 41 31 08 50 41 31 08 60 41 31 08 70 41 19 04 80 41 19 04 88 41 19 04 90 41 +19 04 98 41 19 04 a0 41 19 04 a8 41 19 04 b0 41 19 04 b8 41 19 04 c0 41 19 04 c8 41 19 04 d0 41 +19 04 d8 41 19 04 e0 41 19 04 e8 41 19 04 f0 41 19 04 f8 41 0c 02 00 42 0c 02 04 42 0c 02 08 42 +0c 02 0c 42 0c 02 10 42 0c 02 14 42 0c 02 18 42 0c 02 1c 42 0c 02 20 42 0c 02 24 42 0c 02 28 42 +0c 02 2c 42 0c 02 30 42 0c 02 34 42 0c 02 38 42 0c 02 3c 42 0c 02 40 42 0c 02 44 42 0c 02 48 42 +0c 02 4c 42 0c 02 50 42 0c 02 54 42 0c 02 58 42 0c 02 5c 42 0c 02 60 42 0c 02 64 42 0c 02 68 42 +0c 02 6c 42 0c 02 70 42 0c 02 74 42 0c 02 78 42 0c 02 7c 42 06 01 80 42 06 01 82 42 06 01 84 42 +06 01 86 42 06 01 88 42 06 01 8a 42 06 01 8c 42 06 01 8e 42 06 01 90 42 06 01 92 42 06 01 94 42 +06 01 96 42 06 01 98 42 06 01 9a 42 06 01 9c 42 06 01 9e 42 06 01 a0 42 06 01 a2 42 06 01 a4 42 +06 01 a6 42 06 01 a8 42 06 01 aa 42 06 01 ac 42 06 01 ae 42 06 01 b0 42 06 01 b2 42 06 01 b4 42 +06 01 b6 42 06 01 b8 42 06 01 ba 42 06 01 bc 42 06 01 be 42 06 01 c0 42 06 01 c2 42 06 01 c4 42 +06 01 c6 42 06 01 c8 42 06 01 ca 42 06 01 cc 42 06 01 ce 42 06 01 d0 42 06 01 d2 42 06 01 d4 42 +06 01 d6 42 06 01 d8 42 06 01 da 42 06 01 dc 42 06 01 de 42 06 01 e0 42 06 01 e2 42 06 01 e4 42 +06 01 e6 42 06 01 e8 42 06 01 ea 42 06 01 ec 42 06 01 ee 42 06 01 f0 42 06 01 f2 42 06 01 f4 42 +06 01 f6 42 06 01 f8 42 06 01 fa 42 06 01 fc 42 06 01 fe 42 83 00 00 43 83 00 01 43 83 00 02 43 +83 00 03 43 83 00 04 43 83 00 05 43 83 00 06 43 83 00 07 43 83 00 08 43 83 00 09 43 83 00 0a 43 +83 00 0b 43 83 00 0c 43 83 00 0d 43 83 00 0e 43 83 00 0f 43 83 00 10 43 83 00 11 43 83 00 12 43 +83 00 13 43 83 00 14 43 83 00 15 43 83 00 16 43 83 00 17 43 83 00 18 43 83 00 19 43 83 00 1a 43 +83 00 1b 43 83 00 1c 43 83 00 1d 43 83 00 1e 43 83 00 1f 43 83 00 20 43 83 00 21 43 83 00 22 43 +83 00 23 43 83 00 24 43 83 00 25 43 83 00 26 43 83 00 27 43 83 00 28 43 83 00 29 43 83 00 2a 43 +83 00 2b 43 83 00 2c 43 83 00 2d 43 83 00 2e 43 83 00 2f 43 83 00 30 43 83 00 31 43 83 00 32 43 +83 00 33 43 83 00 34 43 83 00 35 43 83 00 36 43 83 00 37 43 83 00 38 43 83 00 39 43 83 00 3a 43 +83 00 3b 43 83 00 3c 43 83 00 3d 43 83 00 3e 43 83 00 3f 43 83 00 40 43 83 00 41 43 83 00 42 43 +83 00 43 43 83 00 44 43 83 00 45 43 83 00 46 43 83 00 47 43 83 00 48 43 83 00 49 43 83 00 4a 43 +83 00 4b 43 83 00 4c 43 83 00 4d 43 83 00 4e 43 83 00 4f 43 83 00 50 43 83 00 51 43 83 00 52 43 +83 00 53 43 83 00 54 43 83 00 55 43 83 00 56 43 83 00 57 43 83 00 58 43 83 00 59 43 83 00 5a 43 +83 00 5b 43 83 00 5c 43 83 00 5d 43 83 00 5e 43 83 00 5f 43 83 00 60 43 83 00 61 43 83 00 62 43 +83 00 63 43 83 00 64 43 83 00 65 43 83 00 66 43 83 00 67 43 83 00 68 43 83 00 69 43 83 00 6a 43 +83 00 6b 43 83 00 6c 43 83 00 6d 43 83 00 6e 43 83 00 6f 43 83 00 70 43 83 00 71 43 83 00 72 43 +83 00 73 43 83 00 74 43 83 00 75 43 83 00 76 43 83 00 77 43 83 00 78 43 83 00 79 43 83 00 7a 43 +83 00 7b 43 83 00 7c 43 83 00 7d 43 83 00 7e 43 83 00 7f 43 a6 9b 44 3b 4e 62 80 3f 27 31 00 40 +27 31 40 40 93 18 80 40 93 18 a0 40 93 18 c0 40 93 18 e0 40 4a 0c 00 41 4a 0c 10 41 4a 0c 20 41 +4a 0c 30 41 4a 0c 40 41 4a 0c 50 41 4a 0c 60 41 4a 0c 70 41 25 06 80 41 25 06 88 41 25 06 90 41 +25 06 98 41 25 06 a0 41 25 06 a8 41 25 06 b0 41 25 06 b8 41 25 06 c0 41 25 06 c8 41 25 06 d0 41 +25 06 d8 41 25 06 e0 41 25 06 e8 41 25 06 f0 41 25 06 f8 41 12 03 00 42 12 03 04 42 12 03 08 42 +12 03 0c 42 12 03 10 42 12 03 14 42 12 03 18 42 12 03 1c 42 12 03 20 42 12 03 24 42 12 03 28 42 +12 03 2c 42 12 03 30 42 12 03 34 42 12 03 38 42 12 03 3c 42 12 03 40 42 12 03 44 42 12 03 48 42 +12 03 4c 42 12 03 50 42 12 03 54 42 12 03 58 42 12 03 5c 42 12 03 60 42 12 03 64 42 12 03 68 42 +12 03 6c 42 12 03 70 42 12 03 74 42 12 03 78 42 12 03 7c 42 89 01 80 42 89 01 82 42 89 01 84 42 +89 01 86 42 89 01 88 42 89 01 8a 42 89 01 8c 42 89 01 8e 42 89 01 90 42 89 01 92 42 89 01 94 42 +89 01 96 42 89 01 98 42 89 01 9a 42 89 01 9c 42 89 01 9e 42 89 01 a0 42 89 01 a2 42 89 01 a4 42 +89 01 a6 42 89 01 a8 42 89 01 aa 42 89 01 ac 42 89 01 ae 42 89 01 b0 42 89 01 b2 42 89 01 b4 42 +89 01 b6 42 89 01 b8 42 89 01 ba 42 89 01 bc 42 89 01 be 42 89 01 c0 42 89 01 c2 42 89 01 c4 42 +89 01 c6 42 89 01 c8 42 89 01 ca 42 89 01 cc 42 89 01 ce 42 89 01 d0 42 89 01 d2 42 89 01 d4 42 +89 01 d6 42 89 01 d8 42 89 01 da 42 89 01 dc 42 89 01 de 42 89 01 e0 42 89 01 e2 42 89 01 e4 42 +89 01 e6 42 89 01 e8 42 89 01 ea 42 89 01 ec 42 89 01 ee 42 89 01 f0 42 89 01 f2 42 89 01 f4 42 +89 01 f6 42 89 01 f8 42 89 01 fa 42 89 01 fc 42 89 01 fe 42 c5 00 00 43 c5 00 01 43 c5 00 02 43 +c5 00 03 43 c5 00 04 43 c5 00 05 43 c5 00 06 43 c5 00 07 43 c5 00 08 43 c5 00 09 43 c5 00 0a 43 +c5 00 0b 43 c5 00 0c 43 c5 00 0d 43 c5 00 0e 43 c5 00 0f 43 c5 00 10 43 c5 00 11 43 c5 00 12 43 +c5 00 13 43 c5 00 14 43 c5 00 15 43 c5 00 16 43 c5 00 17 43 c5 00 18 43 c5 00 19 43 c5 00 1a 43 +c5 00 1b 43 c5 00 1c 43 c5 00 1d 43 c5 00 1e 43 c5 00 1f 43 c5 00 20 43 c5 00 21 43 c5 00 22 43 +c5 00 23 43 c5 00 24 43 c5 00 25 43 c5 00 26 43 c5 00 27 43 c5 00 28 43 c5 00 29 43 c5 00 2a 43 +c5 00 2b 43 c5 00 2c 43 c5 00 2d 43 c5 00 2e 43 c5 00 2f 43 c5 00 30 43 c5 00 31 43 c5 00 32 43 +c5 00 33 43 c5 00 34 43 c5 00 35 43 c5 00 36 43 c5 00 37 43 c5 00 38 43 c5 00 39 43 c5 00 3a 43 +c5 00 3b 43 c5 00 3c 43 c5 00 3d 43 c5 00 3e 43 c5 00 3f 43 c5 00 40 43 c5 00 41 43 c5 00 42 43 +c5 00 43 43 c5 00 44 43 c5 00 45 43 c5 00 46 43 c5 00 47 43 c5 00 48 43 c5 00 49 43 c5 00 4a 43 +c5 00 4b 43 c5 00 4c 43 c5 00 4d 43 c5 00 4e 43 c5 00 4f 43 c5 00 50 43 c5 00 51 43 c5 00 52 43 +c5 00 53 43 c5 00 54 43 c5 00 55 43 c5 00 56 43 c5 00 57 43 c5 00 58 43 c5 00 59 43 c5 00 5a 43 +c5 00 5b 43 c5 00 5c 43 c5 00 5d 43 c5 00 5e 43 c5 00 5f 43 c5 00 60 43 c5 00 61 43 c5 00 62 43 +c5 00 63 43 c5 00 64 43 c5 00 65 43 c5 00 66 43 c5 00 67 43 c5 00 68 43 c5 00 69 43 c5 00 6a 43 +c5 00 6b 43 c5 00 6c 43 c5 00 6d 43 c5 00 6e 43 c5 00 6f 43 c5 00 70 43 c5 00 71 43 c5 00 72 43 +c5 00 73 43 c5 00 74 43 c5 00 75 43 c5 00 76 43 c5 00 77 43 c5 00 78 43 c5 00 79 43 c5 00 7a 43 +c5 00 7b 43 c5 00 7c 43 c5 00 7d 43 c5 00 7e 43 c5 00 7f 43 6f 12 83 3b 12 83 80 3f 89 41 00 40 +89 41 40 40 c5 20 80 40 c5 20 a0 40 c5 20 c0 40 c5 20 e0 40 62 10 00 41 62 10 10 41 62 10 20 41 +62 10 30 41 62 10 40 41 62 10 50 41 62 10 60 41 62 10 70 41 31 08 80 41 31 08 88 41 31 08 90 41 +31 08 98 41 31 08 a0 41 31 08 a8 41 31 08 b0 41 31 08 b8 41 31 08 c0 41 31 08 c8 41 31 08 d0 41 +31 08 d8 41 31 08 e0 41 31 08 e8 41 31 08 f0 41 31 08 f8 41 19 04 00 42 19 04 04 42 19 04 08 42 +19 04 0c 42 19 04 10 42 19 04 14 42 19 04 18 42 19 04 1c 42 19 04 20 42 19 04 24 42 19 04 28 42 +19 04 2c 42 19 04 30 42 19 04 34 42 19 04 38 42 19 04 3c 42 19 04 40 42 19 04 44 42 19 04 48 42 +19 04 4c 42 19 04 50 42 19 04 54 42 19 04 58 42 19 04 5c 42 19 04 60 42 19 04 64 42 19 04 68 42 +19 04 6c 42 19 04 70 42 19 04 74 42 19 04 78 42 19 04 7c 42 0c 02 80 42 0c 02 82 42 0c 02 84 42 +0c 02 86 42 0c 02 88 42 0c 02 8a 42 0c 02 8c 42 0c 02 8e 42 0c 02 90 42 0c 02 92 42 0c 02 94 42 +0c 02 96 42 0c 02 98 42 0c 02 9a 42 0c 02 9c 42 0c 02 9e 42 0c 02 a0 42 0c 02 a2 42 0c 02 a4 42 +0c 02 a6 42 0c 02 a8 42 0c 02 aa 42 0c 02 ac 42 0c 02 ae 42 0c 02 b0 42 0c 02 b2 42 0c 02 b4 42 +0c 02 b6 42 0c 02 b8 42 0c 02 ba 42 0c 02 bc 42 0c 02 be 42 0c 02 c0 42 0c 02 c2 42 0c 02 c4 42 +0c 02 c6 42 0c 02 c8 42 0c 02 ca 42 0c 02 cc 42 0c 02 ce 42 0c 02 d0 42 0c 02 d2 42 0c 02 d4 42 +0c 02 d6 42 0c 02 d8 42 0c 02 da 42 0c 02 dc 42 0c 02 de 42 0c 02 e0 42 0c 02 e2 42 0c 02 e4 42 +0c 02 e6 42 0c 02 e8 42 0c 02 ea 42 0c 02 ec 42 0c 02 ee 42 0c 02 f0 42 0c 02 f2 42 0c 02 f4 42 +0c 02 f6 42 0c 02 f8 42 0c 02 fa 42 0c 02 fc 42 0c 02 fe 42 06 01 00 43 06 01 01 43 06 01 02 43 +06 01 03 43 06 01 04 43 06 01 05 43 06 01 06 43 06 01 07 43 06 01 08 43 06 01 09 43 06 01 0a 43 +06 01 0b 43 06 01 0c 43 06 01 0d 43 06 01 0e 43 06 01 0f 43 06 01 10 43 06 01 11 43 06 01 12 43 +06 01 13 43 06 01 14 43 06 01 15 43 06 01 16 43 06 01 17 43 06 01 18 43 06 01 19 43 06 01 1a 43 +06 01 1b 43 06 01 1c 43 06 01 1d 43 06 01 1e 43 06 01 1f 43 06 01 20 43 06 01 21 43 06 01 22 43 +06 01 23 43 06 01 24 43 06 01 25 43 06 01 26 43 06 01 27 43 06 01 28 43 06 01 29 43 06 01 2a 43 +06 01 2b 43 06 01 2c 43 06 01 2d 43 06 01 2e 43 06 01 2f 43 06 01 30 43 06 01 31 43 06 01 32 43 +06 01 33 43 06 01 34 43 06 01 35 43 06 01 36 43 06 01 37 43 06 01 38 43 06 01 39 43 06 01 3a 43 +06 01 3b 43 06 01 3c 43 06 01 3d 43 06 01 3e 43 06 01 3f 43 06 01 40 43 06 01 41 43 06 01 42 43 +06 01 43 43 06 01 44 43 06 01 45 43 06 01 46 43 06 01 47 43 06 01 48 43 06 01 49 43 06 01 4a 43 +06 01 4b 43 06 01 4c 43 06 01 4d 43 06 01 4e 43 06 01 4f 43 06 01 50 43 06 01 51 43 06 01 52 43 +06 01 53 43 06 01 54 43 06 01 55 43 06 01 56 43 06 01 57 43 06 01 58 43 06 01 59 43 06 01 5a 43 +06 01 5b 43 06 01 5c 43 06 01 5d 43 06 01 5e 43 06 01 5f 43 06 01 60 43 06 01 61 43 06 01 62 43 +06 01 63 43 06 01 64 43 06 01 65 43 06 01 66 43 06 01 67 43 06 01 68 43 06 01 69 43 06 01 6a 43 +06 01 6b 43 06 01 6c 43 06 01 6d 43 06 01 6e 43 06 01 6f 43 06 01 70 43 06 01 71 43 06 01 72 43 +06 01 73 43 06 01 74 43 06 01 75 43 06 01 76 43 06 01 77 43 06 01 78 43 06 01 79 43 06 01 7a 43 +06 01 7b 43 06 01 7c 43 06 01 7d 43 06 01 7e 43 06 01 7f 43 0b d7 a3 3b d7 a3 80 3f ec 51 00 40 +ec 51 40 40 f6 28 80 40 f6 28 a0 40 f6 28 c0 40 f6 28 e0 40 7b 14 00 41 7b 14 10 41 7b 14 20 41 +7b 14 30 41 7b 14 40 41 7b 14 50 41 7b 14 60 41 7b 14 70 41 3d 0a 80 41 3d 0a 88 41 3d 0a 90 41 +3d 0a 98 41 3d 0a a0 41 3d 0a a8 41 3d 0a b0 41 3d 0a b8 41 3d 0a c0 41 3d 0a c8 41 3d 0a d0 41 +3d 0a d8 41 3d 0a e0 41 3d 0a e8 41 3d 0a f0 41 3d 0a f8 41 1f 05 00 42 1f 05 04 42 1f 05 08 42 +1f 05 0c 42 1f 05 10 42 1f 05 14 42 1f 05 18 42 1f 05 1c 42 1f 05 20 42 1f 05 24 42 1f 05 28 42 +1f 05 2c 42 1f 05 30 42 1f 05 34 42 1f 05 38 42 1f 05 3c 42 1f 05 40 42 1f 05 44 42 1f 05 48 42 +1f 05 4c 42 1f 05 50 42 1f 05 54 42 1f 05 58 42 1f 05 5c 42 1f 05 60 42 1f 05 64 42 1f 05 68 42 +1f 05 6c 42 1f 05 70 42 1f 05 74 42 1f 05 78 42 1f 05 7c 42 8f 02 80 42 8f 02 82 42 8f 02 84 42 +8f 02 86 42 8f 02 88 42 8f 02 8a 42 8f 02 8c 42 8f 02 8e 42 8f 02 90 42 8f 02 92 42 8f 02 94 42 +8f 02 96 42 8f 02 98 42 8f 02 9a 42 8f 02 9c 42 8f 02 9e 42 8f 02 a0 42 8f 02 a2 42 8f 02 a4 42 +8f 02 a6 42 8f 02 a8 42 8f 02 aa 42 8f 02 ac 42 8f 02 ae 42 8f 02 b0 42 8f 02 b2 42 8f 02 b4 42 +8f 02 b6 42 8f 02 b8 42 8f 02 ba 42 8f 02 bc 42 8f 02 be 42 8f 02 c0 42 8f 02 c2 42 8f 02 c4 42 +8f 02 c6 42 8f 02 c8 42 8f 02 ca 42 8f 02 cc 42 8f 02 ce 42 8f 02 d0 42 8f 02 d2 42 8f 02 d4 42 +8f 02 d6 42 8f 02 d8 42 8f 02 da 42 8f 02 dc 42 8f 02 de 42 8f 02 e0 42 8f 02 e2 42 8f 02 e4 42 +8f 02 e6 42 8f 02 e8 42 8f 02 ea 42 8f 02 ec 42 8f 02 ee 42 8f 02 f0 42 8f 02 f2 42 8f 02 f4 42 +8f 02 f6 42 8f 02 f8 42 8f 02 fa 42 8f 02 fc 42 8f 02 fe 42 48 01 00 43 48 01 01 43 48 01 02 43 +48 01 03 43 48 01 04 43 48 01 05 43 48 01 06 43 48 01 07 43 48 01 08 43 48 01 09 43 48 01 0a 43 +48 01 0b 43 48 01 0c 43 48 01 0d 43 48 01 0e 43 48 01 0f 43 48 01 10 43 48 01 11 43 48 01 12 43 +48 01 13 43 48 01 14 43 48 01 15 43 48 01 16 43 48 01 17 43 48 01 18 43 48 01 19 43 48 01 1a 43 +48 01 1b 43 48 01 1c 43 48 01 1d 43 48 01 1e 43 48 01 1f 43 48 01 20 43 48 01 21 43 48 01 22 43 +48 01 23 43 48 01 24 43 48 01 25 43 48 01 26 43 48 01 27 43 48 01 28 43 48 01 29 43 48 01 2a 43 +48 01 2b 43 48 01 2c 43 48 01 2d 43 48 01 2e 43 48 01 2f 43 48 01 30 43 48 01 31 43 48 01 32 43 +48 01 33 43 48 01 34 43 48 01 35 43 48 01 36 43 48 01 37 43 48 01 38 43 48 01 39 43 48 01 3a 43 +48 01 3b 43 48 01 3c 43 48 01 3d 43 48 01 3e 43 48 01 3f 43 48 01 40 43 48 01 41 43 48 01 42 43 +48 01 43 43 48 01 44 43 48 01 45 43 48 01 46 43 48 01 47 43 48 01 48 43 48 01 49 43 48 01 4a 43 +48 01 4b 43 48 01 4c 43 48 01 4d 43 48 01 4e 43 48 01 4f 43 48 01 50 43 48 01 51 43 48 01 52 43 +48 01 53 43 48 01 54 43 48 01 55 43 48 01 56 43 48 01 57 43 48 01 58 43 48 01 59 43 48 01 5a 43 +48 01 5b 43 48 01 5c 43 48 01 5d 43 48 01 5e 43 48 01 5f 43 48 01 60 43 48 01 61 43 48 01 62 43 +48 01 63 43 48 01 64 43 48 01 65 43 48 01 66 43 48 01 67 43 48 01 68 43 48 01 69 43 48 01 6a 43 +48 01 6b 43 48 01 6c 43 48 01 6d 43 48 01 6e 43 48 01 6f 43 48 01 70 43 48 01 71 43 48 01 72 43 +48 01 73 43 48 01 74 43 48 01 75 43 48 01 76 43 48 01 77 43 48 01 78 43 48 01 79 43 48 01 7a 43 +48 01 7b 43 48 01 7c 43 48 01 7d 43 48 01 7e 43 48 01 7f 43 a6 9b c4 3b 9c c4 80 3f 4e 62 00 40 +4e 62 40 40 27 31 80 40 27 31 a0 40 27 31 c0 40 27 31 e0 40 93 18 00 41 93 18 10 41 93 18 20 41 +93 18 30 41 93 18 40 41 93 18 50 41 93 18 60 41 93 18 70 41 4a 0c 80 41 4a 0c 88 41 4a 0c 90 41 +4a 0c 98 41 4a 0c a0 41 4a 0c a8 41 4a 0c b0 41 4a 0c b8 41 4a 0c c0 41 4a 0c c8 41 4a 0c d0 41 +4a 0c d8 41 4a 0c e0 41 4a 0c e8 41 4a 0c f0 41 4a 0c f8 41 25 06 00 42 25 06 04 42 25 06 08 42 +25 06 0c 42 25 06 10 42 25 06 14 42 25 06 18 42 25 06 1c 42 25 06 20 42 25 06 24 42 25 06 28 42 +25 06 2c 42 25 06 30 42 25 06 34 42 25 06 38 42 25 06 3c 42 25 06 40 42 25 06 44 42 25 06 48 42 +25 06 4c 42 25 06 50 42 25 06 54 42 25 06 58 42 25 06 5c 42 25 06 60 42 25 06 64 42 25 06 68 42 +25 06 6c 42 25 06 70 42 25 06 74 42 25 06 78 42 25 06 7c 42 12 03 80 42 12 03 82 42 12 03 84 42 +12 03 86 42 12 03 88 42 12 03 8a 42 12 03 8c 42 12 03 8e 42 12 03 90 42 12 03 92 42 12 03 94 42 +12 03 96 42 12 03 98 42 12 03 9a 42 12 03 9c 42 12 03 9e 42 12 03 a0 42 12 03 a2 42 12 03 a4 42 +12 03 a6 42 12 03 a8 42 12 03 aa 42 12 03 ac 42 12 03 ae 42 12 03 b0 42 12 03 b2 42 12 03 b4 42 +12 03 b6 42 12 03 b8 42 12 03 ba 42 12 03 bc 42 12 03 be 42 12 03 c0 42 12 03 c2 42 12 03 c4 42 +12 03 c6 42 12 03 c8 42 12 03 ca 42 12 03 cc 42 12 03 ce 42 12 03 d0 42 12 03 d2 42 12 03 d4 42 +12 03 d6 42 12 03 d8 42 12 03 da 42 12 03 dc 42 12 03 de 42 12 03 e0 42 12 03 e2 42 12 03 e4 42 +12 03 e6 42 12 03 e8 42 12 03 ea 42 12 03 ec 42 12 03 ee 42 12 03 f0 42 12 03 f2 42 12 03 f4 42 +12 03 f6 42 12 03 f8 42 12 03 fa 42 12 03 fc 42 12 03 fe 42 89 01 00 43 89 01 01 43 89 01 02 43 +89 01 03 43 89 01 04 43 89 01 05 43 89 01 06 43 89 01 07 43 89 01 08 43 89 01 09 43 89 01 0a 43 +89 01 0b 43 89 01 0c 43 89 01 0d 43 89 01 0e 43 89 01 0f 43 89 01 10 43 89 01 11 43 89 01 12 43 +89 01 13 43 89 01 14 43 89 01 15 43 89 01 16 43 89 01 17 43 89 01 18 43 89 01 19 43 89 01 1a 43 +89 01 1b 43 89 01 1c 43 89 01 1d 43 89 01 1e 43 89 01 1f 43 89 01 20 43 89 01 21 43 89 01 22 43 +89 01 23 43 89 01 24 43 89 01 25 43 89 01 26 43 89 01 27 43 89 01 28 43 89 01 29 43 89 01 2a 43 +89 01 2b 43 89 01 2c 43 89 01 2d 43 89 01 2e 43 89 01 2f 43 89 01 30 43 89 01 31 43 89 01 32 43 +89 01 33 43 89 01 34 43 89 01 35 43 89 01 36 43 89 01 37 43 89 01 38 43 89 01 39 43 89 01 3a 43 +89 01 3b 43 89 01 3c 43 89 01 3d 43 89 01 3e 43 89 01 3f 43 89 01 40 43 89 01 41 43 89 01 42 43 +89 01 43 43 89 01 44 43 89 01 45 43 89 01 46 43 89 01 47 43 89 01 48 43 89 01 49 43 89 01 4a 43 +89 01 4b 43 89 01 4c 43 89 01 4d 43 89 01 4e 43 89 01 4f 43 89 01 50 43 89 01 51 43 89 01 52 43 +89 01 53 43 89 01 54 43 89 01 55 43 89 01 56 43 89 01 57 43 89 01 58 43 89 01 59 43 89 01 5a 43 +89 01 5b 43 89 01 5c 43 89 01 5d 43 89 01 5e 43 89 01 5f 43 89 01 60 43 89 01 61 43 89 01 62 43 +89 01 63 43 89 01 64 43 89 01 65 43 89 01 66 43 89 01 67 43 89 01 68 43 89 01 69 43 89 01 6a 43 +89 01 6b 43 89 01 6c 43 89 01 6d 43 89 01 6e 43 89 01 6f 43 89 01 70 43 89 01 71 43 89 01 72 43 +89 01 73 43 89 01 74 43 89 01 75 43 89 01 76 43 89 01 77 43 89 01 78 43 89 01 79 43 89 01 7a 43 +89 01 7b 43 89 01 7c 43 89 01 7d 43 89 01 7e 43 89 01 7f 43 42 60 e5 3b 60 e5 80 3f b0 72 00 40 +b0 72 40 40 58 39 80 40 58 39 a0 40 58 39 c0 40 58 39 e0 40 ac 1c 00 41 ac 1c 10 41 ac 1c 20 41 +ac 1c 30 41 ac 1c 40 41 ac 1c 50 41 ac 1c 60 41 ac 1c 70 41 56 0e 80 41 56 0e 88 41 56 0e 90 41 +56 0e 98 41 56 0e a0 41 56 0e a8 41 56 0e b0 41 56 0e b8 41 56 0e c0 41 56 0e c8 41 56 0e d0 41 +56 0e d8 41 56 0e e0 41 56 0e e8 41 56 0e f0 41 56 0e f8 41 2b 07 00 42 2b 07 04 42 2b 07 08 42 +2b 07 0c 42 2b 07 10 42 2b 07 14 42 2b 07 18 42 2b 07 1c 42 2b 07 20 42 2b 07 24 42 2b 07 28 42 +2b 07 2c 42 2b 07 30 42 2b 07 34 42 2b 07 38 42 2b 07 3c 42 2b 07 40 42 2b 07 44 42 2b 07 48 42 +2b 07 4c 42 2b 07 50 42 2b 07 54 42 2b 07 58 42 2b 07 5c 42 2b 07 60 42 2b 07 64 42 2b 07 68 42 +2b 07 6c 42 2b 07 70 42 2b 07 74 42 2b 07 78 42 2b 07 7c 42 96 03 80 42 96 03 82 42 96 03 84 42 +96 03 86 42 96 03 88 42 96 03 8a 42 96 03 8c 42 96 03 8e 42 96 03 90 42 96 03 92 42 96 03 94 42 +96 03 96 42 96 03 98 42 96 03 9a 42 96 03 9c 42 96 03 9e 42 96 03 a0 42 96 03 a2 42 96 03 a4 42 +96 03 a6 42 96 03 a8 42 96 03 aa 42 96 03 ac 42 96 03 ae 42 96 03 b0 42 96 03 b2 42 96 03 b4 42 +96 03 b6 42 96 03 b8 42 96 03 ba 42 96 03 bc 42 96 03 be 42 96 03 c0 42 96 03 c2 42 96 03 c4 42 +96 03 c6 42 96 03 c8 42 96 03 ca 42 96 03 cc 42 96 03 ce 42 96 03 d0 42 96 03 d2 42 96 03 d4 42 +96 03 d6 42 96 03 d8 42 96 03 da 42 96 03 dc 42 96 03 de 42 96 03 e0 42 96 03 e2 42 96 03 e4 42 +96 03 e6 42 96 03 e8 42 96 03 ea 42 96 03 ec 42 96 03 ee 42 96 03 f0 42 96 03 f2 42 96 03 f4 42 +96 03 f6 42 96 03 f8 42 96 03 fa 42 96 03 fc 42 96 03 fe 42 cb 01 00 43 cb 01 01 43 cb 01 02 43 +cb 01 03 43 cb 01 04 43 cb 01 05 43 cb 01 06 43 cb 01 07 43 cb 01 08 43 cb 01 09 43 cb 01 0a 43 +cb 01 0b 43 cb 01 0c 43 cb 01 0d 43 cb 01 0e 43 cb 01 0f 43 cb 01 10 43 cb 01 11 43 cb 01 12 43 +cb 01 13 43 cb 01 14 43 cb 01 15 43 cb 01 16 43 cb 01 17 43 cb 01 18 43 cb 01 19 43 cb 01 1a 43 +cb 01 1b 43 cb 01 1c 43 cb 01 1d 43 cb 01 1e 43 cb 01 1f 43 cb 01 20 43 cb 01 21 43 cb 01 22 43 +cb 01 23 43 cb 01 24 43 cb 01 25 43 cb 01 26 43 cb 01 27 43 cb 01 28 43 cb 01 29 43 cb 01 2a 43 +cb 01 2b 43 cb 01 2c 43 cb 01 2d 43 cb 01 2e 43 cb 01 2f 43 cb 01 30 43 cb 01 31 43 cb 01 32 43 +cb 01 33 43 cb 01 34 43 cb 01 35 43 cb 01 36 43 cb 01 37 43 cb 01 38 43 cb 01 39 43 cb 01 3a 43 +cb 01 3b 43 cb 01 3c 43 cb 01 3d 43 cb 01 3e 43 cb 01 3f 43 cb 01 40 43 cb 01 41 43 cb 01 42 43 +cb 01 43 43 cb 01 44 43 cb 01 45 43 cb 01 46 43 cb 01 47 43 cb 01 48 43 cb 01 49 43 cb 01 4a 43 +cb 01 4b 43 cb 01 4c 43 cb 01 4d 43 cb 01 4e 43 cb 01 4f 43 cb 01 50 43 cb 01 51 43 cb 01 52 43 +cb 01 53 43 cb 01 54 43 cb 01 55 43 cb 01 56 43 cb 01 57 43 cb 01 58 43 cb 01 59 43 cb 01 5a 43 +cb 01 5b 43 cb 01 5c 43 cb 01 5d 43 cb 01 5e 43 cb 01 5f 43 cb 01 60 43 cb 01 61 43 cb 01 62 43 +cb 01 63 43 cb 01 64 43 cb 01 65 43 cb 01 66 43 cb 01 67 43 cb 01 68 43 cb 01 69 43 cb 01 6a 43 +cb 01 6b 43 cb 01 6c 43 cb 01 6d 43 cb 01 6e 43 cb 01 6f 43 cb 01 70 43 cb 01 71 43 cb 01 72 43 +cb 01 73 43 cb 01 74 43 cb 01 75 43 cb 01 76 43 cb 01 77 43 cb 01 78 43 cb 01 79 43 cb 01 7a 43 +cb 01 7b 43 cb 01 7c 43 cb 01 7d 43 cb 01 7e 43 cb 01 7f 43 6f 12 03 3c 25 06 81 3f 12 83 00 40 +12 83 40 40 89 41 80 40 89 41 a0 40 89 41 c0 40 89 41 e0 40 c5 20 00 41 c5 20 10 41 c5 20 20 41 +c5 20 30 41 c5 20 40 41 c5 20 50 41 c5 20 60 41 c5 20 70 41 62 10 80 41 62 10 88 41 62 10 90 41 +62 10 98 41 62 10 a0 41 62 10 a8 41 62 10 b0 41 62 10 b8 41 62 10 c0 41 62 10 c8 41 62 10 d0 41 +62 10 d8 41 62 10 e0 41 62 10 e8 41 62 10 f0 41 62 10 f8 41 31 08 00 42 31 08 04 42 31 08 08 42 +31 08 0c 42 31 08 10 42 31 08 14 42 31 08 18 42 31 08 1c 42 31 08 20 42 31 08 24 42 31 08 28 42 +31 08 2c 42 31 08 30 42 31 08 34 42 31 08 38 42 31 08 3c 42 31 08 40 42 31 08 44 42 31 08 48 42 +31 08 4c 42 31 08 50 42 31 08 54 42 31 08 58 42 31 08 5c 42 31 08 60 42 31 08 64 42 31 08 68 42 +31 08 6c 42 31 08 70 42 31 08 74 42 31 08 78 42 31 08 7c 42 19 04 80 42 19 04 82 42 19 04 84 42 +19 04 86 42 19 04 88 42 19 04 8a 42 19 04 8c 42 19 04 8e 42 19 04 90 42 19 04 92 42 19 04 94 42 +19 04 96 42 19 04 98 42 19 04 9a 42 19 04 9c 42 19 04 9e 42 19 04 a0 42 19 04 a2 42 19 04 a4 42 +19 04 a6 42 19 04 a8 42 19 04 aa 42 19 04 ac 42 19 04 ae 42 19 04 b0 42 19 04 b2 42 19 04 b4 42 +19 04 b6 42 19 04 b8 42 19 04 ba 42 19 04 bc 42 19 04 be 42 19 04 c0 42 19 04 c2 42 19 04 c4 42 +19 04 c6 42 19 04 c8 42 19 04 ca 42 19 04 cc 42 19 04 ce 42 19 04 d0 42 19 04 d2 42 19 04 d4 42 +19 04 d6 42 19 04 d8 42 19 04 da 42 19 04 dc 42 19 04 de 42 19 04 e0 42 19 04 e2 42 19 04 e4 42 +19 04 e6 42 19 04 e8 42 19 04 ea 42 19 04 ec 42 19 04 ee 42 19 04 f0 42 19 04 f2 42 19 04 f4 42 +19 04 f6 42 19 04 f8 42 19 04 fa 42 19 04 fc 42 19 04 fe 42 0c 02 00 43 0c 02 01 43 0c 02 02 43 +0c 02 03 43 0c 02 04 43 0c 02 05 43 0c 02 06 43 0c 02 07 43 0c 02 08 43 0c 02 09 43 0c 02 0a 43 +0c 02 0b 43 0c 02 0c 43 0c 02 0d 43 0c 02 0e 43 0c 02 0f 43 0c 02 10 43 0c 02 11 43 0c 02 12 43 +0c 02 13 43 0c 02 14 43 0c 02 15 43 0c 02 16 43 0c 02 17 43 0c 02 18 43 0c 02 19 43 0c 02 1a 43 +0c 02 1b 43 0c 02 1c 43 0c 02 1d 43 0c 02 1e 43 0c 02 1f 43 0c 02 20 43 0c 02 21 43 0c 02 22 43 +0c 02 23 43 0c 02 24 43 0c 02 25 43 0c 02 26 43 0c 02 27 43 0c 02 28 43 0c 02 29 43 0c 02 2a 43 +0c 02 2b 43 0c 02 2c 43 0c 02 2d 43 0c 02 2e 43 0c 02 2f 43 0c 02 30 43 0c 02 31 43 0c 02 32 43 +0c 02 33 43 0c 02 34 43 0c 02 35 43 0c 02 36 43 0c 02 37 43 0c 02 38 43 0c 02 39 43 0c 02 3a 43 +0c 02 3b 43 0c 02 3c 43 0c 02 3d 43 0c 02 3e 43 0c 02 3f 43 0c 02 40 43 0c 02 41 43 0c 02 42 43 +0c 02 43 43 0c 02 44 43 0c 02 45 43 0c 02 46 43 0c 02 47 43 0c 02 48 43 0c 02 49 43 0c 02 4a 43 +0c 02 4b 43 0c 02 4c 43 0c 02 4d 43 0c 02 4e 43 0c 02 4f 43 0c 02 50 43 0c 02 51 43 0c 02 52 43 +0c 02 53 43 0c 02 54 43 0c 02 55 43 0c 02 56 43 0c 02 57 43 0c 02 58 43 0c 02 59 43 0c 02 5a 43 +0c 02 5b 43 0c 02 5c 43 0c 02 5d 43 0c 02 5e 43 0c 02 5f 43 0c 02 60 43 0c 02 61 43 0c 02 62 43 +0c 02 63 43 0c 02 64 43 0c 02 65 43 0c 02 66 43 0c 02 67 43 0c 02 68 43 0c 02 69 43 0c 02 6a 43 +0c 02 6b 43 0c 02 6c 43 0c 02 6d 43 0c 02 6e 43 0c 02 6f 43 0c 02 70 43 0c 02 71 43 0c 02 72 43 +0c 02 73 43 0c 02 74 43 0c 02 75 43 0c 02 76 43 0c 02 77 43 0c 02 78 43 0c 02 79 43 0c 02 7a 43 +0c 02 7b 43 0c 02 7c 43 0c 02 7d 43 0c 02 7e 43 0c 02 7f 43 bd 74 13 3c e9 26 81 3f 75 93 00 40 +75 93 40 40 ba 49 80 40 ba 49 a0 40 ba 49 c0 40 ba 49 e0 40 dd 24 00 41 dd 24 10 41 dd 24 20 41 +dd 24 30 41 dd 24 40 41 dd 24 50 41 dd 24 60 41 dd 24 70 41 6f 12 80 41 6f 12 88 41 6f 12 90 41 +6f 12 98 41 6f 12 a0 41 6f 12 a8 41 6f 12 b0 41 6f 12 b8 41 6f 12 c0 41 6f 12 c8 41 6f 12 d0 41 +6f 12 d8 41 6f 12 e0 41 6f 12 e8 41 6f 12 f0 41 6f 12 f8 41 37 09 00 42 37 09 04 42 37 09 08 42 +37 09 0c 42 37 09 10 42 37 09 14 42 37 09 18 42 37 09 1c 42 37 09 20 42 37 09 24 42 37 09 28 42 +37 09 2c 42 37 09 30 42 37 09 34 42 37 09 38 42 37 09 3c 42 37 09 40 42 37 09 44 42 37 09 48 42 +37 09 4c 42 37 09 50 42 37 09 54 42 37 09 58 42 37 09 5c 42 37 09 60 42 37 09 64 42 37 09 68 42 +37 09 6c 42 37 09 70 42 37 09 74 42 37 09 78 42 37 09 7c 42 9c 04 80 42 9c 04 82 42 9c 04 84 42 +9c 04 86 42 9c 04 88 42 9c 04 8a 42 9c 04 8c 42 9c 04 8e 42 9c 04 90 42 9c 04 92 42 9c 04 94 42 +9c 04 96 42 9c 04 98 42 9c 04 9a 42 9c 04 9c 42 9c 04 9e 42 9c 04 a0 42 9c 04 a2 42 9c 04 a4 42 +9c 04 a6 42 9c 04 a8 42 9c 04 aa 42 9c 04 ac 42 9c 04 ae 42 9c 04 b0 42 9c 04 b2 42 9c 04 b4 42 +9c 04 b6 42 9c 04 b8 42 9c 04 ba 42 9c 04 bc 42 9c 04 be 42 9c 04 c0 42 9c 04 c2 42 9c 04 c4 42 +9c 04 c6 42 9c 04 c8 42 9c 04 ca 42 9c 04 cc 42 9c 04 ce 42 9c 04 d0 42 9c 04 d2 42 9c 04 d4 42 +9c 04 d6 42 9c 04 d8 42 9c 04 da 42 9c 04 dc 42 9c 04 de 42 9c 04 e0 42 9c 04 e2 42 9c 04 e4 42 +9c 04 e6 42 9c 04 e8 42 9c 04 ea 42 9c 04 ec 42 9c 04 ee 42 9c 04 f0 42 9c 04 f2 42 9c 04 f4 42 +9c 04 f6 42 9c 04 f8 42 9c 04 fa 42 9c 04 fc 42 9c 04 fe 42 4e 02 00 43 4e 02 01 43 4e 02 02 43 +4e 02 03 43 4e 02 04 43 4e 02 05 43 4e 02 06 43 4e 02 07 43 4e 02 08 43 4e 02 09 43 4e 02 0a 43 +4e 02 0b 43 4e 02 0c 43 4e 02 0d 43 4e 02 0e 43 4e 02 0f 43 4e 02 10 43 4e 02 11 43 4e 02 12 43 +4e 02 13 43 4e 02 14 43 4e 02 15 43 4e 02 16 43 4e 02 17 43 4e 02 18 43 4e 02 19 43 4e 02 1a 43 +4e 02 1b 43 4e 02 1c 43 4e 02 1d 43 4e 02 1e 43 4e 02 1f 43 4e 02 20 43 4e 02 21 43 4e 02 22 43 +4e 02 23 43 4e 02 24 43 4e 02 25 43 4e 02 26 43 4e 02 27 43 4e 02 28 43 4e 02 29 43 4e 02 2a 43 +4e 02 2b 43 4e 02 2c 43 4e 02 2d 43 4e 02 2e 43 4e 02 2f 43 4e 02 30 43 4e 02 31 43 4e 02 32 43 +4e 02 33 43 4e 02 34 43 4e 02 35 43 4e 02 36 43 4e 02 37 43 4e 02 38 43 4e 02 39 43 4e 02 3a 43 +4e 02 3b 43 4e 02 3c 43 4e 02 3d 43 4e 02 3e 43 4e 02 3f 43 4e 02 40 43 4e 02 41 43 4e 02 42 43 +4e 02 43 43 4e 02 44 43 4e 02 45 43 4e 02 46 43 4e 02 47 43 4e 02 48 43 4e 02 49 43 4e 02 4a 43 +4e 02 4b 43 4e 02 4c 43 4e 02 4d 43 4e 02 4e 43 4e 02 4f 43 4e 02 50 43 4e 02 51 43 4e 02 52 43 +4e 02 53 43 4e 02 54 43 4e 02 55 43 4e 02 56 43 4e 02 57 43 4e 02 58 43 4e 02 59 43 4e 02 5a 43 +4e 02 5b 43 4e 02 5c 43 4e 02 5d 43 4e 02 5e 43 4e 02 5f 43 4e 02 60 43 4e 02 61 43 4e 02 62 43 +4e 02 63 43 4e 02 64 43 4e 02 65 43 4e 02 66 43 4e 02 67 43 4e 02 68 43 4e 02 69 43 4e 02 6a 43 +4e 02 6b 43 4e 02 6c 43 4e 02 6d 43 4e 02 6e 43 4e 02 6f 43 4e 02 70 43 4e 02 71 43 4e 02 72 43 +4e 02 73 43 4e 02 74 43 4e 02 75 43 4e 02 76 43 4e 02 77 43 4e 02 78 43 4e 02 79 43 4e 02 7a 43 +4e 02 7b 43 4e 02 7c 43 4e 02 7d 43 4e 02 7e 43 4e 02 7f 43 0b d7 23 3c ae 47 81 3f d7 a3 00 40 +d7 a3 40 40 ec 51 80 40 ec 51 a0 40 ec 51 c0 40 ec 51 e0 40 f6 28 00 41 f6 28 10 41 f6 28 20 41 +f6 28 30 41 f6 28 40 41 f6 28 50 41 f6 28 60 41 f6 28 70 41 7b 14 80 41 7b 14 88 41 7b 14 90 41 +7b 14 98 41 7b 14 a0 41 7b 14 a8 41 7b 14 b0 41 7b 14 b8 41 7b 14 c0 41 7b 14 c8 41 7b 14 d0 41 +7b 14 d8 41 7b 14 e0 41 7b 14 e8 41 7b 14 f0 41 7b 14 f8 41 3d 0a 00 42 3d 0a 04 42 3d 0a 08 42 +3d 0a 0c 42 3d 0a 10 42 3d 0a 14 42 3d 0a 18 42 3d 0a 1c 42 3d 0a 20 42 3d 0a 24 42 3d 0a 28 42 +3d 0a 2c 42 3d 0a 30 42 3d 0a 34 42 3d 0a 38 42 3d 0a 3c 42 3d 0a 40 42 3d 0a 44 42 3d 0a 48 42 +3d 0a 4c 42 3d 0a 50 42 3d 0a 54 42 3d 0a 58 42 3d 0a 5c 42 3d 0a 60 42 3d 0a 64 42 3d 0a 68 42 +3d 0a 6c 42 3d 0a 70 42 3d 0a 74 42 3d 0a 78 42 3d 0a 7c 42 1f 05 80 42 1f 05 82 42 1f 05 84 42 +1f 05 86 42 1f 05 88 42 1f 05 8a 42 1f 05 8c 42 1f 05 8e 42 1f 05 90 42 1f 05 92 42 1f 05 94 42 +1f 05 96 42 1f 05 98 42 1f 05 9a 42 1f 05 9c 42 1f 05 9e 42 1f 05 a0 42 1f 05 a2 42 1f 05 a4 42 +1f 05 a6 42 1f 05 a8 42 1f 05 aa 42 1f 05 ac 42 1f 05 ae 42 1f 05 b0 42 1f 05 b2 42 1f 05 b4 42 +1f 05 b6 42 1f 05 b8 42 1f 05 ba 42 1f 05 bc 42 1f 05 be 42 1f 05 c0 42 1f 05 c2 42 1f 05 c4 42 +1f 05 c6 42 1f 05 c8 42 1f 05 ca 42 1f 05 cc 42 1f 05 ce 42 1f 05 d0 42 1f 05 d2 42 1f 05 d4 42 +1f 05 d6 42 1f 05 d8 42 1f 05 da 42 1f 05 dc 42 1f 05 de 42 1f 05 e0 42 1f 05 e2 42 1f 05 e4 42 +1f 05 e6 42 1f 05 e8 42 1f 05 ea 42 1f 05 ec 42 1f 05 ee 42 1f 05 f0 42 1f 05 f2 42 1f 05 f4 42 +1f 05 f6 42 1f 05 f8 42 1f 05 fa 42 1f 05 fc 42 1f 05 fe 42 8f 02 00 43 8f 02 01 43 8f 02 02 43 +8f 02 03 43 8f 02 04 43 8f 02 05 43 8f 02 06 43 8f 02 07 43 8f 02 08 43 8f 02 09 43 8f 02 0a 43 +8f 02 0b 43 8f 02 0c 43 8f 02 0d 43 8f 02 0e 43 8f 02 0f 43 8f 02 10 43 8f 02 11 43 8f 02 12 43 +8f 02 13 43 8f 02 14 43 8f 02 15 43 8f 02 16 43 8f 02 17 43 8f 02 18 43 8f 02 19 43 8f 02 1a 43 +8f 02 1b 43 8f 02 1c 43 8f 02 1d 43 8f 02 1e 43 8f 02 1f 43 8f 02 20 43 8f 02 21 43 8f 02 22 43 +8f 02 23 43 8f 02 24 43 8f 02 25 43 8f 02 26 43 8f 02 27 43 8f 02 28 43 8f 02 29 43 8f 02 2a 43 +8f 02 2b 43 8f 02 2c 43 8f 02 2d 43 8f 02 2e 43 8f 02 2f 43 8f 02 30 43 8f 02 31 43 8f 02 32 43 +8f 02 33 43 8f 02 34 43 8f 02 35 43 8f 02 36 43 8f 02 37 43 8f 02 38 43 8f 02 39 43 8f 02 3a 43 +8f 02 3b 43 8f 02 3c 43 8f 02 3d 43 8f 02 3e 43 8f 02 3f 43 8f 02 40 43 8f 02 41 43 8f 02 42 43 +8f 02 43 43 8f 02 44 43 8f 02 45 43 8f 02 46 43 8f 02 47 43 8f 02 48 43 8f 02 49 43 8f 02 4a 43 +8f 02 4b 43 8f 02 4c 43 8f 02 4d 43 8f 02 4e 43 8f 02 4f 43 8f 02 50 43 8f 02 51 43 8f 02 52 43 +8f 02 53 43 8f 02 54 43 8f 02 55 43 8f 02 56 43 8f 02 57 43 8f 02 58 43 8f 02 59 43 8f 02 5a 43 +8f 02 5b 43 8f 02 5c 43 8f 02 5d 43 8f 02 5e 43 8f 02 5f 43 8f 02 60 43 8f 02 61 43 8f 02 62 43 +8f 02 63 43 8f 02 64 43 8f 02 65 43 8f 02 66 43 8f 02 67 43 8f 02 68 43 8f 02 69 43 8f 02 6a 43 +8f 02 6b 43 8f 02 6c 43 8f 02 6d 43 8f 02 6e 43 8f 02 6f 43 8f 02 70 43 8f 02 71 43 8f 02 72 43 +8f 02 73 43 8f 02 74 43 8f 02 75 43 8f 02 76 43 8f 02 77 43 8f 02 78 43 8f 02 79 43 8f 02 7a 43 +8f 02 7b 43 8f 02 7c 43 8f 02 7d 43 8f 02 7e 43 8f 02 7f 43 59 39 34 3c 73 68 81 3f 39 b4 00 40 +39 b4 40 40 1d 5a 80 40 1d 5a a0 40 1d 5a c0 40 1d 5a e0 40 0e 2d 00 41 0e 2d 10 41 0e 2d 20 41 +0e 2d 30 41 0e 2d 40 41 0e 2d 50 41 0e 2d 60 41 0e 2d 70 41 87 16 80 41 87 16 88 41 87 16 90 41 +87 16 98 41 87 16 a0 41 87 16 a8 41 87 16 b0 41 87 16 b8 41 87 16 c0 41 87 16 c8 41 87 16 d0 41 +87 16 d8 41 87 16 e0 41 87 16 e8 41 87 16 f0 41 87 16 f8 41 44 0b 00 42 44 0b 04 42 44 0b 08 42 +44 0b 0c 42 44 0b 10 42 44 0b 14 42 44 0b 18 42 44 0b 1c 42 44 0b 20 42 44 0b 24 42 44 0b 28 42 +44 0b 2c 42 44 0b 30 42 44 0b 34 42 44 0b 38 42 44 0b 3c 42 44 0b 40 42 44 0b 44 42 44 0b 48 42 +44 0b 4c 42 44 0b 50 42 44 0b 54 42 44 0b 58 42 44 0b 5c 42 44 0b 60 42 44 0b 64 42 44 0b 68 42 +44 0b 6c 42 44 0b 70 42 44 0b 74 42 44 0b 78 42 44 0b 7c 42 a2 05 80 42 a2 05 82 42 a2 05 84 42 +a2 05 86 42 a2 05 88 42 a2 05 8a 42 a2 05 8c 42 a2 05 8e 42 a2 05 90 42 a2 05 92 42 a2 05 94 42 +a2 05 96 42 a2 05 98 42 a2 05 9a 42 a2 05 9c 42 a2 05 9e 42 a2 05 a0 42 a2 05 a2 42 a2 05 a4 42 +a2 05 a6 42 a2 05 a8 42 a2 05 aa 42 a2 05 ac 42 a2 05 ae 42 a2 05 b0 42 a2 05 b2 42 a2 05 b4 42 +a2 05 b6 42 a2 05 b8 42 a2 05 ba 42 a2 05 bc 42 a2 05 be 42 a2 05 c0 42 a2 05 c2 42 a2 05 c4 42 +a2 05 c6 42 a2 05 c8 42 a2 05 ca 42 a2 05 cc 42 a2 05 ce 42 a2 05 d0 42 a2 05 d2 42 a2 05 d4 42 +a2 05 d6 42 a2 05 d8 42 a2 05 da 42 a2 05 dc 42 a2 05 de 42 a2 05 e0 42 a2 05 e2 42 a2 05 e4 42 +a2 05 e6 42 a2 05 e8 42 a2 05 ea 42 a2 05 ec 42 a2 05 ee 42 a2 05 f0 42 a2 05 f2 42 a2 05 f4 42 +a2 05 f6 42 a2 05 f8 42 a2 05 fa 42 a2 05 fc 42 a2 05 fe 42 d1 02 00 43 d1 02 01 43 d1 02 02 43 +d1 02 03 43 d1 02 04 43 d1 02 05 43 d1 02 06 43 d1 02 07 43 d1 02 08 43 d1 02 09 43 d1 02 0a 43 +d1 02 0b 43 d1 02 0c 43 d1 02 0d 43 d1 02 0e 43 d1 02 0f 43 d1 02 10 43 d1 02 11 43 d1 02 12 43 +d1 02 13 43 d1 02 14 43 d1 02 15 43 d1 02 16 43 d1 02 17 43 d1 02 18 43 d1 02 19 43 d1 02 1a 43 +d1 02 1b 43 d1 02 1c 43 d1 02 1d 43 d1 02 1e 43 d1 02 1f 43 d1 02 20 43 d1 02 21 43 d1 02 22 43 +d1 02 23 43 d1 02 24 43 d1 02 25 43 d1 02 26 43 d1 02 27 43 d1 02 28 43 d1 02 29 43 d1 02 2a 43 +d1 02 2b 43 d1 02 2c 43 d1 02 2d 43 d1 02 2e 43 d1 02 2f 43 d1 02 30 43 d1 02 31 43 d1 02 32 43 +d1 02 33 43 d1 02 34 43 d1 02 35 43 d1 02 36 43 d1 02 37 43 d1 02 38 43 d1 02 39 43 d1 02 3a 43 +d1 02 3b 43 d1 02 3c 43 d1 02 3d 43 d1 02 3e 43 d1 02 3f 43 d1 02 40 43 d1 02 41 43 d1 02 42 43 +d1 02 43 43 d1 02 44 43 d1 02 45 43 d1 02 46 43 d1 02 47 43 d1 02 48 43 d1 02 49 43 d1 02 4a 43 +d1 02 4b 43 d1 02 4c 43 d1 02 4d 43 d1 02 4e 43 d1 02 4f 43 d1 02 50 43 d1 02 51 43 d1 02 52 43 +d1 02 53 43 d1 02 54 43 d1 02 55 43 d1 02 56 43 d1 02 57 43 d1 02 58 43 d1 02 59 43 d1 02 5a 43 +d1 02 5b 43 d1 02 5c 43 d1 02 5d 43 d1 02 5e 43 d1 02 5f 43 d1 02 60 43 d1 02 61 43 d1 02 62 43 +d1 02 63 43 d1 02 64 43 d1 02 65 43 d1 02 66 43 d1 02 67 43 d1 02 68 43 d1 02 69 43 d1 02 6a 43 +d1 02 6b 43 d1 02 6c 43 d1 02 6d 43 d1 02 6e 43 d1 02 6f 43 d1 02 70 43 d1 02 71 43 d1 02 72 43 +d1 02 73 43 d1 02 74 43 d1 02 75 43 d1 02 76 43 d1 02 77 43 d1 02 78 43 d1 02 79 43 d1 02 7a 43 +d1 02 7b 43 d1 02 7c 43 d1 02 7d 43 d1 02 7e 43 d1 02 7f 43 a6 9b 44 3c 37 89 81 3f 9c c4 00 40 +9c c4 40 40 4e 62 80 40 4e 62 a0 40 4e 62 c0 40 4e 62 e0 40 27 31 00 41 27 31 10 41 27 31 20 41 +27 31 30 41 27 31 40 41 27 31 50 41 27 31 60 41 27 31 70 41 93 18 80 41 93 18 88 41 93 18 90 41 +93 18 98 41 93 18 a0 41 93 18 a8 41 93 18 b0 41 93 18 b8 41 93 18 c0 41 93 18 c8 41 93 18 d0 41 +93 18 d8 41 93 18 e0 41 93 18 e8 41 93 18 f0 41 93 18 f8 41 4a 0c 00 42 4a 0c 04 42 4a 0c 08 42 +4a 0c 0c 42 4a 0c 10 42 4a 0c 14 42 4a 0c 18 42 4a 0c 1c 42 4a 0c 20 42 4a 0c 24 42 4a 0c 28 42 +4a 0c 2c 42 4a 0c 30 42 4a 0c 34 42 4a 0c 38 42 4a 0c 3c 42 4a 0c 40 42 4a 0c 44 42 4a 0c 48 42 +4a 0c 4c 42 4a 0c 50 42 4a 0c 54 42 4a 0c 58 42 4a 0c 5c 42 4a 0c 60 42 4a 0c 64 42 4a 0c 68 42 +4a 0c 6c 42 4a 0c 70 42 4a 0c 74 42 4a 0c 78 42 4a 0c 7c 42 25 06 80 42 25 06 82 42 25 06 84 42 +25 06 86 42 25 06 88 42 25 06 8a 42 25 06 8c 42 25 06 8e 42 25 06 90 42 25 06 92 42 25 06 94 42 +25 06 96 42 25 06 98 42 25 06 9a 42 25 06 9c 42 25 06 9e 42 25 06 a0 42 25 06 a2 42 25 06 a4 42 +25 06 a6 42 25 06 a8 42 25 06 aa 42 25 06 ac 42 25 06 ae 42 25 06 b0 42 25 06 b2 42 25 06 b4 42 +25 06 b6 42 25 06 b8 42 25 06 ba 42 25 06 bc 42 25 06 be 42 25 06 c0 42 25 06 c2 42 25 06 c4 42 +25 06 c6 42 25 06 c8 42 25 06 ca 42 25 06 cc 42 25 06 ce 42 25 06 d0 42 25 06 d2 42 25 06 d4 42 +25 06 d6 42 25 06 d8 42 25 06 da 42 25 06 dc 42 25 06 de 42 25 06 e0 42 25 06 e2 42 25 06 e4 42 +25 06 e6 42 25 06 e8 42 25 06 ea 42 25 06 ec 42 25 06 ee 42 25 06 f0 42 25 06 f2 42 25 06 f4 42 +25 06 f6 42 25 06 f8 42 25 06 fa 42 25 06 fc 42 25 06 fe 42 12 03 00 43 12 03 01 43 12 03 02 43 +12 03 03 43 12 03 04 43 12 03 05 43 12 03 06 43 12 03 07 43 12 03 08 43 12 03 09 43 12 03 0a 43 +12 03 0b 43 12 03 0c 43 12 03 0d 43 12 03 0e 43 12 03 0f 43 12 03 10 43 12 03 11 43 12 03 12 43 +12 03 13 43 12 03 14 43 12 03 15 43 12 03 16 43 12 03 17 43 12 03 18 43 12 03 19 43 12 03 1a 43 +12 03 1b 43 12 03 1c 43 12 03 1d 43 12 03 1e 43 12 03 1f 43 12 03 20 43 12 03 21 43 12 03 22 43 +12 03 23 43 12 03 24 43 12 03 25 43 12 03 26 43 12 03 27 43 12 03 28 43 12 03 29 43 12 03 2a 43 +12 03 2b 43 12 03 2c 43 12 03 2d 43 12 03 2e 43 12 03 2f 43 12 03 30 43 12 03 31 43 12 03 32 43 +12 03 33 43 12 03 34 43 12 03 35 43 12 03 36 43 12 03 37 43 12 03 38 43 12 03 39 43 12 03 3a 43 +12 03 3b 43 12 03 3c 43 12 03 3d 43 12 03 3e 43 12 03 3f 43 12 03 40 43 12 03 41 43 12 03 42 43 +12 03 43 43 12 03 44 43 12 03 45 43 12 03 46 43 12 03 47 43 12 03 48 43 12 03 49 43 12 03 4a 43 +12 03 4b 43 12 03 4c 43 12 03 4d 43 12 03 4e 43 12 03 4f 43 12 03 50 43 12 03 51 43 12 03 52 43 +12 03 53 43 12 03 54 43 12 03 55 43 12 03 56 43 12 03 57 43 12 03 58 43 12 03 59 43 12 03 5a 43 +12 03 5b 43 12 03 5c 43 12 03 5d 43 12 03 5e 43 12 03 5f 43 12 03 60 43 12 03 61 43 12 03 62 43 +12 03 63 43 12 03 64 43 12 03 65 43 12 03 66 43 12 03 67 43 12 03 68 43 12 03 69 43 12 03 6a 43 +12 03 6b 43 12 03 6c 43 12 03 6d 43 12 03 6e 43 12 03 6f 43 12 03 70 43 12 03 71 43 12 03 72 43 +12 03 73 43 12 03 74 43 12 03 75 43 12 03 76 43 12 03 77 43 12 03 78 43 12 03 79 43 12 03 7a 43 +12 03 7b 43 12 03 7c 43 12 03 7d 43 12 03 7e 43 12 03 7f 43 f4 fd 54 3c fc a9 81 3f fe d4 00 40 +fe d4 40 40 7f 6a 80 40 7f 6a a0 40 7f 6a c0 40 7f 6a e0 40 3f 35 00 41 3f 35 10 41 3f 35 20 41 +3f 35 30 41 3f 35 40 41 3f 35 50 41 3f 35 60 41 3f 35 70 41 a0 1a 80 41 a0 1a 88 41 a0 1a 90 41 +a0 1a 98 41 a0 1a a0 41 a0 1a a8 41 a0 1a b0 41 a0 1a b8 41 a0 1a c0 41 a0 1a c8 41 a0 1a d0 41 +a0 1a d8 41 a0 1a e0 41 a0 1a e8 41 a0 1a f0 41 a0 1a f8 41 50 0d 00 42 50 0d 04 42 50 0d 08 42 +50 0d 0c 42 50 0d 10 42 50 0d 14 42 50 0d 18 42 50 0d 1c 42 50 0d 20 42 50 0d 24 42 50 0d 28 42 +50 0d 2c 42 50 0d 30 42 50 0d 34 42 50 0d 38 42 50 0d 3c 42 50 0d 40 42 50 0d 44 42 50 0d 48 42 +50 0d 4c 42 50 0d 50 42 50 0d 54 42 50 0d 58 42 50 0d 5c 42 50 0d 60 42 50 0d 64 42 50 0d 68 42 +50 0d 6c 42 50 0d 70 42 50 0d 74 42 50 0d 78 42 50 0d 7c 42 a8 06 80 42 a8 06 82 42 a8 06 84 42 +a8 06 86 42 a8 06 88 42 a8 06 8a 42 a8 06 8c 42 a8 06 8e 42 a8 06 90 42 a8 06 92 42 a8 06 94 42 +a8 06 96 42 a8 06 98 42 a8 06 9a 42 a8 06 9c 42 a8 06 9e 42 a8 06 a0 42 a8 06 a2 42 a8 06 a4 42 +a8 06 a6 42 a8 06 a8 42 a8 06 aa 42 a8 06 ac 42 a8 06 ae 42 a8 06 b0 42 a8 06 b2 42 a8 06 b4 42 +a8 06 b6 42 a8 06 b8 42 a8 06 ba 42 a8 06 bc 42 a8 06 be 42 a8 06 c0 42 a8 06 c2 42 a8 06 c4 42 +a8 06 c6 42 a8 06 c8 42 a8 06 ca 42 a8 06 cc 42 a8 06 ce 42 a8 06 d0 42 a8 06 d2 42 a8 06 d4 42 +a8 06 d6 42 a8 06 d8 42 a8 06 da 42 a8 06 dc 42 a8 06 de 42 a8 06 e0 42 a8 06 e2 42 a8 06 e4 42 +a8 06 e6 42 a8 06 e8 42 a8 06 ea 42 a8 06 ec 42 a8 06 ee 42 a8 06 f0 42 a8 06 f2 42 a8 06 f4 42 +a8 06 f6 42 a8 06 f8 42 a8 06 fa 42 a8 06 fc 42 a8 06 fe 42 54 03 00 43 54 03 01 43 54 03 02 43 +54 03 03 43 54 03 04 43 54 03 05 43 54 03 06 43 54 03 07 43 54 03 08 43 54 03 09 43 54 03 0a 43 +54 03 0b 43 54 03 0c 43 54 03 0d 43 54 03 0e 43 54 03 0f 43 54 03 10 43 54 03 11 43 54 03 12 43 +54 03 13 43 54 03 14 43 54 03 15 43 54 03 16 43 54 03 17 43 54 03 18 43 54 03 19 43 54 03 1a 43 +54 03 1b 43 54 03 1c 43 54 03 1d 43 54 03 1e 43 54 03 1f 43 54 03 20 43 54 03 21 43 54 03 22 43 +54 03 23 43 54 03 24 43 54 03 25 43 54 03 26 43 54 03 27 43 54 03 28 43 54 03 29 43 54 03 2a 43 +54 03 2b 43 54 03 2c 43 54 03 2d 43 54 03 2e 43 54 03 2f 43 54 03 30 43 54 03 31 43 54 03 32 43 +54 03 33 43 54 03 34 43 54 03 35 43 54 03 36 43 54 03 37 43 54 03 38 43 54 03 39 43 54 03 3a 43 +54 03 3b 43 54 03 3c 43 54 03 3d 43 54 03 3e 43 54 03 3f 43 54 03 40 43 54 03 41 43 54 03 42 43 +54 03 43 43 54 03 44 43 54 03 45 43 54 03 46 43 54 03 47 43 54 03 48 43 54 03 49 43 54 03 4a 43 +54 03 4b 43 54 03 4c 43 54 03 4d 43 54 03 4e 43 54 03 4f 43 54 03 50 43 54 03 51 43 54 03 52 43 +54 03 53 43 54 03 54 43 54 03 55 43 54 03 56 43 54 03 57 43 54 03 58 43 54 03 59 43 54 03 5a 43 +54 03 5b 43 54 03 5c 43 54 03 5d 43 54 03 5e 43 54 03 5f 43 54 03 60 43 54 03 61 43 54 03 62 43 +54 03 63 43 54 03 64 43 54 03 65 43 54 03 66 43 54 03 67 43 54 03 68 43 54 03 69 43 54 03 6a 43 +54 03 6b 43 54 03 6c 43 54 03 6d 43 54 03 6e 43 54 03 6f 43 54 03 70 43 54 03 71 43 54 03 72 43 +54 03 73 43 54 03 74 43 54 03 75 43 54 03 76 43 54 03 77 43 54 03 78 43 54 03 79 43 54 03 7a 43 +54 03 7b 43 54 03 7c 43 54 03 7d 43 54 03 7e 43 54 03 7f 43 42 60 65 3c c1 ca 81 3f 60 e5 00 40 +60 e5 40 40 b0 72 80 40 b0 72 a0 40 b0 72 c0 40 b0 72 e0 40 58 39 00 41 58 39 10 41 58 39 20 41 +58 39 30 41 58 39 40 41 58 39 50 41 58 39 60 41 58 39 70 41 ac 1c 80 41 ac 1c 88 41 ac 1c 90 41 +ac 1c 98 41 ac 1c a0 41 ac 1c a8 41 ac 1c b0 41 ac 1c b8 41 ac 1c c0 41 ac 1c c8 41 ac 1c d0 41 +ac 1c d8 41 ac 1c e0 41 ac 1c e8 41 ac 1c f0 41 ac 1c f8 41 56 0e 00 42 56 0e 04 42 56 0e 08 42 +56 0e 0c 42 56 0e 10 42 56 0e 14 42 56 0e 18 42 56 0e 1c 42 56 0e 20 42 56 0e 24 42 56 0e 28 42 +56 0e 2c 42 56 0e 30 42 56 0e 34 42 56 0e 38 42 56 0e 3c 42 56 0e 40 42 56 0e 44 42 56 0e 48 42 +56 0e 4c 42 56 0e 50 42 56 0e 54 42 56 0e 58 42 56 0e 5c 42 56 0e 60 42 56 0e 64 42 56 0e 68 42 +56 0e 6c 42 56 0e 70 42 56 0e 74 42 56 0e 78 42 56 0e 7c 42 2b 07 80 42 2b 07 82 42 2b 07 84 42 +2b 07 86 42 2b 07 88 42 2b 07 8a 42 2b 07 8c 42 2b 07 8e 42 2b 07 90 42 2b 07 92 42 2b 07 94 42 +2b 07 96 42 2b 07 98 42 2b 07 9a 42 2b 07 9c 42 2b 07 9e 42 2b 07 a0 42 2b 07 a2 42 2b 07 a4 42 +2b 07 a6 42 2b 07 a8 42 2b 07 aa 42 2b 07 ac 42 2b 07 ae 42 2b 07 b0 42 2b 07 b2 42 2b 07 b4 42 +2b 07 b6 42 2b 07 b8 42 2b 07 ba 42 2b 07 bc 42 2b 07 be 42 2b 07 c0 42 2b 07 c2 42 2b 07 c4 42 +2b 07 c6 42 2b 07 c8 42 2b 07 ca 42 2b 07 cc 42 2b 07 ce 42 2b 07 d0 42 2b 07 d2 42 2b 07 d4 42 +2b 07 d6 42 2b 07 d8 42 2b 07 da 42 2b 07 dc 42 2b 07 de 42 2b 07 e0 42 2b 07 e2 42 2b 07 e4 42 +2b 07 e6 42 2b 07 e8 42 2b 07 ea 42 2b 07 ec 42 2b 07 ee 42 2b 07 f0 42 2b 07 f2 42 2b 07 f4 42 +2b 07 f6 42 2b 07 f8 42 2b 07 fa 42 2b 07 fc 42 2b 07 fe 42 96 03 00 43 96 03 01 43 96 03 02 43 +96 03 03 43 96 03 04 43 96 03 05 43 96 03 06 43 96 03 07 43 96 03 08 43 96 03 09 43 96 03 0a 43 +96 03 0b 43 96 03 0c 43 96 03 0d 43 96 03 0e 43 96 03 0f 43 96 03 10 43 96 03 11 43 96 03 12 43 +96 03 13 43 96 03 14 43 96 03 15 43 96 03 16 43 96 03 17 43 96 03 18 43 96 03 19 43 96 03 1a 43 +96 03 1b 43 96 03 1c 43 96 03 1d 43 96 03 1e 43 96 03 1f 43 96 03 20 43 96 03 21 43 96 03 22 43 +96 03 23 43 96 03 24 43 96 03 25 43 96 03 26 43 96 03 27 43 96 03 28 43 96 03 29 43 96 03 2a 43 +96 03 2b 43 96 03 2c 43 96 03 2d 43 96 03 2e 43 96 03 2f 43 96 03 30 43 96 03 31 43 96 03 32 43 +96 03 33 43 96 03 34 43 96 03 35 43 96 03 36 43 96 03 37 43 96 03 38 43 96 03 39 43 96 03 3a 43 +96 03 3b 43 96 03 3c 43 96 03 3d 43 96 03 3e 43 96 03 3f 43 96 03 40 43 96 03 41 43 96 03 42 43 +96 03 43 43 96 03 44 43 96 03 45 43 96 03 46 43 96 03 47 43 96 03 48 43 96 03 49 43 96 03 4a 43 +96 03 4b 43 96 03 4c 43 96 03 4d 43 96 03 4e 43 96 03 4f 43 96 03 50 43 96 03 51 43 96 03 52 43 +96 03 53 43 96 03 54 43 96 03 55 43 96 03 56 43 96 03 57 43 96 03 58 43 96 03 59 43 96 03 5a 43 +96 03 5b 43 96 03 5c 43 96 03 5d 43 96 03 5e 43 96 03 5f 43 96 03 60 43 96 03 61 43 96 03 62 43 +96 03 63 43 96 03 64 43 96 03 65 43 96 03 66 43 96 03 67 43 96 03 68 43 96 03 69 43 96 03 6a 43 +96 03 6b 43 96 03 6c 43 96 03 6d 43 96 03 6e 43 96 03 6f 43 96 03 70 43 96 03 71 43 96 03 72 43 +96 03 73 43 96 03 74 43 96 03 75 43 96 03 76 43 96 03 77 43 96 03 78 43 96 03 79 43 96 03 7a 43 +96 03 7b 43 96 03 7c 43 96 03 7d 43 96 03 7e 43 96 03 7f 43 90 c2 75 3c 85 eb 81 3f c3 f5 00 40 +c3 f5 40 40 e1 7a 80 40 e1 7a a0 40 e1 7a c0 40 e1 7a e0 40 71 3d 00 41 71 3d 10 41 71 3d 20 41 +71 3d 30 41 71 3d 40 41 71 3d 50 41 71 3d 60 41 71 3d 70 41 b8 1e 80 41 b8 1e 88 41 b8 1e 90 41 +b8 1e 98 41 b8 1e a0 41 b8 1e a8 41 b8 1e b0 41 b8 1e b8 41 b8 1e c0 41 b8 1e c8 41 b8 1e d0 41 +b8 1e d8 41 b8 1e e0 41 b8 1e e8 41 b8 1e f0 41 b8 1e f8 41 5c 0f 00 42 5c 0f 04 42 5c 0f 08 42 +5c 0f 0c 42 5c 0f 10 42 5c 0f 14 42 5c 0f 18 42 5c 0f 1c 42 5c 0f 20 42 5c 0f 24 42 5c 0f 28 42 +5c 0f 2c 42 5c 0f 30 42 5c 0f 34 42 5c 0f 38 42 5c 0f 3c 42 5c 0f 40 42 5c 0f 44 42 5c 0f 48 42 +5c 0f 4c 42 5c 0f 50 42 5c 0f 54 42 5c 0f 58 42 5c 0f 5c 42 5c 0f 60 42 5c 0f 64 42 5c 0f 68 42 +5c 0f 6c 42 5c 0f 70 42 5c 0f 74 42 5c 0f 78 42 5c 0f 7c 42 ae 07 80 42 ae 07 82 42 ae 07 84 42 +ae 07 86 42 ae 07 88 42 ae 07 8a 42 ae 07 8c 42 ae 07 8e 42 ae 07 90 42 ae 07 92 42 ae 07 94 42 +ae 07 96 42 ae 07 98 42 ae 07 9a 42 ae 07 9c 42 ae 07 9e 42 ae 07 a0 42 ae 07 a2 42 ae 07 a4 42 +ae 07 a6 42 ae 07 a8 42 ae 07 aa 42 ae 07 ac 42 ae 07 ae 42 ae 07 b0 42 ae 07 b2 42 ae 07 b4 42 +ae 07 b6 42 ae 07 b8 42 ae 07 ba 42 ae 07 bc 42 ae 07 be 42 ae 07 c0 42 ae 07 c2 42 ae 07 c4 42 +ae 07 c6 42 ae 07 c8 42 ae 07 ca 42 ae 07 cc 42 ae 07 ce 42 ae 07 d0 42 ae 07 d2 42 ae 07 d4 42 +ae 07 d6 42 ae 07 d8 42 ae 07 da 42 ae 07 dc 42 ae 07 de 42 ae 07 e0 42 ae 07 e2 42 ae 07 e4 42 +ae 07 e6 42 ae 07 e8 42 ae 07 ea 42 ae 07 ec 42 ae 07 ee 42 ae 07 f0 42 ae 07 f2 42 ae 07 f4 42 +ae 07 f6 42 ae 07 f8 42 ae 07 fa 42 ae 07 fc 42 ae 07 fe 42 d7 03 00 43 d7 03 01 43 d7 03 02 43 +d7 03 03 43 d7 03 04 43 d7 03 05 43 d7 03 06 43 d7 03 07 43 d7 03 08 43 d7 03 09 43 d7 03 0a 43 +d7 03 0b 43 d7 03 0c 43 d7 03 0d 43 d7 03 0e 43 d7 03 0f 43 d7 03 10 43 d7 03 11 43 d7 03 12 43 +d7 03 13 43 d7 03 14 43 d7 03 15 43 d7 03 16 43 d7 03 17 43 d7 03 18 43 d7 03 19 43 d7 03 1a 43 +d7 03 1b 43 d7 03 1c 43 d7 03 1d 43 d7 03 1e 43 d7 03 1f 43 d7 03 20 43 d7 03 21 43 d7 03 22 43 +d7 03 23 43 d7 03 24 43 d7 03 25 43 d7 03 26 43 d7 03 27 43 d7 03 28 43 d7 03 29 43 d7 03 2a 43 +d7 03 2b 43 d7 03 2c 43 d7 03 2d 43 d7 03 2e 43 d7 03 2f 43 d7 03 30 43 d7 03 31 43 d7 03 32 43 +d7 03 33 43 d7 03 34 43 d7 03 35 43 d7 03 36 43 d7 03 37 43 d7 03 38 43 d7 03 39 43 d7 03 3a 43 +d7 03 3b 43 d7 03 3c 43 d7 03 3d 43 d7 03 3e 43 d7 03 3f 43 d7 03 40 43 d7 03 41 43 d7 03 42 43 +d7 03 43 43 d7 03 44 43 d7 03 45 43 d7 03 46 43 d7 03 47 43 d7 03 48 43 d7 03 49 43 d7 03 4a 43 +d7 03 4b 43 d7 03 4c 43 d7 03 4d 43 d7 03 4e 43 d7 03 4f 43 d7 03 50 43 d7 03 51 43 d7 03 52 43 +d7 03 53 43 d7 03 54 43 d7 03 55 43 d7 03 56 43 d7 03 57 43 d7 03 58 43 d7 03 59 43 d7 03 5a 43 +d7 03 5b 43 d7 03 5c 43 d7 03 5d 43 d7 03 5e 43 d7 03 5f 43 d7 03 60 43 d7 03 61 43 d7 03 62 43 +d7 03 63 43 d7 03 64 43 d7 03 65 43 d7 03 66 43 d7 03 67 43 d7 03 68 43 d7 03 69 43 d7 03 6a 43 +d7 03 6b 43 d7 03 6c 43 d7 03 6d 43 d7 03 6e 43 d7 03 6f 43 d7 03 70 43 d7 03 71 43 d7 03 72 43 +d7 03 73 43 d7 03 74 43 d7 03 75 43 d7 03 76 43 d7 03 77 43 d7 03 78 43 d7 03 79 43 d7 03 7a 43 +d7 03 7b 43 d7 03 7c 43 d7 03 7d 43 d7 03 7e 43 d7 03 7f 43 01 00 00 00 0c 00 00 00 01 02 00 00 +00 00 00 00 10 27 00 00 00 00 00 00 00 00 00 00 00 00 00 00 09 ba d7 1a f4 c0 f8 98 ca fc 10 18 +04 6d 2e ed 02 a1 36 3b 02 c2 34 25 75 bb 75 8a 44 83 b7 1c 88 05 a4 3a 45 89 dc 13 b4 13 55 d5 +82 c9 54 6a a0 69 ba 8f 38 c6 cb f7 1f 75 d4 4a 03 e8 85 f1 88 2d 73 68 6d 80 1a ab 2c 1a c0 bd +d7 c8 f8 61 51 99 62 35 50 e8 d2 0e 20 22 37 82 88 d2 57 32 eb 8d 26 1f 48 4b b4 8b c7 c7 91 30 +48 3a cd 7b 76 93 84 62 12 e2 93 99 d0 94 5a 0f d5 19 28 44 3d 21 14 57 72 b3 2f 53 b9 44 93 37 +4f ca 92 37 40 95 bc f6 4c 59 0c ab 5d f9 cd a9 65 a8 02 bb 34 d4 cd b3 b4 14 52 08 2d 81 34 fd +6c d6 33 c3 01 1b 99 ae de db c1 30 99 d1 0d 49 91 af b7 28 ad 7b 15 c5 3c 48 46 b0 0c 82 f6 49 +b6 e7 1c 5c 45 ad e5 cb 63 9c 53 39 0c a4 1e c8 b6 52 b5 82 3b 8d 55 d3 39 cc 7f 62 a0 c6 bf a3 +b1 00 c7 6a 00 2e ca df a9 cd ea 08 b6 7a f1 11 18 e1 93 46 f7 dc 50 81 7a 85 f7 c7 25 fa 18 f2 +66 dd 4e d5 5f bc 49 44 aa aa b8 a1 57 4e 8c 60 ac f5 b7 36 f7 ea a1 29 d2 a5 e2 3a 64 91 04 31 +4a c6 16 93 60 97 b0 3c 6b 31 47 98 eb 56 1d c5 25 53 9b 45 de dd ad 95 47 a5 bf 1e e3 a8 5f b6 +60 ca 19 dc a3 33 a3 a8 a0 6b ee 30 72 7d 19 96 7e 60 c8 54 10 1f 31 30 10 97 ad 72 4f 03 af b7 +75 d2 0b ad 4f 8c b7 bb be ec 38 42 48 13 a7 d9 66 97 e6 d9 cf 76 07 02 63 a5 71 ec af 5b 80 03 +9a 40 36 f4 e2 23 ff 18 11 80 53 24 40 4c 7c ba fd 0a ae 92 45 70 40 9f 01 85 73 4f 4a 67 28 9b +92 04 46 a6 e8 3c 1e f0 73 00 17 7a 85 27 46 d7 4d 90 61 7e c7 4e 8b f9 ce 12 06 69 6b 9a 06 d7 +b4 9f cd ea c4 3a f3 96 17 b3 3d de 74 0e 92 39 bf 18 58 63 9e 96 3e 0a 88 b9 4c 69 90 5c 7c a8 +54 67 20 d5 d8 b3 7a 68 3c 2f 4b c2 0e c7 47 9b 8c 50 79 6a 5f e7 e7 62 5f 3d 2c 7d 02 0e 2f a4 +56 6f 22 d1 c7 18 b9 2b 0c 23 dd a6 a3 0d e2 9a c1 b6 21 26 08 8c 39 d5 43 d2 60 0b 6e 8b 9d ee +f7 2d 52 6b 89 b8 fa b9 e4 16 81 58 96 1b 16 9c d8 21 7b c4 78 05 d2 07 c7 8d 3f c7 b7 f2 1d d5 +72 ca da 77 e7 5b 87 53 da 20 26 92 d1 d2 d6 3f 80 d5 d1 69 19 e0 c7 5c 63 75 bd b1 33 5d 94 0c +46 d9 27 29 5d 93 0d a4 d1 9e e8 a5 d4 5a c2 be 08 bc 1a 89 b6 ad 4d 77 46 b3 5d 9b 6c 88 18 69 +c7 24 76 52 16 f5 90 84 32 bc 83 24 15 65 70 67 9e bc 32 17 a1 30 aa 41 c3 1d e1 1c 26 ce 58 01 +e1 85 49 ef 55 c5 ab e8 46 82 6d 43 a4 f8 ab e3 68 58 02 6a 66 b9 f8 57 49 01 bd 42 21 5a c4 50 +b0 2c 73 3f 44 2b bc 83 0b 2a c3 79 2a 5e c1 d3 13 5b 71 bd 60 4a b2 28 1b b0 05 1c 1d 95 6f 0a +11 a4 90 ad 79 67 eb 3f 1c 97 b5 62 c7 cc 9d 88 96 52 04 00 95 0a ff 7f ff ff 37 3e ff ff ff ff +30 ff 33 35 be 3f fd 3a ff 7b ff ff 3c ff ff ff ff ff bc f5 ff 28 33 ff 39 6f ff 31 76 6f ff 30 +ff ff ff 37 ff fb 34 ff ff f5 ff ff ff 34 7a 6c ff bd ff 3e ee ff ed ff 77 72 ff 38 b9 ff ff ff +ff 37 ff ff 3c ff ff ff ff b3 3d ff ff ff ff 3d 3e 3c ff 2d ff ff ff ff 38 b8 ff 33 35 bd ff 3e +32 ff 37 ff f9 ff 7e b9 ff ff ff 6d 2f ff ff fc 3e 3f ff 6c 34 38 ff 3a ff ff 31 ff ff 2e ff f7 +ff ff 3b 3c ff 7d 9c ff af fa f0 79 e7 be 70 31 ff 63 34 ff ff 3b ff f0 ff ff fb 28 3e ff 3a ff +78 ff ff ff ff ff ff ff ff ff 31 31 ff ff ff ff f8 e6 3c 36 33 df ff ff 26 30 3a 1d a4 ff ff ff +ff ff ff ff a7 ff ff 74 ff ff 26 b8 ff ff df fc 6c ff f3 f1 28 ff 31 ff ff ff bf ff af ff ff 76 +79 ff ff ff 5c ff ff ff ff ff ff ff 6d ac 3d ff 31 ff ff 32 b1 f4 ff ff 51 ff ff ff ff ff ff 38 +3d 3e 3b 37 21 ff 31 ff 6e fd ff ff 77 ff 3b ff 9b ff ff ff ff ff ff 6c ff 30 f2 ff 2e ff ff ff +b4 2a 7c f0 ff b7 ed d6 af 7e f5 29 33 e3 7e 1a 39 f0 3b 78 ff ff f3 37 ff ff de 26 ff ff 72 fc +ff aa f6 a2 35 ff 33 ff ff ff 36 31 ff 3e ff ff b0 2a ff 63 6d a5 ff 7d ff ff 25 ff ff ff ae 3a +ff 3f e4 f2 3e ed 2d 6d 24 78 3f ff 39 7b ff 39 ff d3 bf 3a ff e6 25 6c ff f4 ff 6d 1b 2a ff 31 +ff ff ff ff 3a 6f 7e 33 ff 7b 33 ff 39 ff ff ac 32 ed 32 20 36 ff f4 6c ff ff dd ff ff ff ff ff +ff 3c 61 30 ff e0 f2 ff ff ff ff ff ff ff f7 6d ff 2d 9d bf b5 bf bf 35 7c 33 bf ff ff 18 ff a3 +38 ac 28 ff bf b3 bf ff 3b fa 7e ff ff ff 2b ff 33 ff ff ff 2f 3a ff ff ff 96 37 ff ff 39 ff ff +77 31 ff 7e ab 26 ff ff 26 ff f9 ff bf 7e ff ff 32 ff ff ff ff ff ff ff ff 6f f4 ff ff 25 ff ff +6e d5 de ff ff ff ff ff 3c ff ea 34 ff ff ff ff b8 ac ff ff 3a ff ff ff bf 29 ff 7b 3c ff ff ff +ff 3d ff ff ff 30 ff bd ff ff f9 ff ff ff f9 ff ff ff 7e ff ff ff 24 ff 7f ff ae 7f ff ff ff ff +ff 30 7e 31 ff ff 7c 31 fb f7 ff 7f ff ff ff 29 ff b3 f4 24 ff 6d 19 ff ff ff fc ff ff ff 24 ff +ff dc 36 ff ff f4 62 b5 ff ff ff fc ff ff ff 1e 77 ff ff ff ff 6f 34 f0 ff 38 ff 39 ff 30 ff 37 +ff 26 a7 ff ff 3c 3d bd ff ff ff 3b ff 16 ff 39 ff ff 30 ff ff 75 30 6b ff ff ff ff 38 36 3e bb +6e ff fd 7f 6f b3 bf ff 32 df dc f1 3b b6 ff 37 60 23 ff ff 2f ff ff ff ff b7 1d 7f ff ff 2f e9 +af f4 de ff ff ff ea ff 32 ff ff fc ad e7 ff 78 ff 1e 2f e7 ff 78 f9 b9 6d ff ff ba f4 67 ff b7 +2b ff ff 7b 30 ff ff d9 3d 23 ff 76 ff 7b 3b 33 d5 ff ff 6d bf ff a6 7f af 76 b7 62 ff 97 ff 73 +77 e0 ff aa 32 ff 34 7d ff ff ac 32 ff ff ff 7e ff ff ff f7 ff ff ff 5f ff ff ff 70 ff ff ff f2 +34 ff ff 64 ff ff ff 30 ff ff ff bd 3e ff 39 ff ff ff ff 33 ff 7c ff ff ff ff 7e cf 2c ff ff 36 +31 ff ff 23 ff ff ff 1d b8 ff ff 66 2e ff 3d ae ff ff fc bb ff fa ff ff ff 38 3f ff ff ff af ff +ff ff 27 bd ff 3d ad ff ff ff ff b8 3c ef 30 6c 27 55 ff 7b ff 2e ff ff ff dc ff ff ff ff ff 7d +ff 6d ff ff ff eb ff ff ff 2c bf 7f ff 3c 3c ff ff b6 ff ff ff 6f ff ff ff 3a ff ff 3e f9 ff 38 +ff ff ff ff ff ff ff ff 34 f9 2a ff 78 30 ff 34 ff 2e 28 6a bc 2f f5 1c 3d 76 ff 65 ff 31 bf ff +3c ff ff 27 ff e9 34 35 fb 6d 2f 36 e7 ff f4 ff ff ff 2d ff ff ff ff bd f8 ff ff b8 6c be 3b ff +6d ff ff ff af ff ff 39 ff ff fd 32 69 ff ff 6c 27 ff 24 ff bf ff 3a 72 3d ff ff 29 da ff ff ff +64 ff ea 73 2b b8 ae ff ff ff f4 ff ff 30 78 7e 60 b3 ff ff d8 38 3c 63 b5 7e ff 7f ff ed f3 ff +ff ac da ff 25 23 37 ff 3a ff 29 ff ff ff e7 2e 3d 6d ff ff ec ff 3c ff ff 32 ff 30 fb ff 76 36 +1f ff f3 ff ff ff fd 6d ff ff 3f 62 ff fb 38 ff ff 3a ff b5 f4 ff ff 79 38 ff ff ff ff e0 ff f2 +ff ff ff 6d ff ff 3d 3d ff ac ff ff 77 26 ff ff ff ff ff fb ff 30 ff 33 ff ff fd 6f ff fc 3a ff +ff 32 3e ff ff df a6 ff ff e6 ff ff ff f4 ff dd bf ff ff ff ff ff ff 35 fb 30 ff ff 60 ff ff ff +ff ff ff ff ff ff ff fc ff 9a 3a 37 ff ff ff ae ff ff ff ff 77 3d ff ff ff 33 ff 6b 3d ff ff ff +26 e8 ff ff 31 ff ff ff ec ff ff ff ff ff 37 39 ff ff ff ff 1c fa fd ff ff ff ff 6a bd ff 38 77 +ff e9 ff 36 35 2a 63 2d 32 ff ff ff ff ff 30 ff ff 7e 32 30 ff ff fd ff 6c ff 30 ff bf ff 3c f7 +3e 39 ff 71 1b e7 36 ff 55 e2 ff ff ff ff ff ff 99 ff ff ff ff ff 3a 79 2b ff ff 25 29 ff af ff +e1 f1 34 7f 3e ff 3d 3c 33 ff 5d ff d4 ff ff 37 6e ff fd ff b6 ff bf ff ff 3a 30 ff ff ff 13 ff +aa ff ff 6a ff b3 ff ff ff ff a2 f5 f1 38 ff e7 2e fa ff da 78 ff ff fc ff ff ff ff ff ff ff ff +ee be ff ff 3b ff ff 24 dd b8 bf df 0b a5 ff b9 ff ff ff 38 ff ff ff ff be ff ff 3a 3b bf ff ff +ff b8 f6 ff ff ff ff fe bf 1b ff ff ff 64 bf ff ff ff ad ff ff ff e4 32 ff ff ff d1 ff 6d 36 7f +ff ff ff 79 ff 38 d7 ff ff 7d a1 6f 2d 75 25 ff 1f ff 32 ff ac d8 2a ff bf ff 2d 7d 7a 36 cb b8 +bf ff 31 ff 31 75 ff b8 34 2a 31 fc 27 ff 26 6f ff 34 a6 26 9e 75 2c fc 2b ff 2d 25 bf ff 16 b5 +b0 ff 3f 18 34 6f e5 fe ff 1a ff 7a 3d aa f6 ff 32 2b 3b da ad 16 ff ff ff ff 3b ff 7f 7d 3f ff +b9 13 ff 36 ff ff ff e8 ff ff ff 72 b8 bc ff ac ff ff af 70 b8 ff a5 29 ff ff ff ff ff 3c da 3e +ff ff ff ff ff 36 ff 7f ff 7d ff ff 7c 30 34 ff 3a ff 31 7e af ff ff 74 ff 1d ac a2 ff ff ff ff +1f b3 ff ff 21 27 37 ff ff 20 f5 ff 27 ff ff ff 2a 2a ff ff ff ff f1 32 3a ff ff 37 ff f4 ff 34 +2b ff ff ff ff ff ff 2f 36 36 ff ff ff b3 3d ff 28 ff 3c ff ff 22 ff 33 3b ff ff ff ff ff 37 30 +ff ff ff ff ff ff ff ff 21 ff fd 27 36 ff 22 3e 6d ff 9d 37 2a ff 3a 53 76 ff e9 ff 5e ff 6b ff +31 ff af ff 7c ff ff ff ff ff 61 ff 26 ff 32 ff ff ff ff ff 34 39 ff 3c 72 ff ff e7 dd ff ff ff +6e ff fd ff 26 ff ae ff 3a ff 3a ff 63 ff ff ff ff 29 ff 7e 6e ff af 35 68 ff ff 1c bf b3 ff ff +20 7d ff ff ff ff ff 63 ff ff ff ff b7 32 f5 31 ff ff 9f ff b7 a2 14 ff ff 2a 26 ff 2a ff ae ff +ff ff ff ff 77 27 ff 33 da ea f3 ff ff ff 36 6f 15 a2 ff ff 75 ff 3f e1 25 7d 7e db ff ff 2d af +3d 2a 37 2b 3e 30 ac dd de 32 ff e2 ff 7c d9 6d ff 3a 19 7f ff 23 ff 7f 33 e9 76 ff 7f d6 ff 27 +ff 3a ae 39 ff 2f ff ff 3d 67 ff 20 3e 2f 37 31 ff df 2e ff ff ff 62 3b ff ff 27 ff ff ff ff 6b +eb 30 af d3 ff 3c ff 35 ff d3 ff 37 ce 26 3a ae ff ff ff 28 2f 6f ff ff ff 31 2d 7f ff 30 3f ff +af ff f6 ff ff 3f 3e ff 24 ff 23 30 ff d1 1c 7f 34 e0 26 ff ff be 2d f7 69 ff ac 3b 23 ff ff 33 +28 39 ff 7e 2e f4 f4 ff ff ac fe ff b8 34 36 7f 39 9e da ff fb b2 ff ff 64 f4 37 e6 76 2a ff fc +fb e6 3c 3e b9 3a bf f2 3a ff ff aa ff ff 60 ff 33 ff 24 ff ff ff ff ff bf f0 a9 62 38 ff 28 20 +ec ff a4 2a bf fb 39 ff 3b ff ff ff ff ff ff 3a ff ff 00 00 00 00 ff ff f1 b8 ff 01 00 00 23 01 +00 00 3a 00 00 00 c7 01 00 00 75 00 00 00 0d 00 00 00 69 00 00 00 ec 00 00 00 07 01 00 00 92 01 +00 00 84 00 00 00 28 01 00 00 ea 00 00 00 06 01 00 00 9a 00 00 00 31 01 00 00 16 01 00 00 01 01 +00 00 e8 00 00 00 6f 00 00 00 bc 00 00 00 ca 00 00 00 08 01 00 00 c0 01 00 00 e2 00 00 00 13 01 +00 00 6f 01 00 00 ea 01 00 00 dc 01 00 00 85 00 00 00 6e 01 00 00 a1 01 00 00 5b 01 00 00 3b 01 +00 00 e4 00 00 00 b6 01 00 00 06 00 00 00 ed 00 00 00 f2 01 00 00 0d 01 00 00 a8 00 00 00 a7 00 +00 00 4d 00 00 00 70 00 00 00 fb 00 00 00 1b 00 00 00 da 01 00 00 d8 01 00 00 ac 00 00 00 81 00 +00 00 e6 01 00 00 b5 01 00 00 c9 00 00 00 bf 01 00 00 a6 00 00 00 07 00 00 00 b1 01 00 00 b1 00 +00 00 95 01 00 00 53 00 00 00 a2 01 00 00 a9 01 00 00 ed 01 00 00 7b 01 00 00 ef 01 00 00 46 00 +00 00 89 00 00 00 d1 00 00 00 c3 01 00 00 21 01 00 00 91 00 00 00 eb 00 00 00 34 00 00 00 25 00 +00 00 49 01 00 00 66 01 00 00 6e 00 00 00 69 01 00 00 3b 00 00 00 c2 00 00 00 63 01 00 00 3c 00 +00 00 0b 01 00 00 0f 01 00 00 b3 01 00 00 0f 00 00 00 a6 01 00 00 c5 01 00 00 41 00 00 00 5e 01 +00 00 b4 01 00 00 5a 01 00 00 4c 00 00 00 7f 00 00 00 9f 01 00 00 36 00 00 00 e3 01 00 00 e5 01 +00 00 14 01 00 00 94 00 00 00 be 01 00 00 59 01 00 00 5e 00 00 00 90 01 00 00 2b 00 00 00 73 01 +00 00 09 00 00 00 0a 00 00 00 0a 01 00 00 59 00 00 00 02 01 00 00 47 00 00 00 c7 00 00 00 38 01 +00 00 da 00 00 00 3f 00 00 00 de 01 00 00 74 00 00 00 36 01 00 00 c2 01 00 00 71 00 00 00 f0 00 +00 00 37 00 00 00 a0 01 00 00 09 01 00 00 bf 00 00 00 58 00 00 00 5f 00 00 00 0e 01 00 00 8c 01 +00 00 f9 00 00 00 8f 00 00 00 8e 00 00 00 e0 00 00 00 17 00 00 00 9a 01 00 00 95 00 00 00 9c 00 +00 00 4d 01 00 00 6c 00 00 00 55 00 00 00 39 01 00 00 4e 00 00 00 41 01 00 00 22 01 00 00 df 01 +00 00 f7 01 00 00 16 00 00 00 90 00 00 00 1a 00 00 00 02 00 00 00 56 00 00 00 54 00 00 00 5c 01 +00 00 cf 01 00 00 6d 01 00 00 b7 00 00 00 5b 00 00 00 fe 01 00 00 15 01 00 00 9d 01 00 00 ad 01 +00 00 cd 01 00 00 26 00 00 00 51 01 00 00 83 00 00 00 12 00 00 00 c6 00 00 00 c9 01 00 00 87 00 +00 00 eb 01 00 00 27 00 00 00 43 00 00 00 6a 00 00 00 0e 00 00 00 1e 00 00 00 50 00 00 00 32 01 +00 00 3a 01 00 00 bb 00 00 00 2b 01 00 00 5d 00 00 00 f5 00 00 00 18 01 00 00 53 01 00 00 52 00 +00 00 35 00 00 00 f0 01 00 00 ba 01 00 00 93 00 00 00 d3 00 00 00 6c 01 00 00 60 00 00 00 d8 00 +00 00 b4 00 00 00 1c 00 00 00 8a 00 00 00 98 01 00 00 5d 01 00 00 57 00 00 00 80 01 00 00 d7 01 +00 00 d3 01 00 00 96 00 00 00 63 00 00 00 d4 00 00 00 db 01 00 00 51 00 00 00 ee 01 00 00 f2 00 +00 00 8b 01 00 00 d2 00 00 00 97 01 00 00 9b 01 00 00 65 00 00 00 c8 01 00 00 a4 01 00 00 ca 01 +00 00 55 01 00 00 b5 00 00 00 1e 01 00 00 dd 00 00 00 2e 01 00 00 29 01 00 00 8e 01 00 00 26 01 +00 00 57 01 00 00 3f 01 00 00 8d 00 00 00 c5 00 00 00 c8 00 00 00 bb 01 00 00 7a 01 00 00 dc 00 +00 00 d5 01 00 00 fd 00 00 00 1b 01 00 00 4a 00 00 00 86 00 00 00 aa 01 00 00 2f 01 00 00 fe 00 +00 00 40 01 00 00 1f 00 00 00 9f 00 00 00 cd 00 00 00 e3 00 00 00 68 01 00 00 92 00 00 00 00 01 +00 00 2d 00 00 00 64 01 00 00 78 01 00 00 d0 00 00 00 b2 00 00 00 f1 01 00 00 9e 00 00 00 82 01 +00 00 50 01 00 00 e9 01 00 00 f8 01 00 00 e5 00 00 00 ef 00 00 00 ce 00 00 00 19 01 00 00 84 01 +00 00 aa 00 00 00 ac 01 00 00 48 01 00 00 11 00 00 00 fa 01 00 00 b2 01 00 00 bc 01 00 00 8a 01 +00 00 2a 01 00 00 ae 00 00 00 f6 00 00 00 5c 00 00 00 f5 01 00 00 ba 00 00 00 ad 00 00 00 7e 00 +00 00 88 00 00 00 32 00 00 00 11 01 00 00 24 00 00 00 3e 00 00 00 9e 01 00 00 96 01 00 00 56 01 +00 00 72 01 00 00 be 00 00 00 99 00 00 00 28 00 00 00 42 01 00 00 79 01 00 00 3e 01 00 00 42 00 +00 00 6a 01 00 00 1d 00 00 00 13 00 00 00 b9 01 00 00 fa 00 00 00 b7 01 00 00 38 00 00 00 25 01 +00 00 29 00 00 00 68 00 00 00 b0 01 00 00 2c 00 00 00 4c 01 00 00 15 00 00 00 3d 00 00 00 22 00 +00 00 75 01 00 00 fc 01 00 00 d0 01 00 00 cc 01 00 00 ec 01 00 00 d6 01 00 00 6b 01 00 00 60 01 +00 00 c6 01 00 00 4a 01 00 00 83 01 00 00 d2 01 00 00 a4 00 00 00 2a 00 00 00 04 00 00 00 0c 01 +00 00 45 01 00 00 7c 00 00 00 7d 00 00 00 76 00 00 00 8f 01 00 00 1d 01 00 00 7a 00 00 00 4b 00 +00 00 a9 00 00 00 18 00 00 00 44 01 00 00 5a 00 00 00 d7 00 00 00 e7 00 00 00 fd 01 00 00 37 01 +00 00 cb 00 00 00 e6 00 00 00 cf 00 00 00 2c 01 00 00 3c 01 00 00 4f 00 00 00 20 00 00 00 fc 00 +00 00 67 01 00 00 af 01 00 00 b6 00 00 00 e4 01 00 00 de 00 00 00 94 01 00 00 71 01 00 00 a2 00 +00 00 bd 01 00 00 a3 01 00 00 17 01 00 00 33 01 00 00 d4 01 00 00 00 00 00 00 0b 00 00 00 24 01 +00 00 c1 00 00 00 b0 00 00 00 30 00 00 00 e1 00 00 00 65 01 00 00 f3 01 00 00 48 00 00 00 1a 01 +00 00 03 01 00 00 c3 00 00 00 1f 01 00 00 30 01 00 00 88 01 00 00 54 01 00 00 2e 00 00 00 47 01 +00 00 db 00 00 00 00 02 00 00 67 00 00 00 c1 01 00 00 b8 00 00 00 2f 00 00 00 76 01 00 00 d1 01 +00 00 bd 00 00 00 34 01 00 00 8b 00 00 00 ab 01 00 00 c4 00 00 00 77 00 00 00 f1 00 00 00 c0 00 +00 00 05 01 00 00 dd 01 00 00 64 00 00 00 e8 01 00 00 9d 00 00 00 7e 01 00 00 1c 01 00 00 4e 01 +00 00 0c 00 00 00 4b 01 00 00 e0 01 00 00 6d 00 00 00 74 01 00 00 f9 01 00 00 a5 00 00 00 f6 01 +00 00 70 01 00 00 10 00 00 00 85 01 00 00 e2 01 00 00 b9 00 00 00 61 01 00 00 43 01 00 00 77 01 +00 00 21 00 00 00 80 00 00 00 44 00 00 00 f7 00 00 00 e1 01 00 00 61 00 00 00 d9 01 00 00 99 01 +00 00 78 00 00 00 8c 00 00 00 a7 01 00 00 3d 01 00 00 d9 00 00 00 31 00 00 00 a8 01 00 00 03 00 +00 00 7c 01 00 00 79 00 00 00 4f 01 00 00 62 01 00 00 b8 01 00 00 98 00 00 00 10 01 00 00 7f 01 +00 00 35 01 00 00 cb 01 00 00 46 01 00 00 23 00 00 00 9b 00 00 00 04 01 00 00 45 00 00 00 af 00 +00 00 e9 00 00 00 33 00 00 00 9c 01 00 00 52 01 00 00 a1 00 00 00 05 00 00 00 cc 00 00 00 14 00 +00 00 f8 00 00 00 82 00 00 00 fb 01 00 00 49 00 00 00 a3 00 00 00 c4 01 00 00 20 01 00 00 2d 01 +00 00 62 00 00 00 72 00 00 00 08 00 00 00 f4 00 00 00 5f 01 00 00 7b 00 00 00 81 01 00 00 f3 00 +00 00 39 00 00 00 97 00 00 00 e7 01 00 00 6b 00 00 00 a5 01 00 00 87 01 00 00 93 01 00 00 58 01 +00 00 73 00 00 00 91 01 00 00 27 01 00 00 01 00 00 00 ce 01 00 00 7d 01 00 00 f4 01 00 00 ee 00 +00 00 ff 00 00 00 8d 01 00 00 89 01 00 00 ae 01 00 00 12 01 00 00 df 00 00 00 19 00 00 00 ab 00 +00 00 b3 00 00 00 d6 00 00 00 d5 00 00 00 40 00 00 00 86 01 00 00 66 00 00 00 a0 00 00 00 00 00 +00 00 00 00 00 00 ae 01 00 00 00 00 00 00 27 03 00 00 00 00 00 00 92 04 00 00 00 00 00 00 f3 05 +00 00 00 00 00 00 25 07 00 00 00 00 00 00 88 08 00 00 00 00 00 00 d1 09 00 00 00 00 00 00 1e 0b +00 00 00 00 00 00 85 0c 00 00 00 00 00 00 d1 0d 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 42 10 +00 00 00 00 00 00 7d 11 00 00 00 00 00 00 b9 12 00 00 00 00 00 00 21 14 00 00 00 00 00 00 82 15 +00 00 00 00 00 00 f0 16 00 00 00 00 00 00 11 18 00 00 00 00 00 00 38 19 00 00 00 00 00 00 5c 1a +00 00 00 00 00 00 94 1b 00 00 00 00 00 00 a3 1c 00 00 00 00 00 00 ed 1d 00 00 00 00 00 00 35 1f +00 00 00 00 00 00 4c 20 00 00 00 00 00 00 70 21 00 00 00 00 00 00 92 22 00 00 00 00 00 00 ab 23 +00 00 00 00 00 00 da 24 00 00 00 00 00 00 d0 25 00 00 00 00 00 00 ec 26 00 00 00 00 00 00 c4 27 +00 00 00 00 00 00 00 00 20 00 3a 00 50 00 6f 00 88 00 a8 00 c8 00 dc 00 fc 00 1b 01 36 01 40 01 +55 01 70 01 90 01 00 00 1f 00 33 00 47 00 50 00 5f 00 7a 00 8e 00 a2 00 be 00 dd 00 f8 00 18 01 +36 01 47 01 5f 01 00 00 18 00 31 00 44 00 59 00 78 00 91 00 af 00 bd 00 d7 00 ea 00 fb 00 0d 01 +25 01 3d 01 58 01 00 00 1c 00 34 00 4d 00 69 00 82 00 91 00 9c 00 b5 00 c3 00 e3 00 01 01 1d 01 +30 01 3f 01 4d 01 00 00 0c 00 20 00 40 00 4d 00 6b 00 7e 00 8c 00 a5 00 b6 00 c6 00 d6 00 ec 00 +f8 00 0b 01 26 01 00 00 10 00 24 00 44 00 57 00 75 00 90 00 ad 00 c6 00 e1 00 f6 00 0e 01 27 01 +35 01 45 01 50 01 00 00 16 00 32 00 41 00 56 00 69 00 75 00 8b 00 a1 00 b1 00 c3 00 e3 00 fc 00 +11 01 1e 01 36 01 00 00 16 00 24 00 37 00 42 00 62 00 7a 00 99 00 ae 00 bb 00 cf 00 ee 00 09 01 +12 01 28 01 3a 01 00 00 16 00 32 00 44 00 5a 00 69 00 7e 00 94 00 a4 00 bb 00 d1 00 e3 00 ff 00 +1e 01 3d 01 51 01 00 00 16 00 2a 00 3f 00 54 00 6e 00 81 00 a0 00 b9 00 d1 00 e3 00 f4 00 0c 01 +18 01 2b 01 3e 01 00 00 17 00 31 00 41 00 4e 00 59 00 68 00 77 00 8b 00 a9 00 b9 00 cd 00 e3 00 +fd 00 0b 01 1f 01 00 00 12 00 2c 00 3f 00 4d 00 5e 00 75 00 85 00 98 00 a9 00 b5 00 c8 00 db 00 +00 01 18 01 2b 01 00 00 0e 00 20 00 2e 00 42 00 61 00 76 00 8b 00 a1 00 ae 00 c0 00 d4 00 e5 00 +f7 00 07 01 22 01 00 00 18 00 26 00 3d 00 4c 00 5e 00 6e 00 85 00 99 00 ae 00 c2 00 d2 00 e8 00 +f7 00 17 01 2d 01 00 00 09 00 1a 00 33 00 4b 00 5a 00 75 00 94 00 a9 00 bc 00 d3 00 f3 00 10 01 +1e 01 32 01 4a 01 00 00 18 00 2b 00 47 00 5e 00 73 00 93 00 a9 00 bd 00 d5 00 eb 00 04 01 15 01 +2a 01 41 01 52 01 00 00 12 00 27 00 39 00 56 00 6d 00 80 00 9a 00 b2 00 d1 00 e8 00 05 01 10 01 +2a 01 47 01 62 01 00 00 0a 00 20 00 2b 00 38 00 49 00 57 00 68 00 7f 00 9a 00 b1 00 c7 00 dd 00 +ec 00 ff 00 0a 01 00 00 0c 00 1c 00 35 00 45 00 5a 00 6f 00 81 00 95 00 a6 00 ae 00 c1 00 df 00 +f1 00 0a 01 1b 01 00 00 19 00 31 00 47 00 53 00 67 00 7a 00 90 00 98 00 ac 00 c2 00 d0 00 e3 00 +f7 00 07 01 16 01 00 00 17 00 2f 00 42 00 55 00 5e 00 71 00 7f 00 97 00 a9 00 b3 00 cd 00 e2 00 +02 01 0c 01 2a 01 00 00 1a 00 24 00 37 00 52 00 64 00 78 00 86 00 98 00 a3 00 b4 00 c8 00 d3 00 +e1 00 ed 00 f9 00 00 00 12 00 2c 00 40 00 5c 00 73 00 88 00 9e 00 aa 00 b9 00 c2 00 dc 00 ed 00 +fe 00 14 01 34 01 00 00 16 00 30 00 3a 00 4d 00 6a 00 89 00 9b 00 ad 00 c0 00 ce 00 e3 00 f3 00 +03 01 1a 01 34 01 00 00 15 00 2a 00 36 00 46 00 54 00 69 00 7a 00 8c 00 96 00 9e 00 b2 00 cb 00 +db 00 ed 00 05 01 00 00 0f 00 25 00 39 00 4d 00 5a 00 70 00 88 00 a1 00 b9 00 c9 00 d8 00 e5 00 +fc 00 06 01 18 01 00 00 1b 00 28 00 32 00 41 00 53 00 67 00 7a 00 86 00 95 00 aa 00 c5 00 db 00 +e8 00 fe 00 0d 01 00 00 0d 00 1d 00 2e 00 3d 00 52 00 5c 00 6d 00 89 00 9c 00 ae 00 be 00 ce 00 +e3 00 f7 00 0d 01 00 00 12 00 1d 00 33 00 43 00 5e 00 74 00 85 00 a1 00 af 00 bf 00 d2 00 e6 00 +01 01 17 01 28 01 00 00 13 00 27 00 39 00 4c 00 5a 00 67 00 81 00 8d 00 94 00 a1 00 b7 00 c6 00 +d1 00 db 00 e9 00 00 00 0e 00 2d 00 3e 00 4b 00 5d 00 6d 00 7f 00 93 00 a2 00 b6 00 cd 00 d8 00 +e8 00 f7 00 0c 01 00 00 0c 00 22 00 2f 00 3d 00 53 00 5c 00 68 00 71 00 7e 00 8f 00 9b 00 aa 00 +ba 00 cb 00 d4 00 00 00 20 00 1a 00 16 00 1f 00 19 00 20 00 20 00 14 00 20 00 1f 00 1b 00 0a 00 +15 00 1b 00 20 00 1e 00 1f 00 14 00 14 00 09 00 0f 00 1b 00 14 00 14 00 1c 00 1f 00 1b 00 20 00 +1e 00 11 00 18 00 1a 00 18 00 19 00 13 00 15 00 1f 00 19 00 1e 00 0e 00 1a 00 13 00 11 00 12 00 +18 00 18 00 1b 00 13 00 1c 00 18 00 19 00 1c 00 19 00 0f 00 0b 00 19 00 0e 00 20 00 1e 00 1c 00 +13 00 0e 00 0e 00 14 00 0c 00 14 00 20 00 0c 00 1e 00 13 00 0e 00 19 00 11 00 10 00 10 00 16 00 +0c 00 13 00 1b 00 0c 00 10 00 14 00 20 00 13 00 1d 00 1b 00 1d 00 19 00 1b 00 13 00 18 00 18 00 +0e 00 10 00 0a 00 12 00 16 00 1c 00 0e 00 15 00 13 00 0c 00 16 00 16 00 10 00 12 00 20 00 19 00 +15 00 0d 00 18 00 13 00 16 00 0d 00 13 00 0a 00 20 00 18 00 1f 00 15 00 0d 00 14 00 1f 00 1b 00 +09 00 16 00 12 00 13 00 16 00 1c 00 11 00 16 00 0f 00 15 00 16 00 10 00 17 00 16 00 11 00 1c 00 +1f 00 1f 00 14 00 16 00 16 00 14 00 15 00 15 00 1a 00 13 00 1f 00 19 00 18 00 12 00 11 00 18 00 +0c 00 13 00 13 00 0d 00 16 00 1a 00 10 00 0d 00 0a 00 0f 00 0f 00 13 00 1e 00 10 00 13 00 16 00 +1a 00 0e 00 14 00 10 00 12 00 1a 00 12 00 0d 00 11 00 17 00 10 00 13 00 10 00 0c 00 13 00 12 00 +10 00 18 00 13 00 17 00 0d 00 12 00 0e 00 14 00 1f 00 15 00 15 00 16 00 0c 00 12 00 14 00 11 00 +12 00 10 00 1b 00 19 00 18 00 0d 00 17 00 0f 00 11 00 10 00 17 00 14 00 15 00 14 00 10 00 16 00 +0f 00 20 00 16 00 0f 00 09 00 11 00 19 00 18 00 0f 00 1b 00 1f 00 15 00 13 00 17 00 20 00 1d 00 +0d 00 14 00 18 00 1e 00 17 00 12 00 1c 00 17 00 14 00 20 00 16 00 14 00 18 00 16 00 19 00 11 00 +15 00 17 00 11 00 0f 00 12 00 14 00 12 00 1d 00 17 00 13 00 1a 00 18 00 1f 00 17 00 1d 00 0a 00 +1a 00 1d 00 1b 00 0b 00 0a 00 16 00 0a 00 0c 00 11 00 0d 00 10 00 17 00 1b 00 17 00 16 00 16 00 +0f 00 12 00 0a 00 17 00 0c 00 10 00 19 00 10 00 15 00 15 00 12 00 14 00 11 00 07 00 13 00 1e 00 +12 00 19 00 11 00 0b 00 19 00 18 00 16 00 0b 00 14 00 13 00 16 00 08 00 14 00 16 00 0d 00 12 00 +14 00 10 00 0f 00 0d 00 16 00 18 00 13 00 13 00 08 00 13 00 0d 00 17 00 12 00 0a 00 1a 00 14 00 +20 00 09 00 1e 00 0e 00 1a 00 0a 00 13 00 1b 00 11 00 14 00 0e 00 12 00 0b 00 11 00 14 00 0a 00 +0e 00 0b 00 0c 00 16 00 11 00 1a 00 14 00 1c 00 17 00 15 00 16 00 0c 00 0e 00 09 00 1a 00 11 00 +11 00 16 00 20 00 15 00 16 00 1a 00 09 00 13 00 1d 00 1f 00 11 00 12 00 13 00 0e 00 15 00 10 00 +0f 00 17 00 1a 00 14 00 15 00 15 00 0b 00 10 00 0e 00 15 00 10 00 12 00 09 00 07 00 14 00 16 00 +0f 00 12 00 18 00 12 00 0f 00 16 00 14 00 14 00 0d 00 16 00 18 00 19 00 18 00 10 00 0f 00 0d 00 +16 00 0a 00 12 00 0c 00 1b 00 0d 00 09 00 0f 00 12 00 14 00 13 00 0c 00 0f 00 15 00 1b 00 16 00 +0d 00 16 00 0e 00 15 00 0d 00 10 00 11 00 0f 00 15 00 09 00 11 00 1c 00 13 00 12 00 10 00 10 00 +15 00 14 00 16 00 0c 00 12 00 0a 00 15 00 10 00 1b 00 16 00 11 00 1c 00 0e 00 10 00 13 00 13 00 +1b 00 16 00 11 00 06 00 12 00 14 00 11 00 13 00 0e 00 0d 00 1a 00 0c 00 07 00 0d 00 16 00 0f 00 +0b 00 09 00 0e 00 0c 00 0d 00 1f 00 10 00 0d 00 12 00 10 00 12 00 14 00 0f 00 13 00 17 00 0a 00 +10 00 0f 00 15 00 0f 00 0b 00 15 00 0b 00 0e 00 15 00 07 00 0b 00 08 00 0c 00 10 00 0b 00 0e 00 +0f 00 10 00 08 00 02 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 +00 1c 05 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 00 10 14 0f 06 01 +01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 00 10 01 08 01 05 06 1d 09 01 01 01 01 01 01 01 +01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 00 10 02 03 02 0a 3a 07 01 01 01 01 01 01 01 01 01 +01 01 01 01 01 01 01 01 00 0f 07 1c 09 03 04 0e 04 20 01 01 01 01 01 01 01 01 01 01 01 01 01 01 +01 01 01 01 01 01 01 01 00 12 03 0b 01 05 02 01 41 0a 05 12 01 01 01 01 01 01 01 01 01 01 01 01 +01 01 01 01 01 01 01 01 00 08 0a 0d 02 05 09 23 28 11 0b 01 03 06 01 01 01 01 01 01 00 07 09 14 +03 10 13 09 03 05 01 13 07 04 07 01 20 05 01 01 01 01 01 01 01 01 01 01 01 01 01 01 00 0c 01 01 +0b 08 0e 01 01 11 14 08 03 06 10 04 03 04 34 01 01 01 01 01 01 01 01 01 01 01 01 00 1c 1e 02 05 +17 3e 2d 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 00 0e 0e 2c 30 10 1f 30 01 01 +00 09 0c 0c 33 02 13 05 13 07 1f 04 10 1c 03 01 01 01 01 01 01 00 09 18 09 01 0e 19 02 05 0d 1b +20 13 04 01 28 01 01 01 01 01 01 01 01 01 01 01 00 09 02 02 35 02 0e 02 1c 11 16 05 0c 0f 01 10 +09 06 04 0f 03 01 01 01 01 01 01 01 01 01 01 01 05 09 08 02 01 01 01 01 1e 04 0e 04 03 24 37 0d +0a 12 22 01 01 01 01 01 01 01 01 01 01 01 00 01 01 01 01 04 11 01 05 05 2c 02 01 06 12 2d 4c 16 +09 02 01 01 01 01 01 01 01 01 01 01 01 00 03 10 0b 22 02 07 03 11 18 05 06 21 02 17 01 0d 2d 1c +01 00 04 02 01 0c 03 02 06 12 20 09 21 17 25 3b 18 0a 01 01 01 00 11 01 6d 24 0c 0c 5c 01 00 2e +11 02 05 1c 34 6f 08 03 09 01 01 01 01 00 04 02 06 01 0e 04 02 0a 0e 0e 12 0c 2c 24 0f 36 0b 09 +01 0f 01 01 01 01 01 01 00 05 06 04 03 0c 39 07 01 1b 04 0a 15 2e 4c 0e 01 01 01 01 00 04 1b 03 +31 09 05 02 04 06 33 02 0d 6b 10 01 01 01 01 01 00 0f 03 25 0b 08 0c 0d 1a 05 02 0b 10 0a 05 20 +1b 1b 05 21 05 01 01 01 01 01 01 01 00 03 06 06 01 3e 02 02 01 02 01 08 03 02 19 75 07 0a 04 02 +01 09 19 03 09 01 01 01 01 01 01 00 0f 01 0b 53 32 10 05 0f 46 0d 03 0d 16 01 01 01 01 01 01 01 +01 01 01 01 01 01 00 15 05 27 28 01 0c 10 08 01 09 12 24 02 25 01 0d 02 07 0c 1e 0b 05 02 03 01 +01 01 01 01 01 01 00 01 0a 04 12 02 10 0c 0d 0c 07 0a 06 0c 0f 06 0b 01 28 13 03 10 0b 07 01 11 +12 33 01 01 00 29 1a 09 16 53 11 0a 25 1a 17 09 0e 0b 0e 01 01 00 11 01 04 3f 01 0d 0b 02 05 05 +02 0b 0e 31 17 14 08 08 29 2c 03 01 01 00 03 04 09 05 02 03 08 15 10 0c 0d 06 40 02 01 09 03 04 +05 24 2d 08 31 14 01 00 7b 11 06 07 0f 17 0d 0a 07 0b 02 10 20 06 11 07 16 05 0b 01 01 01 01 01 +03 02 01 02 03 01 08 07 0d 05 3d 01 15 26 0f 0a 20 06 04 05 1f 24 31 01 01 16 08 0c 02 07 24 3d +13 15 0a 21 02 4d 02 27 04 01 01 01 1b 0a 06 06 26 02 31 41 1e 01 1f 01 2d 01 05 29 01 01 01 01 +01 01 06 08 15 0e 04 0b 02 03 0b 04 04 11 04 0e 1b 1f 01 0a 4b 45 1a 01 01 01 01 01 01 01 01 01 +26 07 01 08 3d 1c 08 13 01 06 74 0c 16 0d 0c 0c 07 01 01 01 01 01 01 01 01 05 01 1c 02 09 01 23 +0e 0b 16 11 02 02 01 01 0a 05 01 09 05 0b 48 05 47 0f 18 01 01 01 01 07 1d 0a 2e 1d 08 18 4b 2e +15 58 01 01 01 03 02 2a 09 2b 08 27 01 02 2e 26 17 07 14 0a 02 0e 1d 03 04 1a 08 07 01 01 01 05 +17 04 10 41 36 17 2b 0b 25 06 37 09 1a 0c 01 01 01 01 2e 04 0f 08 13 47 03 08 0c 1e 03 05 08 69 +1e 1a 00 01 0c 08 1b 03 14 25 01 12 1a 2e 5b 3e 06 1a 0b 01 00 01 22 08 0e 1b 04 30 02 0b 31 1c +01 03 05 26 28 20 07 04 22 03 04 01 01 21 03 12 59 2d 05 2e 3d 04 0f 21 10 01 04 04 02 01 03 01 +04 0a 01 01 01 13 11 01 0a 0d 09 01 2d 0d 34 0d 49 12 06 30 0e 0e 02 0c 03 02 03 06 07 01 01 01 +06 02 1c 02 03 2d 1d 05 01 08 29 0b 03 03 27 0c 6d 08 01 08 09 16 03 03 0b 2d 11 04 07 12 24 0a +38 01 07 05 1d 0e 02 2f 0b 06 1e 03 01 01 01 08 06 1a 0e 15 48 17 0f 01 04 12 1e 06 0c 07 0a 02 +12 13 1e 05 2b 11 00 01 04 1e 10 30 07 13 0d 11 02 02 28 3b 17 0c 02 33 06 23 16 01 01 01 01 1c +04 04 06 04 04 07 01 02 0b 11 23 30 17 0b 05 29 02 33 10 23 1a 15 04 08 01 01 01 01 16 0b 15 03 +0f 50 0b 02 09 18 01 06 1c 07 3a 0e 1d 10 01 01 3a 01 01 01 01 0d 2a 18 33 0b 2d 10 34 14 55 22 +1c 01 01 01 65 4e 04 08 4b 6f 0c 04 08 17 01 07 17 05 01 08 04 09 08 11 0e 05 38 0a 05 11 2f 0b +0b 05 0b 29 12 4c 10 01 34 6e 1a 07 27 1f 02 32 29 01 43 01 01 02 0b 08 0f 01 07 02 02 03 2a 07 +08 08 14 64 03 06 18 04 21 02 14 07 26 03 1b 03 01 07 10 01 01 00 02 08 05 23 01 01 03 0a 21 22 +05 3e 01 02 04 03 01 18 3c 0f 1e 04 2d 01 15 02 02 06 0d 02 03 16 25 03 02 04 2e 05 02 01 07 01 +5f 0f 0d 03 02 3f 07 31 14 12 05 04 05 01 01 02 08 33 0a 01 35 07 20 29 02 01 1a 08 59 26 13 1a +17 01 02 2c 0e 0a 32 07 01 fe 01 0d 07 01 0d 0b 0c 02 03 0a 35 39 12 64 03 40 1e 30 2c 03 05 02 +12 08 27 09 0b 01 07 1c 0d 2d 10 19 15 7e 04 17 28 05 01 02 0f 28 02 0e 6f 49 06 22 0b 7e 05 00 +02 08 0a 07 1f 09 02 53 2c 41 0a 1b 0b 1a 17 35 07 0a 0f 02 01 02 04 05 03 07 12 09 04 10 0b 03 +05 07 1b 25 03 13 01 01 12 1e 07 07 36 04 09 3a 32 16 01 1d 1e 04 02 84 01 4c 3c 08 23 39 03 0a +02 0c 16 0c 0e 36 01 01 01 06 05 36 2e 11 0d 27 38 0b 2e 0e 03 01 0a 07 01 01 01 01 01 01 02 38 +01 06 05 7d 4e 0a 0c 28 17 0f 19 0e 04 0b 03 02 07 14 06 23 08 12 1f 48 23 69 2b 3c 06 03 0c 02 +0b 08 0a 05 07 17 0d 09 33 1b 03 0f 06 06 06 18 15 10 01 04 42 48 15 1c 02 09 31 06 0b 01 02 0a +05 2c 68 15 20 12 44 1b 03 11 23 07 17 2a 30 03 37 56 16 04 22 24 06 0b 1b 02 06 06 0a 12 29 03 +05 12 34 06 02 02 0e 3e 5f 03 17 34 01 05 03 01 08 0e 0c 11 09 1d 2e 03 28 0d 17 0b 13 18 65 03 +0c 0e 14 1d 02 0f 0c 47 74 22 2f 03 1a 2b 0d 05 01 01 03 0f 17 01 26 10 22 2c 31 32 59 04 03 0c +0a 29 06 03 01 04 07 0a 33 23 05 02 03 04 07 05 02 21 14 26 0d 1f 4e 23 01 03 44 04 0c 23 38 21 +3a 1c 6b 29 10 05 0a 01 02 07 1c 13 04 02 03 4c 06 0e 26 2c 3a 03 32 1b 02 03 04 01 33 24 06 51 +0a 02 24 01 03 2c 1f 03 44 0d 03 16 1c 05 02 17 01 05 25 02 07 0f 14 01 05 04 21 04 0b 17 19 43 +09 03 0c 44 05 05 05 01 01 03 05 07 08 08 18 13 11 0d 3c 0b 09 19 04 1b 19 09 2a 5c 02 03 07 01 +01 16 08 1f 0d 11 0c 0b 1c 07 0e 13 01 01 14 01 1a 07 03 0e 09 0c 02 8a 01 14 03 0f 07 05 24 0b +03 01 0a 1a 10 01 0f 17 10 0f 1a 2d 02 01 25 36 29 31 01 07 01 03 05 01 03 0c 01 05 06 23 03 13 +16 02 03 0a 27 03 02 2e 36 10 10 05 01 5d 01 01 3a 03 00 2f 10 07 05 03 01 0f 03 19 1c 16 06 0a +05 14 46 02 26 1a 17 03 4f 0f 01 03 07 12 10 07 0c 08 04 09 06 03 02 4c 16 04 0a 0a 03 06 28 23 +44 0c 0b 0c 21 22 10 02 03 3c 03 0c 2b 03 03 06 01 04 07 01 22 8b 01 01 86 01 01 23 1f 06 05 03 +08 59 0b 16 07 0d 10 12 02 2e 04 19 18 03 16 13 0f 0d 19 03 01 04 42 09 02 07 0d 09 13 12 09 02 +05 01 02 02 01 2e 03 0f 03 ea 01 01 17 36 0e 06 4e 65 23 01 04 08 29 21 3c 04 11 16 03 2c 02 14 +0a 14 22 08 25 0a 06 3c 4b 3e 09 0d 41 0c 1f 62 35 83 01 2f 11 03 13 06 23 09 04 01 0b 20 23 02 +1f 4e 06 0d 08 30 81 01 03 1c 05 02 25 07 07 12 47 0b 04 03 1e 0d 2c 06 0c 12 0b 2a 1d 1a 03 06 +0e 02 1e 0b 0c 04 05 05 07 49 32 0b 07 16 3d 19 01 10 25 0a 28 06 01 01 06 0b 04 10 09 1d 2a 04 +05 ad 01 05 0e 02 24 04 0f 04 14 06 12 02 20 0c 03 0b 40 39 01 23 22 45 25 35 01 06 0b 01 04 24 +11 29 0e 2a 28 24 04 05 11 3c 2a 0c 2a 06 0e 06 28 04 11 06 0e 3e 29 24 7e 16 0a 35 11 04 09 12 +11 06 01 1b 19 05 02 46 0c 12 14 30 07 0b 1e 13 32 0d 35 04 05 0e 44 0a 0d 23 48 08 10 1f 0c 18 +11 13 0e 0d 03 0d 03 0e 47 04 4f 0f 0b 33 36 15 15 13 29 1b 11 02 1d 13 3e 03 01 08 0f 01 38 07 +10 04 14 12 07 15 26 4b 16 52 24 04 02 15 17 07 38 03 15 01 02 01 01 02 03 01 49 09 16 28 2c 09 +06 0b 19 14 06 1e 25 01 01 01 01 10 11 2b 14 06 03 0a 0e 16 04 1c 15 18 2a 02 5a 09 17 40 05 02 +05 02 0d 01 01 03 1d 07 03 0e 24 06 28 0f 05 47 11 16 20 25 02 18 04 25 52 04 27 1f 18 06 15 27 +0b 1d 02 02 39 5d 04 08 0e 04 2d 01 01 0c 12 05 0a 0b 0c 06 09 12 5b 12 26 03 14 6f 08 07 04 16 +02 08 30 07 0e 18 17 46 17 16 03 33 34 02 17 40 1a 04 50 10 02 08 18 32 2c 0b 06 04 03 0a 15 04 +05 78 29 05 01 09 06 04 77 06 21 0e 27 06 03 85 01 38 34 10 05 04 57 0b 01 37 31 0e 12 2b 06 10 +4f 06 3f 16 05 07 03 01 04 52 15 8f 01 0a 3d 29 09 63 04 05 01 1f 09 02 02 10 02 32 03 01 03 02 +03 0e 03 01 06 4c 07 0a 31 2d 0e 1e 20 05 0d 1d 14 03 04 05 01 0b 07 06 26 10 01 01 03 23 09 0a +57 0f 02 15 01 27 1e 02 05 3e 2e 02 03 03 10 03 0f 0d 0d 02 17 17 09 06 02 04 2b 1a 0d 06 0c 05 +04 09 0e 34 11 23 0c 29 34 0e 05 04 06 21 0b 09 42 03 0a 01 59 01 06 04 01 01 0c 1e 11 70 35 05 +06 32 43 02 04 02 19 38 0a 36 49 32 05 01 29 2b 2e 0c 04 13 66 07 38 29 01 04 0f 46 0f 09 02 01 +05 02 01 09 01 04 08 09 08 1c 04 08 17 06 1d 01 02 0d 11 35 2f 1c 11 02 0a 28 38 0e 16 1b 01 05 +04 13 04 10 03 0c 32 11 29 03 02 28 05 01 02 18 03 07 23 13 0b 12 0c 12 0f 37 05 14 05 2b 66 0c +0f 42 1e 05 13 0c 0e 01 09 02 2f 07 33 31 01 1d 01 01 14 16 18 29 1f 09 61 00 05 04 0d 25 23 21 +58 0c 18 07 1b 0a 39 05 2d 0e 0e 05 0e 18 10 02 37 0a 10 0b 59 1a 12 05 01 03 0e 2e 28 58 05 38 +17 24 0f 1a 04 02 35 16 01 03 2c 19 02 1b 0a 37 1c 05 09 2d 08 04 01 05 0f 05 01 07 01 3c 06 12 +07 2c 22 03 0a 03 05 02 03 08 1c 0e 2c 12 03 1b 05 03 10 0f 08 01 26 08 16 04 03 31 8e 01 3a 52 +16 10 05 01 1e 11 1f 22 0b 0f 08 19 13 01 13 02 18 05 03 16 12 5e 14 53 05 13 0d 15 02 4b 08 0a +21 0f 04 2a 3c 7f 01 4b 02 08 05 01 1b 19 12 10 14 06 42 34 0b 2d 2e 26 06 04 0a 10 05 16 20 12 +08 1b 07 01 10 01 0a 4a 16 02 04 2b 30 05 7a 18 05 02 00 05 36 41 04 04 19 5c 05 37 06 02 19 03 +40 5a 05 06 01 0a 23 06 38 01 01 07 35 23 02 03 03 03 0c 04 2c 01 03 41 3c 05 0a 2b 30 0d 18 34 +05 34 02 1a 02 11 2c 02 42 15 0b 17 16 02 01 00 05 2d 2b 89 01 02 31 09 1c 03 06 1f 1a 0b 12 1e +2c 06 01 1c 24 01 03 07 07 12 01 2b 03 01 35 09 0d 04 01 06 0b 09 04 10 06 03 5a 1d 1d 06 08 12 +4a 24 38 08 08 07 08 03 03 06 05 1e 1a 19 09 09 05 0c 20 03 04 12 18 01 16 0f 13 01 06 15 36 01 +07 11 24 09 09 08 08 03 55 01 03 07 01 0a 03 0d 0a 0f 07 11 09 02 1d 11 36 13 02 06 15 1a 4a 0e +0c 0a 18 2b 05 0b 28 32 09 06 04 04 42 0a 30 06 12 03 23 37 0f 01 04 3d 03 05 28 40 11 05 03 05 +33 10 36 17 03 06 15 01 09 4f 0f 73 1d 4b 03 11 08 06 07 10 07 0c 37 03 03 08 07 06 0c 47 15 0c +1d 12 3c 02 08 15 0e 01 0e 31 0a 3c 2d 1e 11 06 1a 06 08 4b 01 01 19 0e 09 07 2b 03 43 02 09 04 +2a 05 1c 03 28 11 31 0a 03 18 0e 51 01 04 09 15 12 29 0b 22 02 1e 17 0e 2d 26 0a 01 1d 04 25 02 +01 1e 06 1f 20 12 16 0f 08 05 3e 05 0a 06 04 01 0a 1d 04 06 18 0e 29 12 02 56 07 17 25 0a 08 16 +0e 10 22 1d 0e 44 00 14 12 02 06 29 40 0a 28 0f 0f 08 04 1b 11 0e 26 18 10 02 07 01 0c 0f 16 03 +02 03 0f 08 06 0e 18 33 02 10 03 1d 02 04 05 21 1f 11 0a 24 0c 0b 0a 1b 6d 02 0a 11 06 04 10 15 +01 08 0a 09 18 16 0a 0a 07 06 04 19 17 01 47 55 06 01 04 25 23 3b 20 0b 09 50 0a 18 15 02 35 6c +05 1e 10 07 0d 01 03 03 27 3d 10 0d 0a 21 43 1e 09 06 58 08 14 30 0b 16 0e 06 16 16 2d 0c 20 11 +28 0c 09 01 48 1c 15 10 0e 0b 07 14 08 17 2f 02 10 0e 0e 05 33 05 35 0d 3f 1d 29 31 6d 05 01 10 +1c 1e 0f 28 0c 1d 15 34 17 05 11 0f 1f 01 0c 53 37 3b 22 0c 28 06 46 01 11 0b 24 02 0a 15 03 53 +06 12 0e 06 12 1a 01 ef 01 0b 03 03 3d 0c 01 0e 0b 07 09 07 44 08 05 06 1f 0b 06 04 06 05 03 53 +06 0c 07 13 10 8e 01 14 07 06 04 31 08 2e 02 06 16 15 01 1a 02 06 01 13 4b 03 4d 01 0b 40 22 03 +03 07 00 07 01 0f 44 16 2f 3d 0b 44 02 0a 03 26 04 04 07 0c 25 18 02 02 05 35 0a 6d 1f 43 59 2a +0a 08 06 11 06 14 39 02 85 01 07 79 35 64 06 04 09 11 14 05 27 1c 1d 2f 0d 07 2d 13 52 0b 05 14 +01 16 47 0f 56 05 1c 16 00 0b 01 15 09 20 11 22 03 24 06 01 06 1e 0d 0c 05 31 cf 01 01 03 1b 01 +06 03 04 0a 1c 08 32 11 02 02 02 06 03 04 1b 02 08 1a 18 29 1d 28 04 03 0a 47 08 17 28 03 0c 3b +15 3c 40 10 13 09 2f 03 14 29 08 13 1c 24 4c 01 08 03 1b 01 1e 07 04 16 ab 01 01 33 0c 01 08 04 +19 0a 04 14 03 0e 21 02 17 23 09 07 0a 09 04 16 5c 16 1b 77 00 08 10 0d 24 04 08 01 01 04 02 04 +02 15 0a 18 07 34 38 0c 04 12 03 5b 37 09 37 23 01 3d 08 4d 16 6f 0f 2c 1b 19 18 02 08 07 09 3e +1d 08 07 33 09 20 06 16 48 01 19 01 5c 1d 04 0c 08 0b 17 0a 15 0a 09 11 0f 38 2d 2b 06 06 12 0d +08 12 57 0a 12 10 03 37 1d 04 02 09 0d 2b 26 39 1f 02 08 13 0b 25 05 04 06 29 02 22 06 06 12 17 +14 02 04 0d 35 03 07 10 1a 62 01 08 03 04 01 17 06 27 03 08 09 0e 0c 18 19 03 17 19 db 01 14 8d +01 03 1a 07 1d 28 4b 0a 07 1a 7a 06 0a 08 2e 03 24 52 1f 17 01 02 43 16 11 14 38 0b 46 07 08 0d +05 03 02 0f 11 49 1d 06 03 2f 01 1a 16 09 2d 06 09 11 2a 2b 0b 09 09 0f 05 01 08 3c 16 19 57 1f +18 12 47 0f 4c 09 1b 2a 03 01 19 2c 26 0e 08 09 14 14 34 3b 07 01 5e 1b 09 27 06 0a 04 03 0b 01 +13 58 48 0c 09 8a 01 06 1a 0d 12 10 13 08 70 08 30 3e 4f 05 74 09 2a 14 3d 35 02 34 08 1a 3d 0e +2a 09 09 0d 0a 16 1a 15 0c 03 02 02 17 07 04 47 12 0b 1f 02 11 27 0d 13 27 bc 01 09 26 1e 01 07 +02 03 05 60 02 0a 5a 03 51 1d 5b 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +09 19 0b 0b 1a 0e 34 05 0d 11 07 21 02 09 31 13 0b 2c 02 1a 2a 0c 10 34 09 16 18 0f 01 07 58 05 +03 0a 05 14 29 50 13 11 06 4b 08 20 09 08 02 62 22 05 0f 12 21 04 23 02 07 0b 10 21 19 14 02 07 +3f 15 21 15 24 8c 01 0f 1f 26 02 34 16 44 07 13 09 45 04 05 01 08 5c 17 1d 1a 5a 0b 02 0e 15 29 +02 17 09 1b 0a 02 34 55 31 24 66 0b 06 07 0a 09 0a 3b 44 44 28 08 01 01 01 11 3b 45 0b 06 06 09 +07 04 40 01 06 04 10 0a 14 09 06 08 05 0c 06 32 2d 26 15 04 09 0e 0c 20 1c 07 2e 08 0a 0a 0a 09 +11 1f 05 0a 05 30 04 0b 06 04 2b 28 06 01 41 17 04 47 0c 10 04 44 05 18 0a 13 01 0e 28 05 33 03 +06 0c 28 07 41 07 07 2d 03 39 13 23 19 0a 18 09 07 02 06 6c 24 17 01 07 02 2e 11 03 26 2a 13 0c +10 0b 47 0e 03 23 06 4f 15 03 40 b0 01 11 1a 22 06 04 45 35 01 0a 07 15 1b 07 01 63 16 32 18 2d +0b 04 0a 30 0d 35 08 43 0c 02 2c 29 07 02 06 21 40 04 0a 04 0b 06 0a 0c 1f 0d 13 30 32 04 01 03 +14 02 04 24 34 75 16 1a 06 03 2b 1a 03 02 3b 28 28 0b 35 08 02 11 0e 10 29 3c 1e 69 06 06 03 24 +3b 1b 12 0d 33 15 14 20 16 18 0d 0f 06 33 09 16 03 01 0e 02 01 07 02 09 01 58 0c 12 03 02 07 19 +36 0c 0d 04 0a 32 1c 1a 2f 06 03 01 22 05 0f 07 01 01 06 60 08 0a 14 10 0b 37 0f 23 03 0a 11 02 +1f 31 1e 10 2b 06 1e 0e 10 06 10 01 03 14 07 2f 49 01 08 1a 0d 0e 23 09 14 05 6b 3a 0e 02 0f 8e +01 03 11 0a 16 13 07 20 73 02 14 07 09 05 39 08 17 21 0c 0e 11 04 10 01 17 28 0a 44 01 68 0a 0a +0a 07 12 02 0d 45 7b 19 1b 0a 9c 01 62 19 0d 03 05 15 03 2f 14 14 23 01 12 18 07 0a 20 29 04 74 +08 2d 2a 01 04 6b 02 0c 15 10 11 0a 16 46 02 40 42 13 1d 06 0c 16 05 04 03 03 0a 03 0b 01 58 17 +1c 09 0b 01 02 01 0d 34 19 08 0d 60 01 05 14 40 05 01 12 09 09 2c 0b 23 4d 02 04 02 05 0e 15 35 +01 11 2e 10 05 2d 1e 0c 2a 03 32 0b 1f 0b 23 05 1b 05 0e 40 1b 10 01 39 0a 50 15 0f 0e 06 25 06 +06 10 2b 3c 05 13 01 56 18 1a 3a 05 07 09 3b 0c 02 46 07 17 04 0b 04 52 07 03 0c 08 4c 0f 12 28 +04 2b 2f 0e 05 0c 1e 2a 02 02 28 27 04 06 06 1c 1d 48 20 1b 00 20 47 03 07 01 09 0b 02 0a 03 0d +1d 26 14 03 1c 02 0b 1e 09 04 36 07 45 08 05 03 08 04 02 04 0c 33 0e 0e 06 0e 3f 04 03 22 06 16 +10 03 04 39 01 11 22 08 3b 45 0c 3f 72 04 16 33 19 09 02 14 08 1a 01 02 39 0c 15 50 21 46 37 23 +50 6b 00 0d 1d 1c 30 39 0e 0a 01 07 09 0b 05 43 25 6a 43 0d 07 0a 01 0d 1b 04 07 06 08 2b 16 08 +1e 20 05 2b 04 04 0d 2c 0c 32 19 08 0d 1f 03 29 05 1b 06 0a 4a 07 08 08 3c 2b 12 02 1e 09 02 03 +02 07 16 2d 0d 03 4e 0d 05 3e 29 24 0a 1f 19 28 05 09 61 0d 1a 2e 0e 04 1a 0b 05 09 1a 33 01 01 +1b 12 04 10 38 03 11 0a 0d 02 4a 03 05 13 0d 14 0b 06 17 09 11 01 06 02 08 01 15 01 03 26 0c 19 +11 01 0b 35 38 0d 19 09 0b 01 05 0f 11 0d 6d 17 05 01 2f 1c 03 28 09 09 04 01 21 37 1b 18 07 0f +29 0a 20 02 5f 09 18 12 1b 12 04 33 02 05 01 30 12 1e 0b 02 69 0d 0a 01 12 18 1b 04 02 3e 13 13 +08 06 0e 02 46 02 03 40 1f 01 27 05 0e 0e 04 01 02 06 11 02 28 59 05 0d 01 02 04 10 10 02 27 0e +03 1b 08 04 0c 03 17 0e 01 0d 0c 0b 0d 16 09 0c 30 22 01 01 02 36 02 01 12 0d 04 31 0f 01 02 39 +01 01 33 07 01 14 0e 14 25 0d 2c a9 01 4b 03 04 11 4a 04 18 04 08 41 0e 29 24 1b 01 0a 26 03 03 +2e 03 14 02 1a 42 05 2c 3e 02 38 0e 69 1c 04 3c 21 08 1a 1a 01 05 49 22 15 09 01 03 0d 04 06 04 +03 0b 05 06 08 21 02 08 19 20 16 09 02 01 06 20 0a 4a 08 0e 4a 06 03 03 05 12 2f 09 0a 07 05 0f +03 0e 1f 15 11 05 02 01 06 24 1c 07 03 06 06 0a 0e 04 09 91 01 12 07 6b 0a 0e 04 07 1b 1e 39 01 +0b 8a 01 0b 02 03 08 1a 04 12 30 39 0e 25 09 3a 09 01 01 02 16 0d 01 04 06 08 02 1d 02 02 11 0a +01 2b 2b 02 23 5e 2a 02 0e 30 01 09 38 06 02 0b 03 2f 0d 0c 17 08 58 29 06 1b 11 06 2f 05 07 0e +01 28 17 29 1d 16 03 03 3d 01 15 03 03 32 84 01 07 15 06 0b 1b 02 14 08 02 19 07 0c 08 17 3d 2c +07 07 14 0f 1c 10 02 02 02 03 02 2f 21 02 07 03 07 2e 01 0d 5a 23 11 01 01 21 02 1a 0c 0d 04 0d +54 01 04 4e 09 0c 09 06 22 19 24 1e 05 03 22 09 08 06 01 18 1e 1f 19 04 40 1c 11 62 2c 07 0e 0b +07 17 40 04 02 2d 06 05 17 04 01 01 05 13 08 08 41 02 34 02 54 32 0f 0e 01 25 09 24 07 0f 01 16 +40 1f 05 01 41 03 03 0e 03 05 41 50 0f 01 05 0b 10 43 04 06 04 02 2b 02 0e 3a 14 03 04 22 35 04 +05 0d 2a 17 06 0f 21 03 48 03 66 18 01 2a 09 05 0c 01 5a 04 22 2e 0f 0d 38 16 06 19 02 08 44 17 +0d 05 16 31 12 11 2d 09 2b 14 01 0f 0d 17 52 3e 08 08 03 1d 02 06 03 32 11 01 0a 02 27 22 10 03 +18 02 0f 11 67 02 36 04 3a 04 01 2f 26 43 40 05 0b 06 05 0f 26 41 0f 36 09 52 14 01 16 49 1d 07 +1d 1d 0f 55 04 08 53 01 11 27 08 09 27 26 41 01 06 16 13 31 0f 0f 13 0f 8a 01 2f 05 19 0f 03 0a +01 02 12 0e 01 24 1a 03 42 0f 0a 17 02 49 01 02 17 2a 3a 05 0e 08 2c 09 4c 0f 3e 0f 01 0b 1c 19 +4e 02 0d 03 01 1b 12 16 04 05 09 32 03 08 12 0c 65 04 04 17 01 04 09 06 0f 09 2c 2f 1d 01 26 2e +0a 11 0d 06 03 58 06 1b 06 02 0f 08 17 06 0e 10 04 07 20 50 09 0f 41 22 04 04 01 15 43 05 3f 18 +15 0b 10 09 01 47 0f 07 41 19 0d 16 0f 02 02 03 31 01 0a 04 52 0d 02 0f 09 03 2d 06 10 20 0b 05 +07 1a 33 24 4e 05 04 28 0f 02 12 15 09 1d 01 0e 10 12 01 16 10 0d 01 07 03 09 06 17 02 01 01 01 +07 07 2c 11 0a 0f 19 01 03 16 19 06 2a 19 0a 01 26 2b 22 10 02 06 01 0d 2a 03 10 0a 1e 02 13 54 +14 08 05 25 01 1f 3f 01 30 03 10 0a 09 15 13 02 07 04 07 62 04 19 01 26 06 0e 0a 1a 0d 1a 0b 0f +10 12 0b 09 0f 22 12 10 13 13 39 ad 01 22 06 2b 45 32 05 0b 0b 1d 0f 35 01 02 0e 01 07 03 17 4b +08 0d 13 15 09 12 0e 35 09 06 05 44 10 04 10 15 16 0a 13 02 01 06 01 0f 15 12 18 23 1d 06 23 09 +0f 5d 0c 08 08 07 06 0b 11 10 05 07 05 05 40 05 04 27 14 0b 0d 03 0b 1d 02 16 1b 2b 08 02 17 05 +6a 0b 07 09 10 05 08 c3 01 1a 12 0b 31 18 44 10 10 04 4b 24 2d 06 6c 4f 23 39 11 1b 0d 0a 36 1a +01 26 09 03 2f 54 03 03 08 49 0c 06 02 04 11 19 11 08 18 22 03 3b 1e 09 03 97 01 12 15 0e 02 31 +28 6a 05 05 0e 86 01 29 12 1f 20 04 07 0b 25 01 07 01 09 22 42 16 41 75 04 12 43 13 2b 04 06 01 +02 69 8c 01 36 02 14 12 42 26 07 19 3f 0c 02 07 01 0d 03 05 17 9b 01 1d 00 13 03 09 2c 3c 1f 0f +1c 30 0e 10 0a 05 15 02 09 0e 44 04 15 10 11 13 08 5e 10 05 1a 07 17 10 07 0b 2c 25 0f 04 02 09 +08 19 0e 01 07 0a 11 08 45 04 14 1e 2c 1a 12 01 1a 07 1b 43 13 08 02 05 0f 0a 14 0b 01 06 25 04 +27 14 03 03 06 09 39 04 04 08 64 0f 03 43 01 27 04 0e 01 03 25 0f 61 0e 06 31 12 09 10 36 12 05 +07 1e 09 2b 06 2e 47 0b 1b 09 09 02 13 28 6c 06 06 6b 10 02 5b 01 31 03 0e 0b 0a 05 14 1a 29 27 +9b 01 03 09 03 0a 12 1d 1c 01 0d 02 17 08 08 15 6a 0e 01 19 29 52 2f 03 8f 01 14 01 4d 0c 02 18 +03 0b 0f 10 11 0a 0d 05 05 01 02 2d 0c 2e 0a 38 0d 15 07 04 09 49 64 02 10 02 0b 19 40 15 3d 0e +06 27 07 03 3d 1d 4e 14 04 25 1c 2d 13 2b 29 02 15 02 10 03 01 02 07 01 10 04 0a 27 3c 04 0e 29 +25 28 22 0c 01 0e 15 19 5a 0a 13 3a 08 0b 32 5a 15 21 15 0d 0c 0c 15 1b 24 16 06 07 08 02 22 41 +03 08 16 1e 27 19 0b 52 03 20 0c 16 11 0b 2b 1e 17 2a 03 1e 06 3a 15 06 0f 0d 0e 2b 0a 40 05 06 +12 04 06 01 08 03 09 09 12 51 01 63 07 2b 20 01 79 08 16 1c 4d 45 37 1e 15 07 12 18 02 0a 18 14 +04 06 0b 1b 35 04 01 15 11 01 07 03 16 2c 20 22 06 61 57 0f 0d 36 0f 16 2a c1 01 16 13 1b 6a 17 +01 06 23 3b 19 11 09 1b 04 07 10 06 14 2a 02 03 17 02 01 04 12 02 19 02 06 0a 1f 01 25 2b 21 0c +05 03 02 14 1b 10 1c 1f 1c 11 0a 01 04 1a 17 0c 17 4f 31 0b 03 31 09 03 08 07 28 01 23 05 22 01 +6c 14 01 09 23 1a 0b 06 0a 3f 03 23 03 04 07 14 01 01 01 21 5f 01 03 2e 04 14 06 19 32 03 3e 01 +07 29 02 15 0b 12 26 10 03 11 04 26 17 01 05 45 0b 33 0c 22 07 a8 01 1d 18 5a 02 2b 1a 1f 01 10 +01 02 02 0d 02 56 0f 23 0a 01 07 18 07 1b 20 0c 01 18 49 13 18 12 0a 0f 12 01 01 02 07 1a 08 3d +08 42 08 14 08 43 0d 03 01 00 18 09 04 09 01 01 09 1d 0f 15 18 17 37 3d 08 0e 21 08 31 09 58 84 +01 1b 2b 37 07 2a 04 24 21 42 27 18 18 09 07 23 0d 4a 32 14 01 34 01 1c 43 01 01 1b 13 1e 15 18 +22 07 36 08 08 02 14 3b 22 01 1c 10 0d 1f 39 0f 02 24 18 10 67 41 04 1a 09 3c 01 17 39 01 13 12 +0c 08 02 0d 0e 03 0e 07 19 09 3a 53 58 05 4a 3f 19 07 03 33 06 0b 12 02 12 08 0d 0f 06 15 0b 05 +27 22 33 76 19 0a 09 08 02 15 06 43 27 02 02 13 1f 27 24 0d 3d 1f 06 0c 05 0b 0e 0b 29 06 06 07 +28 05 34 0a 19 30 b6 01 19 34 01 32 07 1b 0e 01 47 0b 06 10 15 03 11 01 18 98 01 19 1e 12 08 01 +01 07 02 08 04 58 35 11 33 70 04 0b 12 04 2b 19 01 03 63 07 49 07 21 12 22 1a 12 01 02 05 06 1a +09 15 52 11 3c 34 5a 01 04 24 40 18 11 07 1a 46 40 81 01 0b 02 19 1e 03 02 2e 49 13 1a 13 06 05 +0e 86 01 36 0a 2d 28 09 04 01 04 0e 0e 28 01 0f 30 05 04 1a 01 2b 0e 1b 04 03 0b 04 05 29 01 47 +02 02 48 1b 2c 34 15 0a 07 07 0d 1a 51 43 03 1b 07 03 09 1c 02 2f 0f 13 23 02 54 0f 10 1a 1a 3d +0a 19 01 33 2f 01 1b 0a 14 1a 0a 7e 01 0a 02 14 10 1a 42 59 29 2d 12 c9 01 1a 1a 35 1f 0c 2f 3e +05 21 0a 02 10 01 1d 05 0e 3d 13 13 19 1a 01 20 be 01 0d 01 24 12 1a 56 0d 30 09 00 85 01 05 02 +01 02 35 05 0d 4c 03 02 03 15 02 04 1d 1e 17 04 3e 08 1f 1a 01 02 1c 13 16 03 09 21 06 6c 0e 09 +35 02 0d 1c 20 1a 4d 2d 2b 3a 09 0f 28 26 45 1a 2c 1a 06 4f 09 20 01 27 04 11 09 27 03 0f 11 10 +10 05 18 14 1b 0c 04 10 01 1b 21 88 01 10 02 14 0c 18 28 16 0b 15 2f 03 0b 08 39 09 06 01 5a 2b +0a 04 30 0a 10 18 08 14 3a 03 0d 12 11 15 0b 0f 0a 05 09 03 01 03 16 02 06 04 03 02 01 0c 1b 0a +16 08 cc 01 39 0a 1c 46 0c 02 25 33 42 01 0c 05 1a 02 07 15 06 06 19 01 2a 08 02 0c 04 12 0d 02 +02 10 45 06 1a 07 1b 1e 08 39 15 37 4b 37 30 2a 11 34 01 03 05 74 13 02 41 03 05 04 02 04 02 05 +05 08 01 5f 09 23 22 17 01 0d 06 1f 09 05 1b 3e 3e 05 07 02 0c 25 48 01 24 06 1f 1c 07 09 01 3b +03 41 02 15 06 22 19 59 03 04 13 00 1c 04 1a 28 34 05 1a 12 05 1e 07 29 07 05 0a 05 2c 0a 09 09 +06 02 07 04 10 19 06 16 02 14 0c 2e 09 11 09 0e 81 01 07 01 0c 01 64 1c 1c 09 01 1d 17 1a 29 01 +27 19 15 02 0e 4d 11 16 07 35 19 1b 1d 0b 21 01 3d 1e 5c 07 2b 03 01 20 4d 39 1d 14 31 06 2b 0b +4c 0f 05 1a 02 2c 05 0b 09 52 2a 1e 0b 12 63 4f 0d 06 56 05 11 0d 5a 05 19 0e 3e 1c 01 02 05 49 +1e 07 41 09 01 52 02 55 1e 0a 0a 1c 26 56 2f 07 07 01 01 2a 12 2f 21 01 1c 0b 08 02 1e b8 01 49 +1c 02 0f 0c 2e 3d 2d 1f 10 02 30 06 07 40 0c 5d 1b 1c 4a 31 11 1f 38 0a 05 48 10 8c 01 27 0e 06 +3f 20 6c 04 09 34 09 04 76 0f 0c 75 02 20 1a 30 23 01 06 05 60 05 04 16 09 06 07 0b 10 0e 07 0c +19 1d 27 1d 03 02 07 02 07 01 4b 8e 01 03 16 18 40 0e 02 17 08 1a 02 04 0d 4e 15 02 06 05 01 12 +15 07 06 4b 0a 2c 02 09 02 0b 15 01 38 2d 0c 20 58 08 01 26 04 24 08 01 0b 02 03 3d 0b 11 06 39 +14 4c 1d 0e 13 0f 03 11 20 19 02 0f 02 09 02 27 02 16 0c 04 0d 2d 5f 09 13 06 0b 11 01 05 1c 21 +05 06 03 1c 07 36 0c 17 0a 08 04 1e 03 01 27 0f 0a 09 51 01 2d 43 22 30 1f 0d 03 21 1b 22 02 03 +01 34 01 0a 08 02 10 01 5e 3f 13 22 09 0e 08 21 02 09 1d 2a 2d 09 01 1a 15 23 01 50 0d 01 02 04 +09 23 15 1a 3c 0a 0b 3c 0c 7d 03 16 2a 23 11 01 03 19 17 25 0b a7 01 28 33 34 0b 1e 23 7f 34 07 +14 29 12 5d 07 23 44 01 02 01 08 19 01 08 06 24 05 12 04 0a 16 02 25 1d 01 01 1f 0f 05 61 05 23 +73 37 5a 37 03 06 17 05 0d 01 1b 0f 25 07 15 02 24 0a 0e 30 2c 19 02 1d 35 09 14 0b 1e 26 0b 2f +3a 24 29 05 08 02 05 24 6b 3d 1f 2a 04 03 09 02 07 35 06 02 03 08 1a 24 0c 14 2c 03 21 24 06 31 +06 05 0a 08 18 1c 0a 03 01 1c 10 15 06 03 0e 03 01 03 18 02 0a 11 0d 24 0a 93 01 46 03 01 35 34 +08 01 01 04 11 03 09 25 02 0a 06 21 09 24 06 28 03 05 03 06 09 33 21 26 03 39 0b 4d 15 06 11 1c +03 14 27 24 09 01 05 04 05 03 07 12 25 1e 26 04 07 06 26 16 02 32 11 0a 0f 06 03 2a 47 24 09 37 +2c 1d cd 01 10 1d 3f 24 47 08 08 3c 23 02 08 0e 06 02 01 45 02 0c 51 23 05 0a 16 0e 01 03 08 12 +15 07 02 22 12 06 14 0e 02 04 02 2b 15 24 2b 07 15 27 35 01 04 05 01 24 01 09 05 06 01 05 04 02 +01 29 07 4c 02 01 30 08 11 47 08 14 06 15 1e 06 04 04 0c 12 11 15 01 24 08 2e 35 0e 0a 05 21 17 +88 01 0c 01 02 01 0b 65 25 27 0b 10 08 0b 63 01 11 66 07 19 05 52 07 08 0d 07 25 14 0a 06 4b 73 +0a 07 04 2c 05 02 2a 05 13 23 15 25 01 25 54 0c 0f 0a 2b 53 62 13 15 22 04 07 18 29 04 09 31 12 +17 27 09 2f 7e 01 04 04 14 1c 28 0b 10 0b 02 05 25 6e 03 21 4a 30 02 1b 05 2c 18 43 01 04 12 03 +26 02 05 25 48 05 82 01 0c 3f 03 16 03 08 26 31 26 32 3a 0d 02 04 04 0c 12 0b 36 15 12 02 22 0d +04 09 02 1e 09 5c 0e 26 15 02 37 05 1b 0d 3d 15 17 13 06 27 02 09 0f 07 04 09 01 34 0d 06 06 02 +33 26 40 0c 39 0e 08 01 0f 03 0a 0c 04 01 01 5e 0e 01 1e 49 2d 27 04 02 01 53 02 07 06 02 0d 07 +03 0a 25 0d 03 12 4d 1a 01 3f 27 06 19 2e 25 05 0f 09 07 30 3e 15 10 17 04 1a 28 34 0e 0c 01 27 +07 46 07 02 04 5f 03 8c 01 7e 02 28 0a 38 2a 07 24 05 63 01 2b 10 08 21 14 0c 0c 0b 2f 02 02 58 +23 3a 43 56 19 0b 0c 02 20 2d 01 0c 2c 0c 24 10 1c 6e 06 1d 0a 1f 05 0f 11 09 0d 15 04 0e 29 03 +0a 84 01 06 0f 0e 0d 2e 21 22 61 2d 08 0a 02 29 08 20 14 02 01 25 02 01 05 13 18 58 7d 12 32 0b +10 29 2f 6e a7 01 09 06 15 46 17 29 03 75 45 02 81 01 39 01 29 0b 01 1b 03 13 02 02 21 4a 01 02 +02 24 3f 34 08 48 17 00 2b 03 0f 40 02 0b 15 33 06 0b 03 02 02 02 2d 04 43 06 1c 1c 4f 00 00 00 +0b 20 12 02 83 01 16 01 11 4b 1e 10 12 0e 01 1c 2c 4d 13 20 31 0b 35 08 3b 1d 13 02 0b 12 33 02 +05 05 2c 19 17 13 01 45 22 04 06 07 22 0e 02 24 02 01 09 06 15 01 54 08 1e 09 2d 39 54 2f 03 18 +2c 10 2d 03 2e 19 0b 02 01 24 0b 07 2d 6d 1c 11 1c 36 0f 08 23 16 02 11 09 08 4f 2d 5f 0a 02 01 +2a 05 01 05 01 05 1b 2c 50 0d 01 0f 40 01 15 1c 01 0d 21 01 01 03 03 03 02 02 24 13 0a 36 2f 03 +45 3a 28 06 1c 1b 13 0b 04 78 04 33 33 04 27 02 23 23 04 1a 0a 09 01 25 05 2e 21 09 05 1b 31 02 +16 04 4b 09 09 40 0f 15 0c 26 0d 09 29 22 4a 01 02 04 04 06 22 03 23 14 02 17 43 13 30 03 06 09 +05 0f 49 1d 03 03 20 50 02 26 15 0e 1a 09 0f 15 01 18 11 03 30 09 1d 14 25 01 0b 24 28 14 01 01 +07 21 02 09 04 11 02 34 1b 14 2c 09 21 31 06 1c 06 0c 02 29 01 03 05 21 0a 12 2b 12 05 09 15 12 +14 12 19 0a 4a 32 16 0d 2b 07 04 08 12 1a 24 07 07 18 28 28 4a 32 05 03 2e 36 12 1c 09 29 31 32 +40 03 06 2b 32 57 3d 25 15 27 0e 12 04 0e 0d 37 1e 32 08 0a 01 03 01 15 03 03 02 0a 57 0b 03 26 +05 06 14 01 10 8c 01 11 32 38 07 19 5c 34 11 3a 28 0e 33 06 01 02 54 24 19 01 28 16 21 1e 01 1a +29 1a 0a 05 33 27 41 27 17 1a 70 0c 02 02 3d 26 33 0a 02 06 39 0a 0c 02 29 05 0e 0d 1a 02 01 0e +07 06 08 1a 1a 0c 24 09 0a 19 18 18 1b 41 32 48 39 04 06 4f 05 15 23 13 41 08 1b 5f 05 06 1d 22 +d1 01 34 15 43 0d 4d 10 43 13 04 17 2a 08 23 0c 33 32 08 01 59 6e 05 05 03 08 06 1a 12 0d 0a 3a +12 08 13 0e 26 03 40 1d 05 1f 02 17 03 16 4c 03 2a 21 25 02 06 07 0f 35 0d 32 25 1e 14 39 4e 1c +02 09 01 33 01 09 01 04 10 28 35 50 0b 65 01 07 02 73 15 1f 0d 05 35 19 0c 69 07 03 04 02 02 33 +25 0c 61 22 10 36 05 53 0b 16 05 41 11 06 0c 24 0e 01 07 1c 30 05 02 08 14 22 1b 1c 0d 32 1a 34 +36 03 1f 1a 0f 1c 02 09 02 07 10 12 03 1e 0e 06 0a 10 04 02 10 38 04 01 08 02 19 58 05 2f 19 05 +2f 11 01 07 14 01 05 4b 0b 01 0c 38 09 58 2e 4a 0c 0a 39 0b 12 06 21 17 38 03 09 05 64 21 4e 20 +0a 06 06 1c 10 03 25 03 0f 08 0c 1e 02 03 39 0b 01 0f 09 7c 22 04 05 09 99 01 1c 11 29 39 41 40 +28 01 04 05 3d 07 5d 05 04 0a 12 05 0e 04 07 0f 03 0c 39 05 07 44 0a 57 09 59 17 21 1b 19 45 3a +09 03 7d 0f 23 56 0c 47 07 10 02 01 10 01 31 1e 1b 02 05 3f 05 62 2b 23 18 0e 3c 19 08 0c 20 0b +3b 03 36 10 0a 3f 28 01 59 05 01 49 09 09 3d 38 03 04 04 55 05 27 43 03 01 04 02 0c 06 2a 28 03 +28 04 23 30 33 09 1c 02 26 50 88 01 53 06 3c 0e 1a 32 05 0a 42 07 05 3a 03 06 46 42 18 08 0e 3d +03 05 01 60 17 0a 03 18 07 02 01 2f 1e 07 0c 39 0a 0d 07 03 05 04 02 01 0a 01 13 3e 26 5f 3d 11 +2b 04 0b 2a 0e 1b 07 01 01 03 03 03 01 1b 00 3f 3c 1b 14 04 02 02 03 10 0c 07 03 3c 2c 10 0e 7a +46 3a 2a 34 03 14 11 3d 03 0a 2e 10 05 55 02 10 05 3c 49 02 0a 38 4b 03 0f 25 1b 30 11 32 02 0c +0e 1c 18 18 32 37 05 03 0e 1d 37 04 08 13 38 02 1d 03 02 08 07 3d 05 0d 2b 25 07 03 18 09 09 02 +54 0a 03 24 47 16 19 05 23 43 50 03 01 05 28 01 01 03 07 3d 20 09 15 01 38 32 13 0d 05 03 09 44 +7a 03 2c 01 06 46 10 76 0f 02 10 44 0e 48 09 4b 01 2c 09 12 1c 2c 2f 12 06 06 05 11 0b 44 a9 01 +0d 19 22 2e 22 24 3d 09 0d 37 10 82 01 03 1d 05 02 25 1f 22 0d 03 17 04 01 0f 0d 03 15 3a 70 10 +66 08 17 05 18 02 37 38 18 17 03 0a 05 1f 33 2e 11 3a 09 48 33 13 09 11 10 02 06 0a 06 0a 04 03 +36 05 05 04 03 02 05 06 07 44 0e 18 06 05 1c 27 2f 04 0c 0d 09 27 18 0e 14 02 1f 20 11 03 0d 3e +08 07 0a 2b 72 0f 03 01 15 0c 1b 2c 0f 2c 03 1f 38 0f 38 3e 01 06 09 0d 1e 02 01 0c 20 12 07 02 +0c 0c 05 04 0b 05 2d 01 03 01 0f 1c 49 64 22 7d 04 29 05 13 0b 07 0e 1b 1e 01 4b 0c 4a 28 0b 1b +22 06 04 08 29 10 35 1e 02 0d 4d 05 03 06 01 05 02 05 03 05 27 29 48 34 31 28 39 03 08 4e 07 09 +03 02 0b 02 2a 62 03 01 04 09 08 06 a5 01 0c 07 0e 2d 21 13 24 0e 0b 23 47 44 09 18 0a 02 0c 21 +02 02 03 0b 0c 02 02 03 05 05 1c 0e 18 37 36 0a 07 01 09 0c 06 17 0c 34 04 03 05 11 10 43 5c 06 +03 0e 5a 02 13 39 09 52 11 24 04 02 12 18 12 50 17 04 0d 52 19 46 fa 01 14 12 52 22 20 06 82 01 +53 05 11 1c 05 09 08 06 03 05 02 05 34 52 04 0d 03 05 06 4f 32 30 0a 04 3e 05 4c 05 06 05 05 0d +04 05 50 0c 28 03 12 3f 14 11 12 17 06 9a 01 11 04 01 09 55 19 0b 01 0b 01 69 27 61 02 35 12 0b +16 01 06 03 03 01 52 04 1a 74 0a 4d 13 0d 0f 09 50 0e 12 0d 57 3f 3f 47 0a 3b 13 05 0c 15 23 0f +01 57 08 0c 08 3b 13 04 0e 29 11 16 1f 01 02 2a 05 01 22 13 0c 08 06 1d 04 0c 0b 58 2d 2c 15 09 +56 1c 0b 21 1b 5f 0f 59 0f 03 59 0c 35 1c 59 3c 22 1f 0f 06 19 43 23 06 0a 10 40 5b 13 02 02 01 +04 37 2f 11 01 0f 03 16 2c 07 04 0e 1f 49 17 04 09 5b 3c 3d 20 1a 28 21 20 04 1d 21 05 0f 0d 20 +5e 05 1f 34 2d 02 40 40 2b 42 0d 61 02 01 a3 01 4f 3f 01 2e 3c 64 12 52 0d 19 48 4e 01 0c 11 14 +05 03 0a 67 0b 48 05 1e 12 a8 01 43 02 0d 08 03 6a 07 8b 01 62 03 01 09 23 2d 17 0f 09 08 6a 07 +01 13 07 04 07 03 06 0d 38 09 16 3a 01 02 06 3a 3e 01 0b 03 03 02 03 0e 02 04 02 03 11 6a 0f 89 +01 21 02 27 12 10 1f 22 24 11 06 0a 08 04 52 18 15 0f 03 74 19 06 3f 46 09 22 1f 6a 25 01 0e 22 +0e 28 3b 02 19 3b 06 20 17 1d 01 10 0e 6b 2c 04 0a 2b 23 10 07 22 40 62 03 05 10 06 06 6b 06 01 +3c 05 50 08 03 30 04 02 06 17 10 01 38 37 04 6c 08 0f 07 05 01 0b 32 0c 1d 17 42 06 16 01 09 07 +42 19 25 6f 03 13 04 52 1a 0a 42 0e 28 43 19 07 0f 11 36 03 39 42 06 3b 82 01 03 0c 07 01 01 04 +31 1b 09 0e 04 01 72 17 6a 0b 05 40 03 04 01 01 04 07 53 03 0d 0a 04 18 01 01 02 0c 02 74 05 0d +03 0a 0e ae 01 2a 4f 25 76 07 05 31 3b 06 42 74 03 09 05 05 0d 14 04 0e 79 2e 03 01 35 2c 02 42 +11 15 0c 09 28 38 0b 79 01 0e 19 3c 0a 08 34 0f 1a 18 14 10 05 06 03 19 03 21 18 06 90 01 04 04 +02 1a 57 19 1d 24 09 13 01 2b 14 12 80 01 06 4c 05 06 1c 02 03 5b 02 79 85 01 1c 1b 17 1b 01 04 +04 39 01 1b 21 0e 04 07 10 2e 08 0a 09 0f 86 01 18 07 bc 01 1c 20 31 12 04 06 09 52 35 4d 2f 33 +05 0b 05 01 32 45 20 09 06 8c 01 05 26 03 05 26 19 33 0e 0d 02 2c 01 0b 01 08 17 39 03 02 1c 8c +01 2f 0d 0e 28 33 cc 01 90 01 1b 08 01 27 0d 0d 5b 2a 5d 16 9b 01 29 7a 02 0c 1c 07 45 0d 30 64 +18 2e 05 0b 3d 89 01 06 0d 1b 68 42 03 24 06 16 08 03 0b 03 36 0f 0c 84 01 0f 10 d4 01 44 32 01 +46 20 09 21 0e 10 02 0e 9f 01 10 1e 15 02 58 06 2a 04 08 0b 12 57 fb 01 05 27 17 02 01 20 0a 0a +0c 29 04 28 0c 05 0e 1c 99 01 0a 02 0a 08 11 04 2a 10 32 24 2a 12 33 c7 01 0f 15 2f 16 0e 09 24 +de 01 ee 01 a7 02 19 04 03 05 23 01 0d 1a 39 0d 04 01 02 10 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +98 5c 38 5d 4c 5e 04 5c b0 5b 10 5b 10 5d 50 5b 60 5c a8 5d b0 5d 00 57 d8 5a a0 5a d4 5c 68 5c +20 57 00 5b a8 5e e0 57 68 5b 08 5e 24 5c d0 56 a0 5d 74 5e 50 5b 10 58 f0 57 dc 5c c8 5a 3c 5e +60 50 ec 5d 9c 5d 94 5d 68 5a f4 5d 6c 5d 7c 5c 38 5c c0 59 f0 5b 60 53 30 5c 08 58 58 5e b4 5d +98 58 e0 56 e4 5d e8 59 e0 57 68 5b f0 5d e0 56 c0 50 b4 5d 34 5f 60 58 e8 5a 08 5b f0 5b d8 5c +34 5e 44 5d c0 55 e0 58 24 5c d4 5c 08 5c 40 56 d8 5d 64 5e 08 5e c0 51 60 56 40 59 c4 5d e0 5c +a0 58 08 5b b4 5e 68 5d 20 5f 58 5b 28 5c 68 5e b0 5a f0 5b 68 5d a8 58 20 5e d0 5a b8 59 b0 5b +a4 5c f0 5a d0 5b c8 5b 74 5e e0 58 04 5d f8 59 58 5f 38 5f 40 5b bc 5c f4 5c 78 5a 80 5b d4 5c +6c 5c 58 5c 74 5d 00 54 b8 59 28 5a 48 5e f0 54 c4 5e f8 5d fc 5d b8 59 88 5b 20 56 68 5c 00 4d +b4 5c 90 59 ec 5c c0 4d 28 5d 10 5b 70 5e 6c 5c a0 5b 68 5a 04 5e 20 55 50 59 70 5b f4 5c c0 4f +18 5a b0 5d 58 5f e0 58 28 5e a8 5a c0 56 c0 54 90 56 e0 5c 54 5c 90 59 b0 58 70 55 f8 59 0c 5d +28 5a 6c 5c a4 5e a0 5d 80 50 c8 5a a0 5a dc 5c 58 5c c0 5c 60 5d f8 5b a8 5d 28 5e d4 5f 40 5d +90 58 4c 5d 44 5f ac 5c 54 5d 50 56 60 56 60 5d 68 5d 68 5d e0 5f f0 56 10 54 10 54 3c 5e b0 5e +c8 5a 6c 5c 2c 5d 00 47 cc 5d a0 53 70 5c 40 5d 80 55 5c 5f f4 5c f0 5b b8 58 18 58 c0 5c 48 5c +70 59 84 5f 5c 5c d0 57 84 5e 78 5e c0 5c 00 58 18 5a 08 5d 64 5e b8 5c 20 57 d8 58 94 5d 98 5a +b0 58 28 5d c8 5c b0 58 14 5d 80 58 a0 51 00 5a fc 5c 58 5a 68 5e 00 5d 88 5c 00 59 84 5c d8 5b +90 5d 90 59 80 5c dc 5c b0 5c 88 5b a0 54 f4 5d 20 58 98 5a a0 5d 40 51 30 5e f0 59 4c 5d e0 5d +d8 5b 88 59 f0 5b 00 52 10 54 60 5e 3c 5d 04 5c 10 5b 18 5b 94 5d 90 54 70 5b 30 54 30 59 54 5d +c0 54 90 5e 04 5e 3c 5d d8 5d 20 54 40 5e 20 53 a0 55 80 5c 14 5f 90 55 a0 5d 50 57 a8 5d 98 5d +88 5f 14 5c c8 5b 00 58 30 5e 88 5b 04 5d f0 56 1c 5e 24 5f dc 5e 70 54 64 5e 40 51 d8 59 b0 5b +e8 5c 38 5f 88 5f 50 55 b8 5c 70 58 fc 5f 80 59 20 59 10 5e 40 5e 40 4c f8 5f c8 5a f8 5e b0 5b +08 5c b8 5e f4 5c 10 5d 80 48 40 5f 5c 5d 18 5e 80 5d b8 5f c8 5d 80 52 30 5a 78 58 f8 5d 20 56 +40 5d b0 5d f8 59 40 55 b4 5c 1c 5c 28 5e c4 5c 94 5c ac 5f 48 59 78 5c c8 5a 40 51 10 5f 98 5d +a4 5d 30 59 b4 5f 24 5f 7c 5d a0 59 e8 58 a0 5c c8 58 f8 5f a0 5c 80 51 3c 5e 20 53 4c 5c 60 5b +20 5f 70 5d 04 5d 38 58 e0 52 60 5c 78 5d 60 51 88 5a 28 5d f0 5f 40 55 18 58 24 5f 5c 5d f8 58 +d0 5f 40 5b d0 5d 70 5e a4 5e 90 5c c0 5c e0 57 38 5d 60 54 4c 5d f0 58 60 5a 90 58 cc 5d 80 59 +ec 5d 1c 5c 94 5f 50 58 c0 5b 54 5c 78 5a 00 56 d0 55 b0 5a 10 5f b0 55 38 5c 3c 5c 80 4d a8 5e +c8 5e 48 5a 00 5b 50 5b 40 4d 58 5d 90 5b 34 5e 80 52 38 5d 80 5c 28 58 20 5c 2c 5d 94 5c e0 5b +a4 5c 98 5c 30 54 08 5b 00 5c f0 57 60 5e 44 5f 5c 5c a0 5a 20 5c 18 59 64 5d 74 5c b0 5d cc 5c +a0 59 a0 5a 84 5c 00 5d 08 5b 90 59 80 55 c0 5e 08 5d 50 5f f8 5b 60 58 90 54 e4 5c b8 5d ac 5f +70 5d d4 5c b8 5b 38 5d 80 53 00 51 e4 5e 90 59 10 5c a0 5c 90 59 c0 4d 20 58 68 5c f0 5e f0 5f +cc 5e 44 5d 58 5b b8 5e a0 5d a0 54 84 5e 70 57 b0 54 98 5d a8 5e 78 58 24 5d 88 58 c0 57 7c 5e +34 5e 34 5c a0 5d 50 54 00 42 a0 5c fc 5c f0 55 18 58 d8 5c 08 5a 30 5a 50 56 20 58 78 5f e0 59 +00 60 48 5d 24 5e 88 5a 54 5d c0 57 a8 58 fc 5c b0 5c 3c 5e a0 58 a0 5c 30 59 64 5f e0 5c 04 5d +a8 58 cc 5c 38 5c 80 53 20 5e 3c 5c bc 5c 98 59 80 5e 14 5f 88 5b 80 58 c0 50 b8 58 18 59 04 5e +84 5c 88 5d 6c 5f f0 54 b0 56 0c 5e bc 5d c0 4f 70 5d 4c 5f 38 5b 68 5a 00 4c e4 5e c0 5b 7c 5d +98 59 10 5a f8 5d f0 59 80 56 00 5d 94 5c e8 5b e0 5d d4 5f 7c 5c f8 5d f0 55 90 5e 20 51 bc 5e +20 56 c0 50 70 5b 20 55 e8 5b b4 5e 2c 5c d0 5b 38 59 c0 57 c0 5a e8 5b 30 59 f8 59 c8 5b 1c 5d +a0 57 c0 50 38 5d 7c 5c 20 5b 00 5b 84 5d 20 57 e8 5b 9c 5c 80 4c 40 54 d8 5a 18 5b b4 5d 10 5a +d0 5a 00 59 64 5d 60 53 00 59 d8 58 20 5d a8 58 44 5f c8 5d 20 5a 08 5d 08 5a a0 57 90 58 b0 56 +48 59 c0 50 b0 5e 50 5a e0 5f b8 5e 6c 5c 60 52 1c 5e ec 5f 50 59 00 4b e8 5b c0 5c d8 5c e0 54 +8c 5c d8 5c 30 5b 04 5e 58 59 a0 5d a8 5b 00 45 44 5f 60 5d e0 56 98 58 58 5c 00 53 58 5b 00 5f +f0 5b cc 5c e0 59 78 5a 28 5a 80 54 7c 5c 40 55 f4 5d 30 5d c0 52 30 5b a0 50 70 5e 98 5d d4 5f +a0 51 28 59 ec 5f b0 5a 8c 5f 54 5d 88 5c 80 49 e8 59 30 56 c4 5d a0 5b e0 56 44 5e 7c 5e 3c 5d +7c 5d 64 5e 3c 5f 38 5a 44 5e cc 5f 38 5f c8 5b d0 5c 30 5f 60 5a 20 5d 30 5a 80 59 c4 5e 34 5f +30 58 30 5f f0 5b e4 5d f4 5d a8 5c 64 5e 60 5b 20 58 e4 5f 00 5c ec 5c e0 57 20 5e a0 5a 88 5e +10 5b e0 53 48 5d f0 58 e0 59 cc 5d 80 59 20 53 8c 5e 28 5a 60 51 1c 5e a8 59 38 5f 28 5d a8 58 +b8 5a 2c 5c a0 5d 04 5c 08 5b bc 5f 98 5d 10 5e 08 5e 08 5d 00 48 e0 59 b0 57 40 59 34 5c a0 53 +c0 50 74 5c 8c 5f 08 59 88 5f b0 59 d8 59 80 4b 90 58 38 5d 98 58 e0 59 60 58 90 56 80 59 30 55 +10 5d 2c 5d 98 5b d8 5b d4 5e f4 5f e8 59 f0 59 f0 57 44 5c c0 55 60 57 48 5a c0 52 c0 58 a4 5d +58 5b f0 5c 64 5d 40 50 78 5b 00 40 c8 58 60 54 60 58 60 5b 70 55 c0 57 a4 5c 58 59 6c 5c 74 5e +b8 5e 48 5a 90 5e 98 5f 80 5c 5c 5f 4c 5c b8 5c ac 5d a0 5c a4 5c a8 5a 24 5c cc 5c 60 52 6c 5f +2c 5c 14 5c 5c 5f c8 5d c4 5c d0 5d 10 5a 84 5d 80 58 18 58 40 50 0c 5d 30 57 18 5d 10 5d 70 5e +d8 5a b0 5f 2c 5e e8 59 30 57 68 5d c4 5d 80 49 d0 5f a8 58 8c 5d 88 5a c8 5e 5c 5f a0 5c 9c 5c +c0 4e 40 4d b4 5e 18 58 1c 5e a8 5a c0 4f 24 5c 18 5e c0 5e 14 5e f0 5b d4 5f d8 5e cc 5c 04 5f +00 4d d8 5e e0 50 00 4a f0 5d 80 5e 28 58 20 51 40 4e 44 5c a0 58 30 5c e0 54 48 5f f8 59 dc 5e +80 5c a8 58 18 5d 80 53 e8 59 84 5c 30 5a 80 57 b8 5d 80 4b a0 58 30 57 74 5c 7c 5c 58 5c 10 5c +80 4e 78 5a 1c 5e 00 59 f0 57 00 60 54 5d 78 5e 18 5b b4 5e 04 5e 40 5e e0 5f b4 5e f0 57 a8 5d +70 57 94 5d 58 58 5c 5e 1c 5d 80 5d 5c 5d d8 5a c0 59 e8 5c 30 56 48 59 70 59 18 59 14 5e 54 5d +70 5c 8c 5d 10 5e 68 5e 50 58 10 5d b8 59 30 5b 1c 5e a0 5a 20 5b 30 55 70 5e 4c 5d 94 5f 28 5e +b0 57 84 5e e8 5b cc 5d e0 5c 50 5c 74 5e 14 5f c0 57 50 58 10 59 00 4f 6c 5e 88 5a 50 55 b4 5e +80 55 50 5d e4 5f f4 5d 80 58 10 5d 6c 5d f0 5e 18 5f 00 53 b8 5a d4 5e d8 5c 50 59 98 5e 24 5d +80 4a 98 5f 0c 5f 0c 5e e8 5a 4c 5f 94 5f d8 5f 08 5c 98 58 60 54 04 5c e8 5b e0 56 e0 5a 08 58 +00 4f f0 59 64 5c 70 5f f8 5c b4 5c 50 59 00 60 98 5d 40 4f dc 5c 70 55 c0 5f e0 57 24 5d 84 5f +c0 4c c0 5b 98 5e 44 5d 80 4b 98 5a 90 56 5c 5f 28 59 18 5e b8 59 60 53 d0 57 b0 55 bc 5d 14 5c +60 5a 00 60 e8 5b 14 5e f8 5c a0 5f 7c 5f 10 56 20 51 b0 56 f0 54 80 52 f0 5e a0 54 28 5c fc 5d +20 5c 4c 5f 18 5b 50 5c 20 57 3c 5e 18 5e bc 5e 68 5d d0 5e cc 5d 00 4a 5c 5f 04 5d ec 5e 44 5c +00 57 f0 59 c4 5f e8 5c 84 5d 18 5c 20 5c 80 4f 60 5b a0 55 a0 5d 88 5c c8 5c d0 59 38 5b 54 5c +40 4d f0 5d 80 58 50 5e c0 4e e0 57 28 5d 58 5e 58 5a 3c 5c c0 4f 00 4e a8 5f a0 51 24 5d dc 5e +e0 51 2c 5c 70 59 b4 5c 2c 5f 60 5f 90 56 1c 5d a0 5a 20 51 b0 5c f8 59 d0 5d f0 58 f8 59 30 5c +00 4b d4 5f 44 5c e0 5f a0 54 60 5e 30 59 3c 5f d0 56 94 5d 68 5a 88 5a ec 5e 44 5e ac 5f 0c 5d +40 4d 1c 5d 98 5d 88 5b 40 54 44 5f a8 5b b0 5f c0 52 08 59 98 58 e0 54 d0 5d c0 5c e4 5d 00 46 +b4 5c 40 5b e0 5e 50 58 70 5a f8 5e c4 5e 18 5a d8 58 98 5c f8 59 68 5e e0 5c 90 54 8c 5e 20 5e +80 4a 18 5b 94 5f 14 5f 40 4c e0 57 6c 5c 60 5c fc 5d 40 5a e4 5d 00 5b 78 5a b0 56 c0 4f c0 5e +40 51 2c 5e 10 58 b8 5b 28 5b 3c 5d e0 5f 60 54 b0 5c 00 45 9c 5d 70 54 70 5e 40 5c a0 50 40 5d +68 5c 48 5b ec 5d e8 59 c4 5d e0 58 a8 5e 20 57 70 5d 00 3c 28 5d 28 59 c0 4e 80 4d 24 5f 50 59 +d8 59 38 5b bc 5c 88 5b 00 48 88 59 08 5e 78 5a 00 3c f8 5e 54 5e 24 5e d8 5d e0 59 b0 59 88 58 +90 59 c0 53 64 5c 88 5a 60 52 c0 50 58 58 40 58 a0 52 f0 5d a0 5c 00 50 90 5b 00 50 48 5b c4 5f +50 58 60 57 38 5a 44 5e 80 4f 80 4c 8c 5e 20 5a f0 55 40 5b 9c 5f 00 44 40 55 b0 5a 00 57 d8 58 +28 5b 40 51 8c 5f 90 5b 00 50 b8 58 10 56 68 58 00 56 c0 5b bc 5d 50 5a 2c 5e 50 56 20 51 90 5b +20 56 80 4b d0 5e 44 5e 30 54 08 5d bc 5d 90 58 80 4b c8 5e c4 5d 68 5b e8 5e 00 44 00 60 68 5a +40 57 50 58 c0 5e 50 58 18 5a 80 52 58 5c 8c 5d 14 5c 4c 5d 5c 5f a0 50 70 5e 40 57 b0 56 bc 5f +38 5b d0 54 b0 59 00 44 d0 59 00 52 48 5d 80 50 40 5a b0 5c 4c 5e c0 5b 38 5e a0 5d 80 55 c4 5f +48 5a 70 5c e0 59 20 58 90 5a 94 5d e0 5d c0 4f 38 5a 10 5b 40 5d 40 5e 28 5c d0 58 40 52 90 5a +b4 5d 20 59 34 5d 80 4a 40 57 70 59 28 5c 00 40 b8 59 e0 50 a0 5f 78 58 bc 5c 50 5b 10 5f 4c 5c +a0 54 90 5c 18 5b 10 5b 84 5d 40 5b d0 5b ec 5d 00 51 24 5e e8 5e 60 5d 74 5c 08 58 9c 5e 38 5c +08 5b 90 57 f0 5a a0 5c 54 5f 30 54 58 5b 00 4d d8 59 08 5d e0 5f f0 54 38 5d 6c 5c 10 54 10 5d +f0 5d 60 55 78 5d f0 5b 70 5f a4 5c 34 5e 40 5a 00 4c 70 5a 48 5e d0 58 c0 4c 20 50 b0 5b e0 5d +f8 58 40 5b 0c 5e 04 5f a0 51 d0 5b b8 58 70 57 10 55 2c 5d 20 5b d0 55 38 5b 80 55 4c 5c 80 49 +50 54 30 58 50 5f b8 5d e0 56 78 5a 04 5e 70 5c f8 58 a4 5e 48 5c 94 5f 00 4f 34 5c f4 5e 90 5c +88 5b d0 5a 40 57 00 4d d8 5b 1c 5c 24 5f b8 58 94 5c 84 5f e8 5f 08 5e 68 5d 60 58 f4 5d a4 5e +80 4a 70 55 44 5d b4 5c 20 5a 10 56 a8 5d 50 5c ac 5c 38 59 a0 5d b4 5d 00 58 68 5d e0 53 68 5f +e0 59 30 58 88 59 a0 55 ec 5c 00 4b e4 5c 38 59 cc 5c 18 5a 84 5e 00 55 80 57 d0 5b 1c 5d 80 50 +f8 5a 54 5c 48 5a 00 4d f0 59 e0 53 e0 5d 00 56 20 51 40 56 c0 5e 00 42 80 50 dc 5f 30 56 20 5c +90 54 50 55 98 5d cc 5f e8 5f 00 5f e0 5e 90 57 08 58 ac 5c cc 5d 00 57 c0 51 b8 5c 94 5d c0 5b +38 5c e4 5c cc 5e 48 5d 10 5c 2c 5f c0 57 60 54 40 51 58 5f d0 5f 90 54 60 58 00 45 80 48 f8 58 +e0 57 40 50 f0 5d b0 5f c0 56 20 5a 30 5a c8 5e 28 5b 88 5f 44 5d 28 59 40 51 e0 53 90 58 c8 5d +18 59 c8 58 c0 5a e0 53 80 49 8c 5d 40 5c 70 54 50 56 ec 5e 30 5d d4 5d 98 5c f0 58 1c 5c f0 57 +b0 5c a0 5b d4 5d 60 5c 30 5a 34 5c f8 5a 80 4f 40 50 d0 56 38 5b 40 53 d8 58 84 5c 50 54 20 59 +44 5f cc 5e 00 4f d4 5d 80 4d 9c 5d e8 5c 90 55 ac 5e 84 5c 8c 5d e0 54 a0 55 58 5b ec 5e 8c 5d +10 5f 90 5b 90 56 e0 5c 7c 5d 98 5e d4 5c 00 4f 3c 5c 70 59 cc 5e e0 52 00 58 08 5b 00 55 20 5d +50 57 0c 5f 60 52 fc 5d 38 5b 64 5e a0 56 10 55 80 5e 5c 5c 78 5c 60 58 c8 58 68 58 b0 5b 6c 5e +5c 5e 10 5c 60 53 c0 55 fc 5e 38 58 d4 5e 40 56 50 5c 74 5e 70 56 90 57 38 5a 14 5c 54 5f 40 5f +5c 5e 9c 5f b0 54 b0 55 50 58 90 5e 90 56 e0 52 78 58 c8 58 0c 5c c8 5b 00 56 e0 56 e0 5c c0 5a +08 5e e0 5b 00 54 50 56 30 5d 00 51 38 58 00 4a a8 58 70 5e 08 5a e0 56 08 58 cc 5e 58 5d 28 5b +ec 5f c8 5a c0 4c 60 56 28 58 54 5e 40 52 80 57 54 5f b0 5b 90 5f 20 5a 34 5d d0 5d e0 5c 30 5f +f0 58 90 57 50 54 e0 57 3c 5d 48 58 68 5c 80 5e 38 59 68 5f 64 5f 90 57 60 52 c0 5b 84 5d f4 5d +24 5e 9c 5d c0 51 3c 5d 78 5e 60 5b 48 5b 9c 5d 04 5e 14 5e d8 5a 40 5c c8 5c 78 5c 90 57 f0 5b +48 5d 00 4e e0 58 c0 4c 58 59 58 5d 00 52 88 5a a0 5b 60 5e f0 5e 60 5d 80 4c 40 4f 30 5a 64 5c +40 58 00 58 78 59 c8 5d 94 5d fc 5f e4 5f 90 57 60 5e c4 5d a8 5c e0 59 58 5a c0 52 ac 5d e4 5f +8c 5f 40 5b 44 5d a0 51 68 5a d0 5e 7c 5f 90 5c ec 5f 60 57 00 60 e0 57 60 58 68 5c 70 5f c0 5f +48 5e 78 58 20 55 1c 5c 28 58 20 55 60 5d 20 59 00 4d 04 5f 64 5f 48 5b 28 5c c0 4d bc 5c 20 58 +20 51 30 5a 10 54 58 59 f8 59 e8 5b b8 59 88 5e 98 58 e8 5d f4 5c 00 4b 80 57 c4 5c c8 5d e8 5a +c4 5c f8 5f 00 47 f0 5a 18 5b 20 5b e8 58 a8 5b 00 50 48 5c 24 5f 90 5a 70 5b 90 54 30 5b 48 5c +90 5f 20 5d e0 58 40 54 ac 5e 78 59 20 54 a4 5e 2c 5e 90 5c 6c 5c 00 58 c0 4f 2c 5c 00 44 80 55 +ac 5c 08 5e 00 51 cc 5c b4 5c 20 58 c0 4c 80 57 40 5d 58 5a 30 5a f0 58 00 4d a4 5c 8c 5d a0 51 +c0 5e c0 4f 00 4c 28 59 40 5e a0 5f c0 51 88 5a c0 52 b0 5e 54 5c 1c 5c f0 54 10 54 24 5e 90 5c +28 58 38 5b f4 5c 60 5d ac 5f 14 5d b8 5b 98 5c 08 5e 48 5a 50 58 d8 5a 74 5e b0 5a f0 57 b8 59 +90 5b 40 5a 80 59 00 5e 90 5f 18 5a 78 5e 10 5c 30 5a 20 5c e8 5f a0 5a 2c 5e a0 59 5c 5c 74 5e +00 55 48 58 c0 5b 08 5b 34 5f 58 5d 70 5c 88 5d a8 5c 10 54 58 5d 00 57 bc 5e 28 5c a0 5e a0 56 +48 5b 54 5f 90 5c a8 5b 54 5e 38 5a 20 55 60 5e 88 5b 48 58 60 5c 00 56 88 5f 40 4e 70 56 24 5d +c0 56 08 5d 7c 5f c8 5f c4 5f d8 5a 0c 5d f4 5e 80 59 94 5c bc 5f c8 5b 50 56 90 54 c0 56 d0 54 +00 42 9c 5c f0 5c 40 5c 28 5e 00 50 04 5d 9c 5d 14 5c b8 5e 10 58 20 5b d0 5c b8 5d c0 51 30 5a +a8 5b c8 58 fc 5c 28 5d 48 5f 18 5a 28 5e 08 59 a0 54 f0 5e d8 5a a8 58 18 5f 80 54 d8 5b 80 48 +18 5e 38 5a 44 5f 18 5c a4 5d 20 59 60 50 e8 5e 78 5b 68 5a 20 58 80 57 50 57 80 59 f0 56 78 5a +58 5c 24 5c c8 5f e4 5d d0 5e 78 5c cc 5f 98 5a 00 4c e0 53 9c 5e 00 57 f0 5f 0c 5e 90 56 c4 5e +20 59 58 59 10 5e 38 5a a4 5f 78 5f a8 5c 60 5f b8 5f 80 50 84 5c 58 5a 98 59 50 5a 04 5c b8 5d +a4 5c 2c 5f 30 5f 84 5d 28 5f 38 5e 88 5a bc 5c a0 50 c0 5f e0 5a 60 53 88 5a 24 5d 90 57 98 5d +00 5d ec 5f c8 5e 20 5b ec 5f e4 5e a4 5f 00 60 c0 5c ec 5c 98 58 ac 5c 64 5f e8 59 c8 5a d8 5c +a8 58 fc 5d bc 5d 50 5c 60 5e 80 4f 9c 5e c0 5f 10 58 00 54 78 5f a8 5e 28 5a b0 54 40 5c a8 5a +88 5a cc 5c d8 5c 80 4c c8 5f 48 5c 18 59 98 5d 8c 5d b8 5e a8 5c 00 5b 60 58 88 59 58 5b 80 51 +40 5b 40 57 64 5e 74 5d 6c 5f 8c 5e c8 58 40 59 68 5c f0 58 ac 5c 00 3c b0 57 00 46 28 59 80 52 +00 58 c0 4c 60 5c e0 53 b0 5d 8c 5c 08 5a b8 5f 90 5d f0 5e c8 5b 74 5c 50 5e 88 5a 2c 5f 80 5a +48 5c 98 59 10 5e fc 5f e0 5f 60 54 98 5e 94 5c 7c 5d 60 5a d0 56 a4 5c 2c 5f f0 5a 28 5f e0 5e +c0 55 80 4c 58 59 3c 5e 60 5c 80 52 00 5c 98 5b 50 58 c8 59 dc 5f 84 5c 9c 5d 78 5d 28 5d 20 54 +d0 5b cc 5e 80 48 a0 5f 54 5f e0 55 80 5c a4 5e a0 5b 38 5c f8 5d 00 4a 50 5b b0 55 68 58 98 5b +b4 5f d8 5e c4 5e e0 5e 80 5b 80 49 0c 5e 9c 5e 10 5b d8 58 8c 5e 20 58 58 5f 38 59 38 59 7c 5e +c0 58 ac 5f b0 5d 10 5d 78 5d 80 53 a8 5a c4 5e 10 58 34 5f 80 58 98 5c a8 58 00 3c 08 5b 60 5c +14 5d 88 5f a0 59 34 5f 68 5e 98 5a 80 54 20 5c 3c 5c c0 5a c4 5d 80 56 84 5f 60 5f 68 5d 70 5a +b8 5b 10 5b b8 5a 4c 5e 94 5e 8c 5f dc 5d f0 5f fc 5e 74 5f 2c 5e 90 59 b0 58 20 5e 80 5b 40 5a +8c 5c 14 5c 78 5c 88 5c 40 5b 64 5d 28 5e f8 5a 88 5e e8 5e 04 5f e8 5e 98 5a 80 50 e8 58 c0 5d +38 5d 0c 5d b0 5a ac 5c d4 5e 00 59 f0 5a 18 59 90 5d 6c 5e d4 5e c8 5e c4 5d cc 5f 00 5a 70 5a +74 5f 18 5a 98 5a 00 50 14 5d 70 59 28 5d ec 5c 64 5d 94 5c a4 5c 28 5d e0 5e 18 5d a0 56 40 5d +68 5a 6c 5e a0 58 d0 58 88 5f c0 58 88 5d 90 5d 90 5e e4 5d f4 5c c0 5c 3c 5f a4 5e 58 5b f4 5f +48 5d 50 56 10 56 2c 5f 10 5e b0 55 28 5e 10 5d 2c 5e e8 5b 70 5d 70 5b 84 5e 50 5c 9c 5e 64 5d +04 5c a0 53 a0 50 e8 5a 98 5f b8 5c 18 5b 44 5d 90 5f fc 5c 80 50 50 59 9c 5c 64 5d cc 5c 34 5d +d0 5e 00 5a e0 51 90 54 b4 5f 10 59 08 5d 40 50 a8 5a 6c 5e 60 5a 44 5c 84 5e 40 4f 54 5d 70 5b +48 5f a0 5a b8 5e d8 59 a0 5c e0 50 0c 5e 80 5d 84 5c a0 5f e0 50 50 5c d8 5b 48 5b 68 5a 30 55 +70 59 0c 5d e0 50 3c 5f a0 5c 90 57 40 58 48 5b dc 5d 94 5c 08 58 14 5d 30 5f 00 51 a8 5a dc 5c +18 5b 50 5d 10 5c 60 58 08 5a 58 5b d8 5a 50 58 80 5b 78 5f 00 50 20 5a 48 5f d0 59 40 50 78 5a +40 5f 80 5a 88 58 4c 5c 4c 5e 30 57 78 5e 80 56 70 58 18 5f a4 5c b0 58 b0 5b 0c 5f 00 5a 30 5f +ac 5c a8 5c ac 5c 20 5e 40 54 18 5a 58 5a 10 5a 24 5f 2c 5e 68 5a 94 5d 58 5c a0 53 80 5e 40 54 +a0 5d 10 5b b0 55 20 59 d8 5a 00 54 e0 59 d0 56 2c 5e 24 5d 60 5f f0 5f 04 5d 60 50 c0 5a d0 58 +d0 55 00 51 e8 59 10 55 28 59 dc 5c 80 5d 1c 5e 98 5d ec 5f 68 5a 48 5a 20 5e a0 5c 70 5c 40 59 +e0 58 0c 5f ec 5c 00 52 e8 5e e0 5c 80 5c d0 59 20 5f 0c 5e a0 50 08 5f fc 5f 08 5e 68 5e 94 5d +a0 55 a8 58 d0 54 0c 5c 7c 5e e8 5e f0 59 50 55 5c 5f 30 5f 9c 5c 58 5b 48 5c 20 52 a0 53 d0 54 +f4 5c a0 50 80 58 c8 5d d4 5c 04 5d f0 57 4c 5f 20 5f e8 5e 40 5c 84 5e 70 55 a0 55 20 5b e4 5e +e0 53 28 5c 10 58 10 56 40 5c 60 57 40 5a 30 5c a0 5e d4 5e 48 5f 58 5e 14 5c c0 4c 50 5f 3c 5f +1c 5d f0 57 e4 5d e4 5e 18 5a f0 57 a0 59 38 5f 28 5e cc 5e 68 5d e0 50 74 5f 68 58 e0 55 88 5d +84 5d e0 59 98 5a 08 5d d8 59 40 52 d0 58 c0 55 bc 5f 50 59 30 59 e0 53 d8 5d d8 5e 80 53 80 5a +e0 5e 60 58 04 5e d0 5f 30 5f e4 5e a0 5e 70 58 58 5e d0 58 30 59 2c 5f c0 4d 88 5a 10 5b c0 5c +3c 5f 60 55 20 59 e0 57 c0 4d d0 5a 00 5e d0 58 0c 5e ec 5c 90 5e 40 56 a0 5d 8c 5c e8 58 80 55 +a0 57 0c 5d 10 5d 8c 5e 84 5c 90 57 f0 5e 10 54 b4 5c 10 59 7c 5d 20 5d c8 59 28 59 5c 5e 80 4b +0c 5f 84 5c d0 5d 78 58 c8 59 c0 53 30 5d c8 5c f8 5c 3c 5f c8 5e 08 59 80 4d 78 5e 00 58 80 51 +80 4e bc 5d 08 5b 00 4b a0 5c 60 5b e4 5f a0 5b 0c 5d b8 5f 90 5f 68 5c a0 5f 00 3c 58 5a 2c 5c +c8 5a e0 5b d0 5f 80 48 68 58 cc 5c a8 5f 40 52 9c 5f 80 4e d4 5f 7c 5d 40 5c e8 5c 60 5f a0 5c +64 5f c0 5f b8 5b 58 5d 00 4c 28 58 f0 5e 78 5d e4 5d 14 5c 8c 5c 64 5e c0 53 40 50 58 5b 40 4d +28 5e 14 5e f8 5f 00 56 70 56 bc 5e 28 5c 60 57 c8 5c 50 59 90 5a 2c 5f 9c 5f bc 5c d8 5e 00 40 +30 56 98 5b ec 5d 40 52 48 5e 40 4e 48 5f 8c 5d 84 5f 80 59 04 5f e8 59 60 51 40 5b f8 5d 20 5a +d8 5c 38 59 8c 5c 00 46 58 59 a0 56 d8 5c c0 56 dc 5e 20 5a f0 5a 4c 5c a8 59 18 5e 94 5c 68 5b +4c 5f a0 57 f4 5d c8 58 5c 5c 50 5d 00 60 a0 58 10 58 38 5a b0 5b 1c 5d 78 5a 90 54 f4 5f a0 54 +c4 5c 14 5d dc 5c 80 52 e0 57 20 50 e0 5d 94 5e 10 5f c0 53 d0 5a 80 4b f8 5c c0 5c 20 5f a0 52 +20 5a d8 59 c8 5f 70 56 04 5d 00 55 90 5c 38 5d 80 58 e0 5a 60 53 40 5d 18 58 78 59 40 5e e0 56 +a8 59 50 56 54 5c 58 5f 54 5c 28 5c cc 5e c0 55 44 5c b8 59 44 5d 80 4c 10 55 10 59 b0 59 40 55 +00 54 30 58 90 5b c0 4c e0 57 60 5b e0 58 50 5c 9c 5d 40 54 30 5e 90 57 1c 5c 20 5f 30 56 40 52 +94 5f a0 59 64 5c 90 59 14 5c 70 5b 00 54 2c 5d c0 54 50 5b 98 59 c0 4f 78 58 00 4a 34 5d b0 54 +94 5c bc 5f 40 5e 60 53 7c 5e 50 5a cc 5f 60 56 98 5b 40 4e 7c 5e 70 54 68 5d 70 5f dc 5c 10 5b +e4 5f b8 58 0c 5e 90 54 d8 58 98 5b 50 58 00 60 30 5e 08 5c 24 5d 00 50 f0 5d a0 5c 70 57 90 55 +b0 56 20 52 70 5c 80 4e b8 59 00 4b d8 5d f0 5e 08 5a c8 59 78 5f a0 5e 0c 5d 8c 5c a0 5d 58 5a +c8 5a 00 4a 48 5e c0 57 00 42 90 57 e8 5c 38 5b a0 58 00 5e 18 59 c0 53 48 5d 20 53 5c 5e 60 51 +fc 5c 30 58 00 3c 60 50 2c 5d 28 5a 64 5f 7c 5e c4 5f 40 52 70 5b 88 5c 8c 5c 94 5e 94 5f 00 55 +50 58 a4 5d 00 46 a8 59 78 5f 74 5f 94 5f 68 58 04 5c 68 5a 18 5c 20 55 28 59 90 5f 30 5d b0 57 +c0 5d 24 5d f4 5c d0 58 70 54 c8 59 50 5c cc 5e a8 58 78 5c 50 55 80 5a 00 49 60 57 5c 5e 70 56 +e8 5a c8 5c a8 5f 60 53 cc 5e f0 5c 40 54 c0 54 a0 5f e4 5d 08 5d 10 5a 20 55 c8 58 80 51 70 5a +d4 5c 80 50 cc 5d 80 50 bc 5d 50 59 70 55 80 4a 60 56 30 5a b8 5c 74 5f 08 5a d0 59 68 59 94 5d +00 58 d0 5e 50 5c 30 55 f0 5b 4c 5f b0 58 a0 51 00 50 e0 53 3c 5d ac 5c ac 5d d0 54 98 5b e8 5e +00 5b 1c 5f 9c 5e 8c 5c f0 5d 48 5d 50 59 60 51 a8 5b c0 4c a8 5a c4 5d 80 50 80 4a 94 5e d0 58 +a4 5c a8 5e 04 5c f0 5b 7c 5f 38 58 28 5b 60 52 a0 50 f8 5a a0 51 10 5e 5c 5c 68 59 40 5f 30 5d +20 59 8c 5d ec 5f e0 53 e4 5f 60 55 18 5b 50 58 80 4e c0 5b 5c 5d e0 58 f0 5e 74 5d 3c 5f 04 5f +d0 58 c0 4e 5c 5e 74 5f 28 5a 70 55 40 57 00 4b 80 51 30 5e 28 5c 54 5c 70 57 e0 59 c0 4d 58 5b +d0 59 50 5e 8c 5f 00 49 30 5a b0 5e 28 59 d0 54 80 57 40 5d 08 5a b8 5f d8 5c 94 5d e0 58 18 58 +8c 5c 50 59 a8 5f 00 45 10 5a 58 5e 40 53 38 59 80 48 90 57 f0 56 a0 56 70 5e 50 59 5c 5d 20 59 +dc 5f a8 5a 94 5d 58 5d 8c 5d d0 56 a0 56 e8 58 80 53 58 5f f0 56 d0 5d 90 5a 98 5d 50 58 bc 5e +a0 57 20 5d a0 51 d0 57 c8 5f e8 59 50 5b b0 56 0c 5c c0 4f 68 5c d8 59 60 52 50 5d c0 4c 40 5f +e0 53 b0 57 d0 5e 3c 5c 00 50 6c 5c 80 57 40 57 90 57 38 5a 84 5c 9c 5f 68 58 40 50 b0 57 d0 54 +e0 54 b0 5d a4 5d 5c 5c d8 5b 94 5e 60 53 30 56 dc 5c f8 58 00 3c 64 5e 80 4b 00 42 28 5b 4c 5c +70 55 38 5d 38 5a 38 5f 00 55 50 58 90 57 54 5d 10 56 98 59 68 5c 88 5b a8 5e c4 5f 68 5e 20 5e +58 5b d0 59 c0 59 38 5b f0 55 b0 5e 00 59 9c 5c 58 5a 74 5e e8 5b 94 5d f8 5e 28 5a 64 5e 34 5d +40 4d c0 56 a8 5c 84 5c b0 54 00 58 00 58 74 5f 14 5c 0c 5e f0 5b 20 5c a4 5d 08 59 fc 5d 60 59 +a0 5b 34 5c b0 58 dc 5e b8 59 d0 54 94 5f 08 5c 10 5c 14 5f 38 5d 14 5e 00 5e 34 5d f0 5f 94 5e +f0 5c c4 5d 18 5f 28 5f e0 52 98 5f e4 5c e0 50 e0 5d 44 5e 8c 5c 70 5e a0 5a b8 5f f8 5d 70 5c +00 52 20 5f 28 5e b8 5d 40 5b c0 55 38 58 c0 51 b4 5e e0 56 5c 5f 68 59 4c 5f fc 5d 74 5e 20 5c +34 5c 20 51 34 5d 10 5f b8 5a e0 54 90 5c 18 5a b4 5d 00 5d e0 59 d8 5f dc 5f 38 5d ec 5f 60 59 +a0 5e 58 59 dc 5f a4 5f f0 56 38 5c f4 5d e0 51 90 58 60 55 90 5a 24 5e c0 5f 40 5e 98 5d f4 5d +28 5a f0 55 ec 5f 60 5f 7c 5d a0 56 a0 50 a0 55 f0 57 00 5b 78 5e 04 5e c8 5d ac 5c c8 5c 28 5c +3c 5f 80 50 fc 5e e0 58 80 4b 9c 5c 10 54 24 5e 20 5d 78 59 8c 5c 10 5d 40 59 f0 5e 08 5b b0 5d +80 57 a0 58 e0 5f d8 5f 1c 5c 80 56 70 59 f8 5e 08 5d e0 55 80 54 f0 5f 2c 5d a0 5f 48 5d a4 5d +84 5c 78 5a 28 58 6c 5c 38 5c 58 5a 28 5f 80 59 a8 5f 10 57 a4 5c a0 59 00 5e 0c 5f a4 5e e8 59 +60 54 98 5d c4 5d 00 5d b8 59 d0 54 d8 5c d0 5b 20 5d c0 57 c0 5c ec 5e 00 4b b4 5f 6c 5e 40 52 +54 5e c0 5a 30 59 2c 5c c0 50 28 5a bc 5e 54 5f c4 5e 20 5a ec 5d b4 5d e0 5a 00 55 d4 5e d4 5e +48 5b 8c 5c 84 5e c0 53 80 4e 30 54 00 4f 10 58 70 59 80 50 90 5c 68 5a 2c 5e 30 56 14 5d 74 5e +a8 5b 8c 5e f0 5e 40 5e b0 57 98 5a 20 58 54 5e f8 5c 90 54 30 54 20 5d b0 55 d8 5e 48 59 94 5d +e8 5b 00 42 64 5f 48 5e e8 59 a8 59 00 53 40 58 00 4d 20 58 08 5f bc 5f 7c 5c c8 58 b0 5e d0 5a +60 5e 70 5c 0c 5c b0 58 80 4a e0 50 8c 5c d0 57 6c 5c 40 54 00 52 cc 5f dc 5f 88 5c f8 5c 70 57 +70 5b 80 4c e8 5e e8 5f 80 54 98 5e 00 57 80 49 c0 58 80 4c 6c 5f f0 55 c0 5d e4 5e cc 5e 28 5a +ec 5e 24 5d 10 55 b0 57 d8 59 f0 58 80 56 b4 5c 58 5d 80 5b d0 5d b0 5f b8 58 f0 5e 0c 5f 70 5d +30 5b c8 5c bc 5d 30 54 c4 5c 00 57 a0 50 cc 5f 48 58 dc 5e f8 5f 00 59 c0 57 0c 5d 90 59 f4 5c +70 5b 7c 5d a8 5f 00 5b e0 5f 54 5c 40 53 c0 5f 8c 5f 94 5d 90 59 ec 5c 20 5a d0 57 a4 5f c0 55 +00 48 10 59 60 5d bc 5f d4 5f 00 5e 40 55 34 5c 94 5d c8 5c 84 5f d0 5f 20 57 d0 5a 54 5c 98 5e +b0 57 68 5a 14 5d c0 56 00 5e 30 5a 28 59 d8 5f 48 59 f4 5f 20 56 48 5f 34 5c a0 54 00 5d 70 5e +c0 54 08 5d 18 58 e0 55 1c 5f 60 50 f0 56 f0 5d 30 59 60 5c 7c 5e 20 5a 48 59 dc 5d 50 5e c0 4e +00 57 60 53 ec 5f e0 52 64 5d 00 4e 04 5d 38 5e 80 48 30 59 e8 5e 3c 5f 00 55 60 5a 3c 5f c8 5c +94 5e 80 5d 80 58 40 5d 58 5d 00 52 20 5b 64 5c 28 59 00 5e 50 59 98 5b 98 58 fc 5c 10 58 48 59 +b8 59 20 5b 10 5b 58 58 50 58 80 4b 20 57 dc 5e f0 54 c8 5b 78 5b 40 53 c0 50 20 58 a0 57 a0 52 +38 5b 9c 5e 6c 5f 00 42 00 3c 28 5c 18 5b 34 5f a0 56 3c 5f bc 5d e8 5f f0 5a 68 5b a8 58 00 58 +70 5d 70 5e 54 5f c0 4d 04 5f 40 4c 5c 5d f8 5c 38 5e e4 5f 80 57 b0 55 40 4d 20 50 08 58 54 5c +e0 50 e0 5f ac 5f 30 5a 1c 5d 14 5e 2c 5c f8 5b 14 5c 9c 5c 44 5f 48 5a b0 59 04 5e 74 5e e0 54 +a0 5c 94 5e d0 5c 20 5e b0 58 28 5d 48 5f 80 50 60 53 50 5f 48 5d 18 5f 30 58 b0 54 b8 5d 98 58 +30 57 7c 5f 9c 5f b4 5e 90 5f 3c 5d 1c 5d 28 5c 50 58 68 5e f8 5b 3c 5c c0 50 f4 5c 48 58 60 5a +2c 5e 90 5f 00 5a c0 4d 70 5c 78 5e 80 54 78 5e a0 52 24 5d ec 5f 60 59 70 56 00 54 00 5b e0 5c +8c 5e e0 5b c0 56 c0 5b e0 5f b8 5c 08 5a 80 48 a0 5a ec 5e 40 5e 04 5d 58 5a 84 5e 44 5c 90 54 +a0 54 28 5e 00 58 b4 5d f0 5f 28 5e 04 5e 78 5b a8 5d 28 5a 04 5d 60 5c a8 58 f4 5c 40 4f f4 5d +60 5c 90 5d 00 48 18 5f 84 5f 98 5d d0 5d 58 59 80 5b 38 5f 54 5d 3c 5f 30 5c 80 50 9c 5e 80 50 +14 5d a4 5f 68 5a b8 58 ec 5d 40 5a 54 5e 44 5c bc 5e 68 5f 50 5c 04 5d 00 48 44 5f b4 5e 40 4e +c0 5c b8 5d 44 5d 34 5f 04 5e 64 5f 30 5b 00 46 98 5a 60 56 2c 5e 94 5c 00 3c a4 5f 34 5c e8 58 +78 5b 30 5f cc 5c b8 5b 68 59 1c 5c 28 5a 30 5c ac 5e c8 59 e8 59 18 5f 20 56 54 5c d8 5b dc 5e +88 5a 2c 5e c8 59 cc 5d 58 5a 94 5f 0c 5d 28 58 d0 5d fc 5d 7c 5f c8 5f b8 5c 24 5f 20 5c c4 5d +18 5a c0 5f 40 56 80 5f 40 5c 6c 5f 78 5f 40 58 10 55 90 5a 18 5a 7c 5d 40 57 d0 56 dc 5f 08 5a +80 4c 7c 5d 38 5c 40 50 10 5f 18 58 70 57 a4 5d cc 5d 50 57 a4 5c 60 57 f4 5c 70 56 d8 5a 48 58 +00 5d a8 5c 78 5b 64 5d e0 5d 58 5a 00 49 c8 5b 24 5d e0 51 44 5e 64 5f 94 5e 40 51 20 5a f0 54 +a0 55 88 5c 70 54 08 5a 7c 5d 00 59 40 54 50 56 30 5b 9c 5d 18 58 a8 5b 6c 5c 40 52 08 5c 08 59 +60 56 18 5a fc 5d a0 59 50 59 60 54 60 59 e0 50 a0 58 50 55 6c 5d 5c 5e 70 56 40 5d 04 5d 40 57 +0c 5e 4c 5f 04 5d f0 5b c8 5b b0 58 b8 59 80 4c 74 5e b8 5c 98 5e 14 5f 0c 5d 40 5a 00 46 08 5c +48 5a 68 5d 7c 5e 40 54 40 5e b0 59 70 5c 3c 5f 70 5a 20 55 58 5a 68 5f 70 5c f8 59 70 5e 30 59 +c0 52 cc 5d 70 59 d8 5f 68 58 60 51 80 53 f8 5d 14 5d 50 56 e4 5c b0 5c 28 5a c0 4c e0 59 28 5c +c8 5c 78 5a 74 5c 80 50 d0 58 58 5b 90 56 a4 5d 28 5e 30 54 d8 5d 40 57 6c 5f c0 57 a0 53 10 54 +00 56 64 5d 00 53 00 5c 54 5c 00 4d 80 5b 18 5d b8 58 08 59 98 59 80 58 40 5b 28 5d 8c 5d d0 5d +c0 59 a8 59 60 59 bc 5c 30 5f 00 58 88 5b 2c 5e b4 5e 00 4f 24 5d 0c 5f c8 5b 74 5e d4 5c 8c 5e +c8 58 68 58 28 5b 30 5a b0 5f 28 5d 10 5d b0 5a ec 5c 60 52 b0 5b 04 5f 94 5d 80 4a ec 5e 4c 5e +60 5e 5c 5d b4 5f c0 52 c0 59 40 5a 40 56 e8 5f bc 5d 10 54 00 5c c0 5a 60 5a 18 5c c0 5b f4 5d +9c 5f b0 5d a8 59 5c 5e c0 5b 00 55 60 59 4c 5e 70 54 9c 5d f0 5b 80 4e c0 5d e0 56 f0 56 84 5e +68 5f 30 59 e0 5b c0 52 78 5e 14 5f c0 56 38 5e 78 5a d0 5c d0 56 10 56 1c 5d 70 59 cc 5d 54 5d +1c 5e bc 5d 98 5d e8 5d f8 5d cc 5f 00 4d cc 5e c0 4f d0 5a 9c 5c b0 5a d8 58 a0 5b e8 5d e8 5c +4c 5e c8 5d f0 57 c8 5e 7c 5d 40 5b 00 49 1c 5d 10 57 e8 5e 84 5f fc 5c 74 5c 70 58 64 5c 44 5d +38 5c 50 57 f0 58 e8 5e b8 5b 70 59 00 49 d4 5d 80 57 e0 51 04 5c 80 5b 70 57 8c 5d 40 5f 5c 5c +e0 5f 28 5d f0 59 c0 5e 24 5d 10 56 00 4a 34 5c e0 55 80 50 c8 5b e0 5b 20 5e 38 5f b0 5c b8 5e +b0 55 b0 58 b0 57 d8 5a 1c 5e 98 5d c0 51 b8 5c 90 54 74 5e dc 5e 70 56 c0 5e 14 5e a4 5f 70 5a +14 5c c8 59 00 4f 28 5f 50 55 90 5f 80 4d a8 5a 30 56 f4 5c 10 5e d0 59 30 5f f0 5a 5c 5d 5c 5f +f0 5f 00 5b d8 58 dc 5c 10 5c c8 58 08 5a 60 52 c0 55 a0 59 30 54 40 55 f4 5f 00 56 1c 5f 00 60 +54 5f dc 5c 4c 5e 24 5d 04 5f 6c 5d c0 58 90 5c 20 51 3c 5d 58 59 a0 50 74 5d 04 5f 00 51 f4 5f +a8 5c 00 54 a8 5a f0 56 80 56 30 56 30 59 34 5f c0 5c 40 4e 30 5c f0 55 90 5c a8 5c 40 52 24 5c +d8 5c 80 51 70 54 04 5d 54 5f bc 5e c0 54 90 54 a0 58 e8 5c 9c 5c 74 5d cc 5c a8 58 78 5d e8 5c +f8 5a 40 5f 30 55 80 54 70 57 48 5e e0 5e 0c 5f 70 5c b8 5c e8 5c 88 59 50 5d 00 49 10 55 48 5d +30 5b 14 5f 40 58 b0 55 40 53 fc 5c 10 5c a4 5d b0 54 80 50 04 5d 90 55 e8 5c a8 59 c0 55 88 5b +bc 5c d0 5c c0 5a 04 5c a0 57 bc 5f 00 5e 40 59 80 4c 58 5b c0 52 b0 55 78 5d 50 55 40 4d ec 5e +3c 5d 18 59 d8 5c e0 58 80 51 00 52 90 5d 00 57 78 5c e0 56 60 5a f8 5b 70 5c d0 58 70 57 dc 5e +94 5f e0 54 f8 5d 88 59 88 5a 20 5b dc 5d 60 5a 24 5f e0 58 94 5f e0 58 c8 5d 30 54 c0 52 98 5b +30 56 a0 5b 48 5b c8 5b 80 49 34 5c b8 5d 7c 5f 40 54 18 5e b4 5c f4 5c e8 5b 48 59 10 54 30 5b +80 4c f8 5b b0 56 20 52 80 48 f0 5f f8 5c a8 5c b0 5a 3c 5d 00 46 80 50 a0 50 c0 5a 00 44 b8 5c +44 5d 58 5a cc 5c 60 51 68 5d 80 5d 00 60 6c 5e 04 5c 88 5c 48 5a 98 59 f0 5e 58 5e 08 5a 98 5e +f0 56 00 58 80 48 7c 5e a0 5a 54 5f 94 5d 18 5b 88 59 4c 5d 68 5e 90 5b 18 5d 10 57 38 5e 38 5b +b0 58 6c 5d 40 57 00 4d c0 54 18 59 c0 58 7c 5d 80 5e 44 5d f0 55 50 58 60 53 00 4f 48 5e 80 5b +d4 5f ac 5d 8c 5d 80 52 c0 58 28 5e ec 5f 18 5e 20 59 cc 5f 00 51 80 4b b0 5b 20 55 7c 5d f4 5f +8c 5c 40 4e fc 5f 38 58 80 55 84 5f 00 59 c8 5f e0 5f 44 5f e0 5f d0 58 f0 5a 60 52 d8 5c d8 5a +70 54 0c 5f c8 58 70 5f 08 58 90 56 70 5c e0 5b 70 56 70 5f c0 53 c4 5c 6c 5f 14 5c 58 5c 4c 5f +68 5c 48 5a f4 5d 00 49 68 5f c0 52 d8 5c c0 4f 40 4e c0 52 3c 5e 00 4d 44 5e 90 54 e0 5c 68 5f +4c 5f 80 57 88 5b 00 3c 2c 5f 64 5e 10 54 00 58 90 5d 64 5c 84 5d 70 57 4c 5f b0 5a 7c 5e 38 5d +68 5e 48 5d 10 57 00 58 50 5e 00 3c 1c 5c c0 53 00 5c 94 5e fc 5c 10 57 b0 5e 7c 5e a0 52 00 5e +f4 5f 00 46 40 56 24 5e 10 5b c0 56 60 56 30 57 80 5e f0 5c f8 5e 30 54 50 5f 20 5a 00 4c 54 5c +c0 5a 40 5c b0 57 74 5c f8 5f 0c 5e 40 52 f8 5a f0 5b 80 5f 4c 5f c8 5c d4 5d 20 50 f0 5e 50 55 +f8 5f 30 5f ec 5e 34 5c 50 54 38 59 f0 5c d4 5c 80 51 48 5d c0 5b 78 5a 64 5e 60 50 74 5f f8 59 +38 5d 30 5f 60 54 28 59 78 5a c0 4d bc 5e 14 5c f8 5a 38 5c 30 59 70 5b 28 5e 78 5e c0 5d 54 5c +8c 5c c0 5c 0c 5f 7c 5f 80 5a 90 5e 5c 5e 9c 5e 90 5c 2c 5f 08 5f 30 5d e0 59 f0 59 a0 51 30 56 +c8 5e 28 5f 40 52 24 5d a0 55 4c 5d c0 58 28 5b 88 5c 98 5e ec 5c 60 58 e0 54 d8 5f 90 5d 18 59 +d0 56 84 5f f0 59 28 5c a0 52 74 5f f0 54 70 57 a0 59 ac 5d 0c 5e 88 5d e8 5f 68 58 18 5d c0 56 +38 5a 24 5c a0 56 5c 5e c0 54 ec 5e ec 5c 78 58 d0 5e e8 5e 20 57 50 56 dc 5f 90 59 00 4b 80 4d +f8 5a 00 5d 50 5e 68 5f 28 59 98 5e e0 5e c8 5d 18 5d 3c 5f a0 50 80 53 f8 5a e0 5b 00 55 20 53 +b0 5e 84 5d d8 5a 60 50 28 5d 50 57 c0 5f e8 5f d0 5f 60 5d 50 5d 70 5d f4 5c 40 50 a8 5e b4 5f +c4 5e e8 5f c0 57 e4 5e 80 55 c8 5a ec 5d 34 5c cc 5c 88 5e 30 5b 70 5e 00 56 20 50 30 5e a0 5b +10 5e e8 5e b8 5a 30 59 94 5c 40 52 e0 57 74 5d a4 5d f4 5d 18 58 48 5c 00 4b c0 5c f8 5f 10 5f +98 5f 8c 5e e0 53 80 4a 38 59 a4 5c 68 5b 40 53 8c 5e d4 5f e0 59 c0 5f ac 5d 60 56 c0 5a 74 5c +b4 5d 00 58 90 5b 08 5a 18 5e 28 5f ac 5e 48 5f dc 5d 18 5d 28 5f 30 5f a0 55 00 58 3c 5c 00 5d +00 5c 74 5d 08 58 14 5c 68 5b 20 5a 84 5e e8 5f e0 59 e4 5e 00 51 40 5f 00 5d 80 4d 30 57 00 52 +60 58 f0 5d 20 51 a0 53 58 5f 10 5e 8c 5c 24 5f d0 5a a0 5e 38 5c 8c 5c 18 5d 20 52 90 57 90 57 +e8 5e c0 4e 1c 5f cc 5f 84 5e a0 57 08 5d e4 5c f8 59 f8 5f c0 5a 70 5a 10 59 5c 5d 40 59 40 59 +20 5d 50 56 2c 5e a4 5d 60 57 a8 58 98 58 80 53 bc 5f 7c 5d a8 59 00 4b 18 5d 20 57 4c 5e e0 5e +64 5f a0 5b 18 5e e0 5f 0c 5c 94 5f d8 59 00 5e 38 5a f8 5e 48 5b 08 5e ac 5f 58 5d b0 59 40 5c +a0 58 88 58 8c 5f f4 5e 40 5d d0 5d d0 55 80 52 54 5e 4c 5e c0 50 70 56 b8 5c 40 57 40 50 0c 5d +a0 5f 90 5d 68 5e d0 5f 88 5c e8 5f 4c 5c 00 3c 00 45 d8 5e 70 54 00 47 c0 5f 00 53 f8 5f 38 5a +dc 5f bc 5c 70 5c c0 5d 94 5c 20 5f 88 5e c0 51 e0 57 c4 5c d8 5f c8 59 d4 5e 50 5f 1c 5e b0 58 +18 5c 4c 5f b4 5c ac 5d c0 5c 48 5f 88 58 c8 58 0c 5d 58 59 d4 5e 00 56 d8 5c f0 5f 08 5a c8 59 +b4 5f a4 5f d8 58 30 5a 40 4c 14 5c e4 5d ec 5c 38 5c 40 5c 00 5f f8 5e 00 45 e0 5a 00 5a 00 47 +88 59 60 58 58 5b e4 5d a0 50 88 5e 9c 5d 60 54 08 5e 80 59 48 5d 9c 5e 88 58 b0 5d 9c 5e 68 5d +c4 5f 98 5d 58 5f 80 55 10 5b 68 5a 98 58 60 59 c0 58 e0 5e 20 5e f4 5e 44 5f 98 5d 30 56 48 58 +b8 5f 60 50 54 5c 4c 5f 00 5a c8 5a d8 5f f0 57 3c 5f 00 47 1c 5e 80 4d 60 52 e0 5e 74 5c 40 5a +20 5d 08 5d 3c 5c b8 5f b0 5e 40 5d a0 53 28 5b 18 5e f8 58 78 5b b8 5e 40 4f 60 5a 74 5e 50 59 +28 5e 10 5a 24 5e 44 5e 24 5d c0 54 60 59 00 57 d0 5c e0 50 08 5a ec 5e b4 5d a4 5d 68 5a 88 59 +70 5d 70 5c 00 5d f0 5d 2c 5f a0 5b 7c 5f 28 58 60 59 b0 5d c0 4c 20 53 60 57 80 51 58 5c a8 5a +98 5f 80 4a f0 5e b8 5f 7c 5d 58 5a d8 5e 10 55 20 5e e0 52 80 49 38 5d 24 5e 00 40 d0 5c 90 5e +88 5d e0 5e 80 48 34 5f e0 5e 48 5f 80 5d e0 56 88 59 20 57 20 59 18 5c 00 5d a0 55 18 5d 40 5e +a8 5f b8 5f 7c 5c 98 5e 8c 5d 3c 5f 60 59 ec 5e e4 5e a0 5b 74 5c 40 53 c4 5f a0 57 5c 5e 80 5c +00 5d 50 5a a0 51 00 5d e0 59 a8 5a e4 5c 70 5a 28 5f 40 4e f0 5b 7c 5d a4 5f 00 3c cc 5e b8 5b +84 5f a0 5c dc 5e 00 42 a4 5c e4 5d c4 5f 40 4e 00 53 60 58 f0 5a 40 50 c8 59 bc 5c 00 46 f4 5e +90 5e c0 50 6c 5e c0 56 8c 5d e0 54 08 5c 8c 5d 70 59 78 5d 5c 5e 90 58 f0 54 98 5f 40 57 9c 5d +38 5a 00 5f 7c 5e b8 58 f0 55 00 5c 4c 5c 40 52 20 52 24 5d 48 5a 6c 5e 3c 5d 7c 5f 60 55 b0 5f +20 54 18 58 84 5f 00 5b ac 5e 60 5c 40 53 f0 57 18 5d 00 4f ac 5f 20 5a b0 5b c0 52 60 53 c4 5c +f0 5e 1c 5c b8 5f c8 5c 30 57 80 53 1c 5c a4 5e f0 54 00 48 dc 5f 04 5c 1c 5d 90 5a c0 4e f0 56 +c0 56 20 5d 80 50 f0 55 80 56 20 5b f0 5b 28 58 b0 59 20 54 58 5e f0 58 bc 5c b4 5d 80 53 70 5f +50 5e 80 5f b0 5d 58 5b 80 4c 98 59 28 5f ec 5f 10 56 68 5c 88 5f 04 5c d0 5a 40 5c 00 59 00 5f +b8 5d 70 58 90 54 d8 59 00 56 e4 5f 58 58 b8 5f 58 5a 0c 5f 70 5d 10 5b d0 55 e4 5f 80 54 38 5d +b8 5f 68 5a 00 3c 60 56 40 4d dc 5c 68 5b 48 5a 00 4c f4 5f 38 5c 88 5d a4 5e a0 5e 40 5a 40 56 +9c 5c 78 59 78 5c 50 5b c0 58 2c 5f 28 59 7c 5e 90 54 40 5b 80 48 1c 5e f0 5b bc 5f b8 59 14 5d +58 5a 98 5a e0 54 6c 5c 64 5d 0c 5d 98 5a 94 5f 80 53 d8 5d 38 5e d8 59 68 5a f8 5e f8 5d 80 51 +5c 5f 30 56 48 5c 40 57 e0 50 78 5d 40 53 f0 5e c0 54 f0 56 e8 5c ac 5f 20 56 9c 5c 40 5d c8 59 +00 5f 90 5a 14 5d 38 5e 98 58 c0 54 c0 59 f4 5d 80 49 50 58 a0 59 90 56 ac 5c c0 57 68 59 20 51 +34 5e a0 55 98 58 28 5b 00 4f c0 5a 9c 5e e8 5f 80 4a e0 5b a0 54 00 42 00 42 24 5e 40 57 40 51 +f4 5e 18 58 e0 56 f0 5e f0 59 bc 5f 44 5e f0 5c 60 52 78 5d 00 5d 4c 5d 88 5e b0 56 b0 5e 10 54 +d4 5d 90 5f 20 55 74 5c b8 58 b0 5b 08 5d d0 5e 60 52 20 55 4c 5c 00 3c c8 5e 74 5c 00 51 dc 5c +38 5f f0 56 a0 59 44 5d f8 5c e8 5b 6c 5d 80 5e 80 5b 04 5e 50 5f 1c 5f cc 5d 7c 5e a0 5d 00 5b +50 5e 10 5b 40 4d 64 5c a0 58 e0 59 90 5f 60 56 00 58 94 5e 28 58 00 49 18 5d d8 5b d0 5e fc 5d +18 5f c4 5e d8 5a c0 59 80 4f b0 59 d0 56 58 5c 30 55 00 5f 40 50 e0 55 e4 5e d8 5f 40 55 14 5f +10 5a ec 5e 40 56 c0 4d d0 56 8c 5d 00 5a b4 5e 00 46 50 57 68 58 50 5c 28 5d 68 5a 48 5b 00 4f +78 5b 38 5e 80 4d 90 59 60 51 68 5a b0 5c 7c 5f cc 5e 10 54 d0 56 14 5e 78 5e 8c 5d c8 5d c8 5c +30 5a 78 5b e0 54 38 5f 40 5a 30 57 68 5d b8 5c d0 56 20 57 a0 59 7c 5e 90 5d 10 5d 38 5b e0 5b +90 58 e8 59 30 54 70 54 70 5b 88 59 d0 5f d4 5f 80 4e 18 5c 00 47 60 50 74 5f 38 5a 6c 5d 7c 5e +cc 5e 20 5d 0c 5c 40 55 20 5c 70 54 0c 5c 8c 5c e0 56 50 56 38 58 24 5e 08 5c a0 5a 58 5e 88 5a +70 55 50 5d c0 53 c8 5f b8 5c 90 5a ac 5f 20 5d 20 5d 14 5e e0 56 a0 51 b0 5f 68 5c cc 5f 78 5a +00 57 58 5e e0 5b b8 5b 38 5e 90 55 a0 5b dc 5e 50 5d 48 5b b8 5b f8 5a 88 5a 74 5f c0 56 50 58 +1c 5c 00 5f 00 46 40 53 80 59 60 52 f0 5d 04 5f 5c 5e e8 5d fc 5f 80 5f f8 5d b0 58 a0 5d cc 5c +00 4f 90 58 80 51 60 55 e4 5e 94 5c d4 5f c8 5a 08 59 80 54 14 5c d0 59 e4 5d 70 59 08 5e 04 5e +5c 5f fc 5c 6c 5d f0 5d c0 51 c0 5c a0 51 60 5c 6c 5f 3c 5e 20 59 c0 5b 20 58 40 59 ec 5c b0 5a +10 5d e0 52 28 5c 6c 5f 84 5e e8 58 0c 5f e4 5d 64 5e 80 5e 58 5c 40 4f 54 5c 70 55 ac 5c 9c 5f +80 49 a0 52 e8 5a 3c 5c 30 5a 40 52 40 5a 68 5e f4 5d a0 59 70 55 80 58 30 5c 30 5b 08 58 34 5c +58 5e b4 5e 1c 5c 8c 5c c0 52 c0 52 90 5f 80 51 9c 5e 10 54 98 59 40 54 c0 5c e0 50 10 59 b8 5e +bc 5e f8 5f 80 53 c8 5c 20 50 00 46 10 59 10 5b bc 5d 50 5a 40 59 30 59 40 5e 98 5c e0 5a dc 5e +28 5e 7c 5f 4c 5f 98 58 c0 57 80 4c 08 5f e4 5d 80 4e 90 59 40 5f 40 5e 80 4d cc 5c 10 5f 74 5f +80 5d a8 5f f0 57 ac 5e 24 5e 30 58 e4 5c 6c 5e c8 5b e0 5d 8c 5f a8 5b 30 5c 18 5d 30 55 90 5e +74 5e 48 59 54 5d ac 5f f8 59 20 5f 04 5e f0 58 a8 59 d0 59 90 58 48 5b 54 5f 80 50 c0 52 f0 5e +c4 5f f0 5e 28 58 c0 5e 94 5c 10 54 10 5f 38 5d 00 4b 00 4e 88 5e a0 5f 20 53 d4 5c c0 5a 94 5f +bc 5e e4 5d 80 5b 60 54 60 5b 70 54 28 5c 00 42 00 3c 74 5c 00 51 b8 58 78 5b b8 58 90 5e 30 56 +e0 5a 40 5c d0 5d 98 59 20 53 e8 5d 40 4d 20 52 00 46 80 50 40 51 b0 55 c0 53 50 55 08 5e 50 54 +f4 5d fc 5c d0 59 d8 59 38 5d 00 50 80 54 84 5f cc 5e c0 5f 60 57 c8 5f b0 5c f8 5f 50 5b 18 5a +dc 5f 90 5f b0 5a 30 5b 68 5b 28 59 90 58 90 5d 50 5c 00 5f 80 57 e8 5c a8 5f 40 5e c8 58 74 5e +7c 5f d0 5d 60 5f e4 5d a0 5c b4 5c 1c 5f 10 5f 60 5d 90 5e a0 52 d0 59 90 57 60 5f 90 5a 0c 5f +a4 5d 60 56 d8 5d 20 5d b0 57 b8 59 a0 58 78 5f 90 5e 18 5e 80 50 50 56 b8 5b c4 5e 08 5b d0 56 +c8 5d e0 5b dc 5d 94 5d b8 5f d0 58 58 59 04 5f 9c 5c e8 5c 50 5e 9c 5d 00 4d 18 5e 80 5f 40 4c +a8 58 24 5d f0 5a 58 59 84 5e 60 5f 80 54 a8 5d 48 5e 20 5b a0 57 bc 5d 1c 5d 20 58 e0 59 90 5f +c0 56 ac 5e c0 5d 88 58 18 58 a8 5a 6c 5f c8 59 e0 56 d8 5d f0 56 d0 5e e0 57 58 58 40 4f 48 58 +dc 5e 58 5f 44 5c 00 51 18 5c 4c 5f f0 5b 1c 5d 30 59 58 5f 3c 5e 40 5a c0 53 44 5e 00 47 70 57 +f8 59 c0 5f e0 56 c0 5b 20 51 e4 5f 84 5e 78 5d ac 5d 38 5d b4 5d 08 5a 90 57 ec 5e 60 5e d4 5e +80 48 a8 5d 34 5d 40 4f 30 54 08 5f 14 5d 80 5d 40 5e 2c 5f 60 5e 50 5f 34 5e 50 5e a0 52 10 5d +80 5d 20 51 00 57 30 55 20 51 e4 5c e0 5b 18 5a 8c 5f 8c 5e c0 5c c0 5d a4 5f c8 5f c0 55 a0 5f +e4 5e 98 5e 98 58 a0 5c 88 59 80 4e 40 53 00 4d e4 5f ac 5e fc 5e b4 5e 40 4f 8c 5e 00 4f d8 5b +bc 5f b0 5a 10 54 f0 55 88 5c 4c 5d c0 55 94 5c 40 59 e4 5f 70 5b a4 5d 7c 5c b8 5d 10 57 c0 5f +80 58 e8 5c b4 5c d8 5f c0 54 e0 5c 20 50 a8 5f b4 5f fc 5f c0 5a cc 5f 88 5c a0 5e 78 58 20 52 +e0 54 20 54 c0 59 60 52 50 55 28 5f 64 5e 78 5e 78 5e 90 58 d0 5a 50 5e 90 57 a8 58 18 5e 04 5d +80 55 10 55 b0 56 18 5c d0 56 a8 59 2c 5d 10 58 90 56 80 50 c8 5a b8 5d a0 52 a8 5d f0 5b 90 59 +9c 5c 50 5a 00 47 e0 55 14 5e 44 5e c0 59 10 5b c0 4e 38 5a 40 54 48 5a f0 5d 00 5b 58 5a c0 5c +48 58 10 56 f8 5b e0 5c 28 59 98 5f ac 5d 58 5e a0 57 28 59 a0 50 1c 5e 10 57 c0 55 c0 4c 60 59 +18 5a 00 49 70 59 90 5c d0 5e 30 5f a0 5e 20 54 08 59 20 5a bc 5c a0 5d c0 4d 94 5e 18 59 a0 5e +80 56 60 50 54 5e b8 59 a0 50 5c 5e e0 52 70 5f f8 5d 38 58 70 58 50 5c 40 56 48 59 f8 5d b0 5a +40 4d 00 42 90 56 a0 53 0c 5c 4c 5e 40 4e 90 5a 98 5e 28 5a 20 59 b8 5b 40 50 00 60 2c 5e 24 5d +60 5e 70 58 68 5b 00 40 60 5d a4 5d a0 50 a8 5f 80 4f e0 53 94 5c 38 5e 00 54 08 58 48 5b 98 5e +1c 5c 20 53 6c 5d 18 5e 40 5b 9c 5f 80 56 50 55 28 59 c0 5d f0 55 58 5a 04 5c c0 4f a8 59 50 5a +00 4b 50 5a 90 5a 98 5d d8 5a 1c 5d c0 54 74 5d c0 58 1c 5c e0 51 38 59 18 5b c8 5a ac 5f 28 5b +98 59 e0 52 e8 5a c8 58 b8 5a a0 59 94 5d 00 45 d8 5f 50 5b 38 59 80 5a f0 55 d8 5e 30 5a 60 56 +10 58 30 56 10 56 8c 5c f8 5b e0 53 20 55 90 5a 40 52 84 5e 00 49 80 48 a4 5c 00 5b e0 5a f0 56 +80 4c d4 5c 60 55 9c 5c 40 56 60 5a 74 5f a0 59 80 5a f4 5d 00 47 f0 55 d8 5e 70 58 88 5b 48 5a +80 49 50 55 e8 5a 68 5e a0 58 a0 54 a8 5c e0 54 18 5e 64 5d d0 54 e0 5a dc 5f b0 5d 00 58 a0 52 +70 56 c0 58 e0 51 c0 52 54 5e 68 59 48 5a 00 55 c0 58 08 5d 00 3c cc 5c 30 58 64 5f b0 5a 10 5b +fc 5e c0 52 a8 58 94 5d 30 5c 80 4a 80 4e 30 55 2c 5c 18 58 00 40 cc 5e 00 5a 40 4c 80 5d 08 5a +b0 5a c0 55 80 4d 58 5c 38 58 04 5f 04 5e 00 50 a0 5f ec 5d 10 59 60 5f a0 56 e0 54 80 57 40 57 +58 5e a8 5b a8 5e 90 5d 60 52 98 59 5c 5f 30 5b c0 5c 00 5a 80 51 dc 5e 90 56 30 5c 08 59 90 58 +00 3c f4 5c c8 5b c8 5c 18 58 18 59 d8 5d fc 5d 88 5b 40 5a f0 56 14 5c 40 5d 20 5a 20 53 50 56 +98 59 0c 5d 2c 5d f0 56 e0 58 b0 55 00 54 40 57 d0 59 20 51 10 56 5c 5e 00 44 80 50 f0 5b 20 53 +f0 5f 60 56 78 5c e8 59 2c 5f a4 5f e4 5c a0 5d 74 5e 2c 5e 40 59 cc 5c 8c 5c b0 5d 18 5f 40 4c +60 57 8c 5d 70 5c d8 59 18 5e 00 5e a0 53 b8 5c 28 5e c8 59 30 5a 78 5b 34 5e 50 5f 60 56 c0 4c +38 5a 20 59 98 5f 30 59 f8 5f bc 5f cc 5d dc 5f ac 5c 8c 5d 00 3c 28 5c 80 57 40 55 00 4c 50 55 +58 5c fc 5c 30 5b d8 5e 70 5a 7c 5e 00 4c 70 5b 40 5f c8 5f 40 4f 08 5c c0 5d 70 5b b0 57 10 5f +10 57 50 5f 90 5e ec 5e d0 57 20 5b 70 54 30 5b a8 5f 80 57 c0 5a 14 5d 2c 5e 40 51 f8 5f bc 5e +24 5c 10 5b 1c 5c 8c 5d c0 58 78 5f 00 58 80 52 9c 5f 60 55 80 4e d8 5c 34 5c b0 5c ec 5d 00 4e +30 59 88 5f 70 5a c0 5c 40 5d 00 44 d8 5b 50 57 08 59 3c 5c a0 59 20 5f 10 54 58 5f 00 5d 34 5e +00 5f 00 55 f0 55 40 58 c0 5d 00 4d 48 5d 80 51 a4 5c 68 5b 90 58 f8 5a 34 5c c0 5e e4 5c 30 5f +20 51 80 58 80 49 54 5e 30 56 80 50 84 5f 60 51 1c 5d 90 5f c0 51 b4 5f c0 51 50 5f 4c 5d 14 5d +a0 54 40 5f c0 4e 24 5f fc 5e 70 59 b8 5f a0 55 c4 5f 6c 5e a0 5d 2c 5c 80 4f 44 5e a8 58 fc 5d +80 54 58 58 28 59 20 52 ac 5d 24 5c a0 54 88 5c 40 59 4c 5f b8 5f b0 5d a0 57 34 5f c4 5d 10 5e +00 5a ec 5f 38 5e e0 5e d4 5d a0 5d e0 5f 4c 5f 28 5e 70 5d 28 5a d0 5b e0 51 7c 5c b0 5b 88 5c +40 59 dc 5f b8 5f e8 5e 88 5b 0c 5f 40 4f d8 5c 20 5f a8 5e 70 56 94 5d 30 59 00 4e f8 5d 60 5e +a0 5c 80 5d 88 59 48 5f 14 5f 10 59 b8 5e 48 5b 08 5a 2c 5e 80 56 64 5c 28 59 2c 5f a0 52 10 56 +20 58 48 5f 90 56 00 58 e8 5a 88 5a 08 59 b4 5f e0 53 70 5e 64 5c a8 59 d0 55 ac 5f 0c 5d b0 58 +4c 5c 28 5b 50 54 c8 5a e0 5d 40 5e b0 58 e0 54 80 52 60 57 d0 56 78 5f 88 58 78 5e 38 5f 08 58 +c8 5c a8 5b f8 5d cc 5f 00 5d 60 5e 00 54 c0 51 64 5f 94 5e 30 58 b0 5d 00 49 9c 5d 90 55 00 46 +a0 51 a0 5c c0 55 60 5c a0 54 c8 58 88 5b 80 55 50 5e 64 5e 48 59 3c 5e 28 5f 18 5e 50 5d 40 4f +80 5a 6c 5f 2c 5d a4 5c d4 5f 84 5f 08 5c 74 5c d0 54 68 5b 80 52 38 58 e0 57 44 5d 90 58 c8 5a +d8 59 c8 5e e0 53 28 5f 8c 5c 24 5d 7c 5d d0 5d f0 55 50 54 40 4c 68 5d e0 54 60 59 18 5d e0 51 +30 55 c0 4c f4 5f 6c 5c 30 5b b8 5c b8 5e 2c 5f c8 5a 34 5f 30 5d 8c 5d 80 51 e0 52 4c 5e 00 4a +94 5f 00 45 20 58 70 5b a4 5d 04 5e c8 5c 00 54 40 56 90 58 88 5a 40 59 0c 5c f8 58 20 5a b0 5f +3c 5d 0c 5c f4 5c 14 5f 74 5c f4 5c e0 5d 20 55 00 40 e0 5a 20 52 6c 5e 00 56 80 4a 88 5a 8c 5d +14 5f 80 56 b0 56 bc 5f 28 5f 0c 5d b0 59 c0 5d 70 5a 84 5d 88 5a 44 5d 40 59 58 58 70 5d b4 5d +20 5e 78 5c 60 52 10 54 f4 5c 38 5a 6c 5c 00 47 e8 5b 68 5d 50 56 58 5b ec 5e c4 5c 94 5f 4c 5d +80 5e 14 5e 74 5e 94 5f 94 5c 70 57 30 5c 38 5a 88 5e f0 5b a0 5e ec 5e 70 5b b0 5a 00 40 10 5e +38 5f 20 54 fc 5d 04 5d 18 58 40 5e d0 54 58 5b 64 5e 80 53 c0 58 c0 54 2c 5f ec 5e 10 5a e0 5d +e0 5e 84 5f c0 56 f4 5f 90 59 64 5f 94 5d bc 5e 14 5e 60 5c a0 5f 90 5d b4 5e dc 5f d4 5c 84 5f +e8 5a 78 5f 28 5c 74 5f e0 50 70 5f b0 56 30 58 00 58 08 59 68 5e 28 59 70 5a d4 5c 20 57 98 5b +04 5f 8c 5f e0 50 ec 5e a8 5d f4 5e 90 5d b4 5e c0 5f c8 5c 9c 5c e8 58 90 57 0c 5f c0 4f d8 5a +38 58 14 5c c0 59 48 5e ac 5c 68 5f f8 5d e8 5c 28 5d c0 4d c0 4f 1c 5c 38 5c d4 5f 30 5e 98 5e +d0 54 a0 58 a0 5b cc 5f fc 5c d4 5e 10 54 d4 5f a4 5c b8 59 38 58 38 5f 80 52 60 51 80 54 e8 5d +1c 5c 90 5f 00 5e 78 5d 30 54 20 5c a0 57 c0 5d c0 54 a4 5f 40 50 6c 5d 8c 5d 40 56 44 5c cc 5d +70 5e 70 5b f0 5f ac 5e 04 5f f0 5d 20 58 70 59 38 5d a0 50 60 5a 1c 5f e0 5f c8 5c 58 5b b0 58 +00 44 0c 5f 40 50 44 5e 10 57 70 5c 44 5d 80 50 f4 5e 18 58 b4 5f a8 59 d8 5e 04 5d a4 5d 98 5e +00 4c 70 5f 40 5b 04 5e bc 5d 58 5b 14 5e c0 58 d8 5e 38 58 f0 5c 60 5f f4 5d 84 5f c0 52 28 5c +c8 5d d4 5e 90 5f f0 5b d0 56 8c 5e 64 5e 68 5c 00 5b f0 5b 00 42 60 52 a4 5f e8 5b c0 4d 70 5d +50 58 bc 5e 70 5d 20 54 70 57 c8 5a b0 58 48 58 94 5c 90 55 f0 55 08 5e 94 5f b0 58 40 5c 20 5f +00 4a 40 53 48 5b a0 5b 74 5f 94 5c 80 59 c0 52 50 55 10 5a a0 52 2c 5e 64 5f c4 5e c8 59 64 5c +c0 51 c0 55 00 4e d8 5c a8 5e 40 55 10 54 d8 5c 10 5b 50 55 c8 5b f0 56 24 5d bc 5f 1c 5f 8c 5f +50 56 00 49 d0 5d f0 5e 00 44 20 56 f8 58 00 52 e8 59 30 55 84 5c e4 5c b4 5e 24 5e 40 5b a8 5f +20 56 ac 5f 40 4c 00 54 f0 5d c8 5d 84 5d 2c 5d c0 5b 80 49 a0 54 b0 58 34 5f 2c 5e 78 5b 14 5d +9c 5d e0 50 00 4b 78 5c 98 5a 28 5f 30 56 4c 5e f0 5c c0 58 30 56 d8 5a 68 5e 28 5a a0 5d a8 5b +c0 4e c8 5e 60 5b 40 54 00 44 7c 5f b8 5e ac 5d c0 53 50 58 a4 5c 50 5f ac 5c 68 5b ac 5d 18 5e +08 5d 7c 5e 98 5e 70 5f 80 4a f4 5f 04 5f 7c 5c 30 5e ec 5f 40 52 64 5e 20 5f 14 5f 50 59 18 5c +20 50 b8 5a a0 53 20 5b 00 5c 00 5e c0 5c 18 5c a0 59 c0 51 1c 5c b0 57 94 5f c0 5d 00 45 78 5b +80 53 70 5b 30 56 68 5e fc 5f 9c 5e b0 5a e0 5b 40 4f 38 5f ec 5c 04 5c 04 5c a0 5b d8 5f d0 5d +5c 5c bc 5f 28 5b fc 5f 20 5f 60 5d fc 5c 50 54 e0 52 c0 5d 80 4f f8 5d 30 5b 58 59 90 5d 00 59 +e0 51 08 5f 80 56 d0 5f dc 5c 20 5c 40 51 78 5d 0c 5d e0 52 00 53 e0 5d d4 5c a0 59 bc 5c 90 5e +00 55 a4 5e 28 5c a4 5d 80 51 c0 53 f0 5b 38 5c b8 59 e0 50 c0 52 30 5a a8 5f 24 5e 84 5e fc 5d +40 5a a4 5f 38 58 74 5f d8 5d 68 5d f8 5f b0 59 50 58 28 5b 60 52 24 5d 00 5f 38 59 c0 58 f0 5e +00 46 e8 5b d8 5b 68 5f 5c 5f d0 54 98 5c c0 5e 54 5e 40 55 30 55 c0 50 c8 5e e8 5c a4 5c 00 5f +60 57 98 58 88 5a 38 5c 80 49 84 5e 28 5e e4 5f 50 5a f0 5f 00 47 90 5a 24 5e 18 5a 58 58 e8 5e +00 48 58 59 a8 5e 1c 5d c0 50 80 4f e0 5f 88 5f c0 53 20 53 a0 53 14 5e c0 52 70 59 90 5d 74 5f +b0 5c 18 5b 78 58 60 5f d0 5b 14 5e c8 58 d8 5c e8 5f b0 59 b4 5e 38 5e 10 57 80 50 d0 5e 90 54 +10 57 0c 5e 2c 5d 28 5f f0 57 60 5f 80 5d a0 52 c0 4c 70 58 20 58 08 5f 58 5e 38 5e 74 5c 50 58 +20 50 a0 5c 40 51 00 5e 58 59 48 5d 20 52 90 5c f8 5a e0 56 40 55 f8 5a bc 5f 58 5d 58 5a d4 5f +54 5c 98 5b a0 5f 00 5a 00 52 80 4c 40 4c c4 5d ec 5c b4 5d 80 5b 90 56 f8 5a 58 58 00 5f 80 4a +80 49 00 60 90 5b 04 5d 80 59 f0 55 e4 5c f0 5f 80 51 b0 57 58 5c 70 54 68 5c ac 5d a0 5c 20 51 +98 5b ac 5e 18 5c 30 5f 10 5c 60 5d f8 5c 04 5e a0 5f b4 5d b8 5f f0 59 d8 5f b0 5d 90 58 00 45 +80 53 74 5c 94 5c 90 5f 60 5c c0 57 a8 58 00 4b c0 5c f0 55 84 5e 58 5f 70 5d c8 5e 08 5c 80 5e +50 5c b8 58 78 5f 98 5f a4 5c 6c 5f b0 5e c0 4f 00 5a a0 54 90 5b e0 5d 00 5f a0 53 a4 5c 00 44 +78 59 f0 5a 60 51 7c 5e e0 52 d0 5b 98 59 bc 5c a8 5d 00 5d 20 5c 30 5f 18 5f c0 54 00 5f 90 5a +a0 54 00 57 2c 5c 40 52 bc 5e 14 5e 60 50 90 5d b8 58 80 50 bc 5d 28 5d 3c 5e c0 54 80 4d 40 53 +00 42 28 59 00 47 a0 58 74 5d 98 5d d8 5b e0 59 10 5c 58 5a 88 5d d8 5d 38 5f 00 50 80 51 40 58 +30 58 0c 5e 50 55 00 42 4c 5f 98 5f 80 4e 90 5c 74 5f b8 58 00 46 80 51 c4 5f 50 54 b8 5d 18 59 +b8 59 84 5c 00 56 90 59 48 5f 34 5e 40 55 80 5f 90 58 50 57 40 5e 78 5f f0 55 40 5e 88 5e f8 5e +2c 5c 8c 5e 40 54 1c 5d 28 5e 5c 5d 24 5f 1c 5f 80 4e c0 4d 30 5e fc 5e 20 5b c0 4d 08 59 d8 5a +48 5c 28 5b a0 57 4c 5f 30 5b 70 5a 08 59 20 5b d0 54 00 47 90 56 78 5b b0 54 b0 5a d4 5c 70 55 +30 5b 34 5c 80 48 00 5d ac 5e d8 5e 58 5d 70 57 50 55 70 5b 18 5c 50 5f f0 5d 30 55 b0 5b f4 5c +20 53 20 59 08 5c 0c 5e 80 4b f8 5c 20 58 94 5c 70 59 00 51 c0 57 00 40 10 5e 00 53 80 4b d0 55 +48 5d 0c 5c fc 5e 58 5a d8 5a 14 5f 98 58 58 5d 20 56 d0 5b 80 4a cc 5f b8 5e 98 5a 30 59 94 5e +c0 51 d8 5d 58 58 20 5f 20 52 a0 51 94 5c b8 58 e0 5b 40 4c 30 55 90 5b 00 5d b0 5f 80 57 e0 51 +50 58 28 5d c8 58 f4 5d b0 54 80 4d a8 5a 30 58 70 58 24 5e 18 58 5c 5f d8 5f 70 57 18 5e c0 54 +64 5d 80 57 40 4c 68 5b 5c 5c f0 59 04 5c c0 50 d8 5a 20 57 84 5f 50 59 70 54 70 5a 80 4c 00 5e +7c 5c 88 58 d0 55 90 5d 68 5c e0 5d 00 4e 00 48 c0 50 38 5d 80 5f c0 58 10 5f 00 57 20 52 f0 58 +98 58 00 50 60 57 88 5e 50 56 90 59 50 58 a0 53 90 5a e0 52 a0 52 70 59 c4 5d a0 5b 40 54 80 57 +f4 5d 00 47 b8 5a a8 5e a0 55 60 58 f0 5b 90 54 78 5f 5c 5c 28 5f 50 58 a0 5e 40 54 e0 5d 60 56 +f0 5a 80 55 00 50 20 54 70 57 00 47 f0 5a 00 5e 34 5d d0 5a 80 5c d8 5b 08 58 88 5f 00 46 d8 5c +b8 59 90 56 80 55 40 4e c0 56 48 5f 58 5d 90 56 a0 53 80 55 fc 5f e8 5f 48 5b 38 58 80 50 e0 58 +10 5c a0 5a b0 5a ac 5d 84 5d b8 5f 80 59 80 48 78 5b 60 59 80 4a 50 5e 18 5f f0 55 00 54 80 52 +a8 5e 38 5c 80 54 80 56 4c 5f 9c 5d 00 5b b8 5c a8 59 40 50 a4 5d d4 5e c0 4c 58 58 00 50 70 59 +f0 5d 44 5e 20 5b f8 5d 78 5d 80 58 40 4f 80 56 98 5c 14 5c 74 5e 30 58 84 5d e0 5e 00 4a 18 58 +60 57 48 5a c0 51 f0 5d a8 5f b0 55 28 59 60 59 30 57 8c 5f 18 5c c8 5a 3c 5f 28 58 30 58 c0 4e +5c 5f fc 5d d0 55 30 56 5c 5f 88 5b 98 5b 44 5d c0 4d e0 52 30 5f d4 5e 70 5a 70 55 40 50 d0 5c +e8 5c 00 52 a8 5f fc 5f 70 56 98 5c 00 54 00 53 28 5f 00 5d 50 57 70 56 9c 5d 30 55 b4 5c 00 49 +38 5b 80 49 98 58 c4 5f 00 56 b0 5c 00 4e b4 5d d0 58 34 5c f8 5a 8c 5e a8 5a 20 50 10 5b 20 50 +10 54 00 51 c0 58 84 5f b8 5b a0 50 d8 5f f4 5f d0 58 70 5d 60 5b 0c 5e c0 4c cc 5f 74 5f 68 58 +64 5d 68 59 00 4b 00 4d 40 51 78 5e f0 5a 24 5c 1c 5f 5c 5e 60 5e c0 5d 10 5f e0 50 c8 5f f0 5f +48 5e 40 51 80 50 88 58 f0 5e b0 5d 88 5a e0 51 30 59 4c 5d 74 5f a0 5f 30 57 48 5f 88 59 54 5f +1c 5f 48 5e 40 54 b8 5a 98 5c 00 50 20 57 e0 5b c0 4e 10 59 6c 5c 18 5b 28 5c 80 4e b8 5d 30 57 +f0 5d 78 59 00 42 70 5a c0 4f 80 48 80 4f b8 5b 94 5d a8 5b c0 5b 60 5c 30 5e 30 54 64 5f 80 4e +a4 5f 04 5d 70 5d a4 5f 88 5e 28 5e 40 4f 80 4a 44 5c b4 5f 18 5e 10 57 50 54 e0 59 00 4f 40 4f +b4 5f 58 5b 40 57 e8 5f e0 54 c4 5e cc 5c 40 54 80 4f 20 50 80 50 58 58 e0 53 a8 58 60 50 b0 5d +00 50 dc 5c e0 55 d4 5f 00 5a fc 5c d0 54 f8 58 00 60 00 42 08 5f a0 5a d4 5f f4 5c 38 5f a0 50 +88 5e e4 5c d0 54 68 5e 68 5f a0 54 20 51 60 52 00 50 2c 5d d0 55 bc 5c 70 55 f8 5b b0 5a 20 5a +38 5f 34 5e 70 55 a0 5e 64 5e 48 58 80 50 98 5f 44 5e 78 5e 10 5c d8 5c 6c 5c 64 5e 60 55 00 49 +58 5d 18 58 00 45 90 5b 4c 5d 38 5b c0 56 48 5d 64 5c 30 5c 20 5e 4c 5d f4 5c ec 5d 30 54 00 49 +fc 5c 20 5b 00 55 10 5e 68 5f f8 59 68 5a 08 5f 30 54 30 5d 00 45 9c 5d a4 5f 80 5e 70 57 58 59 +74 5d e4 5e 80 4a 30 54 f8 5f 58 5c f8 5e 74 5d 78 5e c8 59 60 56 f0 5d 1c 5f 50 57 00 54 dc 5d +38 5e 30 56 30 55 84 5e 6c 5f 10 58 90 5c 40 5b 2c 5d 80 4b 40 51 f0 59 10 5d a0 5f 10 55 20 5a +20 5b d0 54 60 52 60 5a c0 50 c0 56 f8 58 48 5f 5c 5f 40 56 e4 5e 18 5f 00 42 70 54 80 4c f0 59 +3c 5d 40 51 40 4c 80 53 78 5a 4c 5d 80 50 14 5f 4c 5e 38 58 b0 56 e8 59 40 54 0c 5c 88 58 80 52 +80 4f c0 4e a0 57 b4 5e 5c 5e 4c 5d 3c 5c 84 5f f8 5f 2c 5e 78 59 98 5e c0 5b 10 5b 98 5c 18 5b +80 53 cc 5d 00 4c 4c 5c dc 5d 40 54 f0 56 50 5d 80 4c b4 5c 0c 5d 40 4d 1c 5d 48 5e 8c 5e 80 49 +30 54 38 5d 00 50 68 5c 0c 5e 68 59 e8 5a 28 59 00 4e c0 5b 20 51 b0 5b 10 57 bc 5f ac 5e e8 5a +90 5e 0c 5e 80 48 0c 5e a0 5a e8 58 e0 50 8c 5e 50 5a a0 53 68 5a 80 59 98 5d 24 5e a0 57 e0 50 +80 55 84 5f 00 5a 00 60 f8 5f 00 50 80 5b 00 5b b0 5e 78 5a 00 4c 70 55 d4 5e e0 58 a8 5b 00 44 +f8 5b d0 54 68 59 3c 5d 78 5c 2c 5e ec 5c a8 5f c4 5e 00 45 50 5f 08 5a 50 5e 70 58 d0 58 00 47 +b0 54 54 5e b0 5d d0 58 6c 5e 40 4e 40 5f b0 5f 70 5f f4 5c a0 51 50 54 c0 57 58 5d 00 5a 68 59 +30 56 9c 5d b0 55 fc 5c 50 5e 00 45 dc 5f c0 56 f0 55 54 5d 20 57 ac 5d 68 59 70 54 18 59 00 55 +d0 56 60 5b 80 4f 20 52 98 5e 74 5e 34 5d 98 5e 48 5a fc 5f 70 59 60 53 30 5e 44 5c 50 56 20 51 +d4 5e 50 5f 08 59 70 5e 24 5f 40 51 f0 58 ec 5f 60 50 c0 55 14 5d 08 58 40 52 30 5c c0 58 dc 5d +78 58 b0 56 c8 58 74 5f d4 5c 80 50 88 58 20 58 f0 58 00 54 e0 5a c0 5e a0 58 e0 57 98 5c 00 57 +94 5f cc 5c 40 51 f8 5f d0 59 00 4f 2c 5c ac 5f 58 5f f0 58 f0 55 38 5b 98 5a 94 5d 80 4c 60 50 +3c 5c f0 5c c0 4c 94 5d 20 56 40 57 d4 5e 70 5d 40 5b 00 4b 98 5b d4 5e 00 60 68 59 30 54 98 59 +fc 5e c0 57 f0 58 a8 5c f0 5e 10 54 50 5b 70 54 e0 53 50 5a f0 58 f4 5f f8 5d f0 5e e8 5b 98 5a +10 5f 24 5e 18 5e 1c 5d 4c 5e e0 54 00 49 00 5a d0 5a 00 55 30 5e b8 5f 10 5f f4 5f a8 5f 08 5d +60 55 fc 5d 80 48 20 51 c0 5f 00 50 48 5d 30 5e 84 5d d8 5a e0 52 e0 59 30 55 f8 5e 04 5f b0 57 +18 5b 00 44 80 54 40 5c 40 4f 68 5f 58 59 b4 5f c0 55 70 5d e8 58 d4 5f 5c 5d d0 5c b0 55 ac 5e +60 50 e4 5f c8 58 cc 5e 00 60 88 5d 58 5f 10 59 98 5b 78 58 88 5a c4 5e 4c 5c 00 3c 98 5f c0 4c +b0 59 70 5e 90 54 00 40 c0 54 80 4f c0 4f 78 5e 74 5e c0 52 98 5d 60 54 90 5d 88 5e 68 59 44 5c +50 56 40 5e 70 56 e0 50 00 50 b0 54 c8 59 18 5b d8 58 40 59 80 55 c8 5c 7c 5e a0 5f ac 5f 00 4d +00 49 d4 5d 40 5a 00 46 28 5a a0 57 70 58 b8 5a 98 59 08 58 a8 5a c8 5f d8 5b 44 5e 18 5c 80 51 +08 5a 30 57 a0 56 d0 5f 38 5b fc 5f 80 52 9c 5f 00 56 c4 5d 90 58 60 5d d0 5c 84 5f 2c 5c 00 59 +c0 4c 44 5e 94 5d 90 5c 54 5f 48 59 f0 54 04 5f 08 5c a0 54 bc 5c c4 5f 04 5c 90 5e fc 5f 48 5d +f0 5e 84 5c 18 58 68 5e 60 58 a0 5f 30 57 c8 59 54 5e c4 5f 8c 5f d8 5f e0 56 9c 5f f0 5f 24 5c +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +0c 5f 80 53 90 57 04 5f 98 5a 78 5d cc 5c 6c 5d 30 55 d0 58 30 55 78 5f 78 5b 08 5f c0 52 30 59 diff --git a/core/tests/fixtures/diskann_interleaved_4bit_v1.hex b/core/tests/fixtures/diskann_interleaved_4bit_v1.hex new file mode 100644 index 0000000..73d129f --- /dev/null +++ b/core/tests/fixtures/diskann_interleaved_4bit_v1.hex @@ -0,0 +1,384 @@ +44 41 4e 4e 01 00 00 00 00 01 00 00 3d 00 00 00 04 00 00 00 00 00 00 00 11 00 00 00 00 00 00 00 +00 00 00 00 04 00 00 00 08 00 00 00 9a 99 99 3f 0b 00 00 00 00 00 00 00 02 00 00 00 04 00 00 00 +00 10 00 00 04 00 00 00 03 00 00 00 02 00 00 00 08 00 00 00 07 00 00 00 00 30 00 00 00 00 00 00 +00 10 00 00 00 00 00 00 2c 01 00 00 00 00 00 00 2c 11 00 00 00 00 00 00 31 00 00 00 00 00 00 00 +5d 11 00 00 00 00 00 00 11 00 00 00 00 00 00 00 6e 11 00 00 00 00 00 00 44 00 00 00 00 00 00 00 +b2 11 00 00 00 00 00 00 54 00 00 00 00 00 00 00 00 20 00 00 00 00 00 00 00 10 00 00 00 00 00 00 +00 30 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +44 50 51 31 01 00 00 00 04 00 00 00 02 00 00 00 04 00 00 00 10 00 00 00 03 00 00 00 00 00 00 00 +00 00 00 00 02 00 00 00 04 00 00 00 00 00 00 00 00 00 00 3f 00 00 80 3f 00 00 c0 3f 00 00 00 40 +00 00 20 40 00 00 40 40 00 00 60 40 00 00 80 40 00 00 90 40 00 00 a0 40 00 00 b0 40 00 00 c0 40 +00 00 d0 40 00 00 e0 40 00 00 f0 40 6f 12 83 3a 89 41 00 3f c5 20 80 3f c5 20 c0 3f 62 10 00 40 +62 10 20 40 62 10 40 40 62 10 60 40 31 08 80 40 31 08 90 40 31 08 a0 40 31 08 b0 40 31 08 c0 40 +31 08 d0 40 31 08 e0 40 31 08 f0 40 6f 12 03 3b 12 83 00 3f 89 41 80 3f 89 41 c0 3f c5 20 00 40 +c5 20 20 40 c5 20 40 40 c5 20 60 40 62 10 80 40 62 10 90 40 62 10 a0 40 62 10 b0 40 62 10 c0 40 +62 10 d0 40 62 10 e0 40 62 10 f0 40 a6 9b 44 3b 9c c4 00 3f 4e 62 80 3f 4e 62 c0 3f 27 31 00 40 +27 31 20 40 27 31 40 40 27 31 60 40 93 18 80 40 93 18 90 40 93 18 a0 40 93 18 b0 40 93 18 c0 40 +93 18 d0 40 93 18 e0 40 93 18 f0 40 01 00 00 00 08 00 00 00 11 00 00 00 00 00 00 00 0c fe ff ff +ff ff ff ff 00 00 00 00 00 00 00 00 58 63 4d 6e 42 79 37 84 2c 8f 21 9a 16 a5 0b b0 00 ff ff 77 +ff 66 ff 55 ff 44 ff 33 ff 22 ff 11 ff 00 10 00 00 00 0e 00 00 00 0c 00 00 00 0a 00 00 00 08 00 +00 00 06 00 00 00 04 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 03 00 00 00 05 00 00 00 07 00 +00 00 09 00 00 00 0b 00 00 00 0d 00 00 00 0f 00 00 00 08 00 00 00 00 00 00 00 a7 00 00 00 00 00 +00 00 00 00 0a 00 14 00 1e 00 28 00 32 00 3c 00 46 00 50 00 5a 00 64 00 6e 00 78 00 82 00 8c 00 +96 00 00 00 02 00 02 00 02 00 02 00 02 00 02 00 02 00 02 00 02 00 02 00 02 00 02 00 02 00 02 00 +02 00 01 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 48 01 48 02 48 03 48 01 01 80 48 81 48 82 48 83 48 00 03 00 47 02 47 04 47 05 47 00 04 00 49 +01 49 03 49 04 49 01 04 00 46 02 46 03 46 05 46 02 04 80 49 81 49 83 49 84 49 03 04 00 45 01 45 +03 45 04 45 04 04 00 4a 02 4a 03 4a 05 4a 05 04 00 44 01 44 02 44 03 44 06 04 80 4a 82 4a 83 4a +85 4a 07 04 00 42 02 42 03 42 05 42 08 04 00 4b 02 4b 04 4b 05 4b 09 04 00 40 01 40 02 40 03 40 +0a 04 80 4b 82 4b 84 4b 86 4b 0b 04 00 3c 01 3c 02 3c 03 3c 0c 04 00 4c 01 4c 02 4c 03 4c 0d 00 +00 00 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 diff --git a/core/tests/fixtures/diskann_raw_row_ids_v1.hex b/core/tests/fixtures/diskann_raw_row_ids_v1.hex new file mode 100644 index 0000000..46b4dc4 --- /dev/null +++ b/core/tests/fixtures/diskann_raw_row_ids_v1.hex @@ -0,0 +1,385 @@ +44 41 4e 4e 01 00 00 00 00 01 00 00 1f 00 00 00 01 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 +00 00 00 00 02 00 00 00 04 00 00 00 9a 99 99 3f 0d 00 00 00 00 00 00 00 01 00 00 00 08 00 00 00 +00 10 00 00 04 00 00 00 03 00 00 00 01 00 00 00 04 00 00 00 07 00 00 00 0c 30 00 00 00 00 00 00 +00 10 00 00 00 00 00 00 28 04 00 00 00 00 00 00 28 14 00 00 00 00 00 00 38 00 00 00 00 00 00 00 +60 14 00 00 00 00 00 00 03 00 00 00 00 00 00 00 63 14 00 00 00 00 00 00 0c 00 00 00 00 00 00 00 +6f 14 00 00 00 00 00 00 14 00 00 00 00 00 00 00 00 20 00 00 00 00 00 00 00 10 00 00 00 00 00 00 +00 30 00 00 00 00 00 00 0c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +44 50 51 31 01 00 00 00 01 00 00 00 01 00 00 00 08 00 00 00 00 01 00 00 02 00 00 00 00 00 00 00 +00 00 00 00 01 00 00 00 00 00 00 00 00 00 80 3f 00 00 00 40 00 00 40 40 00 00 80 40 00 00 a0 40 +00 00 c0 40 00 00 e0 40 00 00 00 41 00 00 10 41 00 00 20 41 00 00 30 41 00 00 40 41 00 00 50 41 +00 00 60 41 00 00 70 41 00 00 80 41 00 00 88 41 00 00 90 41 00 00 98 41 00 00 a0 41 00 00 a8 41 +00 00 b0 41 00 00 b8 41 00 00 c0 41 00 00 c8 41 00 00 d0 41 00 00 d8 41 00 00 e0 41 00 00 e8 41 +00 00 f0 41 00 00 f8 41 00 00 00 42 00 00 04 42 00 00 08 42 00 00 0c 42 00 00 10 42 00 00 14 42 +00 00 18 42 00 00 1c 42 00 00 20 42 00 00 24 42 00 00 28 42 00 00 2c 42 00 00 30 42 00 00 34 42 +00 00 38 42 00 00 3c 42 00 00 40 42 00 00 44 42 00 00 48 42 00 00 4c 42 00 00 50 42 00 00 54 42 +00 00 58 42 00 00 5c 42 00 00 60 42 00 00 64 42 00 00 68 42 00 00 6c 42 00 00 70 42 00 00 74 42 +00 00 78 42 00 00 7c 42 00 00 80 42 00 00 82 42 00 00 84 42 00 00 86 42 00 00 88 42 00 00 8a 42 +00 00 8c 42 00 00 8e 42 00 00 90 42 00 00 92 42 00 00 94 42 00 00 96 42 00 00 98 42 00 00 9a 42 +00 00 9c 42 00 00 9e 42 00 00 a0 42 00 00 a2 42 00 00 a4 42 00 00 a6 42 00 00 a8 42 00 00 aa 42 +00 00 ac 42 00 00 ae 42 00 00 b0 42 00 00 b2 42 00 00 b4 42 00 00 b6 42 00 00 b8 42 00 00 ba 42 +00 00 bc 42 00 00 be 42 00 00 c0 42 00 00 c2 42 00 00 c4 42 00 00 c6 42 00 00 c8 42 00 00 ca 42 +00 00 cc 42 00 00 ce 42 00 00 d0 42 00 00 d2 42 00 00 d4 42 00 00 d6 42 00 00 d8 42 00 00 da 42 +00 00 dc 42 00 00 de 42 00 00 e0 42 00 00 e2 42 00 00 e4 42 00 00 e6 42 00 00 e8 42 00 00 ea 42 +00 00 ec 42 00 00 ee 42 00 00 f0 42 00 00 f2 42 00 00 f4 42 00 00 f6 42 00 00 f8 42 00 00 fa 42 +00 00 fc 42 00 00 fe 42 00 00 00 43 00 00 01 43 00 00 02 43 00 00 03 43 00 00 04 43 00 00 05 43 +00 00 06 43 00 00 07 43 00 00 08 43 00 00 09 43 00 00 0a 43 00 00 0b 43 00 00 0c 43 00 00 0d 43 +00 00 0e 43 00 00 0f 43 00 00 10 43 00 00 11 43 00 00 12 43 00 00 13 43 00 00 14 43 00 00 15 43 +00 00 16 43 00 00 17 43 00 00 18 43 00 00 19 43 00 00 1a 43 00 00 1b 43 00 00 1c 43 00 00 1d 43 +00 00 1e 43 00 00 1f 43 00 00 20 43 00 00 21 43 00 00 22 43 00 00 23 43 00 00 24 43 00 00 25 43 +00 00 26 43 00 00 27 43 00 00 28 43 00 00 29 43 00 00 2a 43 00 00 2b 43 00 00 2c 43 00 00 2d 43 +00 00 2e 43 00 00 2f 43 00 00 30 43 00 00 31 43 00 00 32 43 00 00 33 43 00 00 34 43 00 00 35 43 +00 00 36 43 00 00 37 43 00 00 38 43 00 00 39 43 00 00 3a 43 00 00 3b 43 00 00 3c 43 00 00 3d 43 +00 00 3e 43 00 00 3f 43 00 00 40 43 00 00 41 43 00 00 42 43 00 00 43 43 00 00 44 43 00 00 45 43 +00 00 46 43 00 00 47 43 00 00 48 43 00 00 49 43 00 00 4a 43 00 00 4b 43 00 00 4c 43 00 00 4d 43 +00 00 4e 43 00 00 4f 43 00 00 50 43 00 00 51 43 00 00 52 43 00 00 53 43 00 00 54 43 00 00 55 43 +00 00 56 43 00 00 57 43 00 00 58 43 00 00 59 43 00 00 5a 43 00 00 5b 43 00 00 5c 43 00 00 5d 43 +00 00 5e 43 00 00 5f 43 00 00 60 43 00 00 61 43 00 00 62 43 00 00 63 43 00 00 64 43 00 00 65 43 +00 00 66 43 00 00 67 43 00 00 68 43 00 00 69 43 00 00 6a 43 00 00 6b 43 00 00 6c 43 00 00 6d 43 +00 00 6e 43 00 00 6f 43 00 00 70 43 00 00 71 43 00 00 72 43 00 00 73 43 00 00 74 43 00 00 75 43 +00 00 76 43 00 00 77 43 00 00 78 43 00 00 79 43 00 00 7a 43 00 00 7b 43 00 00 7c 43 00 00 7d 43 +00 00 7e 43 00 00 7f 43 00 00 00 00 40 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 80 ff ff ff ff ff ff ff 7f +01 00 02 01 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 03 00 02 00 01 +00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +01 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 80 3f 00 00 00 00 00 00 00 40 diff --git a/core/tests/fixtures/diskann_v1.hex b/core/tests/fixtures/diskann_v1.hex new file mode 100644 index 0000000..27cb976 --- /dev/null +++ b/core/tests/fixtures/diskann_v1.hex @@ -0,0 +1,385 @@ +44 41 4e 4e 01 00 00 00 00 01 00 00 1f 00 00 00 01 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 +00 00 00 00 02 00 00 00 04 00 00 00 9a 99 99 3f 2a 00 00 00 00 00 00 00 01 00 00 00 08 00 00 00 +00 10 00 00 04 00 00 00 03 00 00 00 02 00 00 00 02 00 00 00 07 00 00 00 02 30 00 00 00 00 00 00 +00 10 00 00 00 00 00 00 28 04 00 00 00 00 00 00 28 14 00 00 00 00 00 00 20 00 00 00 00 00 00 00 +48 14 00 00 00 00 00 00 01 00 00 00 00 00 00 00 49 14 00 00 00 00 00 00 04 00 00 00 00 00 00 00 +4d 14 00 00 00 00 00 00 0c 00 00 00 00 00 00 00 00 20 00 00 00 00 00 00 00 10 00 00 00 00 00 00 +00 30 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +44 50 51 31 01 00 00 00 01 00 00 00 01 00 00 00 08 00 00 00 00 01 00 00 02 00 00 00 00 00 00 00 +00 00 00 00 01 00 00 00 00 00 00 00 00 00 80 3e 00 00 00 3f 00 00 40 3f 00 00 80 3f 00 00 a0 3f +00 00 c0 3f 00 00 e0 3f 00 00 00 40 00 00 10 40 00 00 20 40 00 00 30 40 00 00 40 40 00 00 50 40 +00 00 60 40 00 00 70 40 00 00 80 40 00 00 88 40 00 00 90 40 00 00 98 40 00 00 a0 40 00 00 a8 40 +00 00 b0 40 00 00 b8 40 00 00 c0 40 00 00 c8 40 00 00 d0 40 00 00 d8 40 00 00 e0 40 00 00 e8 40 +00 00 f0 40 00 00 f8 40 00 00 00 41 00 00 04 41 00 00 08 41 00 00 0c 41 00 00 10 41 00 00 14 41 +00 00 18 41 00 00 1c 41 00 00 20 41 00 00 24 41 00 00 28 41 00 00 2c 41 00 00 30 41 00 00 34 41 +00 00 38 41 00 00 3c 41 00 00 40 41 00 00 44 41 00 00 48 41 00 00 4c 41 00 00 50 41 00 00 54 41 +00 00 58 41 00 00 5c 41 00 00 60 41 00 00 64 41 00 00 68 41 00 00 6c 41 00 00 70 41 00 00 74 41 +00 00 78 41 00 00 7c 41 00 00 80 41 00 00 82 41 00 00 84 41 00 00 86 41 00 00 88 41 00 00 8a 41 +00 00 8c 41 00 00 8e 41 00 00 90 41 00 00 92 41 00 00 94 41 00 00 96 41 00 00 98 41 00 00 9a 41 +00 00 9c 41 00 00 9e 41 00 00 a0 41 00 00 a2 41 00 00 a4 41 00 00 a6 41 00 00 a8 41 00 00 aa 41 +00 00 ac 41 00 00 ae 41 00 00 b0 41 00 00 b2 41 00 00 b4 41 00 00 b6 41 00 00 b8 41 00 00 ba 41 +00 00 bc 41 00 00 be 41 00 00 c0 41 00 00 c2 41 00 00 c4 41 00 00 c6 41 00 00 c8 41 00 00 ca 41 +00 00 cc 41 00 00 ce 41 00 00 d0 41 00 00 d2 41 00 00 d4 41 00 00 d6 41 00 00 d8 41 00 00 da 41 +00 00 dc 41 00 00 de 41 00 00 e0 41 00 00 e2 41 00 00 e4 41 00 00 e6 41 00 00 e8 41 00 00 ea 41 +00 00 ec 41 00 00 ee 41 00 00 f0 41 00 00 f2 41 00 00 f4 41 00 00 f6 41 00 00 f8 41 00 00 fa 41 +00 00 fc 41 00 00 fe 41 00 00 00 42 00 00 01 42 00 00 02 42 00 00 03 42 00 00 04 42 00 00 05 42 +00 00 06 42 00 00 07 42 00 00 08 42 00 00 09 42 00 00 0a 42 00 00 0b 42 00 00 0c 42 00 00 0d 42 +00 00 0e 42 00 00 0f 42 00 00 10 42 00 00 11 42 00 00 12 42 00 00 13 42 00 00 14 42 00 00 15 42 +00 00 16 42 00 00 17 42 00 00 18 42 00 00 19 42 00 00 1a 42 00 00 1b 42 00 00 1c 42 00 00 1d 42 +00 00 1e 42 00 00 1f 42 00 00 20 42 00 00 21 42 00 00 22 42 00 00 23 42 00 00 24 42 00 00 25 42 +00 00 26 42 00 00 27 42 00 00 28 42 00 00 29 42 00 00 2a 42 00 00 2b 42 00 00 2c 42 00 00 2d 42 +00 00 2e 42 00 00 2f 42 00 00 30 42 00 00 31 42 00 00 32 42 00 00 33 42 00 00 34 42 00 00 35 42 +00 00 36 42 00 00 37 42 00 00 38 42 00 00 39 42 00 00 3a 42 00 00 3b 42 00 00 3c 42 00 00 3d 42 +00 00 3e 42 00 00 3f 42 00 00 40 42 00 00 41 42 00 00 42 42 00 00 43 42 00 00 44 42 00 00 45 42 +00 00 46 42 00 00 47 42 00 00 48 42 00 00 49 42 00 00 4a 42 00 00 4b 42 00 00 4c 42 00 00 4d 42 +00 00 4e 42 00 00 4f 42 00 00 50 42 00 00 51 42 00 00 52 42 00 00 53 42 00 00 54 42 00 00 55 42 +00 00 56 42 00 00 57 42 00 00 58 42 00 00 59 42 00 00 5a 42 00 00 5b 42 00 00 5c 42 00 00 5d 42 +00 00 5e 42 00 00 5f 42 00 00 60 42 00 00 61 42 00 00 62 42 00 00 63 42 00 00 64 42 00 00 65 42 +00 00 66 42 00 00 67 42 00 00 68 42 00 00 69 42 00 00 6a 42 00 00 6b 42 00 00 6c 42 00 00 6d 42 +00 00 6e 42 00 00 6f 42 00 00 70 42 00 00 71 42 00 00 72 42 00 00 73 42 00 00 74 42 00 00 75 42 +00 00 76 42 00 00 77 42 00 00 78 42 00 00 79 42 00 00 7a 42 00 00 7b 42 00 00 7c 42 00 00 7d 42 +00 00 7e 42 00 00 7f 42 01 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 diff --git a/core/tests/fixtures/ivf_hnsw_flat_v1.hex b/core/tests/fixtures/ivf_hnsw_flat_v1.hex deleted file mode 100644 index d88c2d7..0000000 --- a/core/tests/fixtures/ivf_hnsw_flat_v1.hex +++ /dev/null @@ -1,7 +0,0 @@ -4c 46 48 49 01 00 00 00 02 00 00 00 02 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 02 00 00 00 -08 00 00 00 03 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 -00 00 00 00 00 00 00 00 00 00 20 41 00 00 20 41 80 00 00 00 00 00 00 00 01 00 00 00 11 00 00 00 -26 00 00 00 00 00 00 00 a6 00 00 00 00 00 00 00 01 00 00 00 11 00 00 00 26 00 00 00 00 00 00 00 -07 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 52 47 57 48 01 00 00 00 01 00 00 -00 01 00 00 00 00 63 00 00 00 00 00 00 00 01 00 00 00 00 00 00 20 41 00 00 20 41 52 47 57 48 01 -00 00 00 01 00 00 00 01 00 00 00 00 diff --git a/core/tests/fixtures/ivf_hnsw_sq_v1.hex b/core/tests/fixtures/ivf_hnsw_sq_v1.hex deleted file mode 100644 index 1715feb..0000000 --- a/core/tests/fixtures/ivf_hnsw_sq_v1.hex +++ /dev/null @@ -1,8 +0,0 @@ -51 53 48 49 01 00 00 00 02 00 00 00 02 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 02 00 00 00 -08 00 00 00 03 00 00 00 00 00 00 00 00 00 80 3f 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 -00 00 00 00 00 00 00 00 00 00 80 3f 00 00 80 3f 00 00 00 00 00 00 00 00 00 00 80 3f 00 00 80 3f -00 00 00 00 00 00 00 00 00 00 80 3f 00 00 80 3f 00 00 00 00 00 00 00 00 00 00 20 41 00 00 20 41 -b0 00 00 00 00 00 00 00 01 00 00 00 11 00 00 00 20 00 00 00 00 00 00 00 d0 00 00 00 00 00 00 00 -01 00 00 00 11 00 00 00 20 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 01 00 00 00 00 00 00 52 -47 57 48 01 00 00 00 01 00 00 00 01 00 00 00 00 63 00 00 00 00 00 00 00 01 00 00 00 00 00 00 52 -47 57 48 01 00 00 00 01 00 00 00 01 00 00 00 00 diff --git a/core/tests/fixtures/ivf_rq_v1.hex b/core/tests/fixtures/ivf_rq_v1.hex index df50d56..cce3ac8 100644 --- a/core/tests/fixtures/ivf_rq_v1.hex +++ b/core/tests/fixtures/ivf_rq_v1.hex @@ -1,8 +1,11 @@ -51 52 56 49 01 00 00 00 08 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 -15 7c 4a 7f b9 79 37 9e 03 00 00 00 01 00 00 00 01 00 00 00 01 00 00 00 01 00 00 00 00 00 00 00 -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +51 52 56 49 01 00 00 00 08 00 00 00 40 00 00 00 02 00 00 00 00 00 00 00 07 00 00 00 04 00 00 00 +03 00 00 00 00 00 00 00 15 7c 4a 7f b9 79 37 9e 04 00 00 00 08 00 00 00 02 00 00 00 03 00 00 00 +00 00 00 3f 00 00 00 3f 00 00 00 3f 00 00 00 3f 00 00 00 3f 00 00 00 3f 00 00 00 3f 00 00 00 3f 00 00 20 41 00 00 20 41 00 00 20 41 00 00 20 41 00 00 20 41 00 00 20 41 00 00 20 41 00 00 20 41 -a0 00 00 00 00 00 00 00 02 00 00 00 02 00 00 00 c8 00 00 00 00 00 00 00 01 00 00 00 01 00 00 00 -07 00 00 00 00 00 00 00 02 00 00 00 00 23 00 ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 80 3f -00 00 80 3f 00 00 00 00 63 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 -00 00 +a0 00 00 00 00 00 00 00 02 00 00 00 02 00 00 00 1a 01 00 00 00 00 00 00 01 00 00 00 01 00 00 00 +07 00 00 00 00 00 00 00 02 00 00 00 40 00 00 00 00 23 a4 5b bf 40 34 cb 5d b2 fa 05 9d 62 74 8b +e4 1b 4b b4 40 bf cb 34 62 8d 04 fb d2 2d 9b 64 03 fc 49 b6 40 bf cf 30 a2 4d 05 fa 60 9f f9 06 +79 86 0c f3 8c 73 e9 16 23 cc f0 0f 60 9f d1 2e 6f 90 00 00 c0 40 00 00 00 c0 85 18 7b bf 85 18 +7b bf bf 03 2e 40 bf 03 2e 40 00 00 c0 40 00 00 00 c0 bf 7f 0a be bf 7f 0a be 63 00 00 00 00 00 +00 00 01 00 00 00 20 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 diff --git a/core/tests/fixtures/ivf_sq_v1.hex b/core/tests/fixtures/ivf_sq_v1.hex new file mode 100644 index 0000000..b2d7d46 --- /dev/null +++ b/core/tests/fixtures/ivf_sq_v1.hex @@ -0,0 +1,6 @@ +51 53 56 49 01 00 00 00 02 00 00 00 02 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 08 00 00 00 +03 00 00 00 00 00 00 00 00 00 80 3f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 80 3f 00 00 80 3f 00 00 00 00 00 00 00 00 00 00 80 3f 00 00 80 3f +00 00 00 00 00 00 00 00 00 00 80 3f 00 00 80 3f 00 00 00 00 00 00 00 00 00 00 20 41 00 00 20 41 +a0 00 00 00 00 00 00 00 01 00 00 00 01 00 00 00 af 00 00 00 00 00 00 00 01 00 00 00 01 00 00 00 +00 00 07 00 00 00 00 00 00 00 01 00 00 00 00 00 00 63 00 00 00 00 00 00 00 01 00 00 00 00 diff --git a/core/tests/storage_format_fixtures.rs b/core/tests/storage_format_fixtures.rs index 8f52ca5..3ce9e2f 100644 --- a/core/tests/storage_format_fixtures.rs +++ b/core/tests/storage_format_fixtures.rs @@ -15,22 +15,23 @@ // specific language governing permissions and limitations // under the License. +use paimon_vindex_core::diskann::{ + DiskAnnBuildDistance, DiskAnnBuildParams, DiskAnnIndex, DiskAnnRawVectorEncoding, + DiskAnnStorageLayout, +}; +use paimon_vindex_core::diskann_io::write_diskann_index; use paimon_vindex_core::distance::MetricType; -use paimon_vindex_core::hnsw::HnswBuildParams; use paimon_vindex_core::index::{ IndexType, VectorIndexMetadata, VectorIndexReader, VectorSearchParams, }; use paimon_vindex_core::io::{write_index, PosWriter}; use paimon_vindex_core::ivfflat::IVFFlatIndex; use paimon_vindex_core::ivfflat_io::write_ivfflat_index; -use paimon_vindex_core::ivfhnswflat::IVFHNSWFlatIndex; -use paimon_vindex_core::ivfhnswflat_io::write_ivfhnswflat_index; -use paimon_vindex_core::ivfhnswsq::IVFHNSWSQIndex; -use paimon_vindex_core::ivfhnswsq_io::write_ivfhnswsq_index; use paimon_vindex_core::ivfpq::IVFPQIndex; use paimon_vindex_core::ivfrq::IVFRQIndex; use paimon_vindex_core::ivfrq_io::write_ivfrq_index; -use paimon_vindex_core::rq::RQCodeFactors; +use paimon_vindex_core::ivfsq::IVFSQIndex; +use paimon_vindex_core::ivfsq_io::write_ivfsq_index; use paimon_vindex_core::sq::ScalarQuantizer; use std::fmt::Write as _; use std::io::Cursor; @@ -45,7 +46,7 @@ struct FixtureCase { metric: MetricType, total_vectors: i64, pq_m: Option, - hnsw: Option, + pq_bits: Option, query: Vec, params: VectorSearchParams, expected_first_id: i64, @@ -62,6 +63,7 @@ fn storage_format_v1_golden_fixtures_match_current_writers_and_readers() { "{} writer output changed", case.name ); + assert_diskann_fixture_variant(&case, &fixture); let mut reader = VectorIndexReader::open(Cursor::new(fixture)).unwrap(); assert_metadata(&reader.metadata(), &case); @@ -87,6 +89,66 @@ fn storage_format_v1_golden_fixtures_match_current_writers_and_readers() { } } +fn assert_diskann_fixture_variant(case: &FixtureCase, fixture: &[u8]) { + if case.index_type != IndexType::DiskAnn { + return; + } + assert_eq!(read_u32(fixture, 60) as usize, case.pq_bits.unwrap()); + let expected_encoding = if case.name == "diskann_raw_row_ids_v1" { + DiskAnnRawVectorEncoding::F32 + } else { + DiskAnnRawVectorEncoding::F16 + }; + let element_size = match expected_encoding { + DiskAnnRawVectorEncoding::F32 => 4, + DiskAnnRawVectorEncoding::F16 => 2, + }; + assert_eq!(read_u32(fixture, 76), expected_encoding as u32); + assert_eq!( + read_u32(fixture, 80) as usize, + case.dimension * element_size + ); + match case.name { + "diskann_compact_multipage_v1" => { + assert!( + read_u64(fixture, 184) > 4096, + "fixture must span multiple adjacency pages" + ); + assert!( + read_u64(fixture, 200) > 4096, + "fixture must contain more raw-vector data than one logical page" + ); + } + "diskann_interleaved_4bit_v1" => { + assert_ne!( + read_u32(fixture, 12) & (1 << 5), + 0, + "fixture must select the interleaved layout" + ); + assert_eq!( + read_u64(fixture, 200), + 0, + "interleaved layout must not have a separate vector section" + ); + } + compact_name => { + assert_eq!( + read_u64(fixture, 200), + case.total_vectors as u64 * case.dimension as u64 * element_size as u64, + "{compact_name} compact raw vectors must be densely packed" + ); + if compact_name == "diskann_raw_row_ids_v1" { + let row_ids_offset = read_u64(fixture, 112) as usize; + assert_eq!( + read_u32(fixture, row_ids_offset), + 0, + "extreme row IDs must select raw i64 encoding" + ); + } + } + } +} + #[test] fn storage_format_v1_golden_fixtures_support_search_warmup() { for case in fixture_cases() { @@ -123,16 +185,18 @@ fn storage_format_v1_golden_fixtures_support_search_warmup() { fn print_storage_format_v1_fixture_hex() { for case in fixture_cases() { println!("-- {} --", case.name); - println!("{}", bytes_to_hex(&(case.build)())); + let fixture = bytes_to_hex(&(case.build)()); + println!("{}", fixture); + if std::env::var_os("UPDATE_STORAGE_FORMAT_FIXTURES").is_some() { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures") + .join(format!("{}.hex", case.name)); + std::fs::write(path, fixture).unwrap(); + } } } fn fixture_cases() -> Vec { - let hnsw = HnswBuildParams { - m: 2, - ef_construction: 8, - max_level: 3, - }; vec![ FixtureCase { name: "ivf_flat_v1", @@ -144,7 +208,7 @@ fn fixture_cases() -> Vec { metric: MetricType::L2, total_vectors: 3, pq_m: None, - hnsw: None, + pq_bits: None, query: vec![0.0, 0.0], params: VectorSearchParams::new(2, 2), expected_first_id: 7, @@ -159,7 +223,7 @@ fn fixture_cases() -> Vec { metric: MetricType::L2, total_vectors: 3, pq_m: Some(1), - hnsw: None, + pq_bits: Some(8), query: vec![0.0], params: VectorSearchParams::new(2, 2), expected_first_id: 10, @@ -174,7 +238,7 @@ fn fixture_cases() -> Vec { metric: MetricType::L2, total_vectors: 3, pq_m: Some(2), - hnsw: None, + pq_bits: Some(4), query: vec![0.0, 0.0], params: VectorSearchParams::new(2, 2), expected_first_id: 5, @@ -189,44 +253,228 @@ fn fixture_cases() -> Vec { metric: MetricType::L2, total_vectors: 3, pq_m: None, - hnsw: None, + pq_bits: None, query: vec![0.0; 8], params: VectorSearchParams::new(2, 2), - expected_first_id: 7, + expected_first_id: 42, }, FixtureCase { - name: "ivf_hnsw_flat_v1", - fixture_hex: include_str!("fixtures/ivf_hnsw_flat_v1.hex"), - build: build_ivf_hnsw_flat_fixture, - index_type: IndexType::IvfHnswFlat, + name: "ivf_sq_v1", + fixture_hex: include_str!("fixtures/ivf_sq_v1.hex"), + build: build_ivf_sq_fixture, + index_type: IndexType::IvfSq, dimension: 2, nlist: 2, metric: MetricType::L2, total_vectors: 2, pq_m: None, - hnsw: Some(hnsw), + pq_bits: Some(8), query: vec![0.0, 0.0], - params: VectorSearchParams::with_ef_search(1, 2, 8), + params: VectorSearchParams::new(1, 2), expected_first_id: 7, }, FixtureCase { - name: "ivf_hnsw_sq_v1", - fixture_hex: include_str!("fixtures/ivf_hnsw_sq_v1.hex"), - build: build_ivf_hnsw_sq_fixture, - index_type: IndexType::IvfHnswSq, - dimension: 2, - nlist: 2, + name: "diskann_v1", + fixture_hex: include_str!("fixtures/diskann_v1.hex"), + build: build_diskann_fixture, + index_type: IndexType::DiskAnn, + dimension: 1, + nlist: 1, metric: MetricType::L2, - total_vectors: 2, - pq_m: None, - hnsw: Some(hnsw), - query: vec![0.0, 0.0], - params: VectorSearchParams::with_ef_search(1, 2, 8), + total_vectors: 1, + pq_m: Some(1), + pq_bits: Some(8), + query: vec![0.0], + params: VectorSearchParams::with_l_search(1, 4), expected_first_id: 7, }, + FixtureCase { + name: "diskann_compact_multipage_v1", + fixture_hex: include_str!("fixtures/diskann_compact_multipage_v1.hex"), + build: build_diskann_compact_multipage_fixture, + index_type: IndexType::DiskAnn, + dimension: 16, + nlist: 1, + metric: MetricType::L2, + total_vectors: 513, + pq_m: Some(4), + pq_bits: Some(8), + query: vec![0.0; 16], + params: VectorSearchParams::with_l_search(1, 513), + expected_first_id: 10_000, + }, + FixtureCase { + name: "diskann_interleaved_4bit_v1", + fixture_hex: include_str!("fixtures/diskann_interleaved_4bit_v1.hex"), + build: build_diskann_interleaved_4bit_fixture, + index_type: IndexType::DiskAnn, + dimension: 4, + nlist: 1, + metric: MetricType::L2, + total_vectors: 17, + pq_m: Some(2), + pq_bits: Some(4), + query: vec![0.0; 4], + params: VectorSearchParams::with_l_search(1, 16), + expected_first_id: -500, + }, + FixtureCase { + name: "diskann_raw_row_ids_v1", + fixture_hex: include_str!("fixtures/diskann_raw_row_ids_v1.hex"), + build: build_diskann_raw_row_ids_fixture, + index_type: IndexType::DiskAnn, + dimension: 1, + nlist: 1, + metric: MetricType::L2, + total_vectors: 3, + pq_m: Some(1), + pq_bits: Some(8), + query: vec![0.0], + params: VectorSearchParams::with_l_search(1, 4), + expected_first_id: i64::MIN, + }, ] } +fn build_diskann_fixture() -> Vec { + let mut index = DiskAnnIndex::new( + 1, + MetricType::L2, + 1, + DiskAnnBuildParams { + max_degree: 2, + build_search_list_size: 4, + alpha: 1.2, + seed: 42, + memory_budget_bytes: 1024 * 1024, + ..DiskAnnBuildParams::default() + }, + ); + index.pq.centroids = (0..256).map(|code| code as f32 * 0.25).collect(); + index.pq.rebuild_norms_cache(); + index.ids = vec![7]; + index.vectors = vec![0.0]; + write_diskann_fixture(index) +} + +fn build_diskann_compact_multipage_fixture() -> Vec { + let dimension = 16; + let count = 513; + let mut index = DiskAnnIndex::new( + dimension, + MetricType::L2, + 4, + DiskAnnBuildParams { + max_degree: 32, + build_search_list_size: 64, + alpha: 1.2, + seed: 7, + memory_budget_bytes: 16 * 1024 * 1024, + storage_layout: DiskAnnStorageLayout::Compact, + build_distance: DiskAnnBuildDistance::FullPrecision, + ..DiskAnnBuildParams::default() + }, + ); + index.pq.centroids = (0..dimension) + .flat_map(|coordinate| (0..256).map(move |code| code as f32 + coordinate as f32 * 0.001)) + .collect(); + index.pq.rebuild_norms_cache(); + index.ids = (0..count).map(|node| 10_000 + node as i64 * 7).collect(); + index.vectors = (0..count) + .flat_map(|node| { + (0..dimension).map(move |coordinate| { + if node == 0 { + 0.0 + } else { + // Integer-valued coordinates keep every squared L2 + // accumulation exactly representable in f32. The golden + // graph is therefore stable across SIMD implementations + // and coverage instrumentation. + let mut value = (node as u32) + .wrapping_mul(747_796_405) + .wrapping_add((coordinate as u32).wrapping_mul(2_891_336_453)) + .wrapping_add(277_803_737); + value = (value ^ (value >> 16)).wrapping_mul(2_246_822_519); + value ^= value >> 13; + 1.0 + (value % 512) as f32 + } + }) + }) + .collect(); + write_diskann_fixture(index) +} + +fn build_diskann_interleaved_4bit_fixture() -> Vec { + let dimension = 4; + let count = 17; + let mut index = DiskAnnIndex::with_pq_bits( + dimension, + MetricType::L2, + 2, + 4, + DiskAnnBuildParams { + max_degree: 4, + build_search_list_size: 8, + alpha: 1.2, + seed: 11, + memory_budget_bytes: 1024 * 1024, + storage_layout: DiskAnnStorageLayout::Interleaved, + raw_vector_encoding: DiskAnnRawVectorEncoding::F16, + ..DiskAnnBuildParams::default() + }, + ); + index.pq.centroids = (0..dimension) + .flat_map(|coordinate| { + (0..16).map(move |code| code as f32 * 0.5 + coordinate as f32 * 0.001) + }) + .collect(); + index.pq.rebuild_norms_cache(); + index.ids = (0..count).map(|node| -500 + node as i64 * 11).collect(); + index.vectors = (0..count) + .flat_map(|node| { + (0..dimension) + .map(move |coordinate| node as f32 + coordinate as f32 * 0.001 * node as f32) + }) + .collect(); + write_diskann_fixture(index) +} + +fn build_diskann_raw_row_ids_fixture() -> Vec { + let mut index = DiskAnnIndex::new( + 1, + MetricType::L2, + 1, + DiskAnnBuildParams { + max_degree: 2, + build_search_list_size: 4, + alpha: 1.2, + seed: 13, + memory_budget_bytes: 1024 * 1024, + raw_vector_encoding: DiskAnnRawVectorEncoding::F32, + ..DiskAnnBuildParams::default() + }, + ); + index.pq.centroids = (0..256).map(|code| code as f32).collect(); + index.pq.rebuild_norms_cache(); + index.ids = vec![i64::MIN, 0, i64::MAX]; + index.vectors = vec![0.0, 1.0, 2.0]; + write_diskann_fixture(index) +} + +fn write_diskann_fixture(index: DiskAnnIndex) -> Vec { + // Pin the builder to one worker so the golden graph is independent of the + // host's Rayon worker count and scheduling. + rayon::ThreadPoolBuilder::new() + .num_threads(1) + .build() + .unwrap() + .install(|| { + let mut buf = Vec::new(); + write_diskann_index(&index, &mut PosWriter::new(&mut buf)).unwrap(); + buf + }) +} + fn build_ivf_flat_fixture() -> Vec { let index = IVFFlatIndex { d: 2, @@ -270,64 +518,33 @@ fn build_ivf_pq_4bit_fixture() -> Vec { } fn build_ivf_rq_fixture() -> Vec { - let mut index = IVFRQIndex::new(8, 2, MetricType::L2); - index.quantizer_centroids = vec![ - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, - ]; - index.ids = vec![vec![42, 7], vec![99]]; - index.codes = vec![vec![0xFF, 0x00], vec![0x00]]; - index.factors = vec![ - vec![ - RQCodeFactors { - residual_norm_sqr: 1.0, - vector_norm_sqr: 1.0, - dp_multiplier: 0.0, - }, - RQCodeFactors::zero(), - ], - vec![RQCodeFactors::zero()], + let data = vec![ + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 10.0, 10.0, + 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, ]; + let mut index = IVFRQIndex::with_bits(8, 2, 4, MetricType::L2); + index.train(&data, 3); + index.add(&data, &[42, 7, 99], 3); let mut buf = Vec::new(); write_ivfrq_index(&index, &mut PosWriter::new(&mut buf)).unwrap(); buf } -fn build_ivf_hnsw_flat_fixture() -> Vec { - let mut index = IVFHNSWFlatIndex::new(2, 2, MetricType::L2, fixture_hnsw_params()); - index.flat.quantizer_centroids = vec![0.0, 0.0, 10.0, 10.0]; - index.flat.ids = vec![vec![7], vec![99]]; - index.flat.vectors = vec![vec![0.0, 0.0], vec![10.0, 10.0]]; - index.build_graphs().unwrap(); - - let mut buf = Vec::new(); - write_ivfhnswflat_index(&index, &mut PosWriter::new(&mut buf)).unwrap(); - buf -} - -fn build_ivf_hnsw_sq_fixture() -> Vec { +fn build_ivf_sq_fixture() -> Vec { let sq = ScalarQuantizer::with_dimension_bounds(2, vec![0.0, 0.0], vec![1.0, 1.0]); - let mut index = IVFHNSWSQIndex::new(2, 2, MetricType::L2, fixture_hnsw_params()); + let mut index = IVFSQIndex::new(2, 2, MetricType::L2); index.quantizer_centroids = vec![0.0, 0.0, 10.0, 10.0]; index.sq = sq.clone(); index.list_sqs = vec![sq; 2]; index.ids = vec![vec![7], vec![99]]; index.codes = vec![vec![0, 0], vec![0, 0]]; - index.build_graphs().unwrap(); let mut buf = Vec::new(); - write_ivfhnswsq_index(&index, &mut PosWriter::new(&mut buf)).unwrap(); + write_ivfsq_index(&index, &mut PosWriter::new(&mut buf)).unwrap(); buf } -fn fixture_hnsw_params() -> HnswBuildParams { - HnswBuildParams { - m: 2, - ef_construction: 8, - max_level: 3, - } -} - fn assert_metadata(metadata: &VectorIndexMetadata, case: &FixtureCase) { assert_eq!( metadata.index_type, case.index_type, @@ -347,13 +564,11 @@ fn assert_metadata(metadata: &VectorIndexMetadata, case: &FixtureCase) { case.name ); assert_eq!(metadata.pq_m, case.pq_m, "{} pq m", case.name); + assert_eq!(metadata.pq_bits, case.pq_bits, "{} pq bits", case.name); assert_eq!( - metadata - .hnsw - .map(|params| (params.m, params.ef_construction, params.max_level)), - case.hnsw - .map(|params| (params.m, params.ef_construction, params.max_level)), - "{} hnsw params", + metadata.rq_bits, + (case.index_type == IndexType::IvfRq).then_some(4), + "{} rq bits", case.name ); } @@ -374,6 +589,14 @@ fn hex_to_bytes(hex: &str) -> Vec { .collect() } +fn read_u32(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap()) +} + +fn read_u64(bytes: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(bytes[offset..offset + 8].try_into().unwrap()) +} + fn bytes_to_hex(bytes: &[u8]) -> String { let mut hex = String::new(); for (idx, byte) in bytes.iter().enumerate() { diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 5010c07..3f42d68 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -20,6 +20,7 @@ project(paimon-vindex-cpp-test CXX) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) +find_package(Threads REQUIRED) set(PAIMON_VINDEX_FFI_LIB "" CACHE FILEPATH "Path to libpaimon_vindex_ffi shared library") if(NOT PAIMON_VINDEX_FFI_LIB) @@ -34,4 +35,4 @@ endif() add_executable(test_vindex_cpp test_vindex.cpp) target_include_directories(test_vindex_cpp PRIVATE ${CMAKE_SOURCE_DIR}/../include) -target_link_libraries(test_vindex_cpp ${PAIMON_VINDEX_FFI_LIB}) +target_link_libraries(test_vindex_cpp ${PAIMON_VINDEX_FFI_LIB} Threads::Threads) diff --git a/cpp/test_vindex.cpp b/cpp/test_vindex.cpp index 9cc31d6..d5a6a61 100644 --- a/cpp/test_vindex.cpp +++ b/cpp/test_vindex.cpp @@ -20,10 +20,12 @@ #include "paimon_vindex.hpp" #include +#include #include #include #include #include +#include #include #define ASSERT_EQ(a, b) do { \ @@ -127,7 +129,7 @@ static void run_roundtrip( const std::vector>& options, uint32_t expected_index_type, size_t expected_pq_m, - size_t expected_hnsw_m) { + size_t expected_pq_bits) { std::vector data = roundtrip_data(); std::vector ids = roundtrip_ids(); paimon::vindex::Trainer trainer(options); @@ -143,48 +145,92 @@ static void run_roundtrip( writer.write_index(make_output(buf)); ASSERT_TRUE(!buf.data.empty()); - paimon::vindex::Reader reader(make_input(buf)); + paimon::vindex::Reader* active_reader = nullptr; + bool reentrant_attempted = false; + bool reentrant_rejected = false; + auto input = make_input(buf); + if (expected_index_type == PAIMON_VINDEX_INDEX_TYPE_IVF_FLAT) { + auto base_read = input.read_ranges_fn; + input.read_ranges_fn = + [&, base_read](paimon::vindex::ReadRequest* requests, size_t request_count) { + if (active_reader != nullptr && !reentrant_attempted) { + reentrant_attempted = true; + try { + active_reader->metadata(); + } catch (const paimon::vindex::Error& error) { + reentrant_rejected = + std::string(error.what()).find("reentrant native-handle operation") != + std::string::npos; + } + } + return base_read(requests, request_count); + }; + } + paimon::vindex::Reader reader( + std::move(input), + static_cast(4ULL * 1024 * 1024 * 1024)); + active_reader = &reader; auto metadata = reader.metadata(); ASSERT_EQ(metadata.index_type, expected_index_type); ASSERT_EQ(metadata.dimension, kRoundtripDimension); - ASSERT_EQ(metadata.nlist, 4); + ASSERT_EQ( + metadata.nlist, + expected_index_type == PAIMON_VINDEX_INDEX_TYPE_DISKANN ? 1 : 4); ASSERT_EQ(metadata.metric, PAIMON_VINDEX_METRIC_L2); ASSERT_EQ(metadata.total_vectors, kRoundtripVectorCount); ASSERT_EQ(metadata.pq_m, expected_pq_m); - ASSERT_EQ(metadata.hnsw_m, expected_hnsw_m); + ASSERT_EQ(metadata.pq_bits, expected_pq_bits); + ASSERT_EQ( + metadata.rq_bits, + expected_index_type == PAIMON_VINDEX_INDEX_TYPE_IVF_RQ ? 5 : 0); + if (expected_index_type == PAIMON_VINDEX_INDEX_TYPE_DISKANN) { + ASSERT_EQ(metadata.diskann_max_degree, 8); + ASSERT_EQ(metadata.diskann_build_search_list_size, 16); + ASSERT_TRUE(std::fabs(metadata.diskann_alpha - 1.2f) < 1e-6f); + auto read_plan = reader.read_plan(); + ASSERT_EQ(read_plan.memory_budget_bytes, 4ULL * 1024 * 1024 * 1024); + ASSERT_TRUE(read_plan.window_bytes > 0); + } reader.optimize_for_search(); + if (expected_index_type == PAIMON_VINDEX_INDEX_TYPE_IVF_FLAT) { + ASSERT_TRUE(reentrant_attempted); + ASSERT_TRUE(reentrant_rejected); + } auto query = query_for_center(0.0f); - auto result = reader.search(query.data(), paimon::vindex::SearchParams{2, 4, 16}); + if (expected_index_type == PAIMON_VINDEX_INDEX_TYPE_DISKANN) { + auto calibrated_width = reader.calibrate_search_width(query.data(), 1, 2); + ASSERT_TRUE( + calibrated_width == 100 || + calibrated_width == 200 || + calibrated_width == 400); + } + auto search_params = expected_index_type == PAIMON_VINDEX_INDEX_TYPE_DISKANN + ? paimon::vindex::SearchParams::automatic(2) + : paimon::vindex::SearchParams{2, 4}; + if (expected_index_type == PAIMON_VINDEX_INDEX_TYPE_DISKANN) { + reader.warmup_queries(query.data(), 1, 32); + } + auto result = reader.search(query.data(), search_params); ASSERT_EQ(result.ids.size(), 2); assert_id_in_cluster(result.ids[0], 0); ASSERT_TRUE(std::isfinite(result.distances[0])); if (expected_index_type == PAIMON_VINDEX_INDEX_TYPE_IVF_PQ) { ASSERT_TRUE(buf.max_read_request_count > 1); } - if (expected_index_type == PAIMON_VINDEX_INDEX_TYPE_IVF_RQ) { - auto query_bits_result = - reader.search(query.data(), paimon::vindex::SearchParams{2, 4, 16, 4}); - assert_id_in_cluster(query_bits_result.ids[0], 0); - ASSERT_TRUE(std::isfinite(query_bits_result.distances[0])); - } - auto query0 = query_for_center(0.0f); auto query1 = query_for_center(20.0f); std::vector queries; queries.insert(queries.end(), query0.begin(), query0.end()); queries.insert(queries.end(), query1.begin(), query1.end()); - auto batch = reader.search_batch(queries.data(), 2, paimon::vindex::SearchParams{1, 4, 16}); + auto batch_params = expected_index_type == PAIMON_VINDEX_INDEX_TYPE_DISKANN + ? paimon::vindex::SearchParams::diskann(1, 100) + : paimon::vindex::SearchParams{1, 4}; + auto batch = reader.search_batch(queries.data(), 2, batch_params); ASSERT_EQ(batch.ids.size(), 2); assert_id_in_cluster(batch.ids[0], 0); assert_id_in_cluster(batch.ids[1], 1); - if (expected_index_type == PAIMON_VINDEX_INDEX_TYPE_IVF_RQ) { - auto query_bits_batch = - reader.search_batch(queries.data(), 2, paimon::vindex::SearchParams{1, 4, 16, 8}); - assert_id_in_cluster(query_bits_batch.ids[0], 0); - assert_id_in_cluster(query_bits_batch.ids[1], 1); - } printf("PASS %s\n", name); } @@ -208,11 +254,10 @@ static void test_supported_index_roundtrips() { {"dimension", "8"}, {"nlist", "4"}, {"metric", "l2"}, - {"pq.m", "4"}, }, PAIMON_VINDEX_INDEX_TYPE_IVF_PQ, - 4, - 0); + 2, + 8); run_roundtrip( "ivf_rq_roundtrip", @@ -220,6 +265,7 @@ static void test_supported_index_roundtrips() { {"index.type", "ivf_rq"}, {"dimension", "8"}, {"nlist", "4"}, + {"rq.bits", "5"}, {"metric", "l2"}, }, PAIMON_VINDEX_INDEX_TYPE_IVF_RQ, @@ -227,33 +273,58 @@ static void test_supported_index_roundtrips() { 0); run_roundtrip( - "ivf_hnsw_flat_roundtrip", + "ivf_sq_roundtrip", { - {"index.type", "ivf_hnsw_flat"}, + {"index.type", "ivf_sq"}, {"dimension", "8"}, {"nlist", "4"}, {"metric", "l2"}, - {"hnsw.m", "4"}, }, - PAIMON_VINDEX_INDEX_TYPE_IVF_HNSW_FLAT, + PAIMON_VINDEX_INDEX_TYPE_IVF_SQ, 0, - 4); + 8); run_roundtrip( - "ivf_hnsw_sq_roundtrip", + "diskann_roundtrip", { - {"index.type", "ivf_hnsw_sq"}, + {"index.type", "diskann"}, {"dimension", "8"}, - {"nlist", "4"}, {"metric", "l2"}, - {"hnsw.m", "4"}, + {"pq.m", "4"}, + {"pq.bits", "4"}, + {"diskann.max-degree", "8"}, + {"diskann.build-search-list-size", "16"}, }, - PAIMON_VINDEX_INDEX_TYPE_IVF_HNSW_SQ, - 0, + PAIMON_VINDEX_INDEX_TYPE_DISKANN, + 4, 4); } +static void test_worker_callback_reentry_is_rejected() { + int callback_context = 0; + paimon::vindex::detail::NativeHandleMutex mutex; + mutex.set_callback_context(&callback_context); + std::atomic rejected(false); + + std::lock_guard operation(mutex); + std::thread callback_worker([&]() { + paimon::vindex::detail::NativeCallbackScope callback_scope(&callback_context); + try { + std::lock_guard reentrant(mutex); + } catch (const paimon::vindex::Error& error) { + rejected.store( + std::string(error.what()).find("reentrant native-handle operation") != + std::string::npos, + std::memory_order_relaxed); + } + }); + callback_worker.join(); + ASSERT_TRUE(rejected.load(std::memory_order_relaxed)); + printf("PASS worker_callback_reentry_is_rejected\n"); +} + int main() { test_supported_index_roundtrips(); + test_worker_callback_reentry_is_rejected(); return 0; } diff --git a/docs/api.html b/docs/api.html index 1e76835..576bbf7 100644 --- a/docs/api.html +++ b/docs/api.html @@ -18,7 +18,7 @@

VIPaimon Vector Index - +
@@ -31,7 +31,7 @@

API and language bindings

- +

Public integration layers

@@ -48,28 +48,43 @@

Public integration layers

Shared lifecycle

01Create a Trainer
Parse and validate options
02Submit one or more
training batches
03Finish training and
create a one-shot Writer
04Add row IDs / vectors
and write the file
05Detect file magic
and execute searches
-
  • Vectors are contiguous f32 values; length must equal vector_count × dimension.
  • Training data may arrive in batches, but the Trainer accumulates the full set. Batching avoids oversized language-level arrays; it does not reduce native peak memory.
  • A Writer may receive production vectors in multiple batches. Row-ID count must equal vector count.
  • Readers expose metadata, single-query search, batch search, and Roaring64-filtered variants.
  • Files carry their type and model sections. Callers do not pass index options again when opening a Reader.
+
  • Vectors are contiguous f32 values; length must equal vector_count × dimension.
  • Training data may arrive in batches. Every IVF trainer keeps a deterministic reservoir of at most max(65,536, 64 × resolved nlist) vectors. DiskANN starts from a 50,000-row cap and lowers it when necessary so the retained sample, optional cosine-normalized copy, codebook, and parallel PQ-training scratch fit diskann.memory-budget-bytes. Sampling is independent of batch boundaries.
  • The Python and Java one-shot train helpers infer dimension from the matrix and use its row count for automatic nlist. When the matrix is only a sample, pass the final corpus size as expected-vector-count. Streaming Trainer APIs require a concrete dimension before their first batch.
  • A Writer may receive production vectors in multiple batches. Row-ID count must equal vector count.
  • Readers expose metadata, single-query search, batch search, and Roaring64-filtered variants.
  • Files carry their type and resolved model sections. Callers do not pass index options again when opening a Reader.

Shared search parameters

- - - + + +
ParameterApplies toDescription
top_kAll indexesNumber of nearest neighbors returned for each query.
nprobeAll indexesNumber of IVF lists probed. Larger values usually improve recall while increasing I/O and compute.
ef_searchIVF-HNSW-FLAT / SQPer-list graph-search breadth. 0 resolves to max(top_k, 32).
query_bitsIVF-RQ only0 keeps the float-query byte-LUT path; 4 / 8 use quantized-query bit-plane scans.
Auto widthAll indexesIVF starts at max(8, ceil(nlist/16)), adds enough average-list capacity for at least 4 × top_k candidates, scales for filter selectivity, and progressively doubles when a filtered result is short. DiskANN uses a calibrated width when present, otherwise max(100, 2 × top_k).
nprobeIVF familiesExplicit expert override for the number of lists probed. A tagged IVF width is rejected by DiskANN rather than silently ignored.
l_searchDiskANNExplicit expert override for graph search-list size; it is clamped to at least top_k. A tagged DiskANN width is rejected by IVF indexes.
-
Stored RQ bits and query bits are differentIVF-RQ currently writes 1-bit vector codes. query_bits is a per-request query quantization setting. Current Readers reject reserved multi-bit file payloads.
+

Use the binding's automatic search-parameter constructor. For DiskANN, calibrate_search_width / calibrateSearchWidth evaluates widths 100, 200, and 400 on representative queries and remembers the smallest adjacent pair with at least 98% Top-K result overlap. This is a stability proxy, not a ground-truth recall guarantee; explicit widths always win.

+
IVF-RQ width is a build optionrq.bits accepts 1–8 and defaults to 4. It is persisted in the file and reported as rq_bits / rqBits metadata; search has no separate bit-width switch.
+
+ +
+

Reader options for DiskANN

+

Reader options are accepted by every binding and affect DiskANN only. Other index families continue to use their existing positional-I/O behavior.

+
+ + +
ConceptRust / PythonC / C++ / JavaDefault
Random-read latency hintInput capability estimated_random_read_latency_nanosC/C++ field / Java estimatedRandomReadLatencyNanos()0: reuse the mandatory header read's elapsed time; positive values bypass measurement
Total Reader memorymemory_budget_bytesmemory_budget_bytes / constructor argument4 GiB; automatically partitioned among required resident state, a profile-sized adjacency prefix, a bounded cold-adjacency LRU, and a bounded raw-vector LRU
+

The Reader has no public storage-tier switch. DiskANN selects an internal tier once during open from the latency hint or mandatory header-read timing; its latency, window, and beam policy then remain stable. read_plan / readPlan exposes that policy together with the current effective preload and shared-cache capacities. Those capacities are zero before resident initialization and may shrink when lazy row-ID lookup state consumes the same total budget. Cache partitioning remains an internal decision. The input callback receives all positional ranges for one round together and should execute them concurrently.

+

A storage adapter may additionally advertise preferred_window_bytes and a maximum range count named max_ranges_per_pread in Rust or max_ranges_per_read in C/C++/Python. Zero means unspecified. DiskANN rounds the requested window to complete 4 KiB logical pages, bounds it to 1 MiB, limits each pread batch to the advertised range count, and keeps physical alignment concerns inside the storage adapter. Build-time deployment-profile remains separate because it can select the persisted compact or interleaved layout. See DiskANN reader tuning.

Search warm-up

-

After opening a Reader and before repeated searches, call the language-specific warm-up method. It builds process-local caches without changing the file or results.

-
- - +

After opening a Reader and before repeated searches, initialize resident state and, for DiskANN, optionally replay a small representative query set. Warm-up builds process-local caches without changing the file or search results.

+
LanguageMethodCurrent behavior
Rustoptimize_for_searchIVF-PQ builds residual-L2 precomputed tables. IVF-HNSW-SQ builds SQ decode LUTs for filtered scans and fallback. Other indexes primarily preload metadata.
Cpaimon_vindex_reader_optimize_for_search
C++optimize_for_search
JavaoptimizeForSearch
Pythonoptimize_for_search
+ + + + +
LanguageResident initializationRepresentative-query warm-upDiskANN width calibration
Rustoptimize_for_searchwarmup_queriescalibrate_search_width
Cpaimon_vindex_reader_optimize_for_searchpaimon_vindex_reader_warmup_queriespaimon_vindex_reader_calibrate_search_width
C++optimize_for_searchwarmup_queriescalibrate_search_width
JavaoptimizeForSearchwarmupQueriescalibrateSearchWidth
Pythonoptimize_for_searchwarmup_queriescalibrate_search_width
-

The IVF-HNSW-SQ LUT is not used by normal unfiltered graph traversal, so warm-up is not expected to accelerate that path.

+

optimize_for_search is optional for correctness: the first search performs the same initialization lazily. It builds IVF-PQ residual-L2 tables or DiskANN resident PQ/row-ID state and its automatically budgeted hot adjacency prefix. DiskANN warmup_queries then executes top-1 graph traversal and persisted-vector rerank for each supplied query, priming immutable adjacency and raw-vector LRUs. Other index families treat representative warm-up as resident initialization only.

@@ -77,18 +92,16 @@

Rust

Rust · train, write, and search
use std::fs::File;
 
 use paimon_vindex_core::distance::MetricType;
-use paimon_vindex_core::hnsw::HnswBuildParams;
 use paimon_vindex_core::index::{
     VectorIndexConfig, VectorIndexReader, VectorIndexTrainer,
     VectorIndexWriter, VectorSearchParams,
 };
 use paimon_vindex_core::io::PosWriter;
 
-let config = VectorIndexConfig::IvfHnswSq {
+let config = VectorIndexConfig::IvfSq {
     dimension: 128,
     nlist: 1024,
     metric: MetricType::L2,
-    hnsw: HnswBuildParams::default(),
 };
 
 let training = VectorIndexTrainer::train(
@@ -103,22 +116,21 @@ 
Rust

 let file = File::open("vectors.pvindex")?;
 let mut reader = VectorIndexReader::open(file)?;
 reader.optimize_for_search()?;
-let params = VectorSearchParams::with_ef_search(10, 16, 80);
+let params = VectorSearchParams::automatic(10);
 let (ids, distances) = reader.search(&query, params)?;
Rust · other configurations
VectorIndexConfig::IvfFlat {
     dimension: 128, nlist: 1024, metric: MetricType::L2,
 };
-VectorIndexConfig::IvfPq {
-    dimension: 128, nlist: 1024, m: 16,
-    metric: MetricType::L2, use_opq: false,
-};
+VectorIndexConfig::ivf_pq(
+    128, 1024, MetricType::L2, false,
+)?;
 VectorIndexConfig::IvfRq {
-    dimension: 128, nlist: 1024, metric: MetricType::L2,
+    dimension: 128, nlist: 1024, bits: 4, metric: MetricType::L2,
 };
-VectorIndexConfig::IvfHnswFlat {
+VectorIndexConfig::IvfSq {
     dimension: 128, nlist: 1024, metric: MetricType::L2,
-    hnsw: HnswBuildParams::default(),
 };
+

The IVF-PQ constructor uses the default relative PQ-code budget and resolves a concrete m. In every option-map API, pq.m is optional: pq.code-ratio=0.0625 is the default, and an explicit pq.m takes precedence. Metadata and the on-disk header expose the resolved value.

@@ -151,11 +163,14 @@

C FFI

int64_t ids[10]; float distances[10]; PaimonVindexSearchParams params = { - .top_k = 10, .nprobe = 16, .ef_search = 80, .query_bits = 0, + .top_k = 10, + .search_width = PAIMON_VINDEX_SEARCH_WIDTH_AUTO, + .width = 0, }; +paimon_vindex_reader_warmup_queries(reader, representative_queries, 8, 0); paimon_vindex_reader_search(reader, query, params, ids, distances, 10); paimon_vindex_reader_free(reader);
-

PaimonVindexOutputFile and PaimonVindexInputFile are callback structures. The input callback receives every positional range in one I/O batch, allowing object-store implementations to issue reads concurrently. C callers own result buffers and pass their capacity explicitly.

+

PaimonVindexOutputFile and PaimonVindexInputFile are callback structures. The input callback receives every positional range in one I/O batch, allowing object-store implementations to issue reads concurrently. DiskANN batch search may also invoke the callback concurrently from separate query workers, so the callback and its context must be thread-safe. The input structure also carries the three optional read-capability hints described above. C callers own result buffers and pass their capacity explicitly. Metadata reports pq_m/pq_bits for PQ-backed indexes and rq_bits for IVF-RQ. IVF-SQ reports pq_m=0 and pq_bits=8 to expose its scalar-code width.

@@ -178,22 +193,17 @@

C++ RAII

paimon::vindex::Reader reader(input_file); auto metadata = reader.metadata(); reader.optimize_for_search(); +reader.warmup_queries(representative_queries.data(), 8); auto result = reader.search( - query.data(), paimon::vindex::SearchParams{10, 16, 80}); + query.data(), paimon::vindex::SearchParams::automatic(10));

Java / JNI

The package is org.apache.paimon.index.vector. String options map directly to Paimon table and index properties; Rust parses and validates them when a Trainer is created.

-
Java · build and search
int dimension = 128;
-Map<String, String> options = new HashMap<>();
-options.put("index.type", "ivf_hnsw_sq");
-options.put("dimension", Integer.toString(dimension));
-options.put("nlist", "1024");
+          
Java · build and search
Map<String, String> options = new HashMap<>();
+options.put("index.type", "ivf_sq");
 options.put("metric", "l2");
-options.put("hnsw.m", "20");
-options.put("hnsw.ef-construction", "150");
-options.put("hnsw.max-level", "7");
 
 try (VectorIndexTraining training =
              VectorIndexTrainer.train(options, trainingVectors, trainingCount);
@@ -206,7 +216,7 @@ 
Java / JNI

     VectorIndexMetadata metadata = reader.metadata();
     reader.optimizeForSearch();
     VectorSearchResult result = reader.search(
-            query, new VectorSearchParams(10, 16, 80, 0));
+            query, VectorSearchParams.automatic(10));
 }

           
Batching a large training set

           
Java · Trainer-owned native staging
try (VectorIndexTrainer trainer = VectorIndexTrainer.create(options)) {
@@ -219,12 +229,12 @@ 
Batching a large training set

         writer.writeIndex(vectorIndexOutput);
     }
 }

-          
Staging avoids one very large Java float[] and its array-length limit. Batches remain accumulated in native memory until finishTraining(), so native peak training memory is unchanged.

+          
Staging avoids one very large Java float[] and its array-length limit. Native reservoir sampling bounds retained training rows, but streaming creation still needs concrete dimension and either concrete nlist or expected-vector-count.

Python

-

The Python package loads libpaimon_vindex_ffi with ctypes. A single search returns one-dimensional NumPy arrays. search_batch accepts a two-dimensional query array and returns arrays shaped (query_count, top_k).

+

The C++, Java, and Python Readers serialize operations on one native handle across threads. Calling another method on that same handle from its input/output callback is rejected as reentrant—even when DiskANN invokes the callback from a native query worker—instead of deadlocking or re-entering a mutable native handle; use a separate handle if a callback needs vector-index work. The Python package loads libpaimon_vindex_ffi with ctypes. A single search returns one-dimensional NumPy arrays. search_batch accepts a two-dimensional query array and returns arrays shaped (query_count, top_k).

Python
from paimon_vindex import (
     SearchParams, VectorIndexReader,
     VectorIndexTrainer, VectorIndexWriter,
@@ -233,15 +243,14 @@ 
Python

 class VectorIndexInput:
     def __init__(self, data: bytes):
         self.data = data
+        self.estimated_random_read_latency_nanos = 20_000_000
+        self.preferred_window_bytes = 64 * 1024
+        self.max_ranges_per_read = 32
 
     def pread_many(self, ranges):
         return [self.data[pos : pos + length] for pos, length in ranges]
 
-options = {
-    "index.type": "ivf_hnsw_sq", "dimension": "128",
-    "nlist": "1024", "metric": "l2", "hnsw.m": "20",
-    "hnsw.ef-construction": "150", "hnsw.max-level": "7",
-}
+options = {"index.type": "ivf_sq", "metric": "l2"}
 training = VectorIndexTrainer.train(options, training_vectors)
 writer = VectorIndexWriter(training)
 writer.add_vectors(row_ids, vectors)
@@ -249,8 +258,9 @@ 
Python

 
 reader = VectorIndexReader(VectorIndexInput(index_bytes))
 reader.optimize_for_search()
+reader.warmup_queries(representative_queries)
 ids, distances = reader.search(
-    query, SearchParams(top_k=10, nprobe=16, ef_search=80))
+ query, SearchParams.automatic(top_k=10))
@@ -263,6 +273,7 @@

Metadata filter pushdown

Pythonsearch(..., filter_bytes=...) / search_batch(..., filter_bytes=...)
Row-ID constraintRoaringTreemap uses the u64 domain, so row IDs must be non-negative to match the filter. Filters are query payloads and are never written into index files.
+

DiskANN maps sparse filters through its persisted row-ID lookup after first use; dense filters scan resident row IDs once per batch. It normally scans PQ codes only for matching nodes, then exactly reranks a bounded candidate set. A quality-gated broad filter may use ordinary graph navigation and post-filter its candidates, with a complete matching-node PQ fallback before raw-vector reads. Benchmark filter-heavy traffic separately because the safe PQ path remains linear in the number of matching nodes.

diff --git a/docs/development.html b/docs/development.html index fffb847..8eaae98 100644 --- a/docs/development.html +++ b/docs/development.html @@ -18,7 +18,7 @@
VIPaimon Vector Index - +
@@ -26,12 +26,12 @@

Build · test · measure

Development and benchmarks

Run the standard Rust checks, exercise the C/C++, Java, and Python integrations, and measure ANN and filtered-query behavior with reproducible workloads.

Cargo workspaceCMake smoke testsMaven / pytestCriterion benchmarks
- +

Repository layout

- + @@ -52,28 +52,85 @@

Standard Rust checks

ANN benchmark

-

ann_bench compares IVF-PQ, IVF-RQ, IVF-HNSW-FLAT, and IVF-HNSW-SQ under one workload. It reports build time, Reader open/load time, first-query latency, batch throughput, and serialized index size.

+

ann_bench compares IVF-FLAT, IVF-SQ, IVF-PQ, IVF-RQ, and DiskANN under one workload. By default it generates independent clustered queries and computes exact Top-10 ground truth. It can instead read standard fvecs/ivecs files and use their published ground truth. Every CSV row starts with a dataset label and reports vector and training counts, raw-dataset bytes, total build time plus train/add/write stages, process peak RSS, warm-up time, serialized bytes, Recall@10, first/P50/P95 latency, sequential and batch throughput, positional-read rounds, range count, and bytes.

Shell · defaults
cargo bench -p paimon-vindex-core --bench ann_bench -- --nocapture
-
Shell · explicit workload
ANN_N=100000 ANN_NQ=1000 ANN_D=128 ANN_K=10 \
-ANN_NLIST=256 ANN_NPROBE=16 ANN_PQ_M=16 \
-ANN_HNSW_M=20 ANN_HNSW_EF_CONSTRUCTION=150 \
-ANN_HNSW_EF_SEARCH=80 ANN_RQ_QUERY_BITS=0 \
+          
Shell · documented 100k comparison
ANN_N=100000 ANN_NQ=64 ANN_D=64 ANN_K=10 \
+ANN_NLIST=256 ANN_NPROBE=32 ANN_PQ_CODE_RATIO=0.0625 \
+ANN_DISKANN_L_SEARCH=200 \
+ANN_RQ_BITS=4 \
 cargo bench -p paimon-vindex-core --bench ann_bench -- --nocapture

-
DirectoryContentsPrimary verification
core/Indexes, quantizers, HNSW, I/O, format fixtures, and benchmarkscargo test -p paimon-vindex-core
core/Indexes, quantizers, I/O, format fixtures, and benchmarkscargo test -p paimon-vindex-core
ffi/C ABI and shared libraryC smoke test
include/C++ RAII headerC++ smoke test
jni/Rust JNI bridgeJava Maven tests
VariableMeaning
ANN_N / ANN_NQIndexed vector count / query count
ANN_D / ANN_KDimension / top K
ANN_NLIST / ANN_NPROBEIVF list count / probed lists
ANN_PQ_MIVF-PQ subquantizer count
ANN_HNSW_*HNSW build and query parameters
ANN_RQ_QUERY_BITSIVF-RQ query quantization: 0 / 4 / 8
-

disk_scope=index_bytes means the serialized vector-index file itself, excluding the outer Paimon file and manifest.

+
Shell · 1024 physical dimensions, exactly 10 GiB of raw vectors
ANN_DATA_GIB=10 ANN_D=1024 ANN_NOISE_DIMENSIONS=64 \
+ANN_TRAIN_N=65536 ANN_NQ=8 ANN_K=10 \
+ANN_NLIST=1 ANN_NPROBE=1 ANN_PQ_CODE_RATIO=0.0625 ANN_CLUSTERS=64 \
+ANN_DISKANN_L_SEARCHES=200,1000 \
+ANN_DISKANN_MEMORY_BUDGET_BYTES=34359738368 \
+ANN_INDEXES=DISKANN ANN_KEEP_INDEXES=1 \
+ANN_OUTPUT_DIR=/path/on/target/ssd \
+cargo bench -p paimon-vindex-core --bench ann_bench
+
Shell · complete five-index 10 GiB comparison, isolated processes
for index in IVF_FLAT IVF_SQ IVF_PQ IVF_RQ DISKANN; do
+  ANN_DATA_GIB=10 ANN_D=1024 ANN_NOISE_DIMENSIONS=64 \
+  ANN_TRAIN_N=65536 ANN_NQ=8 ANN_K=10 \
+  ANN_NLIST=1024 ANN_NPROBE=64 ANN_PQ_CODE_RATIO=0.0625 ANN_CLUSTERS=64 \
+    ANN_DISKANN_L_SEARCHES=200,1000 \
+  ANN_DISKANN_MEMORY_BUDGET_BYTES=34359738368 \
+  ANN_INDEXES="$index" ANN_OUTPUT_DIR=/path/on/target/ssd \
+  cargo bench -p paimon-vindex-core --bench ann_bench
+done
+

Public SIFT1M, GIST1M, and GloVe-100 data

+

ANN-Benchmarks publishes dense HDF5 files with independent test queries and exact Top-100 neighbors. Convert them once to the streaming-friendly fvecs/ivecs format. --query-limit=1000 keeps all recorded datasets at the same query count while retaining enough queries for stable Recall@10.

+
Shell · download and convert a public dataset
curl -LO https://ann-benchmarks.com/sift-128-euclidean.hdf5
+python3 -m pip install h5py numpy
+python3 tools/convert_ann_benchmarks.py \
+  sift-128-euclidean.hdf5 /data/sift1m \
+  --prefix sift1m --query-limit 1000
+
Shell · convert GloVe angular data for the common L2 benchmark
curl -LO https://ann-benchmarks.com/glove-100-angular.hdf5
+python3 tools/convert_ann_benchmarks.py \
+  glove-100-angular.hdf5 /data/glove100 \
+  --prefix glove100 --query-limit 1000 --normalize-l2
+
Shell · run all public-data indexes in isolated processes
RAYON_NUM_THREADS=12 ANN_DATASET_NAME=sift1m \
+ANN_BASE_FVECS=/data/sift1m/sift1m_base.fvecs \
+ANN_QUERY_FVECS=/data/sift1m/sift1m_query.fvecs \
+ANN_GROUND_TRUTH_IVECS=/data/sift1m/sift1m_ground_truth.ivecs \
+ANN_K=10 ANN_NLIST=1024 ANN_NPROBE=64 ANN_PQ_CODE_RATIO=0.0625 \
+ANN_DISKANN_L_SEARCH=100 \
+ANN_DISKANN_RAW_VECTOR_ENCODING=f16 \
+ANN_DISKANN_MEMORY_BUDGET_BYTES=8589934592 \
+ANN_STORAGE_CASES=local_ssd_warm_cache,remote_cache_2ms,object_store_20ms \
+ANN_OUTPUT_DIR=/path/on/target/ssd \
+cargo bench -p paimon-vindex-core --bench ann_bench
+

The benchmark infers ANN_N, ANN_NQ, and ANN_D from the public files, chooses ANN_TRAIN_N=min(N, max(65,536, 64 × nlist)), and automatically starts one child process per selected index so each peak-RSS value is independent. Explicit shape variables remain available as assertions. For GIST1M, download and convert gist-960-euclidean.hdf5; for GloVe-100, use the command above. Unit-normalized L2 has the same neighbor ordering as cosine for non-zero vectors; that equivalence was used by the recorded GloVe run. DiskANN now also accepts native cosine and inner-product metrics, which should be benchmarked against ground truth generated with the same production metric. Keep the other recorded settings unchanged. The default ANN_PQ_CODE_RATIO=0.0625 resolves automatically to m=32, m=240, and m=25 for SIFT, GIST, and GloVe respectively. Set ANN_PQ_M only to benchmark an explicit expert override.

+

At 1024 dimensions, ANN_DATA_GIB=10 derives exactly 2,621,440 vectors. Run large comparisons one index per process by changing ANN_INDEXES; this isolates process peak RSS and, unless ANN_KEEP_INDEXES=1, deletes that index after all selected serving cases. A comma-separated subset or all is convenient for smaller workloads. The benchmark retains only the selected training prefix, then adds the entire data set. The clustered generator is round-robin by cluster, so a prefix still covers every cluster when ANN_TRAIN_N ≥ ANN_CLUSTERS.

+
Physical versus intrinsic dimensionANN_NOISE_DIMENSIONS=64 spreads independent per-vector noise evenly over 64 of the 1024 stored dimensions; the other dimensions retain cluster-center signal. This creates a repeatable storage/compute scale workload without pretending that 1024 independent uniform-noise dimensions resemble production embeddings. With all 1024 dimensions independently noisy, same-cluster distances concentrate so strongly that exact Top-10 membership is nearly arbitrary: a 65k diagnostic produced local Recall@10 of 0.0125 at l_search=200 and only 0.178 at l_search=5000. Use a public or production corpus for an algorithm-quality claim.
+
VariableMeaning
ANN_DATASET_NAMEComma-free label written to every CSV row
ANN_BASE_FVECS / ANN_QUERY_FVECS / ANN_GROUND_TRUTH_IVECSOptional public-data files; set all three together. Their vector count, query count, dimension, and ground-truth width are inspected before the full payload is loaded.
ANN_N / ANN_NQ / ANN_DInferred for public files; an explicit value is validated as a shape assertion. Generated workloads retain defaults of 20,000 / 64 / 64.
ANN_DATA_GIBGenerated-workload raw float-vector GiB; derives n = floor(bytes / (4d)) and cannot be combined with public file paths.
ANN_TRAIN_NOptional training-sample override; omitted values resolve to min(N, max(65,536, 64 × nlist)).
ANN_KTop K
ANN_INDEXESall or a comma-separated subset of the five index names. Public multi-index runs automatically isolate each index in a child process.
ANN_STORAGE_CASESall or a comma-separated subset of local_ssd_warm_cache, remote_cache_2ms, and object_store_20ms. The default runs all three.
ANN_NLIST / ANN_NPROBEIVF list count / probed lists
ANN_PQ_CODE_RATIOIVF-PQ and DiskANN PQ-code/raw-vector byte ratio; default 0.0625 and automatically resolves a valid m
ANN_PQ_MOptional explicit subquantizer-count override; takes precedence over automatic sizing
ANN_RQ_BITSPersisted IVF-RQ bit width, 1–8; default 4
ANN_DISKANN_L_SEARCH / ANN_DISKANN_L_SEARCHESOne DiskANN search-list size, or a comma-separated sweep over one built index
ANN_DISKANN_BUILD_DISTANCEproduct_quantized by default; use full_precision as a graph-quality control
ANN_DISKANN_RAW_VECTOR_ENCODINGf16 by default; use f32 as an exact-distance and numeric-range control. The resolved value is written to every CSV row.
ANN_DISKANN_MEMORY_BUDGET_BYTESDiskANN's internal preflight build budget; default 8 GiB and excludes the benchmark's source-vector allocation
ANN_NOISE_DIMENSIONS / ANN_CLUSTERS / ANN_SEEDGenerated-workload distribution and reproducibility controls; ignored by public file loading.
ANN_OUTPUT_DIRDirectory whose filesystem supplies the local-storage path
ANN_KEEP_INDEXES1 retains generated index files; the default deletes each index after all selected serving cases
ANN_REUSE_INDEX_PATHSkips training/build and queries one retained index; requires a single ANN_INDEXES value
+
Large-run memory boundaryThe current public writer API accepts an in-memory vector slice and each index writer retains its encoded or raw build state. A raw 10 GiB DiskANN run therefore needs roughly the 10 GiB source allocation plus DiskANN's separately budgeted build state. On a 48 GiB machine, use a 32 GiB DiskANN budget and run one index per process. This benchmark validates large immutable builds; it is not a streaming-ingest benchmark.
+

The benchmark executes three serving cases by default. local_ssd_warm_cache uses the real index file with no artificial delay. remote_cache_2ms adds a fixed 2 ms delay per pread call while reading all ranges in that call concurrently. object_store_20ms uses the ObjectStore read plan: open/optimization and sequential-query latency are the measured CPU/I/O time plus 20 ms for every observed dependent read round, avoiding hours of idle wall-clock sleep while retaining the exact fixed-latency model; batch QPS still injects the real 20 ms sleep so concurrent-client overlap is measured rather than inferred. These models do not include bandwidth or operating-system cache misses. DiskANN uses a dedicated range-I/O pool separate from Rayon query workers, preventing a saturated batch from starving nested positional reads; IVF retains the shared compute pool because its batch read scheduling is not nested inside per-query sessions. The unified IVF Reader reuses the 64-byte type-dispatch header and loads resident metadata in one further contiguous read, reducing initialization from three rounds to two without changing any format. DiskANN is opened explicitly as LocalStorage, RemoteStorage, or ObjectStore with the default 4 GiB Reader budget, which is automatically partitioned into resident state and caches. The sequential sweep and batch call use separate opened-and-optimized Readers, preventing all benchmark queries from prewarming their own batch measurement. CSV output records optimization rounds, ranges, and bytes separately from query I/O. Use ANN_STORAGE_CASES for a focused rerun against a retained index. See the recorded public-corpus results and interpretation.

-
-

IVF-HNSW-SQ filter benchmark

-

The filter-heavy benchmark measures batch filtered search before and after warm-up and verifies that both paths return identical results. It isolates the SQ decode LUT used by scan and fallback paths.

-
Shell · defaults
cargo bench -p paimon-vindex-core \
-  --bench ivfhnswsq_filter_bench -- --nocapture
-
Shell · explicit workload
FILTER_BENCH_N=50000 FILTER_BENCH_NQ=500 FILTER_BENCH_D=128 \
-FILTER_BENCH_NLIST=64 FILTER_BENCH_NPROBE=32 \
-FILTER_BENCH_EF_SEARCH=80 FILTER_BENCH_FILTER_STRIDES=1,4,16,64 \
-cargo bench -p paimon-vindex-core \
-  --bench ivfhnswsq_filter_bench -- --nocapture
-

FILTER_BENCH_FILTER_STRIDES simulates different selectivities through row-ID strides. Analyze direct scans, graph search, and insufficient-result backfill separately.

+
+

DiskANN access-pattern benchmark

+

diskann_bench measures the page-oriented search path separately from the IVF benchmark. With no dataset paths it generates a small deterministic correctness workload. SIFT1M mode reads standard .fvecs and .ivecs files.

+
Shell · generated smoke workload
RAYON_NUM_THREADS=4 \
+DISKANN_BENCH_ACCEPTANCE=1 DISKANN_BENCH_SIMULATED_RTT_MS=1 \
+DISKANN_BENCH_CONCURRENCY=4 \
+DISKANN_BENCH_STORAGE_LAYOUT=compact \
+DISKANN_BENCH_RAW_VECTOR_ENCODING=f32 \
+DISKANN_BENCH_BUILD_DISTANCE=product_quantized \
+cargo bench -p paimon-vindex-core --bench diskann_bench
+
Shell · SIFT1M acceptance
DISKANN_BASE_FVECS=/data/sift_base.fvecs \
+DISKANN_QUERY_FVECS=/data/sift_query.fvecs \
+DISKANN_GROUND_TRUTH_IVECS=/data/sift_groundtruth.ivecs \
+DISKANN_BENCH_INDEX_DIR=/mnt/local-nvme \
+RAYON_NUM_THREADS=16 \
+DISKANN_BENCH_CONCURRENCY=16 \
+DISKANN_BENCH_READER_MEMORY_BUDGET_BYTES=4294967296 \
+DISKANN_BENCH_ACCEPTANCE=1 \
+DISKANN_BENCH_FILTERED_MATRIX=1 \
+DISKANN_BENCH_SIMULATED_RTT_MS=5 \
+cargo bench -p paimon-vindex-core --bench diskann_bench
+

The benchmark writes a temporary .dann file under DISKANN_BENCH_INDEX_DIR (or the system temporary directory), reopens it through a benchmark-private SeekRead adapter, and deletes it on exit. This adapter exercises native positional I/O without adding a local-file type to the public core API. Use DISKANN_BENCH_STORAGE_LAYOUT=compact|interleaved, DISKANN_BENCH_RAW_VECTOR_ENCODING=f32|f16, and DISKANN_BENCH_BUILD_DISTANCE=full_precision|product_quantized for controlled comparisons. PQ defaults to DISKANN_BENCH_PQ_CODE_RATIO=0.0625; DISKANN_BENCH_PQ_M is an optional explicit override in 1..=dimension and DISKANN_BENCH_PQ_BITS selects 4 or 8 bits. Generated workloads can be scaled with DISKANN_BENCH_SMOKE_N, DISKANN_BENCH_SMOKE_NQ, and DISKANN_BENCH_SMOKE_DIMENSION; exact top-10 truth is generated automatically. CSV rows inject representative latency hints for all four internal read tiers at l_search 50, 100, and 200 so timing noise does not make planner comparisons ambiguous. DISKANN_BENCH_SIMULATED_RTT_MS adds the same fixed delay to every pread round for every tier. Set DISKANN_BENCH_WARMUP_QUERIES=0 to measure resident-only initialization; its default 8 replays representative queries before latency sampling. DISKANN_BENCH_READER_MEMORY_BUDGET_BYTES controls the one public Reader budget; the benchmark records automatically resolved cache behavior through search statistics. Rows additionally identify layout, raw-vector encoding, build distance, selected graph-shard count, and complete parallel-session query counts.

+

Set DISKANN_BENCH_FILTERED_MATRIX=1 to add random and contiguous/cluster-correlated filters at 0.01%, 0.1%, 1%, 10%, 50%, and 100% selectivity. The matrix compares adaptive results with a forced exhaustive matching-node baseline. It reports the actual executed strategy and graph fallback count from diskann_search_stats, Recall@10, elapsed time, I/O, PQ distance evaluations versus PQ code loads, query chunks, rerank chunks, candidate references, unique raw windows, batch LRU hits/misses/evictions, and parallel exact-rerank work. Under DISKANN_BENCH_ACCEPTANCE=1, every graph-enabled cell must stay within one absolute Recall@10 percentage point of the exhaustive baseline.

+

The benchmark warms one Reader. DISKANN_BENCH_CONCURRENCY=1 repeatedly invokes the single-query API; larger values invoke the native batch path at that width. Retained sessions run complete queries concurrently for small batches and all interleaved-layout batches. Large compact batches retain shared vector-window rerank. Graph sessions clone only the storage handle and share resident PQ/row-ID/code allocations, validation state, hot prefix, and cold-adjacency LRU. Local results use the operating-system page cache after index creation; set DISKANN_BENCH_INDEX_DIR to the production SSD mount and run cold-cache testing with an external, platform-appropriate cache-control procedure.

+
Acceptance scopeWhen enabled, the gate requires unfiltered Recall@10 ≥ 0.90, filtered graph Recall@10 within one point of exhaustive matching-node scan, build time below two hours, peak RSS below 4 GiB, and at most seven graph rounds plus one exact-rerank round per ObjectStore query. The generated workload validates mechanics only. SIFT1M or a representative production dataset remains a required non-CI release check.
@@ -121,13 +178,13 @@

Python ctypes tests

cd python PAIMON_VINDEX_LIB_PATH=../target/release pip install -e ".[test]" PAIMON_VINDEX_LIB_PATH=../target/release pytest -v
-

The suite uses NumPy inputs and covers all index types, RQ query bits, single/batch shapes, and error paths.

+

The suite uses NumPy inputs and covers all index types, persisted RQ bit metadata, single/batch shapes, and error paths.

Storage format and compatibility

-

Changes to headers, flags, sections, sort order, or graph encoding must update both format fixtures and documentation. The normative v1 field definitions live in STORAGE_FORMAT.md.

-

Reader compatibility

Unknown versions, required flags, non-zero reserved bytes, negative counts, and out-of-bounds sections must fail rather than be guessed.

Stable fixtures

core/tests/fixtures stores a hexadecimal fixture for each v1 index family to detect accidental layout drift.

Outer integrity

Vector-index files do not embed a checksum. Length and checksum validation belong to the outer Paimon file and manifest layer.

+

Changes to headers, flags, sections, sort order, or graph encoding must update both format fixtures and documentation. The normative v1 field definitions live in core/STORAGE_FORMAT.md.

+

Reader compatibility

Unknown versions, required flags, non-zero reserved bytes, negative counts, and out-of-bounds sections must fail rather than be guessed.

Stable fixtures

core/tests/fixtures stores byte-exact hexadecimal fixtures for every v1 index family. DiskANN additionally freezes compact multi-page, 4-bit interleaved, FOR row-ID, and raw row-ID variants.

Outer integrity

Vector-index files do not embed a checksum. Length and checksum validation belong to the outer Paimon file and manifest layer.

diff --git a/docs/diskann.html b/docs/diskann.html new file mode 100644 index 0000000..1f8067e --- /dev/null +++ b/docs/diskann.html @@ -0,0 +1,272 @@ + + + + + + + + DiskANN · Paimon Vector Index + + + + + +
+ VIPaimon Vector Index + +
+
+ +
+
+

A global Vamana graph designed for paged vector access

+

DiskANN

+

Keep compact PQ navigation data in memory, read graph and raw-vector records on demand, then rerank the best candidates with persisted F32 or F16 values. Choose an interleaved local-SSD layout or a compact remote-range layout; both are queried through the same abstract storage interface.

+
Native Rust VamanaResident 4/8-bit PQExact L2 rerankMagic: DANN
+
+ +
+ +
+
+

Positioning and trade-offs

+
NavigationGlobal Vamana graph
Resident dataPQ model + IDs + codes
Paged dataGraph + raw vectors
DistanceL2 / IP / Cosine
+
+

Good fit

    +
  • The immutable raw-vector set is too large to keep in RAM but fits on local NVMe or SSD.
    • +
  • High recall is required without scanning multiple complete IVF lists.
    • +
  • Indexes are built offline and replaced atomically rather than updated node by node.
    • +
  • Object storage is the system of record and query nodes cache index files on local SSD.
    • +
  • Direct object-store queries can tolerate a small number of batched range-read rounds.
    • +
+

Poor fit

    +
  • The metric-specific recall target cannot be validated on representative queries before deployment.
    • +
  • Vectors change frequently and the serving graph must be updated incrementally.
    • +
  • The full index is already memory-resident and the lowest possible tail latency is the only goal.
    • +
  • Filtered queries dominate and routinely inspect a large resident ID/PQ population.
    • +
  • Build peak memory, build duration, or serialized raw-vector size is the primary constraint.
    • +
+
+
Disk-backed does not mean a minimum-size filev1 stores every rerank vector in addition to PQ codes and graph edges. The balanced default uses F16, halving that payload and its read bandwidth at the cost of quantized final distances; explicit F32 remains available when exact rerank distances matter. The main advantage is bounded resident data and range-granular query I/O.
+
Measured positionOn the public SIFT1M, GIST1M, and GloVe-100 ANN-Benchmarks corpora, with a PQ code budget equal to 6.25% of raw vector bytes and the balanced F16 default, l_search=100 reached Recall@10 0.9915 / 0.9336 / 0.8355 at 1.50 / 1.83 / 1.90 ms P95 from a warm local filesystem cache. Under the fixed-2-ms-per-read remote model it reached 0.9808 / 0.8482 / 0.8029 recall at 15.97 / 14.22 / 18.18 ms P95. The trade-off is build time: 74 seconds, 11 minutes 26 seconds, and 2 minutes 33 seconds, much slower than IVF. See the public-corpus comparison for the complete matrix and methodology.
+

How it compares with this repository's other indexes

+
+ + + +
ChoicePrefer it whenPrefer DiskANN when
IVF-FLATYou need a simple recall baseline, three metrics, and predictable list scans.Scanning raw vectors in enough lists exceeds the CPU or I/O budget.
IVF-PQ / IVF-RQSerialized size and sequential compact-code throughput matter more than reranking persisted vectors.You can afford rerank vectors on disk and need graph-guided candidate generation plus F32-exact or F16-quantized final distances.
IVF-SQOne-byte residual dimensions and complete selected-list scans meet the recall and bandwidth target.A global graph and page-granular adjacency/vector reads better match large local-SSD indexes.
+
+ +
+

Build and search architecture

+

Build

+
+
1

Train PQ

Split each vector into pq.m balanced contiguous chunks and train 2^pq.bits centroids per chunk. Training uses deterministic reservoir sampling capped at 50,000 vectors, so training memory does not grow with the full corpus.

+
2

Build Vamana

Initialize fixed-capacity adjacency in parallel, then navigate build candidates with symmetric PQ centroid-distance tables by default. L2 uses the triangle-inequality robust-prune rule, inner product uses an occluding rule, and cosine normalizes vectors before using the equivalent L2 graph ordering. Reverse-edge overflow pruning, centroid entry selection, and reachability repair remain metric-aware and full precision, so quantization accelerates discovery without becoming the final edge-quality check.

+
3

Reorder for locality

Breadth-first reorder compact node blocks in place, remap row IDs, PQ codes, raw vectors, and graph edges together, then sort every adjacency list once for serialization.

+
4

Write one file

Serialize a 256-byte header, persistent row-ID/adjacency indexes, and either graph pages plus a dense vector-record section or page-contained [raw vector][compressed adjacency] records.

+
+

Unfiltered search

+
    +
  1. Load the PQ codebook, adaptively packed row IDs, PQ codes, and the block-compressed adjacency locator index into resident memory.
    2. +
  2. Use the PQ distance table, four-code neighbor batches, and a reusable contiguous heap frontier to traverse the Vamana graph with search-list size L.
    3. +
  3. Read only adjacency windows needed by the selected beam.
    4. +
  4. Read raw-vector windows for at most min(L, max(4 × top_k, 64)) candidates.
    5. +
  5. Compute squared-L2 distances from the persisted F32/F16 records and return the best top_k.
    6. +
+

Every sorted adjacency list independently uses canonical delta-varint encoding only when it is smaller than fixed-width IDs; otherwise it falls back to raw little-endian u32. Locator positions use one u64 base per 16 nodes plus a u16 relative offset and u16 degree/encoding value per node, for 4 × N + 8 × ceil(N / 16) bytes. Raw fallback guarantees that compression cannot increase adjacency payload bytes or logical page count.

+
Storage choice versus upstream DiskANNThe legacy Microsoft PQFlashIndex packs fixed-width node records into 4 KiB sectors and keeps PQ navigation data resident. This implementation retains the proven resident-PQ / on-demand-rerank split, but stores variable-length compressed adjacency separately from dense F16/F32 vectors so degree slack does not consume vector-section bytes. It follows current DiskANN3 in keeping storage behind a provider abstraction: query code depends on SeekRead, never a local-file type.
+
Third pre-release format reviewThe current Microsoft Rust workspace emphasizes provider/accessor-driven asynchronous graph search, while the original SSD design contributes resident PQ navigation, sector reads, and a BFS-derived hot cache. This v1 already has the corresponding abstract SeekRead sessions, resident PQ codes, compact 4 KiB adjacency pages, breadth-first locality reorder, hot prefix, bounded LRUs, and persisted-vector rerank. Keeping dense vectors separate in the default compact layout avoids fixed-degree sector slack and remains better suited to remote coalescing. No on-disk change survived this review. Compact batch rerank now hashes candidate windows before sorting only the unique windows into deterministic I/O order; median local batch time changed from 117 / 223 / 162 ms to 111 / 215 / 159 ms on SIFT/GIST/GloVe. A broader Vec sort/dedup replacement for graph window planners was rejected after regressing local SIFT/GIST by 7–13%.
+
F16-quantized or F32-exact rerankThe balanced default evaluates decoded binary16 values. Explicit F32 returns exact distances for reranked candidates and is also selected by the high-recall preset. Recall remains approximate in both modes because PQ-guided graph traversal and the finite rerank set decide which candidates reach that stage.
+
+ +
+

SSD and object-store deployment

+
+

Preferred production path

    +
  1. Publish the immutable index to S3, OSS, HDFS, or another durable store.
    2. +
  2. Download or cache the complete file on the query node.
    3. +
  3. Open the Reader and warm the resident sections before serving traffic. The input latency selects the read plan automatically.
    4. +
  4. Replace files atomically when a new snapshot is ready.
    5. +
+

Direct remote path

    +
  1. Implement concurrent positional reads in the storage callback.
    2. +
  2. Advertise representative random-read latency when the adapter already knows it; otherwise opening measures the mandatory header read.
    3. +
  3. Give the Reader one total memory budget; it automatically sizes the adjacency prefix and query caches for the selected internal tier.
    4. +
  4. Budget P95/P99 for graph rounds plus one persisted-vector rerank round.
    5. +
+
+
+ + + + +
Internal tierRead windowUnfiltered graph beamIntent
Memory4 KiB16 nodes per roundMinimize copied bytes while reducing callback rounds for an in-memory immutable source.
LocalStorage16 KiB4 nodes per roundCoalesce nearby graph and rerank records while keeping local-SSD byte amplification bounded.
RemoteStorage32 KiB16 nodes per roundBalance fixed network latency against bandwidth and byte amplification.
ObjectStore64 KiB16 nodes per roundMinimize expensive request rounds when additional transferred bytes are acceptable.
+

All tiers use exactly the same file. Quality-gated filtered graph traversal uses the separately validated beam width 4 for every tier; the table's wider beams apply to ordinary unfiltered graph search. The Reader selects exactly once while opening: below 50 µs selects Memory, below 750 µs selects LocalStorage, below 3 ms selects RemoteStorage, and 3 ms or more selects ObjectStore. SeekReadCapabilities::estimated_random_read_latency_nanos bypasses measurement when an adapter already knows representative latency; zero reuses the mandatory header read's elapsed time and adds no probe I/O. Window and range-count capabilities can refine the selected plan; physical alignment remains encapsulated by the storage adapter. A remote callback receives all ranges for a round together and should issue them concurrently.

+
Measured local-window choiceIn a window-only A/B on the same retained SIFT index, 4 / 8 / 16 / 32 KiB produced Recall@10 0.9910 / 0.9912 / 0.9913 / 0.9913, P95 1.87 / 1.71 / 1.41 / 1.39 ms, and batch throughput 6,649 / 6,931 / 8,127 / 9,587 QPS. The 32 KiB plan read 1.25 GiB across 1,000 sequential queries versus 0.69 GiB at 16 KiB for an 18% batch gain and negligible P95 gain, so the local default stops at 16 KiB. The later F16 SIMD optimization lifts the final 16 KiB batch result further without changing that read-amplification comparison. The remote and object-store internal tiers keep their latency-oriented 32 / 64 KiB plans.
+
+ +
+

Usage

+
Properties · recommended baseline
index.type=diskann
+dimension=128
+metric=l2
+deployment-profile=local_storage
+diskann.build-preset=balanced
+
Rust · abstract storage reader
use paimon_vindex_core::index::{
+    VectorIndexReader, VectorIndexReaderOptions, VectorSearchParams,
+};
+
+let options = VectorIndexReaderOptions::new(4 * 1024 * 1024 * 1024);
+// `input` comes from the storage/cache layer and implements SeekRead.
+let mut reader = VectorIndexReader::open_with_options(input, options)?;
+reader.optimize_for_search()?;
+let plan = reader.read_plan(); // Current effective DiskANN I/O/cache diagnostics.
+reader.warmup_queries(&representative_queries, representative_query_count, 100)?;
+reader.calibrate_search_width(&representative_queries, representative_query_count, 10)?;
+let params = VectorSearchParams::automatic(10);
+let (ids, squared_l2) = reader.search(&query, params)?;
+

The core API depends only on SeekRead; the application storage adapter owns local-cache, object-store, and transport details. Its pread implementation should issue all ranges in a search round concurrently and may report immutable-source capabilities so DiskANN does not overfill a callback or choose the wrong coalescing size. Its I/O executor must remain runnable when CPU query workers are saturated: recursively scheduling range reads onto that same fully occupied worker pool can starve the reads while every query waits on single-flight cache entries. The benchmark therefore uses an independent range-I/O pool for DiskANN. The generic Read + Seek adapter is a sequential compatibility path, not the recommended production integration.

+

The parent Reader and retained search-session clones share one immutable raw-vector LRU. Query-local caches hold borrowed Arc windows, so concurrent reranks for the same range are single-flighted instead of reading duplicate bytes. Cache hits and oldest eviction update linked recency metadata in constant time, and capacity is charged by retained Vec::capacity(). Misses in one search are read together; a query whose working set is larger than its local-reference allowance bypasses local retention, while batch rerank processes bounded chunks. DiskAnnSearchStats resets per top-level search and exposes raw-vector hits, storage misses, and evictions across both cache layers.

+

F16 compact-vector rerank on AArch64 loads unaligned half-precision lanes, rejects non-finite values, converts them to F32, and accumulates L2 in the same NEON loop. This removes the temporary decode buffer and the second SIMD pass while preserving the scalar fallback and sentinel behavior on other targets.

+

Cold adjacency windows outside the automatically sized hot prefix use a separate bounded LRU shared by the Reader and its batch workers. Concurrent misses for the same window are single-flighted: one worker performs the positional read while the others wait and then borrow the immutable payload. Successful reads move their existing Vec allocation into shared ownership without copying the payload. Capacity is charged by actual retained allocation capacity and clipped to both the adjacency section and the remaining total memory budget. Budgets of at least 1 MiB use 16 independently locked capacity shards. Every graph worker also bounds query-local cold-window references to 8 MiB and releases least-recently-used windows between beam rounds.

+

DiskANN retains cloned storage handles as reusable search sessions. Small batches (up to four queries per worker) run graph traversal and persisted-vector rerank concurrently end to end, avoiding a serial parent-rerank bottleneck; the interleaved layout always uses this path because the graph read already contains the vector. Larger compact-layout batches keep the object-store-friendly path: workers return node IDs, the parent unions raw-vector windows, and rerank streams chunks of at most 64 MiB and 1024 ranges. Unsupported sources fall back to complete serial queries. Single and batch rerank retain only the best top_k results per query in bounded heaps. Rust storage adapters, C/Python range callbacks, and JNI inputs opt into reusable sessions through try_clone_reader. Every clone must expose the same immutable byte sequence and be safe for concurrent calls.

+
Python · direct object-store reader
reader = VectorIndexReader(
+    input_with_concurrent_pread_many,
+    memory_budget_bytes=4 * 1024 * 1024 * 1024,
+)
+reader.optimize_for_search()
+plan = reader.read_plan()
+reader.warmup_queries(representative_queries, l_search=100)
+reader.calibrate_search_width(representative_queries, top_k=10)
+ids, distances = reader.search(
+    query, SearchParams.automatic(top_k=10)
+)
+

The C, C++, Java/JNI, and Python APIs expose the same latency hint, total Reader memory budget, and concrete read-plan diagnostics. See the shared API guide for lifecycle and callback ownership.

+
+ +
+

Build parameters

+
+ + + + + + + + + + + + + + + + + +
ParameterDefault / validationWhat it controlsWhen increased
dimensionInferred by Java/Python one-shot training; required by streaming APIs; 1..=1024Raw vector width and PQ codebook size.Raw-vector payload and rerank work grow linearly.
metricRequired: l2, inner_product, or cosineDistance used by graph build, PQ navigation, and exact reranking. Inner-product scores are negative dot products; cosine scores are 1 - cosine.Semantic, not inferred. Cosine vectors and queries are normalized internally; zero vectors retain distance 1.
target-recallOptional; 0..=1Selects fast_build at ≤ 0.85, high_recall at ≥ 0.97, and balanced between them when no preset is pinned.This is a deterministic starting policy, not a measured guarantee; validate on held-out queries.
max-bytes-per-vectorOptional positive persisted-size budgetGuides PQ width and may select 4-bit PQ and F16 rerank records. Configuration is rejected before training when the estimated row bytes plus amortized fixed data exceed the budget.A larger allowance preserves more PQ/raw precision. The check is conservative; compression, headers, and alignment mean it remains a per-vector sizing bound rather than an exact final-file-size promise.
deployment-profileauto, memory, local_storage, remote_storage, or object_storeSelects an interleaved local layout when one record fits 4 KiB; remote/object profiles select compact layout.It describes the intended serving medium, not a local-file implementation.
diskann.build-presetfast_build, balanced, or high_recall; inferred from target recall when omittedResolves R, Lbuild, alpha, raw encoding, and build distance as one coherent baseline.Higher-recall presets increase graph build work, file bytes, and memory.
pq.code-ratio0.0625; finite and in (0, 0.25] for 8-bit or (0, 0.125] for 4-bitTarget ratio between resident PQ-code bytes and raw f32-vector bytes. The builder selects the nearest m and distributes dimensions across balanced chunks.Usually reduces PQ error when increased, but grows resident memory and per-candidate lookup work.
pq.mOptional expert override; 1..=dimensionConcrete PQ chunk count. Explicit values take precedence over pq.code-ratio; exact chunk offsets are persisted in the self-describing codebook.Use only for a measured override of automatic sizing. Non-divisible dimensions and odd 4-bit values are valid.
pq.bits8; must be 4 or 8Centroids and stored bits per PQ chunk. Four-bit codes pack two chunks per byte, use 16-entry query tables, and require a zero high padding nibble when m is odd.Eight bits generally improve graph-navigation recall; four bits reduce codebook, resident codes, training work, and lookup-table size. Rebuild and benchmark both.
diskann.max-degree64; 1..=1023, preserving page-contained raw fallbackMaximum graph out-degree R.May improve connectivity and recall; increases graph bytes, build work, and page density cost.
diskann.build-search-list-sizeOmitted: max(100, R); explicit values must be ≥ RCandidate width Lbuild during Vamana construction.Usually improves graph quality while increasing build CPU and per-worker scratch.
diskann.alpha1.2; finite and ≥ 1Second-pass robust-prune threshold.Higher values prune candidates less aggressively; validate degree, recall, and graph behavior empirically.
diskann.seed42Random neighbor initialization and build order.It is not a quality knob; fix it for reproducible comparisons.
diskann.memory-budget-bytes8 GiB; > 0Selects the normal parallel graph build when it fits. Otherwise DiskANN automatically chooses 2–64 coarse, overlapping shards, builds and merges one shard at a time, then performs global robust pruning and connectivity repair. It rejects only when neither plan fits the estimate. The 8 GiB default also leaves enough internal build-state budget for the standard one-million-vector, 960-dimensional GIST corpus; the source-vector slice supplied by the caller is separate.A larger budget reduces or avoids sharding and usually shortens build time. This is an internal peak estimate, not an operating-system memory limit.
diskann.storage-layoutauto or omitted; explicit compact/interleaved overridescompact stores graph pages separately from densely packed vector records; interleaved stores each vector immediately before its adjacency list and requires E × dimension + 4 × R ≤ 4096, where E is 4 for F32 and 2 for F16.Automatic selection follows deployment-profile; pin only after measuring a deployment-specific exception.
diskann.raw-vector-encodingauto or omitted; preset/budget resolves F32 or F16Controls the persisted rerank-vector element width for both layouts. Compact F32/F16 payloads are exactly 4 × d × N / 2 × d × N bytes with no per-page padding.Explicit F32 preserves original rerank distances. Explicit F16 halves raw-vector I/O but must be recall-tested.
diskann.build-distanceauto or omitted; preset resolves PQ or full precisionSelects build-traversal distance. Both modes use full precision for robust pruning and connectivity repair.high_recall uses full precision; balanced/fast presets use PQ guidance.
+

Starting values

+
    +
  • Start with diskann.build-preset=balanced, the automatic pq.code-ratio=0.0625, and the intended deployment-profile. The balanced preset resolves to 8-bit PQ, R=64, Lbuild=100, alpha 1.2, F16, and PQ-guided construction unless an explicit option changes the representation.
    • +
  • For 8-bit PQ, the default resolves to m=32 at 128 dimensions and m=240 at 960 dimensions. Other shapes use the nearest count and balanced chunks whose widths differ by at most one. Inspect metadata for the resolved value and use explicit pq.m only after a representative recall measurement.
    • +
  • Test pq.bits=4 only as a separate build. Accept it when the smaller resident footprint and faster lookup tables preserve the required Recall@K across the full l_search sweep.
    • +
  • Use the F16 default for the storage/I/O baseline, but compare diskann.raw-vector-encoding=f32 when exact final distances or an unusual numeric range matters. F16 can change top-k ordering even when graph candidates are identical and rejects values outside the finite binary16 range.
    • +
  • If online l_search must become very large to reach the target, test Lbuild=150/200 before increasing R.
    • +
  • Increase R only after recording file size, build peak RSS, graph-page bytes, and recall. Rebuilds are required for every build-parameter change.
    • +
+
Why automatic relative sizing mattersOn public GIST1M, a fixed m=64 uses only 1.67% of the raw 960-dimensional vector bytes and DiskANN saturated near 0.74 Recall@10. The default ratio instead resolves to m=240, the same relative budget as m=32 on SIFT1M; local Recall@10 then reached 0.973 at l_search=200 and 0.991 at l_search=500.
+
+ +
+

Query parameters

+
+ + + + +
ParameterDiskANN meaningGuidance
top_kRequested result count.Results are padded with -1 and f32::MAX if fewer candidates survive.
AutoUses a Reader-calibrated width when available; otherwise max(100, 2 × top_k). Adaptive filtered graph candidates require an effective value of at least 200 in the initial quality-gated release.Preferred production default. Calibrate on representative queries, then validate the selected width against exact ground truth.
l_searchExplicit graph search-list size L=max(top_k, l_search).Expert recall/latency override. Sweep 50, 100, 200, and 400 when the automatic or calibrated result misses the workload target.
nprobeInvalid for DiskANN. The tagged search API rejects an IVF width instead of ignoring it.DiskANN has one global graph and reports logical nlist=1.
+
Large l_search has two costsIt expands more graph nodes and can read more adjacency pages. Unfiltered graph search uses max(4 × top_k, 64) as the exact-rerank seed count, then also reranks later graph candidates whose raw vectors fall in the same already selected read windows. The number of exact distance evaluations can therefore exceed the seed count without adding vector-read windows; candidates in any other window remain approximate-only. Increasing l_search still improves discovery, but does not imply exact reranking of every visited candidate.
+
+ +
+

Reader and warm-up parameters

+
+ + +
ParameterDefaultPurpose and guidance
estimated_random_read_latency_nanos (input capability)0Optional representative random-read latency. Zero measures the mandatory header read; a positive value selects the same internal plan without timing noise.
memory_budget_bytes4 GiBTotal Reader budget. Required PQ/model/row-ID state is reserved first; the remainder is partitioned across an automatically sized hot adjacency prefix, a cold-adjacency LRU, and a raw-vector LRU, each clipped to its actual section size.
+

Read-plan selection is part of open, not optimize_for_search. Call optimize_for_search before accepting traffic when predictable first-query latency matters: it loads resident data, preloads the automatically sized adjacency prefix, and validates every covered 4 KiB logical page in parallel before publishing that prefix. Follow it with warmup_queries to populate query-dependent caches and calibrate_search_width to select an automatic width. None is required for correctness; the first non-empty query initializes resident data lazily.

+
Resident size approximationLet K=2^pq.bits and C=pq.m for 8-bit or ceil(pq.m / 2) for 4-bit. The steady required state is approximately 4 × K × dimension + ceil(N × row_id_bit_width / 8) + C × N + 4 × N + 8 × ceil(N / 16) + adjacency_page_count + 4 × K × pq.m bytes, plus headers, decode scratch, and allocator overhead. The one public memory_budget_bytes value covers this steady state and the automatically retained adjacency/raw-vector caches. Query-local and active batch scratch remain transient and are separately bounded by the implementation.
+
+ +
+

Filtered search behavior

+

Every Roaring64 filter is first translated into a bitmap of matching internal nodes. Filters with at most N / 16 values lazily load the persisted (row_id, node_id) order and resolve duplicate row IDs with binary-search ranges. Dense filters use a sequential decoder over resident raw or FOR-bitpacked row IDs. If the optional lookup would exceed the Reader memory budget, sparse queries transparently use the sequential path. A filtered batch performs this translation once and shares the bitmap with every candidate worker.

+

The safe baseline scans matching resident PQ codes and keeps T=min(M, max(4 × top_k, 64)) candidates. A filtered batch evaluates this path in node-major tiles of up to four queries: each matching PQ code is loaded once per tile while each query keeps an independent bounded heap. The tile shrinks automatically when necessary so active PQ distance tables occupy at most 2 MiB. The resident scan does not clone or read the storage source. The graph path traverses the ordinary unfiltered graph, post-filters its candidates, and falls back to that complete matching-node PQ scan before any raw-vector read when fewer than T matches survive. It never filters graph edges, so the filter cannot disconnect traversal.

+
Initial graph quality gateThe checked formula is adaptive_l=max(resolved l_search, 2 × ceil(T × N / M)), capped at N, with estimated graph work min(N, adaptive_l × (R + 1)). Graph work must be at most half the matching-node scan. Recall-matrix evidence additionally keeps 50% filters and effective l_search<200 on scan; the initial graph-enabled cell is 100% matching with effective l_search≥200. The coalesced RemoteStorage and ObjectStore plans also require the complete adjacency section to be preloaded.
+

Benefit

  • Sparse filters avoid an O(N) row-ID scan after first use.
  • Dense batch filters scan row IDs once per batch rather than once per query.
  • Quality-gated all-row filters can reuse normal graph navigation.
  • Node-major PQ tiles reuse code loads across up to four queries.
  • Batch rerank reads each shared vector window once per chunk.
  • F32-exact or F16-quantized final distances with unchanged sentinel behavior.

Cost

  • The optional lookup occupies 4 × N resident bytes.
  • The first sparse query reads and validates that section.
  • A filtered batch retains its internal-node bitmap and at most one 1024-query chunk of candidate sets until rerank completes.
  • Most filtered cells intentionally remain linear in matching PQ codes to protect recall.
+

Negative stored row IDs and bitmap values above i64::MAX never match. Duplicate row IDs return every matching internal node before bounded PQ candidate selection.

+
+ +
+

Recommended tuning order

+
    +
  1. Fix acceptance criteria. Use production queries and filters; record Recall@K, P50/P95/P99, QPS, index bytes, peak RSS, build duration, read rounds, and bytes read.
    2. +
  2. Choose storage layout and characterize read latency. Benchmark interleaved on the cached local path, exercise the latency-hint or measured remote path for bandwidth-balanced network reads, and test compact when minimizing object-store request rounds matters most. Include file-size and byte-amplification results.
    3. +
  3. Calibrate, then verify search width. Run calibrate_search_width on representative queries, validate the chosen 100/200/400 width against exact ground truth, and use an explicit l_search only when the workload target requires it.
    4. +
  4. Tune PQ. Sweep pq.code-ratio around 0.0625 and record the resolved pq.m, then compare a separate 4-bit build. Both the ratio and bit width change resident memory and traversal accuracy.
    5. +
  5. Improve graph quality. Compare product_quantized against the full_precision control, then raise Lbuild and consider R. Do not change several build knobs in one experiment.
    6. +
  6. Size Reader memory. Sweep the single total memory budget after graph parameters stabilize. The Reader repartitions it automatically; use search statistics and read rounds to identify whether a larger working set has material benefit.
    7. +
  7. Run production-scale validation. Include cold/warm cache, concurrent queries, selective and broad filters, corrupt/truncated input, and restart/rollover behavior.
    8. +
+
+ +
+

Capacity model

+
+ + + + + + + + + + + + + + + + +
ResourceApproximationControlled by
Raw-vector file payloadCompact: exactly E × dimension × N, with E=4 for F32 and E=2 for F16. Interleaved records add adjacency-page tail padding.Dimension, vector count, encoding, and layout.
PQ-code resident/file payloadpq.m × N for 8-bit; ceil(pq.m / 2) × N for 4-bitpq.m and pq.bits.
Row IDsFile: 32 + ceil(N × w / 8); raw fallback 32 + 8 × N. Resident storage omits the 32-byte header.Vector count and global row-ID span; w is the minimum width in 0..63.
Lazy row-ID order4 × NSparse filters and resident budget.
Resident adjacency locators4 × N + 8 × ceil(N / 16) (about 4.5 bytes per node)Vector count.
Hot adjacency prefixAutomatically up to 16 MiB for memory/local latency, 32 MiB for remote latency, or 64 MiB for object-store latency, clipped to the adjacency section and remaining total budgetMeasured or hinted latency, adjacency size, required resident state, and memory_budget_bytes.
Cold-adjacency LRUMemory/local latency tiers cap it at 64 MiB. Remote/object-store latency tiers may use all remaining budget needed by the cold adjacency section after the raw-vector split; all cases are clipped to section size and total budget.Shared with retained batch workers; split across 16 shards for production-sized budgets.
Adaptive adjacency payloadAt most 4 × total_edges, usually less after BFS-remapped delta-varint encoding, plus page-tail paddingActual graph degrees, ID locality, and diskann.max-degree.
PQ codebook4 × 2^pq.bits × dimensionDimension and pq.bits; independent of pq.m.
Build peakRaw vectors + IDs + row-ID encoding scratch + codes + codebook + row-ID order + adjacency locators + max(graph-build, in-place graph-remap). PQ training keeps a deterministic reservoir of at most 50,000 rows, reduced automatically so the retained sample, optional cosine-normalized copy, codebook, and parallel KMeans scratch fit the build budget. The low-level Rust DiskAnnIndex::train returns an error when even the minimum one-worker plan cannot fit. If the parallel graph estimate exceeds the budget, overlapping coarse shards are built and merged one at a time; the final compact adjacency remains approximately 4 × N × R + 2 × N.N, dimension, pq.m, pq.bits, R, Lbuild, memory budget, and Rayon worker count.
Query-local adjacencyAt most 8 MiB of retained cold-window allocation capacity/references after each beam round, plus the active beamInternal safety bound; stats report peak bytes and evictions.
Raw-vector cache and query buffersOne shared immutable raw-vector LRU, plus up to 8 MiB of reusable/local-reference capacity per active worker under a 64 MiB aggregate worker allowance. Memory/local latency tiers cap the shared LRU at 64 MiB; remote/object-store latency tiers may use the remaining Reader budget up to the vector-section size.The shared LRU is clipped to the total Reader budget. Session scratch and borrowed references persist for reuse across batch calls without duplicating payload bytes.
Graph candidate heapsL retained candidates plus at most 2 × L live frontier entries; contiguous capacity is reused between queriesl_search and graph worker count; evicted lazy entries are periodically compacted.
Graph visited scratchMinimum of dense ceil(N / 8) bytes or a reusable deterministic open-addressed table sized from L × RVector count, l_search, graph degree, and graph worker count; scan/rerank-only paths allocate neither form.
Filtered PQ query tileUp to four distance tables and bounded candidate heaps, with distance-table bytes capped at 2 MiB per active tilepq.m, pq.bits, and Rayon worker count; unusually large tables automatically reduce the tile width.
Parallel exact-rerank referencesOne borrowed record reference per candidate in the active read chunk. In unfiltered graph search, references may exceed the seed count because every later candidate in an already selected raw-vector window is included; the unique-window count remains seed-bounded.Allocated only when candidate references × dimension ≥ 16384; bounded by the 1024-query, 64 MiB, and 1024-range chunk limits. Benchmark stats expose both candidate references and unique windows.
+

These formulas omit headers, alignment, allocator overhead, and per-query caches. Use measured RSS and serialized bytes as the final authority.

+
+ +
+

Current implementation boundaries

+
    +
  • v1 supports squared L2, negative-dot-product inner product, and 1 - cosine, with dimensions up to 1024 and 4/8-bit PQ.
    • +
  • The index is static. FreshDiskANN-style insert, delete, consolidation, and graph repair are not implemented.
    • +
  • Serialization builds the complete graph in memory. Raw vectors are stored as explicit F32 or F16 scalar records; there is no PQ-only result mode.
    • +
  • Recall is sensitive to both PQ capacity and graph/search budgets. On the public SIFT1M and GIST1M runs, the default ratio resolved to m=32 and m=240. DiskANN Recall@10 was 0.995 and 0.973 at l_search=200, while GIST rose to 0.991 at l_search=500; use the public-corpus procedure rather than synthetic vectors when validating a new ratio.
    • +
  • Remote graph traversal has dependent I/O rounds. It improved high-recall single-query latency over raw IVF scans at 2 ms RTT in the measured workload, but compact IVF-PQ won batch throughput when its lower recall was acceptable.
    • +
  • Direct remote reads rely on the caller's range-read callback for actual concurrency.
    • +
  • The file has strict structural validation but no embedded checksum; integrity belongs to the outer Paimon file/manifest layer.
    • +
  • Generated correctness and I/O acceptance workloads pass, but production readiness still requires SIFT1M or representative full-scale recall, soak, concurrency, and failure testing.
    • +
+
Recommended maturity label: previewThe implementation has cross-language tests, corruption checks, memory budgets, and local/remote acceptance instrumentation. Do not label it generally available until representative large-scale benchmarks and operational validation meet the deployment's SLOs.
+ +
+
+
+
+
Apache Paimon Vector IndexDiskANN · v1 preview
+ + diff --git a/docs/index.html b/docs/index.html index 442719b..91f85ec 100644 --- a/docs/index.html +++ b/docs/index.html @@ -9,7 +9,7 @@ - + Index Selection Guide · Paimon Vector Index @@ -29,8 +29,8 @@ IVF-FLAT IVF-PQ IVF-RQ - HNSW-FLAT - HNSW-SQ + IVF-SQ + DiskANN
@@ -46,9 +46,9 @@

Index selection guide · v1

Five indexes.
One selection map.

-

Every implementation uses IVF to narrow the search space. The real difference is what happens inside each selected list: scan full vectors, scan compact codes, or traverse a local HNSW graph. This guide compares recall, latency, storage, build cost, and object-store I/O.

+

Four implementations use IVF to narrow the search space before scanning raw vectors or compact codes. DiskANN instead uses one global Vamana graph with resident PQ navigation and paged F16/F32 reranking. This guide compares recall, latency, storage, build cost, and object-store I/O.

- L2 / Inner Product / Cosine + All indexes: L2 / IP / Cosine Batch search Roaring row filters Magic-based detection @@ -56,21 +56,22 @@

Five indexes.
One selection map.

-

A shared two-stage search

-

nlist controls the number of coarse partitions. nprobe controls how many partitions a query reads. Only the second stage changes between exact scans, quantized scans, and graph traversal.

+

IVF's shared two-stage search

+

For the four IVF families, nlist controls the number of coarse partitions and nprobe controls how many partitions a query reads. Only the stored representation and scan kernel change.

+
DiskANN follows a different pathIt traverses one global Vamana graph using resident PQ codes, reads adjacency pages on demand, then reranks persisted F16 or F32 vectors from either the same interleaved pages or a separate compact section. It has no IVF lists; the tagged query API uses l_search and rejects an IVF nprobe override.
01Preprocess vectors
Normalize for cosine
02Measure distance to
IVF coarse centroids
03Select the nearest
nprobe lists
-
04Scan codes or
traverse local graphs
+
04Scan raw vectors or
compact codes
05Merge candidates into
the global top K
@@ -82,15 +83,16 @@

Usage and development

API and language bindings

The Trainer / Writer / Reader lifecycle with complete Rust, C, C++, Java/JNI, and Python examples.

Open the API guide →
- +

Search and filtering

Understand automatic width, explicit IVF nprobe, DiskANN l_search calibration, warm-up, and Roaring64 filters.

Explore the search API →

Development and benchmarks

Repository modules, Rust checks, cross-language smoke tests, ANN benchmarks, and storage compatibility.

Open the development guide →
+

DiskANN deployment and tuning

Understand the memory, local, remote, and object-store profiles, resident-memory model, build parameters, and production-readiness boundary.

Open the DiskANN guide →

Core differences

-

This table describes the implementation in this repository, not a general promise made by similarly named algorithms elsewhere. Storage estimates omit row IDs, IVF centroids, offset tables, graph edges, and headers unless noted.

+

This table describes the implementation in this repository, not a general promise made by similarly named algorithms elsewhere. Storage estimates omit row IDs, model sections, offset tables, alignment, and headers unless noted.

@@ -101,51 +103,146 @@

Core differences

- + - - - - - + + + + +
IndexPer-list representationPer-list searchMain payload per vectorAccuracy profileBuild costPrimary controlsBest fit
IndexRepresentationCandidate searchMain payload per vectorAccuracy profileBuild costPrimary controlsBest fit
IVF-FLATRaw f32 vectorsExact distance scan in probed listsAbout 4d bytesNo quantization loss; recall mainly depends on nprobeLownlist, nprobeBaselines, recall-first workloads, moderate scale
IVF-PQ8-bit PQ codes; optional OPQDistance-table lookup over compact codesAbout m bytesPQ reconstruction error; OPQ may improve uneven subspacesMedium to highpq.m, use-opqLarge-scale general use and memory–throughput balance
IVF-RQ1-bit rotated residual code + three factorsByte LUT or quantized-query popcount scanAbout ceil(d/8)+12 bytesThe most aggressive distance approximationLow to mediumquery_bitsHigh dimensions, extreme compression, low training budget
IVF-HNSW-FLATRaw vectors + one HNSW graph per listGraph traversal with scan fallback4d bytes + graphRaw-vector distances; candidate generation is approximateHighhnsw.m, ef_searchRecall and latency first when individual lists are large
IVF-HNSW-SQ8-bit scalar residual codes + graphGraph traversal over decoded vectorsd bytes + graphBoth SQ and graph-search approximationHighhnsw.*, ef_searchGraph search when raw-vector storage is too large
IVF-PQ8-bit PQ codes; optional OPQDistance-table lookup over compact codesAbout m bytesPQ reconstruction error; OPQ may improve uneven subspacesMedium to highAutomatic nlist/nprobe/pq.m; target-based OPQ; explicit overridesMinimum file and selected-list bytes among IVF when the measured 0.58–0.74 recall band is sufficient
IVF-SQ8-bit scalar residual codes with per-list boundsSIMD code scan in probed listsAbout d bytesPer-coordinate scalar quantization lossLowAutomatic nlist/nprobe; explicit overridesHighest measured compact batch throughput when 0.80–0.86 recall meets the target
IVF-RQMulti-bit rotated residual levels + coarse/full factorsBounded sign-plane scan, then full bit-plane refinementDefault about padded_d/2+20 bytesMeasured 0.82–0.91 Recall@10 across GloVe-100, SIFT1M, and GIST1M at four bitsLow to mediumAutomatic nlist/nprobe; budget-based bits; explicit overridesBest measured compact recall when 3–4.5× lower batch throughput than IVF-SQ is acceptable
DiskANNGlobal Vamana + resident PQ + persisted rerank vectorsPQ-guided graph traversal and F32/F16 rerankE·d + pq.m + 4(R+1) bytes, approximately; E=4 or 2Approximate candidate discovery; F32-exact or F16-quantized distances for reranked candidatesVery highBuild preset + deployment/capacity objectives; calibrated automatic l_searchImmutable L2 on local SSD when high recall and sub-MiB query reads repay the long build
IVF-FLATRaw f32 vectorsExact distance scan in probed listsAbout 4d bytesNo quantization loss; recall mainly depends on nprobeLowAutomatic nlist/nprobe; explicit overridesRecall ceiling, frequent rebuilds, IP/cosine, or production sets whose scan bytes are affordable
+
+
+

Measured comparison: public SIFT1M, GIST1M, and GloVe-100 corpora

+

This repository's unified benchmark builds all five indexes over standard public vectors, searches the same independent public queries, and scores every result against published exact neighbors. The results below are the homepage's sole performance evidence.

+
+ +

Benchmark setup

+
This is not the zero-configuration benchmarkRunning cargo bench -p paimon-vindex-core --bench ann_bench without public file paths uses a 20k-vector, 64-dimensional generated smoke workload. Reproducing the results below requires the public files, recorded IVF and DiskANN search settings, a fixed worker count, and an output directory on the storage device being measured. File shape, training count, and multi-index process isolation are automatic.
+

The real-data run uses the public ANN-Benchmarks SIFT1M, GIST1M, and GloVe-100 files, the first 1,000 independent test queries, and their published Top-100 exact neighbors. Recall@10 compares only the first ten published neighbors. SIFT and GIST contain one million base vectors with 128 and 960 dimensions. GloVe contains 1,183,514 vectors with 100 dimensions and angular ground truth; its base and query vectors are L2-normalized during conversion so the common L2 benchmark produces the same neighbor ordering as cosine. The benchmark supplies 65,536 base vectors to every trainer; DiskANN bounds PQ training memory with a deterministic reservoir of at most 50,000 vectors.

+
+ + + + + + + + + +
SourceParametersRecorded valueReproduction rule
Public dataANN_BASE_FVECS, ANN_QUERY_FVECS, ANN_GROUND_TRUTH_IVECSSIFT1M, GIST1M, or normalized GloVe-100 converted filesSet all three together; otherwise the benchmark generates synthetic vectors.
Dataset shapeANN_N, ANN_NQ, ANN_D1,000,000 / 1,183,514, 1,000, 128 / 960 / 100Inferred from the public files. An explicit value becomes a shape assertion and fails before the full dataset is loaded if it differs.
Training inputANN_TRAIN_N65,536Inferred as min(N, max(65,536, 64 × nlist)). DiskANN deterministically retains at most 50,000 of these vectors for bounded PQ training; the IVF trainers consume all 65,536.
IVF searchANN_NLIST, ANN_NPROBE1,024, 64Set explicitly; benchmark defaults are 64 and 8.
DiskANN searchANN_DISKANN_L_SEARCH100May be omitted; this is the benchmark default and the automatic value for k=10.
Process and device isolationANN_INDEXES, ANN_OUTPUT_DIR, RAYON_NUM_THREADSAll five indexes, target APFS path, 12 threadsPublic multi-index runs automatically spawn one child process per index. Set a subset only when needed; set the device path and worker count explicitly.
Matching defaultsANN_K, ANN_PQ_CODE_RATIO, ANN_RQ_BITS10, 0.0625, 4May be omitted; the reproduction command pins them so a future default change cannot silently alter the comparison.
DiskANN build defaultspq.bits, R, Lbuild, alpha, memory budget, layout, raw-vector encoding, build distance8, 64, 100, 1.2, 8 GiB, compact, F16, product-quantizedUse the benchmark defaults. Leave ANN_PQ_M unset so pq.code-ratio resolves the concrete value.
Reader/I/O modelAutomatic read plan, Reader budget, simulated latencyLatency-derived local/remote/object-store plans, 4 GiB automatically partitioned budget, 0/2/20 ms per read roundFixed by the current benchmark implementation; ANN_STORAGE_CASES selects a focused subset. DiskANN range reads use an I/O pool independent of query workers; all five indexes were refreshed after their current reader, storage, and parallel-scan work.
+
Equal relative PQ budgetThe default pq.code-ratio=0.0625 automatically resolves SIFT to pq.m=32, GIST to pq.m=240, and GloVe to pq.m=25. Every code occupies 6.25% as many bytes as its raw f32 vector and leaves four dimensions per PQ sub-vector. The concrete value is persisted in index metadata; use explicit pq.m only as an expert override.
+

The run was recorded on 25 July 2026 using an Apple M4 Pro with 12 logical CPUs and 48 GiB RAM, a release build with Rust 1.95, real APFS files with warm operating-system pages, and the automatic 4 GiB DiskANN Reader budget. The reproduction command pins Rayon to 12 workers instead of relying on automatic host parallelism. The modeled serving profiles add 2 ms or 20 ms per positional-read round while executing all ranges in that round concurrently. DiskANN's benchmark adapter runs those ranges on a separate 12-worker I/O pool so a full query-worker pool cannot starve nested reads; this models the independent executor required of a production concurrent storage callback. For the 20 ms profile, open/optimization and sequential-query latency are computed as measured CPU/I/O time plus 20 ms per observed round; batch QPS retains literal delay injection so query overlap is measured. IVF multi-range calls are bounded to 64 MiB, so an all-query GIST batch uses 4 IVF-PQ, 15 IVF-SQ, or 59 IVF-FLAT payload rounds instead of submitting hundreds of MiB or several GiB as one unbounded call. Unified IVF Readers now reuse the 64-byte dispatch header, so opening and loading resident metadata takes two positional-read rounds rather than three. Each dataset's three DiskANN profile rows reuse the same built graph. Sequential and batch measurements use separately opened and optimized Readers, so the batch does not inherit query-dependent windows from the sequential sweep. Batch QPS measures one search_batch call over all 1,000 public queries; it is not concurrent-client QPS. See the complete public-data command.

+ +

Build, file, and peak process memory

+
+ + + + + +
IndexSIFT buildSIFT file / RSSGIST buildGIST file / RSSGloVe buildGloVe file / RSS
IVF-PQ8.74 s0.032 / 0.88 GiB55.4 s0.230 / 5.00 GiB7.92 s0.030 / 0.85 GiB
IVF-SQ3.93 s0.122 / 0.79 GiB22.7 s0.907 / 5.09 GiB3.86 s0.113 / 0.71 GiB
IVF-RQ3.92 s0.080 / 0.71 GiB23.5 s0.471 / 4.65 GiB4.03 s0.095 / 0.68 GiB
DiskANN74.0 s0.361 / 1.51 GiB11 min 26 s2.089 / 7.94 GiB2 min 33 s0.396 / 1.45 GiB
IVF-FLAT4.05 s0.479 / 1.83 GiB24.9 s3.582 / 12.74 GiB4.10 s0.443 / 1.60 GiB
+
IVF-SQ build and scan refreshThe add path now borrows L2/IP input, assigns rows once, and encodes lists in parallel with one residual scratch vector per active list task instead of materializing an additional N × d residual matrix. In the immediately preceding same-machine run, SIFT/GIST/GloVe peak RSS was 1.81 / 12.92 / 1.60 GiB; it is now 0.79 / 5.09 / 0.71 GiB. A Top-K threshold fast path skips hash work for candidates that cannot enter the heap: local P95 is now 0.79 / 3.56 / 0.71 ms and batch throughput is 11,082 / 1,502 / 12,962 QPS. An experimental list-major batch scan was slower on SIFT/GIST and was not retained. The blocked-code format, file size, read bytes, and measured Recall@10 remain unchanged.
+
Final IVF-PQ read/write reviewThe Reader keeps the v1 code suffix zero-copy and aligned, orders selected lists by persisted offset, decodes the common one-byte row-ID delta without entering the generic varint loop, and initializes each distance accumulator from the first PQ column instead of zero-filling it in a separate pass. In three-run same-file A/B medians, the row-ID fast path alone changed SIFT/GIST/GloVe batch throughput from 7,528 / 987 / 8,693 to 7,649 / 1,026 / 8,943 QPS. The writer now performs a 32×32 cache-blocked transpose into one list-sized reusable buffer; SIFT and GIST serialization changed from 47 / 221 ms to 36 / 150 ms while producing the same v1 bytes. The final complete profile rerun reports 8,353 / 1,094 / 9,330 warm-local batch QPS and 0.71 / 2.19 / 0.64 ms P95. Faiss-style list-major query bucketing, parallel ID decode, one contiguous all-query table, and worker-level GIST scratch retention all regressed at least one public corpus and were removed. Faiss FastScan's 4-bit, 32-row design remains a different accuracy/format choice; conventional Faiss and Lance support the retained 8-bit/transposed locality choices. Recall, file size, and query bytes are unchanged, so published IVF-PQ remains v1.
+
Latest IVF-FLAT storage and scan reviewThe v1 writer retains only sort permutations and encoded IDs, materializes one sorted raw-vector list at a time, and reproduced all three prior public files byte-for-byte. The Reader now receives list bytes directly into an f32-aligned allocation; an internal prefix of at most three bytes keeps the raw-vector suffix aligned despite variable-length row IDs, so search no longer allocates and decodes a second vector payload. Together with the strict partial-L2 cutoff, the complete public rerun changed SIFT/GIST/GloVe local batch throughput from 6,570 / 559 / 6,345 to 8,510 / 875 / 9,502 QPS and P95 from 5.31 / 47.04 / 4.76 ms to 1.88 / 11.38 / 1.40 ms. Recall@10, file version, file bytes, and bytes read are unchanged; these are complete-run results, not best-of measurements.
+
Pre-release storage changes produced measurable savingsThe compact IVF-RQ factor layout removes one unused F32 value per row: SIFT/GIST/GloVe files fell from 90.3 / 509.8 / 106.8 MB to 86.3 / 505.8 / 102.1 MB without changing the estimator; the refreshed Recall@10 values are 0.9148 / 0.9039 / 0.8203. Changing the balanced DiskANN default from F32 to F16 rerank vectors reduced the same three files from 0.599 / 3.877 / 0.617 GiB to 0.361 / 2.089 / 0.396 GiB. The current warm-local Recall@10 values are 0.9915 / 0.9336 / 0.8355. Timing changes also include the intervening reader/cache and graph-build work.
+
Latest IVF-RQ build and scan refreshThe add path assigns rows once and encodes independent lists in parallel with list-local residual, rotation, code, and factor scratch. SIFT/GIST/GloVe add time fell from 5.96 / 39.11 / 6.48 seconds to 3.61 / 21.51 / 3.76 seconds; complete build time is now 3.92 / 23.5 / 4.03 seconds. The unfiltered scanner now removes the per-lane filter test from coarse-code accumulation, and the final Top-K threshold avoids hash-map work after both estimation stages. Local batch throughput is now 3,074 / 444 / 2,917 QPS; a strict GIST same-file A/B improved batch time by 8.7%. The rebuilt files remain byte-for-byte identical to the serial baseline, so recall, storage, and bytes read are unchanged.
+
DiskANN read-path reviewDiskANN converts F16 rerank vectors and accumulates L2 directly in one AArch64 NEON loop; its local profile coalesces 16 KiB windows. The final compact-layout batch rerank groups candidate windows with a hash table, then sorts only the unique windows into deterministic I/O order. Against the immediately preceding ordered-map control, median local batch time changed from 117 / 223 / 162 ms to 111 / 215 / 159 ms on SIFT/GIST/GloVe. Local P95 remains 1.50 / 1.83 / 1.90 ms; representative batch throughput is 9,009 / 4,651 / 6,289 QPS with a separate range-I/O executor. Remote and object-store plans remain 32 / 64 KiB. A broader Vec sort/dedup replacement for graph window planners was not retained because it regressed local SIFT/GIST batch time by 7–13%.
+
Final open-source cross-check and format decisionFaiss FastScan still trades down to 4-bit lookup tables and a 32-row layout, so it is not a transparent replacement for the published 8-bit IVF-PQ v1. Faiss Panorama's additional level-oriented energy data was not needed to keep the existing IVF-FLAT v1 progressive cutoff. Lance's partition prefetch and transposed PQ storage match the current batched Readers; its prepared transposed L2 target is explicitly aimed at small target sets such as PQ codebooks, not large flat lists. Faiss RaBitQ's blocked multi-bit scan remains structurally aligned with IVF-RQ, while a direct-factor RQ payload experiment regressed SIFT batch throughput by 10–15% and was removed. DiskANN3's asynchronous provider, beam, and working-set model remains aligned with SeekRead, the latency-derived read planner, and the bounded caches. No measured result justified a v2 migration for IVF-PQ or IVF-FLAT, and no byte-layout change was retained for the pre-release IVF-SQ, IVF-RQ, or DiskANN formats.
+

DiskANN spends almost all build time constructing one global graph: about 18× IVF-FLAT on SIFT, 28× on GIST, and 37× on GloVe. The balanced F16 default makes its files about 42% smaller than IVF-FLAT on GIST and 11% smaller on GloVe, but they remain much larger than the compact IVF encodings because persisted rerank vectors, resident codes, and graph edges are all material. Peak RSS remains below the raw IVF writers because the DiskANN writer does not retain a second full raw-vector organization.

+ +

Warm local-storage result

+
+ + + + + +
Index / searchSIFT Recall / P95 / batch QPS / readGIST Recall / P95 / batch QPS / readGloVe Recall / P95 / batch QPS / read
IVF-PQ0.7142 / 0.71 ms / 8,353 / 2.18 MiB0.7410 / 2.19 ms / 1,094 / 17.84 MiB0.5819 / 0.64 ms / 9,330 / 1.84 MiB
IVF-SQ0.8627 / 0.79 ms / 11,082 / 8.38 MiB0.8577 / 3.56 ms / 1,502 / 70.95 MiB0.8036 / 0.71 ms / 12,962 / 6.99 MiB
IVF-RQ0.9148 / 1.20 ms / 3,074 / 5.54 MiB0.9039 / 4.41 ms / 444 / 37.02 MiB0.8203 / 1.23 ms / 2,917 / 5.89 MiB
DiskANN0.9915 / 1.50 ms / 9,009 / 0.66 MiB0.9336 / 1.83 ms / 4,651 / 0.83 MiB0.8355 / 1.90 ms / 6,289 / 0.96 MiB
IVF-FLAT0.9937 / 1.88 ms / 8,510 / 33.19 MiB0.9549 / 11.38 ms / 875 / 283.40 MiB0.8832 / 1.40 ms / 9,502 / 27.57 MiB
+

IVF-SQ is the compact-throughput choice: it leads the compact indexes on all three local batch runs. IVF-RQ uses smaller files and raises recall from 0.8627 / 0.8577 / 0.8036 to 0.9148 / 0.9039 / 0.8203, but batch throughput falls by about 3.4–4.4×. IVF-PQ is smaller and faster than RQ, but its 0.5819–0.7410 recall makes it a capacity-first choice rather than a default accuracy compromise. DiskANN is compelling on SIFT and especially GIST: it reads below 1 MiB per query on both, and on GIST greatly outpaces IVF-FLAT. It is not automatically best on GloVe, where IVF-FLAT has higher recall, lower P95, and higher batch throughput at the recorded settings. Treat l_search and nprobe as calibration points whenever the displayed recall misses the production gate.

+ +

Remote cache with 2 ms per I/O round

+
+ + + + + +
Index / searchSIFT Recall / P95 / batch QPS / roundsGIST Recall / P95 / batch QPS / roundsGloVe Recall / P95 / batch QPS / rounds
IVF-PQ0.7142 / 6.16 ms / 7,282 / 1.00.7410 / 7.12 ms / 1,162 / 1.00.5819 / 5.94 ms / 8,243 / 1.0
IVF-SQ0.8627 / 6.38 ms / 9,345 / 1.00.8577 / 11.36 ms / 1,359 / 1.90.8036 / 6.16 ms / 10,502 / 1.0
IVF-RQ0.9148 / 7.05 ms / 2,916 / 1.00.9039 / 7.80 ms / 434 / 1.00.8203 / 6.74 ms / 3,080 / 1.0
DiskANN0.9808 / 15.97 ms / 3,089 / 1.70.8482 / 14.22 ms / 2,175 / 1.50.8029 / 18.18 ms / 2,203 / 2.1
IVF-FLAT0.9937 / 7.31 ms / 6,763 / 1.00.9549 / 29.73 ms / 846 / 5.00.8832 / 6.60 ms / 7,394 / 1.0
+

IVF-PQ and IVF-RQ load each query's selected compact lists in one concurrent multi-range round. IVF-SQ does the same on SIFT/GloVe; GIST's 960-dimensional payload crosses the 64 MiB per-call guard on 89% of queries and averages 1.9 rounds. IVF-FLAT also uses bounded concurrent multi-range reads: SIFT/GloVe fit in one round, while the 283 MiB GIST payload averages 5.0. IVF-RQ preserves the strongest compact-IVF recall with 7.05 / 7.80 / 6.74 ms P95. Parallel IVF-FLAT has much higher recall and competitive fixed-latency results on SIFT/GloVe, but transfers 28–33 MiB per query; this model does not charge bandwidth. DiskANN's adaptive coalescing and caches reduce the average sequential request count to 1.5–1.7 rounds on SIFT/GIST and 2.1 rounds on GloVe.

+ +

Object store with 20 ms per I/O round

+
+ + + + + +
Index / searchSIFT Recall / P95 / batch QPS / roundsGIST Recall / P95 / batch QPS / roundsGloVe Recall / P95 / batch QPS / rounds
IVF-PQ0.7142 / 20.70 ms / 6,831 / 1.00.7410 / 22.14 ms / 1,054 / 1.00.5819 / 20.62 ms / 8,299 / 1.0
IVF-SQ0.8627 / 20.77 ms / 6,781 / 1.00.8577 / 43.50 ms / 854 / 1.90.8036 / 20.68 ms / 7,175 / 1.0
IVF-RQ0.9148 / 21.05 ms / 2,688 / 1.00.9039 / 24.34 ms / 392 / 1.00.8203 / 21.04 ms / 2,767 / 1.0
DiskANN0.9808 / 60.57 ms / 537 / 1.00.8483 / 41.19 ms / 1,011 / 1.00.8033 / 80.84 ms / 421 / 1.2
IVF-FLAT0.9937 / 21.80 ms / 3,076 / 1.00.9549 / 130.49 ms / 349 / 5.00.8832 / 21.22 ms / 2,948 / 1.0
+

At 20 ms per round, IVF-RQ is the strongest measured compact one-round option: it reaches 0.90-class recall on SIFT/GIST and 0.8203 on GloVe. IVF-SQ is faster when its lower recall is enough, and IVF-PQ is smaller when stronger quantization loss is acceptable. IVF-FLAT now looks competitive on one-round SIFT/GloVe in this fixed-latency model, but that result assumes 28–33 MiB transfers have no bandwidth cost; GIST's 283 MiB and five rounds expose the boundary. DiskANN averages about one modeled round after warmup, but dependent graph rounds remain visible in P95. A complete local SSD cache remains its preferred deployment.

+
The automatic read plan affects approximate searchThe latency-derived local tier uses graph beam 4 while remote and object-store tiers use beam 16, so the same l_search can return different approximate candidates; this is visible for both GIST and GloVe at l_search=100. The tiers use 16 KiB, 32 KiB, and 64 KiB coalescing windows respectively. Storage latency itself does not change ground truth. Compare indexes with the same latency and capability hints when isolating media effects.
+
Remote-model boundaryThe vectors and exact neighbors are public corpus data, but the 2 ms and 20 ms profiles are controlled I/O models rather than measurements from a production cache or object store. They add fixed latency without modeling bandwidth, cache misses, TLS, retries, throttling, request limits, or tail-latency variance.
+
+

Choose by constraint

There is no best index independent of data distribution. Narrow the field to one or two candidates, then evaluate Recall@K, P95/P99 latency, file size, build time, and object-store bytes on real queries.

+
Practical default orderFirst reject indexes that cannot meet the measured recall target. Build IVF-FLAT to establish the corpus-specific IVF ceiling. If a compact representation is required, choose IVF-SQ for throughput, IVF-RQ for recall, or IVF-PQ for minimum bytes. Evaluate DiskANN separately for immutable data served from local SSD; do not select it only because the collection is large or assume L2 results transfer to another metric.
+

Measured recommendation matrix

+
+ + + + + + + + +
Production constraintStart withEvidence from this runMove away when
Establish a recall ceiling or debug ranking qualityIVF-FLATHighest measured recall on all three corpora: 0.9937 / 0.9549 / 0.8832, with roughly four-second SIFT/GloVe builds.The 28–283 MiB selected-list reads or raw-vector file size exceed the serving budget.
Highest compact batch throughputIVF-SQ11,082 / 1,502 / 12,962 local batch QPS at 0.8627 / 0.8577 / 0.8036 recall; files are about one quarter of IVF-FLAT.The recall gate is above SQ, or one byte per dimension is still too large.
Strongest recall in a compact IVF fileIVF-RQ0.9148 / 0.9039 / 0.8203 recall in files smaller than IVF-SQ, with one sequential read round per query in all three modeled profiles.Batch throughput is the primary SLO; the four-bit scanner is 3–4.5× slower than SQ in the local run.
Minimum index file and compact-IVF scan bytesIVF-PQThe smallest files—0.032 / 0.230 / 0.030 GiB—and the smallest IVF selected-list reads at 1.84–17.84 MiB, with strong batch throughput.0.5819–0.7410 recall is below the gate; increase the PQ budget or choose SQ/RQ instead.
High-recall immutable data on local SSDDiskANN, checked against IVF-FLAT for the same metricThe recorded L2-equivalent run reached 0.9915 / 0.9336 / 0.8355 recall with 0.66 / 0.83 / 0.96 MiB reads; SIFT/GIST P95 is 1.50 / 1.83 ms.Metric-specific recall misses the gate, rebuilds are frequent, the file is not locally cached, preview maturity is unacceptable, or the corpus behaves like GloVe at l_search=100.
Frequent rebuilds or rapidly changing snapshotsIVF-FLAT, IVF-SQ, or IVF-RQThese build in about 4 seconds on SIFT/GloVe and 23–25 seconds on GIST; IVF-PQ is about 2× slower and DiskANN is 18–37× slower than IVF-FLAT.The serving phase dominates lifetime cost enough to justify PQ training or graph construction.
Direct 2/20 ms remote or object-store readsCompact IVF selected by recall: PQ → SQ → RQPQ and RQ use one sequential multi-range round here; SQ does so on SIFT/GloVe and averages 1.9 rounds on GIST. Choose successively more recall at greater bytes or CPU cost.Bandwidth, request limits, or real tail latency invalidate the fixed-latency model; prefer a complete local SSD cache and rerun the benchmark.
Inner product or cosineIVF-FLAT as the recall control; DiskANN as an additional candidate for immutable local-SSD servingAll five implementations support L2, IP, and cosine. DiskANN normalizes cosine internally and uses metric-aware graph construction and exact reranking, but the displayed public-corpus matrix was recorded through the L2-equivalent benchmark path.The selected configuration misses its metric-specific recall gate—retune nprobe, representation width, OPQ, or l_search before deployment.
+
A displayed winner can still be the wrong choiceThese recommendations apply to the recorded nlist=1024, nprobe=64, PQ ratio, RQ bits, and l_search=100. For example, the current GloVe run does not reach 0.90 recall with any index, and current GIST reaches 0.95 only with IVF-FLAT. If a required recall threshold is not present in the table, tune and rebuild rather than choosing the closest result.
-

I need a trustworthy baseline

Start with IVF-FLAT. It keeps raw vectors, which separates IVF partition loss from quantization loss.

Explore IVF-FLAT →
-

The index must be tiny

Evaluate IVF-RQ first. If recall is insufficient, try IVF-PQ and increase pq.m to trade space for accuracy.

Explore IVF-RQ →
-

I want a proven compromise

IVF-PQ is the conventional starting point. It compresses each vector to m bytes and scans with lookup tables.

Explore IVF-PQ →
-

Per-list scans dominate CPU

Consider IVF-HNSW-FLAT. The graph reduces distance evaluations but increases build time and serialized size.

Explore HNSW-FLAT →
-

I want graphs without raw-vector size

IVF-HNSW-SQ replaces 4-byte floats with one byte per dimension while retaining a graph in each list.

Explore HNSW-SQ →
-

Data lives in S3, OSS, or HDFS

Control nprobe and list size first. Readers use positional I/O and batch adjacent selected-list ranges where possible.

Understand the I/O model →
+

I need a trustworthy baseline

Start with IVF-FLAT. It exposes the IVF partition ceiling without quantization loss and rebuilds quickly.

Explore IVF-FLAT →
+

I need compact high recall

Choose IVF-RQ when its 0.82–0.91 measured recall matters more than batch throughput; compare every result with the IVF-FLAT ceiling.

Explore IVF-RQ →
+

I need the smallest index

Choose IVF-PQ when its corpus-specific recall passes the gate. It is the capacity-first option, not the automatic middle ground.

Explore IVF-PQ →
+

I need compact batch speed

Choose IVF-SQ when one byte per dimension fits and its measured 0.80–0.86 recall is enough; it is the fastest compact scanner here.

Explore IVF-SQ →
+

Raw vectors exceed RAM but fit local SSD

Evaluate DiskANN for immutable L2, IP, or cosine data when high recall and sub-MiB query reads justify a much slower build; retain IVF-FLAT as the metric-specific accuracy control.

Explore DiskANN →
+

Data lives in S3, OSS, or HDFS

Prefer durable publication plus a complete local SSD cache. For direct remote reads, start with a compact IVF index when one-round scans meet recall; use DiskANN only after measuring its corpus-dependent coalesced graph rounds.

Compare deployment modes →
-

How parameters interact

Build parameters define static structures; query parameters define per-request work. Changing nlist usually changes the useful nprobe range as well.

+

How parameters interact

Build parameters define static structures; query parameters define per-request work. For IVF, changing nlist usually changes the useful nprobe range. DiskANN instead couples graph build quality with online l_search.

+
Automate numeric work, keep semantics explicitindex.type and metric remain required because changing either changes persistence and result meaning. Rust callers can use recommend_index as an advisory starting point and must explicitly accept its result. For measured offline sweeps, select_calibrated_candidate chooses the smallest candidate satisfying supplied recall, byte, and build-time objectives and returns no result when the sample does not meet them.
- - - - - - - - - + + + + + + + + + + + + + + + + +
ParameterStageIndexesTypical effect when increasedConstraint / default
nlistBuildAllShorter lists and more coarse centroids; a fixed nprobe covers less of the collectionRequired, > 0
nprobeQueryAllReads more lists; recall usually rises with latency and I/OProvided by caller
pq.mBuildIVF-PQLonger codes, often lower quantization error, more scan workRequired; d % m == 0
use-opqBuildIVF-PQAdds training and matrix cost; may improve PQ qualityDefault false
hnsw.mBuildHNSW variantsDenser graph, more build work and graph bytes, higher recall potentialDefault 20
hnsw.ef-constructionBuildHNSW variantsWider build search, usually better graph quality and slower buildsDefault 150
hnsw.max-levelBuildHNSW variantsAllows higher navigation layers; it is not a query budgetDefault 7
ef_searchQueryHNSW variantsWider traversal, usually higher recall and latency0 means max(top_k, 32)
query_bitsQueryIVF-RQ only4/8 use quantized-query bitwise scans; results depend on data and hardware0, 4, or 8
index.typeBuildAllChanges the persisted algorithm and its capability boundaryRequired; recommendation is advisory, never silently applied
metricBuildAllChanges training, ranking, and ground-truth semanticsRequired and never inferred
dimensionBuildAllChanges representation width and distance workInferred by Java/Python one-shot training; required by streaming APIs
nlistBuildIVF familiesShorter lists and more coarse centroids; a fixed nprobe covers less of the collectionAuto: nearest power of two around √N, with at least 64 rows of training density per list; requires expected-vector-count
nprobeQueryIVF familiesReads more lists; recall usually rises with latency and I/OAuto is K-, N-, nlist-, and filter-selectivity-aware; explicit values are expert overrides
pq.code-ratioBuildIVF-PQ, DiskANNRaises or lowers the automatically inferred code bytes and subquantizer countDefault 0.0625; finite and positive
pq.mBuildIVF-PQ, DiskANNExpert override for the inferred subquantizer count; larger values often reduce quantization error but add lookup workOptional; d % m == 0
rq.bitsBuildIVF-RQMore persisted bit planes improve reconstruction and usually recall while increasing file bytes, I/O, and scan workAuto from max-bytes-per-vector; otherwise 4
use-opqBuildIVF-PQAdds training and matrix cost; may improve PQ qualityAuto enables at target-recall ≥ 0.9; explicit true/false wins
target-recallBuild objectiveIVF-PQ, DiskANNSelects OPQ and a coherent DiskANN build presetStarting policy only; validate measured recall on held-out queries
max-bytes-per-vectorBuild objective and preflight boundIVF-PQ, IVF-RQ, DiskANNReduces code width and may select 4-bit/F16 DiskANN storage; rejects configurations whose conservative persisted-size estimate exceeds the boundIncludes estimated row bytes and, when expected-vector-count is set, amortized fixed data; not an exact final-file-size promise
max-build-secondsOffline calibration objectiveMeasured candidate setsRejects candidates whose measured build time exceeds the targetAccepted by VectorIndexBuildPlan; direct Trainer creation rejects it because build time cannot be safely guessed from hardware
diskann.build-presetBuildDiskANNMoves together across degree, construction width, encoding, and build distancefast_build, balanced, or high_recall; inferred from target recall
deployment-profileBuildDiskANNSelects interleaved layout for eligible memory/local serving and compact layout for remote/object servingExplicit layout/encoding/build-distance overrides always win
estimated_random_read_latency_nanosReader input capabilityDiskANNSelects the internal read window, graph beam, and automatic cache partition without probe I/O0 measures the mandatory header read; positive values are useful for known remote/cache latency
l_searchQueryDiskANNLarger DiskANN candidate list, usually higher recall and latencyAuto uses calibrated 100/200/400 when available, otherwise max(100, 2k)
memory_budget_bytesReaderDiskANNControls required resident state plus automatically partitioned adjacency/raw-vector caches4 GiB; cache sub-budgets are internal
-

Data-lake storage and I/O

Every file begins with a 64-byte v1 header, followed by model sections, an offset table, and list payloads. The Reader dispatches on the first four-byte magic and uses the offset table for positional reads.

-
Byte orderLittle-endian
Type discoveryFirst 4-byte magic
Row IDsSorted delta-varints
IntegrityOuter Paimon file layer
+

Data-lake storage and I/O

IVF files begin with a 64-byte v1 header and use model/list sections. DiskANN uses a 256-byte header, page-aligned resident/adjacency data, and either densely packed compact vector records or interleaved page-contained records. The Reader dispatches on the first four-byte magic and uses positional reads for both layouts.

+
Byte orderLittle-endian
Type discoveryFirst 4-byte magic
Row IDsIVF delta varints / DiskANN adaptive packed
IntegrityOuter Paimon file layer
Format boundaryv1 files have no footer, checksum, compression envelope, or schema registry. Roaring filters are query payloads and are not embedded. Readers reject unknown versions, required flags, non-zero reserved bytes, and malformed sections.
@@ -154,9 +251,9 @@

Choose by constraint

1

Build ground truth

Generate exact top K using the production metric, realistic filters, and edge cases such as zero vectors.

2

Measure IVF-FLAT

Establish the recall ceiling caused by probing only nprobe lists and record bytes read.

-
3

Compare compression

Use the same nlist/nprobe for IVF-PQ and IVF-RQ to isolate added approximation and storage savings.

-
4

Add graphs only when needed

If list scans dominate P95/P99, compare HNSW-FLAT and HNSW-SQ, including build time and graph bytes.

-
5

Apply production budgets

Set thresholds for Recall@K, P99, file size, build time, and remote bytes rather than optimizing average latency alone.

+
3

Compare compression

Use the same nlist/nprobe for IVF-SQ, IVF-PQ, and IVF-RQ to isolate added approximation and storage savings.

+
4

Evaluate disk-backed search

When raw data exceeds RAM, compare DiskANN on local SSD and realistic remote storage, including cold/warm caches and read rounds.

+
5

Apply production budgets

Set thresholds for Recall@K, P99, file size, build time, RSS, and remote bytes rather than optimizing average latency alone.

diff --git a/docs/ivf-flat.html b/docs/ivf-flat.html index e65cb98..fbcb649 100644 --- a/docs/ivf-flat.html +++ b/docs/ivf-flat.html @@ -9,11 +9,11 @@ IVF-FLAT · Paimon Vector Index -
VIPaimon Vector Index
+
VIPaimon Vector Index

Exact scan inside selected lists

IVF-FLAT

Use IVF coarse partitions to reduce the candidate set, then retain and scan full f32 vectors inside each selected list. IVF-FLAT is the clearest baseline for separating partition recall loss from quantization loss.

No in-list quantizationAbout 4d bytes / vectorSimple buildMagic: IVFL
- +

Positioning and trade-offs

@@ -25,9 +25,17 @@

Positioning and trade-offs

Design and data flow

Build

-
1

Preprocess training vectors

Cosine mode applies L2 normalization. L2 and inner product keep the input representation.

2

Train coarse centroids

K-Means produces nlist × d f32 centroid values.

3

Assign vectors

Each vector enters its nearest coarse list with its complete processed vector and row ID.

4

Sort and serialize

Each non-empty list is sorted by signed row ID; IDs use delta-varints and vectors follow the same order.

+
1

Preprocess training vectors

Cosine mode applies L2 normalization. L2 and inner product keep the input representation.

2

Train coarse centroids

K-Means produces nlist × d f32 centroid values.

3

Assign vectors

Each vector enters its nearest coarse list with its complete processed vector and row ID.

4

Sort and serialize

The writer retains compact sort permutations and delta-varint IDs, then materializes and writes one sorted raw-vector list at a time instead of duplicating every list in memory.

Search

-
  1. Apply the same preprocessing used during construction.
  2. Measure the query against every IVF centroid and select the closest nprobe lists.
  3. Read those list payloads through the offset table.
  4. Decode row IDs and compute the true metric against every raw vector. A Roaring filter skips disallowed IDs.
  5. Merge all list results in a top-K heap. Missing entries are padded with -1 / f32::MAX.
+
  1. Apply the same preprocessing used during construction.
  2. Measure the query against every IVF centroid and select the closest nprobe lists.
  3. Read those list payloads through the offset table in bounded concurrent multi-range calls.
  4. Decode row IDs and compute the true metric against every raw vector. A Roaring filter skips disallowed IDs.
  5. For at least 1,048,576 distance components, scan independent lists on Rayon workers. Batch search keeps list-major locality and scans every loaded list for all queries that selected it.
  6. Merge list-local results in the original list order. Missing entries are padded with -1 / f32::MAX.
+
+ +
+

Public-corpus measurements

+

Apple M4 Pro, 12 Rayon workers, one million SIFT/GIST vectors or 1,183,514 GloVe vectors, 1,000 published queries, nlist=1024, nprobe=64, Top-10, and warm APFS pages. Times are release-build measurements from 25 July 2026.

+
DatasetRecall@10Local P95Local batch QPSRead / query
SIFT1M, 128d0.99371.88 ms8,51033.19 MiB
GIST1M, 960d0.954911.38 ms875283.40 MiB
GloVe-100, 100d0.88321.40 ms9,50227.57 MiB
+
Latest complete same-file rerunThe Reader receives each list directly into an f32-aligned allocation and scans the raw-vector suffix without allocating a second decoded payload. An internal prefix of at most three bytes compensates for the variable-length row-ID prefix; the persisted bytes do not change. Together with the strict partial-L2 cutoff, this changed SIFT/GIST/GloVe batch throughput from 6,570 / 559 / 6,345 to 8,510 / 875 / 9,502 QPS and P95 from 5.31 / 47.04 / 4.76 ms to 1.88 / 11.38 / 1.40 ms. Recall, file bytes, and read bytes stayed identical. The table reports the complete rerun rather than a best-of result.
+

The remote model groups every query's selected ranges into calls capped at 64 MiB. SIFT/GloVe average one round; GIST averages five because its raw-vector payload is much larger. Fixed 2/20 ms latency therefore produced P95 7.31/21.80 ms on SIFT, 29.73/130.49 ms on GIST, and 6.60/21.22 ms on GloVe. Batch throughput was 6,763/3,076 QPS on SIFT, 846/349 on GIST, and 7,394/2,948 on GloVe. These modeled numbers do not charge bandwidth, so they should not be read as evidence that transferring tens or hundreds of MiB directly from an object store is cheap.

@@ -60,7 +68,7 @@

Usage

Parameters

-
ParameterRequirementPurposeEffect when increased
dimensionRequired, > 0Input dimensionLinearly increases compute and vector payload
nlistRequired, > 0IVF partition countShorter average lists and a larger centroid table; recalibrate nprobe
metricl2, inner_product, or cosine; default L2Training, assignment, and search distanceNot a numeric control; it must match ground truth
top_kQuery-time, > 0Requested resultsIncreases heap and output work
nprobeQuery-time, > 0Lists to probeUsually improves recall while increasing scan work and I/O
+
ParameterRequirementPurposeEffect when increased
dimensionInferred by Java/Python one-shot training; otherwise required and > 0Input dimensionLinearly increases compute and vector payload
nlistAuto from expected-vector-count, or explicit > 0IVF partition countShorter average lists and a larger centroid table; automatic nprobe follows the resolved value
metricRequired: l2, inner_product, or cosineTraining, assignment, and search distanceSemantic, not inferred; it must match ground truth
top_kQuery-time, > 0Requested resultsIncreases heap and output work
nprobeAutomatic by default; explicit 1 to nlistLists to probeAuto accounts for K, average list size, and filter selectivity; explicit values remain available for measured overrides
@@ -70,7 +78,7 @@

v1 storage layout

Fixed 64-byte header

OffsetSizeFieldDescription
04magic: u32IVFL
44version: u32Currently 1
84dimension: i32d
124nlist: i32IVF list count
164metric: u320=L2, 1=IP, 2=Cosine
208total_vectors: i64Total vector count
284flags: u32Bit 0: delta-varint IDs; required in v1
3232ReservedMust be all zero

Offset table and list payloads

-

Each offset entry is (offset: i64, count: i32, id_bytes_len: i32), or 16 bytes. A non-empty list contains:

+

Each offset entry is (offset: i64, count: i32, id_bytes_len: i32), or 16 bytes. The Reader validates that list counts match the header before serving a query. A non-empty list contains:

FieldType / sizeDescription
base_idi64First sorted row ID
id_bytes_leni32Encoded ID stream length
id_bytesVariableOne unsigned LEB128 delta per ID; the first delta is zero
vectorscount × d × f32Raw vectors in sorted-ID order
@@ -82,12 +90,12 @@

Capacity estimate

Tuning order

-
  1. Fix the metric. Build exact ground truth with the production metric and include zero-vector edge cases for cosine.
  2. Choose an average list size. Use N / nlist to balance long scans against centroid metadata and many small reads.
  3. Sweep nprobe. Record Recall@K, P95/P99, selected lists, and bytes read.
  4. Measure batch search. Readers submit batched positional ranges, which often matters more than single-query latency on object stores.
  5. Only then compress or add graphs. Move to IVF-PQ/RQ for space; evaluate HNSW when per-list CPU is the bottleneck.
+
  1. Fix the metric. Build exact ground truth with the production metric and include zero-vector edge cases for cosine.
  2. Start automatic. Supply the final corpus count, inspect the resolved nlist, and use automatic query width.
  3. Calibrate only if needed. Sweep explicit nprobe around the automatic value and record Recall@K, P95/P99, selected lists, and bytes read.
  4. Measure batch search. Readers submit bounded multi-range batches, which often matters more than single-query latency on object stores.
  5. Only then compress. Try IVF-SQ for a one-byte-per-dimension scan, IVF-PQ for stronger compression, or IVF-RQ for the strongest measured compact-IVF recall.

Implementation boundaries

-
  • Index files do not contain checksums; the outer Paimon file and manifest layer provides integrity.
  • Roaring64 filters are query payloads. Negative row IDs cannot match the RoaringTreemap domain.
  • Readers reject unknown versions, non-zero reserved bytes, unknown flags, negative counts, and out-of-bounds sections.
  • optimizeForSearch() does not change files or results; for IVF-FLAT it primarily preloads metadata.
+
  • Index files do not contain checksums; the outer Paimon file and manifest layer provides integrity.
  • Roaring64 filters are query payloads. Negative row IDs cannot match the RoaringTreemap domain.
  • Readers reject unknown versions, non-zero reserved bytes, unknown flags, negative counts, mismatched total counts, and out-of-bounds sections.
  • The search-only list payload owns an aligned f32 allocation and reads bytes into it directly; the public list-materialization API still returns owned row IDs and vectors.
  • optimizeForSearch() does not change files or results; for IVF-FLAT it preloads centroids and the offset table in one contiguous read. The unified Reader reuses its type-dispatch header, so open plus metadata initialization takes two read rounds.
diff --git a/docs/ivf-hnsw-flat.html b/docs/ivf-hnsw-flat.html deleted file mode 100644 index 61cd7f4..0000000 --- a/docs/ivf-hnsw-flat.html +++ /dev/null @@ -1,109 +0,0 @@ - - - -IVF-HNSW-FLAT · Paimon Vector Index - - -
VIPaimon Vector Index
-
-

A local HNSW graph in every IVF list

IVF-HNSW-FLAT

Retain IVF-FLAT raw vectors but replace the default linear list scan with one HNSW graph per non-empty IVF list. Graph traversal reduces distance evaluations before candidates are merged across lists.

Raw f32 vectorsOne graph per listTunable ef_searchMagic: IHFL
-
- -
-
-

Positioning and trade-offs

-
Vector representationd × f32
In-list candidatesApproximate HNSW traversal
Extra storageLayered adjacency graph
BuildParallel graphs before write
-

Good fit

  • IVF-FLAT list distance calculations dominate CPU.
  • Candidate distances should still use raw vectors.
  • Lists are large enough to amortize graph overhead.
  • Slower construction and larger files are acceptable.

Poor fit

  • File size or remote bytes are the main constraint.
  • Lists are already short and cheap to scan.
  • Vectors are appended frequently and graphs must stay current.
  • Build time and peak memory are tightly constrained.
-
This is not one global HNSWIVF first selects nprobe lists, then searches each local graph. Recall is constrained by both IVF list selection and each graph’s ef_search.
-
- -
-

Design and data flow

-

Build

-
1

Train IVF

As in IVF-FLAT, normalize for cosine and learn nlist coarse centroids.

2

Store raw vectors

Assign each vector to its nearest list with the full processed vector and row ID.

3

Build list graphs

Before serialization, the Writer builds one independent HNSW per non-empty list; lists build in parallel.

4

Sort and remap

Vectors are sorted by row ID and local graph node references are remapped to the same order.

5

Serialize adjacency

Store entry point, levels, and delta-varint adjacency groups.

-

Search

-
  1. Select the nearest nprobe IVF lists.
  2. Read each selected list’s IDs, raw vectors, and graph section, then reconstruct the local graph.
  3. Navigate from the entry point through upper levels; search level 0 with the ef_search candidate width.
  4. Merge candidates from all selected graphs in one deduplicating top-K heap.
-

The unified API resolves ef_search=0 to max(top_k, 32). The graph implementation also ensures the effective width is at least top_k.

-
- -
-

Why filtering can fall back to a scan

-

A filter controls which row IDs may enter the result, but filtered entry or path nodes must not make distant valid points unreachable. The shared HNSW search uses two safeguards:

-
  1. If the number of allowed IDs across all probed lists is ≤ max(ef_search, top_k), scan those lists directly with the filter.
  2. Otherwise search the graphs first. If fewer than top_k filtered results remain, scan all probed lists to backfill.
-
Accuracy meaningThe fallback uses raw vectors, so distances are exact within the probed lists and allowed ID set. Unprobed IVF lists remain outside the candidate set.
-
- -
-

Usage

-
Java · build
Map<String, String> options = new HashMap<>();
-options.put("index.type", "ivf_hnsw_flat");
-options.put("dimension", "128");
-options.put("nlist", "1024");
-options.put("metric", "l2");
-options.put("hnsw.m", "20");
-options.put("hnsw.ef-construction", "150");
-options.put("hnsw.max-level", "7");
-
-try (VectorIndexTraining training =
-             VectorIndexTrainer.train(options, trainingVectors, trainingCount);
-     VectorIndexWriter writer = new VectorIndexWriter(training)) {
-    writer.addVectors(rowIds, vectors, vectorCount);
-    writer.writeIndex(vectorIndexOutput); // builds every list graph first
-}
-
Java · search
try (VectorIndexReader reader = new VectorIndexReader(vectorIndexInput)) {
-    VectorSearchParams params = new VectorSearchParams(10, 16)
-            .withEfSearch(80);
-    VectorSearchResult result = reader.search(query, params);
-}
-
Rust · configuration
let config = VectorIndexConfig::IvfHnswFlat {
-    dimension: 128,
-    nlist: 1024,
-    metric: MetricType::L2,
-    hnsw: HnswBuildParams::default(), // m=20, ef=150, max_level=7
-};
-let params = VectorSearchParams::with_ef_search(10, 16, 80);
-
- -
-

Parameters

-
ParameterDefault / requirementPurposeTypical effect when increased
nlist> 0Graph count and average list sizeMore, smaller graphs; fixed nprobe covers less of the collection
hnsw.m20, > 0Maximum high-level degree; level 0 allows 2mDenser graph, more bytes and build work, higher recall potential
hnsw.ef-construction150, > 0Candidate width while inserting nodesSlower builds and usually better graph quality
hnsw.max-level7, > 0Random level capMay add upper navigation layers; not a query depth
ef_search0 → max(k,32)Search width in each graphMore distances and usually better in-list recall
nprobeQuery parameterGraphs searchedMore complete list payloads read and traversed
-
- -
-

v1 storage layout

-
64 B headerHNSW parameters and flags
IVF centersnlist × d × f32
Offset tablenlist × 24 B
ListsIDs + raw vectors + graph
-

Fixed 64-byte header

-
OffsetSizeFieldDescription
04magicIHFL / 0x4948464C
44version1
84dimensiond
124nlistList count
164metric0=L2, 1=IP, 2=Cosine
208total_vectorsTotal count
284hnsw.mGraph degree parameter
324ef_constructionBuild width
364max_levelLevel cap
404flagsBit 0 delta IDs; bit 1 v1 graph encoding; both required
4420ReservedMust be zero
-

Each offset entry is (offset: i64, count: i32, graph_bytes_len: i32, payload_bytes_len: i64). A non-empty list stores its ID stream, count × d × f32 raw vectors, then the graph section.

-

Shared HNSW graph section

-
FieldEncodingDescription
graph_magicu32HWGR / 0x48574752
graph_versionu32Currently 1
graph_flagsu32Bit 0 delta-varint adjacency; required
graph_countVarintLocal node count
entry_pointVarintEntry local ID
max_observed_levelVarintActual highest level
level[node]One varint per nodeNode’s highest level
Adjacency groupsDegree + neighbor deltasSorted neighbors encoded as unsigned LEB128 deltas
-
- -
-

I/O and capacity

-
Important Reader behaviorThe graph does not make an object store return only visited nodes. The Reader first loads each selected list’s complete payload—IDs, all raw vectors, and graph—then traverses the graph in memory. HNSW reduces distance calculations, not selected-list bytes.
-

Approximate size is 64 + 4×nlist×d + 24×nlist + N×4d + encoded_ids + graph_bytes. Graph size depends on actual levels, degrees, and varint lengths and should be measured on representative data.

-

Adjacent selected-list ranges may be coalesced to reduce remote requests, potentially reading the gap between them.

-
- -
-

Tuning order

-
  1. Measure IVF-FLAT with identical nlist/nprobe and confirm that CPU scans, rather than remote I/O, are the bottleneck.
  2. Keep default build parameters and sweep ef_search first.
  3. If high ef_search still misses the target, increase hnsw.m or ef-construction and rebuild.
  4. Benchmark filtered queries independently; selective filters may trigger scans and have a different latency profile.
  5. Record graph bytes, build peak memory, and serialization time alongside online latency.
-
- -
-

Implementation boundaries

-
  • Every add invalidates in-memory graphs; the unified Writer rebuilds all list graphs before write.
  • Every non-empty v1 list must contain a graph. Readers reject missing or zero-length graph payloads.
  • Graph nodes use list-local IDs while row IDs are encoded separately. Serialization remaps both consistently after sorting.
  • optimize_for_search() primarily loads metadata and does not keep every graph resident.
- -
-
-
-
-
Apache Paimon Vector IndexIVF-HNSW-FLAT · v1
- - diff --git a/docs/ivf-hnsw-sq.html b/docs/ivf-hnsw-sq.html deleted file mode 100644 index 782c5c7..0000000 --- a/docs/ivf-hnsw-sq.html +++ /dev/null @@ -1,112 +0,0 @@ - - - -IVF-HNSW-SQ · Paimon Vector Index - - -
VIPaimon Vector Index
-
-

Scalar-quantized residuals plus local HNSW

IVF-HNSW-SQ

Quantize each IVF-list residual to one byte per dimension, then build a local HNSW over decoded vectors. This reduces the main vector payload from 4d to d bytes while retaining graph-based list search.

1 byte / dimensionPer-list SQ boundsOne HNSW per listMagic: IHSQ
-
- -
-
-

Positioning and trade-offs

-
Vector payloadd bytes
Quantization scopePer-list, per-dimension bounds
In-list candidatesHNSW over decoded vectors
Approximation sourcesIVF + SQ + graph
-

Good fit

  • HNSW-style list search is useful, but raw-vector files are too large.
  • One-byte-per-dimension accuracy meets the recall target.
  • Lists are large enough that linear SQ scans remain expensive.
  • Complex construction is acceptable in exchange for query CPU.

Poor fit

  • Raw-vector exact distance is required.
  • Lists are too short to amortize graph overhead.
  • The smallest possible file is required; IVF-PQ/RQ are often smaller.
  • Query working memory must also remain at one byte per dimension.
-
Difference from IVF-PQSQ stores one byte for each dimension and decodes simply, but code length is fixed at d. PQ stores m subspace-centroid IDs and usually compresses further. HNSW-SQ additionally organizes decoded vectors into a graph.
-
- -
-

Design: residual SQ plus local graphs

-

Training scalar quantizers

-

Training vectors are preprocessed for the metric, assigned to IVF lists, and converted to residuals r = x − c[list]. The implementation learns both global per-dimension min/max bounds and separate per-list bounds. Non-empty lists use their local bounds; empty lists retain the global bounds as a stable fallback.

-

Each residual dimension maps linearly to 0..255. Decoding restores an approximate residual and adds the list centroid:

-
Decoded representationx̂[j] = centroid[list][j] + decode(code[j], list_min[j], list_max[j])
-

Why the graph uses decoded vectors

-

Graph edges and search distances must match a representation that can be reconstructed from the file. Before writing, the Writer decodes each list’s SQ codes and builds HNSW over x̂. The file stores codes, bounds, and adjacency—not a duplicate copy of decoded f32 vectors.

-
1

Train IVF

Learn coarse centroids and assignments for training vectors.

2

Measure residual bounds

Collect global and per-list, per-dimension min/max values.

3

Encode production vectors

Use the assigned list’s bounds to produce d code bytes.

4

Decode and build graphs

Restore x̂ and build one HNSW per non-empty list.

5

Sort and serialize

Write row IDs, codes, and graph nodes in one consistent order.

-
- -
-

Search, filtering, and warm-up

-
  1. Select the nearest nprobe IVF lists and read each complete payload.
  2. Decode codes through per-list bounds and centroids, then rebuild searchable HNSW graphs.
  3. Search decoded-vector graphs with ef_search and merge across lists.
  4. For highly selective filters or insufficient graph results, scan SQ codes as a fallback.
-

optimizeForSearch() builds one SQ decode LUT per list. These LUTs primarily accelerate filtered scans and fallback. Normal unfiltered graph traversal uses decoded vectors, so the LUT is not expected to improve it materially.

-
Recall is relative to the quantized representationEven a full fallback scan compares SQ-decoded vectors, not originals. It eliminates graph misses inside selected lists but cannot eliminate SQ error or unprobed IVF lists.
-
- -
-

Usage

-
Java · build
Map<String, String> options = new HashMap<>();
-options.put("index.type", "ivf_hnsw_sq");
-options.put("dimension", "128");
-options.put("nlist", "1024");
-options.put("metric", "l2");
-options.put("hnsw.m", "20");
-options.put("hnsw.ef-construction", "150");
-options.put("hnsw.max-level", "7");
-
-try (VectorIndexTraining training =
-             VectorIndexTrainer.train(options, trainingVectors, trainingCount);
-     VectorIndexWriter writer = new VectorIndexWriter(training)) {
-    writer.addVectors(rowIds, vectors, vectorCount);
-    writer.writeIndex(vectorIndexOutput); // decodes and builds graphs first
-}
-
Java · search and warm-up
try (VectorIndexReader reader = new VectorIndexReader(vectorIndexInput)) {
-    reader.optimizeForSearch(); // SQ scan/fallback decode LUTs
-    VectorSearchParams params = new VectorSearchParams(10, 16)
-            .withEfSearch(80);
-    VectorSearchResult result = reader.search(query, params);
-}
-
Rust · configuration
let config = VectorIndexConfig::IvfHnswSq {
-    dimension: 128,
-    nlist: 1024,
-    metric: MetricType::L2,
-    hnsw: HnswBuildParams::default(),
-};
-let params = VectorSearchParams::with_ef_search(10, 16, 80);
-
- -
-

Parameters

-
ParameterDefault / requirementPurposeTypical effect when increased
nlist> 0List, SQ-bound set, and graph countShorter average lists; linearly larger bounds and centroid metadata
hnsw.m20, > 0Adjacency densityHigher recall potential, graph bytes, and build cost
hnsw.ef-construction150, > 0Build candidate widthUsually better graphs and slower construction
hnsw.max-level7, > 0Level capMay add upper navigation layers
ef_search0 → max(k,32)Per-graph query widthMore decoded-vector distances and usually fewer graph misses
nprobeQuery parameterGraphs read and rebuiltI/O, decode work, memory, and traversal all increase
-
- -
-

v1 storage layout

-
64 B headerHNSW + SQ summary
Global bounds2 × d × f32
Per-list bounds2 × nlist × d × f32
IVF centersnlist × d × f32
Offset tablenlist × 24 B
ListsIDs + SQ codes + graph
-

Fixed 64-byte header

-
OffsetSizeFieldDescription
04magicIHSQ / 0x49485351
44version1
84dimensiond
124nlistList count
164metric0=L2, 1=IP, 2=Cosine
208total_vectorsTotal count
284hnsw.mGraph degree parameter
324ef_constructionBuild width
364max_levelLevel cap
404sq_min_summaryMinimum summary of global bounds
444sq_max_summaryMaximum summary of global bounds
484flagsBit 0 delta IDs; bit 1 v1 graph encoding
5212ReservedMust be zero
-

Model sections and list payloads

-

The header is followed by global minimums, global maximums, every list’s minimums and maximums, IVF centroids, and the offset table. Offset entries match HNSW-FLAT: (offset: i64, count: i32, graph_bytes_len: i32, payload_bytes_len: i64).

-

A non-empty list stores its ID header and stream, count × d SQ code bytes, then the shared HWGR graph section. Entry point, levels, degrees, and neighbors use the same v1 delta-varint encoding as HNSW-FLAT.

-
- -
-

Capacity, I/O, and working memory

-
Approximate file size64 + 8d + 8×nlist×d + 4×nlist×d + 24×nlist + N×d + encoded_ids + graph_bytes
-

The first three products represent global bounds, per-list bounds, and IVF centroids. For N=1,000,000, d=128, nlist=1024, SQ codes are about 122 MiB—one quarter of IVF-FLAT vector payloads—plus about 1.5 MiB of bounds and centroids, IDs, and graph bytes.

-
A smaller file does not imply one-byte query memoryThe Reader loads each selected list’s complete payload and decodes SQ codes to f32 vectors to rebuild searchable HNSW. The transient working set contains codes, decoded vectors, and the graph.
-
- -
-

Tuning order

-
  1. Compare IVF-HNSW-FLAT with identical parameters to isolate SQ recall loss and storage savings.
  2. Hold graph build parameters fixed and sweep ef_search. If high ef cannot approach HNSW-FLAT, graph density is not the limiting factor.
  3. Inspect training samples per list. Tiny lists produce unstable bounds and do not amortize graph overhead; consider reducing nlist.
  4. Benchmark filtered workloads separately after warm-up to cover the SQ LUT path.
  5. Record full-list read bytes and decode peak memory, especially for batch search and high nprobe.
-
- -
-

Implementation boundaries

-
  • SQ codes are fixed at one byte per dimension; the options API has no bit-width control.
  • Graphs are built and searched over SQ-decoded vectors, so reported distances reflect the quantized representation.
  • Every vector append invalidates graphs; the unified Writer rebuilds them before serialization.
  • Header min/max values are summaries. Complete per-dimension bounds live in sections after the header.
  • optimize_for_search primarily helps SQ scan/fallback, not ordinary unfiltered graph traversal.
- -
-
-
-
-
Apache Paimon Vector IndexIVF-HNSW-SQ · v1
- - diff --git a/docs/ivf-pq.html b/docs/ivf-pq.html index 0bab8b6..a0dfdb3 100644 --- a/docs/ivf-pq.html +++ b/docs/ivf-pq.html @@ -9,11 +9,11 @@ IVF-PQ · Paimon Vector Index -
VIPaimon Vector Index
+
VIPaimon Vector Index

Product-quantized inverted lists

IVF-PQ

Split each d-dimensional vector into m subspaces and store only one centroid ID per subspace. The unified configuration currently uses 8-bit codes, reducing the main vector payload from 4d bytes to about m bytes.

High compression8-bit PQOptional OPQMagic: IVPQ
- +

Positioning and trade-offs

@@ -45,7 +45,7 @@

Usage

options.put("dimension", "128"); options.put("nlist", "1024"); options.put("metric", "l2"); -options.put("pq.m", "16"); // 128 % 16 == 0 +options.put("pq.code-ratio", "0.0625"); // optional; default 0.0625 options.put("use-opq", "false"); // optional; default false try (VectorIndexTraining training = @@ -60,18 +60,18 @@

Usage

VectorSearchParams params = new VectorSearchParams(10, 16); VectorSearchResult result = reader.search(query, params); }
-
Rust · configuration
let config = VectorIndexConfig::IvfPq {
-    dimension: 128,
-    nlist: 1024,
-    m: 16,
-    metric: MetricType::L2,
-    use_opq: false,
-};
+
Rust · configuration
let config = VectorIndexConfig::ivf_pq(
+    128,
+    1024,
+    MetricType::L2,
+    false,
+)?;
+

The default relative code budget resolves this 128-dimensional 8-bit index to m=32. Option maps can change the target with pq.code-ratio or set pq.m explicitly. The resolved concrete m, rather than the ratio, is stored in the file.

Parameters

-
ParameterRequirement / defaultPurposeTuning meaning
dimension> 0Input dimension dMust be divisible by pq.m
nlist> 0IVF partition countControls list length, centroid cost, and the nprobe curve
pq.mRequired, > 0; d % m == 0Subspace count and 8-bit code bytesLarger values often lower quantization error but linearly increase codes and scan work
use-opqDefault falseLearn and apply an orthogonal rotationTrades training and query work for possible recall improvement
nprobeQuery parameterLists readAffects IVF recall loss and scan volume
+
ParameterRequirement / defaultPurposeTuning meaning
dimensionInferred by Java/Python one-shot training; otherwise > 0Input dimension dThe inferred or explicit pq.m must divide it
nlistAuto from expected-vector-count, or explicit > 0IVF partition countControls list length, centroid cost, and automatic nprobe
pq.code-ratioDefault 0.0625; finite and in (0, 0.25]Target ratio between PQ-code bytes and raw f32-vector bytesThe closest valid divisor is selected; max-bytes-per-vector can supply the target instead
pq.mOptional expert override; > 0; d % m == 0Concrete subspace count and 8-bit code bytesTakes precedence over automatic sizing; use after representative recall measurements
use-opqAuto enables when target-recall ≥ 0.9; explicit true/false winsLearn and apply an orthogonal rotationTrades training and query work for possible recall improvement
nprobeAutomatic by default; explicit expert overrideLists readAuto accounts for K, average list size, and filter selectivity
@@ -84,6 +84,14 @@

List payload

An offset entry is (offset: i64, count: i32, id_bytes_len: i32). A non-empty list stores base_id: i64, id_bytes_len: i32, the ID stream, then transposed codes. The 8-bit layout is codes[sub][vector]. The format also defines a 4-bit layout with two subquantizers per byte.

+
+

I/O and batching

+

The Reader loads the fixed header in one positional read and the contiguous OPQ matrix, IVF centers, PQ codebook, and offset table in one further read. The outer type dispatcher adds one small magic read. A query then supplies all selected non-empty list ranges in one capability-bounded multi-range round instead of issuing one request per field or list.

+

Batch search deduplicates the union of probed lists, sorts them by persisted offset, reads every unique list once, and scans queries in parallel. A single query uses the same physical-order read plan, which avoids backward seeks for a serial fallback SeekRead. Each query reuses its distance-table and candidate-distance scratch across lists. Internally, the Reader receives list bytes directly into a 16-byte-aligned typed allocation and exposes the persisted code suffix as a borrowed slice, avoiding a second code allocation; public list materialization continues to return owned arrays. The common one-byte row-ID delta has a checked fast path, and an 8-bit scan initializes distances from the first code column instead of zero-filling the scratch in a separate pass. The persisted codes[sub][vector] layout already matches this table-driven scan, so the improvements remain compatible with IVF-PQ v1.

+
Open-source and same-file cross-checkConventional Faiss IVF-PQ uses 8-bit subquantizers by default, prefetches selected inverted lists, and offers a list-major large-batch path. FastScan obtains register-resident tables by changing to 4-bit codes and 32-row blocks. Lance PQ storage loads partition ranges and transposes code columns. The current v1 format and physical-order multi-range reads already retain the relevant 8-bit choices. Faiss-style list-major query bucketing, parallel ID decode, one contiguous all-query table, and worker-level scratch retention were tested and removed because each regressed at least one of SIFT1M, GIST1M, and GloVe-100. In three-run same-file medians, the retained one-byte ID fast path changed batch throughput from 7,528 / 987 / 8,693 to 7,649 / 1,026 / 8,943 QPS. The final complete warm-local rerun reached 8,353 / 1,094 / 9,330 QPS at unchanged recall and bytes read. No v2 migration is justified.
+

The writer keeps only each list's row-order permutation and encoded IDs, performs a 32×32 cache-blocked transpose, and reuses one list-sized output buffer across all lists. It never retains an index-sized sorted-code copy. Same-machine SIFT/GIST serialization changed from 47 / 221 ms to 36 / 150 ms, and the reference transpose test plus v1 golden fixtures verify unchanged output bytes.

+
+

Capacity and memory estimates

Approximate 8-bit size without OPQ64 + 4×nlist×d + 4×256×d + 16×nlist + N×m + encoded_ids
@@ -93,7 +101,7 @@

Capacity and memory estimates

Tuning order

-
  1. Measure IVF-FLAT with identical nlist/nprobe to establish the no-quantization baseline.
  2. Test several divisors of d for m; compare file size, Recall@K, and P99 together.
  3. With m fixed, sweep nprobe to separate coarse-partition misses from PQ rank error.
  4. Test OPQ only when PQ loss is visible and the training budget allows it.
  5. Call optimizeForSearch() once for long-lived Readers so precomputation is outside the hot path.
+
  1. Measure IVF-FLAT with identical nlist/nprobe to establish the no-quantization baseline.
  2. Start with pq.code-ratio=0.0625, then sweep nearby ratios; compare resolved m, file size, Recall@K, and P99 together.
  3. With the PQ budget fixed, sweep nprobe to separate coarse-partition misses from PQ rank error.
  4. Test OPQ only when PQ loss is visible and the training budget allows it.
  5. Call optimizeForSearch() once for long-lived Readers so precomputation is outside the hot path.
diff --git a/docs/ivf-rq.html b/docs/ivf-rq.html index 7d8f51d..dcddf75 100644 --- a/docs/ivf-rq.html +++ b/docs/ivf-rq.html @@ -6,43 +6,35 @@ --> -IVF-RQ · Paimon Vector Index +IVF-RQ · Paimon Vector Index -
VIPaimon Vector Index
+
VIPaimon Vector Index
-

RaBitQ-style rotated residual codes

IVF-RQ

Apply a reproducible Kac rotation to each IVF residual, retain only the sign of every dimension, and estimate distance with three f32 correction factors. Storage is extremely compact and no learned quantization codebook is required.

1 bit / dimension+ 12 bytes of factorsquery_bits 0 / 4 / 8Magic: IVRQ
+

Multi-bit rotated residual quantization

IVF-RQ

Partition vectors with IVF, spread each residual through a deterministic orthogonal transform, and store centered scalar levels as bit planes. A cheap sign-plane estimate rejects weak candidates before the remaining planes are evaluated.

4 bits / dimension by default1–8 build-time bitstwo-stage scanMagic: IVRQ
- +

Positioning and trade-offs

-
Primary coded / 8 bytes
Per-vector factors3 × f32
Quantizer trainingNo learned codebook
Dimension constraintd % 8 == 0
-

Good fit

  • High-dimensional, very large collections where file size is the first constraint.
  • Limited training budget and no desire to train PQ/OPQ codebooks.
  • Aggressive distance approximation is acceptable and can be tuned against ground truth.
  • The query path should switch between float LUTs and bitwise scans.

Poor fit

  • The dimension is not a multiple of eight.
  • In-list accuracy must approach raw-vector distances.
  • Exact reranking is required but raw vectors are not stored elsewhere.
  • Consumers require unimplemented multi-bit file codes.
-
Do not confuse the two bit widthsStored vector codes are always 1 bit per dimension today. Query values query_bits=4/8 quantize the query residual for a bitwise scan; they do not change index size.
+
Default codepadded_d / 2 bytes
Per-vector factors5 × f32
Learned modelIVF centroids only
DimensionAny positive value
+

Good fit

  • You need higher recall than IVF-SQ at a smaller serialized size.
  • Training time and model complexity should stay close to IVF-FLAT.
  • The source supports one concurrent multi-range read for selected lists.
  • Approximate in-list ranking is acceptable and measured against real ground truth.

Poor fit

  • Raw-vector or exact reranking accuracy is mandatory.
  • Sub-millisecond local latency matters more than 0.90-class recall.
  • The collection is highly mutable; this implementation writes immutable files.
  • Resident PQ tables and lower recall are acceptable in exchange for a still smaller IVF-PQ index.
+
Why it remains IVF-RQThe public index family name is unchanged. The pre-release 1-bit/query-bit experiment was replaced completely: bit width is now a property of persisted data, not a per-query switch.

Design and distance estimation

-

Residuals and rotation

-

A vector is assigned to its nearest IVF centroid and converted to residual r = x − c. A deterministic Kac rotation derived from (d, rotation_seed, rotation_rounds) spreads residual energy more evenly across dimensions. The parameters live in the header, so a Reader reconstructs the exact transform without an external model.

-

Per-vector data

-
  • Sign code: one positive/negative bit for every rotated residual dimension.
  • residual_norm_sqr: squared residual norm.
  • vector_norm_sqr: squared norm of the processed vector.
  • dp_multiplier: scale factor used to reconstruct the approximate inner product.
-
1

Train IVF

Learn only the nlist coarse centroids. Rotation is seed-derived, not learned.

2

Compute residuals

Subtract the assigned centroid after metric preprocessing.

3

Apply Kac rotation

Mix dimensions through deterministic rounds of paired rotations.

4

Encode signs

Pack eight dimension signs into each byte and compute three factors.

5

Approximate distance

Rotate the query residual identically and combine sign sums with the factors.

-
- -
-

The three query_bits modes

-
ValueQuery representationScan pathGuidance
0Float rotated query residualLists with at least 64 entries build a 256-entry signed-sum LUT for each code byte; shorter lists sum dimensions directlyDefault; use it as the recall and latency baseline
4One sign plane + three magnitude planesBitwise and popcount approximationCompact query representation with additional query quantization error
8One sign plane + seven magnitude planesMore popcount plane accumulationsFiner than 4-bit queries, with more scan work
-

Only 0, 4, and 8 are accepted, and non-zero values are valid only for IVF-RQ. Passing them to another index is rejected by the unified Reader.

+
1

Train and assign IVF

Learn nlist coarse centroids and form r = x − c.

2

Rotate deterministically

Pad to a multiple of 64, then apply four rounds of random signs, 64-wide normalized FHT, and seeded permutation. The transform is orthogonal and reconstructed from the header seed.

3

Quantize centered levels

Choose 1–8 bits and refine one per-vector scale for three rounds. Store levels MSB first as bit planes.

4

Estimate in two stages

The MSB sign plane and its three factors produce an estimate plus a deterministic reconstruction-error bound. Only candidates whose lower bound can enter Top-K evaluate all planes and the full factors.

5

Scan blocked data

Codes and factors are transposed within 32-vector blocks. The rotated query and byte LUT are built once and reused across every selected list.

+

The full estimator is exact for a query equal to the encoded source vector, while ranking other vectors through the scaled reconstruction. L2, inner product, and cosine have metric-specific additive/rescale factors; cosine inputs are normalized before IVF assignment.

Usage

-
Java · build
Map<String, String> options = new HashMap<>();
+          
Java · build and query
Map<String, String> options = new HashMap<>();
 options.put("index.type", "ivf_rq");
-options.put("dimension", "128"); // must be divisible by 8
+options.put("dimension", "128");
 options.put("nlist", "1024");
+options.put("rq.bits", "4"); // optional; 4 is the default
 options.put("metric", "l2");
 
 try (VectorIndexTraining training =
@@ -50,53 +42,63 @@ 
Usage

      VectorIndexWriter writer = new VectorIndexWriter(training)) {
     writer.addVectors(rowIds, vectors, vectorCount);
     writer.writeIndex(vectorIndexOutput);
-}

-          
Java · compare query precision
try (VectorIndexReader reader = new VectorIndexReader(vectorIndexInput)) {
-    VectorSearchParams floatQuery = new VectorSearchParams(10, 16);
-    VectorSearchParams query4Bit = floatQuery.withQueryBits(4);
-    VectorSearchParams query8Bit = floatQuery.withQueryBits(8);
+}
 
-    VectorSearchResult baseline = reader.search(query, floatQuery);
-    VectorSearchResult fast4Bit = reader.search(query, query4Bit);
-    VectorSearchResult fine8Bit = reader.search(query, query8Bit);
+try (VectorIndexReader reader = new VectorIndexReader(vectorIndexInput)) {
+    VectorSearchResult result =
+            reader.search(query, new VectorSearchParams(10, 64));
+    int storedBits = reader.metadata().rqBits();
 }

-          
Rust · configuration and search
let config = VectorIndexConfig::IvfRq {
+          
Rust · configuration
let config = VectorIndexConfig::IvfRq {
     dimension: 128,
     nlist: 1024,
+    bits: 4,
     metric: MetricType::L2,
 };
-let params = VectorSearchParams::with_query_bits(10, 16, 4);

+let params = VectorSearchParams::new(10, 64);

Parameters

-
ParameterRequirement / defaultPurposeNote
dimension> 0 and divisible by 8Vector dimensionValidated before construction
nlist> 0IVF partition countMust still be tuned with nprobe
metricDefault L2L2 / IP / cosine estimatorCosine vectors are normalized first
query_bits0 (default), 4, or 8Query residual representationQuery-only; never serialized
nprobeQuery parameterLists probedIVF misses and RQ rank error are separate recall sources
+
ParameterRequirement / defaultPurposeGuidance
dimensionInferred by Java/Python one-shot training; otherwise > 0Logical vector dimensionStorage pads internally to a multiple of 64.
nlistAuto from expected-vector-count, or explicit > 0IVF partition countCompare the resolved value with the same IVF-FLAT baseline.
rq.bits1–8; auto from max-bytes-per-vector, otherwise 4Persisted residual level widthHigher values increase recall, file bytes, I/O, and scan work linearly.
metricRequiredL2 / inner product / cosineSemantic, not inferred; fixed in the file.
nprobeAutomatic by default; explicit expert overrideLists probedAuto accounts for K, average list size, and filter selectivity.
+
No query-side bit widthThe Reader always evaluates the representation stored in the file. Changing rq.bits requires rebuilding the index; this keeps one file's accuracy and cost contract stable.
+
+ +
+

Public-corpus measurements

+

Apple M4 Pro, 12 Rayon workers, one million base vectors, 1,000 published queries, nlist=1024, nprobe=64, Top-10, warm APFS pages. Times are release-build measurements from 25 July 2026.

+
DatasetBuildFileRecall@10Local P95Local batch QPSRead / query
SIFT1M, 128d3.92 s86.3 MB0.91481.20 ms3,0745.81 MB
GIST1M, 960d23.5 s505.8 MB0.90394.41 ms44438.81 MB
GloVe-100, 100d4.03 s102.1 MB0.82031.23 ms2,9176.18 MB
+

On SIFT1M, a controlled bit sweep produced Recall@10 0.7233 / 0.8365 / 0.9149 / 0.9530 / 0.9731 for 2 / 3 / 4 / 5 / 6 bits. The compact v1 factor layout removes four unused bytes per row, reducing the corresponding files from 58.3 / 74.3 / 90.3 / 106.3 / 122.3 MB to approximately 54.3 / 70.3 / 86.3 / 102.3 / 118.3 MB without changing the distance estimator. Four bits is the first point above 0.90 and is therefore the default.

+
GIST1M storage modelRecall@10P95Batch QPSQuery rounds
Warm local storage0.90394.41 ms4441
Remote cache, fixed 2 ms0.90397.80 ms4341
Object store, fixed 20 ms0.903924.34 ms3921
+
What changed in this runRows are assigned to IVF lists once, then independent lists are encoded in parallel with task-local residual, rotation, code, and factor scratch. The unfiltered scan removes the per-lane filter test from coarse-code accumulation, and the final estimator checks the Top-K threshold before entering the duplicate-aware heap. Relative to the immediately preceding same-machine build, add time fell 39% / 45% / 42% on SIFT/GIST/GloVe; a strict GIST same-file query A/B improved batch time by 8.7%. Rebuilt file hashes match the serial baseline exactly, so these are execution-path changes rather than a format or estimator change.
+

The fixed-latency rows model one concurrent multi-range round and do not model bandwidth, retries, TLS, or throttling. Batch QPS is one 1,000-query call, not independent clients.

v1 storage layout

-
64 B headerRotation and format fields
IVF centersnlist × d × f32
Offset tablenlist × 16 B
ListsIDs + sign codes + factors
-

Fixed 64-byte header

-
OffsetSizeFieldDescription
04magicIVRQ / 0x49565251
44version1
84dimensiond
124nlistList count
164metric0=L2, 1=IP, 2=Cosine
204flagsBit 0: delta-varint IDs
248total_vectorsTotal count
328rotation_seedDeterministic rotation seed
404rotation_roundsRotation rounds
444code_sizeceil(d / 8)
484num_bitsMust currently equal 1
524rotation_type1 = deterministic Kac
564factor_layout1 = three f32 factors
604rq_format_flagsOptional section flags; current Writer emits zero
-

List payload

-
FieldSizeDescription
IDsbase_id + len + varintsSorted delta-varint row IDs
codescount × ceil(d/8)Rotated-residual sign plane
ex_codesOptionalReserved for future multi-bit codes; not written today
factorscount × 3 × f32(residual_norm_sqr, vector_norm_sqr, dp_multiplier)
error_factorOptional count × f32Reserved refinement data; not written today
+
64 B headerShape, bits, transform, layout
IVF centersnlist × d × f32
Offset tablenlist × 16 B
ListsIDs + blocked planes + blocked factors
+

The header records logical and padded dimensions, metric, required layout flags, persisted rq.bits, total vectors, rotation seed/rounds, plane bytes, rotation type 2, and compact factor layout 3. Every non-empty list begins with base_id, encoded-ID length, and code length, followed by sorted delta-varint IDs.

+

Within each 32-vector block, bytes are ordered by plane, byte position, then lane. For more than one bit, the five structure-of-arrays fields are coarse (f_add, f_rescale, f_error) followed by full (f_add, f_rescale). The omitted full error was never read: the full estimate is the final ranking stage and does not produce another lower bound. Readers reject the old pre-release layouts rather than guessing their meaning.

+
Open-source cross-checkFaiss IVFRaBitQ FastScan also groups database vectors in 32-lane blocks and separates RaBitQ correction factors from packed codes. Lance's RaBitQ kernels likewise use blocked multi-bit codes and architecture-specific dot products. The v1 plane/byte/lane layout and SoA factors already preserve the important scan locality, while sorted delta-varint row IDs avoid fixed-width ID overhead. A quantized-LUT SIMD port remains future work because its rounding error must be incorporated into the conservative first-stage bound; the current release keeps exact F32 table sums.
+
Why the remaining factors stay F32Faiss additive quantizers can encode norms with qint8 or qint4. IVF-RQ cannot apply that choice blindly to its coarse factors: their error term makes the first-stage lower bound conservative, and inward rounding could incorrectly prune a true Top-K candidate. v1 takes the lossless 4-byte saving by deleting only the unused final-stage error; lower-precision factor encodings remain gated on a proof-preserving rounding rule and public-corpus recall data.
+
I/O contractOpen reads the 64-byte header once. Resident initialization reads centroids plus the offset table in one contiguous round. Search groups selected list payloads into at most 64 MiB per round and also honors SeekReadCapabilities.max_ranges_per_pread. Each returned payload remains the backing allocation for its blocked codes, so the reader decodes only IDs and factors instead of copying the usually much larger code region.

Capacity estimate

-
Approximate current 1-bit size64 + 4×nlist×d + 16×nlist + N×(d/8 + 12) + encoded_ids
-

For N=1,000,000, d=128, nlist=1024, codes and factors use 28 × N bytes, or about 26.7 MiB. This is larger than the code-only payload of IVF-PQ with m=16 because every RQ vector carries 12 bytes of factors, but RQ needs no PQ codebook.

+
Approximate default size64 + 4×nlist×d + 16×nlist + N×(4×padded_d/8 + 20) + encoded_ids
+

For one bit, only the two estimate factors are stored, so the factor term is 8 bytes. For 2–8 bits it is 20 bytes. The formula excludes small per-list headers and delta-varint ID variability.

Tuning order

-
  1. Use IVF-FLAT to fix nlist/nprobe and measure coarse-partition loss.
  2. Use query_bits=0 as the RQ baseline.
  3. Test 4 and 8 independently. Do not assume 4 is always fastest or 8 always has the best recall; dimension, hardware, and list length matter.
  4. Inspect short and long lists separately. The float path only builds byte LUTs for lists with at least 64 entries.
  5. If recall remains insufficient, compare IVF-PQ rather than increasing nprobe merely to mask quantization rank error.
+
  1. Run IVF-FLAT with the target nlist/nprobe to establish the partition recall ceiling.
  2. Start IVF-RQ at the default four bits.
  3. If recall is low for both indexes, increase nprobe. If only IVF-RQ is low, try five bits before increasing I/O through more lists.
  4. Compare IVF-SQ when simpler/faster scans matter; compare IVF-PQ when minimum size matters.
  5. Validate the final choice on a public or production corpus, including batch and the real storage adapter.

Implementation boundaries

-
  • The current Writer fixes num_bits=1, rotation_type=1, and factor_layout=1, with no optional RQ sections.
  • The v1 header reserves values for future 2/4/8-bit stored codes, but current Readers reject those payloads.
  • query_bits is not serialized and may change per request on the same Reader.
  • optimize_for_search() currently preloads metadata and does not change RQ distance results.
- +
  • The file is immutable; updates require a new index file.
  • Distances are approximate and raw vectors are not retained for exact reranking.
  • Four transform rounds and compact factor layout 3 are fixed for v1; readers reject other values.
  • Batch scan parallelism uses Rayon while sharing each loaded list payload across queries; a single query also scans independent lists in parallel once its candidate count reaches 8,192.
  • optimize_for_search() loads and validates resident metadata; it does not change search results.
+
diff --git a/docs/ivf-sq.html b/docs/ivf-sq.html new file mode 100644 index 0000000..436f88b --- /dev/null +++ b/docs/ivf-sq.html @@ -0,0 +1,74 @@ + + + +IVF-SQ · Paimon Vector Index + +
VIPaimon Vector Index
+
+

Per-list residual scalar quantization

IVF-SQ

Store one unsigned byte per residual dimension and scan the selected IVF lists directly. IVF-SQ targets the gap between raw IVF-FLAT and aggressively compressed IVF-PQ/RQ without paying for a graph in every list.

1 byte / dimensionPer-list boundsSIMD scanMagic: IVSQ
+
+ +
+
+

Position

+

IVF-SQ preserves the IVF partitioning model and replaces every residual f32 component with an 8-bit scalar code. It usually occupies about one quarter of IVF-FLAT's vector payload. Unlike IVF-PQ, each dimension is quantized independently, so there is no subquantizer-count parameter or codebook lookup table.

+
Use it whenIVF-FLAT recall is good, its raw-vector payload or scan bandwidth is too large, and a one-byte-per-dimension representation fits the budget. Prefer IVF-PQ for much smaller codes; prefer DiskANN when high-recall large-scale local-SSD search needs page-granular graph traversal.
+
+
+

Build and search

+
  1. Train the IVF coarse centroids and assign training vectors to lists.
  2. Subtract each list centroid and learn per-dimension minimum/maximum residual bounds for that list.
  3. Encode every residual coordinate to an unsigned byte. Empty training lists use the global residual bounds.
  4. At query time, select nprobe lists, load their sorted row IDs and codes in one multi-range read, scan the codes with SIMD L2 or inner-product kernels, and merge the top K.
+

Cosine input is normalized through the shared metric preprocessing path. Filters are checked while scanning, so excluded rows do not enter the top-K heap.

+
+
+

Configuration

+
Options API
index.type = ivf_sq
+dimension = 128
+nlist = 1024
+metric = l2
+
Rust
let config = VectorIndexConfig::IvfSq {
+    dimension: 128,
+    nlist: 1024,
+    metric: MetricType::L2,
+};
+let params = VectorSearchParams::new(10, 16);
+
+
+

Parameters

+
ParameterRequirementEffect
dimensionInferred by Java/Python one-shot training; otherwise > 0Each vector uses exactly d SQ-code bytes.
nlistAuto from expected-vector-count, or explicit > 0 and no larger than training countMore lists shorten scans but enlarge centroid and per-list-bound metadata.
metricRequired: L2, inner product, or cosineSelects preprocessing and the distance kernel.
nprobeAutomatic by default; explicit 1 to nlistAuto accounts for K, average list size, and filter selectivity; explicit values provide a measured override.
+

The scalar code width is fixed at 8 bits in v1. There is deliberately no sq.bits, graph-width, or search-width option.

+
+
+

Stable v1 storage

+
64 B headerIVSQ v1
Global bounds2 × d × f32
Per-list bounds2 × nlist × d × f32
IVF centersnlist × d × f32
Offset tablenlist × 16 B
Listsblocked SQ codes + delta IDs
+

Every non-empty list is sorted by signed row ID before writing. Codes come first and are transposed within up-to-32-row blocks, with dimension before row lane, so SIMD evaluates multiple candidates together and the reader scans directly from the list payload allocation. The trailing IDs use the shared delta-varint encoding and remain aligned with code lanes. The normative byte layout and golden fixture are in the storage-format specification.

+
+
+

Open-source comparison

+

Faiss IVF-SQ provides residual SQ4/SQ6/SQ8/F16 encodings, parallel add, and query-parallel scanning over generic inverted lists. Milvus Knowhere builds on the same scanner model and adds concurrent inverted-list mutation. This implementation deliberately fixes the first immutable format at SQ8, uses per-list per-dimension residual bounds, stores compressed sorted IDs instead of fixed eight-byte IDs, and transposes each 32-row code block for its CPU SIMD kernels.

+

The comparison did produce two build changes: non-cosine inputs are borrowed instead of copied, and assigned lists are encoded in parallel with one reusable residual vector per worker. It did not justify changing the persisted layout. SQ4/SQ6 would overlap IVF-PQ/RQ, quantile clipping would introduce another corpus-sensitive accuracy parameter, and compressing the relatively small resident bounds would save little beside the N × d code payload. The existing codes-first payload already allows one bounded multi-range operation and reuse of the read allocation as the scan buffer.

+
+
+

I/O and batching

+

Open reads the fixed header and contiguous resident metadata in two positional operations; the outer type dispatcher adds one small magic read. A query submits selected list ranges through the abstract positional-read interface in capability- and 64 MiB-bounded multi-range batches. SIFT1M and GloVe-100 use one payload round per query at nprobe=64; 960-dimensional GIST1M averages 1.9. Batch search first deduplicates the lists selected across queries, loads each unique list once across the bounded rounds, and then scans queries in parallel. This is especially useful when a remote adapter executes the supplied ranges concurrently.

+

The add path retains the caller's L2/IP slice without copying it, partitions assigned row positions by list, and encodes those lists in parallel. Each active list task reuses one d-component residual buffer and one code buffer; the add path never materializes an N × d residual matrix. The writer retains only each list's row-order permutation and encoded IDs. It generates one list's blocked codes directly from the in-memory row-major codes, writes them, and releases the temporary buffer before processing the next list.

+

During scanning, a candidate whose distance cannot improve a full Top-K heap is rejected before row-ID hashing. Batch remains query-parallel after a measured list-major experiment regressed SIFT/GIST throughput; list payloads are still deduplicated and read once.

+

IVF-SQ still reads complete selected lists. At high nprobe, scan bytes grow linearly; DiskANN is the better fit when the workload requires small page-granular reads from a large local-SSD index.

+
+
+

Public benchmarks

+

On the documented Apple M4 Pro run with one million-scale public vectors, nlist=1024, nprobe=64, k=10, and 12 Rayon workers:

+
DatasetRecall@10Build / peak RSSWarm P95 / batch QPSRead/query
SIFT1M0.86273.93 s / 0.79 GiB0.79 ms / 11,0828.38 MiB
GIST1M0.857722.7 s / 5.09 GiB3.56 ms / 1,50270.95 MiB
GloVe-1000.80363.86 s / 0.71 GiB0.71 ms / 12,9626.99 MiB
+

Compared with the immediately preceding implementation on the same files, peak RSS dropped by 56–61%, local P95 improved by 3–8%, and batch throughput improved by about 8–61%, depending on dimension and cache behavior. File bytes and read bytes are unchanged.

+ +
+
+
+
+
Apache Paimon Vector IndexIVF-SQ · v1
+ + diff --git a/ffi/Cargo.toml b/ffi/Cargo.toml index 3a66c0b..d3f33ce 100644 --- a/ffi/Cargo.toml +++ b/ffi/Cargo.toml @@ -17,17 +17,19 @@ [package] name = "paimon-vindex-ffi" -version = "0.2.0" +version = "0.3.0" edition = "2021" description = "Apache Paimon Vector Index C FFI bindings" license = "Apache-2.0" +repository = "https://github.com/apache/paimon-vector-index" +homepage = "https://paimon.apache.org" [lib] name = "paimon_vindex_ffi" crate-type = ["cdylib"] [dependencies] -paimon-vindex-core = { path = "../core", version = "0.2.0" } +paimon-vindex-core = { path = "../core", version = "0.3.0" } [build-dependencies] cbindgen = "0.27" diff --git a/ffi/src/lib.rs b/ffi/src/lib.rs index 6e9eb94..7fa2c2c 100644 --- a/ffi/src/lib.rs +++ b/ffi/src/lib.rs @@ -19,10 +19,11 @@ use paimon_vindex_core::distance::MetricType; use paimon_vindex_core::index::{ - VectorIndexConfig, VectorIndexMetadata, VectorIndexReader, VectorIndexTrainer, - VectorIndexTraining, VectorIndexWriter, VectorSearchParams, + SearchWidth, VectorIndexConfig, VectorIndexMetadata, VectorIndexReadPlan, VectorIndexReader, + VectorIndexReaderOptions, VectorIndexTrainer, VectorIndexTraining, VectorIndexWriter, + VectorSearchParams, }; -use paimon_vindex_core::io::{ReadRequest, SeekRead, SeekWrite}; +use paimon_vindex_core::io::{ReadRequest, SeekRead, SeekReadCapabilities, SeekWrite}; use std::cell::RefCell; use std::collections::HashMap; use std::ffi::{CStr, CString}; @@ -33,14 +34,18 @@ use std::{ptr, slice}; pub const PAIMON_VINDEX_INDEX_TYPE_IVF_FLAT: u32 = 0; pub const PAIMON_VINDEX_INDEX_TYPE_IVF_PQ: u32 = 1; -pub const PAIMON_VINDEX_INDEX_TYPE_IVF_HNSW_FLAT: u32 = 2; -pub const PAIMON_VINDEX_INDEX_TYPE_IVF_HNSW_SQ: u32 = 3; pub const PAIMON_VINDEX_INDEX_TYPE_IVF_RQ: u32 = 4; +pub const PAIMON_VINDEX_INDEX_TYPE_DISKANN: u32 = 5; +pub const PAIMON_VINDEX_INDEX_TYPE_IVF_SQ: u32 = 6; pub const PAIMON_VINDEX_METRIC_L2: u32 = 0; pub const PAIMON_VINDEX_METRIC_INNER_PRODUCT: u32 = 1; pub const PAIMON_VINDEX_METRIC_COSINE: u32 = 2; +pub const PAIMON_VINDEX_SEARCH_WIDTH_AUTO: u32 = 0; +pub const PAIMON_VINDEX_SEARCH_WIDTH_IVF_NPROBE: u32 = 1; +pub const PAIMON_VINDEX_SEARCH_WIDTH_DISKANN_L_SEARCH: u32 = 2; + thread_local! { static LAST_ERROR: RefCell> = const { RefCell::new(None) }; } @@ -172,14 +177,24 @@ pub struct PaimonVindexReadRequest { } #[repr(C)] +#[derive(Clone, Copy)] pub struct PaimonVindexInputFile { pub ctx: *mut c_void, /// Reads every request in the batch, preferably concurrently. /// + /// DiskANN batch search may invoke this callback concurrently from multiple + /// threads. The callback and `ctx` must therefore be thread-safe. + /// /// Request descriptors and their buffers are valid only for the duration of /// the callback and must not be retained by the implementation. pub read_ranges_fn: Option c_int>, + /// Estimated latency of one random read in nanoseconds, or zero to let + /// DiskANN use the mandatory header read as its measurement. + pub estimated_random_read_latency_nanos: u64, + /// Zero means unspecified for the remaining capability fields. + pub preferred_window_bytes: usize, + pub max_ranges_per_read: usize, } struct FfiInputFile { @@ -212,6 +227,18 @@ impl SeekRead for FfiInputFile { Err(io::Error::other("read_ranges_fn is null")) } } + + fn try_clone_reader(&self) -> io::Result> { + Ok(Some(Self { raw: self.raw })) + } + + fn read_capabilities(&self) -> SeekReadCapabilities { + SeekReadCapabilities { + estimated_random_read_latency_nanos: self.raw.estimated_random_read_latency_nanos, + preferred_window_bytes: self.raw.preferred_window_bytes, + max_ranges_per_pread: self.raw.max_ranges_per_read, + } + } } // ======================== Common structs ======================== @@ -224,18 +251,39 @@ pub struct PaimonVindexMetadata { pub metric: u32, pub total_vectors: i64, pub pq_m: usize, - pub hnsw_m: usize, - pub hnsw_ef_construction: usize, - pub hnsw_max_level: usize, + pub pq_bits: usize, + pub rq_bits: usize, + pub diskann_max_degree: usize, + pub diskann_build_search_list_size: usize, + pub diskann_alpha: f32, } #[repr(C)] #[derive(Clone, Copy)] pub struct PaimonVindexSearchParams { pub top_k: usize, - pub nprobe: usize, - pub ef_search: usize, - pub query_bits: usize, + pub search_width: u32, + pub width: usize, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct PaimonVindexReaderOptions { + pub memory_budget_bytes: usize, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct PaimonVindexReadPlan { + pub random_read_latency_nanos: u64, + pub window_bytes: usize, + pub max_ranges_per_read: usize, + pub graph_beam_width: usize, + pub filtered_graph_beam_width: usize, + pub adjacency_preload_bytes: usize, + pub adjacency_cache_bytes: usize, + pub raw_vector_cache_bytes: usize, + pub memory_budget_bytes: usize, } pub struct PaimonVindexTrainerHandle { @@ -255,10 +303,10 @@ pub struct PaimonVindexReaderHandle { } fn metadata_to_ffi(metadata: VectorIndexMetadata) -> PaimonVindexMetadata { - let (hnsw_m, hnsw_ef_construction, hnsw_max_level) = metadata - .hnsw - .map(|h| (h.m, h.ef_construction, h.max_level)) - .unwrap_or((0, 0, 0)); + let (diskann_max_degree, diskann_build_search_list_size, diskann_alpha) = metadata + .diskann + .map(|d| (d.max_degree, d.build_search_list_size, d.alpha)) + .unwrap_or((0, 0, 0.0)); PaimonVindexMetadata { index_type: metadata.index_type as u32, dimension: metadata.dimension, @@ -266,9 +314,25 @@ fn metadata_to_ffi(metadata: VectorIndexMetadata) -> PaimonVindexMetadata { metric: metric_code(metadata.metric), total_vectors: metadata.total_vectors, pq_m: metadata.pq_m.unwrap_or(0), - hnsw_m, - hnsw_ef_construction, - hnsw_max_level, + pq_bits: metadata.pq_bits.unwrap_or(0), + rq_bits: metadata.rq_bits.unwrap_or(0), + diskann_max_degree, + diskann_build_search_list_size, + diskann_alpha, + } +} + +fn read_plan_to_ffi(plan: VectorIndexReadPlan) -> PaimonVindexReadPlan { + PaimonVindexReadPlan { + random_read_latency_nanos: plan.random_read_latency_nanos, + window_bytes: plan.window_bytes, + max_ranges_per_read: plan.max_ranges_per_read, + graph_beam_width: plan.graph_beam_width, + filtered_graph_beam_width: plan.filtered_graph_beam_width, + adjacency_preload_bytes: plan.adjacency_preload_bytes, + adjacency_cache_bytes: plan.adjacency_cache_bytes, + raw_vector_cache_bytes: plan.raw_vector_cache_bytes, + memory_budget_bytes: plan.memory_budget_bytes, } } @@ -447,13 +511,21 @@ fn copy_search_result( Ok(()) } -fn search_params_from_ffi(params: PaimonVindexSearchParams) -> VectorSearchParams { - VectorSearchParams { - top_k: params.top_k, - nprobe: params.nprobe, - ef_search: params.ef_search, - query_bits: params.query_bits, +fn search_params_from_ffi(params: PaimonVindexSearchParams) -> Result { + let search_width = match params.search_width { + PAIMON_VINDEX_SEARCH_WIDTH_AUTO => SearchWidth::Auto, + PAIMON_VINDEX_SEARCH_WIDTH_IVF_NPROBE => SearchWidth::IvfNProbe, + PAIMON_VINDEX_SEARCH_WIDTH_DISKANN_L_SEARCH => SearchWidth::DiskAnnLSearch, + value => return Err(format!("invalid search width type: {value}")), + }; + if search_width == SearchWidth::Auto && params.width != 0 { + return Err("automatic search width must have width=0".to_string()); } + Ok(VectorSearchParams { + top_k: params.top_k, + search_width, + width: params.width, + }) } // ======================== Trainer / Writer ======================== @@ -644,10 +716,29 @@ pub unsafe extern "C" fn paimon_vindex_writer_write_index( #[no_mangle] pub unsafe extern "C" fn paimon_vindex_reader_open( input_file: PaimonVindexInputFile, +) -> *mut PaimonVindexReaderHandle { + unsafe { + paimon_vindex_reader_open_with_options( + input_file, + PaimonVindexReaderOptions { + memory_budget_bytes: 4 * 1024 * 1024 * 1024, + }, + ) + } +} + +#[no_mangle] +pub unsafe extern "C" fn paimon_vindex_reader_open_with_options( + input_file: PaimonVindexInputFile, + options: PaimonVindexReaderOptions, ) -> *mut PaimonVindexReaderHandle { ffi_ptr(|| { let input = FfiInputFile { raw: input_file }; - let reader = VectorIndexReader::open(input).map_err(|e| format!("open reader: {}", e))?; + let reader = VectorIndexReader::open_with_options( + input, + VectorIndexReaderOptions::new(options.memory_budget_bytes), + ) + .map_err(|e| format!("open reader: {}", e))?; Ok(Box::into_raw(Box::new(PaimonVindexReaderHandle { inner: reader, }))) @@ -680,6 +771,27 @@ pub unsafe extern "C" fn paimon_vindex_reader_metadata( }) } +#[no_mangle] +pub unsafe extern "C" fn paimon_vindex_reader_read_plan( + handle: *const PaimonVindexReaderHandle, + out: *mut PaimonVindexReadPlan, +) -> c_int { + ffi_status(|| { + if out.is_null() { + return Err("out pointer is null".to_string()); + } + let handle = unsafe { reader_ref(handle) }?; + let plan = handle + .inner + .read_plan() + .ok_or_else(|| "read plan is only available for DiskANN".to_string())?; + unsafe { + *out = read_plan_to_ffi(plan); + } + Ok(()) + }) +} + #[no_mangle] pub unsafe extern "C" fn paimon_vindex_reader_optimize_for_search( handle: *mut PaimonVindexReaderHandle, @@ -693,6 +805,50 @@ pub unsafe extern "C" fn paimon_vindex_reader_optimize_for_search( }) } +#[no_mangle] +pub unsafe extern "C" fn paimon_vindex_reader_warmup_queries( + handle: *mut PaimonVindexReaderHandle, + queries: *const f32, + query_count: usize, + l_search: usize, +) -> c_int { + ffi_status(|| { + let handle = unsafe { reader_mut(handle) }?; + let query_len = checked_len(query_count, handle.inner.dimension(), "warmup queries")?; + let queries = unsafe { const_slice(queries, query_len, "warmup queries") }?; + handle + .inner + .warmup_queries(queries, query_count, l_search) + .map_err(|e| format!("warmup_queries: {}", e)) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn paimon_vindex_reader_calibrate_search_width( + handle: *mut PaimonVindexReaderHandle, + queries: *const f32, + query_count: usize, + top_k: usize, + out_l_search: *mut usize, +) -> c_int { + ffi_status(|| { + if out_l_search.is_null() { + return Err("out_l_search pointer is null".to_string()); + } + let handle = unsafe { reader_mut(handle) }?; + let query_len = checked_len(query_count, handle.inner.dimension(), "calibration queries")?; + let queries = unsafe { const_slice(queries, query_len, "calibration queries") }?; + let resolved = handle + .inner + .calibrate_search_width(queries, query_count, top_k) + .map_err(|e| format!("calibrate_search_width: {e}"))?; + unsafe { + *out_l_search = resolved; + } + Ok(()) + }) +} + #[no_mangle] pub unsafe extern "C" fn paimon_vindex_reader_search( handle: *mut PaimonVindexReaderHandle, @@ -705,7 +861,7 @@ pub unsafe extern "C" fn paimon_vindex_reader_search( ffi_status(|| { let handle = unsafe { reader_mut(handle) }?; let query = unsafe { const_slice(query, handle.inner.dimension(), "query") }?; - let params = search_params_from_ffi(params); + let params = search_params_from_ffi(params)?; let (ids, distances) = handle .inner .search(query, params) @@ -736,7 +892,7 @@ pub unsafe extern "C" fn paimon_vindex_reader_search_with_roaring_filter( let handle = unsafe { reader_mut(handle) }?; let query = unsafe { const_slice(query, handle.inner.dimension(), "query") }?; let filter = unsafe { const_slice(roaring_filter, roaring_filter_len, "roaring_filter") }?; - let params = search_params_from_ffi(params); + let params = search_params_from_ffi(params)?; let (ids, distances) = handle .inner .search_with_roaring_filter(query, params, filter) @@ -766,7 +922,7 @@ pub unsafe extern "C" fn paimon_vindex_reader_search_batch( let handle = unsafe { reader_mut(handle) }?; let query_len = checked_len(query_count, handle.inner.dimension(), "queries")?; let queries = unsafe { const_slice(queries, query_len, "queries") }?; - let params = search_params_from_ffi(params); + let params = search_params_from_ffi(params)?; let expected_len = checked_len(query_count, params.top_k, "batch result")?; let (ids, distances) = handle .inner @@ -800,7 +956,7 @@ pub unsafe extern "C" fn paimon_vindex_reader_search_batch_with_roaring_filter( let query_len = checked_len(query_count, handle.inner.dimension(), "queries")?; let queries = unsafe { const_slice(queries, query_len, "queries") }?; let filter = unsafe { const_slice(roaring_filter, roaring_filter_len, "roaring_filter") }?; - let params = search_params_from_ffi(params); + let params = search_params_from_ffi(params)?; let expected_len = checked_len(query_count, params.top_k, "batch result")?; let (ids, distances) = handle .inner @@ -855,6 +1011,9 @@ mod tests { let raw = PaimonVindexInputFile { ctx: (&mut state as *mut BatchReadState).cast(), read_ranges_fn: Some(read_ranges), + estimated_random_read_latency_nanos: 0, + preferred_window_bytes: 0, + max_ranges_per_read: 0, }; let mut input = FfiInputFile { raw }; let mut first = [0u8; 3]; @@ -883,6 +1042,9 @@ mod tests { let raw = PaimonVindexInputFile { ctx: (&mut state as *mut BatchReadState).cast(), read_ranges_fn: Some(read_ranges), + estimated_random_read_latency_nanos: 0, + preferred_window_bytes: 0, + max_ranges_per_read: 0, }; let mut input = FfiInputFile { raw }; @@ -890,4 +1052,46 @@ mod tests { assert_eq!(state.calls, 0); } + + #[test] + fn ffi_input_file_clones_reuse_the_callback_context() { + let mut state = BatchReadState { + data: Vec::new(), + calls: 0, + range_count: 0, + }; + let input = FfiInputFile { + raw: PaimonVindexInputFile { + ctx: (&mut state as *mut BatchReadState).cast(), + read_ranges_fn: Some(read_ranges), + estimated_random_read_latency_nanos: 0, + preferred_window_bytes: 0, + max_ranges_per_read: 0, + }, + }; + + let clone = input + .try_clone_reader() + .unwrap() + .expect("FFI positional callbacks should support concurrent reader clones"); + + assert_eq!(clone.raw.ctx, input.raw.ctx); + assert_eq!( + clone.raw.read_ranges_fn.map(|callback| callback as usize), + input.raw.read_ranges_fn.map(|callback| callback as usize) + ); + } + + #[test] + fn ffi_search_parameters_preserve_diskann_width() { + let params = search_params_from_ffi(PaimonVindexSearchParams { + top_k: 10, + search_width: PAIMON_VINDEX_SEARCH_WIDTH_DISKANN_L_SEARCH, + width: 200, + }) + .unwrap(); + + assert_eq!(params.search_width, SearchWidth::DiskAnnLSearch); + assert_eq!(params.width, 200); + } } diff --git a/include/paimon_vindex.hpp b/include/paimon_vindex.hpp index 294e482..6fa01f1 100644 --- a/include/paimon_vindex.hpp +++ b/include/paimon_vindex.hpp @@ -26,8 +26,10 @@ extern "C" { #include #include #include +#include #include #include +#include #include #include @@ -54,11 +56,99 @@ struct OutputFile { using ReadRequest = PaimonVindexReadRequest; struct InputFile { + // DiskANN batch search may invoke this callback from multiple worker threads. std::function read_ranges_fn; + // Optional positional-read capabilities. Zero leaves the policy unspecified. + uint64_t estimated_random_read_latency_nanos = 0; + size_t preferred_window_bytes = 0; + size_t max_ranges_per_read = 0; }; namespace detail { +class NativeCallbackScope { +public: + explicit NativeCallbackScope(const void* context) noexcept + : context_(context), previous_(current()) { + current() = this; + } + + NativeCallbackScope(const NativeCallbackScope&) = delete; + NativeCallbackScope& operator=(const NativeCallbackScope&) = delete; + + ~NativeCallbackScope() { + current() = previous_; + } + + static bool active_for(const void* context) noexcept { + if (context == nullptr) return false; + for (auto* scope = current(); scope != nullptr; scope = scope->previous_) { + if (scope->context_ == context) return true; + } + return false; + } + +private: + static NativeCallbackScope*& current() noexcept { + static thread_local NativeCallbackScope* scope = nullptr; + return scope; + } + + const void* context_; + NativeCallbackScope* previous_; +}; + +class NativeHandleMutex { +public: + void lock() { + const auto current = std::this_thread::get_id(); + { + std::lock_guard state_lock(state_mutex_); + if (owner_ == current || + NativeCallbackScope::active_for(callback_context_)) { + throw Error("reentrant native-handle operation is not allowed"); + } + } + operation_mutex_.lock(); + std::lock_guard state_lock(state_mutex_); + owner_ = current; + } + + bool try_lock() { + const auto current = std::this_thread::get_id(); + { + std::lock_guard state_lock(state_mutex_); + if (owner_ == current || + NativeCallbackScope::active_for(callback_context_)) { + throw Error("reentrant native-handle operation is not allowed"); + } + } + if (!operation_mutex_.try_lock()) return false; + std::lock_guard state_lock(state_mutex_); + owner_ = current; + return true; + } + + void unlock() noexcept { + { + std::lock_guard state_lock(state_mutex_); + owner_ = std::thread::id(); + } + operation_mutex_.unlock(); + } + + void set_callback_context(const void* context) { + std::lock_guard state_lock(state_mutex_); + callback_context_ = context; + } + +private: + std::mutex operation_mutex_; + std::mutex state_mutex_; + std::thread::id owner_; + const void* callback_context_ = nullptr; +}; + inline int stream_write(void* ctx, const uint8_t* data, size_t len) noexcept { try { auto* cbs = static_cast(ctx); @@ -92,6 +182,7 @@ inline int input_read_ranges( void* ctx, PaimonVindexReadRequest* raw_requests, size_t request_count) noexcept { + NativeCallbackScope callback_scope(ctx); try { auto* cbs = static_cast(ctx); return cbs->read_ranges_fn(raw_requests, request_count); @@ -109,9 +200,23 @@ struct Metadata { uint32_t metric = 0; int64_t total_vectors = 0; size_t pq_m = 0; - size_t hnsw_m = 0; - size_t hnsw_ef_construction = 0; - size_t hnsw_max_level = 0; + size_t pq_bits = 0; + size_t rq_bits = 0; + size_t diskann_max_degree = 0; + size_t diskann_build_search_list_size = 0; + float diskann_alpha = 0.0f; +}; + +struct ReadPlan { + uint64_t random_read_latency_nanos = 0; + size_t window_bytes = 0; + size_t max_ranges_per_read = 0; + size_t graph_beam_width = 0; + size_t filtered_graph_beam_width = 0; + size_t adjacency_preload_bytes = 0; + size_t adjacency_cache_bytes = 0; + size_t raw_vector_cache_bytes = 0; + size_t memory_budget_bytes = 0; }; struct SearchResult { @@ -121,21 +226,38 @@ struct SearchResult { struct SearchParams { size_t top_k = 0; - size_t nprobe = 0; - size_t ef_search = 0; - size_t query_bits = 0; + uint32_t search_width = PAIMON_VINDEX_SEARCH_WIDTH_AUTO; + size_t width = 0; + + SearchParams(size_t top_k, size_t nprobe) + : top_k(top_k), + search_width(PAIMON_VINDEX_SEARCH_WIDTH_IVF_NPROBE), + width(nprobe) {} + + static SearchParams automatic(size_t top_k) { + SearchParams params; + params.top_k = top_k; + return params; + } - SearchParams(size_t top_k, size_t nprobe, size_t ef_search = 0, size_t query_bits = 0) - : top_k(top_k), nprobe(nprobe), ef_search(ef_search), query_bits(query_bits) {} + static SearchParams diskann(size_t top_k, size_t l_search) { + SearchParams params; + params.top_k = top_k; + params.search_width = PAIMON_VINDEX_SEARCH_WIDTH_DISKANN_L_SEARCH; + params.width = l_search; + return params; + } PaimonVindexSearchParams to_ffi() const { PaimonVindexSearchParams params; params.top_k = top_k; - params.nprobe = nprobe; - params.ef_search = ef_search; - params.query_bits = query_bits; + params.search_width = search_width; + params.width = width; return params; } + +private: + SearchParams() = default; }; class Training { @@ -328,39 +450,61 @@ class Writer { class Reader { public: - explicit Reader(InputFile input) : input_(std::make_shared(std::move(input))) { + explicit Reader(InputFile input) + : Reader( + std::move(input), + static_cast(4ULL * 1024 * 1024 * 1024)) {} + + Reader(InputFile input, size_t memory_budget_bytes) + : input_(std::make_shared(std::move(input))) { + native_handle_mutex_.set_callback_context(input_.get()); PaimonVindexInputFile raw; raw.ctx = input_.get(); raw.read_ranges_fn = detail::input_read_ranges; - handle_ = paimon_vindex_reader_open(raw); + raw.estimated_random_read_latency_nanos = + input_->estimated_random_read_latency_nanos; + raw.preferred_window_bytes = input_->preferred_window_bytes; + raw.max_ranges_per_read = input_->max_ranges_per_read; + PaimonVindexReaderOptions options; + options.memory_budget_bytes = memory_budget_bytes; + handle_ = paimon_vindex_reader_open_with_options(raw, options); if (!handle_) throw Error("failed to open vector index reader"); } Reader(const Reader&) = delete; Reader& operator=(const Reader&) = delete; - Reader(Reader&& other) noexcept - : handle_(other.handle_), input_(std::move(other.input_)) { + Reader(Reader&& other) { + std::lock_guard lock(other.native_handle_mutex_); + handle_ = other.handle_; + input_ = std::move(other.input_); + native_handle_mutex_.set_callback_context(input_.get()); + other.native_handle_mutex_.set_callback_context(nullptr); other.handle_ = nullptr; } - Reader& operator=(Reader&& other) noexcept { + Reader& operator=(Reader&& other) { if (this != &other) { + std::scoped_lock lock(native_handle_mutex_, other.native_handle_mutex_); if (handle_) paimon_vindex_reader_free(handle_); handle_ = other.handle_; input_ = std::move(other.input_); + native_handle_mutex_.set_callback_context(input_.get()); + other.native_handle_mutex_.set_callback_context(nullptr); other.handle_ = nullptr; } return *this; } ~Reader() { + std::lock_guard lock(native_handle_mutex_); if (handle_) paimon_vindex_reader_free(handle_); } Metadata metadata() const { + std::lock_guard lock(native_handle_mutex_); PaimonVindexMetadata raw; - check(paimon_vindex_reader_metadata(handle_, &raw)); + check(paimon_vindex_reader_metadata(require_open(), &raw)); Metadata result; result.index_type = raw.index_type; result.dimension = raw.dimension; @@ -368,22 +512,59 @@ class Reader { result.metric = raw.metric; result.total_vectors = raw.total_vectors; result.pq_m = raw.pq_m; - result.hnsw_m = raw.hnsw_m; - result.hnsw_ef_construction = raw.hnsw_ef_construction; - result.hnsw_max_level = raw.hnsw_max_level; + result.pq_bits = raw.pq_bits; + result.rq_bits = raw.rq_bits; + result.diskann_max_degree = raw.diskann_max_degree; + result.diskann_build_search_list_size = raw.diskann_build_search_list_size; + result.diskann_alpha = raw.diskann_alpha; return result; } void optimize_for_search() { - check(paimon_vindex_reader_optimize_for_search(handle_)); + std::lock_guard lock(native_handle_mutex_); + check(paimon_vindex_reader_optimize_for_search(require_open())); + } + + void warmup_queries( + const float* queries, size_t query_count, size_t l_search = 0) { + std::lock_guard lock(native_handle_mutex_); + check(paimon_vindex_reader_warmup_queries( + require_open(), queries, query_count, l_search)); + } + + size_t calibrate_search_width( + const float* queries, size_t query_count, size_t top_k = 10) { + std::lock_guard lock(native_handle_mutex_); + size_t l_search = 0; + check(paimon_vindex_reader_calibrate_search_width( + require_open(), queries, query_count, top_k, &l_search)); + return l_search; + } + + ReadPlan read_plan() const { + std::lock_guard lock(native_handle_mutex_); + PaimonVindexReadPlan raw; + check(paimon_vindex_reader_read_plan(require_open(), &raw)); + ReadPlan result; + result.random_read_latency_nanos = raw.random_read_latency_nanos; + result.window_bytes = raw.window_bytes; + result.max_ranges_per_read = raw.max_ranges_per_read; + result.graph_beam_width = raw.graph_beam_width; + result.filtered_graph_beam_width = raw.filtered_graph_beam_width; + result.adjacency_preload_bytes = raw.adjacency_preload_bytes; + result.adjacency_cache_bytes = raw.adjacency_cache_bytes; + result.raw_vector_cache_bytes = raw.raw_vector_cache_bytes; + result.memory_budget_bytes = raw.memory_budget_bytes; + return result; } SearchResult search(const float* query, SearchParams params) { + std::lock_guard lock(native_handle_mutex_); SearchResult result; result.ids.resize(params.top_k); result.distances.resize(params.top_k); check(paimon_vindex_reader_search( - handle_, + require_open(), query, params.to_ffi(), result.ids.data(), @@ -396,12 +577,13 @@ class Reader { const float* query, SearchParams params, const uint8_t* filter, - size_t filter_len) { + size_t filter_len) { + std::lock_guard lock(native_handle_mutex_); SearchResult result; result.ids.resize(params.top_k); result.distances.resize(params.top_k); check(paimon_vindex_reader_search_with_roaring_filter( - handle_, + require_open(), query, params.to_ffi(), filter, @@ -414,14 +596,15 @@ class Reader { SearchResult search_batch( const float* queries, - size_t query_count, - SearchParams params) { + size_t query_count, + SearchParams params) { + std::lock_guard lock(native_handle_mutex_); const size_t result_len = query_count * params.top_k; SearchResult result; result.ids.resize(result_len); result.distances.resize(result_len); check(paimon_vindex_reader_search_batch( - handle_, + require_open(), queries, query_count, params.to_ffi(), @@ -435,14 +618,15 @@ class Reader { const float* queries, size_t query_count, SearchParams params, - const uint8_t* filter, - size_t filter_len) { + const uint8_t* filter, + size_t filter_len) { + std::lock_guard lock(native_handle_mutex_); const size_t result_len = query_count * params.top_k; SearchResult result; result.ids.resize(result_len); result.distances.resize(result_len); check(paimon_vindex_reader_search_batch_with_roaring_filter( - handle_, + require_open(), queries, query_count, params.to_ffi(), @@ -455,6 +639,12 @@ class Reader { } private: + PaimonVindexReaderHandle* require_open() const { + if (!handle_) throw Error("vector index reader is closed"); + return handle_; + } + + mutable detail::NativeHandleMutex native_handle_mutex_; PaimonVindexReaderHandle* handle_ = nullptr; std::shared_ptr input_; }; diff --git a/java/pom.xml b/java/pom.xml index ef99a41..ee303e8 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -30,7 +30,7 @@ org.apache.paimon paimon-vector-index-java - 0.2.0-SNAPSHOT + 0.3.0-SNAPSHOT jar Apache Paimon Vector Index Java diff --git a/java/src/main/java/org/apache/paimon/index/vector/VectorIndexInput.java b/java/src/main/java/org/apache/paimon/index/vector/VectorIndexInput.java index dca4430..f8a7aee 100644 --- a/java/src/main/java/org/apache/paimon/index/vector/VectorIndexInput.java +++ b/java/src/main/java/org/apache/paimon/index/vector/VectorIndexInput.java @@ -19,5 +19,27 @@ public interface VectorIndexInput { + /** + * Reads all requested ranges. DiskANN batch search may call this method concurrently from + * multiple native worker threads, so implementations must be thread-safe. + */ void pread(long[] positions, byte[][] buffers); + + /** + * Estimated end-to-end latency of one representative random read in nanoseconds, or zero to + * let DiskANN use the mandatory header read as its measurement. + */ + default long estimatedRandomReadLatencyNanos() { + return 0L; + } + + /** Efficient coalesced random-read window, or zero when unspecified. */ + default long preferredReadWindowBytes() { + return 0L; + } + + /** Maximum ranges accepted by one pread call, or zero when unlimited/unspecified. */ + default long maxRangesPerRead() { + return 0L; + } } diff --git a/java/src/main/java/org/apache/paimon/index/vector/VectorIndexMetadata.java b/java/src/main/java/org/apache/paimon/index/vector/VectorIndexMetadata.java index 48c49e0..89b431f 100644 --- a/java/src/main/java/org/apache/paimon/index/vector/VectorIndexMetadata.java +++ b/java/src/main/java/org/apache/paimon/index/vector/VectorIndexMetadata.java @@ -25,9 +25,11 @@ public final class VectorIndexMetadata { private final String metric; private final long totalVectors; private final int pqM; - private final int hnswM; - private final int hnswEfConstruction; - private final int hnswMaxLevel; + private final int pqBits; + private final int rqBits; + private final int diskAnnMaxDegree; + private final int diskAnnBuildSearchListSize; + private final float diskAnnAlpha; public VectorIndexMetadata( String indexType, @@ -36,9 +38,11 @@ public VectorIndexMetadata( String metric, long totalVectors, int pqM, - int hnswM, - int efConstruction, - int maxLevel) { + int pqBits, + int rqBits, + int diskAnnMaxDegree, + int diskAnnBuildSearchListSize, + float diskAnnAlpha) { if (indexType == null) { throw new NullPointerException("indexType"); } @@ -51,9 +55,11 @@ public VectorIndexMetadata( this.metric = metric; this.totalVectors = totalVectors; this.pqM = pqM; - this.hnswM = hnswM; - this.hnswEfConstruction = efConstruction; - this.hnswMaxLevel = maxLevel; + this.pqBits = pqBits; + this.rqBits = rqBits; + this.diskAnnMaxDegree = diskAnnMaxDegree; + this.diskAnnBuildSearchListSize = diskAnnBuildSearchListSize; + this.diskAnnAlpha = diskAnnAlpha; } public String indexType() { @@ -80,16 +86,24 @@ public int pqM() { return pqM; } - public int hnswM() { - return hnswM; + public int pqBits() { + return pqBits; } - public int hnswEfConstruction() { - return hnswEfConstruction; + public int rqBits() { + return rqBits; } - public int hnswMaxLevel() { - return hnswMaxLevel; + public int diskAnnMaxDegree() { + return diskAnnMaxDegree; + } + + public int diskAnnBuildSearchListSize() { + return diskAnnBuildSearchListSize; + } + + public float diskAnnAlpha() { + return diskAnnAlpha; } } diff --git a/java/src/main/java/org/apache/paimon/index/vector/VectorIndexNative.java b/java/src/main/java/org/apache/paimon/index/vector/VectorIndexNative.java index 49edc6d..cbbd0a9 100644 --- a/java/src/main/java/org/apache/paimon/index/vector/VectorIndexNative.java +++ b/java/src/main/java/org/apache/paimon/index/vector/VectorIndexNative.java @@ -45,10 +45,18 @@ private VectorIndexNative() {} static native long openReader(Object streamInput); + static native long openReaderWithOptions(Object streamInput, long memoryBudgetBytes); + static native VectorIndexMetadata metadata(long ptr); static native void optimizeForSearch(long ptr); + static native void warmupQueries(long ptr, float[] queries, int queryCount, int lSearch); + + static native int calibrateSearchWidth(long ptr, float[] queries, int queryCount, int topK); + + static native VectorIndexReadPlan readPlan(long ptr); + static native VectorSearchResult search(long ptr, float[] query, VectorSearchParams params); static native VectorSearchResult searchWithRoaringFilter( diff --git a/java/src/main/java/org/apache/paimon/index/vector/VectorIndexReadPlan.java b/java/src/main/java/org/apache/paimon/index/vector/VectorIndexReadPlan.java new file mode 100644 index 0000000..2076535 --- /dev/null +++ b/java/src/main/java/org/apache/paimon/index/vector/VectorIndexReadPlan.java @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.paimon.index.vector; + +/** The concrete DiskANN read policy derived from the input and Reader memory budget. */ +public final class VectorIndexReadPlan { + + private final long randomReadLatencyNanos; + private final long windowBytes; + private final long maxRangesPerRead; + private final long graphBeamWidth; + private final long filteredGraphBeamWidth; + private final long adjacencyPreloadBytes; + private final long adjacencyCacheBytes; + private final long rawVectorCacheBytes; + private final long memoryBudgetBytes; + + public VectorIndexReadPlan( + long randomReadLatencyNanos, + long windowBytes, + long maxRangesPerRead, + long graphBeamWidth, + long filteredGraphBeamWidth, + long adjacencyPreloadBytes, + long adjacencyCacheBytes, + long rawVectorCacheBytes, + long memoryBudgetBytes) { + this.randomReadLatencyNanos = randomReadLatencyNanos; + this.windowBytes = windowBytes; + this.maxRangesPerRead = maxRangesPerRead; + this.graphBeamWidth = graphBeamWidth; + this.filteredGraphBeamWidth = filteredGraphBeamWidth; + this.adjacencyPreloadBytes = adjacencyPreloadBytes; + this.adjacencyCacheBytes = adjacencyCacheBytes; + this.rawVectorCacheBytes = rawVectorCacheBytes; + this.memoryBudgetBytes = memoryBudgetBytes; + } + + public long randomReadLatencyNanos() { + return randomReadLatencyNanos; + } + + public long windowBytes() { + return windowBytes; + } + + public long maxRangesPerRead() { + return maxRangesPerRead; + } + + public long graphBeamWidth() { + return graphBeamWidth; + } + + public long filteredGraphBeamWidth() { + return filteredGraphBeamWidth; + } + + public long adjacencyPreloadBytes() { + return adjacencyPreloadBytes; + } + + public long adjacencyCacheBytes() { + return adjacencyCacheBytes; + } + + public long rawVectorCacheBytes() { + return rawVectorCacheBytes; + } + + public long memoryBudgetBytes() { + return memoryBudgetBytes; + } +} diff --git a/java/src/main/java/org/apache/paimon/index/vector/VectorIndexReader.java b/java/src/main/java/org/apache/paimon/index/vector/VectorIndexReader.java index 91de525..0dee537 100644 --- a/java/src/main/java/org/apache/paimon/index/vector/VectorIndexReader.java +++ b/java/src/main/java/org/apache/paimon/index/vector/VectorIndexReader.java @@ -20,15 +20,25 @@ public final class VectorIndexReader implements AutoCloseable { private final Object nativeHandleLock = new Object(); + private final NativeCallbackContext nativeCallbackContext = new NativeCallbackContext(); private long nativePtr; private Thread nativeHandleOwner; private VectorIndexMetadata metadata; public VectorIndexReader(VectorIndexInput input) { + this(input, 4L * 1024 * 1024 * 1024); + } + + public VectorIndexReader(VectorIndexInput input, long memoryBudgetBytes) { if (input == null) { throw new NullPointerException("input"); } - this.nativePtr = VectorIndexNative.openReader(input); + if (memoryBudgetBytes < 0) { + throw new IllegalArgumentException("memoryBudgetBytes must be non-negative"); + } + this.nativePtr = + VectorIndexNative.openReaderWithOptions( + new CallbackTrackingInput(input, nativeCallbackContext), memoryBudgetBytes); } private VectorIndexReader(long nativePtr) { @@ -40,6 +50,7 @@ static VectorIndexReader fromNativePointerForTesting(long nativePtr) { } public VectorIndexMetadata metadata() { + rejectCallbackReentry(); synchronized (nativeHandleLock) { enterNativeHandle(); try { @@ -67,6 +78,7 @@ public long totalVectors() { } public void optimizeForSearch() { + rejectCallbackReentry(); synchronized (nativeHandleLock) { enterNativeHandle(); try { @@ -77,9 +89,62 @@ public void optimizeForSearch() { } } + public void warmupQueries(float[] queries, int queryCount, int lSearch) { + if (queries == null) { + throw new NullPointerException("queries"); + } + if (queryCount < 0) { + throw new IllegalArgumentException("queryCount must be non-negative"); + } + if (lSearch < 0) { + throw new IllegalArgumentException("lSearch must be non-negative"); + } + rejectCallbackReentry(); + synchronized (nativeHandleLock) { + enterNativeHandle(); + try { + VectorIndexNative.warmupQueries(requireOpen(), queries, queryCount, lSearch); + } finally { + exitNativeHandle(); + } + } + } + + public int calibrateSearchWidth(float[] queries, int queryCount, int topK) { + if (queries == null) { + throw new NullPointerException("queries"); + } + if (queryCount <= 0 || topK <= 0) { + throw new IllegalArgumentException("queryCount and topK must be positive"); + } + rejectCallbackReentry(); + synchronized (nativeHandleLock) { + enterNativeHandle(); + try { + return VectorIndexNative.calibrateSearchWidth( + requireOpen(), queries, queryCount, topK); + } finally { + exitNativeHandle(); + } + } + } + + public VectorIndexReadPlan readPlan() { + rejectCallbackReentry(); + synchronized (nativeHandleLock) { + enterNativeHandle(); + try { + return VectorIndexNative.readPlan(requireOpen()); + } finally { + exitNativeHandle(); + } + } + } + public VectorSearchResult search(float[] query, VectorSearchParams params) { validateQuery(query); validateParams(params); + rejectCallbackReentry(); synchronized (nativeHandleLock) { enterNativeHandle(); try { @@ -97,6 +162,7 @@ public VectorSearchResult search( if (roaringFilter == null) { throw new NullPointerException("roaringFilter"); } + rejectCallbackReentry(); synchronized (nativeHandleLock) { enterNativeHandle(); try { @@ -114,6 +180,7 @@ public VectorSearchBatchResult searchBatch( throw new NullPointerException("queries"); } validateParams(params); + rejectCallbackReentry(); synchronized (nativeHandleLock) { enterNativeHandle(); try { @@ -133,6 +200,7 @@ public VectorSearchBatchResult searchBatch( if (roaringFilter == null) { throw new NullPointerException("roaringFilter"); } + rejectCallbackReentry(); synchronized (nativeHandleLock) { enterNativeHandle(); try { @@ -146,6 +214,7 @@ public VectorSearchBatchResult searchBatch( @Override public void close() { + rejectCallbackReentry(); synchronized (nativeHandleLock) { enterNativeHandle(); try { @@ -179,6 +248,13 @@ private long requireOpen() { return nativePtr; } + private void rejectCallbackReentry() { + if (nativeCallbackContext.isActiveOnCurrentThread()) { + throw new IllegalStateException( + "VectorIndexReader native handle is already in use by its input callback"); + } + } + private void enterNativeHandle() { Thread current = Thread.currentThread(); if (nativeHandleOwner == current) { @@ -190,4 +266,66 @@ private void enterNativeHandle() { private void exitNativeHandle() { nativeHandleOwner = null; } + + private static final class NativeCallbackContext { + private final ThreadLocal depth = new ThreadLocal(); + + private void enter() { + Integer currentDepth = depth.get(); + depth.set(currentDepth == null ? 1 : currentDepth + 1); + } + + private void exit() { + Integer currentDepth = depth.get(); + if (currentDepth == null) { + throw new IllegalStateException("input callback scope is not active"); + } + if (currentDepth == 1) { + depth.remove(); + } else { + depth.set(currentDepth - 1); + } + } + + private boolean isActiveOnCurrentThread() { + Integer currentDepth = depth.get(); + return currentDepth != null && currentDepth > 0; + } + } + + private static final class CallbackTrackingInput implements VectorIndexInput { + private final VectorIndexInput delegate; + private final NativeCallbackContext callbackContext; + + private CallbackTrackingInput( + VectorIndexInput delegate, NativeCallbackContext callbackContext) { + this.delegate = delegate; + this.callbackContext = callbackContext; + } + + @Override + public void pread(long[] positions, byte[][] buffers) { + callbackContext.enter(); + try { + delegate.pread(positions, buffers); + } finally { + callbackContext.exit(); + } + } + + @Override + public long estimatedRandomReadLatencyNanos() { + return delegate.estimatedRandomReadLatencyNanos(); + } + + @Override + public long preferredReadWindowBytes() { + return delegate.preferredReadWindowBytes(); + } + + @Override + public long maxRangesPerRead() { + return delegate.maxRangesPerRead(); + } + } } diff --git a/java/src/main/java/org/apache/paimon/index/vector/VectorIndexTrainer.java b/java/src/main/java/org/apache/paimon/index/vector/VectorIndexTrainer.java index 1eec2e3..fef3101 100644 --- a/java/src/main/java/org/apache/paimon/index/vector/VectorIndexTrainer.java +++ b/java/src/main/java/org/apache/paimon/index/vector/VectorIndexTrainer.java @@ -17,14 +17,15 @@ package org.apache.paimon.index.vector; +import java.util.HashMap; import java.util.Map; /** * Builds a trained vector-index state from one or more training batches. * - *

Staging batches avoids requiring one large Java {@code float[]} and its array length limit, - * but all batches are accumulated in native memory until {@link #finishTraining()}; this does not - * reduce native peak training memory. + *

Staging batches avoids requiring one large Java {@code float[]} and its array length limit. + * Native deterministic reservoir sampling retains at most the automatically selected training + * sample, independent of batch boundaries. */ public final class VectorIndexTrainer implements AutoCloseable { @@ -53,7 +54,27 @@ public static VectorIndexTrainer create(Map options) { public static VectorIndexTraining train( Map options, float[] data, int vectorCount) { - try (VectorIndexTrainer trainer = create(options)) { + if (options == null) { + throw new NullPointerException("options"); + } + if (data == null) { + throw new NullPointerException("data"); + } + if (vectorCount <= 0 || data.length % vectorCount != 0) { + throw new IllegalArgumentException( + "data length must be divisible by a positive vectorCount"); + } + Map resolved = new HashMap<>(options); + if (!resolved.containsKey("dimension") || "auto".equals(resolved.get("dimension"))) { + resolved.put("dimension", Integer.toString(data.length / vectorCount)); + } + String indexType = resolved.getOrDefault("index.type", ""); + if (indexType.startsWith("ivf_") + && (!resolved.containsKey("nlist") || "auto".equals(resolved.get("nlist"))) + && !resolved.containsKey("expected-vector-count")) { + resolved.put("expected-vector-count", Integer.toString(vectorCount)); + } + try (VectorIndexTrainer trainer = create(resolved)) { return trainer.addTrainingVectors(data, vectorCount).finishTraining(); } } diff --git a/java/src/main/java/org/apache/paimon/index/vector/VectorSearchParams.java b/java/src/main/java/org/apache/paimon/index/vector/VectorSearchParams.java index b0de1fe..77ab05e 100644 --- a/java/src/main/java/org/apache/paimon/index/vector/VectorSearchParams.java +++ b/java/src/main/java/org/apache/paimon/index/vector/VectorSearchParams.java @@ -19,43 +19,49 @@ public final class VectorSearchParams { + static final int SEARCH_WIDTH_AUTO = 0; + static final int SEARCH_WIDTH_IVF_NPROBE = 1; + static final int SEARCH_WIDTH_DISKANN_L_SEARCH = 2; + private final int topK; - private final int nprobe; - private final int efSearch; - private final int queryBits; + private final int searchWidth; + private final int width; public VectorSearchParams(int topK, int nprobe) { - this(topK, nprobe, 0, 0); + this(topK, SEARCH_WIDTH_IVF_NPROBE, nprobe); } - public VectorSearchParams(int topK, int nprobe, int efSearch, int queryBits) { + private VectorSearchParams(int topK, int searchWidth, int width) { this.topK = topK; - this.nprobe = nprobe; - this.efSearch = efSearch; - this.queryBits = queryBits; + this.searchWidth = searchWidth; + this.width = width; } - public int topK() { - return topK; + public static VectorSearchParams automatic(int topK) { + return new VectorSearchParams(topK, SEARCH_WIDTH_AUTO, 0); } - public int nprobe() { - return nprobe; + public static VectorSearchParams ivf(int topK, int nprobe) { + return new VectorSearchParams(topK, SEARCH_WIDTH_IVF_NPROBE, nprobe); } - public int efSearch() { - return efSearch; + public static VectorSearchParams diskAnn(int topK, int lSearch) { + return new VectorSearchParams(topK, SEARCH_WIDTH_DISKANN_L_SEARCH, lSearch); + } + + public int topK() { + return topK; } - public int queryBits() { - return queryBits; + int searchWidth() { + return searchWidth; } - public VectorSearchParams withEfSearch(int efSearch) { - return new VectorSearchParams(topK, nprobe, efSearch, queryBits); + int width() { + return width; } - public VectorSearchParams withQueryBits(int queryBits) { - return new VectorSearchParams(topK, nprobe, efSearch, queryBits); + public VectorSearchParams withLSearch(int lSearch) { + return diskAnn(topK, lSearch); } } diff --git a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexJavaApiTest.java b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexJavaApiTest.java index cbe37cc..3e86508 100644 --- a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexJavaApiTest.java +++ b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexJavaApiTest.java @@ -27,6 +27,8 @@ public static void main(String[] args) { testSingleResultCopiesArrays(); testBatchResultCopiesArraysAndSlicesRows(); testMetadata(); + testSearchParametersRemainAlgorithmSpecific(); + testReaderRejectsNegativeAdjacencyCacheBudget(); testClosedReaderRejectsOperations(); testClosedTrainerRejectsOperations(); testClosedTrainingRejectsOperations(); @@ -34,6 +36,16 @@ public static void main(String[] args) { testReaderAndWriterApiCompile(); } + private static void testSearchParametersRemainAlgorithmSpecific() { + VectorSearchParams params = new VectorSearchParams(10, 4).withLSearch(200); + + assertEquals(VectorSearchParams.SEARCH_WIDTH_DISKANN_L_SEARCH, params.searchWidth()); + assertEquals(200, params.width()); + assertEquals( + VectorSearchParams.SEARCH_WIDTH_IVF_NPROBE, + new VectorSearchParams(10, 4).searchWidth()); + } + private static void testSingleResultCopiesArrays() { long[] ids = new long[] {11L, 7L}; float[] distances = new float[] {0.1f, 0.3f}; @@ -82,23 +94,34 @@ public void run() { private static void testMetadata() { VectorIndexMetadata metadata = new VectorIndexMetadata( - "ivf_hnsw_flat", - 16, - 4, - "cosine", - 123L, - 0, - 20, - 150, - 7); - assertEquals("ivf_hnsw_flat", metadata.indexType()); + "ivf_sq", 16, 4, "cosine", 123L, 0, 8, 0, 0, 0, 0.0f); + assertEquals("ivf_sq", metadata.indexType()); assertEquals(16, metadata.dimension()); assertEquals(4, metadata.nlist()); assertEquals("cosine", metadata.metric()); assertEquals(123L, metadata.totalVectors()); - assertEquals(20, metadata.hnswM()); - assertEquals(150, metadata.hnswEfConstruction()); - assertEquals(7, metadata.hnswMaxLevel()); + assertEquals(8, metadata.pqBits()); + assertEquals(0, metadata.rqBits()); + + VectorIndexMetadata diskAnnMetadata = + new VectorIndexMetadata( + "diskann", 128, 1, "l2", 1000L, 16, 4, 0, 64, 100, 1.2f); + assertEquals(4, diskAnnMetadata.pqBits()); + assertEquals(0, diskAnnMetadata.rqBits()); + assertEquals(64, diskAnnMetadata.diskAnnMaxDegree()); + assertEquals(100, diskAnnMetadata.diskAnnBuildSearchListSize()); + assertEquals(Float.valueOf(1.2f), Float.valueOf(diskAnnMetadata.diskAnnAlpha())); + } + + private static void testReaderRejectsNegativeAdjacencyCacheBudget() { + assertThrows(IllegalArgumentException.class, new ThrowingRunnable() { + @Override + public void run() { + new VectorIndexReader( + new EmptyVectorIndexInput(), + -1); + } + }); } private static void testClosedReaderRejectsOperations() { @@ -136,6 +159,12 @@ public void run() { reader.optimizeForSearch(); } }); + assertThrows(IllegalStateException.class, new ThrowingRunnable() { + @Override + public void run() { + reader.warmupQueries(new float[] {0.0f}, 1, 0); + } + }); assertThrows(IllegalStateException.class, new ThrowingRunnable() { @Override public void run() { @@ -214,7 +243,7 @@ public void run() { } private static void testReaderAndWriterApiCompile() { - Map options = ivfPqOptions(2, 4, 1); + Map options = ivfPqOptions(2, 4); VectorIndexReader closedReader = VectorIndexReader.fromNativePointerForTesting(0L); closedReader.close(); closedReader.close(); @@ -225,32 +254,33 @@ private static void testReaderAndWriterApiCompile() { if (System.currentTimeMillis() < 0) { VectorIndexReader reader = new VectorIndexReader(new EmptyVectorIndexInput()); + VectorIndexReader coalescedReader = + new VectorIndexReader( + new EmptyVectorIndexInput(), + 4L * 1024 * 1024 * 1024); + coalescedReader.close(); reader.metadata(); reader.indexType(); reader.dimension(); reader.totalVectors(); reader.optimizeForSearch(); + reader.warmupQueries(new float[] {0.0f, 1.0f}, 1, 100); + reader.readPlan(); VectorSearchParams params = new VectorSearchParams(10, 4); - VectorSearchParams hnswParams = params.withEfSearch(32); - VectorSearchParams rqParams = hnswParams.withQueryBits(4); + VectorSearchParams diskAnnParams = params.withLSearch(100); reader.search(new float[] {0.0f, 1.0f}, params); - reader.search(new float[] {0.0f, 1.0f}, hnswParams); - reader.search(new float[] {0.0f, 1.0f}, rqParams); + reader.search(new float[] {0.0f, 1.0f}, diskAnnParams); reader.search(new float[] {0.0f, 1.0f}, params, new byte[] {1, 2}); - reader.search(new float[] {0.0f, 1.0f}, hnswParams, new byte[] {1, 2}); - reader.search(new float[] {0.0f, 1.0f}, rqParams, new byte[] {1, 2}); + reader.search(new float[] {0.0f, 1.0f}, diskAnnParams, new byte[] {1, 2}); reader.searchBatch(new float[] {0.0f, 1.0f, 2.0f, 3.0f}, 2, params); - reader.searchBatch(new float[] {0.0f, 1.0f, 2.0f, 3.0f}, 2, hnswParams); - reader.searchBatch(new float[] {0.0f, 1.0f, 2.0f, 3.0f}, 2, rqParams); - reader.searchBatch(new float[] {0.0f, 1.0f, 2.0f, 3.0f}, 2, params, new byte[] {1, 2}); reader.searchBatch( - new float[] {0.0f, 1.0f, 2.0f, 3.0f}, 2, hnswParams, new byte[] {1, 2}); + new float[] {0.0f, 1.0f, 2.0f, 3.0f}, 2, diskAnnParams); + reader.searchBatch(new float[] {0.0f, 1.0f, 2.0f, 3.0f}, 2, params, new byte[] {1, 2}); reader.searchBatch( new float[] {0.0f, 1.0f, 2.0f, 3.0f}, 2, - rqParams, + diskAnnParams, new byte[] {1, 2}); - VectorIndexTraining training = VectorIndexTrainer.train(options, new float[] {0.0f, 1.0f, 2.0f, 3.0f}, 2); VectorIndexWriter writer = new VectorIndexWriter(training); @@ -279,10 +309,9 @@ private static Map ivfFlatOptions(int dimension, int nlist) { return options; } - private static Map ivfPqOptions(int dimension, int nlist, int m) { + private static Map ivfPqOptions(int dimension, int nlist) { Map options = ivfFlatOptions(dimension, nlist); options.put("index.type", "ivf_pq"); - options.put("pq.m", Integer.toString(m)); options.put("use-opq", "false"); return options; } diff --git a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeHandleSafetyTest.java b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeHandleSafetyTest.java index 563205c..ac348b9 100644 --- a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeHandleSafetyTest.java +++ b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeHandleSafetyTest.java @@ -20,6 +20,8 @@ import java.io.ByteArrayOutputStream; import java.util.HashMap; import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicBoolean; public class VectorIndexNativeHandleSafetyTest { @@ -32,6 +34,7 @@ public static void main(String[] args) { testWriterRejectsReentrantCloseDuringNativeCall(); testReaderRejectsReentrantCloseDuringNativeCall(); + testReaderRejectsWorkerCallbackReentryDuringBatchSearch(); } private static void testWriterRejectsReentrantCloseDuringNativeCall() { @@ -63,6 +66,27 @@ private static void testReaderRejectsReentrantCloseDuringNativeCall() { } } + private static void testReaderRejectsWorkerCallbackReentryDuringBatchSearch() { + WorkerReentrantInput input = new WorkerReentrantInput(buildDiskAnnIndexBytes()); + VectorIndexReader reader = new VectorIndexReader(input, 83446L); + input.setReader(reader); + input.setOperationThread(Thread.currentThread()); + try { + float[] queries = new float[64 * 16]; + Random random = new Random(11L); + for (int offset = 0; offset < queries.length; offset++) { + queries[offset] = (float) random.nextGaussian(); + } + VectorSearchBatchResult result = + reader.searchBatch(queries, 64, VectorSearchParams.diskAnn(1, 100)); + assertEquals(64, result.ids().length); + assertTrue(input.workerReentryAttempted()); + assertTrue(input.workerReentryRejected()); + } finally { + reader.close(); + } + } + private static byte[] buildIndexBytes() { VectorIndexWriter writer = newPopulatedWriter(); ByteArrayPositionOutputStream output = new ByteArrayPositionOutputStream(); @@ -74,6 +98,41 @@ private static byte[] buildIndexBytes() { } } + private static byte[] buildDiskAnnIndexBytes() { + int dimension = 16; + int vectorCount = 5000; + float[] data = new float[vectorCount * dimension]; + long[] ids = new long[vectorCount]; + Random random = new Random(7L); + for (int row = 0; row < vectorCount; row++) { + ids[row] = row; + for (int column = 0; column < dimension; column++) { + data[row * dimension + column] = (float) random.nextGaussian(); + } + } + + Map options = new HashMap(); + options.put("index.type", "diskann"); + options.put("dimension", Integer.toString(dimension)); + options.put("metric", "l2"); + options.put("pq.m", "4"); + options.put("pq.bits", "4"); + options.put("diskann.max-degree", "8"); + options.put("diskann.build-search-list-size", "16"); + + VectorIndexWriter writer = + new VectorIndexWriter( + VectorIndexTrainer.train(options, data, vectorCount)); + ByteArrayPositionOutputStream output = new ByteArrayPositionOutputStream(); + try { + writer.addVectors(ids, data, vectorCount); + writer.writeIndex(output); + return output.toByteArray(); + } finally { + writer.close(); + } + } + private static VectorIndexWriter newPopulatedWriter() { VectorIndexWriter writer = new VectorIndexWriter( @@ -206,4 +265,60 @@ private void tryCloseReaderOnce() { } } } + + public static final class WorkerReentrantInput implements VectorIndexInput { + private final byte[] data; + private final AtomicBoolean reentryAttempted = new AtomicBoolean(); + private final AtomicBoolean reentryRejected = new AtomicBoolean(); + private volatile VectorIndexReader reader; + private volatile Thread operationThread; + + WorkerReentrantInput(byte[] data) { + this.data = data.clone(); + } + + void setReader(VectorIndexReader reader) { + this.reader = reader; + } + + void setOperationThread(Thread operationThread) { + this.operationThread = operationThread; + } + + @Override + public void pread(long[] positions, byte[][] buffers) { + VectorIndexReader currentReader = reader; + if (currentReader != null + && Thread.currentThread() != operationThread + && reentryAttempted.compareAndSet(false, true)) { + try { + currentReader.metadata(); + } catch (IllegalStateException expected) { + reentryRejected.set( + expected.getMessage() != null + && expected.getMessage().contains("input callback")); + } + } + + if (positions.length != buffers.length) { + throw new IllegalArgumentException("positions and buffers length mismatch"); + } + for (int i = 0; i < positions.length; i++) { + long readPosition = positions[i]; + byte[] buffer = buffers[i]; + if (readPosition < 0 || readPosition + buffer.length > data.length) { + throw new IllegalArgumentException("read out of range: " + readPosition); + } + System.arraycopy(data, (int) readPosition, buffer, 0, buffer.length); + } + } + + boolean workerReentryAttempted() { + return reentryAttempted.get(); + } + + boolean workerReentryRejected() { + return reentryRejected.get(); + } + } } diff --git a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeValidationTest.java b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeValidationTest.java index d8f95c0..f872df0 100644 --- a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeValidationTest.java +++ b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeValidationTest.java @@ -41,7 +41,126 @@ public static void main(String[] args) { testStagedTrainingStateValidation(); testReaderValidationComesFromCore(); testReaderRejectsNonFiniteQueries(); + testReaderCapabilityFailuresArePropagatedBeforeOpen(); + testHighLevelTrainingInfersDimensionAndIvfShape(); + testHighLevelTrainingPreservesExpectedVectorCount(); testSupportedIndexRoundtrips(); + testDiskAnnInnerProductAndCosine(); + } + + private static void testReaderCapabilityFailuresArePropagatedBeforeOpen() { + final VectorIndexInput throwingInput = + new VectorIndexInput() { + @Override + public void pread(long[] positions, byte[][] buffers) {} + + @Override + public long preferredReadWindowBytes() { + throw new IllegalStateException("capability boom"); + } + }; + assertThrowsMessage( + IllegalStateException.class, + "capability boom", + new ThrowingRunnable() { + @Override + public void run() { + new VectorIndexReader(throwingInput); + } + }); + + final VectorIndexInput negativeInput = + new VectorIndexInput() { + @Override + public void pread(long[] positions, byte[][] buffers) {} + + @Override + public long preferredReadWindowBytes() { + return -1L; + } + }; + assertThrowsMessage( + RuntimeException.class, + "preferredReadWindowBytes must be non-negative", + new ThrowingRunnable() { + @Override + public void run() { + new VectorIndexReader(negativeInput); + } + }); + + final VectorIndexInput negativeLatencyInput = + new VectorIndexInput() { + @Override + public void pread(long[] positions, byte[][] buffers) {} + + @Override + public long estimatedRandomReadLatencyNanos() { + return -1L; + } + }; + assertThrowsMessage( + RuntimeException.class, + "estimatedRandomReadLatencyNanos must be non-negative", + new ThrowingRunnable() { + @Override + public void run() { + new VectorIndexReader(negativeLatencyInput); + } + }); + } + + private static void testHighLevelTrainingInfersDimensionAndIvfShape() { + Map options = new HashMap(); + options.put("index.type", "ivf_sq"); + options.put("metric", "l2"); + byte[] indexBytes = + buildIndexBytes( + options, roundtripData(), roundtripIds(), ROUNDTRIP_VECTOR_COUNT); + VectorIndexReader reader = + new VectorIndexReader(new ByteArraySeekableInputStream(indexBytes)); + try { + VectorIndexMetadata metadata = reader.metadata(); + assertEquals(ROUNDTRIP_DIMENSION, metadata.dimension()); + assertEquals(8, metadata.nlist()); + VectorSearchResult result = + reader.search(queryForCenter(0.0f), VectorSearchParams.automatic(2)); + assertIdInCluster(result.ids()[0], 0); + } finally { + reader.close(); + } + } + + private static void testHighLevelTrainingPreservesExpectedVectorCount() { + int dimension = 8; + int vectorCount = 64; + float[] data = new float[vectorCount * dimension]; + for (int row = 0; row < vectorCount; row++) { + int cluster = row % 8; + for (int column = 0; column < dimension; column++) { + data[row * dimension + column] = + cluster * 20.0f + column * 0.01f + row * 0.0001f; + } + } + long[] ids = new long[vectorCount]; + for (int i = 0; i < ids.length; i++) { + ids[i] = i; + } + Map options = new HashMap<>(); + options.put("index.type", "ivf_sq"); + options.put("dimension", "auto"); + options.put("nlist", "auto"); + options.put("expected-vector-count", "65536"); + options.put("metric", "l2"); + + byte[] bytes = buildIndexBytes(options, data, ids, vectorCount); + VectorIndexReader reader = + new VectorIndexReader(new ByteArraySeekableInputStream(bytes)); + try { + assertEquals(256, reader.metadata().nlist()); + } finally { + reader.close(); + } } private static void testWriterValidationComesFromCore() { @@ -169,19 +288,14 @@ private static void testStagedTrainingRoundtrip() { runStagedTrainingRoundtrip( "ivf_flat", ivfFlatOptions(ROUNDTRIP_DIMENSION, ROUNDTRIP_NLIST), 0, 0); runStagedTrainingRoundtrip( - "ivf_pq", ivfPqOptions(ROUNDTRIP_DIMENSION, ROUNDTRIP_NLIST, 4), 4, 0); + "ivf_pq", ivfPqOptions(ROUNDTRIP_DIMENSION, ROUNDTRIP_NLIST), 2, 8); runStagedTrainingRoundtrip( "ivf_rq", ivfRqOptions(ROUNDTRIP_DIMENSION, ROUNDTRIP_NLIST), 0, 0); runStagedTrainingRoundtrip( - "ivf_hnsw_flat", - ivfHnswOptions("ivf_hnsw_flat", ROUNDTRIP_DIMENSION, ROUNDTRIP_NLIST), - 0, - 4); - runStagedTrainingRoundtrip( - "ivf_hnsw_sq", - ivfHnswOptions("ivf_hnsw_sq", ROUNDTRIP_DIMENSION, ROUNDTRIP_NLIST), + "ivf_sq", + ivfSqOptions(ROUNDTRIP_DIMENSION, ROUNDTRIP_NLIST), 0, - 4); + 8); } private static void testStagedTrainingStateValidation() { @@ -310,86 +424,135 @@ public void run() { private static void testSupportedIndexRoundtrips() { runRoundtrip("ivf_flat", ivfFlatOptions(ROUNDTRIP_DIMENSION, ROUNDTRIP_NLIST), 0, 0); - runRoundtrip("ivf_pq", ivfPqOptions(ROUNDTRIP_DIMENSION, ROUNDTRIP_NLIST, 4), 4, 0); + runRoundtrip("ivf_pq", ivfPqOptions(ROUNDTRIP_DIMENSION, ROUNDTRIP_NLIST), 2, 8); runRoundtrip("ivf_rq", ivfRqOptions(ROUNDTRIP_DIMENSION, ROUNDTRIP_NLIST), 0, 0); runRoundtrip( - "ivf_hnsw_flat", - ivfHnswOptions("ivf_hnsw_flat", ROUNDTRIP_DIMENSION, ROUNDTRIP_NLIST), + "ivf_sq", + ivfSqOptions(ROUNDTRIP_DIMENSION, ROUNDTRIP_NLIST), 0, - 4); + 8); runRoundtrip( - "ivf_hnsw_sq", - ivfHnswOptions("ivf_hnsw_sq", ROUNDTRIP_DIMENSION, ROUNDTRIP_NLIST), - 0, + "diskann", + diskAnnOptions(ROUNDTRIP_DIMENSION), + 2, 4); } + private static void testDiskAnnInnerProductAndCosine() { + float[] data = + new float[] { + 10.0f, 0.0f, + 1.0f, 1.0f, + 0.0f, 8.0f, + 0.0f, 1.0f, + -1.0f, 0.0f, + 0.0f, -1.0f, + -2.0f, 1.0f, + 1.0f, -2.0f, + -3.0f, -1.0f, + -1.0f, -3.0f, + -4.0f, 0.5f, + 0.5f, -4.0f, + -5.0f, -2.0f, + -2.0f, -5.0f, + -6.0f, -1.0f, + -2.0f, 0.0f + }; + long[] ids = new long[16]; + for (int i = 0; i < ids.length; i++) { + ids[i] = 100L + i; + } + for (String metric : new String[] {"inner_product", "cosine"}) { + Map options = diskAnnOptions(2); + options.put("metric", metric); + options.put("pq.m", "1"); + options.put("diskann.raw-vector-encoding", "f32"); + byte[] indexBytes = buildIndexBytes(options, data, ids, ids.length); + VectorIndexReader reader = + new VectorIndexReader(new ByteArraySeekableInputStream(indexBytes)); + try { + assertEquals(metric, reader.metadata().metric()); + VectorSearchResult result = + reader.search( + new float[] {1.0f, 0.0f}, + new VectorSearchParams(1, 1).withLSearch(16)); + assertEquals(100L, result.ids()[0]); + float expected = "inner_product".equals(metric) ? -10.0f : 0.0f; + assertNear(expected, result.distances()[0], 1e-5f); + } finally { + reader.close(); + } + } + } + private static void runRoundtrip( - String indexType, Map options, int expectedPqM, int expectedHnswM) { + String indexType, Map options, int expectedPqM, int expectedPqBits) { byte[] indexBytes = buildIndexBytes( options, roundtripData(), roundtripIds(), ROUNDTRIP_VECTOR_COUNT); - assertRoundtrip(indexType, indexBytes, expectedPqM, expectedHnswM); + assertRoundtrip(indexType, indexBytes, expectedPqM, expectedPqBits); } private static void runStagedTrainingRoundtrip( - String indexType, Map options, int expectedPqM, int expectedHnswM) { + String indexType, Map options, int expectedPqM, int expectedPqBits) { byte[] indexBytes = buildStagedIndexBytes( options, roundtripData(), roundtripIds(), ROUNDTRIP_VECTOR_COUNT); - assertRoundtrip(indexType, indexBytes, expectedPqM, expectedHnswM); + assertRoundtrip(indexType, indexBytes, expectedPqM, expectedPqBits); } private static void assertRoundtrip( - String indexType, byte[] indexBytes, int expectedPqM, int expectedHnswM) { + String indexType, byte[] indexBytes, int expectedPqM, int expectedPqBits) { VectorIndexReader reader = new VectorIndexReader(new ByteArraySeekableInputStream(indexBytes)); try { VectorIndexMetadata metadata = reader.metadata(); assertEquals(indexType, metadata.indexType()); assertEquals(ROUNDTRIP_DIMENSION, metadata.dimension()); - assertEquals(ROUNDTRIP_NLIST, metadata.nlist()); + assertEquals("diskann".equals(indexType) ? 1 : ROUNDTRIP_NLIST, metadata.nlist()); assertEquals("l2", metadata.metric()); assertEquals((long) ROUNDTRIP_VECTOR_COUNT, metadata.totalVectors()); assertEquals(expectedPqM, metadata.pqM()); - assertEquals(expectedHnswM, metadata.hnswM()); + assertEquals(expectedPqBits, metadata.pqBits()); + assertEquals("ivf_rq".equals(indexType) ? 5 : 0, metadata.rqBits()); + if ("diskann".equals(indexType)) { + VectorIndexReadPlan plan = reader.readPlan(); + assertEquals(4L * 1024 * 1024 * 1024, plan.memoryBudgetBytes()); + if (plan.randomReadLatencyNanos() <= 0 || plan.windowBytes() <= 0) { + throw new AssertionError("DiskANN read plan was not resolved during open"); + } + } reader.optimizeForSearch(); - VectorSearchParams params = new VectorSearchParams(2, 4, 16, 0); + final VectorSearchParams params; + if ("diskann".equals(indexType)) { + int calibrated = reader.calibrateSearchWidth(queryForCenter(0.0f), 1, 2); + if (calibrated != 100 && calibrated != 200 && calibrated != 400) { + throw new AssertionError("unexpected calibrated width: " + calibrated); + } + params = VectorSearchParams.automatic(2); + reader.warmupQueries(queryForCenter(0.0f), 1, 16); + } else { + params = new VectorSearchParams(2, 4); + } VectorSearchResult single = reader.search(queryForCenter(0.0f), params); assertIdInCluster(single.ids()[0], 0); assertFinite(single.distances()[0], indexType + " single distance"); + final VectorSearchParams batchParams; + if ("diskann".equals(indexType)) { + batchParams = new VectorSearchParams(1, 4).withLSearch(16); + } else { + batchParams = new VectorSearchParams(1, 4); + } VectorSearchBatchResult batch = - reader.searchBatch(batchQueries(), 2, new VectorSearchParams(1, 4, 16, 0)); + reader.searchBatch(batchQueries(), 2, batchParams); assertIdInCluster(batch.ids()[0], 0); assertIdInCluster(batch.ids()[1], 1); assertFinite(batch.distances()[0], indexType + " batch distance 0"); assertFinite(batch.distances()[1], indexType + " batch distance 1"); - if ("ivf_rq".equals(indexType)) { - VectorSearchResult queryBitsSingle = - reader.search(queryForCenter(0.0f), params.withQueryBits(4)); - assertIdInCluster(queryBitsSingle.ids()[0], 0); - assertFinite(queryBitsSingle.distances()[0], "ivf_rq queryBits single distance"); - - VectorSearchBatchResult queryBitsBatch = - reader.searchBatch( - batchQueries(), 2, new VectorSearchParams(1, 4, 16, 8)); - assertIdInCluster(queryBitsBatch.ids()[0], 0); - assertIdInCluster(queryBitsBatch.ids()[1], 1); - - assertThrowsMessage( - RuntimeException.class, - "query_bits", - new ThrowingRunnable() { - @Override - public void run() { - reader.search(queryForCenter(0.0f), params.withQueryBits(7)); - } - }); - } } finally { reader.close(); } @@ -462,10 +625,9 @@ private static Map ivfFlatOptions(int dimension, int nlist) { return options; } - private static Map ivfPqOptions(int dimension, int nlist, int m) { + private static Map ivfPqOptions(int dimension, int nlist) { Map options = ivfFlatOptions(dimension, nlist); options.put("index.type", "ivf_pq"); - options.put("pq.m", Integer.toString(m)); options.put("use-opq", "false"); return options; } @@ -473,15 +635,25 @@ private static Map ivfPqOptions(int dimension, int nlist, int m) private static Map ivfRqOptions(int dimension, int nlist) { Map options = ivfFlatOptions(dimension, nlist); options.put("index.type", "ivf_rq"); + options.put("rq.bits", "5"); return options; } - private static Map ivfHnswOptions(String indexType, int dimension, int nlist) { + private static Map ivfSqOptions(int dimension, int nlist) { Map options = ivfFlatOptions(dimension, nlist); - options.put("index.type", indexType); - options.put("hnsw.m", "4"); - options.put("hnsw.ef-construction", "16"); - options.put("hnsw.max-level", "4"); + options.put("index.type", "ivf_sq"); + return options; + } + + private static Map diskAnnOptions(int dimension) { + Map options = new HashMap(); + options.put("index.type", "diskann"); + options.put("dimension", Integer.toString(dimension)); + options.put("metric", "l2"); + options.put("pq.m", "2"); + options.put("pq.bits", "4"); + options.put("diskann.max-degree", "8"); + options.put("diskann.build-search-list-size", "16"); return options; } @@ -542,6 +714,12 @@ private static void assertEquals(long expected, long actual) { } } + private static void assertNear(float expected, float actual, float tolerance) { + if (Math.abs(expected - actual) > tolerance) { + throw new AssertionError("expected " + expected + " but got " + actual); + } + } + private static void assertEquals(Object expected, Object actual) { if (!expected.equals(actual)) { throw new AssertionError("expected " + expected + " but got " + actual); diff --git a/jni/Cargo.toml b/jni/Cargo.toml index c698579..f814f61 100644 --- a/jni/Cargo.toml +++ b/jni/Cargo.toml @@ -17,15 +17,17 @@ [package] name = "paimon-vindex-jni" -version = "0.2.0" +version = "0.3.0" edition = "2021" description = "Apache Paimon Vector Index — JNI bindings for Java" license = "Apache-2.0" +repository = "https://github.com/apache/paimon-vector-index" +homepage = "https://paimon.apache.org" [lib] name = "paimon_vindex_jni" crate-type = ["cdylib"] [dependencies] -paimon-vindex-core = { path = "../core", version = "0.2.0" } +paimon-vindex-core = { path = "../core", version = "0.3.0" } jni = "0.21" diff --git a/jni/src/lib.rs b/jni/src/lib.rs index cfc0ecc..c56aaaa 100644 --- a/jni/src/lib.rs +++ b/jni/src/lib.rs @@ -21,13 +21,14 @@ use jni::objects::{JByteArray, JClass, JFloatArray, JLongArray, JObject, JValue} use jni::sys::{jint, jlong, jobject, jobjectArray}; use jni::JNIEnv; use paimon_vindex_core::index::{ - VectorIndexConfig, VectorIndexMetadata, VectorIndexReader, VectorIndexTrainer, - VectorIndexTraining, VectorIndexWriter, VectorSearchParams, + SearchWidth, VectorIndexConfig, VectorIndexMetadata, VectorIndexReadPlan, VectorIndexReader, + VectorIndexReaderOptions, VectorIndexTrainer, VectorIndexTraining, VectorIndexWriter, + VectorSearchParams, }; use std::any::Any; use std::collections::HashMap; use std::panic::{catch_unwind, AssertUnwindSafe}; -use stream::{JniOutputStream, JniSeekableStream}; +use stream::{read_capabilities, JniOutputStream, JniSeekableStream}; fn throw_and_return(env: &mut JNIEnv, msg: &str) -> T { let _ = env.throw_new("java/lang/RuntimeException", msg); @@ -344,13 +345,19 @@ fn build_metadata(env: &mut JNIEnv, metadata: VectorIndexMetadata) -> jobject { Ok(value) => JObject::from(value), Err(e) => return throw_and_return(env, &format!("new_string(metric): {}", e)), }; - let (hnsw_m, ef_construction, max_level) = metadata - .hnsw - .map(|h| (h.m as jint, h.ef_construction as jint, h.max_level as jint)) - .unwrap_or((0, 0, 0)); + let (diskann_max_degree, diskann_build_search_list_size, diskann_alpha) = metadata + .diskann + .map(|d| { + ( + d.max_degree as jint, + d.build_search_list_size as jint, + d.alpha, + ) + }) + .unwrap_or((0, 0, 0.0)); let result = match env.new_object( class, - "(Ljava/lang/String;IILjava/lang/String;JIIII)V", + "(Ljava/lang/String;IILjava/lang/String;JIIIIIF)V", &[ JValue::Object(&index_type), JValue::Int(metadata.dimension as jint), @@ -358,9 +365,11 @@ fn build_metadata(env: &mut JNIEnv, metadata: VectorIndexMetadata) -> jobject { JValue::Object(&metric), JValue::Long(metadata.total_vectors), JValue::Int(metadata.pq_m.unwrap_or(0) as jint), - JValue::Int(hnsw_m), - JValue::Int(ef_construction), - JValue::Int(max_level), + JValue::Int(metadata.pq_bits.unwrap_or(0) as jint), + JValue::Int(metadata.rq_bits.unwrap_or(0) as jint), + JValue::Int(diskann_max_degree), + JValue::Int(diskann_build_search_list_size), + JValue::Float(diskann_alpha), ], ) { Ok(r) => r, @@ -369,25 +378,60 @@ fn build_metadata(env: &mut JNIEnv, metadata: VectorIndexMetadata) -> jobject { result.into_raw() } +fn build_read_plan(env: &mut JNIEnv, plan: VectorIndexReadPlan) -> jobject { + let class = match env.find_class("org/apache/paimon/index/vector/VectorIndexReadPlan") { + Ok(class) => class, + Err(error) => return throw_and_return(env, &format!("find_class: {error}")), + }; + let usize_to_jlong = |value: usize| i64::try_from(value).unwrap_or(i64::MAX); + let u64_to_jlong = |value: u64| i64::try_from(value).unwrap_or(i64::MAX); + let result = match env.new_object( + class, + "(JJJJJJJJJ)V", + &[ + JValue::Long(u64_to_jlong(plan.random_read_latency_nanos)), + JValue::Long(usize_to_jlong(plan.window_bytes)), + JValue::Long(usize_to_jlong(plan.max_ranges_per_read)), + JValue::Long(usize_to_jlong(plan.graph_beam_width)), + JValue::Long(usize_to_jlong(plan.filtered_graph_beam_width)), + JValue::Long(usize_to_jlong(plan.adjacency_preload_bytes)), + JValue::Long(usize_to_jlong(plan.adjacency_cache_bytes)), + JValue::Long(usize_to_jlong(plan.raw_vector_cache_bytes)), + JValue::Long(usize_to_jlong(plan.memory_budget_bytes)), + ], + ) { + Ok(result) => result, + Err(error) => return throw_and_return(env, &format!("new_object: {error}")), + }; + result.into_raw() +} + fn search_params(env: &mut JNIEnv, params: JObject) -> Result { if params.is_null() { return Err("params is null".to_string()); } let top_k = call_int_method(env, ¶ms, "topK")?; - let nprobe = call_int_method(env, ¶ms, "nprobe")?; - let ef_search = call_int_method(env, ¶ms, "efSearch")?; - let query_bits = call_int_method(env, ¶ms, "queryBits")?; - if top_k < 0 || nprobe < 0 || ef_search < 0 || query_bits < 0 { + let search_width = call_int_method(env, ¶ms, "searchWidth")?; + let width = call_int_method(env, ¶ms, "width")?; + if top_k < 0 || width < 0 { return Err(format!( - "invalid search parameters: topK={}, nprobe={}, efSearch={}, queryBits={}", - top_k, nprobe, ef_search, query_bits + "invalid search parameters: topK={}, searchWidth={}, width={}", + top_k, search_width, width )); } + let search_width = match search_width { + 0 => SearchWidth::Auto, + 1 => SearchWidth::IvfNProbe, + 2 => SearchWidth::DiskAnnLSearch, + value => return Err(format!("invalid search width type: {value}")), + }; + if search_width == SearchWidth::Auto && width != 0 { + return Err("automatic search width must have width=0".to_string()); + } Ok(VectorSearchParams { top_k: top_k as usize, - nprobe: nprobe as usize, - ef_search: ef_search as usize, - query_bits: query_bits as usize, + search_width, + width: width as usize, }) } @@ -648,12 +692,21 @@ pub extern "system" fn Java_org_apache_paimon_index_vector_VectorIndexNative_ope Ok(vm) => vm, Err(e) => return throw_and_return(env, &format!("get_java_vm: {}", e)), }; + let capabilities = match read_capabilities(env, &stream_input) { + Ok(capabilities) => capabilities, + Err(error) => { + if env.exception_check().unwrap_or(false) { + return 0; + } + return throw_and_return(env, &format!("read capabilities: {error}")); + } + }; let global_ref = match env.new_global_ref(stream_input) { Ok(r) => r, Err(e) => return throw_and_return(env, &format!("new_global_ref: {}", e)), }; - let stream = JniSeekableStream::new(jvm, global_ref); + let stream = JniSeekableStream::new(jvm, global_ref, capabilities); let reader = match VectorIndexReader::open(stream) { Ok(reader) => reader, Err(e) => return throw_and_return(env, &format!("open reader: {}", e)), @@ -662,6 +715,51 @@ pub extern "system" fn Java_org_apache_paimon_index_vector_VectorIndexNative_ope }) } +#[no_mangle] +pub extern "system" fn Java_org_apache_paimon_index_vector_VectorIndexNative_openReaderWithOptions( + env: JNIEnv, + _class: JClass, + stream_input: JObject, + memory_budget_bytes: jlong, +) -> jlong { + jni_call(env, |env| { + if memory_budget_bytes < 0 { + return throw_and_return(env, "memory budget bytes must be non-negative"); + } + let memory_budget_bytes = match usize::try_from(memory_budget_bytes) { + Ok(value) => value, + Err(_) => return throw_and_return(env, "memory budget bytes exceed usize"), + }; + let jvm = match env.get_java_vm() { + Ok(vm) => vm, + Err(e) => return throw_and_return(env, &format!("get_java_vm: {}", e)), + }; + let capabilities = match read_capabilities(env, &stream_input) { + Ok(capabilities) => capabilities, + Err(error) => { + if env.exception_check().unwrap_or(false) { + return 0; + } + return throw_and_return(env, &format!("read capabilities: {error}")); + } + }; + let global_ref = match env.new_global_ref(stream_input) { + Ok(r) => r, + Err(e) => return throw_and_return(env, &format!("new_global_ref: {}", e)), + }; + + let stream = JniSeekableStream::new(jvm, global_ref, capabilities); + let reader = match VectorIndexReader::open_with_options( + stream, + VectorIndexReaderOptions::new(memory_budget_bytes), + ) { + Ok(reader) => reader, + Err(e) => return throw_and_return(env, &format!("open reader: {}", e)), + }; + Box::into_raw(Box::new(reader)) as jlong + }) +} + #[no_mangle] pub extern "system" fn Java_org_apache_paimon_index_vector_VectorIndexNative_metadata( env: JNIEnv, @@ -694,6 +792,81 @@ pub extern "system" fn Java_org_apache_paimon_index_vector_VectorIndexNative_opt }) } +#[no_mangle] +pub extern "system" fn Java_org_apache_paimon_index_vector_VectorIndexNative_warmupQueries( + env: JNIEnv, + _class: JClass, + ptr: jlong, + queries: JFloatArray, + query_count: jint, + l_search: jint, +) { + jni_call_void(env, |env| { + let reader = match deref_reader(ptr) { + Some(reader) => reader, + None => return throw_and_return(env, "null native pointer (reader already freed?)"), + }; + if query_count < 0 || l_search < 0 { + return throw_and_return(env, "warmup query count and lSearch must be non-negative"); + } + let query_buf = match read_float_array(env, &queries, "warmup queries") { + Ok(buf) => buf, + Err(e) => return throw_and_return(env, &e), + }; + if let Err(e) = reader.warmup_queries(&query_buf, query_count as usize, l_search as usize) { + throw_and_return::<()>(env, &format!("warmup_queries: {}", e)); + } + }) +} + +#[no_mangle] +pub extern "system" fn Java_org_apache_paimon_index_vector_VectorIndexNative_calibrateSearchWidth( + env: JNIEnv, + _class: JClass, + ptr: jlong, + queries: JFloatArray, + query_count: jint, + top_k: jint, +) -> jint { + jni_call(env, |env| { + let reader = match deref_reader(ptr) { + Some(reader) => reader, + None => return throw_and_return(env, "null native pointer (reader already freed?)"), + }; + if query_count <= 0 || top_k <= 0 { + return throw_and_return(env, "calibration queryCount and topK must be positive"); + } + let query_buf = match read_float_array(env, &queries, "calibration queries") { + Ok(buf) => buf, + Err(e) => return throw_and_return(env, &e), + }; + match reader.calibrate_search_width(&query_buf, query_count as usize, top_k as usize) { + Ok(width) => width as jint, + Err(e) => throw_and_return(env, &format!("calibrate_search_width: {e}")), + } + }) +} + +#[no_mangle] +pub extern "system" fn Java_org_apache_paimon_index_vector_VectorIndexNative_readPlan( + env: JNIEnv, + _class: JClass, + ptr: jlong, +) -> jobject { + jni_call(env, |env| { + let reader = match deref_reader(ptr) { + Some(reader) => reader, + None => { + return throw_and_return(env, "null native pointer (reader already freed?)"); + } + }; + match reader.read_plan() { + Some(plan) => build_read_plan(env, plan), + None => throw_and_return(env, "read plan is only available for DiskANN"), + } + }) +} + #[no_mangle] pub extern "system" fn Java_org_apache_paimon_index_vector_VectorIndexNative_search( env: JNIEnv, diff --git a/jni/src/stream.rs b/jni/src/stream.rs index f0b0364..a10bdc9 100644 --- a/jni/src/stream.rs +++ b/jni/src/stream.rs @@ -17,25 +17,30 @@ use jni::objects::{GlobalRef, JByteArray, JObject, JObjectArray, JValue}; use jni::JavaVM; -use paimon_vindex_core::io::{ReadRequest, SeekRead}; +use paimon_vindex_core::io::{ReadRequest, SeekRead, SeekReadCapabilities}; use std::io; use std::sync::Arc; /// JNI-backed input stream that delegates to Java's VectorIndexInput. +#[derive(Clone)] pub struct JniSeekableStream { jvm: Arc, stream_ref: Arc, + capabilities: SeekReadCapabilities, } impl JniSeekableStream { - pub fn new(jvm: JavaVM, stream_ref: GlobalRef) -> Self { + pub fn new(jvm: JavaVM, stream_ref: GlobalRef, capabilities: SeekReadCapabilities) -> Self { JniSeekableStream { jvm: Arc::new(jvm), stream_ref: Arc::new(stream_ref), + capabilities, } } } +const MAX_JNI_RANGES_PER_CALL: usize = 256; + impl SeekRead for JniSeekableStream { /// Positional reads via VectorIndexInput.pread(long[] positions, byte[][] buffers). fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { @@ -48,37 +53,87 @@ impl SeekRead for JniSeekableStream { .attach_current_thread() .map_err(|e| io::Error::other(format!("JNI attach: {}", e)))?; - let positions = env - .new_long_array(ranges.len() as i32) - .map_err(|e| io::Error::other(format!("JNI alloc positions: {}", e)))?; - let position_values: Vec = ranges.iter().map(|range| range.pos as i64).collect(); - env.set_long_array_region(&positions, 0, &position_values) - .map_err(|e| io::Error::other(format!("JNI set positions: {}", e)))?; - - let byte_array_class = env - .find_class("[B") - .map_err(|e| io::Error::other(format!("JNI find byte[] class: {}", e)))?; - let buffers = env - .new_object_array(ranges.len() as i32, byte_array_class, JObject::null()) - .map_err(|e| io::Error::other(format!("JNI alloc buffers: {}", e)))?; - for (idx, range) in ranges.iter().enumerate() { - let jbuf = env - .new_byte_array(range.buf.len() as i32) - .map_err(|e| io::Error::other(format!("JNI alloc range buffer: {}", e)))?; - env.set_object_array_element(&buffers, idx as i32, jbuf) - .map_err(|e| io::Error::other(format!("JNI set buffer: {}", e)))?; + for chunk in ranges.chunks_mut(MAX_JNI_RANGES_PER_CALL) { + let result = env + .with_local_frame(16 + chunk.len() as i32, |env| { + Ok::<_, jni::errors::Error>(pread_chunk(env, self.stream_ref.as_obj(), chunk)) + }) + .map_err(|e| io::Error::other(format!("JNI local frame: {e}")))?; + result?; } + Ok(()) + } - env.call_method( - self.stream_ref.as_obj(), - "pread", - "([J[[B)V", - &[JValue::Object(&positions), JValue::Object(&buffers)], - ) - .map_err(|e| io::Error::other(format!("JNI pread: {}", e)))?; + fn try_clone_reader(&self) -> io::Result> { + Ok(Some(self.clone())) + } + + fn read_capabilities(&self) -> SeekReadCapabilities { + self.capabilities + } +} - copy_java_buffers(&mut env, &buffers, ranges) +pub fn read_capabilities( + env: &mut jni::JNIEnv<'_>, + stream: &JObject<'_>, +) -> Result { + let mut read_hint = |name: &str| -> Result { + let value = env + .call_method(stream, name, "()J", &[]) + .map_err(|error| format!("{name}: {error}"))? + .j() + .map_err(|error| format!("{name}: {error}"))?; + if value < 0 { + return Err(format!("{name} must be non-negative, got {value}")); + } + Ok(value) + }; + Ok(SeekReadCapabilities { + estimated_random_read_latency_nanos: read_hint("estimatedRandomReadLatencyNanos")? as u64, + preferred_window_bytes: usize::try_from(read_hint("preferredReadWindowBytes")?) + .map_err(|_| "preferredReadWindowBytes exceeds usize".to_string())?, + max_ranges_per_pread: usize::try_from(read_hint("maxRangesPerRead")?) + .map_err(|_| "maxRangesPerRead exceeds usize".to_string())?, + }) +} + +fn pread_chunk( + env: &mut jni::JNIEnv<'_>, + stream: &JObject<'_>, + ranges: &mut [ReadRequest<'_>], +) -> io::Result<()> { + let positions = env + .new_long_array(ranges.len() as i32) + .map_err(|e| io::Error::other(format!("JNI alloc positions: {}", e)))?; + let position_values: Vec = ranges.iter().map(|range| range.pos as i64).collect(); + env.set_long_array_region(&positions, 0, &position_values) + .map_err(|e| io::Error::other(format!("JNI set positions: {}", e)))?; + + let byte_array_class = env + .find_class("[B") + .map_err(|e| io::Error::other(format!("JNI find byte[] class: {}", e)))?; + let buffers = env + .new_object_array(ranges.len() as i32, byte_array_class, JObject::null()) + .map_err(|e| io::Error::other(format!("JNI alloc buffers: {}", e)))?; + for (idx, range) in ranges.iter().enumerate() { + let jbuf = env + .new_byte_array(range.buf.len() as i32) + .map_err(|e| io::Error::other(format!("JNI alloc range buffer: {}", e)))?; + env.set_object_array_element(&buffers, idx as i32, &jbuf) + .map_err(|e| io::Error::other(format!("JNI set buffer: {}", e)))?; + env.delete_local_ref(jbuf) + .map_err(|e| io::Error::other(format!("JNI delete range buffer ref: {}", e)))?; } + + env.call_method( + stream, + "pread", + "([J[[B)V", + &[JValue::Object(&positions), JValue::Object(&buffers)], + ) + .map_err(|e| io::Error::other(format!("JNI pread: {}", e)))?; + + copy_java_buffers(env, &buffers, ranges) } fn copy_java_buffers( @@ -114,6 +169,8 @@ fn copy_java_buffers( range.buf[i] = b as u8; } } + env.delete_local_ref(jbuf) + .map_err(|e| io::Error::other(format!("JNI delete returned buffer ref: {}", e)))?; } Ok(()) } @@ -142,21 +199,12 @@ impl paimon_vindex_core::io::SeekWrite for JniOutputStream { .attach_current_thread() .map_err(|e| io::Error::other(format!("JNI attach: {}", e)))?; - let jbuf = env - .new_byte_array(buf.len() as i32) - .map_err(|e| io::Error::other(format!("JNI alloc: {}", e)))?; - - let signed: Vec = buf.iter().map(|&b| b as i8).collect(); - env.set_byte_array_region(&jbuf, 0, &signed) - .map_err(|e| io::Error::other(format!("JNI set_region: {}", e)))?; - - env.call_method( - self.stream_ref.as_obj(), - "write", - "([B)V", - &[jni::objects::JValue::Object(&jbuf)], - ) - .map_err(|e| io::Error::other(format!("JNI write: {}", e)))?; + let result = env + .with_local_frame(8, |env| { + Ok::<_, jni::errors::Error>(write_chunk(env, self.stream_ref.as_obj(), buf)) + }) + .map_err(|e| io::Error::other(format!("JNI output local frame: {e}")))?; + result?; self.pos += buf.len() as u64; Ok(()) @@ -166,3 +214,42 @@ impl paimon_vindex_core::io::SeekWrite for JniOutputStream { self.pos } } + +fn write_chunk(env: &mut jni::JNIEnv<'_>, stream: &JObject<'_>, buf: &[u8]) -> io::Result<()> { + let jbuf = env + .new_byte_array(buf.len() as i32) + .map_err(|e| io::Error::other(format!("JNI alloc: {e}")))?; + let signed = buf.iter().map(|&byte| byte as i8).collect::>(); + env.set_byte_array_region(&jbuf, 0, &signed) + .map_err(|e| io::Error::other(format!("JNI set_region: {e}")))?; + env.call_method(stream, "write", "([B)V", &[JValue::Object(&jbuf)]) + .map_err(|e| io::Error::other(format!("JNI write: {e}")))?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn jni_seekable_stream_is_cloneable_for_parallel_diskann_batch() { + fn assert_clone() {} + assert_clone::(); + } + + #[test] + fn jni_range_calls_are_bounded_to_one_local_reference_frame() { + let range_count = 50_000; + let chunk_sizes = (0..range_count) + .collect::>() + .chunks(MAX_JNI_RANGES_PER_CALL) + .map(<[usize]>::len) + .collect::>(); + + assert_eq!(chunk_sizes.iter().sum::(), range_count); + assert!(chunk_sizes + .iter() + .all(|&size| size <= MAX_JNI_RANGES_PER_CALL)); + assert!(chunk_sizes.len() > 1); + } +} diff --git a/python/paimon_vindex/__init__.py b/python/paimon_vindex/__init__.py index 2113220..99037b9 100644 --- a/python/paimon_vindex/__init__.py +++ b/python/paimon_vindex/__init__.py @@ -15,21 +15,97 @@ # specific language governing permissions and limitations # under the License. +import ctypes +import operator +import threading from dataclasses import dataclass +from enum import IntEnum from typing import Mapping, Optional -import ctypes import numpy as np from . import _ffi from ._ffi import lib +_SIZE_T_MAX = ctypes.c_size_t(-1).value +_UINT64_MAX = ctypes.c_uint64(-1).value + + +def _size_t(value, name: str, *, allow_zero: bool) -> int: + try: + value = operator.index(value) + except TypeError as exc: + raise ValueError(f"{name} must be an integer") from exc + lower_bound = 0 if allow_zero else 1 + if not lower_bound <= value <= _SIZE_T_MAX: + raise ValueError( + f"{name} must be in [{lower_bound}, {_SIZE_T_MAX}]" + ) + return value + + +def _uint64(value, name: str) -> int: + try: + value = operator.index(value) + except TypeError as exc: + raise ValueError(f"{name} must be an integer") from exc + if not 0 <= value <= _UINT64_MAX: + raise ValueError(f"{name} must be in [0, {_UINT64_MAX}]") + return value + + +class _NativeHandleLock: + """Serialize a native handle while rejecting reentry from its callbacks.""" + + def __init__(self): + self._lock = threading.Lock() + self._state_lock = threading.Lock() + self._owner = None + self._callback_depths = {} + + def __enter__(self): + current = threading.get_ident() + with self._state_lock: + if ( + self._owner == current + or self._callback_depths.get(current, 0) > 0 + ): + raise RuntimeError( + "reentrant native-handle operation is not allowed" + ) + self._lock.acquire() + with self._state_lock: + self._owner = current + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + with self._state_lock: + self._owner = None + self._lock.release() + return False + + def _enter_callback(self): + current = threading.get_ident() + with self._state_lock: + self._callback_depths[current] = ( + self._callback_depths.get(current, 0) + 1 + ) + + def _exit_callback(self): + current = threading.get_ident() + with self._state_lock: + depth = self._callback_depths[current] + if depth == 1: + del self._callback_depths[current] + else: + self._callback_depths[current] = depth - 1 + INDEX_TYPES = { 0: "ivf_flat", 1: "ivf_pq", - 2: "ivf_hnsw_flat", - 3: "ivf_hnsw_sq", 4: "ivf_rq", + 5: "diskann", + 6: "ivf_sq", } METRICS = { @@ -39,6 +115,12 @@ } +class SearchWidth(IntEnum): + AUTO = 0 + IVF_NPROBE = 1 + DISKANN_L_SEARCH = 2 + + @dataclass(frozen=True) class VectorIndexMetadata: index_type: str @@ -47,24 +129,78 @@ class VectorIndexMetadata: metric: str total_vectors: int pq_m: Optional[int] = None - hnsw_m: Optional[int] = None - hnsw_ef_construction: Optional[int] = None - hnsw_max_level: Optional[int] = None + pq_bits: Optional[int] = None + rq_bits: Optional[int] = None + diskann_max_degree: Optional[int] = None + diskann_build_search_list_size: Optional[int] = None + diskann_alpha: Optional[float] = None + + +@dataclass(frozen=True) +class VectorIndexReadPlan: + random_read_latency_nanos: int + window_bytes: int + max_ranges_per_read: int + graph_beam_width: int + filtered_graph_beam_width: int + adjacency_preload_bytes: int + adjacency_cache_bytes: int + raw_vector_cache_bytes: int + memory_budget_bytes: int @dataclass(frozen=True) class SearchParams: top_k: int - nprobe: int - ef_search: int = 0 - query_bits: int = 0 + search_width: SearchWidth = SearchWidth.AUTO + width: int = 0 + + def __post_init__(self): + try: + search_width = SearchWidth(self.search_width) + except (TypeError, ValueError) as exc: + raise ValueError("search_width is invalid") from exc + top_k = _size_t(self.top_k, "top_k", allow_zero=False) + if search_width == SearchWidth.AUTO: + width = _size_t(self.width, "automatic search width", allow_zero=True) + if width: + raise ValueError("automatic search width must be zero") + else: + name = ( + "nprobe" + if search_width == SearchWidth.IVF_NPROBE + else "l_search" + ) + width = _size_t(self.width, name, allow_zero=False) + object.__setattr__(self, "top_k", top_k) + object.__setattr__(self, "search_width", search_width) + object.__setattr__(self, "width", width) + + @classmethod + def automatic(cls, top_k: int): + return cls(top_k=top_k) + + @classmethod + def ivf(cls, top_k: int, nprobe: int): + return cls( + top_k=top_k, + search_width=SearchWidth.IVF_NPROBE, + width=nprobe, + ) + + @classmethod + def diskann(cls, top_k: int, l_search: int): + return cls( + top_k=top_k, + search_width=SearchWidth.DISKANN_L_SEARCH, + width=l_search, + ) def to_ffi(self): return _ffi.PaimonVindexSearchParams( self.top_k, - self.nprobe, - self.ef_search, - self.query_bits, + int(self.search_width), + self.width, ) @@ -83,9 +219,11 @@ def _metadata_from_ffi(raw): metric=METRICS.get(raw.metric, f"unknown_{raw.metric}"), total_vectors=raw.total_vectors, pq_m=raw.pq_m or None, - hnsw_m=raw.hnsw_m or None, - hnsw_ef_construction=raw.hnsw_ef_construction or None, - hnsw_max_level=raw.hnsw_max_level or None, + pq_bits=raw.pq_bits or None, + rq_bits=raw.rq_bits or None, + diskann_max_degree=raw.diskann_max_degree or None, + diskann_build_search_list_size=raw.diskann_build_search_list_size or None, + diskann_alpha=raw.diskann_alpha or None, ) @@ -136,9 +274,11 @@ def _option_arrays(options: Mapping[str, str]): return option_items, key_bytes, value_bytes, keys, values -def _make_read_ranges_callback(input): +def _make_read_ranges_callback(input, native_handle_lock=None): @_ffi.READ_RANGES_FN def read_ranges_callback(ctx, requests, request_count): + if native_handle_lock is not None: + native_handle_lock._enter_callback() try: ranges = [ (requests[i].offset, requests[i].len) @@ -155,12 +295,16 @@ def read_ranges_callback(ctx, requests, request_count): return 0 except Exception: return -1 + finally: + if native_handle_lock is not None: + native_handle_lock._exit_callback() return read_ranges_callback class VectorIndexTraining: def __init__(self, handle): + self._native_handle_lock = _NativeHandleLock() self._closed = False self._handle = handle @@ -169,21 +313,24 @@ def _require_open(self): raise RuntimeError("VectorIndexTraining is closed") def _take_handle(self): - self._require_open() - handle = self._handle - self._handle = None - self._closed = True - return handle + with self._native_handle_lock: + self._require_open() + handle = self._handle + self._handle = None + self._closed = True + return handle def close(self): - if self._handle: - lib.paimon_vindex_training_free(self._handle) - self._handle = None - self._closed = True + with self._native_handle_lock: + if self._handle: + lib.paimon_vindex_training_free(self._handle) + self._handle = None + self._closed = True def __enter__(self): - self._require_open() - return self + with self._native_handle_lock: + self._require_open() + return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() @@ -198,6 +345,7 @@ def __del__(self): class VectorIndexTrainer: def __init__(self, options: Mapping[str, str]): + self._native_handle_lock = _NativeHandleLock() self._closed = False ( option_items, @@ -221,7 +369,17 @@ def create(cls, options: Mapping[str, str]): @classmethod def train(cls, options: Mapping[str, str], data): - with cls(options) as trainer: + data = _float32_matrix(data, "data") + resolved_options = dict(options) + if resolved_options.get("dimension") in (None, "auto"): + resolved_options["dimension"] = str(data.shape[1]) + if ( + resolved_options.get("index.type", "").startswith("ivf_") + and resolved_options.get("nlist") in (None, "auto") + and "expected-vector-count" not in resolved_options + ): + resolved_options["expected-vector-count"] = str(data.shape[0]) + with cls(resolved_options) as trainer: return trainer.add_training_vectors(data).finish_training() def _require_open(self): @@ -229,54 +387,63 @@ def _require_open(self): raise RuntimeError("VectorIndexTrainer is closed") def _read_dimension(self): - out = ctypes.c_size_t(0) - rc = lib.paimon_vindex_trainer_dimension(self._handle, ctypes.byref(out)) - if rc != 0: - _check_error("trainer dimension failed") - return out.value + with self._native_handle_lock: + self._require_open() + out = ctypes.c_size_t(0) + rc = lib.paimon_vindex_trainer_dimension( + self._handle, ctypes.byref(out) + ) + if rc != 0: + _check_error("trainer dimension failed") + return out.value @property def dimension(self): - self._require_open() - return self._dimension + with self._native_handle_lock: + self._require_open() + return self._dimension def add_training_vectors(self, data): - self._require_open() data = _float32_matrix(data, "data") - if data.shape[1] != self._dimension: - raise RuntimeError( - f"training data length {data.size} does not match vector count " - f"* dimension {data.shape[0] * self._dimension}" + with self._native_handle_lock: + self._require_open() + if data.shape[1] != self._dimension: + raise RuntimeError( + f"training data length {data.size} does not match vector count " + f"* dimension {data.shape[0] * self._dimension}" + ) + rc = lib.paimon_vindex_trainer_add_training_vectors( + self._handle, + data.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + data.shape[0], ) - rc = lib.paimon_vindex_trainer_add_training_vectors( - self._handle, - data.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), - data.shape[0], - ) - if rc != 0: - _check_error("add training vectors failed") - return self + if rc != 0: + _check_error("add training vectors failed") + return self def finish_training(self): - self._require_open() - handle = self._handle - training = lib.paimon_vindex_trainer_finish(handle) - lib.paimon_vindex_trainer_free(handle) - self._handle = None - self._closed = True - if not training: - _check_error("finish training failed") - return VectorIndexTraining(training) + with self._native_handle_lock: + self._require_open() + handle = self._handle + training = lib.paimon_vindex_trainer_finish(handle) + lib.paimon_vindex_trainer_free(handle) + self._handle = None + self._closed = True + if not training: + _check_error("finish training failed") + return VectorIndexTraining(training) def close(self): - if self._handle: - lib.paimon_vindex_trainer_free(self._handle) - self._handle = None - self._closed = True + with self._native_handle_lock: + if self._handle: + lib.paimon_vindex_trainer_free(self._handle) + self._handle = None + self._closed = True def __enter__(self): - self._require_open() - return self + with self._native_handle_lock: + self._require_open() + return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() @@ -293,6 +460,7 @@ class VectorIndexWriter: def __init__(self, training: VectorIndexTraining): if not isinstance(training, VectorIndexTraining): raise TypeError("training must be a VectorIndexTraining") + self._native_handle_lock = _NativeHandleLock() self._closed = False training_handle = training._take_handle() self._handle = lib.paimon_vindex_writer_open(training_handle) @@ -306,41 +474,47 @@ def _require_open(self): raise RuntimeError("VectorIndexWriter is closed") def _read_dimension(self): - out = ctypes.c_size_t(0) - rc = lib.paimon_vindex_writer_dimension(self._handle, ctypes.byref(out)) - if rc != 0: - _check_error("writer dimension failed") - return out.value + with self._native_handle_lock: + self._require_open() + out = ctypes.c_size_t(0) + rc = lib.paimon_vindex_writer_dimension( + self._handle, ctypes.byref(out) + ) + if rc != 0: + _check_error("writer dimension failed") + return out.value @property def dimension(self): - self._require_open() - return self._dimension + with self._native_handle_lock: + self._require_open() + return self._dimension def add_vectors(self, ids, data): - self._require_open() data = _float32_matrix(data, "data") ids = _int64_vector(ids, "ids") - if data.shape[1] != self._dimension: - raise RuntimeError( - f"vector data length {data.size} does not match vector count " - f"* dimension {data.shape[0] * self._dimension}" - ) - if ids.shape[0] != data.shape[0]: - raise RuntimeError( - f"ids length {ids.shape[0]} does not match vector count {data.shape[0]}" + with self._native_handle_lock: + self._require_open() + if data.shape[1] != self._dimension: + raise RuntimeError( + f"vector data length {data.size} does not match vector count " + f"* dimension {data.shape[0] * self._dimension}" + ) + if ids.shape[0] != data.shape[0]: + raise RuntimeError( + f"ids length {ids.shape[0]} does not match vector count " + f"{data.shape[0]}" + ) + rc = lib.paimon_vindex_writer_add_vectors( + self._handle, + ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)), + data.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + data.shape[0], ) - rc = lib.paimon_vindex_writer_add_vectors( - self._handle, - ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)), - data.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), - data.shape[0], - ) - if rc != 0: - _check_error("add_vectors failed") + if rc != 0: + _check_error("add_vectors failed") def write(self, file): - self._require_open() pos = 0 @_ffi.WRITE_FN @@ -376,18 +550,23 @@ def pos_callback(ctx): output.flush_fn = flush_callback output.get_pos_fn = pos_callback - rc = lib.paimon_vindex_writer_write_index(self._handle, output) - if rc != 0: - _check_error("write index failed") + with self._native_handle_lock: + self._require_open() + rc = lib.paimon_vindex_writer_write_index(self._handle, output) + if rc != 0: + _check_error("write index failed") def close(self): - if self._handle: - lib.paimon_vindex_writer_free(self._handle) - self._handle = None - self._closed = True + with self._native_handle_lock: + if self._handle: + lib.paimon_vindex_writer_free(self._handle) + self._handle = None + self._closed = True def __enter__(self): - return self + with self._native_handle_lock: + self._require_open() + return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() @@ -401,15 +580,46 @@ def __del__(self): class VectorIndexReader: - def __init__(self, input): + def __init__( + self, + input, + memory_budget_bytes: int = 4 * 1024 * 1024 * 1024, + ): + self._native_handle_lock = _NativeHandleLock() self._input = input self._closed = False - self._read_ranges_callback = _make_read_ranges_callback(self._input) + memory_budget_bytes = _size_t( + memory_budget_bytes, "memory_budget_bytes", allow_zero=True + ) + + self._read_ranges_callback = _make_read_ranges_callback( + self._input, self._native_handle_lock + ) input_file = _ffi.PaimonVindexInputFile() input_file.ctx = None input_file.read_ranges_fn = self._read_ranges_callback - self._handle = lib.paimon_vindex_reader_open(input_file) + capability_names = ( + "estimated_random_read_latency_nanos", + "preferred_window_bytes", + "max_ranges_per_read", + ) + capabilities = {} + for name in capability_names: + value = getattr(self._input, name, 0) + if name == "estimated_random_read_latency_nanos": + capabilities[name] = _uint64(value, f"input.{name}") + else: + capabilities[name] = _size_t( + value, f"input.{name}", allow_zero=True + ) + input_file.estimated_random_read_latency_nanos = capabilities[ + "estimated_random_read_latency_nanos" + ] + input_file.preferred_window_bytes = capabilities["preferred_window_bytes"] + input_file.max_ranges_per_read = capabilities["max_ranges_per_read"] + options = _ffi.PaimonVindexReaderOptions(memory_budget_bytes) + self._handle = lib.paimon_vindex_reader_open_with_options(input_file, options) if not self._handle: _check_error("failed to open reader") self._metadata = self.metadata() @@ -435,18 +645,84 @@ def total_vectors(self): return self.metadata().total_vectors def metadata(self): - self._require_open() - raw = _ffi.PaimonVindexMetadata() - rc = lib.paimon_vindex_reader_metadata(self._handle, ctypes.byref(raw)) - if rc != 0: - _check_error("metadata failed") - return _metadata_from_ffi(raw) + with self._native_handle_lock: + self._require_open() + raw = _ffi.PaimonVindexMetadata() + rc = lib.paimon_vindex_reader_metadata( + self._handle, ctypes.byref(raw) + ) + if rc != 0: + _check_error("metadata failed") + return _metadata_from_ffi(raw) def optimize_for_search(self): - self._require_open() - rc = lib.paimon_vindex_reader_optimize_for_search(self._handle) - if rc != 0: - _check_error("optimize_for_search failed") + with self._native_handle_lock: + self._require_open() + rc = lib.paimon_vindex_reader_optimize_for_search(self._handle) + if rc != 0: + _check_error("optimize_for_search failed") + + def warmup_queries(self, queries, l_search: int = 0): + queries = _float32_matrix(queries, "queries") + if queries.shape[1] != self._metadata.dimension: + raise RuntimeError( + f"queries length {queries.size} does not match nq * dimension " + f"{queries.shape[0] * self._metadata.dimension}" + ) + l_search = _size_t(l_search, "l_search", allow_zero=True) + with self._native_handle_lock: + self._require_open() + rc = lib.paimon_vindex_reader_warmup_queries( + self._handle, + queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + queries.shape[0], + l_search, + ) + if rc != 0: + _check_error("warmup_queries failed") + + def calibrate_search_width(self, queries, top_k: int = 10): + queries = _float32_matrix(queries, "queries") + if queries.shape[1] != self._metadata.dimension: + raise RuntimeError( + f"queries length {queries.size} does not match nq * dimension " + f"{queries.shape[0] * self._metadata.dimension}" + ) + top_k = _size_t(top_k, "top_k", allow_zero=False) + with self._native_handle_lock: + self._require_open() + out = ctypes.c_size_t() + rc = lib.paimon_vindex_reader_calibrate_search_width( + self._handle, + queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + queries.shape[0], + top_k, + ctypes.byref(out), + ) + if rc != 0: + _check_error("calibrate_search_width failed") + return out.value + + def read_plan(self): + with self._native_handle_lock: + self._require_open() + raw = _ffi.PaimonVindexReadPlan() + rc = lib.paimon_vindex_reader_read_plan( + self._handle, ctypes.byref(raw) + ) + if rc != 0: + _check_error("read_plan failed") + return VectorIndexReadPlan( + random_read_latency_nanos=raw.random_read_latency_nanos, + window_bytes=raw.window_bytes, + max_ranges_per_read=raw.max_ranges_per_read, + graph_beam_width=raw.graph_beam_width, + filtered_graph_beam_width=raw.filtered_graph_beam_width, + adjacency_preload_bytes=raw.adjacency_preload_bytes, + adjacency_cache_bytes=raw.adjacency_cache_bytes, + raw_vector_cache_bytes=raw.raw_vector_cache_bytes, + memory_budget_bytes=raw.memory_budget_bytes, + ) def _filter_args(self, filter_bytes): if filter_bytes is None: @@ -454,7 +730,6 @@ def _filter_args(self, filter_bytes): return _bytes_buffer(filter_bytes, "filter_bytes") def search(self, query, params: SearchParams, filter_bytes=None): - self._require_open() query = _float32_vector(query, "query") if query.shape[0] != self._metadata.dimension: raise RuntimeError( @@ -465,33 +740,34 @@ def search(self, query, params: SearchParams, filter_bytes=None): ids = np.empty(params.top_k, dtype=np.int64) distances = np.empty(params.top_k, dtype=np.float32) - if filter_bytes is None: - rc = lib.paimon_vindex_reader_search( - self._handle, - query.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), - ffi_params, - ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)), - distances.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), - params.top_k, - ) - else: - filter_buf, filter_len, _ = self._filter_args(filter_bytes) - rc = lib.paimon_vindex_reader_search_with_roaring_filter( - self._handle, - query.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), - ffi_params, - filter_buf, - filter_len, - ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)), - distances.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), - params.top_k, - ) - if rc != 0: - _check_error("search failed") + with self._native_handle_lock: + self._require_open() + if filter_bytes is None: + rc = lib.paimon_vindex_reader_search( + self._handle, + query.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + ffi_params, + ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)), + distances.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + params.top_k, + ) + else: + filter_buf, filter_len, _ = self._filter_args(filter_bytes) + rc = lib.paimon_vindex_reader_search_with_roaring_filter( + self._handle, + query.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + ffi_params, + filter_buf, + filter_len, + ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)), + distances.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + params.top_k, + ) + if rc != 0: + _check_error("search failed") return ids, distances def search_batch(self, queries, params: SearchParams, filter_bytes=None): - self._require_open() queries = _float32_matrix(queries, "queries") if queries.shape[1] != self._metadata.dimension: raise RuntimeError( @@ -503,41 +779,46 @@ def search_batch(self, queries, params: SearchParams, filter_bytes=None): ids = np.empty((queries.shape[0], params.top_k), dtype=np.int64) distances = np.empty((queries.shape[0], params.top_k), dtype=np.float32) - if filter_bytes is None: - rc = lib.paimon_vindex_reader_search_batch( - self._handle, - queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), - queries.shape[0], - ffi_params, - ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)), - distances.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), - result_len, - ) - else: - filter_buf, filter_len, _ = self._filter_args(filter_bytes) - rc = lib.paimon_vindex_reader_search_batch_with_roaring_filter( - self._handle, - queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), - queries.shape[0], - ffi_params, - filter_buf, - filter_len, - ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)), - distances.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), - result_len, - ) - if rc != 0: - _check_error("batch search failed") + with self._native_handle_lock: + self._require_open() + if filter_bytes is None: + rc = lib.paimon_vindex_reader_search_batch( + self._handle, + queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + queries.shape[0], + ffi_params, + ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)), + distances.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + result_len, + ) + else: + filter_buf, filter_len, _ = self._filter_args(filter_bytes) + rc = lib.paimon_vindex_reader_search_batch_with_roaring_filter( + self._handle, + queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + queries.shape[0], + ffi_params, + filter_buf, + filter_len, + ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)), + distances.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + result_len, + ) + if rc != 0: + _check_error("batch search failed") return ids, distances def close(self): - if self._handle: - lib.paimon_vindex_reader_free(self._handle) - self._handle = None - self._closed = True + with self._native_handle_lock: + if self._handle: + lib.paimon_vindex_reader_free(self._handle) + self._handle = None + self._closed = True def __enter__(self): - return self + with self._native_handle_lock: + self._require_open() + return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() @@ -551,7 +832,9 @@ def __del__(self): __all__ = [ + "SearchParams", "VectorIndexMetadata", + "VectorIndexReadPlan", "VectorIndexReader", "VectorIndexTrainer", "VectorIndexTraining", diff --git a/python/paimon_vindex/_ffi.py b/python/paimon_vindex/_ffi.py index fbd87fc..f86936f 100644 --- a/python/paimon_vindex/_ffi.py +++ b/python/paimon_vindex/_ffi.py @@ -116,6 +116,9 @@ class PaimonVindexInputFile(Structure): _fields_ = [ ("ctx", c_void_p), ("read_ranges_fn", READ_RANGES_FN), + ("estimated_random_read_latency_nanos", c_uint64), + ("preferred_window_bytes", c_size_t), + ("max_ranges_per_read", c_size_t), ] @@ -127,18 +130,39 @@ class PaimonVindexMetadata(Structure): ("metric", c_uint32), ("total_vectors", c_int64), ("pq_m", c_size_t), - ("hnsw_m", c_size_t), - ("hnsw_ef_construction", c_size_t), - ("hnsw_max_level", c_size_t), + ("pq_bits", c_size_t), + ("rq_bits", c_size_t), + ("diskann_max_degree", c_size_t), + ("diskann_build_search_list_size", c_size_t), + ("diskann_alpha", c_float), ] class PaimonVindexSearchParams(Structure): _fields_ = [ ("top_k", c_size_t), - ("nprobe", c_size_t), - ("ef_search", c_size_t), - ("query_bits", c_size_t), + ("search_width", c_uint32), + ("width", c_size_t), + ] + + +class PaimonVindexReaderOptions(Structure): + _fields_ = [ + ("memory_budget_bytes", c_size_t), + ] + + +class PaimonVindexReadPlan(Structure): + _fields_ = [ + ("random_read_latency_nanos", c_uint64), + ("window_bytes", c_size_t), + ("max_ranges_per_read", c_size_t), + ("graph_beam_width", c_size_t), + ("filtered_graph_beam_width", c_size_t), + ("adjacency_preload_bytes", c_size_t), + ("adjacency_cache_bytes", c_size_t), + ("raw_vector_cache_bytes", c_size_t), + ("memory_budget_bytes", c_size_t), ] @@ -197,6 +221,12 @@ class PaimonVindexSearchParams(Structure): lib.paimon_vindex_reader_open.argtypes = [PaimonVindexInputFile] lib.paimon_vindex_reader_open.restype = c_void_p +lib.paimon_vindex_reader_open_with_options.argtypes = [ + PaimonVindexInputFile, + PaimonVindexReaderOptions, +] +lib.paimon_vindex_reader_open_with_options.restype = c_void_p + lib.paimon_vindex_reader_free.argtypes = [c_void_p] lib.paimon_vindex_reader_free.restype = None @@ -208,6 +238,26 @@ class PaimonVindexSearchParams(Structure): lib.paimon_vindex_reader_optimize_for_search.argtypes = [c_void_p] lib.paimon_vindex_reader_optimize_for_search.restype = c_int +lib.paimon_vindex_reader_warmup_queries.argtypes = [ + c_void_p, + POINTER(c_float), + c_size_t, + c_size_t, +] +lib.paimon_vindex_reader_warmup_queries.restype = c_int +lib.paimon_vindex_reader_calibrate_search_width.argtypes = [ + c_void_p, + POINTER(c_float), + c_size_t, + c_size_t, + POINTER(c_size_t), +] +lib.paimon_vindex_reader_calibrate_search_width.restype = c_int +lib.paimon_vindex_reader_read_plan.argtypes = [ + c_void_p, + POINTER(PaimonVindexReadPlan), +] +lib.paimon_vindex_reader_read_plan.restype = c_int lib.paimon_vindex_reader_search.argtypes = [ c_void_p, diff --git a/python/pyproject.toml b/python/pyproject.toml index e48cd19..41a8135 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -21,12 +21,17 @@ build-backend = "setuptools.build_meta" [project] name = "paimon-vindex" -version = "0.2.0" +version = "0.3.0" description = "Python bindings for Apache Paimon Vector Index" license = {text = "Apache-2.0"} requires-python = ">=3.9" dependencies = ["numpy"] +[project.urls] +Homepage = "https://paimon.apache.org" +Repository = "https://github.com/apache/paimon-vector-index" +Documentation = "https://github.com/apache/paimon-vector-index#readme" + [project.optional-dependencies] test = ["pytest"] diff --git a/python/tests/test_vindex.py b/python/tests/test_vindex.py index c8c8ebf..8c638f4 100644 --- a/python/tests/test_vindex.py +++ b/python/tests/test_vindex.py @@ -17,11 +17,17 @@ import ctypes import io +import threading import numpy as np import pytest -from paimon_vindex import SearchParams, VectorIndexReader, VectorIndexTrainer, VectorIndexWriter +from paimon_vindex import ( + SearchParams, + VectorIndexReader, + VectorIndexTrainer, + VectorIndexWriter, +) class VectorIndexInput: @@ -56,6 +62,70 @@ def reader_from_bytes(data): return VectorIndexReader(VectorIndexInput(data)) +def test_python_search_parameters_remain_algorithm_specific(): + params = SearchParams.diskann(top_k=10, l_search=200).to_ffi() + + assert params.search_width == 2 + assert params.width == 200 + automatic = SearchParams.automatic(top_k=10).to_ffi() + assert automatic.search_width == 0 + assert automatic.width == 0 + + +@pytest.mark.parametrize( + "factory", + [ + lambda: SearchParams.automatic(top_k=0), + lambda: SearchParams.ivf(top_k=5, nprobe=-1), + lambda: SearchParams.diskann(top_k=5, l_search=-1), + lambda: SearchParams.ivf( + top_k=5, nprobe=ctypes.c_size_t(-1).value + 1 + ), + ], +) +def test_python_search_parameters_reject_values_that_ctypes_would_wrap(factory): + with pytest.raises(ValueError): + factory() + + +def test_python_high_level_training_infers_dimension_and_ivf_shape(): + data = clustered_data(512, 16, 8) + options = {"index.type": "ivf_sq", "metric": "l2"} + ids = np.arange(data.shape[0], dtype=np.int64) + output = io.BytesIO() + + training = VectorIndexTrainer.train(options, data) + with VectorIndexWriter(training) as writer: + writer.add_vectors(ids, data) + writer.write(output) + + with reader_from_bytes(output.getvalue()) as reader: + metadata = reader.metadata() + assert metadata.dimension == 16 + assert metadata.nlist == 8 + result_ids, _ = reader.search(data[0], SearchParams.automatic(top_k=5)) + assert result_ids[0] == 0 + + +def test_python_high_level_training_preserves_explicit_expected_count(): + data = clustered_data(512, 16, 8) + options = { + "index.type": "ivf_sq", + "metric": "l2", + "expected-vector-count": "1000000", + } + ids = np.arange(data.shape[0], dtype=np.int64) + output = io.BytesIO() + + training = VectorIndexTrainer.train(options, data) + with VectorIndexWriter(training) as writer: + writer.add_vectors(ids, data) + writer.write(output) + + with reader_from_bytes(output.getvalue()) as reader: + assert reader.metadata().nlist == 1024 + + def test_python_read_callback_forwards_ranges_in_one_batch(): from paimon_vindex import _make_read_ranges_callback from paimon_vindex import _ffi @@ -84,6 +154,90 @@ def pread_many(self, ranges): assert bytes(second) == bytes([11, 12, 13, 14]) +def test_python_handle_lock_rejects_worker_callback_reentry(): + from paimon_vindex import _NativeHandleLock + + native_handle_lock = _NativeHandleLock() + rejected = [] + + def callback_worker(): + native_handle_lock._enter_callback() + try: + with native_handle_lock: + pass + except RuntimeError as error: + rejected.append("reentrant native-handle operation" in str(error)) + finally: + native_handle_lock._exit_callback() + + with native_handle_lock: + worker = threading.Thread(target=callback_worker) + worker.start() + worker.join(timeout=5) + assert not worker.is_alive() + + assert rejected == [True] + + +def test_python_reader_close_waits_for_an_inflight_native_search(): + index_bytes, data = build_index( + { + "index.type": "ivf_flat", + "dimension": "2", + "nlist": "2", + "metric": "l2", + }, + 2, + n=64, + ) + + class BlockingInput(VectorIndexInput): + def __init__(self, payload): + super().__init__(payload) + self.block_reads = False + self.read_entered = threading.Event() + self.release_read = threading.Event() + + def pread_many(self, ranges): + if self.block_reads: + self.read_entered.set() + assert self.release_read.wait(timeout=5) + return super().pread_many(ranges) + + source = BlockingInput(index_bytes) + reader = VectorIndexReader(source) + source.block_reads = True + search_done = threading.Event() + close_done = threading.Event() + errors = [] + + def search(): + try: + reader.search(data[0], SearchParams.ivf(top_k=5, nprobe=2)) + except Exception as exc: + errors.append(exc) + finally: + search_done.set() + + def close(): + reader.close() + close_done.set() + + search_thread = threading.Thread(target=search) + close_thread = threading.Thread(target=close) + search_thread.start() + assert source.read_entered.wait(timeout=5) + close_thread.start() + assert not close_done.wait(timeout=0.1) + source.release_read.set() + search_thread.join(timeout=5) + close_thread.join(timeout=5) + + assert search_done.is_set() + assert close_done.is_set() + assert errors == [] + + def test_python_ffi_roundtrips_supported_indexes(): configs = [ ( @@ -100,7 +254,6 @@ def test_python_ffi_roundtrips_supported_indexes(): "index.type": "ivf_pq", "dimension": "16", "nlist": "4", - "pq.m": "4", "metric": "l2", "use-opq": "false", }, @@ -117,7 +270,7 @@ def test_python_ffi_roundtrips_supported_indexes(): ), ( { - "index.type": "ivf_hnsw_flat", + "index.type": "ivf_sq", "dimension": "16", "nlist": "4", "metric": "l2", @@ -126,11 +279,13 @@ def test_python_ffi_roundtrips_supported_indexes(): ), ( { - "index.type": "ivf_hnsw_sq", + "index.type": "diskann", "dimension": "16", - "nlist": "4", + "pq.m": "4", + "pq.bits": "4", "metric": "l2", - "hnsw.m": "12", + "diskann.max-degree": "8", + "diskann.build-search-list-size": "16", }, 16, ), @@ -144,14 +299,35 @@ def test_python_ffi_roundtrips_supported_indexes(): assert metadata.index_type == options["index.type"] assert reader.dimension == d assert metadata.total_vectors == 512 - - params = SearchParams(top_k=5, nprobe=4, ef_search=32) + if options["index.type"] == "ivf_pq": + assert metadata.pq_m == 4 + assert metadata.pq_bits == 8 + elif options["index.type"] == "ivf_sq": + assert metadata.pq_m is None + assert metadata.pq_bits == 8 + elif options["index.type"] == "diskann": + assert metadata.pq_m == 4 + assert metadata.pq_bits == 4 + assert metadata.diskann_max_degree == 8 + assert metadata.diskann_build_search_list_size == 16 + assert metadata.diskann_alpha == pytest.approx(1.2) + + params = ( + SearchParams.diskann(top_k=5, l_search=32) + if options["index.type"] == "diskann" + else SearchParams.ivf(top_k=5, nprobe=4) + ) ids, distances = reader.search(data[0], params) reader.optimize_for_search() + if options["index.type"] == "diskann": + reader.warmup_queries(np.vstack([data[0], data[1]]), l_search=32) optimized_ids, optimized_distances = reader.search(data[0], params) assert ids.shape == (5,) assert distances.shape == (5,) - assert ids[0] == 0 + if options["index.type"] == "diskann": + assert ids[0] >= 0 + else: + assert ids[0] == 0 np.testing.assert_array_equal(optimized_ids, ids) np.testing.assert_allclose(optimized_distances, distances, rtol=0, atol=1e-4) @@ -171,7 +347,7 @@ def test_python_ffi_batch_search(): with reader_from_bytes(index_bytes) as reader: ids, distances = reader.search_batch( np.vstack([data[0], data[1]]), - SearchParams(top_k=2, nprobe=2), + SearchParams.ivf(top_k=2, nprobe=2), ) assert ids.shape == (2, 2) assert distances.shape == (2, 2) @@ -179,12 +355,305 @@ def test_python_ffi_batch_search(): assert ids[1, 0] == 1 -def test_python_ffi_ivfrq_query_bits(): +def test_python_diskann_latency_hint_selects_coalesced_read_plan(): + index_bytes, data = build_index( + { + "index.type": "diskann", + "dimension": "16", + "pq.m": "4", + "metric": "l2", + "diskann.max-degree": "8", + "diskann.build-search-list-size": "16", + }, + 16, + ) + source = VectorIndexInput(index_bytes) + source.estimated_random_read_latency_nanos = 20_000_000 + + with VectorIndexReader(source) as reader: + plan = reader.read_plan() + assert plan.random_read_latency_nanos == 20_000_000 + assert plan.window_bytes == 64 * 1024 + ids, distances = reader.search(data[0], SearchParams.diskann(top_k=5, l_search=100)) + + assert ids.shape == (5,) + assert distances.shape == (5,) + assert ids[0] >= 0 + + +def test_python_diskann_automatic_cache_reuses_reads(): + class RecordingInput(VectorIndexInput): + def __init__(self, data): + super().__init__(data) + self.calls = [] + + def pread_many(self, ranges): + self.calls.append(list(ranges)) + return super().pread_many(ranges) + + index_bytes, data = build_index( + { + "index.type": "diskann", + "dimension": "16", + "pq.m": "4", + "metric": "l2", + "diskann.max-degree": "8", + "diskann.build-search-list-size": "16", + }, + 16, + ) + source = RecordingInput(index_bytes) + + with VectorIndexReader(source) as reader: + reader.optimize_for_search() + reader.search(data[0], SearchParams.diskann(top_k=5, l_search=100)) + first_query_calls = len(source.calls) + source.calls.clear() + reader.search(data[0], SearchParams.diskann(top_k=5, l_search=100)) + + assert len(source.calls) <= first_query_calls + + +def test_python_diskann_calibrates_automatic_search_width(): + index_bytes, data = build_index( + { + "index.type": "diskann", + "dimension": "16", + "pq.m": "4", + "metric": "l2", + "diskann.max-degree": "8", + "diskann.build-search-list-size": "16", + }, + 16, + ) + + with reader_from_bytes(index_bytes) as reader: + resolved = reader.calibrate_search_width(data[:4], top_k=5) + assert resolved in {100, 200, 400} + ids, distances = reader.search(data[0], SearchParams.automatic(top_k=5)) + assert ids.shape == (5,) + assert distances.shape == (5,) + + +@pytest.mark.parametrize( + ("metric", "expected_id", "expected_distance"), + [ + ("inner_product", 100, -10.0), + ("cosine", 100, 0.0), + ], +) +def test_python_diskann_supports_ip_and_cosine(metric, expected_id, expected_distance): + data = np.asarray( + [ + [10.0, 0.0], + [1.0, 1.0], + [0.0, 8.0], + [0.0, 1.0], + [-1.0, 0.0], + [0.0, -1.0], + [-2.0, 1.0], + [1.0, -2.0], + [-3.0, -1.0], + [-1.0, -3.0], + [-4.0, 0.5], + [0.5, -4.0], + [-5.0, -2.0], + [-2.0, -5.0], + [-6.0, -1.0], + [-2.0, 0.0], + ], + dtype=np.float32, + ) + options = { + "index.type": "diskann", + "dimension": "2", + "metric": metric, + "pq.m": "1", + "pq.bits": "4", + "diskann.max-degree": "8", + "diskann.build-search-list-size": "16", + "diskann.raw-vector-encoding": "f32", + } + output = io.BytesIO() + training = VectorIndexTrainer.train(options, data) + with VectorIndexWriter(training) as writer: + writer.add_vectors(np.arange(100, 116, dtype=np.int64), data) + writer.write(output) + + with reader_from_bytes(output.getvalue()) as reader: + assert reader.metadata().metric == metric + ids, distances = reader.search( + np.asarray([1.0, 0.0], dtype=np.float32), + SearchParams.diskann(top_k=1, l_search=16), + ) + assert ids[0] == expected_id + assert distances[0] == pytest.approx(expected_distance) + assert ids[0] >= 0 + + +def test_python_diskann_read_plan_resolves_during_open(): + index_bytes, data = build_index( + { + "index.type": "diskann", + "dimension": "16", + "pq.m": "4", + "metric": "l2", + "diskann.max-degree": "8", + "diskann.build-search-list-size": "16", + }, + 16, + ) + + with VectorIndexReader(VectorIndexInput(index_bytes)) as reader: + plan = reader.read_plan() + assert plan.random_read_latency_nanos > 0 + assert plan.window_bytes > 0 + reader.search(data[0], SearchParams.diskann(top_k=5, l_search=100)) + + +def test_python_reader_rejects_negative_memory_budget(): + with pytest.raises(ValueError, match="memory_budget_bytes"): + VectorIndexReader(VectorIndexInput(b""), memory_budget_bytes=-1) + + +def test_python_reader_rejects_negative_latency_hint(): + source = VectorIndexInput(b"") + source.estimated_random_read_latency_nanos = -1 + with pytest.raises(ValueError, match="estimated_random_read_latency_nanos"): + VectorIndexReader(source) + + +def test_python_size_t_arguments_reject_platform_overflow(): + oversized = ctypes.c_size_t(-1).value + 1 + with pytest.raises(ValueError, match="top_k"): + SearchParams.automatic(oversized) + with pytest.raises(ValueError, match="nprobe"): + SearchParams.ivf(5, oversized) + with pytest.raises(ValueError, match="memory_budget_bytes"): + VectorIndexReader(VectorIndexInput(b""), memory_budget_bytes=oversized) + + source = VectorIndexInput(b"") + source.max_ranges_per_read = oversized + with pytest.raises(ValueError, match="max_ranges_per_read"): + VectorIndexReader(source) + + source = VectorIndexInput(b"") + source.estimated_random_read_latency_nanos = ctypes.c_uint64(-1).value + 1 + with pytest.raises(ValueError, match="estimated_random_read_latency_nanos"): + VectorIndexReader(source) + + index_bytes, data = build_index( + { + "index.type": "diskann", + "dimension": "16", + "pq.m": "4", + "metric": "l2", + "diskann.max-degree": "8", + "diskann.build-search-list-size": "16", + }, + 16, + ) + with reader_from_bytes(index_bytes) as reader: + with pytest.raises(ValueError, match="l_search"): + reader.warmup_queries(data[:1], l_search=oversized) + with pytest.raises(ValueError, match="top_k"): + reader.calibrate_search_width(data[:1], top_k=oversized) + + +def test_python_reader_rejects_reentrant_callback_operations(): + index_bytes, data = build_index( + { + "index.type": "ivf_flat", + "dimension": "16", + "nlist": "4", + "metric": "l2", + }, + 16, + ) + + class ReentrantInput(VectorIndexInput): + reader = None + attempted = False + + def pread_many(self, ranges): + if self.reader is not None and not self.attempted: + self.attempted = True + self.reader.close() + return super().pread_many(ranges) + + source = ReentrantInput(index_bytes) + reader = VectorIndexReader(source) + source.reader = reader + try: + with pytest.raises(RuntimeError): + reader.search(data[0], SearchParams.ivf(top_k=5, nprobe=2)) + assert source.attempted + assert reader.metadata().index_type == "ivf_flat" + finally: + reader.close() + + +def test_python_writer_rejects_reentrant_output_callback_operations(): + data = np.arange(128 * 16, dtype=np.float32).reshape(128, 16) + training = VectorIndexTrainer.train( + { + "index.type": "ivf_flat", + "dimension": "16", + "nlist": "4", + "metric": "l2", + }, + data, + ) + + class ReentrantOutput(io.BytesIO): + writer = None + attempted = False + + def write(self, payload): + if self.writer is not None and not self.attempted: + self.attempted = True + self.writer.close() + return super().write(payload) + + output = ReentrantOutput() + writer = VectorIndexWriter(training) + output.writer = writer + try: + with pytest.raises(RuntimeError): + writer.write(output) + assert output.attempted + assert writer.dimension == 16 + finally: + writer.close() + + +def test_python_reader_enforces_configured_resident_memory_budget(): + index_bytes, _ = build_index( + { + "index.type": "diskann", + "dimension": "16", + "pq.m": "4", + "metric": "l2", + "diskann.max-degree": "8", + "diskann.build-search-list-size": "16", + }, + 16, + ) + + with VectorIndexReader( + VectorIndexInput(index_bytes), memory_budget_bytes=1 + ) as reader: + with pytest.raises(RuntimeError, match="reader budget"): + reader.optimize_for_search() + + +def test_python_ffi_ivfrq_build_bits(): index_bytes, data = build_index( { "index.type": "ivf_rq", "dimension": "16", "nlist": "4", + "rq.bits": "5", "metric": "l2", }, 16, @@ -192,23 +661,18 @@ def test_python_ffi_ivfrq_query_bits(): ) with reader_from_bytes(index_bytes) as reader: - for query_bits in (4, 8): - ids, distances = reader.search( - data[7], SearchParams(top_k=5, nprobe=4, query_bits=query_bits) - ) - assert ids.shape == (5,) - assert distances.shape == (5,) - assert ids[0] % 4 == 7 % 4 + assert reader.metadata().rq_bits == 5 + ids, distances = reader.search(data[7], SearchParams.ivf(top_k=5, nprobe=4)) + assert ids.shape == (5,) + assert distances.shape == (5,) + assert ids[0] % 4 == 7 % 4 ids, distances = reader.search_batch( - np.vstack([data[4], data[7]]), SearchParams(top_k=5, nprobe=4, query_bits=4) + np.vstack([data[4], data[7]]), SearchParams.ivf(top_k=5, nprobe=4) ) assert ids[0, 0] % 4 == 4 % 4 assert ids[1, 0] % 4 == 7 % 4 - with pytest.raises(RuntimeError, match="query_bits"): - reader.search(data[0], SearchParams(top_k=5, nprobe=4, query_bits=7)) - def test_python_ffi_delegates_validation(): options = { @@ -232,8 +696,10 @@ def test_python_ffi_delegates_validation(): index_bytes, data = build_index(options, 16) with reader_from_bytes(index_bytes) as reader: with pytest.raises(RuntimeError, match="query length 15"): - reader.search(np.zeros(15, dtype=np.float32), SearchParams(top_k=5, nprobe=2)) - with pytest.raises(RuntimeError, match="k must be greater than 0"): - reader.search(data[0], SearchParams(top_k=0, nprobe=2)) + reader.search(np.zeros(15, dtype=np.float32), SearchParams.ivf(top_k=5, nprobe=2)) + with pytest.raises(ValueError, match="top_k must be"): + reader.search(data[0], SearchParams.ivf(top_k=0, nprobe=2)) with pytest.raises(RuntimeError, match="queries length 15"): - reader.search_batch(np.zeros((1, 15), dtype=np.float32), SearchParams(top_k=5, nprobe=2)) + reader.search_batch( + np.zeros((1, 15), dtype=np.float32), SearchParams.ivf(top_k=5, nprobe=2) + ) diff --git a/tools/README.md b/tools/README.md index ff94ed5..95154e2 100644 --- a/tools/README.md +++ b/tools/README.md @@ -21,6 +21,38 @@ This directory contains helper scripts used by release managers and committers. +## ANN-Benchmarks dataset conversion + +`convert_ann_benchmarks.py` converts a dense +[ANN-Benchmarks](https://github.com/erikbern/ann-benchmarks) HDF5 file into +base/query `fvecs` files and the published-neighbor `ivecs` file accepted by +`core/benches/ann_bench.rs`. + +The script requires `h5py` and `numpy`. For example: + +```bash +python3 tools/convert_ann_benchmarks.py \ + gist-960-euclidean.hdf5 /data/gist1m \ + --prefix gist1m --query-limit 1000 +``` + +The optional query limit applies to both queries and ground truth. It does not +truncate the indexed base vectors. + +Angular/cosine datasets can be converted for the benchmark's common L2 path by +normalizing base and query vectors: + +```bash +python3 tools/convert_ann_benchmarks.py \ + glove-100-angular.hdf5 /data/glove100 \ + --prefix glove100 --query-limit 1000 --normalize-l2 +``` + +For non-zero vectors, squared L2 distance after unit normalization has the same +neighbor ordering as cosine distance. Published neighbor IDs are copied +unchanged. Conversion fails if a vector is zero-length or has a non-finite +norm. + ## Java staging deploy `deploy_java_staging.sh` deploys the Java release candidate artifacts to Apache @@ -92,7 +124,7 @@ https://repository.apache.org/ -> Profile -> User Token ### Find the run id After pushing the RC tag, open the GitHub Actions run for that RC tag. Use the -`Release` workflow run triggered by the tag, for example `v0.2.0-rc1`. +`Release` workflow run triggered by the tag, for example `v0.3.0-rc1`. The run id is the number in the workflow run URL: @@ -114,10 +146,10 @@ checked out locally. Required for the normal release flow: -- `--release-version 0.2.0`: Java artifact version in `java/pom.xml`. This does +- `--release-version 0.3.0`: Java artifact version in `java/pom.xml`. This does not include the RC suffix. - `--rc 1`: RC number. Together with `--release-version`, this derives the tag - `v0.2.0-rc1`. + `v0.3.0-rc1`. - `--run-id 12345678901`: GitHub Actions run id from the RC tag's `Release` workflow URL. The script uses it to download the four `native-*` artifacts. @@ -145,7 +177,7 @@ Always run a dry-run first with the real RC workflow artifacts: ```bash ./tools/deploy_java_staging.sh \ - --release-version 0.2.0 \ + --release-version 0.3.0 \ --rc 1 \ --run-id 12345678901 \ --dry-run @@ -161,7 +193,7 @@ mvn clean verify -Prelease -Dgpg.skip=true -DskipTests It does not sign and does not deploy to Nexus. It verifies: - `java/pom.xml` version matches `--release-version`; -- current checkout matches the RC tag, such as `v0.2.0-rc1`; +- current checkout matches the RC tag, such as `v0.3.0-rc1`; - Java package inputs have no local changes; - the GitHub Actions run is a successful tag-push `Release` workflow run and its commit matches the RC tag; @@ -176,7 +208,7 @@ After the dry-run succeeds, run the same command without `--dry-run`: ```bash ./tools/deploy_java_staging.sh \ - --release-version 0.2.0 \ + --release-version 0.3.0 \ --rc 1 \ --run-id 12345678901 ``` @@ -192,7 +224,7 @@ After that passes, it runs the local Nexus staging deploy: ```bash mvn deploy -Prelease -DskipTests \ - -DstagingDescription="Apache Paimon Vector Index, version 0.2.0, release candidate 1" + -DstagingDescription="Apache Paimon Vector Index, version 0.3.0, release candidate 1" ``` The Maven output contains the Nexus staging repository id, for example: diff --git a/tools/check_license_headers.py b/tools/check_license_headers.py index 7add236..4e5fc22 100755 --- a/tools/check_license_headers.py +++ b/tools/check_license_headers.py @@ -34,11 +34,14 @@ "NOTICE", # Golden test data; adding comments changes the fixture format. "core/tests/fixtures/ivf_flat_v1.hex", - "core/tests/fixtures/ivf_hnsw_flat_v1.hex", - "core/tests/fixtures/ivf_hnsw_sq_v1.hex", + "core/tests/fixtures/ivf_sq_v1.hex", "core/tests/fixtures/ivf_pq_4bit_v1.hex", "core/tests/fixtures/ivf_pq_v1.hex", "core/tests/fixtures/ivf_rq_v1.hex", + "core/tests/fixtures/diskann_v1.hex", + "core/tests/fixtures/diskann_compact_multipage_v1.hex", + "core/tests/fixtures/diskann_interleaved_4bit_v1.hex", + "core/tests/fixtures/diskann_raw_row_ids_v1.hex", } diff --git a/tools/convert_ann_benchmarks.py b/tools/convert_ann_benchmarks.py new file mode 100644 index 0000000..5449380 --- /dev/null +++ b/tools/convert_ann_benchmarks.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert an ANN-Benchmarks dense HDF5 dataset to fvecs/ivecs files.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import h5py +import numpy as np + + +def write_fvecs( + dataset: h5py.Dataset, + path: Path, + batch_rows: int, + row_limit: int | None = None, + normalize_l2: bool = False, +) -> None: + if dataset.ndim != 2: + raise ValueError(f"{dataset.name} must be a two-dimensional dense dataset") + rows, dimension = dataset.shape + if row_limit is not None: + rows = min(rows, row_limit) + with path.open("wb") as output: + for start in range(0, rows, batch_rows): + end = min(start + batch_rows, rows) + values = np.asarray( + dataset[start:end], dtype=" None: + if dataset.ndim != 2: + raise ValueError(f"{dataset.name} must be a two-dimensional dense dataset") + rows, width = dataset.shape + if row_limit is not None: + rows = min(rows, row_limit) + with path.open("wb") as output: + for start in range(0, rows, batch_rows): + end = min(start + batch_rows, rows) + values = np.asarray( + dataset[start:end], dtype=" None: + parser = argparse.ArgumentParser( + description="Convert ANN-Benchmarks train/test/neighbors to fvecs/ivecs" + ) + parser.add_argument("input", type=Path, help="ANN-Benchmarks HDF5 file") + parser.add_argument("output_dir", type=Path, help="destination directory") + parser.add_argument( + "--prefix", + help="output filename prefix; defaults to the HDF5 filename before the first dot", + ) + parser.add_argument("--batch-rows", type=int, default=4096) + parser.add_argument( + "--query-limit", + type=int, + help="convert only the first N test queries and ground-truth rows", + ) + parser.add_argument( + "--normalize-l2", + action="store_true", + help=( + "L2-normalize train and test vectors while preserving published " + "neighbor IDs; use for angular/cosine datasets" + ), + ) + args = parser.parse_args() + if args.batch_rows <= 0: + parser.error("--batch-rows must be positive") + if args.query_limit is not None and args.query_limit <= 0: + parser.error("--query-limit must be positive") + + prefix = args.prefix or args.input.name.split(".", 1)[0] + args.output_dir.mkdir(parents=True, exist_ok=True) + base_path = args.output_dir / f"{prefix}_base.fvecs" + query_path = args.output_dir / f"{prefix}_query.fvecs" + truth_path = args.output_dir / f"{prefix}_ground_truth.ivecs" + + with h5py.File(args.input, "r") as source: + required = {"train", "test", "neighbors"} + missing = required.difference(source.keys()) + if missing: + raise ValueError(f"missing HDF5 datasets: {', '.join(sorted(missing))}") + write_fvecs( + source["train"], + base_path, + args.batch_rows, + normalize_l2=args.normalize_l2, + ) + write_fvecs( + source["test"], + query_path, + args.batch_rows, + row_limit=args.query_limit, + normalize_l2=args.normalize_l2, + ) + write_ivecs( + source["neighbors"], + truth_path, + args.batch_rows, + row_limit=args.query_limit, + ) + + for path in (base_path, query_path, truth_path): + print(f"{path}\t{path.stat().st_size}") + + +if __name__ == "__main__": + main()