feat: DiskSINDI sparse index with batched and merged rerank IO

docs/docs/en/src/advanced/introspection.md

@@ -45,7 +45,7 @@ Two overloads are provided:
 // Dense vector indexes (HGraph, BruteForce, IVF, DiskANN, HNSW)
 auto r = index->CalDistanceById(query_ptr, ids, count, /*calculate_precise_distance=*/true);
 
-// Sparse vector indexes (SINDI, SparseIndex) — wrap the query in a Dataset
+// Sparse vector indexes (SINDI) — wrap the query in a Dataset
 auto query_ds = vsag::Dataset::Make();
 query_ds->NumElements(1)->SparseVectors(/* ... */);
 auto r = index->CalDistanceById(query_ds, ids, count, /*calculate_precise_distance=*/true);

docs/docs/en/src/advanced/search_allocator.md

@@ -43,7 +43,7 @@ index falls back to the allocator that was attached to its owning `Resource`.
 > **Availability.** `Index::SearchWithRequest` has a default implementation that returns an
 > *unsupported* error. Only HGraph, IVF, BruteForce and WARP implement it today
 > (`src/algorithm/{hgraph,ivf,brute_force,warp}.cpp`). For indexes that do not yet override
-> `SearchWithRequest` (HNSW, DiskANN, SINDI, Pyramid, SparseIndex), use the legacy `SearchParam`
+> `SearchWithRequest` (HNSW, DiskANN, SINDI, Pyramid), use the legacy `SearchParam`
 > path described below.
 
 ## Legacy API — `SearchParam::allocator` *(deprecated)*

docs/docs/zh/src/advanced/introspection.md

@@ -41,7 +41,7 @@ if (not index->CheckFeature(vsag::SUPPORT_DELETE_BY_ID)) {
 // 稠密向量索引（HGraph、BruteForce、IVF、DiskANN、HNSW）
 auto r = index->CalDistanceById(query_ptr, ids, count, /*calculate_precise_distance=*/true);
 
-// 稀疏向量索引（SINDI、SparseIndex）—— 用 Dataset 封装查询
+// 稀疏向量索引（SINDI）—— 用 Dataset 封装查询
 auto query_ds = vsag::Dataset::Make();
 query_ds->NumElements(1)->SparseVectors(/* ... */);
 auto r = index->CalDistanceById(query_ds, ids, count, /*calculate_precise_distance=*/true);

docs/docs/zh/src/advanced/search_allocator.md

@@ -37,7 +37,7 @@ auto result = index->SearchWithRequest(req).value();
 
 > **可用性。** `Index::SearchWithRequest` 默认实现会返回 *不支持* 错误。目前只有 HGraph、
 > IVF、BruteForce、WARP 实现了它（`src/algorithm/{hgraph,ivf,brute_force,warp}.cpp`）。对于
-> 尚未 override 的索引（HNSW、DiskANN、SINDI、Pyramid、SparseIndex），请使用下文的旧版
+> 尚未 override 的索引（HNSW、DiskANN、SINDI、Pyramid），请使用下文的旧版
 > `SearchParam` 路径。
 
 ## 旧版 API —— `SearchParam::allocator`（已弃用）

include/vsag/constants.h

@@ -23,6 +23,7 @@ extern const char* const INDEX_FRESH_HNSW;
 extern const char* const INDEX_PYRAMID;
 extern const char* const INDEX_SPARSE;
 extern const char* const INDEX_SINDI;
+extern const char* const INDEX_DISKSINDI;
 extern const char* const INDEX_BRUTE_FORCE;
 extern const char* const INDEX_IVF;
 extern const char* const INDEX_WARP;

include/vsag/index.h

@@ -51,7 +51,20 @@ struct MergeUnit {
     IdMapFunction id_map_func = nullptr;
 };
 
-enum class IndexType { HNSW, DISKANN, HGRAPH, IVF, PYRAMID, BRUTEFORCE, SPARSE, SINDI, WARP };
+enum class IndexType {
+    HNSW = 0,
+    DISKANN = 1,
+    HGRAPH = 2,
+    IVF = 3,
+    PYRAMID = 4,
+    BRUTEFORCE = 5,
+    // Kept for source compatibility with SparseIndex callers; new sparse workloads should
+    // prefer SINDI or DISKSINDI.
+    SPARSE = 6,
+    SINDI = 7,
+    WARP = 8,
+    DISKSINDI = 9,
+};
 
 #define DATA_FLAG_FLOAT32_VECTOR 0x01
 #define DATA_FLAG_INT8_VECTOR 0x02
@@ -462,7 +475,7 @@ class Index {
      *
      * Suitable for dense vector indexes (HGraph, BruteForce, IVF, DiskANN, HNSW).
      * The query must be a contiguous float32 array with dimension matching the index.
-     * For sparse vector indexes (SINDI, SparseIndex), this overload is not applicable;
+     * For sparse vector indexes (SINDI, DiskSINDI), this overload is not applicable;
      * use CalcDistanceById(DatasetPtr, int64_t, bool) instead.
      *
      * @param vector The embedding of the query (float32 array for dense vectors).
@@ -483,7 +496,7 @@ class Index {
     /**
      * @brief Calculate the distance between the query and the vector of the given ID.
      *
-     * Suitable for sparse vector indexes (SINDI, SparseIndex) where vectors
+     * Suitable for sparse vector indexes (SINDI, DiskSINDI) where vectors
      * cannot be represented as a simple float pointer. The Dataset should
      * contain sparse vectors via GetSparseVectors().
      * For dense vector indexes (HGraph, BruteForce, IVF, DiskANN, HNSW),
@@ -509,7 +522,7 @@ class Index {
      *
      * Suitable for dense vector indexes (HGraph, BruteForce, IVF, DiskANN, HNSW).
      * The query must be a contiguous float32 array. For sparse vector indexes
-     * (SINDI, SparseIndex), this overload is not applicable; use
+     * (SINDI, DiskSINDI), this overload is not applicable; use
      * CalDistanceById(DatasetPtr, const int64_t*, int64_t, bool) instead.
      *
      * @param query is the embedding of query (float32 array for dense vectors).
@@ -532,7 +545,7 @@ class Index {
     /**
      * @brief Calculate the distance between the query and the vector of the given ID for batch.
      *
-     * Suitable for sparse vector indexes (SINDI, SparseIndex) where vectors
+     * Suitable for sparse vector indexes (SINDI, DiskSINDI) where vectors
      * cannot be represented as a simple float pointer. The Dataset should
      * contain sparse vectors via GetSparseVectors().
      * For dense vector indexes (HGraph, BruteForce, IVF, DiskANN, HNSW),

src/algorithm/CMakeLists.txt

@@ -16,6 +16,7 @@
 
 add_subdirectory (hnswlib)
 add_subdirectory (sindi)
+add_subdirectory (disksindi)
 add_subdirectory (hgraph)
 add_subdirectory (bruteforce)
 add_subdirectory (ivf)
@@ -35,6 +36,7 @@ set (ALGORITHM_LIBS
         algorithm
         hnswlib
         sindi
+        disksindi
         hgraph
         bruteforce
         ivf

src/algorithm/disksindi/CMakeLists.txt

@@ -0,0 +1,23 @@
+
+# Copyright 2024-present the vsag project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+set(DISKSINDI_SRCS
+        disksindi.cpp
+        disksindi_parameter.cpp
+)
+
+add_library(disksindi OBJECT ${DISKSINDI_SRCS})
+target_link_libraries(disksindi PRIVATE coverage_config vsag_src_common)