disk poc delete vector (#843)

* First step

* comment

* Restore vectors

* Working serializtion

* Load vectors from disk

* load from vector

* Add serializer script

* Change the gitginore

* mark delete

* Remove old tests

* add mark as deleted

* pr changes

* fix the bm

* Fixed the replace entry point

* change order

* Remove prints

* added delete vectors and unit tests

* Added bm and tests

* Removed the freeidvector

* Add repair updates

* remove comments

* change to if continue

* getchached vector data

* remove cache for now

* back to origin

* Fix the stale

* choose candidate by distance

* Add the tests

* reduce call for staged nodes

* Change to map for better performance

* PR suggestopns

* improvments

* PR changes

* Added to ci

* Added download to ci

* Added track delete time to bm

* Added the bm filename

* Change the name of file

* added no gt bm

* PR fix

* Add more validations

* More pr changes

* Added more benchmarks

* Change to l2

* PR

Author: dor-forer

.github/workflows/benchmark.yml

@@ -41,6 +41,7 @@ on:
           - bm-batch-iter-uint8-single
           - bm-batch-iter-uint8-multi
           - bm-updated-fp32-single
+          - bm-hnsw-disk-fp32-single
           - bm-spaces
         description: 'Benchmarks set to run'
         default: benchmarks-all

src/VecSim/algorithms/hnsw/hnsw_disk.h

@@ -58,8 +58,10 @@ HNSWDiskIndex<DataType, DistType>::HNSWDiskIndex(
       indexDataGuard(), visitedNodesHandlerPool(INITIAL_CAPACITY, this->allocator),
       delta_list(), new_elements_meta_data(this->allocator), batchThreshold(0), // Will be restored from file
       pendingVectorIds(this->allocator), pendingMetadata(this->allocator),
-      pendingVectorCount(0), stagedGraphUpdates(this->allocator),
-      stagedNeighborUpdates(this->allocator) {
+      pendingVectorCount(0), pendingDeleteIds(this->allocator),
+      stagedInsertUpdates(this->allocator),
+      stagedDeleteUpdates(this->allocator), stagedRepairUpdates(this->allocator),
+      stagedInsertNeighborUpdates(this->allocator) {
 
     // Restore index fields from file (including batchThreshold)
     this->restoreIndexFields(input);
@@ -264,18 +266,27 @@ void HNSWDiskIndex<DataType, DistType>::saveIndexIMP(std::ofstream &output) {
     if (!pendingVectorIds.empty()) {
         throw std::runtime_error("Serialization error: pendingVectorIds not empty after flush");
     }
-    if (!stagedGraphUpdates.empty()) {
-        throw std::runtime_error("Serialization error: stagedGraphUpdates not empty after flush");
+    if (!stagedInsertUpdates.empty()) {
+        throw std::runtime_error("Serialization error: stagedInsertUpdates not empty after flush");
     }
-    if (!stagedNeighborUpdates.empty()) {
-        throw std::runtime_error("Serialization error: stagedNeighborUpdates not empty after flush");
+    if (!stagedDeleteUpdates.empty()) {
+        throw std::runtime_error("Serialization error: stagedDeleteUpdates not empty after flush");
+    }
+    if (!stagedInsertNeighborUpdates.empty()) {
+        throw std::runtime_error("Serialization error: stagedInsertNeighborUpdates not empty after flush");
     }
     if (!rawVectorsInRAM.empty()) {
         throw std::runtime_error("Serialization error: rawVectorsInRAM not empty after flush");
     }
     if (pendingVectorCount != 0) {
         throw std::runtime_error("Serialization error: pendingVectorCount not zero after flush");
     }
+    if (!stagedRepairUpdates.empty()) {
+        throw std::runtime_error("Serialization error: stagedRepairUpdates not empty after flush");
+    }
+    if (pendingDeleteIds.size() != 0) {
+        throw std::runtime_error("Serialization error: pendingDeleteIds not empty after flush");
+    }
     // Note: delta_list and new_elements_meta_data are currently unused legacy variables
     // but we verify them for future-proofing
     if (!delta_list.empty()) {
@@ -692,8 +703,9 @@ void HNSWDiskIndex<DataType, DistType>::restoreGraph(std::ifstream &input,
     this->pendingVectorIds.clear();
     this->pendingMetadata.clear();
     this->pendingVectorCount = 0;
-    this->stagedGraphUpdates.clear();
-    this->stagedNeighborUpdates.clear();
+    this->stagedInsertUpdates.clear();
+    this->stagedDeleteUpdates.clear();
+    this->stagedInsertNeighborUpdates.clear();
 
     // Resize visited nodes handler pool
     this->visitedNodesHandlerPool.resize(this->curElementCount);