From 345b9d18f0e36f0ceec4f0b2b6c93658657ba71d Mon Sep 17 00:00:00 2001 From: ofiryanai Date: Sun, 19 Apr 2026 18:48:23 +0300 Subject: [PATCH 01/99] MOD-14916 Devirtualize HNSW / brute-force search hot path (#937) * MOD-14916 Devirtualize distance + getElement on HNSW search hot path MOD-14916 / LTK perf investigation. Two virtual dispatches per HNSW candidate added between v2.10.21 and v8.2.6 account for a measurable share of the KNN regression observed in the LTK benchmarks (-38% throughput on Intel). Both are removed here with the minimum possible change. V1 - distance computation: Every calcDistance() call goes through IndexCalculatorInterface's vtable to reach DistanceCalculatorCommon, which then calls the underlying SIMD function pointer. The intermediate vtable hop is pure indirection; the concrete calculator class is fixed for the life of an index. Expose the underlying dist_func via a new pure-virtual getDistFunc() on IndexCalculatorInterface, implemented by DistanceCalculatorCommon. Cache the returned function pointer in VecSimIndexAbstract at construction time and call it directly in calcDistance(), bypassing the virtual dispatch. V2 - vector fetch: HNSWIndex::getDataByInternalId and BruteForceIndex::getDataByInternalId call this->vectors->getElement(id), which is virtual through the RawDataContainer base. DataBlocksContainer is the only concrete implementation, and this->vectors is always a DataBlocksContainer (created and owned by VecSimIndexAbstract's constructor). Use a static_cast to DataBlocksContainer* plus a qualified call to DataBlocksContainer::getElement to skip the vtable lookup. No behavior change; per-candidate distance and neighbor-fetch calls on HNSW / brute force search paths become direct function-pointer / direct-member calls. Headers in index_factories, hnsw_serializer, and brute_force_factory compile cleanly. * MOD-14916 Inline DataBlocksContainer::getElement on HNSW search hot path Follow-up to the previous V1/V2 devirt commit. The static_cast+qualified call in getDataByInternalId removed the vtable lookup but left the larger cost on the table: DataBlocksContainer::getElement was still defined in data_blocks_container.cpp, so every per-candidate neighbor fetch still paid a real out-of-line function call and a bounds-checked blocks.at() lookup. Without LTO the compiler could neither inline the body nor hoist the div/mod in the HNSW hot loop. Move the definition into the header as inline and drop the .at() bounds check to match the v2.10.21 baseline, which used unchecked operator[] and was fully inlined into processCandidate. Also add a getDistFunc() override to DistanceCalculatorDummy in test_components.cpp so BUILD_TESTS still compiles after the pure virtual added in the previous commit. (cherry picked from commit 4ca500a6fbac0fde659717f726e8c23442c37e7c) --- src/VecSim/algorithms/brute_force/brute_force.h | 5 ++++- src/VecSim/algorithms/hnsw/hnsw.h | 4 +++- src/VecSim/containers/data_blocks_container.cpp | 5 ----- src/VecSim/containers/data_blocks_container.h | 6 +++++- src/VecSim/spaces/computer/calculator.h | 5 +++++ src/VecSim/vec_sim_index.h | 16 +++++++++++++--- tests/unit/test_components.cpp | 3 +++ 7 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/VecSim/algorithms/brute_force/brute_force.h b/src/VecSim/algorithms/brute_force/brute_force.h index 3be453024..be19f2f4c 100644 --- a/src/VecSim/algorithms/brute_force/brute_force.h +++ b/src/VecSim/algorithms/brute_force/brute_force.h @@ -43,7 +43,10 @@ class BruteForceIndex : public VecSimIndexAbstract { size_t indexCapacity() const override; std::unique_ptr getVectorsIterator() const; const DataType *getDataByInternalId(idType id) const { - return reinterpret_cast(this->vectors->getElement(id)); + // `vectors` is always a DataBlocksContainer; skip the RawDataContainer vtable. + return reinterpret_cast( + static_cast(this->vectors) + ->DataBlocksContainer::getElement(id)); } VecSimQueryReply *topKQuery(const void *queryBlob, size_t k, VecSimQueryParams *queryParams) const override; diff --git a/src/VecSim/algorithms/hnsw/hnsw.h b/src/VecSim/algorithms/hnsw/hnsw.h index e5994d314..1a3776fa2 100644 --- a/src/VecSim/algorithms/hnsw/hnsw.h +++ b/src/VecSim/algorithms/hnsw/hnsw.h @@ -385,7 +385,9 @@ labelType HNSWIndex::getEntryPointLabel() const { template const char *HNSWIndex::getDataByInternalId(idType internal_id) const { - return this->vectors->getElement(internal_id); + // `vectors` is always a DataBlocksContainer; skip the RawDataContainer vtable on the hot path. + return static_cast(this->vectors) + ->DataBlocksContainer::getElement(internal_id); } template diff --git a/src/VecSim/containers/data_blocks_container.cpp b/src/VecSim/containers/data_blocks_container.cpp index bf63b683a..f6010b7db 100644 --- a/src/VecSim/containers/data_blocks_container.cpp +++ b/src/VecSim/containers/data_blocks_container.cpp @@ -38,11 +38,6 @@ RawDataContainer::Status DataBlocksContainer::addElement(const void *element, si return Status::OK; } -const char *DataBlocksContainer::getElement(size_t id) const { - assert(id < element_count); - return blocks.at(id / this->block_size).getElement(id % this->block_size); -} - RawDataContainer::Status DataBlocksContainer::removeElement(size_t id) { assert(id == element_count - 1); // only the last element can be removed blocks.back().popLastElement(); diff --git a/src/VecSim/containers/data_blocks_container.h b/src/VecSim/containers/data_blocks_container.h index c375590f2..74f66de11 100644 --- a/src/VecSim/containers/data_blocks_container.h +++ b/src/VecSim/containers/data_blocks_container.h @@ -40,7 +40,11 @@ class DataBlocksContainer : public VecsimBaseObject, public RawDataContainer { Status addElement(const void *element, size_t id) override; - const char *getElement(size_t id) const override; + // Inlined so the hot search path (via getDataByInternalId) can fold the indexing arithmetic. + const char *getElement(size_t id) const override { + assert(id < element_count); + return blocks[id / block_size].getElement(id % block_size); + } Status removeElement(size_t id) override; diff --git a/src/VecSim/spaces/computer/calculator.h b/src/VecSim/spaces/computer/calculator.h index a82293700..539325d8e 100644 --- a/src/VecSim/spaces/computer/calculator.h +++ b/src/VecSim/spaces/computer/calculator.h @@ -23,6 +23,9 @@ class IndexCalculatorInterface : public VecsimBaseObject { virtual ~IndexCalculatorInterface() = default; virtual DistType calcDistance(const void *v1, const void *v2, size_t dim) const = 0; + + // Raw distance function; cached by the index to skip the vtable on the hot path. + virtual spaces::dist_func_t getDistFunc() const = 0; }; /** @@ -56,4 +59,6 @@ class DistanceCalculatorCommon DistType calcDistance(const void *v1, const void *v2, size_t dim) const override { return this->dist_func(v1, v2, dim); } + + spaces::dist_func_t getDistFunc() const override { return this->dist_func; } }; diff --git a/src/VecSim/vec_sim_index.h b/src/VecSim/vec_sim_index.h index 88eabea69..e5a5183b7 100644 --- a/src/VecSim/vec_sim_index.h +++ b/src/VecSim/vec_sim_index.h @@ -86,6 +86,7 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { RawDataContainer *vectors; // The raw vectors data container. private: IndexCalculatorInterface *indexCalculator; // Distance calculator. + spaces::dist_func_t cachedDistFunc; // Cached dist func, used on the hot path. PreprocessorsContainerAbstract *preprocessors; // Storage and query preprocessors. size_t inputBlobSize; // The size of input vectors/queries blob in bytes. May differ from dim * @@ -124,8 +125,11 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { metric(params.metric), blockSize(params.blockSize ? params.blockSize : DEFAULT_BLOCK_SIZE), lastMode(EMPTY_MODE), isMulti(params.multi), isDisk(params.isDisk), logCallbackCtx(params.logCtx), - indexCalculator(components.indexCalculator), preprocessors(components.preprocessors), - inputBlobSize(params.inputBlobSize), storedDataSize(params.storedDataSize) { + indexCalculator(components.indexCalculator), + cachedDistFunc(components.indexCalculator ? components.indexCalculator->getDistFunc() + : nullptr), + preprocessors(components.preprocessors), inputBlobSize(params.inputBlobSize), + storedDataSize(params.storedDataSize) { assert(VecSimType_sizeof(vecType)); assert(storedDataSize); assert(inputBlobSize); @@ -146,10 +150,16 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { /** * @brief Calculate the distance between two vectors based on index parameters. * + * Uses the cached dist func to avoid the indexCalculator vtable on the hot path. + * + * @note Precondition: @c cachedDistFunc must be non-null. Subclasses that construct + * this index with a null @c indexCalculator (e.g. SVS, which uses its own + * internal distance kernels) must not call this method. + * * @return the distance between the vectors. */ DistType calcDistance(const void *vector_data1, const void *vector_data2) const { - return indexCalculator->calcDistance(vector_data1, vector_data2, this->dim); + return cachedDistFunc(vector_data1, vector_data2, this->dim); } /** diff --git a/tests/unit/test_components.cpp b/tests/unit/test_components.cpp index bd3c8d642..2f91d2411 100644 --- a/tests/unit/test_components.cpp +++ b/tests/unit/test_components.cpp @@ -33,6 +33,9 @@ class DistanceCalculatorDummy : public DistanceCalculatorInterfacedist_func(7); } + + // Dummy uses a non-standard dist func signature, so the standard slot is unavailable. + spaces::dist_func_t getDistFunc() const override { return nullptr; } }; } // namespace dummyCalcultor From 4e0412e611ca5f30ecdd348d50d765dd9cd43c98 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Tue, 23 Jun 2026 08:57:23 -0700 Subject: [PATCH 02/99] initial --- src/VecSim/algorithms/svs/svs.h | 40 +++---- src/VecSim/algorithms/svs/svs_tiered.h | 140 +++++++++++++++---------- src/VecSim/vec_sim_tiered_index.h | 15 +++ 3 files changed, 116 insertions(+), 79 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 8b03514f0..200fdd51a 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -263,11 +263,6 @@ class SVSIndex : public VecSimIndexAbstract, fl } int deleted_num = 0; - if constexpr (!isMulti) { - // SVS index does not support overriding vectors with the same label - // so we have to delete them first if needed - deleted_num = deleteVectorsImpl(labels, n); - } std::span ids(labels, n); auto processed_blob = this->preprocessForBatchStorage(vectors_data, n); @@ -279,6 +274,11 @@ class SVSIndex : public VecSimIndexAbstract, fl // SVS index instance cannot be empty, so we have to construct it at first rows impl_ = initImpl(points, ids); } else { + if constexpr (!isMulti) { + // SVS index does not support overriding vectors with the same label + // so we have to delete them first if needed + deleted_num = deleteVectorsImpl(labels, n); + } // Add new points to existing SVS index impl_->add_points(points, ids); } @@ -287,7 +287,7 @@ class SVSIndex : public VecSimIndexAbstract, fl } int deleteVectorImpl(const labelType label) { - if (indexLabelCount() == 0 || !impl_->has_id(label)) { + if (indexLabelCount() == 0) { return 0; } @@ -302,20 +302,7 @@ class SVSIndex : public VecSimIndexAbstract, fl return 0; } - // SVS fails if we try to delete non-existing entries - std::vector entries_to_delete; - entries_to_delete.reserve(n); - for (size_t i = 0; i < n; i++) { - if (impl_->has_id(labels[i])) { - entries_to_delete.push_back(labels[i]); - } - } - - if (entries_to_delete.size() == 0) { - return 0; - } - - const auto deleted_num = impl_->delete_entries(entries_to_delete); + const auto deleted_num = impl_->delete_entries(std::span{labels, n}); this->markIndexUpdate(deleted_num); return deleted_num; @@ -553,13 +540,16 @@ class SVSIndex : public VecSimIndexAbstract, fl } double getDistanceFrom_Unsafe(labelType label, const void *vector_data) const override { - if (!impl_ || !impl_->has_id(label)) { - return std::numeric_limits::quiet_NaN(); - }; + if (!impl_) return std::numeric_limits::quiet_NaN(); auto query_datum = std::span{static_cast(vector_data), this->dim}; - auto dist = impl_->get_distance(label, query_datum); - return toVecSimDistance(dist); + try { + auto dist = impl_->get_distance(label, query_datum); + return std::isnan(dist) ? std::numeric_limits::quiet_NaN() + : toVecSimDistance(static_cast(dist)); + } catch (const svs::lib::ANNException &) { + return std::numeric_limits::quiet_NaN(); + } } VecSimQueryReply *topKQuery(const void *queryBlob, size_t k, diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 9bce50f89..947db01a1 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -233,7 +233,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::atomic_flag indexGCScheduled = ATOMIC_FLAG_INIT; // Used to prevent running multiple index update jobs in parallel. // Even if update jobs scheduled sequentially, they can be started in parallel. - mutable std::mutex updateJobMutex; + // mutable std::shared_mutex updateJobMutex; // The reason of following container just to properly destroy jobs which not executed yet SVSMultiThreadJob::JobsRegistry uncompletedJobs; @@ -261,7 +261,6 @@ class TieredSVSIndex : public VecSimTieredIndex { VecSimBatchIterator *flat_iterator; VecSimBatchIterator *svs_iterator; - std::shared_lock svs_lock; // On single value indices, this set holds the IDs of the results that were returned from // the flat buffer. @@ -327,7 +326,6 @@ class TieredSVSIndex : public VecSimTieredIndex { void acquire_svs_iterator() { assert(svs_iterator == nullptr); - this->index->mainIndexGuard.lock_shared(); svs_iterator = index->backendIndex->newBatchIterator( this->flat_iterator->getQueryBlob(), queryParams); } @@ -336,7 +334,6 @@ class TieredSVSIndex : public VecSimTieredIndex { if (svs_iterator != nullptr && svs_iterator != depleted()) { delete svs_iterator; svs_iterator = nullptr; - this->index->mainIndexGuard.unlock_shared(); } } @@ -361,7 +358,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::move(allocator)), index(index), flat_results(this->allocator), svs_results(this->allocator), flat_iterator(index->frontendIndex->newBatchIterator(query_vector, queryParams)), - svs_iterator(nullptr), svs_lock(index->mainIndexGuard, std::defer_lock), + svs_iterator(nullptr), returned_results_set(this->allocator) { if (queryParams) { this->queryParams = @@ -548,7 +545,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto index = static_cast *>(idx); assert(index); // prevent parallel updates - std::lock_guard lock(index->updateJobMutex); + std::lock_guard lock(index->updateJobMutex); // Release the scheduled flag to allow scheduling again index->indexUpdateScheduled.clear(); // Update the SVS index @@ -575,7 +572,6 @@ class TieredSVSIndex : public VecSimTieredIndex { auto index = static_cast *>(idx); assert(index); - std::lock_guard lock{index->mainIndexGuard}; // Release the scheduled flag to allow scheduling again index->indexGCScheduled.clear(); @@ -605,7 +601,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto jobs = SVSMultiThreadJob::createJobs( this->allocator, SVS_BATCH_UPDATE_JOB, updateSVSIndexWrapper, this, total_threads, std::chrono::microseconds(updateJobWaitTime), &uncompletedJobs); - this->submitJobs(jobs); + this->submitUpdateJobs(jobs); } void scheduleSVSIndexGC() { @@ -678,29 +674,24 @@ class TieredSVSIndex : public VecSimTieredIndex { executeTracingCallback("UpdateJob::before_add_to_svs"); { // lock backend index for writing and add vectors there - std::shared_lock main_shared_lock(this->mainIndexGuard); auto svs_index = GetSVSIndex(); assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); if (this->backendIndex->indexSize() == 0) { - // If backend index is empty, we need to initialize it first. svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); - // Upgrade to unique lock to set the new impl - main_shared_lock.unlock(); - std::lock_guard lock(this->mainIndexGuard); + // std::lock_guard lock(this->mainIndexGuard); svs_index->setImpl(std::move(impl)); } else { - // Backend index is initialized - just add the vectors. - main_shared_lock.unlock(); - std::lock_guard lock(this->mainIndexGuard); - // Upgrade to unique lock to add vectors + // std::lock_guard lock(this->mainIndexGuard); + // std::shared_lock lock(this->mainIndexGuard); svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); } } + executeTracingCallback("UpdateJob::after_add_to_svs"); // clean-up frontend index { // lock frontend index for writing and delete moved vectors @@ -732,8 +723,6 @@ class TieredSVSIndex : public VecSimTieredIndex { // delete vectors from backend index that were deleted from the frontend index during // the update process. { - std::lock_guard main_lock(this->mainIndexGuard); - std::sort(deleted_labels_during_update.begin(), deleted_labels_during_update.end()); auto it = std::unique(deleted_labels_during_update.begin(), deleted_labels_during_update.end()); @@ -791,7 +780,6 @@ class TieredSVSIndex : public VecSimTieredIndex { // It is ok to lock everything at once for in-place mode, // but we will have to unlock averything before calling updateSVSIndexWrapper() // so make the minimal needed lock here. - std::shared_lock backend_shared_lock(this->mainIndexGuard); // Backend index initialization data have to be buffered for proper // compression/training. if (this->backendIndex->indexSize() == 0) { @@ -807,18 +795,18 @@ class TieredSVSIndex : public VecSimTieredIndex { // ... move vectors to the backend index. if (frontend_index_size >= this->trainingTriggerThreshold) { // updateSVSIndexWrapper() accures it's own locks - backend_shared_lock.unlock(); + // backend_shared_lock.unlock(); // initialize the SVS index synchonously using current thread only updateSVSIndexWrapper(this, 1); } return ret; } else { // backend index is initialized - we can add the vector directly - backend_shared_lock.unlock(); + // backend_shared_lock.unlock(); auto storage_blob = this->frontendIndex->preprocessForStorage(blob); // prevent update job from running in parallel and lock any access to the backend // index - std::scoped_lock lock(this->updateJobMutex, this->mainIndexGuard); + std::lock_guard lock(this->updateJobMutex); // Set available thread count to 1 for single vector write-in-place operation. // This maintains the contract that single vector operations use exactly one thread. // TODO: Replace this setNumThreads(1) call with an assertion once we establish @@ -840,15 +828,7 @@ class TieredSVSIndex : public VecSimTieredIndex { } } // Remove vector from the backend index if it exists in case of non-MULTI. - auto label_exists = [&]() { - std::shared_lock lock(this->mainIndexGuard); - return svs_index->isLabelExists(label); - }(); - - if (label_exists) { - std::lock_guard lock(this->mainIndexGuard); - ret -= this->backendIndex->deleteVector(label); - } + ret -= this->backendIndex->deleteVector(label); } { // Add vector to the frontend index. std::lock_guard lock(this->flatIndexGuard); @@ -869,7 +849,6 @@ class TieredSVSIndex : public VecSimTieredIndex { { // If main index is empty then update_threshold is trainingTriggerThreshold, // overwise it is updateTriggerThreshold. - std::shared_lock lock(this->mainIndexGuard); update_threshold = this->backendIndex->indexSize() == 0 ? this->trainingTriggerThreshold : this->updateTriggerThreshold; } @@ -936,15 +915,7 @@ class TieredSVSIndex : public VecSimTieredIndex { ret = this->deleteAndRecordSwaps_Unsafe(label); } - label_exists = [&]() { - std::shared_lock lock(this->mainIndexGuard); - return svs_index->isLabelExists(label); - }(); - - if (label_exists) { - std::lock_guard lock(this->mainIndexGuard); - ret += this->backendIndex->deleteVector(label); - } + ret += this->backendIndex->deleteVector(label); return ret; } size_t getNumMarkedDeleted() const override { @@ -953,13 +924,11 @@ class TieredSVSIndex : public VecSimTieredIndex { size_t indexSize() const override { std::shared_lock flat_lock(this->flatIndexGuard); - std::shared_lock main_lock(this->mainIndexGuard); return this->frontendIndex->indexSize() + this->backendIndex->indexSize(); } size_t indexCapacity() const override { std::shared_lock flat_lock(this->flatIndexGuard); - std::shared_lock main_lock(this->mainIndexGuard); return this->frontendIndex->indexCapacity() + this->backendIndex->indexCapacity(); } @@ -997,7 +966,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // the entire training duration (which can take 40-85s on slow machines). // If the mutex is held, training is actively running, so we report // indexUpdateScheduled = true (BACKGROUND_INDEXING = 1). - std::unique_lock lock(this->updateJobMutex, std::try_to_lock); + std::unique_lock lock(this->updateJobMutex, std::try_to_lock); if (lock.owns_lock()) { svsTieredInfo.indexUpdateScheduled = this->indexUpdateScheduled.test() == VecSimBool_TRUE; @@ -1052,17 +1021,81 @@ class TieredSVSIndex : public VecSimTieredIndex { VecSimQueryReply *topKQuery(const void *queryBlob, size_t k, VecSimQueryParams *queryParams) const override { - // SVS implements it's own distance computation functions which may cause sligthly different - // distance values than VecSim Flat Index does, so we always have to merge results with set. - return this->template topKQueryImp(queryBlob, k, queryParams); + // SVS handles its own internal locking for concurrent search + modification, + // so we don't need mainIndexGuard for backend queries. + this->flatIndexGuard.lock_shared(); + + if (this->frontendIndex->indexSize() == 0) { + this->flatIndexGuard.unlock_shared(); + + auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); + const void *processed_query = processed_query_ptr.get(); + return this->backendIndex->topKQuery(processed_query, k, queryParams); + } else { + auto flat_results = this->frontendIndex->topKQuery(queryBlob, k, queryParams); + this->flatIndexGuard.unlock_shared(); + + if (flat_results->code != VecSim_QueryReply_OK) { + assert(flat_results->results.empty()); + return flat_results; + } + + auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); + const void *processed_query = processed_query_ptr.get(); + auto main_results = this->backendIndex->topKQuery(processed_query, k, queryParams); + + if (main_results->code != VecSim_QueryReply_OK) { + VecSimQueryReply_Free(flat_results); + assert(main_results->results.empty()); + return main_results; + } + + return merge_result_lists(main_results, flat_results, k); + } } VecSimQueryReply *rangeQuery(const void *queryBlob, double radius, VecSimQueryParams *queryParams, VecSimQueryReply_Order order) const override { - // SVS implements it's own distance computation functions which may cause sligthly different - // distance values than VecSim Flat Index does, so we always have to merge results with set. - return this->template rangeQueryImp(queryBlob, radius, queryParams, order); + // SVS handles its own internal locking for concurrent search + modification, + // so we don't need mainIndexGuard for backend queries. + this->flatIndexGuard.lock_shared(); + + if (this->frontendIndex->indexSize() == 0) { + this->flatIndexGuard.unlock_shared(); + + auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); + const void *processed_query = processed_query_ptr.get(); + auto res = this->backendIndex->rangeQuery(processed_query, radius, queryParams); + sort_results(res, order); + return res; + } else { + auto flat_results = this->frontendIndex->rangeQuery(queryBlob, radius, queryParams); + this->flatIndexGuard.unlock_shared(); + + if (flat_results->code != VecSim_QueryReply_OK) { + return flat_results; + } + + auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); + const void *processed_query = processed_query_ptr.get(); + auto main_results = + this->backendIndex->rangeQuery(processed_query, radius, queryParams); + + if (BY_SCORE == order) { + sort_results_by_score_then_id(main_results); + sort_results_by_score_then_id(flat_results); + + auto code = main_results->code; + VecSimQueryReply *ret = merge_result_lists(main_results, flat_results, -1); + ret->code = code; + return ret; + } else { // BY_ID + concat_results(main_results, flat_results); + filter_results_by_id(main_results); + return main_results; + } + } } VecSimBatchIterator *newBatchIterator(const void *queryBlob, @@ -1081,7 +1114,6 @@ class TieredSVSIndex : public VecSimTieredIndex { TIERED_LOG(VecSimCommonStrings::LOG_VERBOSE_STRING, "running synchronous GC for tiered SVS index in write-in-place mode"); // In write-in-place mode, we run GC synchronously. - std::lock_guard lock{this->mainIndexGuard}; if (this->backendIndex->indexSize() == 0) { // No need to run GC on an empty index. return; @@ -1099,11 +1131,11 @@ class TieredSVSIndex : public VecSimTieredIndex { void acquireSharedLocks() override { this->flatIndexGuard.lock_shared(); - this->mainIndexGuard.lock_shared(); + // this->mainIndexGuard.lock_shared(); } void releaseSharedLocks() override { - this->mainIndexGuard.unlock_shared(); + // this->mainIndexGuard.unlock_shared(); this->flatIndexGuard.unlock_shared(); } }; diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index d097b1ae4..4f8b109a9 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -50,6 +50,7 @@ class VecSimTieredIndex : public VecSimIndexInterface { mutable std::shared_mutex flatIndexGuard; mutable std::shared_mutex mainIndexGuard; + mutable std::shared_mutex updateJobMutex; void lockMainIndexGuard() const { mainIndexGuard.lock(); #ifdef BUILD_TESTS @@ -76,6 +77,16 @@ class VecSimTieredIndex : public VecSimIndexInterface { jobs.size()); } + void submitUpdateJobs(vecsim_stl::vector &jobs) { + vecsim_stl::vector callbacks(jobs.size(), this->allocator); + for (size_t i = 0; i < jobs.size(); i++) { + callbacks[i] = jobs[i]->Execute; + } + std::shared_lock lock(this->updateJobMutex); + this->SubmitJobsToQueue(this->jobQueue, this->jobQueueCtx, jobs.data(), callbacks.data(), + jobs.size()); + } + /** * @brief Return the union of unique labels in both index tiers (which are not deleted). * This is a debug-only method for tiered indexes that computes the union of labels @@ -181,6 +192,7 @@ VecSimTieredIndex::topKQueryImp(const void *queryBlob, size_ // Simply query the main index and return the results while holding the lock. auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); + assert(false); this->mainIndexGuard.lock_shared(); auto res = this->backendIndex->topKQuery(processed_query, k, queryParams); this->mainIndexGuard.unlock_shared(); @@ -201,6 +213,7 @@ VecSimTieredIndex::topKQueryImp(const void *queryBlob, size_ auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); // Lock the main index and query it. + assert(false); this->mainIndexGuard.lock_shared(); auto main_results = this->backendIndex->topKQuery(processed_query, k, queryParams); this->mainIndexGuard.unlock_shared(); @@ -263,6 +276,7 @@ VecSimTieredIndex::rangeQueryImp(const void *queryBlob, doub auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); // Simply query the main index and return the results while holding the lock. + assert(false); this->mainIndexGuard.lock_shared(); auto res = this->backendIndex->rangeQuery(processed_query, radius, queryParams); this->mainIndexGuard.unlock_shared(); @@ -286,6 +300,7 @@ VecSimTieredIndex::rangeQueryImp(const void *queryBlob, doub auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); // Lock the main index and query it. + assert(false); this->mainIndexGuard.lock_shared(); auto main_results = this->backendIndex->rangeQuery(processed_query, radius, queryParams); this->mainIndexGuard.unlock_shared(); From 3f987c1187917a226de05b036efbd53b3efc0a24 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Fri, 26 Jun 2026 04:17:42 -0700 Subject: [PATCH 03/99] optimization --- src/VecSim/algorithms/svs/svs.h | 15 +- src/VecSim/algorithms/svs/svs_tiered.h | 244 ++++++++++++++++++++++--- src/VecSim/vec_sim_common.h | 1 + src/VecSim/vec_sim_tiered_index.h | 15 -- 4 files changed, 229 insertions(+), 46 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 200fdd51a..9043671a2 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -37,6 +37,7 @@ struct SVSIndexBase { SVSIndexBase() : num_marked_deleted{0} {}; virtual ~SVSIndexBase() = default; + virtual int addVector(const void *vector_data, labelType label) = 0; virtual int addVectors(const void *vectors_data, const labelType *labels, size_t n) = 0; virtual int deleteVectors(const labelType *labels, size_t n) = 0; virtual bool isLabelExists(labelType label) const = 0; @@ -45,7 +46,9 @@ struct SVSIndexBase virtual void setNumThreads(size_t numThreads) = 0; virtual size_t getThreadPoolCapacity() const = 0; virtual bool isCompressed() const = 0; - size_t getNumMarkedDeleted() const { return num_marked_deleted; } + size_t getNumMarkedDeleted() const { + return num_marked_deleted.load(std::memory_order_relaxed); + } // Abstract handler to manage SVS implementation instance // declared to avoid unsafe unique_ptr usage @@ -62,7 +65,7 @@ struct SVSIndexBase protected: // Index marked deleted vectors counter to initiate reindexing if it exceeds threshold // markIndexUpdate() manages this counter - size_t num_marked_deleted; + std::atomic num_marked_deleted; }; /** Thread Management Strategy: @@ -316,11 +319,11 @@ class SVSIndex : public VecSimIndexAbstract, fl // SVS index instance should not be empty if (indexLabelCount() == 0) { this->impl_.reset(); - num_marked_deleted = 0; + num_marked_deleted.store(0, std::memory_order_relaxed); return; } - num_marked_deleted += n; + num_marked_deleted.fetch_add(n, std::memory_order_relaxed); } bool isTwoLevelLVQ(const VecSimSvsQuantBits &qbits) { @@ -506,7 +509,7 @@ class SVSIndex : public VecSimIndexAbstract, fl // Enforce single-threaded execution for single vector operations to ensure optimal // performance and consistent behavior. Callers must set numThreads=1 before calling this // method. - assert(getNumThreads() == 1 && "Can't use more than one thread to insert a single vector"); + // assert(getNumThreads() == 1 && "Can't use more than one thread to insert a single vector"); return addVectorsImpl(vector_data, &label, 1); } @@ -721,7 +724,7 @@ class SVSIndex : public VecSimIndexAbstract, fl // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.compact impl_->compact(); } - num_marked_deleted = 0; + num_marked_deleted.store(0, std::memory_order_relaxed); } #ifdef BUILD_TESTS diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 947db01a1..82fd33b2c 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -13,6 +13,19 @@ #include #include +/** + * Definition of a job that inserts a new vector from flat into SVS Index. + */ +struct SVSInsertJob : public AsyncJob { + labelType label; + idType id; + + SVSInsertJob(std::shared_ptr allocator, labelType label_, idType id_, + JobCallback insertCb, VecSimIndex *index_) + : AsyncJob(allocator, SVS_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} +}; + + /** * @class SVSMultiThreadJob * @brief Represents a multi-threaded asynchronous job for the SVS algorithm. @@ -200,10 +213,11 @@ class SVSMultiThreadJob : public AsyncJob { template class TieredSVSIndex : public VecSimTieredIndex { + using DistType = float; using Self = TieredSVSIndex; - using Base = VecSimTieredIndex; - using flat_index_t = BruteForceIndex; - using backend_index_t = VecSimIndexAbstract; + using Base = VecSimTieredIndex; + using flat_index_t = BruteForceIndex; + using backend_index_t = VecSimIndexAbstract; using svs_index_t = SVSIndexBase; // swaps_journal is used by updateSVSIndex() to track vectors swap operations that were done in @@ -233,11 +247,22 @@ class TieredSVSIndex : public VecSimTieredIndex { std::atomic_flag indexGCScheduled = ATOMIC_FLAG_INIT; // Used to prevent running multiple index update jobs in parallel. // Even if update jobs scheduled sequentially, they can be started in parallel. - // mutable std::shared_mutex updateJobMutex; + mutable std::shared_mutex updateJobMutex; // The reason of following container just to properly destroy jobs which not executed yet SVSMultiThreadJob::JobsRegistry uncompletedJobs; + std::atomic backendReady{false}; + std::atomic backendInitSubmited{false}; + + vecsim_stl::unordered_map> labelToInsertJobs; + // A mapping to hold invalid jobs, so we can dispose them upon index deletion. + vecsim_stl::unordered_map invalidJobs; + idType currInvalidJobId; // A unique arbitrary identifier for accessing invalid jobs + std::mutex invalidJobsLookupGuard; + + size_t flat_buffer_bound; + /// //////////////////////////////////////////////////////////////////////////////////////////////////// // TieredSVS_BatchIterator // @@ -546,10 +571,17 @@ class TieredSVSIndex : public VecSimTieredIndex { assert(index); // prevent parallel updates std::lock_guard lock(index->updateJobMutex); - // Release the scheduled flag to allow scheduling again - index->indexUpdateScheduled.clear(); // Update the SVS index index->updateSVSIndex(availableThreads); + // Release the scheduled flag to allow scheduling again + index->indexUpdateScheduled.clear(); + } + + static void executeInsertJobWrapper(AsyncJob *job) { + auto *insert_job = static_cast(job); + auto *job_index = static_cast *>(insert_job->index); + job_index->executeInsertJob(insert_job); + delete job; } /** @@ -591,6 +623,21 @@ class TieredSVSIndex : public VecSimTieredIndex { #ifdef BUILD_TESTS public: #endif + + void updateInsertJobInternalId(idType prev_id, idType new_id, labelType label) { + // Update the pending job id, due to a swap that was caused after the removal of new_id. + assert(new_id != INVALID_ID && prev_id != INVALID_ID); + auto it = this->labelToInsertJobs.find(label); + if (it != this->labelToInsertJobs.end()) { + // There is a pending job for the label of the swapped last id - update its id. + for (SVSInsertJob *job_it : it->second) { + if (job_it->id == prev_id) { + job_it->id = new_id; + } + } + } + } + void scheduleSVSIndexUpdate() { // do not schedule if scheduled already if (indexUpdateScheduled.test_and_set()) { @@ -601,7 +648,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto jobs = SVSMultiThreadJob::createJobs( this->allocator, SVS_BATCH_UPDATE_JOB, updateSVSIndexWrapper, this, total_threads, std::chrono::microseconds(updateJobWaitTime), &uncompletedJobs); - this->submitUpdateJobs(jobs); + this->submitJobs(jobs); } void scheduleSVSIndexGC() { @@ -648,6 +695,73 @@ class TieredSVSIndex : public VecSimTieredIndex { } } + idType setAndSaveInvalidJob(AsyncJob *job) { + this->invalidJobsLookupGuard.lock(); + job->isValid = false; + idType curInvalidId = currInvalidJobId++; + this->invalidJobs.insert({curInvalidId, job}); + this->invalidJobsLookupGuard.unlock(); + return curInvalidId; + } + + void executeInsertJob(SVSInsertJob *job) { + assert(this->backendReady.load(std::memory_order_acquire)); + + // Note that accessing the job fields should occur with flat index guard held (here and later). + this->flatIndexGuard.lock_shared(); + if (!job->isValid) { + this->flatIndexGuard.unlock_shared(); + // Job has been invalidated in the meantime - nothing to execute, and remove it from the + // lookup. + this->invalidJobsLookupGuard.lock(); + this->invalidJobs.erase(job->id); + this->invalidJobsLookupGuard.unlock(); + return; + } + this->flatIndexGuard.unlock_shared(); + + auto svs_index = GetSVSIndex(); + auto storage_blob = this->frontendIndex->preprocessForStorage(this->frontendIndex->getDataByInternalId(job->id)); + svs_index->addVector(storage_blob.get(), job->label); + + // Remove the vector and the insert job from the flat buffer. + this->flatIndexGuard.lock(); + // The job might have been invalidated due to overwrite in the meantime. In this case, + // it was already deleted and the job has been evicted. Otherwise, we need to do it now. + if (job->isValid) { + // Remove the job pointer from the labelToInsertJobs mapping. + auto &jobs = labelToInsertJobs.at(job->label); + for (size_t i = 0; i < jobs.size(); i++) { + if (jobs[i]->id == job->id) { + jobs.erase(jobs.begin() + (long)i); + break; + } + } + if (labelToInsertJobs.at(job->label).empty()) { + labelToInsertJobs.erase(job->label); + } + // Remove the vector from the flat buffer. + // The flat buffer stores data in a contiguous + // array, so deleting an element may move the last element into the freed slot to keep + // ids dense. Capture the last id's label beforehand (after deletion it is no longer in + // the lookup) so we can later fix up its insert job. + labelType last_vec_label = + this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); + int deleted = this->frontendIndex->deleteVectorById(job->label, job->id); + if (deleted && job->id != this->frontendIndex->indexSize()) { + // If the vector removal caused a swap with the last id, update the relevant insert job. + this->updateInsertJobInternalId(this->frontendIndex->indexSize(), job->id, + last_vec_label); + } + } else { + // Remove the current job from the invalid jobs' lookup, as we are about to delete it now. + this->invalidJobsLookupGuard.lock(); + this->invalidJobs.erase(job->id); + this->invalidJobsLookupGuard.unlock(); + } + this->flatIndexGuard.unlock(); + } + void updateSVSIndex(size_t availableThreads) { std::vector labels_to_move; std::vector vectors_to_move; @@ -658,6 +772,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto flat_index = this->GetFlatIndex(); const auto frontend_index_size = this->frontendIndex->indexSize(); + // fprintf(stderr, "updateSVSIndex: frontend_index_size = %ld\n", frontend_index_size); const size_t dim = flat_index->getDim(); labels_to_move.reserve(frontend_index_size); vectors_to_move.reserve(frontend_index_size * dim); @@ -681,12 +796,9 @@ class TieredSVSIndex : public VecSimTieredIndex { auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); - // std::lock_guard lock(this->mainIndexGuard); svs_index->setImpl(std::move(impl)); } else { - // std::lock_guard lock(this->mainIndexGuard); - // std::shared_lock lock(this->mainIndexGuard); - svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); + // svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); } @@ -709,6 +821,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // improvement int deleted = 0; idType id = labels_to_move.size(); + // fprintf(stderr, "updateSVSIndexMidle: frontend_index_size = %ld\n", this->frontendIndex->indexSize()); while (id-- > 0) { auto label = labels_to_move[id]; // Delete the vector from the frontend index if not in-place updated. @@ -716,6 +829,7 @@ class TieredSVSIndex : public VecSimTieredIndex { deleted += this->frontendIndex->deleteVectorById(label, id); } } + // fprintf(stderr, "updateSVSIndexEnd: frontend_index_size = %ld\n", this->frontendIndex->indexSize()); assert(deleted == std::count_if(labels_to_move.begin(), labels_to_move.end(), [](labelType label) { return label != SKIP_LABEL; }) && "Deleted vectors count does not match the number of labels to delete"); @@ -731,6 +845,7 @@ class TieredSVSIndex : public VecSimTieredIndex { svs_index->deleteVectors(deleted_labels_during_update.data(), deleted_labels_during_update.size()); } + this->backendReady.store(true, std::memory_order_release); } public: @@ -738,11 +853,14 @@ class TieredSVSIndex : public VecSimTieredIndex { const TieredIndexParams &tiered_index_params, std::shared_ptr allocator) : Base(svs_index, bf_index, tiered_index_params, allocator), - uncompletedJobs(this->allocator) { + uncompletedJobs(this->allocator), + labelToInsertJobs(this->allocator), + invalidJobs(this->allocator), + currInvalidJobId(0) { const auto &tiered_svs_params = tiered_index_params.specificParams.tieredSVSParams; // If flatBufferLimit is not initialized (0), use the default update threshold. - const size_t flat_buffer_bound = tiered_index_params.flatBufferLimit == 0 + flat_buffer_bound = tiered_index_params.flatBufferLimit == 0 ? SVS_VAMANA_DEFAULT_UPDATE_THRESHOLD : tiered_index_params.flatBufferLimit; @@ -830,7 +948,9 @@ class TieredSVSIndex : public VecSimTieredIndex { // Remove vector from the backend index if it exists in case of non-MULTI. ret -= this->backendIndex->deleteVector(label); } - { // Add vector to the frontend index. + + if (!this->backendInitSubmited.load(std::memory_order_acquire)) { + // Add vector to the frontend index. std::lock_guard lock(this->flatIndexGuard); const auto ft_ret = this->frontendIndex->addVector(blob, label); @@ -843,20 +963,94 @@ class TieredSVSIndex : public VecSimTieredIndex { deleted_labels_journal.push_back(label); } ret = std::max(ret + ft_ret, 0); - // Check frontend index size to determine if an update job schedule is needed. - frontend_index_size = this->frontendIndex->indexSize(); - } - { - // If main index is empty then update_threshold is trainingTriggerThreshold, - // overwise it is updateTriggerThreshold. - update_threshold = this->backendIndex->indexSize() == 0 ? this->trainingTriggerThreshold - : this->updateTriggerThreshold; + + if (this->frontendIndex->indexSize() >= this->trainingTriggerThreshold) { + this->backendInitSubmited.store(true, std::memory_order_release); + scheduleSVSIndexUpdate(); + // fprintf(stderr, "Init submited: this->frontendIndex->indexSize() = %ld\n", this->frontendIndex->indexSize()); + } + return ret; } - if (frontend_index_size >= update_threshold) { - scheduleSVSIndexUpdate(); + + // fprintf(stderr, "Waiting for backend ready\t"); + // spin-lock, while backend is initilizing + while (!this->backendReady.load(std::memory_order_acquire)) { + __builtin_ia32_pause(); } + // fprintf(stderr, "done: this->frontendIndex->indexSize() = %ld\n", this->frontendIndex->indexSize()); - return ret; + if (this->frontendIndex->indexSize() >= flat_buffer_bound) { + auto storage_blob = this->frontendIndex->preprocessForStorage(blob); + return ret = svs_index->addVector(storage_blob.get(), label); + } else { + this->flatIndexGuard.lock(); + idType new_flat_id = this->frontendIndex->indexSize(); + if (this->frontendIndex->isLabelExists(label) && !this->frontendIndex->isMultiValue()) { + // Overwrite the vector and invalidate its only pending job (since we are not in MULTI). + auto *old_job = this->labelToInsertJobs.at(label).at(0); + old_job->id = this->setAndSaveInvalidJob(old_job); + this->labelToInsertJobs.erase(label); + ret = 0; + // We are going to update the internal id that currently holds the vector associated with + // the given label. + new_flat_id = + dynamic_cast *>(this->frontendIndex) + ->getIdOfLabel(label); + // If we are adding a new element (rather than updating an exiting one) we may need to + // increase index capacity. + } + // If this label already exists, this will do overwrite. + this->frontendIndex->addVector(blob, label); + + AsyncJob *new_insert_job = new (this->allocator) + SVSInsertJob(this->allocator, label, new_flat_id, executeInsertJobWrapper, this); + // Save a pointer to the job, so that if the vector is overwritten, we'll have an indication. + if (this->labelToInsertJobs.find(label) != this->labelToInsertJobs.end()) { + // There's already a pending insert job for this label, add another one (without overwrite, + // only possible in multi index) + assert(this->backendIndex->isMultiValue()); + this->labelToInsertJobs.at(label).push_back((SVSInsertJob *)new_insert_job); + } else { + vecsim_stl::vector new_jobs_vec(1, (SVSInsertJob *)new_insert_job, + this->allocator); + this->labelToInsertJobs.insert({label, new_jobs_vec}); + } + this->flatIndexGuard.unlock(); + + // Insert job to the queue and signal the workers' updater. + this->submitSingleJob(new_insert_job); + return ret; + // } else { + // // // Add vector to the frontend index. + // // std::lock_guard lock(this->flatIndexGuard); + // // const auto ft_ret = this->frontendIndex->addVector(blob, label); + + // // if (ft_ret == 0) { // Vector was overriden - add 'skiping' swap to the journal. + // // assert(!this->backendIndex->isMultiValue() && + // // "addVector() may return 0 for single value indices only"); + // // for (auto id : this->frontendIndex->getElementIds(label)) { + // // this->swaps_journal.emplace_back(SKIP_LABEL, id, id); + // // } + // // deleted_labels_journal.push_back(label); + // // } + // // ret = std::max(ret + ft_ret, 0); + + // // // Check frontend index size to determine if an update job schedule is needed. + // // frontend_index_size = this->frontendIndex->indexSize(); + // // // if (this->backendInitSubmited.load(std::memory_order_acquire)) { + // // // if (frontend_index_size >= this->updateTriggerThreshold) { + // // // if (this->backendReady.load(std::memory_order_acquire)) { + // // // scheduleSVSIndexUpdate(); + // // // } + // // // } + // // // } else { + // // if (frontend_index_size >= this->trainingTriggerThreshold) { + // // this->backendInitSubmited.store(true, std::memory_order_release); + // // scheduleSVSIndexUpdate(); + // // } + // // // } + // // return ret; + } } int deleteAndRecordSwaps_Unsafe(labelType label) { diff --git a/src/VecSim/vec_sim_common.h b/src/VecSim/vec_sim_common.h index 82aceaa1c..03e7870f3 100644 --- a/src/VecSim/vec_sim_common.h +++ b/src/VecSim/vec_sim_common.h @@ -273,6 +273,7 @@ typedef enum { HNSW_SWAP_JOB, HNSW_DISK_JOB, SVS_BATCH_UPDATE_JOB, + SVS_INSERT_VECTOR_JOB, SVS_GC_JOB, INVALID_JOB // to indicate that finding a JobType >= INVALID_JOB is an error } JobType; diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 4f8b109a9..d097b1ae4 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -50,7 +50,6 @@ class VecSimTieredIndex : public VecSimIndexInterface { mutable std::shared_mutex flatIndexGuard; mutable std::shared_mutex mainIndexGuard; - mutable std::shared_mutex updateJobMutex; void lockMainIndexGuard() const { mainIndexGuard.lock(); #ifdef BUILD_TESTS @@ -77,16 +76,6 @@ class VecSimTieredIndex : public VecSimIndexInterface { jobs.size()); } - void submitUpdateJobs(vecsim_stl::vector &jobs) { - vecsim_stl::vector callbacks(jobs.size(), this->allocator); - for (size_t i = 0; i < jobs.size(); i++) { - callbacks[i] = jobs[i]->Execute; - } - std::shared_lock lock(this->updateJobMutex); - this->SubmitJobsToQueue(this->jobQueue, this->jobQueueCtx, jobs.data(), callbacks.data(), - jobs.size()); - } - /** * @brief Return the union of unique labels in both index tiers (which are not deleted). * This is a debug-only method for tiered indexes that computes the union of labels @@ -192,7 +181,6 @@ VecSimTieredIndex::topKQueryImp(const void *queryBlob, size_ // Simply query the main index and return the results while holding the lock. auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); - assert(false); this->mainIndexGuard.lock_shared(); auto res = this->backendIndex->topKQuery(processed_query, k, queryParams); this->mainIndexGuard.unlock_shared(); @@ -213,7 +201,6 @@ VecSimTieredIndex::topKQueryImp(const void *queryBlob, size_ auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); // Lock the main index and query it. - assert(false); this->mainIndexGuard.lock_shared(); auto main_results = this->backendIndex->topKQuery(processed_query, k, queryParams); this->mainIndexGuard.unlock_shared(); @@ -276,7 +263,6 @@ VecSimTieredIndex::rangeQueryImp(const void *queryBlob, doub auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); // Simply query the main index and return the results while holding the lock. - assert(false); this->mainIndexGuard.lock_shared(); auto res = this->backendIndex->rangeQuery(processed_query, radius, queryParams); this->mainIndexGuard.unlock_shared(); @@ -300,7 +286,6 @@ VecSimTieredIndex::rangeQueryImp(const void *queryBlob, doub auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); // Lock the main index and query it. - assert(false); this->mainIndexGuard.lock_shared(); auto main_results = this->backendIndex->rangeQuery(processed_query, radius, queryParams); this->mainIndexGuard.unlock_shared(); From f2fe02f69653bd3df2626ba474631732339a1b04 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Fri, 26 Jun 2026 06:06:39 -0700 Subject: [PATCH 04/99] fix threadpool --- src/VecSim/algorithms/svs/svs_tiered.h | 34 +++----------------------- 1 file changed, 3 insertions(+), 31 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 82fd33b2c..2657679a4 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -797,10 +797,12 @@ class TieredSVSIndex : public VecSimTieredIndex { labels_to_move.size()); svs_index->setImpl(std::move(impl)); + svs_index->setNumThreads(1); } else { - // svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); + svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); + svs_index->setNumThreads(1); } } @@ -1020,36 +1022,6 @@ class TieredSVSIndex : public VecSimTieredIndex { // Insert job to the queue and signal the workers' updater. this->submitSingleJob(new_insert_job); return ret; - // } else { - // // // Add vector to the frontend index. - // // std::lock_guard lock(this->flatIndexGuard); - // // const auto ft_ret = this->frontendIndex->addVector(blob, label); - - // // if (ft_ret == 0) { // Vector was overriden - add 'skiping' swap to the journal. - // // assert(!this->backendIndex->isMultiValue() && - // // "addVector() may return 0 for single value indices only"); - // // for (auto id : this->frontendIndex->getElementIds(label)) { - // // this->swaps_journal.emplace_back(SKIP_LABEL, id, id); - // // } - // // deleted_labels_journal.push_back(label); - // // } - // // ret = std::max(ret + ft_ret, 0); - - // // // Check frontend index size to determine if an update job schedule is needed. - // // frontend_index_size = this->frontendIndex->indexSize(); - // // // if (this->backendInitSubmited.load(std::memory_order_acquire)) { - // // // if (frontend_index_size >= this->updateTriggerThreshold) { - // // // if (this->backendReady.load(std::memory_order_acquire)) { - // // // scheduleSVSIndexUpdate(); - // // // } - // // // } - // // // } else { - // // if (frontend_index_size >= this->trainingTriggerThreshold) { - // // this->backendInitSubmited.store(true, std::memory_order_release); - // // scheduleSVSIndexUpdate(); - // // } - // // // } - // // return ret; } } From f806dca336f35d3be767bc06e0c76f349fb24068 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Fri, 26 Jun 2026 08:42:23 -0700 Subject: [PATCH 05/99] minor fix --- src/VecSim/algorithms/svs/svs_tiered.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 2657679a4..48ea51dc8 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -718,10 +718,10 @@ class TieredSVSIndex : public VecSimTieredIndex { this->invalidJobsLookupGuard.unlock(); return; } + auto storage_blob = this->frontendIndex->preprocessForStorage(this->frontendIndex->getDataByInternalId(job->id)); this->flatIndexGuard.unlock_shared(); auto svs_index = GetSVSIndex(); - auto storage_blob = this->frontendIndex->preprocessForStorage(this->frontendIndex->getDataByInternalId(job->id)); svs_index->addVector(storage_blob.get(), job->label); // Remove the vector and the insert job from the flat buffer. @@ -1005,7 +1005,8 @@ class TieredSVSIndex : public VecSimTieredIndex { this->frontendIndex->addVector(blob, label); AsyncJob *new_insert_job = new (this->allocator) - SVSInsertJob(this->allocator, label, new_flat_id, executeInsertJobWrapper, this); + SVSInsertJob(this->allocator, label, new_flat_id, executeInsertJobWrapper, this); + // Save a pointer to the job, so that if the vector is overwritten, we'll have an indication. if (this->labelToInsertJobs.find(label) != this->labelToInsertJobs.end()) { // There's already a pending insert job for this label, add another one (without overwrite, From 28503c25787a3283f6a2f362555eac06c2e661df Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 29 Jun 2026 08:42:54 -0700 Subject: [PATCH 06/99] minor cleaning --- src/VecSim/algorithms/svs/svs.h | 4 ++-- src/VecSim/algorithms/svs/svs_tiered.h | 28 ++++++++------------------ 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 9043671a2..67e628d6d 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -351,7 +351,7 @@ class SVSIndex : public VecSimIndexAbstract, fl svs_details::getOrDefault(params.leanvec_dim, SVS_VAMANA_DEFAULT_LEANVEC_DIM)}, epsilon{svs_details::getOrDefault(params.epsilon, SVS_VAMANA_DEFAULT_EPSILON)}, is_two_level_lvq{isTwoLevelLVQ(params.quantBits)}, - threadpool_{std::max(size_t{SVS_VAMANA_DEFAULT_NUM_THREADS}, params.num_threads)}, + threadpool_{1}, impl_{nullptr} { logger_ = makeLogger(); } @@ -509,7 +509,7 @@ class SVSIndex : public VecSimIndexAbstract, fl // Enforce single-threaded execution for single vector operations to ensure optimal // performance and consistent behavior. Callers must set numThreads=1 before calling this // method. - // assert(getNumThreads() == 1 && "Can't use more than one thread to insert a single vector"); + assert(getNumThreads() == 1 && "Can't use more than one thread to insert a single vector"); return addVectorsImpl(vector_data, &label, 1); } diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 48ea51dc8..66c8ce530 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -792,17 +792,17 @@ class TieredSVSIndex : public VecSimTieredIndex { auto svs_index = GetSVSIndex(); assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); if (this->backendIndex->indexSize() == 0) { - svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); + // svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); svs_index->setImpl(std::move(impl)); - svs_index->setNumThreads(1); + // svs_index->setNumThreads(1); } else { - svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); + // svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); - svs_index->setNumThreads(1); + // svs_index->setNumThreads(1); } } @@ -938,19 +938,6 @@ class TieredSVSIndex : public VecSimTieredIndex { assert(this->getWriteMode() != VecSim_WriteInPlace && "InPlace mode returns early"); // Async mode - add vector to the frontend index and schedule an update job if needed. - if (!this->backendIndex->isMultiValue()) { - { - std::shared_lock lock(this->flatIndexGuard); - // If the label already exists in the frontend index, we should count it - // to prevent the case when existing vector is moved meanwhile by the update job. - if (this->frontendIndex->isLabelExists(label)) { - ret = -1; - } - } - // Remove vector from the backend index if it exists in case of non-MULTI. - ret -= this->backendIndex->deleteVector(label); - } - if (!this->backendInitSubmited.load(std::memory_order_acquire)) { // Add vector to the frontend index. std::lock_guard lock(this->flatIndexGuard); @@ -969,22 +956,23 @@ class TieredSVSIndex : public VecSimTieredIndex { if (this->frontendIndex->indexSize() >= this->trainingTriggerThreshold) { this->backendInitSubmited.store(true, std::memory_order_release); scheduleSVSIndexUpdate(); - // fprintf(stderr, "Init submited: this->frontendIndex->indexSize() = %ld\n", this->frontendIndex->indexSize()); } return ret; } - // fprintf(stderr, "Waiting for backend ready\t"); // spin-lock, while backend is initilizing while (!this->backendReady.load(std::memory_order_acquire)) { __builtin_ia32_pause(); } - // fprintf(stderr, "done: this->frontendIndex->indexSize() = %ld\n", this->frontendIndex->indexSize()); + this->flatIndexGuard.lock_shared(); if (this->frontendIndex->indexSize() >= flat_buffer_bound) { + this->flatIndexGuard.unlock_shared(); auto storage_blob = this->frontendIndex->preprocessForStorage(blob); + return ret = svs_index->addVector(storage_blob.get(), label); } else { + this->flatIndexGuard.unlock_shared(); this->flatIndexGuard.lock(); idType new_flat_id = this->frontendIndex->indexSize(); if (this->frontendIndex->isLabelExists(label) && !this->frontendIndex->isMultiValue()) { From f2596b36549122ad5802e2c914e992a13b875cc3 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 1 Jul 2026 04:47:03 -0700 Subject: [PATCH 07/99] clean up --- src/VecSim/algorithms/svs/svs_tiered.h | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 66c8ce530..909da8266 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -772,7 +772,6 @@ class TieredSVSIndex : public VecSimTieredIndex { auto flat_index = this->GetFlatIndex(); const auto frontend_index_size = this->frontendIndex->indexSize(); - // fprintf(stderr, "updateSVSIndex: frontend_index_size = %ld\n", frontend_index_size); const size_t dim = flat_index->getDim(); labels_to_move.reserve(frontend_index_size); vectors_to_move.reserve(frontend_index_size * dim); @@ -792,17 +791,13 @@ class TieredSVSIndex : public VecSimTieredIndex { auto svs_index = GetSVSIndex(); assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); if (this->backendIndex->indexSize() == 0) { - // svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); svs_index->setImpl(std::move(impl)); - // svs_index->setNumThreads(1); } else { - // svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); - // svs_index->setNumThreads(1); } } @@ -823,7 +818,6 @@ class TieredSVSIndex : public VecSimTieredIndex { // improvement int deleted = 0; idType id = labels_to_move.size(); - // fprintf(stderr, "updateSVSIndexMidle: frontend_index_size = %ld\n", this->frontendIndex->indexSize()); while (id-- > 0) { auto label = labels_to_move[id]; // Delete the vector from the frontend index if not in-place updated. @@ -831,7 +825,6 @@ class TieredSVSIndex : public VecSimTieredIndex { deleted += this->frontendIndex->deleteVectorById(label, id); } } - // fprintf(stderr, "updateSVSIndexEnd: frontend_index_size = %ld\n", this->frontendIndex->indexSize()); assert(deleted == std::count_if(labels_to_move.begin(), labels_to_move.end(), [](labelType label) { return label != SKIP_LABEL; }) && "Deleted vectors count does not match the number of labels to delete"); @@ -915,14 +908,12 @@ class TieredSVSIndex : public VecSimTieredIndex { // ... move vectors to the backend index. if (frontend_index_size >= this->trainingTriggerThreshold) { // updateSVSIndexWrapper() accures it's own locks - // backend_shared_lock.unlock(); // initialize the SVS index synchonously using current thread only updateSVSIndexWrapper(this, 1); } return ret; } else { // backend index is initialized - we can add the vector directly - // backend_shared_lock.unlock(); auto storage_blob = this->frontendIndex->preprocessForStorage(blob); // prevent update job from running in parallel and lock any access to the backend // index @@ -1286,11 +1277,9 @@ class TieredSVSIndex : public VecSimTieredIndex { void acquireSharedLocks() override { this->flatIndexGuard.lock_shared(); - // this->mainIndexGuard.lock_shared(); } void releaseSharedLocks() override { - // this->mainIndexGuard.unlock_shared(); this->flatIndexGuard.unlock_shared(); } }; From 6dc0722d1903a3592bd8394737bac1d45cbd66ce Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 1 Jul 2026 06:25:52 -0700 Subject: [PATCH 08/99] remove dead code --- deps/ScalableVectorSearch | 2 +- src/VecSim/algorithms/svs/svs.h | 13 +++++-------- src/VecSim/algorithms/svs/svs_tiered.h | 14 ++++++-------- 3 files changed, 12 insertions(+), 17 deletions(-) mode change 160000 => 120000 deps/ScalableVectorSearch diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch deleted file mode 160000 index 02dea9d9c..000000000 --- a/deps/ScalableVectorSearch +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 02dea9d9cd1c73dc03dc0fd91d14a548f1aece70 diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch new file mode 120000 index 000000000..25f58fd1a --- /dev/null +++ b/deps/ScalableVectorSearch @@ -0,0 +1 @@ +/home/drazdobu/libraries.ai.vector-search.svs \ No newline at end of file diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 67e628d6d..a3f2ca8c5 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -543,16 +543,13 @@ class SVSIndex : public VecSimIndexAbstract, fl } double getDistanceFrom_Unsafe(labelType label, const void *vector_data) const override { - if (!impl_) return std::numeric_limits::quiet_NaN(); + if (!impl_ || !impl_->has_id(label)) { + return std::numeric_limits::quiet_NaN(); + }; auto query_datum = std::span{static_cast(vector_data), this->dim}; - try { - auto dist = impl_->get_distance(label, query_datum); - return std::isnan(dist) ? std::numeric_limits::quiet_NaN() - : toVecSimDistance(static_cast(dist)); - } catch (const svs::lib::ANNException &) { - return std::numeric_limits::quiet_NaN(); - } + auto dist = impl_->get_distance(label, query_datum); + return toVecSimDistance(dist); } VecSimQueryReply *topKQuery(const void *queryBlob, size_t k, diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 909da8266..383734248 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -247,7 +247,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::atomic_flag indexGCScheduled = ATOMIC_FLAG_INIT; // Used to prevent running multiple index update jobs in parallel. // Even if update jobs scheduled sequentially, they can be started in parallel. - mutable std::shared_mutex updateJobMutex; + mutable std::mutex updateJobMutex; // The reason of following container just to properly destroy jobs which not executed yet SVSMultiThreadJob::JobsRegistry uncompletedJobs; @@ -570,11 +570,11 @@ class TieredSVSIndex : public VecSimTieredIndex { auto index = static_cast *>(idx); assert(index); // prevent parallel updates - std::lock_guard lock(index->updateJobMutex); - // Update the SVS index - index->updateSVSIndex(availableThreads); + std::lock_guard lock(index->updateJobMutex); // Release the scheduled flag to allow scheduling again index->indexUpdateScheduled.clear(); + // Update the SVS index + index->updateSVSIndex(availableThreads); } static void executeInsertJobWrapper(AsyncJob *job) { @@ -885,7 +885,6 @@ class TieredSVSIndex : public VecSimTieredIndex { int addVector(const void *blob, labelType label) override { int ret = 0; auto svs_index = GetSVSIndex(); - size_t update_threshold = 0; size_t frontend_index_size = 0; // In-Place mode - add vector syncronously to the backend index. @@ -917,7 +916,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto storage_blob = this->frontendIndex->preprocessForStorage(blob); // prevent update job from running in parallel and lock any access to the backend // index - std::lock_guard lock(this->updateJobMutex); + std::lock_guard lock(this->updateJobMutex); // Set available thread count to 1 for single vector write-in-place operation. // This maintains the contract that single vector operations use exactly one thread. // TODO: Replace this setNumThreads(1) call with an assertion once we establish @@ -1047,7 +1046,6 @@ class TieredSVSIndex : public VecSimTieredIndex { int deleteVector(labelType label) override { int ret = 0; - auto svs_index = GetSVSIndex(); // Backend index deletions to be synchronized with the frontend index, // elsewhere there is the risk of labels duplication in both indices which can lead to wrong // results of topK queries. In such case we should behave as if InPlace mode is always set. @@ -1112,7 +1110,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // the entire training duration (which can take 40-85s on slow machines). // If the mutex is held, training is actively running, so we report // indexUpdateScheduled = true (BACKGROUND_INDEXING = 1). - std::unique_lock lock(this->updateJobMutex, std::try_to_lock); + std::unique_lock lock(this->updateJobMutex, std::try_to_lock); if (lock.owns_lock()) { svsTieredInfo.indexUpdateScheduled = this->indexUpdateScheduled.test() == VecSimBool_TRUE; From 402691f7fa80b202a7aaf8be7602c6dcfcbc5e37 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 8 Jul 2026 00:30:16 -0700 Subject: [PATCH 09/99] fixes; tests pass --- src/VecSim/algorithms/svs/svs.h | 285 ++++++++++++------ .../algorithms/svs/svs_serializer_impl.h | 1 + src/VecSim/algorithms/svs/svs_tiered.h | 264 +++++++++------- tests/unit/test_svs_tiered.cpp | 15 +- 4 files changed, 360 insertions(+), 205 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index a3f2ca8c5..7dcdc1d93 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -39,6 +39,7 @@ struct SVSIndexBase virtual ~SVSIndexBase() = default; virtual int addVector(const void *vector_data, labelType label) = 0; virtual int addVectors(const void *vectors_data, const labelType *labels, size_t n) = 0; + virtual int deleteVector(labelType label) = 0; virtual int deleteVectors(const labelType *labels, size_t n) = 0; virtual bool isLabelExists(labelType label) const = 0; virtual size_t indexStorageSize() const = 0; @@ -46,6 +47,8 @@ struct SVSIndexBase virtual void setNumThreads(size_t numThreads) = 0; virtual size_t getThreadPoolCapacity() const = 0; virtual bool isCompressed() const = 0; + virtual bool ready() const = 0; + size_t getNumMarkedDeleted() const { return num_marked_deleted.load(std::memory_order_relaxed); } @@ -116,6 +119,17 @@ class SVSIndex : public VecSimIndexAbstract, fl svs::logging::logger_ptr logger_; // SVS Index implementation instance std::unique_ptr impl_; + mutable std::shared_mutex pimplGuard_; + + std::atomic impl_ready_{false}; + + void setReady() { + this->impl_ready_.store(true, std::memory_order_release); + } + + void setUnready() { + this->impl_ready_.store(false, std::memory_order_release); + } static double toVecSimDistance(float v) { return svs_details::toVecSimDistance(v); } @@ -243,7 +257,7 @@ class SVSIndex : public VecSimIndexAbstract, fl } void setImpl(std::unique_ptr handler) override { - if (impl_ != nullptr) { + if (ready()) { throw std::logic_error("SVSIndex::setImpl called on non-empty impl_"); } @@ -251,7 +265,11 @@ class SVSIndex : public VecSimIndexAbstract, fl if (!svs_handler) { throw std::logic_error("Failed to cast to SVSImplHandler"); } - this->impl_ = std::move(svs_handler->impl); + { + std::lock_guard lock(this->pimplGuard_); + this->impl_ = std::move(svs_handler->impl); + if (this->impl_) setReady(); + } } // Assuming numThreads was updated to reflect the number of available threads before this @@ -273,16 +291,23 @@ class SVSIndex : public VecSimIndexAbstract, fl // Wrap data into SVS SimpleDataView for SVS API auto points = svs::data::SimpleDataView{typed_vectors_data, n, this->dim}; - if (!impl_) { - // SVS index instance cannot be empty, so we have to construct it at first rows - impl_ = initImpl(points, ids); - } else { - if constexpr (!isMulti) { + if constexpr (!isMulti) { + if (ready()) { // SVS index does not support overriding vectors with the same label // so we have to delete them first if needed deleted_num = deleteVectorsImpl(labels, n); } + } + + if (!ready()) { + // SVS index instance cannot be empty, so we have to construct it at first rows + std::lock_guard lock(this->pimplGuard_); + impl_ = initImpl(points, ids); + assert(this->impl_ != nullptr); + setReady(); + } else { // Add new points to existing SVS index + std::shared_lock lock(this->pimplGuard_); impl_->add_points(points, ids); } @@ -294,7 +319,11 @@ class SVSIndex : public VecSimIndexAbstract, fl return 0; } - const auto deleted_num = impl_->delete_entries(std::span{&label, 1}); + int deleted_num = 0; + { + std::shared_lock lock(this->pimplGuard_); + deleted_num = impl_->delete_entries(std::span{&label, 1}); + } this->markIndexUpdate(deleted_num); return deleted_num; @@ -305,7 +334,11 @@ class SVSIndex : public VecSimIndexAbstract, fl return 0; } - const auto deleted_num = impl_->delete_entries(std::span{labels, n}); + int deleted_num = 0; + { + std::shared_lock lock(this->pimplGuard_); + deleted_num = impl_->delete_entries(std::span{labels, n}); + } this->markIndexUpdate(deleted_num); return deleted_num; @@ -313,12 +346,16 @@ class SVSIndex : public VecSimIndexAbstract, fl // Count deletions and consolidate index if needed void markIndexUpdate(size_t n = 1) { - if (!impl_) + if (!ready()) return; // SVS index instance should not be empty if (indexLabelCount() == 0) { - this->impl_.reset(); + std::lock_guard lock(this->pimplGuard_); + { + setUnready(); + this->impl_.reset(); + } num_marked_deleted.store(0, std::memory_order_relaxed); return; } @@ -351,32 +388,59 @@ class SVSIndex : public VecSimIndexAbstract, fl svs_details::getOrDefault(params.leanvec_dim, SVS_VAMANA_DEFAULT_LEANVEC_DIM)}, epsilon{svs_details::getOrDefault(params.epsilon, SVS_VAMANA_DEFAULT_EPSILON)}, is_two_level_lvq{isTwoLevelLVQ(params.quantBits)}, - threadpool_{1}, + threadpool_{std::max(size_t{SVS_VAMANA_DEFAULT_NUM_THREADS}, params.num_threads)}, impl_{nullptr} { logger_ = makeLogger(); } ~SVSIndex() = default; + bool ready() const override { + return this->impl_ready_.load(std::memory_order_acquire); + } + size_t indexSize() const override { return indexStorageSize(); } - size_t indexStorageSize() const override { return impl_ ? impl_->view_data().size() : 0; } + size_t indexStorageSize() const override { + if (ready()) { + std::shared_lock lock(this->pimplGuard_); + return impl_->view_data().size(); + } else { + return 0; + } + } size_t indexCapacity() const override { - return impl_ ? storage_traits_t::storage_capacity(impl_->view_data()) : 0; + if (ready()) { + std::shared_lock lock(this->pimplGuard_); + return storage_traits_t::storage_capacity(impl_->view_data()); + } else { + return 0; + } } size_t indexLabelCount() const override { if constexpr (isMulti) { - return impl_ ? impl_->labelcount() : 0; + if (ready()) { + std::shared_lock lock(this->pimplGuard_); + return impl_->labelcount(); + } else { + return 0; + } } else { - return impl_ ? impl_->size() : 0; + if (ready()) { + std::shared_lock lock(this->pimplGuard_); + return impl_->size(); + } else { + return 0; + } } } vecsim_stl::set getLabelsSet() const override { vecsim_stl::set labels(this->allocator); - if (impl_) { + if (ready()) { + std::shared_lock lock(this->pimplGuard_); impl_->on_ids([&labels](size_t label) { labels.insert(label); }); } return labels; @@ -528,7 +592,12 @@ class SVSIndex : public VecSimIndexAbstract, fl } bool isLabelExists(labelType label) const override { - return impl_ ? impl_->has_id(label) : false; + if (ready()) { + std::shared_lock lock(this->pimplGuard_); + return impl_->has_id(label); + } else { + return false; + } } size_t getNumThreads() const override { return threadpool_.size(); } @@ -543,13 +612,23 @@ class SVSIndex : public VecSimIndexAbstract, fl } double getDistanceFrom_Unsafe(labelType label, const void *vector_data) const override { - if (!impl_ || !impl_->has_id(label)) { + if (!ready()) { return std::numeric_limits::quiet_NaN(); - }; + } + { + std::shared_lock lock(this->pimplGuard_); + if (!impl_->has_id(label)) { + return std::numeric_limits::quiet_NaN(); + } + } + auto query_datum = std::span{static_cast(vector_data), this->dim}; - auto dist = impl_->get_distance(label, query_datum); - return toVecSimDistance(dist); + { + std::shared_lock lock(this->pimplGuard_); + auto dist = impl_->get_distance(label, query_datum); + return toVecSimDistance(dist); + } } VecSimQueryReply *topKQuery(const void *queryBlob, size_t k, @@ -559,36 +638,38 @@ class SVSIndex : public VecSimIndexAbstract, fl if (k == 0 || this->indexLabelCount() == 0) { return rep; } + { + std::shared_lock lock(this->pimplGuard_); + // limit result size to index size + k = std::min(k, this->indexLabelCount()); + + auto processed_query_ptr = this->preprocessQuery(queryBlob); + const void *processed_query = processed_query_ptr.get(); + + auto query = svs::data::ConstSimpleDataView{ + static_cast(processed_query), 1, this->dim}; + auto result = svs::QueryResult{query.size(), k}; + auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams, + is_two_level_lvq); + + auto timeoutCtx = queryParams ? queryParams->timeoutCtx : nullptr; + auto cancel = [timeoutCtx]() { return VECSIM_TIMEOUT(timeoutCtx); }; + + impl_->search(result.view(), query, sp, cancel); + if (cancel()) { + rep->code = VecSim_QueryReply_TimedOut; + return rep; + } - // limit result size to index size - k = std::min(k, this->indexLabelCount()); - - auto processed_query_ptr = this->preprocessQuery(queryBlob); - const void *processed_query = processed_query_ptr.get(); - - auto query = svs::data::ConstSimpleDataView{ - static_cast(processed_query), 1, this->dim}; - auto result = svs::QueryResult{query.size(), k}; - auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams, - is_two_level_lvq); - - auto timeoutCtx = queryParams ? queryParams->timeoutCtx : nullptr; - auto cancel = [timeoutCtx]() { return VECSIM_TIMEOUT(timeoutCtx); }; - - impl_->search(result.view(), query, sp, cancel); - if (cancel()) { - rep->code = VecSim_QueryReply_TimedOut; - return rep; - } - - assert(result.n_queries() == 1); + assert(result.n_queries() == 1); - const auto n_neighbors = result.n_neighbors(); - rep->results.reserve(n_neighbors); + const auto n_neighbors = result.n_neighbors(); + rep->results.reserve(n_neighbors); - for (size_t i = 0; i < n_neighbors; i++) { - rep->results.push_back( - VecSimQueryResult{result.index(0, i), toVecSimDistance(result.distance(0, i))}); + for (size_t i = 0; i < n_neighbors; i++) { + rep->results.push_back( + VecSimQueryResult{result.index(0, i), toVecSimDistance(result.distance(0, i))}); + } } // Workaround for VecSim merge_results() that expects results to be sorted // by score, then by id from both indices. @@ -604,58 +685,61 @@ class SVSIndex : public VecSimIndexAbstract, fl if (radius == 0 || this->indexLabelCount() == 0) { return rep; } + { + std::shared_lock lock(this->pimplGuard_); + + auto timeoutCtx = queryParams ? queryParams->timeoutCtx : nullptr; + auto cancel = [timeoutCtx]() { return VECSIM_TIMEOUT(timeoutCtx); }; + + // Prepare query blob for SVS + auto processed_query_ptr = this->preprocessQuery(queryBlob); + const void *processed_query = processed_query_ptr.get(); + std::span query{static_cast(processed_query), + this->dim}; + + // Base search parameters for the SVS iterator schedule. + auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams, + is_two_level_lvq); + // SVS BatchIterator handles the search in batches + // The batch size is set to the index search window size by default + const size_t batch_size = sp.buffer_config_.get_search_window_size(); + + // Create SVS BatchIterator for range search + // Search result is cached in the iterator and can be accessed by the user + auto svs_it = impl_->make_batch_iterator(query); + svs_it.next(batch_size, cancel); + if (cancel()) { + rep->code = VecSim_QueryReply_TimedOut; + return rep; + } - auto timeoutCtx = queryParams ? queryParams->timeoutCtx : nullptr; - auto cancel = [timeoutCtx]() { return VECSIM_TIMEOUT(timeoutCtx); }; - - // Prepare query blob for SVS - auto processed_query_ptr = this->preprocessQuery(queryBlob); - const void *processed_query = processed_query_ptr.get(); - std::span query{static_cast(processed_query), - this->dim}; - - // Base search parameters for the SVS iterator schedule. - auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams, - is_two_level_lvq); - // SVS BatchIterator handles the search in batches - // The batch size is set to the index search window size by default - const size_t batch_size = sp.buffer_config_.get_search_window_size(); - - // Create SVS BatchIterator for range search - // Search result is cached in the iterator and can be accessed by the user - auto svs_it = impl_->make_batch_iterator(query); - svs_it.next(batch_size, cancel); - if (cancel()) { - rep->code = VecSim_QueryReply_TimedOut; - return rep; - } - - // range search using epsilon - const auto epsilon = queryParams && queryParams->svsRuntimeParams.epsilon != 0 - ? queryParams->svsRuntimeParams.epsilon - : this->epsilon; - - const auto range_search_boundaries = radius * (1.0 + std::abs(epsilon)); - bool keep_searching = true; - - // Loop while iterator cache is not empty and search radius + epsilon is not exceeded - while (keep_searching && svs_it.size() > 0) { - // Iterate over the cached search results - for (auto &neighbor : svs_it) { - const auto dist = toVecSimDistance(neighbor.distance()); - if (dist <= radius) { - rep->results.push_back(VecSimQueryResult{neighbor.id(), dist}); - } else if (dist > range_search_boundaries) { - keep_searching = false; + // range search using epsilon + const auto epsilon = queryParams && queryParams->svsRuntimeParams.epsilon != 0 + ? queryParams->svsRuntimeParams.epsilon + : this->epsilon; + + const auto range_search_boundaries = radius * (1.0 + std::abs(epsilon)); + bool keep_searching = true; + + // Loop while iterator cache is not empty and search radius + epsilon is not exceeded + while (keep_searching && svs_it.size() > 0) { + // Iterate over the cached search results + for (auto &neighbor : svs_it) { + const auto dist = toVecSimDistance(neighbor.distance()); + if (dist <= radius) { + rep->results.push_back(VecSimQueryResult{neighbor.id(), dist}); + } else if (dist > range_search_boundaries) { + keep_searching = false; + } } - } - // If search radius + epsilon is not exceeded, request SVS BatchIterator for the next - // batch - if (keep_searching) { - svs_it.next(batch_size, cancel); - if (cancel()) { - rep->code = VecSim_QueryReply_TimedOut; - return rep; + // If search radius + epsilon is not exceeded, request SVS BatchIterator for the next + // batch + if (keep_searching) { + svs_it.next(batch_size, cancel); + if (cancel()) { + rep->code = VecSim_QueryReply_TimedOut; + return rep; + } } } } @@ -713,7 +797,8 @@ class SVSIndex : public VecSimIndexAbstract, fl } void runGC() override { - if (impl_) { + if (ready()) { + std::shared_lock lock(this->pimplGuard_); // There is documentation for consolidate(): // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.consolidate impl_->consolidate(); diff --git a/src/VecSim/algorithms/svs/svs_serializer_impl.h b/src/VecSim/algorithms/svs/svs_serializer_impl.h index 2780d3457..6f6757922 100644 --- a/src/VecSim/algorithms/svs/svs_serializer_impl.h +++ b/src/VecSim/algorithms/svs/svs_serializer_impl.h @@ -112,6 +112,7 @@ void SVSIndex distance_f(), std::move(threadpool_handle), false, logger_); impl_ = std::make_unique(std::move(loaded)); } + setReady(); } template { std::atomic_flag indexGCScheduled = ATOMIC_FLAG_INIT; // Used to prevent running multiple index update jobs in parallel. // Even if update jobs scheduled sequentially, they can be started in parallel. - mutable std::mutex updateJobMutex; + mutable std::shared_mutex updateJobMutex; // The reason of following container just to properly destroy jobs which not executed yet SVSMultiThreadJob::JobsRegistry uncompletedJobs; - std::atomic backendReady{false}; std::atomic backendInitSubmited{false}; + std::unordered_set ids_to_init_; vecsim_stl::unordered_map> labelToInsertJobs; // A mapping to hold invalid jobs, so we can dispose them upon index deletion. @@ -570,7 +570,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto index = static_cast *>(idx); assert(index); // prevent parallel updates - std::lock_guard lock(index->updateJobMutex); + std::lock_guard lock(index->updateJobMutex); // Release the scheduled flag to allow scheduling again index->indexUpdateScheduled.clear(); // Update the SVS index @@ -580,6 +580,8 @@ class TieredSVSIndex : public VecSimTieredIndex { static void executeInsertJobWrapper(AsyncJob *job) { auto *insert_job = static_cast(job); auto *job_index = static_cast *>(insert_job->index); + // prevent parallel execution with index initilizing job + std::shared_lock lock(job_index->updateJobMutex); job_index->executeInsertJob(insert_job); delete job; } @@ -705,7 +707,8 @@ class TieredSVSIndex : public VecSimTieredIndex { } void executeInsertJob(SVSInsertJob *job) { - assert(this->backendReady.load(std::memory_order_acquire)); + auto svs_index = GetSVSIndex(); + assert(svs_index->ready()); // Note that accessing the job fields should occur with flat index guard held (here and later). this->flatIndexGuard.lock_shared(); @@ -721,7 +724,6 @@ class TieredSVSIndex : public VecSimTieredIndex { auto storage_blob = this->frontendIndex->preprocessForStorage(this->frontendIndex->getDataByInternalId(job->id)); this->flatIndexGuard.unlock_shared(); - auto svs_index = GetSVSIndex(); svs_index->addVector(storage_blob.get(), job->label); // Remove the vector and the insert job from the flat buffer. @@ -763,84 +765,79 @@ class TieredSVSIndex : public VecSimTieredIndex { } void updateSVSIndex(size_t availableThreads) { + std::vector ids_to_move; std::vector labels_to_move; std::vector vectors_to_move; - std::vector deleted_labels_during_update; { // lock frontendIndex from modifications - std::shared_lock flat_lock{this->flatIndexGuard}; + std::lock_guard flat_lock{this->flatIndexGuard}; + auto svs_index = GetSVSIndex(); + bool is_initialization = !svs_index->ready(); auto flat_index = this->GetFlatIndex(); const auto frontend_index_size = this->frontendIndex->indexSize(); + const auto init_batch_size = is_initialization ? ids_to_init_.size() : frontend_index_size; const size_t dim = flat_index->getDim(); - labels_to_move.reserve(frontend_index_size); - vectors_to_move.reserve(frontend_index_size * dim); + ids_to_move.reserve(init_batch_size); + labels_to_move.reserve(init_batch_size); + vectors_to_move.reserve(init_batch_size * dim); + + if (is_initialization) { + for (idType id : ids_to_init_) { + ids_to_move.push_back(id); + labels_to_move.push_back(flat_index->getVectorLabel(id)); + auto data = flat_index->getDataByInternalId(id); + vectors_to_move.insert(vectors_to_move.end(), data, data + dim); + } + ids_to_init_.clear(); + } else { + for (idType id = 0; id < frontend_index_size; id++) { + ids_to_move.push_back(id); + labels_to_move.push_back(flat_index->getVectorLabel(id)); + auto data = flat_index->getDataByInternalId(id); + vectors_to_move.insert(vectors_to_move.end(), data, data + dim); + } + } - for (idType i = 0; i < frontend_index_size; i++) { - labels_to_move.push_back(flat_index->getVectorLabel(i)); - auto data = flat_index->getDataByInternalId(i); - vectors_to_move.insert(vectors_to_move.end(), data, data + dim); + std::sort(ids_to_move.begin(), ids_to_move.end()); + int total_deleted = 0; + for (auto it = ids_to_move.rbegin(); it != ids_to_move.rend(); ++it) { + idType id = *it; + auto label = this->frontendIndex->getVectorLabel(id); + // Delete the vector from the frontend index if not in-place updated. + if (label != SKIP_LABEL) { + labelType last_vec_label = + this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); + int deleted = this->frontendIndex->deleteVectorById(label, id); + if (deleted && id != this->frontendIndex->indexSize()) { + // If the vector removal caused a swap with the last id, update the relevant insert job. + this->updateInsertJobInternalId(this->frontendIndex->indexSize(), id, + last_vec_label); + } + total_deleted += deleted; + } } - // reset journal to the current frontend index state - swaps_journal.clear(); - deleted_labels_journal.clear(); + + assert(total_deleted == std::count_if(labels_to_move.begin(), labels_to_move.end(), + [](labelType label) { return label != SKIP_LABEL; }) && + "Deleted vectors count does not match the number of labels to delete"); + } // release frontend index executeTracingCallback("UpdateJob::before_add_to_svs"); { // lock backend index for writing and add vectors there auto svs_index = GetSVSIndex(); + svs_index->setNumThreads(1); assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); if (this->backendIndex->indexSize() == 0) { auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); - svs_index->setImpl(std::move(impl)); } else { svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); } } - - executeTracingCallback("UpdateJob::after_add_to_svs"); - // clean-up frontend index - { // lock frontend index for writing and delete moved vectors - std::lock_guard lock(this->flatIndexGuard); - - // swap deleted labels journal with the local variable to track deleted labels during - // update - std::swap(deleted_labels_during_update, deleted_labels_journal); - - // Apply swaps from journal to labels_to_move to reflect changes made in meanwhile. - applySwapsToLabelsArray(labels_to_move, this->swaps_journal); - - // delete vectors from the frontend index in reverse order - // it increases the chance of avoiding swaps in the frontend index and performance - // improvement - int deleted = 0; - idType id = labels_to_move.size(); - while (id-- > 0) { - auto label = labels_to_move[id]; - // Delete the vector from the frontend index if not in-place updated. - if (label != SKIP_LABEL) { - deleted += this->frontendIndex->deleteVectorById(label, id); - } - } - assert(deleted == std::count_if(labels_to_move.begin(), labels_to_move.end(), - [](labelType label) { return label != SKIP_LABEL; }) && - "Deleted vectors count does not match the number of labels to delete"); - } - // delete vectors from backend index that were deleted from the frontend index during - // the update process. - { - std::sort(deleted_labels_during_update.begin(), deleted_labels_during_update.end()); - auto it = std::unique(deleted_labels_during_update.begin(), - deleted_labels_during_update.end()); - deleted_labels_during_update.erase(it, deleted_labels_during_update.end()); - auto svs_index = GetSVSIndex(); - svs_index->deleteVectors(deleted_labels_during_update.data(), - deleted_labels_during_update.size()); - } - this->backendReady.store(true, std::memory_order_release); } public: @@ -916,7 +913,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto storage_blob = this->frontendIndex->preprocessForStorage(blob); // prevent update job from running in parallel and lock any access to the backend // index - std::lock_guard lock(this->updateJobMutex); + std::lock_guard lock(this->updateJobMutex); // Set available thread count to 1 for single vector write-in-place operation. // This maintains the contract that single vector operations use exactly one thread. // TODO: Replace this setNumThreads(1) call with an assertion once we establish @@ -931,16 +928,11 @@ class TieredSVSIndex : public VecSimTieredIndex { if (!this->backendInitSubmited.load(std::memory_order_acquire)) { // Add vector to the frontend index. std::lock_guard lock(this->flatIndexGuard); - const auto ft_ret = this->frontendIndex->addVector(blob, label); - - if (ft_ret == 0) { // Vector was overriden - add 'skiping' swap to the journal. - assert(!this->backendIndex->isMultiValue() && - "addVector() may return 0 for single value indices only"); - for (auto id : this->frontendIndex->getElementIds(label)) { - this->swaps_journal.emplace_back(SKIP_LABEL, id, id); - } - deleted_labels_journal.push_back(label); + if (this->frontendIndex->isLabelExists(label)) { + deleteAndUpdateInitIds(label); } + ids_to_init_.insert(this->frontendIndex->indexSize()); + const auto ft_ret = this->frontendIndex->addVector(blob, label); ret = std::max(ret + ft_ret, 0); if (this->frontendIndex->indexSize() >= this->trainingTriggerThreshold) { @@ -950,13 +942,8 @@ class TieredSVSIndex : public VecSimTieredIndex { return ret; } - // spin-lock, while backend is initilizing - while (!this->backendReady.load(std::memory_order_acquire)) { - __builtin_ia32_pause(); - } - this->flatIndexGuard.lock_shared(); - if (this->frontendIndex->indexSize() >= flat_buffer_bound) { + if (svs_index->ready() && (this->frontendIndex->indexSize() >= flat_buffer_bound)) { this->flatIndexGuard.unlock_shared(); auto storage_blob = this->frontendIndex->preprocessForStorage(blob); @@ -966,7 +953,18 @@ class TieredSVSIndex : public VecSimTieredIndex { this->flatIndexGuard.lock(); idType new_flat_id = this->frontendIndex->indexSize(); if (this->frontendIndex->isLabelExists(label) && !this->frontendIndex->isMultiValue()) { + if (this->labelToInsertJobs.count(label) == 0) { + // No pending insert job. + // Just replace vector in the frontend + // If this label already exists, this will do overwrite. + ret = this->frontendIndex->addVector(blob, label); + this->flatIndexGuard.unlock(); + return ret; + } + // Overwrite the vector and invalidate its only pending job (since we are not in MULTI). + // Label exists, but job doesn't. It means the label is used for initialization. + // Just create new job, this job will remove duplicate from the backend. auto *old_job = this->labelToInsertJobs.at(label).at(0); old_job->id = this->setAndSaveInvalidJob(old_job); this->labelToInsertJobs.erase(label); @@ -998,19 +996,26 @@ class TieredSVSIndex : public VecSimTieredIndex { } this->flatIndexGuard.unlock(); + // Here, a worker might ingest the previous vector that was stored under "label" + // (in case of override in non-MULTI index) - so if it's there, we remove it + // we submit the insert job. + if (!this->backendIndex->isMultiValue()) { + if (svs_index->ready()) { + // If we removed the previous vector from both svs and flat in the overwrite process, + // we still return 0 (not -1). + ret = std::max(ret - svs_index->deleteVector(label), 0); + } + } + // Insert job to the queue and signal the workers' updater. this->submitSingleJob(new_insert_job); return ret; } } - int deleteAndRecordSwaps_Unsafe(labelType label) { + void deleteAndUpdateInitIds(labelType label) { auto deleting_ids = this->frontendIndex->getElementIds(label); - // assert if all elements of deleting_ids are unique - assert(std::set(deleting_ids.begin(), deleting_ids.end()).size() == deleting_ids.size() && - "deleting_ids should contain unique ids"); - // Sort deleting_ids by id descending order std::sort(deleting_ids.begin(), deleting_ids.end(), [](const auto &a, const auto &b) { return a > b; }); @@ -1025,43 +1030,94 @@ class TieredSVSIndex : public VecSimTieredIndex { }) && "updated_ids should be a subset of deleting_ids"); - // Record swaps in the journal. - for (auto id : deleting_ids) { - auto it = updated_ids.find(id); - if (it != updated_ids.end()) { - assert(id == it->first && "id in updated_ids should match the id in deleting_ids"); - auto newId = id; - auto oldId = it->second.first; - auto oldLabel = it->second.second; - this->swaps_journal.emplace_back(oldLabel, oldId, newId); - } else { - // No swap, just delete is marked by oldId == newId == deleted id - this->swaps_journal.emplace_back(SKIP_LABEL, id, id); + std::vector new_ids_to_init; + for (auto &it : updated_ids) { + idType prev_id = it.second.first; + idType new_id = it.first; + + if (ids_to_init_.count(prev_id) > 0) { + ids_to_init_.erase(prev_id); + new_ids_to_init.push_back(new_id); + } + labelType updated_vec_label = it.second.second; + this->updateInsertJobInternalId(prev_id, new_id, updated_vec_label); + } + + for (idType new_id : new_ids_to_init) { + ids_to_init_.insert(new_id); + } + } + + int deleteAndRecordSwaps_Unsafe(labelType label) { + auto deleting_ids = this->frontendIndex->getElementIds(label); + if (deleting_ids.size() == 0) return 0; + + // assert if all elements of deleting_ids are unique + assert(std::set(deleting_ids.begin(), deleting_ids.end()).size() == deleting_ids.size() && + "deleting_ids should contain unique ids"); + + // If id is deleted, don't use it for initialization + for (idType id : deleting_ids) { + ids_to_init_.erase(id); + } + + if (this->labelToInsertJobs.count(label) > 0) { + // Invalidate the pending insert job(s) into SVS associated with this label + auto &insert_jobs = this->labelToInsertJobs.at(label); + for (auto *job : insert_jobs) { + job->id = this->setAndSaveInvalidJob(job); } + // Remove the pending insert job(s) from the labelToInsertJobs mapping. + this->labelToInsertJobs.erase(label); } - deleted_labels_journal.push_back(label); + deleteAndUpdateInitIds(label); return deleting_ids.size(); } int deleteVector(labelType label) override { int ret = 0; - // Backend index deletions to be synchronized with the frontend index, - // elsewhere there is the risk of labels duplication in both indices which can lead to wrong - // results of topK queries. In such case we should behave as if InPlace mode is always set. - bool label_exists = [&]() { - std::shared_lock lock(this->flatIndexGuard); - return this->frontendIndex->isLabelExists(label); - }(); - - if (label_exists) { - std::lock_guard lock(this->flatIndexGuard); - ret = this->deleteAndRecordSwaps_Unsafe(label); + + this->flatIndexGuard.lock_shared(); + if (this->frontendIndex->isLabelExists(label)) { + this->flatIndexGuard.unlock_shared(); + this->flatIndexGuard.lock(); + // Check again if the label exists, as it may have been removed while we released the lock. + if (this->frontendIndex->isLabelExists(label)) { + ret = this->deleteAndRecordSwaps_Unsafe(label); + // if (this->labelToInsertJobs.count(label) > 0) { + // // Invalidate the pending insert job(s) into SVS associated with this label + // auto &insert_jobs = this->labelToInsertJobs.at(label); + // for (auto *job : insert_jobs) { + // job->id = this->setAndSaveInvalidJob(job); + // } + // ret += insert_jobs.size(); + // // Remove the pending insert job(s) from the labelToInsertJobs mapping. + // this->labelToInsertJobs.erase(label); + // // Go over the every id that corresponds the label and remove it from the flat buffer. + // // Every delete may cause a swap of the deleted id with the last id, and we return a + // // mapping from id to the original id that resides in this id after the deletion(s) (see + // // an example in this function implementation in MULTI index). + // auto updated_ids = this->frontendIndex->deleteVectorAndGetUpdatedIds(label); + // for (auto &it : updated_ids) { + // idType prev_id = it.second.first; + // labelType updated_vec_label = it.second.second; + // this->updateInsertJobInternalId(prev_id, it.first, updated_vec_label); + // } + // } else { + // // label was deleted during initilization stage + // ret = this->deleteAndRecordSwaps_Unsafe(label); + // } + } + this->flatIndexGuard.unlock(); + } else { + this->flatIndexGuard.unlock_shared(); } ret += this->backendIndex->deleteVector(label); return ret; } + size_t getNumMarkedDeleted() const override { return this->GetSVSIndex()->getNumMarkedDeleted(); } @@ -1110,7 +1166,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // the entire training duration (which can take 40-85s on slow machines). // If the mutex is held, training is actively running, so we report // indexUpdateScheduled = true (BACKGROUND_INDEXING = 1). - std::unique_lock lock(this->updateJobMutex, std::try_to_lock); + std::unique_lock lock(this->updateJobMutex, std::try_to_lock); if (lock.owns_lock()) { svsTieredInfo.indexUpdateScheduled = this->indexUpdateScheduled.test() == VecSimBool_TRUE; diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 3d9c0bd0d..63f9c6a23 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -613,6 +613,12 @@ TYPED_TEST(SVSTieredIndexTest, background_indexing_check) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); } + // The backgroundIndexing flag tracks the batch update job only; individual async + // SVS_INSERT_VECTOR_JOBs may still be in flight, each transiently holding its vector in + // both the flat and backend indices. Drain the queue so the total size is stable before + // asserting the exact count. + mock_thread_pool.thread_pool_join(); + ASSERT_GT(tiered_index->GetBackendIndex()->indexSize(), training_th + second_batch / update_th); ASSERT_LT(tiered_index->GetFlatIndex()->indexSize(), update_th); ASSERT_EQ(tiered_index->indexSize(), second_batch + training_th); @@ -1125,7 +1131,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, markedDeleted) { ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 0); // Move vectors to the backend - mock_thread_pool.thread_iteration(); + while (mock_thread_pool.jobQ.size() > 0) mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), n); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->indexSize(), n); @@ -1222,13 +1228,20 @@ TYPED_TEST(SVSTieredIndexTestBasic, deleteVectorMulti) { ASSERT_EQ(tiered_index->indexSize(), 2); ASSERT_EQ(tiered_index->deleteVector(vec_label), 2); ASSERT_EQ(tiered_index->indexLabelCount(), 0); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 2); + mock_thread_pool.thread_iteration(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); mock_thread_pool.thread_iteration(); ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); // Test deleting a label for which both of its vector's is in SVS index. GenerateAndAddVector(tiered_index, dim, vec_label, vec_label); GenerateAndAddVector(tiered_index, dim, vec_label, other_vec_val); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 2); mock_thread_pool.thread_iteration(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); + mock_thread_pool.thread_iteration(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); ASSERT_EQ(tiered_index->indexLabelCount(), 1); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 2); From 139161f63f0ec9feb924ee4c59669f2138c8ce26 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 8 Jul 2026 06:47:45 -0700 Subject: [PATCH 10/99] refactoring --- src/VecSim/algorithms/svs/svs.h | 4 +- src/VecSim/algorithms/svs/svs_tiered.h | 162 ++++++++++--------------- tests/unit/test_svs_fp16.cpp | 2 +- tests/unit/test_svs_tiered.cpp | 4 +- 4 files changed, 67 insertions(+), 105 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 7dcdc1d93..ee71b7470 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -340,7 +340,9 @@ class SVSIndex : public VecSimIndexAbstract, fl deleted_num = impl_->delete_entries(std::span{labels, n}); } - this->markIndexUpdate(deleted_num); + if (deleted_num > 0) + this->markIndexUpdate(deleted_num); + return deleted_num; } diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 8869b2191..8608de84c 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -232,7 +232,7 @@ class TieredSVSIndex : public VecSimTieredIndex { using swap_record = std::tuple; constexpr static size_t SKIP_LABEL = std::numeric_limits::max(); std::vector swaps_journal; - // deleted_labels_journal is used by updateSVSIndex() to track vectors that were deleted from + // deleted_labels_journal is used by initSVSIndex() to track vectors that were deleted from // Flat index during SVS index updating. The journal contains the deleted labels. These labels // are used to delete the same vectors from the SVS index at the end of the update. std::vector deleted_labels_journal; @@ -553,19 +553,19 @@ class TieredSVSIndex : public VecSimTieredIndex { private: /** - * @brief Updates the SVS index in a thread-safe manner. + * @brief Init the SVS index in a thread-safe manner. * * This static wrapper function performs the following actions: * - Acquires a lock on the index's updateJobMutex to prevent concurrent updates. * - Clears the indexUpdateScheduled flag to allow future scheduling. * - Configures the number of threads for the underlying SVS index update operation. - * - Calls the updateSVSIndex method to perform the actual index update. + * - Calls the initSVSIndex method to perform the actual index update. * * @param idx Pointer to the VecSimIndex to be updated. * @param availableThreads The number of threads available for the update operation. Current * thread us used as well, so the minimal value is 1. */ - static void updateSVSIndexWrapper(VecSimIndex *idx, size_t availableThreads) { + static void initSVSIndexWrapper(VecSimIndex *idx, size_t availableThreads) { assert(availableThreads > 0); auto index = static_cast *>(idx); assert(index); @@ -574,7 +574,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // Release the scheduled flag to allow scheduling again index->indexUpdateScheduled.clear(); // Update the SVS index - index->updateSVSIndex(availableThreads); + index->initSVSIndex(availableThreads); } static void executeInsertJobWrapper(AsyncJob *job) { @@ -640,7 +640,7 @@ class TieredSVSIndex : public VecSimTieredIndex { } } - void scheduleSVSIndexUpdate() { + void scheduleSVSIndexInit() { // do not schedule if scheduled already if (indexUpdateScheduled.test_and_set()) { return; @@ -648,7 +648,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto total_threads = this->GetSVSIndex()->getThreadPoolCapacity(); auto jobs = SVSMultiThreadJob::createJobs( - this->allocator, SVS_BATCH_UPDATE_JOB, updateSVSIndexWrapper, this, total_threads, + this->allocator, SVS_BATCH_UPDATE_JOB, initSVSIndexWrapper, this, total_threads, std::chrono::microseconds(updateJobWaitTime), &uncompletedJobs); this->submitJobs(jobs); } @@ -764,39 +764,38 @@ class TieredSVSIndex : public VecSimTieredIndex { this->flatIndexGuard.unlock(); } - void updateSVSIndex(size_t availableThreads) { + void initSVSIndex(size_t availableThreads) { std::vector ids_to_move; std::vector labels_to_move; std::vector vectors_to_move; { // lock frontendIndex from modifications + // The whole initialization is done under flatIndexGuard std::lock_guard flat_lock{this->flatIndexGuard}; - auto svs_index = GetSVSIndex(); - bool is_initialization = !svs_index->ready(); auto flat_index = this->GetFlatIndex(); - const auto frontend_index_size = this->frontendIndex->indexSize(); - const auto init_batch_size = is_initialization ? ids_to_init_.size() : frontend_index_size; + const auto init_batch_size = ids_to_init_.size(); const size_t dim = flat_index->getDim(); ids_to_move.reserve(init_batch_size); labels_to_move.reserve(init_batch_size); vectors_to_move.reserve(init_batch_size * dim); - if (is_initialization) { - for (idType id : ids_to_init_) { - ids_to_move.push_back(id); - labels_to_move.push_back(flat_index->getVectorLabel(id)); - auto data = flat_index->getDataByInternalId(id); - vectors_to_move.insert(vectors_to_move.end(), data, data + dim); - } - ids_to_init_.clear(); - } else { - for (idType id = 0; id < frontend_index_size; id++) { - ids_to_move.push_back(id); - labels_to_move.push_back(flat_index->getVectorLabel(id)); - auto data = flat_index->getDataByInternalId(id); - vectors_to_move.insert(vectors_to_move.end(), data, data + dim); - } + for (idType id : ids_to_init_) { + ids_to_move.push_back(id); + labels_to_move.push_back(flat_index->getVectorLabel(id)); + auto data = flat_index->getDataByInternalId(id); + vectors_to_move.insert(vectors_to_move.end(), data, data + dim); + } + ids_to_init_.clear(); + + executeTracingCallback("UpdateJob::before_add_to_svs"); + { + auto svs_index = GetSVSIndex(); + svs_index->setNumThreads(1); + assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); + auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), + labels_to_move.size()); + svs_index->setImpl(std::move(impl)); } std::sort(ids_to_move.begin(), ids_to_move.end()); @@ -821,23 +820,7 @@ class TieredSVSIndex : public VecSimTieredIndex { assert(total_deleted == std::count_if(labels_to_move.begin(), labels_to_move.end(), [](labelType label) { return label != SKIP_LABEL; }) && "Deleted vectors count does not match the number of labels to delete"); - } // release frontend index - - executeTracingCallback("UpdateJob::before_add_to_svs"); - { // lock backend index for writing and add vectors there - auto svs_index = GetSVSIndex(); - svs_index->setNumThreads(1); - assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); - if (this->backendIndex->indexSize() == 0) { - auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), - labels_to_move.size()); - svs_index->setImpl(std::move(impl)); - } else { - svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), - labels_to_move.size()); - } - } } public: @@ -887,25 +870,29 @@ class TieredSVSIndex : public VecSimTieredIndex { // In-Place mode - add vector syncronously to the backend index. if (this->getWriteMode() == VecSim_WriteInPlace) { // It is ok to lock everything at once for in-place mode, - // but we will have to unlock averything before calling updateSVSIndexWrapper() + // but we will have to unlock averything before calling initSVSIndexWrapper() // so make the minimal needed lock here. // Backend index initialization data have to be buffered for proper // compression/training. if (this->backendIndex->indexSize() == 0) { // If backend index size is 0, first collect vectors in frontend index // lock in scope to ensure that these will be released before - // updateSVSIndexWrapper() is called. + // initSVSIndexWrapper() is called. { std::lock_guard lock(this->flatIndexGuard); + if (this->frontendIndex->isLabelExists(label)) { + deleteAndUpdateInitIds(label); + } + ids_to_init_.insert(this->frontendIndex->indexSize()); ret = this->frontendIndex->addVector(blob, label); // If frontend size exceeds the update job threshold, ... frontend_index_size = this->frontendIndex->indexSize(); } // ... move vectors to the backend index. if (frontend_index_size >= this->trainingTriggerThreshold) { - // updateSVSIndexWrapper() accures it's own locks + // initSVSIndexWrapper() accures it's own locks // initialize the SVS index synchonously using current thread only - updateSVSIndexWrapper(this, 1); + initSVSIndexWrapper(this, 1); } return ret; } else { @@ -937,7 +924,7 @@ class TieredSVSIndex : public VecSimTieredIndex { if (this->frontendIndex->indexSize() >= this->trainingTriggerThreshold) { this->backendInitSubmited.store(true, std::memory_order_release); - scheduleSVSIndexUpdate(); + scheduleSVSIndexInit(); } return ret; } @@ -1048,33 +1035,6 @@ class TieredSVSIndex : public VecSimTieredIndex { } } - int deleteAndRecordSwaps_Unsafe(labelType label) { - auto deleting_ids = this->frontendIndex->getElementIds(label); - if (deleting_ids.size() == 0) return 0; - - // assert if all elements of deleting_ids are unique - assert(std::set(deleting_ids.begin(), deleting_ids.end()).size() == deleting_ids.size() && - "deleting_ids should contain unique ids"); - - // If id is deleted, don't use it for initialization - for (idType id : deleting_ids) { - ids_to_init_.erase(id); - } - - if (this->labelToInsertJobs.count(label) > 0) { - // Invalidate the pending insert job(s) into SVS associated with this label - auto &insert_jobs = this->labelToInsertJobs.at(label); - for (auto *job : insert_jobs) { - job->id = this->setAndSaveInvalidJob(job); - } - // Remove the pending insert job(s) from the labelToInsertJobs mapping. - this->labelToInsertJobs.erase(label); - } - - deleteAndUpdateInitIds(label); - return deleting_ids.size(); - } - int deleteVector(labelType label) override { int ret = 0; @@ -1084,30 +1044,30 @@ class TieredSVSIndex : public VecSimTieredIndex { this->flatIndexGuard.lock(); // Check again if the label exists, as it may have been removed while we released the lock. if (this->frontendIndex->isLabelExists(label)) { - ret = this->deleteAndRecordSwaps_Unsafe(label); - // if (this->labelToInsertJobs.count(label) > 0) { - // // Invalidate the pending insert job(s) into SVS associated with this label - // auto &insert_jobs = this->labelToInsertJobs.at(label); - // for (auto *job : insert_jobs) { - // job->id = this->setAndSaveInvalidJob(job); - // } - // ret += insert_jobs.size(); - // // Remove the pending insert job(s) from the labelToInsertJobs mapping. - // this->labelToInsertJobs.erase(label); - // // Go over the every id that corresponds the label and remove it from the flat buffer. - // // Every delete may cause a swap of the deleted id with the last id, and we return a - // // mapping from id to the original id that resides in this id after the deletion(s) (see - // // an example in this function implementation in MULTI index). - // auto updated_ids = this->frontendIndex->deleteVectorAndGetUpdatedIds(label); - // for (auto &it : updated_ids) { - // idType prev_id = it.second.first; - // labelType updated_vec_label = it.second.second; - // this->updateInsertJobInternalId(prev_id, it.first, updated_vec_label); - // } - // } else { - // // label was deleted during initilization stage - // ret = this->deleteAndRecordSwaps_Unsafe(label); - // } + auto deleting_ids = this->frontendIndex->getElementIds(label); + if (deleting_ids.size() == 0) return 0; + + // assert if all elements of deleting_ids are unique + assert(std::set(deleting_ids.begin(), deleting_ids.end()).size() == deleting_ids.size() && + "deleting_ids should contain unique ids"); + + // If id is deleted, don't use it for initialization + for (idType id : deleting_ids) { + ids_to_init_.erase(id); + } + + if (this->labelToInsertJobs.count(label) > 0) { + // Invalidate the pending insert job(s) into SVS associated with this label + auto &insert_jobs = this->labelToInsertJobs.at(label); + for (auto *job : insert_jobs) { + job->id = this->setAndSaveInvalidJob(job); + } + // Remove the pending insert job(s) from the labelToInsertJobs mapping. + this->labelToInsertJobs.erase(label); + } + + deleteAndUpdateInitIds(label); + return deleting_ids.size(); } this->flatIndexGuard.unlock(); } else { @@ -1162,7 +1122,7 @@ class TieredSVSIndex : public VecSimTieredIndex { }; { // Use try_lock to avoid blocking the main thread during long-running - // training operations. updateSVSIndexWrapper holds updateJobMutex for + // training operations. initSVSIndexWrapper holds updateJobMutex for // the entire training duration (which can take 40-85s on slow machines). // If the mutex is held, training is actively running, so we report // indexUpdateScheduled = true (BACKGROUND_INDEXING = 1). @@ -1171,7 +1131,7 @@ class TieredSVSIndex : public VecSimTieredIndex { svsTieredInfo.indexUpdateScheduled = this->indexUpdateScheduled.test() == VecSimBool_TRUE; } else { - // Mutex is held by updateSVSIndexWrapper — training is in progress. + // Mutex is held by initSVSIndexWrapper — training is in progress. svsTieredInfo.indexUpdateScheduled = true; } } diff --git a/tests/unit/test_svs_fp16.cpp b/tests/unit/test_svs_fp16.cpp index 59c11c0a1..ded23bd72 100644 --- a/tests/unit/test_svs_fp16.cpp +++ b/tests/unit/test_svs_fp16.cpp @@ -2337,7 +2337,7 @@ class FP16SVSTieredIndexTest : public FP16SVSTest { } } // Submit the index update job. - tiered_index->scheduleSVSIndexUpdate(); + tiered_index->scheduleSVSIndexInit(); ASSERT_EQ(mock_thread_pool.jobQ.size(), mock_thread_pool.thread_pool_size); // Execute the job from the queue and validate that the index was updated properly. diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 63f9c6a23..2980d0e38 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -489,7 +489,7 @@ TYPED_TEST(SVSTieredIndexTest, CreateIndexInstance) { ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 0); // Submit the index update job. - tiered_index->scheduleSVSIndexUpdate(); + tiered_index->scheduleSVSIndexInit(); ASSERT_EQ(mock_thread_pool.jobQ.size(), mock_thread_pool.thread_pool_size); // Execute the job from the queue and validate that the index was updated properly. @@ -1635,7 +1635,7 @@ TYPED_TEST(SVSTieredIndexTest, parallelInsertAdHoc) { tiered_index->submitSingleJob(search_job); } - tiered_index->scheduleSVSIndexUpdate(); + tiered_index->scheduleSVSIndexInit(); mock_thread_pool.thread_pool_join(); EXPECT_EQ(successful_searches, n); From f8efbbe750e891aba5f66b59fe21818f33e4165a Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 15 Jul 2026 09:27:43 -0700 Subject: [PATCH 11/99] fixes for GC --- src/VecSim/algorithms/svs/svs_tiered.h | 46 ++++++++++++++++++-------- src/VecSim/vec_sim_common.h | 1 + 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 8608de84c..2e797e188 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -25,6 +25,15 @@ struct SVSInsertJob : public AsyncJob { : AsyncJob(allocator, SVS_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} }; +/** + * Definition of a job that launches GC on SVS Index. + */ +struct SVSGCJob : public AsyncJob { + SVSGCJob(std::shared_ptr allocator, + JobCallback insertCb, VecSimIndex *index_) + : AsyncJob(allocator, SVS_GC2_JOB, insertCb, index_) {} +}; + /** * @class SVSMultiThreadJob @@ -601,13 +610,17 @@ class TieredSVSIndex : public VecSimTieredIndex { * @note no need to implement extra non-static method, as GC logic is simple enough to be done * here. */ - static void SVSIndexGCWrapper(VecSimIndex *idx, size_t availableThreads) { - assert(availableThreads > 0); - auto index = static_cast *>(idx); - assert(index); + // static void SVSIndexGCWrapper(VecSimIndex *idx) { + static void SVSIndexGCWrapper(AsyncJob *job) { + // assert(availableThreads > 0); + // auto index = static_cast *>(idx); + // assert(index); - // Release the scheduled flag to allow scheduling again - index->indexGCScheduled.clear(); + auto gc_job = static_cast(job); + auto index = static_cast *>(gc_job->index); + + // // Release the scheduled flag to allow scheduling again + // index->indexGCScheduled.clear(); // Do SVS index GC index->backendIndex->log(VecSimCommonStrings::LOG_VERBOSE_STRING, @@ -617,9 +630,13 @@ class TieredSVSIndex : public VecSimTieredIndex { // No need to run GC on an empty index. return; } - svs_index->setNumThreads(std::min(availableThreads, index->backendIndex->indexSize())); + // svs_index->setNumThreads(std::min(availableThreads, index->backendIndex->indexSize())); + svs_index->setNumThreads(1); // VecSimIndexAbstract::runGC() is protected static_cast(index->backendIndex)->runGC(); + + // Release the scheduled flag to allow scheduling again + index->indexGCScheduled.clear(); } #ifdef BUILD_TESTS @@ -659,11 +676,11 @@ class TieredSVSIndex : public VecSimTieredIndex { return; } - auto total_threads = this->GetSVSIndex()->getThreadPoolCapacity(); - auto jobs = SVSMultiThreadJob::createJobs( - this->allocator, SVS_GC_JOB, SVSIndexGCWrapper, this, total_threads, - std::chrono::microseconds(updateJobWaitTime), &uncompletedJobs); - this->submitJobs(jobs); + AsyncJob *new_GC_job = new (this->allocator) + SVSGCJob(this->allocator, SVSIndexGCWrapper, this); + + // Insert job to the queue. + this->submitSingleJob(new_GC_job); } private: @@ -724,6 +741,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto storage_blob = this->frontendIndex->preprocessForStorage(this->frontendIndex->getDataByInternalId(job->id)); this->flatIndexGuard.unlock_shared(); + svs_index->setNumThreads(1); svs_index->addVector(storage_blob.get(), job->label); // Remove the vector and the insert job from the flat buffer. @@ -933,6 +951,7 @@ class TieredSVSIndex : public VecSimTieredIndex { if (svs_index->ready() && (this->frontendIndex->indexSize() >= flat_buffer_bound)) { this->flatIndexGuard.unlock_shared(); auto storage_blob = this->frontendIndex->preprocessForStorage(blob); + svs_index->setNumThreads(1); return ret = svs_index->addVector(storage_blob.get(), label); } else { @@ -1041,7 +1060,7 @@ class TieredSVSIndex : public VecSimTieredIndex { this->flatIndexGuard.lock_shared(); if (this->frontendIndex->isLabelExists(label)) { this->flatIndexGuard.unlock_shared(); - this->flatIndexGuard.lock(); + std::lock_guard flat_lock{this->flatIndexGuard}; // Check again if the label exists, as it may have been removed while we released the lock. if (this->frontendIndex->isLabelExists(label)) { auto deleting_ids = this->frontendIndex->getElementIds(label); @@ -1069,7 +1088,6 @@ class TieredSVSIndex : public VecSimTieredIndex { deleteAndUpdateInitIds(label); return deleting_ids.size(); } - this->flatIndexGuard.unlock(); } else { this->flatIndexGuard.unlock_shared(); } diff --git a/src/VecSim/vec_sim_common.h b/src/VecSim/vec_sim_common.h index 03e7870f3..d800e7cdc 100644 --- a/src/VecSim/vec_sim_common.h +++ b/src/VecSim/vec_sim_common.h @@ -275,6 +275,7 @@ typedef enum { SVS_BATCH_UPDATE_JOB, SVS_INSERT_VECTOR_JOB, SVS_GC_JOB, + SVS_GC2_JOB, INVALID_JOB // to indicate that finding a JobType >= INVALID_JOB is an error } JobType; From cfca2f557046b79fb8ade2c0dfac408df29be615 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 20 Jul 2026 09:35:50 -0700 Subject: [PATCH 12/99] copy blob before flat unlock --- src/VecSim/algorithms/svs/svs_tiered.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 2e797e188..4ebca3e09 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -738,11 +738,16 @@ class TieredSVSIndex : public VecSimTieredIndex { this->invalidJobsLookupGuard.unlock(); return; } - auto storage_blob = this->frontendIndex->preprocessForStorage(this->frontendIndex->getDataByInternalId(job->id)); + + // Copy the vector blob out of the flat buffer while holding flatIndexGuard, so we + // can release the flat lock before indexing into the SVS backend + size_t data_size = this->frontendIndex->getStoredDataSize(); + auto blob_copy = this->getAllocator()->allocate_unique(data_size); + memcpy(blob_copy.get(), this->frontendIndex->getDataByInternalId(job->id), data_size); this->flatIndexGuard.unlock_shared(); svs_index->setNumThreads(1); - svs_index->addVector(storage_blob.get(), job->label); + svs_index->addVector(blob_copy.get(), job->label); // Remove the vector and the insert job from the flat buffer. this->flatIndexGuard.lock(); @@ -812,7 +817,7 @@ class TieredSVSIndex : public VecSimTieredIndex { svs_index->setNumThreads(1); assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), - labels_to_move.size()); + labels_to_move.size()); svs_index->setImpl(std::move(impl)); } @@ -1315,3 +1320,4 @@ class TieredSVSIndex : public VecSimTieredIndex { this->flatIndexGuard.unlock_shared(); } }; + \ No newline at end of file From dcbdf54b155a32f0c52bb9b215dfbd7dd9df9fbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mihai=20Capot=C4=83?= Date: Thu, 23 Jul 2026 22:09:56 -0700 Subject: [PATCH 13/99] Remove null characters from end of file --- src/VecSim/algorithms/svs/svs_tiered.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 4ebca3e09..e3eb441b8 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -1320,4 +1320,3 @@ class TieredSVSIndex : public VecSimTieredIndex { this->flatIndexGuard.unlock_shared(); } }; - \ No newline at end of file From aeae737b61ddcbd20f5f8650b6e191db129ed78e Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Tue, 28 Jul 2026 02:28:19 -0700 Subject: [PATCH 14/99] fixes for gc --- src/VecSim/algorithms/svs/svs.h | 60 ++++++++++--- src/VecSim/algorithms/svs/svs_tiered.h | 109 +++++++++++++++++------ src/VecSim/vec_sim_common.h | 3 +- tests/unit/test_svs_fp16.cpp | 8 ++ tests/unit/test_svs_tiered.cpp | 116 ++++++++++++++----------- 5 files changed, 204 insertions(+), 92 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index ee71b7470..e410529f7 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -41,6 +41,7 @@ struct SVSIndexBase virtual int addVectors(const void *vectors_data, const labelType *labels, size_t n) = 0; virtual int deleteVector(labelType label) = 0; virtual int deleteVectors(const labelType *labels, size_t n) = 0; + virtual void consolidate(const std::vector& labels) = 0; virtual bool isLabelExists(labelType label) const = 0; virtual size_t indexStorageSize() const = 0; virtual size_t getNumThreads() const = 0; @@ -299,21 +300,36 @@ class SVSIndex : public VecSimIndexAbstract, fl } } + this->pimplGuard_.lock_shared(); if (!ready()) { + this->pimplGuard_.unlock_shared(); // SVS index instance cannot be empty, so we have to construct it at first rows std::lock_guard lock(this->pimplGuard_); - impl_ = initImpl(points, ids); - assert(this->impl_ != nullptr); - setReady(); + if (!ready()) { + impl_ = initImpl(points, ids); + assert(this->impl_ != nullptr); + setReady(); + } else { + impl_->add_points(points, ids); + } } else { // Add new points to existing SVS index - std::shared_lock lock(this->pimplGuard_); impl_->add_points(points, ids); + this->pimplGuard_.unlock_shared(); } return n - deleted_num; } + void consolidate(const std::vector& labels) { + std::shared_lock lock(this->pimplGuard_); + if (!ready()) + return; + + size_t n_consolidated = impl_->consolidate(labels); + num_marked_deleted.fetch_sub(n_consolidated, std::memory_order_relaxed); + } + int deleteVectorImpl(const labelType label) { if (indexLabelCount() == 0) { return 0; @@ -354,12 +370,14 @@ class SVSIndex : public VecSimIndexAbstract, fl // SVS index instance should not be empty if (indexLabelCount() == 0) { std::lock_guard lock(this->pimplGuard_); - { - setUnready(); - this->impl_.reset(); + if (indexLabelCountUnsafe() == 0) { + { + setUnready(); + this->impl_.reset(); + } + num_marked_deleted.store(0, std::memory_order_relaxed); + return; } - num_marked_deleted.store(0, std::memory_order_relaxed); - return; } num_marked_deleted.fetch_add(n, std::memory_order_relaxed); @@ -404,8 +422,8 @@ class SVSIndex : public VecSimIndexAbstract, fl size_t indexSize() const override { return indexStorageSize(); } size_t indexStorageSize() const override { + std::shared_lock lock(this->pimplGuard_); if (ready()) { - std::shared_lock lock(this->pimplGuard_); return impl_->view_data().size(); } else { return 0; @@ -413,14 +431,30 @@ class SVSIndex : public VecSimIndexAbstract, fl } size_t indexCapacity() const override { + std::shared_lock lock(this->pimplGuard_); if (ready()) { - std::shared_lock lock(this->pimplGuard_); return storage_traits_t::storage_capacity(impl_->view_data()); } else { return 0; } } + size_t indexLabelCountUnsafe() const { + if constexpr (isMulti) { + if (ready()) { + return impl_->labelcount(); + } else { + return 0; + } + } else { + if (ready()) { + return impl_->size(); + } else { + return 0; + } + } + } + size_t indexLabelCount() const override { if constexpr (isMulti) { if (ready()) { @@ -641,7 +675,6 @@ class SVSIndex : public VecSimIndexAbstract, fl return rep; } { - std::shared_lock lock(this->pimplGuard_); // limit result size to index size k = std::min(k, this->indexLabelCount()); @@ -651,6 +684,7 @@ class SVSIndex : public VecSimIndexAbstract, fl auto query = svs::data::ConstSimpleDataView{ static_cast(processed_query), 1, this->dim}; auto result = svs::QueryResult{query.size(), k}; + std::shared_lock lock(this->pimplGuard_); auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams, is_two_level_lvq); @@ -803,7 +837,7 @@ class SVSIndex : public VecSimIndexAbstract, fl std::shared_lock lock(this->pimplGuard_); // There is documentation for consolidate(): // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.consolidate - impl_->consolidate(); + // impl_->consolidate(); // There is documentation for compact(): // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.compact impl_->compact(); diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 4ebca3e09..c94d3b711 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -25,13 +25,24 @@ struct SVSInsertJob : public AsyncJob { : AsyncJob(allocator, SVS_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} }; +/** + * Definition of a job that launches partial consolidation on SVS Index. + */ +struct SVSConsolidateJob : public AsyncJob { + std::vector labels; + + SVSConsolidateJob(std::shared_ptr allocator, const std::vector& labels_, + JobCallback insertCb, VecSimIndex *index_) + : AsyncJob(allocator, SVS_CONSOLIDATE_JOB, insertCb, index_), labels(labels_) {} +}; + /** * Definition of a job that launches GC on SVS Index. */ struct SVSGCJob : public AsyncJob { SVSGCJob(std::shared_ptr allocator, JobCallback insertCb, VecSimIndex *index_) - : AsyncJob(allocator, SVS_GC2_JOB, insertCb, index_) {} + : AsyncJob(allocator, SVS_GC_JOB, insertCb, index_) {} }; @@ -248,6 +259,7 @@ class TieredSVSIndex : public VecSimTieredIndex { size_t trainingTriggerThreshold; size_t updateTriggerThreshold; + size_t consolidateTriggerThreshold; size_t updateJobWaitTime; // Used to prevent scheduling multiple index update jobs at the same time. // As far as the update job does a batch update, job queue should have just 1 job at the moment. @@ -263,6 +275,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::atomic backendInitSubmited{false}; std::unordered_set ids_to_init_; + std::vector labels_to_consolidate_; vecsim_stl::unordered_map> labelToInsertJobs; // A mapping to hold invalid jobs, so we can dispose them upon index deletion. @@ -610,23 +623,18 @@ class TieredSVSIndex : public VecSimTieredIndex { * @note no need to implement extra non-static method, as GC logic is simple enough to be done * here. */ - // static void SVSIndexGCWrapper(VecSimIndex *idx) { static void SVSIndexGCWrapper(AsyncJob *job) { - // assert(availableThreads > 0); - // auto index = static_cast *>(idx); - // assert(index); - auto gc_job = static_cast(job); auto index = static_cast *>(gc_job->index); - // // Release the scheduled flag to allow scheduling again - // index->indexGCScheduled.clear(); - + std::shared_lock lock(index->updateJobMutex); // Do SVS index GC index->backendIndex->log(VecSimCommonStrings::LOG_VERBOSE_STRING, "running asynchronous GC for tiered SVS index"); auto svs_index = index->GetSVSIndex(); if (index->backendIndex->indexSize() == 0) { + index->indexGCScheduled.clear(); + delete job; // No need to run GC on an empty index. return; } @@ -637,6 +645,18 @@ class TieredSVSIndex : public VecSimTieredIndex { // Release the scheduled flag to allow scheduling again index->indexGCScheduled.clear(); + delete job; + } + + static void SVSIndexConsolidateWrapper(AsyncJob *job) { + auto consolidate_job = static_cast(job); + auto index = static_cast *>(consolidate_job->index); + + std::shared_lock lock(index->updateJobMutex); + auto svs_index = index->GetSVSIndex(); + svs_index->setNumThreads(1); + svs_index->consolidate(consolidate_job->labels); + delete job; } #ifdef BUILD_TESTS @@ -683,6 +703,19 @@ class TieredSVSIndex : public VecSimTieredIndex { this->submitSingleJob(new_GC_job); } + void scheduleSVSIndexConsolidate(labelType label) { + labels_to_consolidate_.push_back(label); + + if (labels_to_consolidate_.size() >= consolidateTriggerThreshold) { + AsyncJob *new_consolidate_job = new (this->allocator) + SVSConsolidateJob(this->allocator, labels_to_consolidate_, SVSIndexConsolidateWrapper, this); + + // Insert job to the queue. + this->submitSingleJob(new_consolidate_job); + labels_to_consolidate_.clear(); + } + } + private: static void applySwapsToLabelsArray(std::vector &labels, const std::vector &swaps) { @@ -746,7 +779,7 @@ class TieredSVSIndex : public VecSimTieredIndex { memcpy(blob_copy.get(), this->frontendIndex->getDataByInternalId(job->id), data_size); this->flatIndexGuard.unlock_shared(); - svs_index->setNumThreads(1); + // svs_index->setNumThreads(1); svs_index->addVector(blob_copy.get(), job->label); // Remove the vector and the insert job from the flat buffer. @@ -792,6 +825,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::vector labels_to_move; std::vector vectors_to_move; + executeTracingCallback("UpdateJob::before_add_to_svs"); { // lock frontendIndex from modifications // The whole initialization is done under flatIndexGuard std::lock_guard flat_lock{this->flatIndexGuard}; @@ -811,13 +845,13 @@ class TieredSVSIndex : public VecSimTieredIndex { } ids_to_init_.clear(); - executeTracingCallback("UpdateJob::before_add_to_svs"); { auto svs_index = GetSVSIndex(); - svs_index->setNumThreads(1); + svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); + svs_index->setNumThreads(1); svs_index->setImpl(std::move(impl)); } @@ -844,6 +878,7 @@ class TieredSVSIndex : public VecSimTieredIndex { [](labelType label) { return label != SKIP_LABEL; }) && "Deleted vectors count does not match the number of labels to delete"); } // release frontend index + executeTracingCallback("UpdateJob::after_add_to_svs"); } public: @@ -872,6 +907,8 @@ class TieredSVSIndex : public VecSimTieredIndex { ? SVS_VAMANA_DEFAULT_TRAINING_THRESHOLD : this->updateTriggerThreshold; + this->consolidateTriggerThreshold = SVS_VAMANA_DEFAULT_CONSOLIDATE_THRESHOLD; + this->trainingTriggerThreshold = tiered_svs_params.trainingTriggerThreshold == 0 ? default_training_threshold @@ -897,7 +934,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // so make the minimal needed lock here. // Backend index initialization data have to be buffered for proper // compression/training. - if (this->backendIndex->indexSize() == 0) { + if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { // If backend index size is 0, first collect vectors in frontend index // lock in scope to ensure that these will be released before // initSVSIndexWrapper() is called. @@ -929,16 +966,22 @@ class TieredSVSIndex : public VecSimTieredIndex { // TODO: Replace this setNumThreads(1) call with an assertion once we establish // a contract that write-in-place mode guarantees numThreads == 1. svs_index->setNumThreads(1); - return this->backendIndex->addVector(storage_blob.get(), label); + int deleted = 0; + if (!this->backendIndex->isMultiValue()) { + deleted = svs_index->deleteVector(label); + if (deleted > 0) + svs_index->consolidate({label}); + } + return this->backendIndex->addVector(storage_blob.get(), label) - deleted; } } assert(this->getWriteMode() != VecSim_WriteInPlace && "InPlace mode returns early"); // Async mode - add vector to the frontend index and schedule an update job if needed. - if (!this->backendInitSubmited.load(std::memory_order_acquire)) { + if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { // Add vector to the frontend index. std::lock_guard lock(this->flatIndexGuard); - if (this->frontendIndex->isLabelExists(label)) { + if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { deleteAndUpdateInitIds(label); } ids_to_init_.insert(this->frontendIndex->indexSize()); @@ -956,7 +999,12 @@ class TieredSVSIndex : public VecSimTieredIndex { if (svs_index->ready() && (this->frontendIndex->indexSize() >= flat_buffer_bound)) { this->flatIndexGuard.unlock_shared(); auto storage_blob = this->frontendIndex->preprocessForStorage(blob); - svs_index->setNumThreads(1); + + if (!this->backendIndex->isMultiValue()) { + auto deleted = svs_index->deleteVector(label); + if (deleted > 0) + scheduleSVSIndexConsolidate(label); + } return ret = svs_index->addVector(storage_blob.get(), label); } else { @@ -989,7 +1037,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // increase index capacity. } // If this label already exists, this will do overwrite. - this->frontendIndex->addVector(blob, label); + ret += this->frontendIndex->addVector(blob, label); AsyncJob *new_insert_job = new (this->allocator) SVSInsertJob(this->allocator, label, new_flat_id, executeInsertJobWrapper, this); @@ -1014,7 +1062,10 @@ class TieredSVSIndex : public VecSimTieredIndex { if (svs_index->ready()) { // If we removed the previous vector from both svs and flat in the overwrite process, // we still return 0 (not -1). - ret = std::max(ret - svs_index->deleteVector(label), 0); + auto deleted = svs_index->deleteVector(label); + if (deleted > 0) + scheduleSVSIndexConsolidate(label); + ret = std::max(ret - deleted, 0); } } @@ -1091,13 +1142,21 @@ class TieredSVSIndex : public VecSimTieredIndex { } deleteAndUpdateInitIds(label); - return deleting_ids.size(); + ret += deleting_ids.size(); } } else { this->flatIndexGuard.unlock_shared(); } - ret += this->backendIndex->deleteVector(label); + auto deleted = this->backendIndex->deleteVector(label); + if (deleted > 0) { + if (this->getWriteMode() == VecSim_WriteInPlace) { + GetSVSIndex()->consolidate({label}); + } else { + scheduleSVSIndexConsolidate(label); + } + } + ret += deleted; return ret; } @@ -1159,10 +1218,11 @@ class TieredSVSIndex : public VecSimTieredIndex { } } info.tieredInfo.specificTieredBackendInfo.svsTieredInfo = svsTieredInfo; + // Background indexing is in progress whenever the flat buffer is non-empty: every vector + // buffered there has either a pending per-vector insert job or a batch-init job that will + // drain it into the backend. info.tieredInfo.backgroundIndexing = - svsTieredInfo.indexUpdateScheduled && info.tieredInfo.frontendCommonInfo.indexSize > 0 - ? VecSimBool_TRUE - : VecSimBool_FALSE; + info.tieredInfo.frontendCommonInfo.indexSize > 0 ? VecSimBool_TRUE : VecSimBool_FALSE; return info; } @@ -1320,4 +1380,3 @@ class TieredSVSIndex : public VecSimTieredIndex { this->flatIndexGuard.unlock_shared(); } }; - \ No newline at end of file diff --git a/src/VecSim/vec_sim_common.h b/src/VecSim/vec_sim_common.h index d800e7cdc..7d87baea0 100644 --- a/src/VecSim/vec_sim_common.h +++ b/src/VecSim/vec_sim_common.h @@ -50,6 +50,7 @@ extern "C" { #define SVS_VAMANA_DEFAULT_TRAINING_THRESHOLD (10 * DEFAULT_BLOCK_SIZE) // 10 * 1024 vectors // Default batch update threshold for SVS index. #define SVS_VAMANA_DEFAULT_UPDATE_THRESHOLD (1 * DEFAULT_BLOCK_SIZE) // 1 * 1024 vectors +#define SVS_VAMANA_DEFAULT_CONSOLIDATE_THRESHOLD (1 * DEFAULT_BLOCK_SIZE) // 1 * 1024 vectors #define SVS_VAMANA_DEFAULT_SEARCH_WINDOW_SIZE 10 // NOTE: No need to have SVS_VAMANA_DEFAULT_SEARCH_BUFFER_CAPACITY // as the default is determined by the search_window_size @@ -275,7 +276,7 @@ typedef enum { SVS_BATCH_UPDATE_JOB, SVS_INSERT_VECTOR_JOB, SVS_GC_JOB, - SVS_GC2_JOB, + SVS_CONSOLIDATE_JOB, INVALID_JOB // to indicate that finding a JobType >= INVALID_JOB is an error } JobType; diff --git a/tests/unit/test_svs_fp16.cpp b/tests/unit/test_svs_fp16.cpp index ded23bd72..95a26e6c2 100644 --- a/tests/unit/test_svs_fp16.cpp +++ b/tests/unit/test_svs_fp16.cpp @@ -2919,13 +2919,21 @@ TYPED_TEST(FP16SVSTieredIndexTest, deleteVectorMulti) { ASSERT_EQ(tiered_index->deleteVector(vec_label), 2); ASSERT_EQ(tiered_index->indexLabelCount(), 0); + + ASSERT_EQ(mock_thread_pool.jobQ.size(), 2); + mock_thread_pool.thread_iteration(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); mock_thread_pool.thread_iteration(); ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); // Test deleting a label for which both of its vector's is in SVS index. this->GenerateAndAddVector(tiered_index, dim, vec_label, vec_label); this->GenerateAndAddVector(tiered_index, dim, vec_label, other_vec_val); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 2); mock_thread_pool.thread_iteration(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); + mock_thread_pool.thread_iteration(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); ASSERT_EQ(tiered_index->indexLabelCount(), 1); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 2); diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 2980d0e38..6ef887f79 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -556,14 +556,16 @@ TYPED_TEST(SVSTieredIndexTest, addVector) { ASSERT_LE(expected_mem, tiered_index->getAllocationSize()); if constexpr (TypeParam::isMulti()) { - // Add another vector under the same label + // Add another vector under the same label. The backend init was already submitted + // by the first vector (thread_pool_size batch-init jobs), so this second vector takes + // the per-vector async path and schedules one additional SVSInsertJob of its own. VecSimIndex_AddVector(tiered_index, vector, vec_label); ASSERT_EQ(tiered_index->indexSize(), 2); ASSERT_EQ(tiered_index->indexLabelCount(), 1); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 2); - // Validate that there still 1 update jobs set - ASSERT_EQ(mock_thread_pool.jobQ.size(), mock_thread_pool.thread_pool_size); + // The batch-init jobs plus the extra per-vector insert job. + ASSERT_EQ(mock_thread_pool.jobQ.size(), mock_thread_pool.thread_pool_size + 1); } } @@ -2937,12 +2939,13 @@ TYPED_TEST(SVSTieredIndexTest, writeInPlaceMode) { ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 2); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); - // Overwrite inplace - only in single-value mode + // In write-in-place mode every mutation runs synchronously, including consolidation + // of soft-deleted slots. size_t expected_marked_deleted = 0; + // Overwrite inplace - only in single-value mode if (!TypeParam::isMulti()) { TEST_DATA_T overwritten_vec[] = {1, 1, 1, 1}; tiered_index->addVector(overwritten_vec, vec_label); - expected_marked_deleted++; ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 3); ASSERT_EQ(tiered_index->indexSize(), 3); ASSERT_EQ(tiered_index->indexLabelCount(), 2); @@ -2950,9 +2953,9 @@ TYPED_TEST(SVSTieredIndexTest, writeInPlaceMode) { ASSERT_EQ(tiered_index->getDistanceFrom_Unsafe(vec_label, overwritten_vec), 0); ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), expected_marked_deleted); } - // Validate that the vector is marked as deleted. + // Validate that the vector is deleted and consolidated synchronously (in-place mode), + // so no marked-deleted entry remains. tiered_index->deleteVector(vec_label); - expected_marked_deleted++; ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), TypeParam::isMulti() ? 2 : 3); ASSERT_EQ(tiered_index->indexLabelCount(), 1); @@ -3142,12 +3145,13 @@ TYPED_TEST(SVSTieredIndexTestBasic, runGCAPI) { auto jobs_before_gc = mock_thread_pool.jobQ.size(); // Run the GC API call, expect that we will clean up the SVS index. VecSimTieredIndex_GC(tiered_index); - // Expected that GC jobs were added to the queue. - ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + mock_thread_pool.thread_pool_size); + // Expected that a single GC job was added to the queue. + ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + 1); // Run GC twice. VecSimTieredIndex_GC(tiered_index); - // Expected that no new GC jobs were added to the queue. - ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + mock_thread_pool.thread_pool_size); + // Expected that no new GC job was added to the queue (indexGCScheduled is still set until the + // pending job runs). + ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + 1); // Wait for any pending jobs to complete. As far as SVS GC is done via a job. mock_thread_pool.init_threads(); mock_thread_pool.thread_pool_join(); @@ -3415,7 +3419,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalSingle) { // update job paused, we have vectors 0-(n-1) in the index, let's do index modifications // Remove vector label=n-2, it is copied to backend index. - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 2), 2); + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 2), 1); // Update vector label=1. EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, 1, 10), 0); // Add a new vector @@ -3423,9 +3427,9 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalSingle) { // Add another one EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, n + 1, n + 1), 1); // Remove vector label=0, it is copied to backend index. - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, 0), 2); + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, 0), 1); // Remove the last vector copied to backend index - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 1), 2); + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 1), 1); // Update vector label=2. EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, 2, 20), 0); // Remove vector label=2. @@ -3447,16 +3451,15 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalSingle) { // 0:deleted, 1: 10, 2: deleted, 3:3, ..., n-2:deleted n-1: 10(n-1), n+1: n+1; // total: n-2 vectors and labels ASSERT_EQ(tiered_index->indexLabelCount(), n - 2); - EXPECT_EQ(tiered_index->GetBackendIndex()->indexLabelCount(), n - 5); + // Nothing remains in flat. + EXPECT_EQ(tiered_index->GetBackendIndex()->indexLabelCount(), n - 2); - // We added 3 vectors to the flat index and removed 5 vectors from the backend index. - // Backend index: 0:deleted, 1:deleted, 2:deleted, 3:3, ..., n-2:deleted, n-1:deleted; - // total: n-5 - EXPECT_EQ(tiered_index->GetBackendIndex()->indexSize(), n); - ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 5); - // Frontend index: 1:10, n-1:10(n-1), n+1:n+1 - ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 3); - ASSERT_EQ(tiered_index->indexSize(), n + tiered_index->GetFlatIndex()->indexSize()); + EXPECT_EQ(tiered_index->GetBackendIndex()->indexSize(), n - 2); + ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 0); + // Flat buffer fully drained by the insert jobs. + ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); + ASSERT_EQ(tiered_index->indexSize(), tiered_index->GetBackendIndex()->indexSize() + + tiered_index->GetFlatIndex()->indexSize()); double abs_err = 1e-2; // Allow a larger relative error for quantization. TEST_DATA_T expected_vector[dim]; @@ -3469,7 +3472,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalSingle) { GenerateVector(expected_vector, dim, 0); ASSERT_TRUE(std::isnan(tiered_index->getDistanceFrom_Unsafe(n - 2, expected_vector))); - // Vector label=1, with value 10 should be in the flat index. + // Vector label=1 was updated to value 10; its latest value is retrievable from the index. GenerateVector(expected_vector, dim, 10); ASSERT_NEAR(tiered_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); @@ -3548,22 +3551,24 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalMulti) { // update job paused, we have vectors 0-(n-1) in the index, let's do index modifications - // Remove vector label=n-2, it is copied to backend index. - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 2), 2); + // Remove vector label=n-2. At the pause it lives only in the backend + //, so a single vector is removed. + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 2), 1); // Add one more vector label=1. EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, 1, 10), 1); // Add a new vector EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, n, n), 1); // Add another one EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, n + 1, n + 1), 1); - // Remove vector label=0, it is copied to backend index. - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, 0), 2); - // Remove the last vector copied to backend index - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 1), 2); + // Remove vector label=0, only in the backend at the pause. + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, 0), 1); + // Remove the last vector, only in the backend at the pause. + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 1), 1); // Add one more vector label=2. EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, 2, 20), 1); - // Remove vector label=2: for multi: old is copied to backend , old + new are in flat - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, 2), 3); + // Remove vector label=2: for multi both the original (2) and the new (20) copies sit in + // the flat buffer at this point, so 2 vectors are removed. + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, 2), 2); // Remove vector label=n - in flat only EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n), 1); // Add vector (n-1) again @@ -3581,15 +3586,16 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalMulti) { // 0: deleted, 1: (1,10), 2: deleted, 3:3, ..., n-2: deleted n-1: 10(n-1), n+1: n+1; // total: n-2 labels, n-1 vectors ASSERT_EQ(tiered_index->indexLabelCount(), n - 2); - EXPECT_EQ(tiered_index->GetBackendIndex()->indexLabelCount(), n - 4); + // In the async-insert design the per-vector insert jobs drain every surviving vector from + // the flat buffer into the backend, so nothing remains in flat. + EXPECT_EQ(tiered_index->GetBackendIndex()->indexLabelCount(), n - 2); - // We added 3 vectors to the flat index and removed 4 vectors from the backend index. - // Backend index: 0:deleted, 1:1, 2:deleted, 3:3, ..., n-2:deleted, n-1:deleted; total: n-4 - EXPECT_EQ(tiered_index->GetBackendIndex()->indexSize(), n); - ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 4); - // Frontend index: 1:10, n-1:10(n-1), n+1:n+1 - ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 3); - ASSERT_EQ(tiered_index->indexSize(), n + tiered_index->GetFlatIndex()->indexSize()); + EXPECT_EQ(tiered_index->GetBackendIndex()->indexSize(), n - 1); + ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 0); + // Flat buffer fully drained by the insert jobs. + ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); + ASSERT_EQ(tiered_index->indexSize(), tiered_index->GetBackendIndex()->indexSize() + + tiered_index->GetFlatIndex()->indexSize()); double abs_err = 1e-2; // Allow a larger relative error for quantization. TEST_DATA_T expected_vector[dim]; @@ -3602,11 +3608,11 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalMulti) { GenerateVector(expected_vector, dim, 0); ASSERT_TRUE(std::isnan(tiered_index->getDistanceFrom_Unsafe(n - 2, expected_vector))); - // There are 2 vectors labeled "1" with values 1 in backend and 10 in flat. - // We expect the minimal distance for the query 10 to be taken from flat index. + // Label "1" is multi-valued and keeps both of its vectors (values 1 and 10). + // The minimal distance for the query 10 should match the second value. GenerateVector(expected_vector, dim, 10); ASSERT_NEAR(tiered_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); - // And the minimal distance for the query 1.0 to be taken from backend + // And the minimal distance for the query 1.0 should match the first value. GenerateVector(expected_vector, dim, 1); ASSERT_NEAR(tiered_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); @@ -3706,8 +3712,9 @@ TYPED_TEST(SVSTieredIndexTestBasic, testDeletedJournalSingle) { mock_thread_pool.thread_pool_join(); - // Verify that vectors labels: {0, 1, 2, n-1} are marked as deleted in the SVS index. - ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 4); + // The update job is paused before it transfers vectors to the backend, so at the pause the + // backend is still empty. + ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 0); // Verify that the deleted vectors are not accessible. double abs_err = 1e-2; // Allow a larger relative error for quantization. @@ -3724,9 +3731,10 @@ TYPED_TEST(SVSTieredIndexTestBasic, testDeletedJournalSingle) { ASSERT_TRUE(std::isnan(tiered_index->getDistanceFrom_Unsafe(label, expected_vector))); } - // label 1 - updated to 10 but deleted in the SVS index - ASSERT_TRUE(flat_index->isLabelExists(1)); - ASSERT_FALSE(svs_index->isLabelExists(1)); + // label 1 - updated to 10 during the pause. The insert job drains it from the flat buffer + // into the backend, so it now lives in the SVS index (not flat) with its updated value. + ASSERT_FALSE(flat_index->isLabelExists(1)); + ASSERT_TRUE(svs_index->isLabelExists(1)); GenerateVector(expected_vector, dim, 10); ASSERT_NEAR(tiered_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); } @@ -3808,8 +3816,9 @@ TYPED_TEST(SVSTieredIndexTestBasic, testDeletedJournalMulti) { mock_thread_pool.thread_pool_join(); - // Verify that vectors labels: {0, 2, n-1} are marked as deleted in the SVS index. - ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 3); + // The update job is paused before it transfers vectors to the backend, so at the pause the + // backend is still empty. + ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 0); // Verify that the deleted vectors are not accessible. double abs_err = 1e-2; // Allow a larger relative error for quantization. @@ -3826,13 +3835,14 @@ TYPED_TEST(SVSTieredIndexTestBasic, testDeletedJournalMulti) { ASSERT_TRUE(std::isnan(tiered_index->getDistanceFrom_Unsafe(label, expected_vector))); } - // label 1 - multi-value 1 (in SVS) and 10 (in flat) - ASSERT_TRUE(flat_index->isLabelExists(1)); + // label 1 - multi-value with both vectors (1 and 10). The insert jobs drain the flat buffer + // into the backend, so both values now live in the SVS index and none remain in flat. + ASSERT_FALSE(flat_index->isLabelExists(1)); ASSERT_TRUE(svs_index->isLabelExists(1)); GenerateVector(expected_vector, dim, 1); ASSERT_NEAR(backend_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); GenerateVector(expected_vector, dim, 10); - ASSERT_NEAR(flat_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); + ASSERT_NEAR(backend_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); } TEST(SVSTieredIndexTest, testThreadPool) { From 07cd22fc967ea38040c3868afc44b4a58e90050b Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 30 Jul 2026 00:17:00 -0700 Subject: [PATCH 15/99] fix writer starvation for compaction --- src/VecSim/algorithms/svs/svs_tiered.h | 64 ++++++++++++++++++++------ 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index c94d3b711..c72503829 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -623,29 +623,55 @@ class TieredSVSIndex : public VecSimTieredIndex { * @note no need to implement extra non-static method, as GC logic is simple enough to be done * here. */ - static void SVSIndexGCWrapper(AsyncJob *job) { - auto gc_job = static_cast(job); - auto index = static_cast *>(gc_job->index); + // static void SVSIndexGCWrapper(AsyncJob *job) { + // auto gc_job = static_cast(job); + // auto index = static_cast *>(gc_job->index); + + // std::shared_lock lock(index->updateJobMutex); + // // Do SVS index GC + // index->backendIndex->log(VecSimCommonStrings::LOG_VERBOSE_STRING, + // "running asynchronous GC for tiered SVS index"); + // auto svs_index = index->GetSVSIndex(); + // if (index->backendIndex->indexSize() == 0) { + // index->indexGCScheduled.clear(); + // delete job; + // // No need to run GC on an empty index. + // return; + // } + // // svs_index->setNumThreads(std::min(availableThreads, index->backendIndex->indexSize())); + // svs_index->setNumThreads(1); + // // VecSimIndexAbstract::runGC() is protected + // fprintf(stderr, "runGC\n"); + // static_cast(index->backendIndex)->runGC(); + // fprintf(stderr, "GC done\n"); + + // // Release the scheduled flag to allow scheduling again + // index->indexGCScheduled.clear(); + // delete job; + // } + + static void SVSIndexGCWrapper(VecSimIndex *idx, size_t availableThreads) { + assert(availableThreads > 0); + auto index = static_cast *>(idx); + assert(index); - std::shared_lock lock(index->updateJobMutex); // Do SVS index GC index->backendIndex->log(VecSimCommonStrings::LOG_VERBOSE_STRING, "running asynchronous GC for tiered SVS index"); auto svs_index = index->GetSVSIndex(); if (index->backendIndex->indexSize() == 0) { - index->indexGCScheduled.clear(); - delete job; // No need to run GC on an empty index. return; } - // svs_index->setNumThreads(std::min(availableThreads, index->backendIndex->indexSize())); - svs_index->setNumThreads(1); + std::lock_guard lock(index->updateJobMutex); + + svs_index->setNumThreads(std::min(availableThreads, index->backendIndex->indexSize())); // VecSimIndexAbstract::runGC() is protected static_cast(index->backendIndex)->runGC(); + svs_index->setNumThreads(1); // Release the scheduled flag to allow scheduling again index->indexGCScheduled.clear(); - delete job; } static void SVSIndexConsolidateWrapper(AsyncJob *job) { @@ -696,11 +722,17 @@ class TieredSVSIndex : public VecSimTieredIndex { return; } - AsyncJob *new_GC_job = new (this->allocator) - SVSGCJob(this->allocator, SVSIndexGCWrapper, this); + auto total_threads = this->GetSVSIndex()->getThreadPoolCapacity(); + auto jobs = SVSMultiThreadJob::createJobs( + this->allocator, SVS_GC_JOB, SVSIndexGCWrapper, this, total_threads, + std::chrono::microseconds(updateJobWaitTime), &uncompletedJobs); + this->submitJobs(jobs); + + // AsyncJob *new_GC_job = new (this->allocator) + // SVSGCJob(this->allocator, SVSIndexGCWrapper, this); - // Insert job to the queue. - this->submitSingleJob(new_GC_job); + // // Insert job to the queue. + // this->submitSingleJob(new_GC_job); } void scheduleSVSIndexConsolidate(labelType label) { @@ -780,7 +812,10 @@ class TieredSVSIndex : public VecSimTieredIndex { this->flatIndexGuard.unlock_shared(); // svs_index->setNumThreads(1); - svs_index->addVector(blob_copy.get(), job->label); + { + std::shared_lock lock(updateJobMutex); + svs_index->addVector(blob_copy.get(), job->label); + } // Remove the vector and the insert job from the flat buffer. this->flatIndexGuard.lock(); @@ -1006,6 +1041,7 @@ class TieredSVSIndex : public VecSimTieredIndex { scheduleSVSIndexConsolidate(label); } + std::shared_lock lock(updateJobMutex); return ret = svs_index->addVector(storage_blob.get(), label); } else { this->flatIndexGuard.unlock_shared(); From f24187b5fdd2a8c462acafbbee8f9a8884ca1471 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Tue, 11 Aug 2026 03:52:21 -0700 Subject: [PATCH 16/99] adopt sigle label consolidation --- src/VecSim/algorithms/svs/svs.h | 4 +-- src/VecSim/algorithms/svs/svs_tiered.h | 40 +++++++++++--------------- tests/unit/test_svs_fp16.cpp | 9 ++++-- tests/unit/test_svs_tiered.cpp | 18 +++++++----- 4 files changed, 36 insertions(+), 35 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index e410529f7..33cfbc334 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -310,11 +310,11 @@ class SVSIndex : public VecSimIndexAbstract, fl assert(this->impl_ != nullptr); setReady(); } else { - impl_->add_points(points, ids); + impl_->add_points(points, ids, /*reuse_empty*/ false); } } else { // Add new points to existing SVS index - impl_->add_points(points, ids); + impl_->add_points(points, ids, /*reuse_empty*/ false); this->pimplGuard_.unlock_shared(); } diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index c72503829..27d2f4d12 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -259,7 +259,6 @@ class TieredSVSIndex : public VecSimTieredIndex { size_t trainingTriggerThreshold; size_t updateTriggerThreshold; - size_t consolidateTriggerThreshold; size_t updateJobWaitTime; // Used to prevent scheduling multiple index update jobs at the same time. // As far as the update job does a batch update, job queue should have just 1 job at the moment. @@ -275,7 +274,6 @@ class TieredSVSIndex : public VecSimTieredIndex { std::atomic backendInitSubmited{false}; std::unordered_set ids_to_init_; - std::vector labels_to_consolidate_; vecsim_stl::unordered_map> labelToInsertJobs; // A mapping to hold invalid jobs, so we can dispose them upon index deletion. @@ -661,6 +659,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto svs_index = index->GetSVSIndex(); if (index->backendIndex->indexSize() == 0) { // No need to run GC on an empty index. + index->indexGCScheduled.clear(); return; } std::lock_guard lock(index->updateJobMutex); @@ -736,16 +735,11 @@ class TieredSVSIndex : public VecSimTieredIndex { } void scheduleSVSIndexConsolidate(labelType label) { - labels_to_consolidate_.push_back(label); + AsyncJob *new_consolidate_job = new (this->allocator) + SVSConsolidateJob(this->allocator, {label}, SVSIndexConsolidateWrapper, this); - if (labels_to_consolidate_.size() >= consolidateTriggerThreshold) { - AsyncJob *new_consolidate_job = new (this->allocator) - SVSConsolidateJob(this->allocator, labels_to_consolidate_, SVSIndexConsolidateWrapper, this); - - // Insert job to the queue. - this->submitSingleJob(new_consolidate_job); - labels_to_consolidate_.clear(); - } + // Insert job to the queue. + this->submitSingleJob(new_consolidate_job); } private: @@ -942,8 +936,6 @@ class TieredSVSIndex : public VecSimTieredIndex { ? SVS_VAMANA_DEFAULT_TRAINING_THRESHOLD : this->updateTriggerThreshold; - this->consolidateTriggerThreshold = SVS_VAMANA_DEFAULT_CONSOLIDATE_THRESHOLD; - this->trainingTriggerThreshold = tiered_svs_params.trainingTriggerThreshold == 0 ? default_training_threshold @@ -1016,18 +1008,20 @@ class TieredSVSIndex : public VecSimTieredIndex { if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { // Add vector to the frontend index. std::lock_guard lock(this->flatIndexGuard); - if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { - deleteAndUpdateInitIds(label); - } - ids_to_init_.insert(this->frontendIndex->indexSize()); - const auto ft_ret = this->frontendIndex->addVector(blob, label); - ret = std::max(ret + ft_ret, 0); + if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { + if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { + deleteAndUpdateInitIds(label); + } + ids_to_init_.insert(this->frontendIndex->indexSize()); + const auto ft_ret = this->frontendIndex->addVector(blob, label); + ret = std::max(ret + ft_ret, 0); - if (this->frontendIndex->indexSize() >= this->trainingTriggerThreshold) { - this->backendInitSubmited.store(true, std::memory_order_release); - scheduleSVSIndexInit(); + if (this->frontendIndex->indexSize() >= this->trainingTriggerThreshold) { + this->backendInitSubmited.store(true, std::memory_order_release); + scheduleSVSIndexInit(); + } + return ret; } - return ret; } this->flatIndexGuard.lock_shared(); diff --git a/tests/unit/test_svs_fp16.cpp b/tests/unit/test_svs_fp16.cpp index 95a26e6c2..dc36615f6 100644 --- a/tests/unit/test_svs_fp16.cpp +++ b/tests/unit/test_svs_fp16.cpp @@ -2857,8 +2857,9 @@ TYPED_TEST(FP16SVSTieredIndexTest, deleteVector) { ASSERT_EQ(tiered_index->indexSize(), 1); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 1); - // Move the vector to SVS by executing the insert job. - mock_thread_pool.thread_iteration(); + // Move the vector to SVS by executing the pending jobs. + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->indexLabelCount(), 1); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 1); // Scalar quantization accuracy is insufficient for this check. @@ -2905,7 +2906,9 @@ TYPED_TEST(FP16SVSTieredIndexTest, deleteVectorMulti) { ASSERT_EQ(tiered_index->deleteVector(vec_label), 2); ASSERT_EQ(tiered_index->indexSize(), 0); ASSERT_EQ(tiered_index->indexLabelCount(), 0); - mock_thread_pool.thread_iteration(); + + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); // Test deleting a label for which both of its vector's is in the flat index. diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 6ef887f79..bbf655667 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -1088,8 +1088,9 @@ TYPED_TEST(SVSTieredIndexTest, deleteVector) { ASSERT_EQ(tiered_index->indexSize(), 1); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 1); - // Move the vector to SVS by executing the insert job. - mock_thread_pool.thread_iteration(); + // Move the vector to SVS by executing the pending jobs. + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->indexLabelCount(), 1); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 1); // Scalar quantization accuracy is insufficient for this check. @@ -1218,6 +1219,8 @@ TYPED_TEST(SVSTieredIndexTestBasic, deleteVectorMulti) { ASSERT_EQ(tiered_index->deleteVector(vec_label), 2); ASSERT_EQ(tiered_index->indexSize(), 0); ASSERT_EQ(tiered_index->indexLabelCount(), 0); + + mock_thread_pool.thread_iteration(); mock_thread_pool.thread_iteration(); ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); @@ -2662,7 +2665,8 @@ TYPED_TEST(SVSTieredIndexTestBasic, overwriteVectorBasic) { ASSERT_EQ(tiered_index->getDistanceFrom_Unsafe(0, overwritten_vec), 0); // Ingest the updated vector to SVS. - mock_thread_pool.thread_iteration(); + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 1); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->indexLabelCount(), 1); @@ -3146,12 +3150,12 @@ TYPED_TEST(SVSTieredIndexTestBasic, runGCAPI) { // Run the GC API call, expect that we will clean up the SVS index. VecSimTieredIndex_GC(tiered_index); // Expected that a single GC job was added to the queue. - ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + 1); + ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + mock_thread_pool.thread_pool_size); // Run GC twice. VecSimTieredIndex_GC(tiered_index); - // Expected that no new GC job was added to the queue (indexGCScheduled is still set until the - // pending job runs). - ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + 1); + // Expected that no new GC jobs were added to the queue (indexGCScheduled is still set until the + // pending jobs run). + ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + mock_thread_pool.thread_pool_size); // Wait for any pending jobs to complete. As far as SVS GC is done via a job. mock_thread_pool.init_threads(); mock_thread_pool.thread_pool_join(); From f4a8f520dda06f0abb9a6799097e71895167f9b1 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Fri, 21 Aug 2026 21:27:51 -0700 Subject: [PATCH 17/99] vecsim separate fin-grain index enabling --- src/VecSim/algorithms/svs/svs.h | 6 ++- src/VecSim/algorithms/svs/svs_extensions.h | 22 +++++++--- .../algorithms/svs/svs_serializer_impl.h | 6 +-- src/VecSim/algorithms/svs/svs_utils.h | 40 ++++++++++++++++--- 4 files changed, 57 insertions(+), 17 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 33cfbc334..663725d7c 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -93,10 +93,12 @@ class SVSIndex : public VecSimIndexAbstract, fl using graph_builder_t = SVSGraphBuilder; using graph_type = typename graph_builder_t::graph_type; + // The concurrent index mirrors the upstream template signature and member surface, so + // the only change needed here is the namespace. using impl_type = std::conditional_t< isMulti, - svs::index::vamana::MultiMutableVamanaIndex, - svs::index::vamana::MutableVamanaIndex>; + svs::concurrent::MultiMutableVamanaIndex, + svs::concurrent::MutableVamanaIndex>; bool forcePreprocessing; diff --git a/src/VecSim/algorithms/svs/svs_extensions.h b/src/VecSim/algorithms/svs/svs_extensions.h index 3903d2289..1700b30c9 100644 --- a/src/VecSim/algorithms/svs/svs_extensions.h +++ b/src/VecSim/algorithms/svs/svs_extensions.h @@ -10,10 +10,20 @@ #pragma once #include "VecSim/algorithms/svs/svs_utils.h" #include "svs/extensions/vamana/scalar.h" +// Tells SQDataset that the concurrent SegmentedBlocked tag is a blocked allocator, so +// that resize()/compact() are not constrained away. Without it a scalar-quantized +// concurrent index fails to compile the moment the index grows. +#include "svs/concurrent/extensions/scalar.h" #if HAVE_SVS_LVQ #include SVS_LVQ_HEADER #include SVS_LEANVEC_HEADER +// The same treatment for the LVQ and LeanVec datasets, each of which keeps its own private +// copy of the blocked-allocator trait. LeanVec additionally picks the allocator for its inner +// datasets through that trait, and its blocked branch hardcodes svs::data::Blocked -- so +// without this the inner storage would quietly lose the grow-stable guarantee. +#include "svs/concurrent/extensions/lvq.h" +#include "svs/concurrent/extensions/leanvec.h" #endif // HAVE_SVS_LVQ // Scalar Quantization traits for SVS @@ -21,7 +31,7 @@ template struct SVSStorageTraits { using element_type = std::int8_t; using allocator_type = svs_details::SVSAllocator; - using blocked_type = svs::data::Blocked>; + using blocked_type = svs::concurrent::SegmentedBlocked>; using index_storage_type = svs::quantization::scalar::SQDataset; @@ -32,7 +42,7 @@ struct SVSStorageTraits { // SVS block size is a power of two, so we can use it directly auto svs_bs = svs_details::SVSBlockSize(block_size, element_size(dim)); allocator_type data_allocator{std::move(allocator)}; - return svs::make_blocked_allocator_handle({svs_bs}, data_allocator); + return svs_details::make_segmented_blocked_allocator_handle({svs_bs}, data_allocator); } static constexpr VecSimSvsQuantBits get_compression_mode() { return VecSimSvsQuant_Scalar; } @@ -89,7 +99,7 @@ template struct SVSStorageTraits 1)>> { using allocator_type = svs_details::SVSAllocator; - using blocked_type = svs::data::Blocked>; + using blocked_type = svs::concurrent::SegmentedBlocked>; using strategy_type = typename svs_details::LVQSelector::strategy; using index_storage_type = svs::quantization::lvq::LVQDataset @@ -163,7 +173,7 @@ struct SVSStorageTraits struct SVSStorageTraits { using allocator_type = svs_details::SVSAllocator; - using blocked_type = svs::data::Blocked>; + using blocked_type = svs::concurrent::SegmentedBlocked>; using index_storage_type = svs::leanvec::LeanDataset, svs::leanvec::UsingLVQ, svs::Dynamic, svs::Dynamic, blocked_type>; @@ -193,7 +203,7 @@ struct SVSStorageTraits { // SVS block size is a power of two, so we can use it directly auto svs_bs = svs_details::SVSBlockSize(block_size, element_size(dim)); allocator_type data_allocator{std::move(allocator)}; - return svs::make_blocked_allocator_handle({svs_bs}, data_allocator); + return svs_details::make_segmented_blocked_allocator_handle({svs_bs}, data_allocator); } template diff --git a/src/VecSim/algorithms/svs/svs_serializer_impl.h b/src/VecSim/algorithms/svs/svs_serializer_impl.h index 6f6757922..5e551ef81 100644 --- a/src/VecSim/algorithms/svs/svs_serializer_impl.h +++ b/src/VecSim/algorithms/svs/svs_serializer_impl.h @@ -93,17 +93,17 @@ void SVSIndex compareMetadataFile(folder_path + "/metadata"); if constexpr (isMulti) { - auto loaded = svs::index::vamana::auto_multi_dynamic_assemble( + auto loaded = svs::concurrent::auto_multi_dynamic_assemble( folder_path + "/config", SVS_LAZY(graph_builder_t::load(folder_path + "/graph", this->blockSize, this->buildParams, this->getAllocator())), SVS_LAZY(storage_traits_t::load(folder_path + "/data", this->blockSize, this->dim, this->getAllocator())), distance_f(), std::move(threadpool_handle), - svs::index::vamana::MultiMutableVamanaLoad::FROM_MULTI, logger_); + svs::concurrent::MultiMutableVamanaLoad::FROM_MULTI, logger_); impl_ = std::make_unique(std::move(loaded)); } else { - auto loaded = svs::index::vamana::auto_dynamic_assemble( + auto loaded = svs::concurrent::auto_dynamic_assemble( folder_path + "/config", SVS_LAZY(graph_builder_t::load(folder_path + "/graph", this->blockSize, this->buildParams, this->getAllocator())), diff --git a/src/VecSim/algorithms/svs/svs_utils.h b/src/VecSim/algorithms/svs/svs_utils.h index 2e240358f..3c1de0f14 100644 --- a/src/VecSim/algorithms/svs/svs_utils.h +++ b/src/VecSim/algorithms/svs/svs_utils.h @@ -14,6 +14,10 @@ #include "svs/core/distance.h" #include "svs/lib/float16.h" #include "svs/index/vamana/dynamic_index.h" +// Pulls in `svs::index::vamana::concurrent` and the `svs::concurrent` alias. The concurrent +// namespace redeclares only the entities it has to replace; every other name (build/search +// parameters, extension points) still resolves to the enclosing `svs::index::vamana`. +#include "svs/concurrent/concurrent.h" #if HAVE_SVS_LVQ #include "svs/cpuid.h" @@ -198,6 +202,21 @@ inline std::pair isSVSQuantBitsSupported(VecSimSvsQuan // unreachable code, but to avoid compiler warning return std::make_pair(VecSimSvsQuant_NONE, false); } + +// The `svs::concurrent` counterpart of `svs::make_blocked_allocator_handle`. +// +// The upstream helper cannot be reused here: it hardcodes `svs::data::Blocked` in its return +// type, so there is no way to ask it for the grow-stable tag. The type-erasing +// `AllocatorHandle` wrapper is what makes the compressed datasets (which take an allocator +// handle rather than a concrete allocator) work with VecSim's own allocator. +template +svs::concurrent::SegmentedBlocked> +make_segmented_blocked_allocator_handle(const svs::data::BlockingParameters ¶meters, + Alloc alloc) { + using handle_type = svs::AllocatorHandle; + return svs::concurrent::SegmentedBlocked{ + parameters, svs::make_allocator_handle(std::move(alloc))}; +} } // namespace svs_details template ; // Used in creating storage + // `SegmentedBlocked` is a drop-in replacement for `svs::data::Blocked` (it derives from + // it and carries the same blocking parameters) that additionally keeps already-published + // elements at a stable address when the dataset grows, which is what allows a concurrent + // reader to keep a pointer into the data while a writer appends. + using blocked_type = svs::concurrent::SegmentedBlocked; // svs::Dynamic means runtime dimensionality in opposite to compile-time dimensionality - using index_storage_type = svs::data::BlockedData; + using index_storage_type = + svs::concurrent::SegmentedBlockedData; static constexpr bool is_compressed() { return false; } @@ -272,9 +296,13 @@ struct SVSStorageTraits { template struct SVSGraphBuilder { using allocator_type = svs_details::SVSAllocator; - using blocked_type = svs::data::Blocked; - using graph_data_type = svs::data::BlockedData; - using graph_type = svs::graphs::SimpleGraph; + using blocked_type = svs::concurrent::SegmentedBlocked; + using graph_data_type = + svs::concurrent::SegmentedBlockedData; + // The concurrent `SimpleGraph` has the same shape as the upstream one but guards each + // adjacency list with a sequence lock, so a reader can traverse an edge list that a + // writer is concurrently rewriting. + using graph_type = svs::concurrent::graphs::SimpleGraph; static blocked_type make_blocked_allocator(size_t block_size, size_t graph_max_degree, std::shared_ptr allocator) { @@ -304,7 +332,7 @@ struct SVSGraphBuilder { // based on the data types, which we found to perform better through heuristic analysis. auto prefetch_parameters = svs::index::vamana::extensions::estimate_prefetch_parameters(data); - auto builder = svs::index::vamana::VamanaBuilder( + auto builder = svs::concurrent::VamanaBuilder( graph, data, std::move(distance), parameters, threadpool, prefetch_parameters, logger); // Specific to the Vamana algorithm: From a603e6dad1f04215e0ec178ef056ebb6abb91b51 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Wed, 26 Aug 2026 21:06:45 -0700 Subject: [PATCH 18/99] Rename SVSIndexBase thread API to match upstream Upstream renamed the three thread-control methods on SVSIndexBase: getNumThreads -> getParallelism setNumThreads -> setParallelism getThreadPoolCapacity -> getPoolSize Adopt those names here ahead of merging upstream/main. This is a pure rename -- 32 lines across 6 files, mechanically verified by reversing the substitution and diffing against the parent commit. Method bodies deliberately keep this branch's threadpool API (size/resize/capacity), since VecSimSVSThreadPool here is still the per-index owned pool. Upstream reworked it into a process-wide singleton with thread renting, sized via VecSim_UpdateThreadPoolSize(); that is genuine divergence to reconcile in the merge, not something a rename should paper over. The point is to remove this conflict class before merging. The names collided on roughly half the affected lines without producing conflict markers, so git resolved some toward upstream and some toward here, yielding a tree that referenced methods it no longer declared. Co-Authored-By: Claude Opus 5 --- src/VecSim/algorithms/svs/svs.h | 20 ++++++++++---------- src/VecSim/algorithms/svs/svs_tiered.h | 26 +++++++++++++------------- tests/benchmark/bm_utils.h | 4 ++-- tests/benchmark/bm_vecsim_svs.h | 2 +- tests/unit/test_svs_fp16.cpp | 6 +++--- tests/unit/test_svs_tiered.cpp | 6 +++--- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 663725d7c..b2a7699ba 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -44,9 +44,9 @@ struct SVSIndexBase virtual void consolidate(const std::vector& labels) = 0; virtual bool isLabelExists(labelType label) const = 0; virtual size_t indexStorageSize() const = 0; - virtual size_t getNumThreads() const = 0; - virtual void setNumThreads(size_t numThreads) = 0; - virtual size_t getThreadPoolCapacity() const = 0; + virtual size_t getParallelism() const = 0; + virtual void setParallelism(size_t parallelism) = 0; + virtual size_t getPoolSize() const = 0; virtual bool isCompressed() const = 0; virtual bool ready() const = 0; @@ -504,8 +504,8 @@ class SVSIndex : public VecSimIndexAbstract, fl .maxCandidatePoolSize = this->buildParams.max_candidate_pool_size, .pruneTo = this->buildParams.prune_to, .useSearchHistory = this->buildParams.use_full_search_history, - .numThreads = this->getThreadPoolCapacity(), - .lastReservedThreads = this->getNumThreads(), + .numThreads = this->getPoolSize(), + .lastReservedThreads = this->getParallelism(), .numberOfMarkedDeletedNodes = this->num_marked_deleted, .searchWindowSize = this->search_window_size, .searchBufferCapacity = this->search_buffer_capacity, @@ -611,14 +611,14 @@ class SVSIndex : public VecSimIndexAbstract, fl // Enforce single-threaded execution for single vector operations to ensure optimal // performance and consistent behavior. Callers must set numThreads=1 before calling this // method. - assert(getNumThreads() == 1 && "Can't use more than one thread to insert a single vector"); + assert(getParallelism() == 1 && "Can't use more than one thread to insert a single vector"); return addVectorsImpl(vector_data, &label, 1); } int addVectors(const void *vectors_data, const labelType *labels, size_t n) override { // Prevent misuse: single vector operations should use addVector(), not addVectors() with // n=1 This ensures proper thread management and API contract enforcement. - assert(!(n == 1 && getNumThreads() > 1) && + assert(!(n == 1 && getParallelism() > 1) && "Can't use more than one thread to insert a single vector"); return addVectorsImpl(vectors_data, labels, n); } @@ -638,10 +638,10 @@ class SVSIndex : public VecSimIndexAbstract, fl } } - size_t getNumThreads() const override { return threadpool_.size(); } - void setNumThreads(size_t numThreads) override { threadpool_.resize(numThreads); } + size_t getParallelism() const override { return threadpool_.size(); } + void setParallelism(size_t parallelism) override { threadpool_.resize(parallelism); } - size_t getThreadPoolCapacity() const override { return threadpool_.capacity(); } + size_t getPoolSize() const override { return threadpool_.capacity(); } bool isCompressed() const override { return storage_traits_t::is_compressed(); } diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 27d2f4d12..a240fcd09 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -636,8 +636,8 @@ class TieredSVSIndex : public VecSimTieredIndex { // // No need to run GC on an empty index. // return; // } - // // svs_index->setNumThreads(std::min(availableThreads, index->backendIndex->indexSize())); - // svs_index->setNumThreads(1); + // // svs_index->setParallelism(std::min(availableThreads, index->backendIndex->indexSize())); + // svs_index->setParallelism(1); // // VecSimIndexAbstract::runGC() is protected // fprintf(stderr, "runGC\n"); // static_cast(index->backendIndex)->runGC(); @@ -664,10 +664,10 @@ class TieredSVSIndex : public VecSimTieredIndex { } std::lock_guard lock(index->updateJobMutex); - svs_index->setNumThreads(std::min(availableThreads, index->backendIndex->indexSize())); + svs_index->setParallelism(std::min(availableThreads, index->backendIndex->indexSize())); // VecSimIndexAbstract::runGC() is protected static_cast(index->backendIndex)->runGC(); - svs_index->setNumThreads(1); + svs_index->setParallelism(1); // Release the scheduled flag to allow scheduling again index->indexGCScheduled.clear(); @@ -679,7 +679,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::shared_lock lock(index->updateJobMutex); auto svs_index = index->GetSVSIndex(); - svs_index->setNumThreads(1); + svs_index->setParallelism(1); svs_index->consolidate(consolidate_job->labels); delete job; } @@ -708,7 +708,7 @@ class TieredSVSIndex : public VecSimTieredIndex { return; } - auto total_threads = this->GetSVSIndex()->getThreadPoolCapacity(); + auto total_threads = this->GetSVSIndex()->getPoolSize(); auto jobs = SVSMultiThreadJob::createJobs( this->allocator, SVS_BATCH_UPDATE_JOB, initSVSIndexWrapper, this, total_threads, std::chrono::microseconds(updateJobWaitTime), &uncompletedJobs); @@ -721,7 +721,7 @@ class TieredSVSIndex : public VecSimTieredIndex { return; } - auto total_threads = this->GetSVSIndex()->getThreadPoolCapacity(); + auto total_threads = this->GetSVSIndex()->getPoolSize(); auto jobs = SVSMultiThreadJob::createJobs( this->allocator, SVS_GC_JOB, SVSIndexGCWrapper, this, total_threads, std::chrono::microseconds(updateJobWaitTime), &uncompletedJobs); @@ -805,7 +805,7 @@ class TieredSVSIndex : public VecSimTieredIndex { memcpy(blob_copy.get(), this->frontendIndex->getDataByInternalId(job->id), data_size); this->flatIndexGuard.unlock_shared(); - // svs_index->setNumThreads(1); + // svs_index->setParallelism(1); { std::shared_lock lock(updateJobMutex); svs_index->addVector(blob_copy.get(), job->label); @@ -876,11 +876,11 @@ class TieredSVSIndex : public VecSimTieredIndex { { auto svs_index = GetSVSIndex(); - svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); + svs_index->setParallelism(std::min(availableThreads, labels_to_move.size())); assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); - svs_index->setNumThreads(1); + svs_index->setParallelism(1); svs_index->setImpl(std::move(impl)); } @@ -990,9 +990,9 @@ class TieredSVSIndex : public VecSimTieredIndex { std::lock_guard lock(this->updateJobMutex); // Set available thread count to 1 for single vector write-in-place operation. // This maintains the contract that single vector operations use exactly one thread. - // TODO: Replace this setNumThreads(1) call with an assertion once we establish + // TODO: Replace this setParallelism(1) call with an assertion once we establish // a contract that write-in-place mode guarantees numThreads == 1. - svs_index->setNumThreads(1); + svs_index->setParallelism(1); int deleted = 0; if (!this->backendIndex->isMultiValue()) { deleted = svs_index->deleteVector(label); @@ -1392,7 +1392,7 @@ class TieredSVSIndex : public VecSimTieredIndex { return; } // Force single thread for write-in-place mode. - this->GetSVSIndex()->setNumThreads(1); + this->GetSVSIndex()->setParallelism(1); // VecSimIndexAbstract::runGC() is protected static_cast(this->backendIndex)->runGC(); return; diff --git a/tests/benchmark/bm_utils.h b/tests/benchmark/bm_utils.h index 88f3e2e5d..e952794ad 100644 --- a/tests/benchmark/bm_utils.h +++ b/tests/benchmark/bm_utils.h @@ -30,9 +30,9 @@ CreateTieredSVSParams(VecSimParams &svs_params, tieredIndexMock &mock_thread_poo template static void verifyNumThreads(TieredSVSIndex *tiered_index, size_t expected_num_threads, size_t expected_capcity, std::string msg = "") { - ASSERT_EQ(tiered_index->GetSVSIndex()->getThreadPoolCapacity(), expected_capcity) + ASSERT_EQ(tiered_index->GetSVSIndex()->getPoolSize(), expected_capcity) << msg << ": thread pool capacity mismatch"; - size_t num_reserved_threads = tiered_index->GetSVSIndex()->getNumThreads(); + size_t num_reserved_threads = tiered_index->GetSVSIndex()->getParallelism(); if (num_reserved_threads < expected_num_threads) { std::cout << msg << ": WARNING: last reserved threads (" << num_reserved_threads << ") is less than expected (" << expected_num_threads << ")." << std::endl; diff --git a/tests/benchmark/bm_vecsim_svs.h b/tests/benchmark/bm_vecsim_svs.h index fe6c92493..c4b167421 100644 --- a/tests/benchmark/bm_vecsim_svs.h +++ b/tests/benchmark/bm_vecsim_svs.h @@ -115,7 +115,7 @@ class BM_VecSimSVS : public BM_VecSimGeneral { tiered_params.primaryIndexParams->algoParams.svsParams.num_threads; size_t num_threads = params_threadpool_size ? params_threadpool_size : mock_thread_pool.thread_pool_size; - tiered_index->GetSVSIndex()->setNumThreads(num_threads); + tiered_index->GetSVSIndex()->setParallelism(num_threads); test_utils::verifyNumThreads(tiered_index, num_threads, num_threads, std::string("CreateTieredSVSIndex")); diff --git a/tests/unit/test_svs_fp16.cpp b/tests/unit/test_svs_fp16.cpp index dc36615f6..f1746fa13 100644 --- a/tests/unit/test_svs_fp16.cpp +++ b/tests/unit/test_svs_fp16.cpp @@ -2242,8 +2242,8 @@ class FP16SVSTieredIndexTest : public FP16SVSTest { } void verifyNumThreads(TieredSVSIndex *tiered_index, size_t expected_num_threads, size_t expected_capcity) { - ASSERT_EQ(tiered_index->GetSVSIndex()->getNumThreads(), expected_num_threads); - ASSERT_EQ(tiered_index->GetSVSIndex()->getThreadPoolCapacity(), expected_capcity); + ASSERT_EQ(tiered_index->GetSVSIndex()->getParallelism(), expected_num_threads); + ASSERT_EQ(tiered_index->GetSVSIndex()->getPoolSize(), expected_capcity); } TieredSVSIndex *CreateTieredSVSIndex(const TieredIndexParams &tiered_params, @@ -2258,7 +2258,7 @@ class FP16SVSTieredIndexTest : public FP16SVSTest { // Set number of available threads to 1 unless specified otherwise, // so we can insert one vector at a time directly to svs. - tiered_index->GetSVSIndex()->setNumThreads(num_available_threads); + tiered_index->GetSVSIndex()->setParallelism(num_available_threads); size_t params_threadpool_size = tiered_params.primaryIndexParams->algoParams.svsParams.num_threads; size_t expected_capacity = diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index bbf655667..e701489d6 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -92,8 +92,8 @@ class SVSTieredIndexTest : public ::testing::Test { void verifyNumThreads(TieredSVSIndex *tiered_index, size_t expected_num_threads, size_t expected_capcity) { - ASSERT_EQ(tiered_index->GetSVSIndex()->getNumThreads(), expected_num_threads); - ASSERT_EQ(tiered_index->GetSVSIndex()->getThreadPoolCapacity(), expected_capcity); + ASSERT_EQ(tiered_index->GetSVSIndex()->getParallelism(), expected_num_threads); + ASSERT_EQ(tiered_index->GetSVSIndex()->getPoolSize(), expected_capcity); } TieredSVSIndex *CreateTieredSVSIndex(const TieredIndexParams &tiered_params, tieredIndexMock &mock_thread_pool, @@ -108,7 +108,7 @@ class SVSTieredIndexTest : public ::testing::Test { // which requires exactly 1 thread. When using tiered index addVector API, // the thread count is managed internally according to the operation and threadpool // capacity, so testing parallelism remains intact. - tiered_index->GetSVSIndex()->setNumThreads(num_available_threads); + tiered_index->GetSVSIndex()->setParallelism(num_available_threads); size_t params_threadpool_size = tiered_params.primaryIndexParams->algoParams.svsParams.num_threads; size_t expected_capacity = From baff6deaf2aa8a1cc157915797d91cb217fef820 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Tue, 1 Sep 2026 21:08:18 -0700 Subject: [PATCH 19/99] CI fixes --- src/VecSim/algorithms/svs/svs.h | 2 +- tests/unit/test_svs_tiered.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 9cf7d494f..b41e7ecdd 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -323,7 +323,7 @@ class SVSIndex : public VecSimIndexAbstract, fl return n - deleted_num; } - void consolidate(const std::vector& labels) { + void consolidate(const std::vector& labels) override { std::shared_lock lock(this->pimplGuard_); if (!ready()) return; diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index e7142f455..b1675b6ca 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -519,7 +519,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, ShrinkDuringScheduledUpdateIsDeferred) { shrink_callback_ran = true; }); - tiered_index->scheduleSVSIndexUpdate(); + tiered_index->scheduleSVSIndexInit(); ASSERT_EQ(mock_thread_pool.jobQ.size(), num_threads); mock_thread_pool.init_threads(); From 325f7b53c5e428aeb69a149a2af598438591238a Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 2 Sep 2026 23:51:44 -0700 Subject: [PATCH 20/99] clean-up commented and unused dead code --- src/VecSim/algorithms/svs/svs_tiered.h | 34 -------------------------- src/VecSim/vec_sim_common.h | 1 - 2 files changed, 35 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 7bdebb0c0..8e03ac3e4 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -640,33 +640,6 @@ class TieredSVSIndex : public VecSimTieredIndex { * @note no need to implement extra non-static method, as GC logic is simple enough to be done * here. */ - // static void SVSIndexGCWrapper(AsyncJob *job) { - // auto gc_job = static_cast(job); - // auto index = static_cast *>(gc_job->index); - - // std::shared_lock lock(index->updateJobMutex); - // // Do SVS index GC - // index->backendIndex->log(VecSimCommonStrings::LOG_VERBOSE_STRING, - // "running asynchronous GC for tiered SVS index"); - // auto svs_index = index->GetSVSIndex(); - // if (index->backendIndex->indexSize() == 0) { - // index->indexGCScheduled.clear(); - // delete job; - // // No need to run GC on an empty index. - // return; - // } - // // svs_index->setParallelism(std::min(availableThreads, index->backendIndex->indexSize())); - // svs_index->setParallelism(1); - // // VecSimIndexAbstract::runGC() is protected - // fprintf(stderr, "runGC\n"); - // static_cast(index->backendIndex)->runGC(); - // fprintf(stderr, "GC done\n"); - - // // Release the scheduled flag to allow scheduling again - // index->indexGCScheduled.clear(); - // delete job; - // } - static void SVSIndexGCWrapper(VecSimIndex *idx, size_t availableThreads) { assert(availableThreads > 0); auto index = static_cast *>(idx); @@ -744,12 +717,6 @@ class TieredSVSIndex : public VecSimTieredIndex { this->allocator, SVS_GC_JOB, SVSIndexGCWrapper, this, std::chrono::microseconds(updateJobWaitTime), &uncompletedJobs); this->submitJobs(jobs); - - // AsyncJob *new_GC_job = new (this->allocator) - // SVSGCJob(this->allocator, SVSIndexGCWrapper, this); - - // // Insert job to the queue. - // this->submitSingleJob(new_GC_job); } void scheduleSVSIndexConsolidate(labelType label) { @@ -823,7 +790,6 @@ class TieredSVSIndex : public VecSimTieredIndex { memcpy(blob_copy.get(), this->frontendIndex->getDataByInternalId(job->id), data_size); this->flatIndexGuard.unlock_shared(); - // svs_index->setParallelism(1); { std::shared_lock lock(updateJobMutex); svs_index->addVector(blob_copy.get(), job->label); diff --git a/src/VecSim/vec_sim_common.h b/src/VecSim/vec_sim_common.h index b3ded66c0..0f1a360cb 100644 --- a/src/VecSim/vec_sim_common.h +++ b/src/VecSim/vec_sim_common.h @@ -50,7 +50,6 @@ extern "C" { #define SVS_VAMANA_DEFAULT_TRAINING_THRESHOLD (10 * DEFAULT_BLOCK_SIZE) // 10 * 1024 vectors // Default batch update threshold for SVS index. #define SVS_VAMANA_DEFAULT_UPDATE_THRESHOLD (1 * DEFAULT_BLOCK_SIZE) // 1 * 1024 vectors -#define SVS_VAMANA_DEFAULT_CONSOLIDATE_THRESHOLD (1 * DEFAULT_BLOCK_SIZE) // 1 * 1024 vectors #define SVS_VAMANA_DEFAULT_SEARCH_WINDOW_SIZE 10 // NOTE: No need to have SVS_VAMANA_DEFAULT_SEARCH_BUFFER_CAPACITY // as the default is determined by the search_window_size From 105109527a1700861a478adbdf981ac9e392da39 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 3 Sep 2026 06:04:36 -0700 Subject: [PATCH 21/99] harmonize test --- tests/unit/test_svs_tiered.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index b1675b6ca..061ff75ec 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -603,7 +603,9 @@ TYPED_TEST(SVSTieredIndexTestBasic, ShrinkDuringScheduledGCIsDeferred) { }); VecSimTieredIndex_GC(tiered_index); - ASSERT_EQ(mock_thread_pool.jobQ.size(), num_threads); + // Each soft-deleted slot has corresponding consolidate() job + size_t num_consolidate = tiered_index->GetSVSIndex()->getNumMarkedDeleted(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), num_threads + num_consolidate); mock_thread_pool.init_threads(); mock_thread_pool.thread_pool_join(); From b8b57e3124539fead941b09752240eab531583d3 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Fri, 4 Sep 2026 13:34:58 -0700 Subject: [PATCH 22/99] minor revisions for CI --- src/VecSim/algorithms/svs/svs_tiered.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 8e03ac3e4..72a8cb357 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -871,7 +871,7 @@ class TieredSVSIndex : public VecSimTieredIndex { } std::sort(ids_to_move.begin(), ids_to_move.end()); - int total_deleted = 0; + [[maybe_unused]] int total_deleted = 0; for (auto it = ids_to_move.rbegin(); it != ids_to_move.rend(); ++it) { idType id = *it; auto label = this->frontendIndex->getVectorLabel(id); @@ -1308,7 +1308,7 @@ class TieredSVSIndex : public VecSimTieredIndex { return main_results; } - return merge_result_lists(main_results, flat_results, k); + return merge_result_lists(main_results, flat_results, k); } } @@ -1345,12 +1345,12 @@ class TieredSVSIndex : public VecSimTieredIndex { sort_results_by_score_then_id(flat_results); auto code = main_results->code; - VecSimQueryReply *ret = merge_result_lists(main_results, flat_results, -1); + VecSimQueryReply *ret = merge_result_lists(main_results, flat_results, -1); ret->code = code; return ret; } else { // BY_ID concat_results(main_results, flat_results); - filter_results_by_id(main_results); + filter_results_by_id(main_results); return main_results; } } From b530842457f4773cf4532286ef17bd52c4aeb198 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 7 Sep 2026 01:27:01 -0700 Subject: [PATCH 23/99] fix build and tests --- src/VecSim/algorithms/svs/svs_tiered.h | 21 ++++++++++++++------- src/VecSim/algorithms/svs/svs_utils.h | 7 +++++++ src/VecSim/index_factories/svs_factory.cpp | 4 +++- tests/unit/test_svs.cpp | 16 +++++++++++++--- tests/unit/test_svs_fp16.cpp | 21 ++++++++++++++++----- tests/unit/test_svs_multi.cpp | 10 +++++++++- tests/unit/test_svs_tiered.cpp | 10 ++++++++-- 7 files changed, 70 insertions(+), 19 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 8e03ac3e4..a55c8185a 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -864,10 +864,17 @@ class TieredSVSIndex : public VecSimTieredIndex { auto svs_index = GetSVSIndex(); svs_index->setParallelism(std::min(availableThreads, labels_to_move.size())); assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); - auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), - labels_to_move.size()); - svs_index->setParallelism(1); - svs_index->setImpl(std::move(impl)); + // The backend may already have been initialized while this job was queued + if (svs_index->ready()) { + svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), + labels_to_move.size()); + svs_index->setParallelism(1); + } else { + auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), + labels_to_move.size()); + svs_index->setParallelism(1); + svs_index->setImpl(std::move(impl)); + } } std::sort(ids_to_move.begin(), ids_to_move.end()); @@ -1308,7 +1315,7 @@ class TieredSVSIndex : public VecSimTieredIndex { return main_results; } - return merge_result_lists(main_results, flat_results, k); + return merge_result_lists(main_results, flat_results, k); } } @@ -1345,12 +1352,12 @@ class TieredSVSIndex : public VecSimTieredIndex { sort_results_by_score_then_id(flat_results); auto code = main_results->code; - VecSimQueryReply *ret = merge_result_lists(main_results, flat_results, -1); + VecSimQueryReply *ret = merge_result_lists(main_results, flat_results, -1); ret->code = code; return ret; } else { // BY_ID concat_results(main_results, flat_results); - filter_results_by_id(main_results); + filter_results_by_id(main_results); return main_results; } } diff --git a/src/VecSim/algorithms/svs/svs_utils.h b/src/VecSim/algorithms/svs/svs_utils.h index 33c1135a9..45860c9ba 100644 --- a/src/VecSim/algorithms/svs/svs_utils.h +++ b/src/VecSim/algorithms/svs/svs_utils.h @@ -373,6 +373,13 @@ struct SVSGraphBuilder { // graph_max_degree neighbors return sizeof(SVSIdType) * (graph_max_degree + 1); } + + // Per-node size of the concurrent graph's reverse-edge index. + // + static constexpr size_t reverse_edges_element_size() { + using reverse_edges_type = typename graph_type::reverse_edges_type; + return sizeof(typename reverse_edges_type::list_type) + sizeof(svs::concurrent::SpinLock); + } }; // A slot in the shared SVS thread pool. Wraps an SVS Thread with an occupancy flag diff --git a/src/VecSim/index_factories/svs_factory.cpp b/src/VecSim/index_factories/svs_factory.cpp index 33adf6fb1..bae7c021d 100644 --- a/src/VecSim/index_factories/svs_factory.cpp +++ b/src/VecSim/index_factories/svs_factory.cpp @@ -234,10 +234,12 @@ size_t EstimateElementSize(const SVSParams *params) { // Assuming that the graph_max_degree can be unset in params. const auto graph_max_degree = svs_details::makeVamanaBuildParameters(*params).graph_max_degree; const auto graph_node_size = SVSGraphBuilder::element_size(graph_max_degree); + const auto reverse_edges_node_size = + SVSGraphBuilder::reverse_edges_element_size(); const auto vector_size = QuantizedVectorSize(params->type, params->quantBits, params->dim, 0, params->leanvec_dim); - return vector_size + graph_node_size; + return vector_size + graph_node_size + reverse_edges_node_size; } size_t EstimateInitialSize(const SVSParams *params, bool is_normalized) { diff --git a/tests/unit/test_svs.cpp b/tests/unit/test_svs.cpp index 27b381327..3105372b2 100644 --- a/tests/unit/test_svs.cpp +++ b/tests/unit/test_svs.cpp @@ -35,6 +35,16 @@ } \ } +// Expected allocation for one block of `block_size` element slots, of which only +// `num_elements` are actually populated. +// +static size_t EstimateBlockSize(const SVSParams ¶ms, size_t block_size, size_t num_elements) { + const size_t reverse_edges_per_slot = SVSGraphBuilder::reverse_edges_element_size(); + const size_t reverse_edges_slots = svs::lib::SegmentedVector(num_elements).capacity(); + return (EstimateElementSize(params) - reverse_edges_per_slot) * block_size + + reverse_edges_per_slot * reverse_edges_slots; +} + // Log callback function to print non-debug log messages static void svsTestLogCallBackNoDebug(void *ctx, const char *level, const char *message) { if (level == nullptr || message == nullptr) { @@ -1996,7 +2006,7 @@ TYPED_TEST(SVSTest, testSizeEstimation) { size_t actual = index->getAllocationSize(); ASSERT_EQ(estimation, actual); - estimation = EstimateElementSize(params) * bs; + estimation = EstimateBlockSize(params, bs, 1); GenerateAndAddVector(index, dim, 0); actual = index->getAllocationSize() - actual; // get the delta @@ -2857,7 +2867,7 @@ TEST(SVSTest, quant_modes) { ASSERT_EQ(VecSimIndex_IndexSize(index), n); - estimation = EstimateElementSize(params) * params.blockSize; + estimation = EstimateBlockSize(params, params.blockSize, n); actual = index->getAllocationSize() - actual; // get the delta ASSERT_GT(actual, 0); // LVQ element size estimation accuracy is low @@ -3219,7 +3229,7 @@ TEST(SVSTest, scalar_quantization_query) { ASSERT_EQ(VecSimIndex_IndexSize(index_sq), n); ASSERT_EQ(index_sq->indexCapacity(), n); - estimation = EstimateElementSize(params) * params.blockSize; + estimation = EstimateBlockSize(params, params.blockSize, n); actual = index_sq->getAllocationSize() - actual; // get the delta ASSERT_GT(actual, 0); ASSERT_GE(estimation * 1.01, actual); diff --git a/tests/unit/test_svs_fp16.cpp b/tests/unit/test_svs_fp16.cpp index 895e8f8a5..63ccf1aab 100644 --- a/tests/unit/test_svs_fp16.cpp +++ b/tests/unit/test_svs_fp16.cpp @@ -40,6 +40,13 @@ using float16 = vecsim_types::float16; } \ } +static size_t EstimateBlockSize(const SVSParams ¶ms, size_t block_size, size_t num_elements) { + const size_t reverse_edges_per_slot = SVSGraphBuilder::reverse_edges_element_size(); + const size_t reverse_edges_slots = svs::lib::SegmentedVector(num_elements).capacity(); + return (EstimateElementSize(params) - reverse_edges_per_slot) * block_size + + reverse_edges_per_slot * reverse_edges_slots; +} + // Log callback function to print non-debug log messages static void svsTestLogCallBackNoDebug(void *ctx, const char *level, const char *message) { if (level == nullptr || message == nullptr) { @@ -1388,7 +1395,7 @@ TYPED_TEST(FP16SVSTest, testSizeEstimation) { size_t actual = index->getAllocationSize(); ASSERT_EQ(estimation, actual); - estimation = EstimateElementSize(params) * bs; + estimation = EstimateBlockSize(params, bs, 1); this->GenerateAndAddVector(index, dim, 0); actual = index->getAllocationSize() - actual; // get the delta @@ -1589,7 +1596,7 @@ TYPED_TEST(FP16SVSTest, quant_modes) { ASSERT_EQ(VecSimIndex_IndexSize(index), n); - estimation = EstimateElementSize(params) * DEFAULT_BLOCK_SIZE; + estimation = EstimateBlockSize(params, DEFAULT_BLOCK_SIZE, n); actual = index->getAllocationSize() - actual; // get the delta ASSERT_GT(actual, 0); // LVQ element size estimation accuracy is low @@ -1666,15 +1673,19 @@ TYPED_TEST(FP16SVSTest, scalar_quantization_query) { const size_t k = 10; const double quant_precision = 1.0 / (1 << 7); // int8 quantization precision + std::default_random_engine gen; + std::uniform_real_distribution dist(-1.0, 1.0); std::vector> dataset(n); for (size_t i = 0; i < n; i++) { - test_utils::populate_float16_vec(dataset[i].data(), dim, i, -1.0, 1.0); + for (size_t j = 0; j < dim; j++) { + dataset[i][j] = vecsim_types::FP32_to_FP16(dist(gen)); + } } std::vector ids(n); std::iota(ids.begin(), ids.end(), 0); float16 query[dim]; - test_utils::populate_float16_vec(query, dim, n, -1.0, 1.0); + this->GenerateVector(query, dim, 0.1f); VecSimQueryReply *fp_results = nullptr; auto verify_res = [&](size_t id, double score, size_t result_rank) { @@ -1724,7 +1735,7 @@ TYPED_TEST(FP16SVSTest, scalar_quantization_query) { ASSERT_EQ(VecSimIndex_IndexSize(index_sq), n); ASSERT_EQ(index_sq->indexCapacity(), n); - estimation = EstimateElementSize(params) * params.blockSize; + estimation = EstimateBlockSize(params, params.blockSize, n); actual = index_sq->getAllocationSize() - actual; // get the delta ASSERT_GT(actual, 0); ASSERT_GE(estimation * 1.01, actual); diff --git a/tests/unit/test_svs_multi.cpp b/tests/unit/test_svs_multi.cpp index 9c75ab8c3..bca0ee705 100644 --- a/tests/unit/test_svs_multi.cpp +++ b/tests/unit/test_svs_multi.cpp @@ -23,6 +23,14 @@ } \ } + +static size_t EstimateBlockSize(const SVSParams ¶ms, size_t block_size, size_t num_elements) { + const size_t reverse_edges_per_slot = SVSGraphBuilder::reverse_edges_element_size(); + const size_t reverse_edges_slots = svs::lib::SegmentedVector(num_elements).capacity(); + return (EstimateElementSize(params) - reverse_edges_per_slot) * block_size + + reverse_edges_per_slot * reverse_edges_slots; +} + // Log callback function to print non-debug log messages static void svsTestLogCallBackNoDebug(void *ctx, const char *level, const char *message) { if (level == nullptr || message == nullptr) { @@ -748,7 +756,7 @@ TYPED_TEST(SVSMultiTest, testSizeEstimation) { size_t actual = index->getAllocationSize(); ASSERT_EQ(estimation, actual); - estimation = EstimateElementSize(params) * bs; + estimation = EstimateBlockSize(params, bs, 1); GenerateAndAddVector(index, dim, 0); actual = index->getAllocationSize() - actual; // get the delta diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index ebf1d5497..d424d8124 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -28,6 +28,11 @@ } \ } +static size_t EstimateReverseEdgesSize(size_t num_elements) { + return SVSGraphBuilder::reverse_edges_element_size() * + svs::lib::SegmentedVector(num_elements).capacity(); +} + // Get available number of CPUs // Returns the number of logical processors on the process // Returns std::thread::hardware_concurrency() if the number of logical processors is not available @@ -1706,8 +1711,9 @@ TYPED_TEST(SVSTieredIndexTest, testSizeEstimation) { } mock_thread_pool.thread_pool_wait(); - // Estimate memory delta for filling up the first block and adding another block. - size_t estimation = VecSimIndex_EstimateElementSize(¶ms) * bs; + const size_t reverse_edges_per_slot = SVSGraphBuilder::reverse_edges_element_size(); + size_t estimation = (VecSimIndex_EstimateElementSize(¶ms) - reverse_edges_per_slot) * bs + + EstimateReverseEdgesSize(n + 1) - EstimateReverseEdgesSize(n); size_t before = index->getAllocationSize(); GenerateAndAddVector(index, dim, bs + n, bs + n); From 7e66fea573b205fe745e1d2fb543cc7a59b7b6ed Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Tue, 8 Sep 2026 02:05:46 -0700 Subject: [PATCH 24/99] fix ub --- src/VecSim/algorithms/svs/svs_tiered.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index f6e10ea30..447085c38 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -790,10 +790,7 @@ class TieredSVSIndex : public VecSimTieredIndex { memcpy(blob_copy.get(), this->frontendIndex->getDataByInternalId(job->id), data_size); this->flatIndexGuard.unlock_shared(); - { - std::shared_lock lock(updateJobMutex); - svs_index->addVector(blob_copy.get(), job->label); - } + svs_index->addVector(blob_copy.get(), job->label); // Remove the vector and the insert job from the flat buffer. this->flatIndexGuard.lock(); From 7213909f861f0b65d74030095dfa01704ddfc39f Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Tue, 8 Sep 2026 22:18:18 -0700 Subject: [PATCH 25/99] use SVS rc --- cmake/svs.cmake | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index 6174d827c..f6aebd87a 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -64,7 +64,7 @@ if(USE_SVS) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.31" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "21.0") if(GLIBCXX_VERSION VERSION_GREATER_EQUAL "11") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.3.2/svs-shared-library-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc1-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") else() message(STATUS "libstdc++ >= GCC 11 is required for Clang SVS binaries - disabling SVS_SHARED_LIB") set(SVS_SHARED_LIB OFF) @@ -76,12 +76,10 @@ if(USE_SVS) else() if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.28") if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "14.0") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.3.2/svs-shared-library-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc1-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") else() - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.3.2/svs-shared-library-reduced.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc1-reduced.tar.gz" CACHE STRING "SVS URL") endif() - elseif(GLIBC_VERSION VERSION_GREATER_EQUAL "2.26") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.3.0/svs-shared-library-0.3.0-reduced-glibc2_26.tar.gz" CACHE STRING "SVS URL") else() message(STATUS "GLIBC >= 2.26 is required for GCC SVS binaries - disabling SVS_SHARED_LIB") set(SVS_SHARED_LIB OFF) From 37884f4d5bf8c9230be31257c07db8f8590315f1 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 10 Sep 2026 09:31:54 +0200 Subject: [PATCH 26/99] fix TestIndexMultiValueJsonReload Co-authored-by: ethanglaser <42726565+ethanglaser@users.noreply.github.com> --- src/VecSim/algorithms/svs/svs_tiered.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 447085c38..d6388a721 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -957,7 +957,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // initSVSIndexWrapper() is called. { std::lock_guard lock(this->flatIndexGuard); - if (this->frontendIndex->isLabelExists(label)) { + if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { deleteAndUpdateInitIds(label); } ids_to_init_.insert(this->frontendIndex->indexSize()); From 586634278ad0a59a9bd521440df5d4050e6d8703 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 10 Sep 2026 00:51:16 -0700 Subject: [PATCH 27/99] clang format --- src/VecSim/algorithms/svs/svs.h | 30 +++---- src/VecSim/algorithms/svs/svs_tiered.h | 103 +++++++++++++------------ src/VecSim/algorithms/svs/svs_utils.h | 6 +- tests/unit/test_svs.cpp | 10 ++- tests/unit/test_svs_multi.cpp | 1 - tests/unit/test_svs_tiered.cpp | 3 +- 6 files changed, 79 insertions(+), 74 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 2a367bad0..c84751cf5 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -41,7 +41,7 @@ struct SVSIndexBase virtual int addVectors(const void *vectors_data, const labelType *labels, size_t n) = 0; virtual int deleteVector(labelType label) = 0; virtual int deleteVectors(const labelType *labels, size_t n) = 0; - virtual void consolidate(const std::vector& labels) = 0; + virtual void consolidate(const std::vector &labels) = 0; virtual bool isLabelExists(labelType label) const = 0; virtual size_t indexStorageSize() const = 0; virtual size_t getParallelism() const = 0; @@ -126,13 +126,9 @@ class SVSIndex : public VecSimIndexAbstract, fl std::atomic impl_ready_{false}; - void setReady() { - this->impl_ready_.store(true, std::memory_order_release); - } + void setReady() { this->impl_ready_.store(true, std::memory_order_release); } - void setUnready() { - this->impl_ready_.store(false, std::memory_order_release); - } + void setUnready() { this->impl_ready_.store(false, std::memory_order_release); } static double toVecSimDistance(float v) { return svs_details::toVecSimDistance(v); } @@ -271,7 +267,8 @@ class SVSIndex : public VecSimIndexAbstract, fl { std::lock_guard lock(this->pimplGuard_); this->impl_ = std::move(svs_handler->impl); - if (this->impl_) setReady(); + if (this->impl_) + setReady(); } } @@ -323,7 +320,7 @@ class SVSIndex : public VecSimIndexAbstract, fl return n - deleted_num; } - void consolidate(const std::vector& labels) override { + void consolidate(const std::vector &labels) override { std::shared_lock lock(this->pimplGuard_); if (!ready()) return; @@ -421,9 +418,7 @@ class SVSIndex : public VecSimIndexAbstract, fl ~SVSIndex() = default; - bool ready() const override { - return this->impl_ready_.load(std::memory_order_acquire); - } + bool ready() const override { return this->impl_ready_.load(std::memory_order_acquire); } size_t indexSize() const override { return indexStorageSize(); } @@ -670,7 +665,6 @@ class SVSIndex : public VecSimIndexAbstract, fl } } - auto query_datum = std::span{static_cast(vector_data), this->dim}; { std::shared_lock lock(this->pimplGuard_); @@ -743,7 +737,7 @@ class SVSIndex : public VecSimIndexAbstract, fl auto processed_query_ptr = this->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); std::span query{static_cast(processed_query), - this->dim}; + this->dim}; // Base search parameters for the SVS iterator schedule. auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams, @@ -763,8 +757,8 @@ class SVSIndex : public VecSimIndexAbstract, fl // range search using epsilon const auto epsilon = queryParams && queryParams->svsRuntimeParams.epsilon != 0 - ? queryParams->svsRuntimeParams.epsilon - : this->epsilon; + ? queryParams->svsRuntimeParams.epsilon + : this->epsilon; const auto range_search_boundaries = radius * (1.0 + std::abs(epsilon)); bool keep_searching = true; @@ -780,8 +774,8 @@ class SVSIndex : public VecSimIndexAbstract, fl keep_searching = false; } } - // If search radius + epsilon is not exceeded, request SVS BatchIterator for the next - // batch + // If search radius + epsilon is not exceeded, request SVS BatchIterator for the + // next batch if (keep_searching) { svs_it.next(batch_size, cancel); if (cancel()) { diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index d6388a721..fd8b17239 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -21,7 +21,7 @@ struct SVSInsertJob : public AsyncJob { idType id; SVSInsertJob(std::shared_ptr allocator, labelType label_, idType id_, - JobCallback insertCb, VecSimIndex *index_) + JobCallback insertCb, VecSimIndex *index_) : AsyncJob(allocator, SVS_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} }; @@ -31,8 +31,9 @@ struct SVSInsertJob : public AsyncJob { struct SVSConsolidateJob : public AsyncJob { std::vector labels; - SVSConsolidateJob(std::shared_ptr allocator, const std::vector& labels_, - JobCallback insertCb, VecSimIndex *index_) + SVSConsolidateJob(std::shared_ptr allocator, + const std::vector &labels_, JobCallback insertCb, + VecSimIndex *index_) : AsyncJob(allocator, SVS_CONSOLIDATE_JOB, insertCb, index_), labels(labels_) {} }; @@ -40,12 +41,10 @@ struct SVSConsolidateJob : public AsyncJob { * Definition of a job that launches GC on SVS Index. */ struct SVSGCJob : public AsyncJob { - SVSGCJob(std::shared_ptr allocator, - JobCallback insertCb, VecSimIndex *index_) - : AsyncJob(allocator, SVS_GC_JOB, insertCb, index_) {} + SVSGCJob(std::shared_ptr allocator, JobCallback insertCb, VecSimIndex *index_) + : AsyncJob(allocator, SVS_GC_JOB, insertCb, index_) {} }; - /** * @class SVSMultiThreadJob * @brief Represents a multi-threaded asynchronous job for the SVS algorithm. @@ -422,8 +421,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::move(allocator)), index(index), flat_results(this->allocator), svs_results(this->allocator), flat_iterator(index->frontendIndex->newBatchIterator(query_vector, queryParams)), - svs_iterator(nullptr), - returned_results_set(this->allocator) { + svs_iterator(nullptr), returned_results_set(this->allocator) { if (queryParams) { this->queryParams = (VecSimQueryParams *)this->allocator->allocate(sizeof(VecSimQueryParams)); @@ -771,7 +769,8 @@ class TieredSVSIndex : public VecSimTieredIndex { auto svs_index = GetSVSIndex(); assert(svs_index->ready()); - // Note that accessing the job fields should occur with flat index guard held (here and later). + // Note that accessing the job fields should occur with flat index guard held (here and + // later). this->flatIndexGuard.lock_shared(); if (!job->isValid) { this->flatIndexGuard.unlock_shared(); @@ -817,12 +816,14 @@ class TieredSVSIndex : public VecSimTieredIndex { this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); int deleted = this->frontendIndex->deleteVectorById(job->label, job->id); if (deleted && job->id != this->frontendIndex->indexSize()) { - // If the vector removal caused a swap with the last id, update the relevant insert job. + // If the vector removal caused a swap with the last id, update the relevant insert + // job. this->updateInsertJobInternalId(this->frontendIndex->indexSize(), job->id, last_vec_label); } } else { - // Remove the current job from the invalid jobs' lookup, as we are about to delete it now. + // Remove the current job from the invalid jobs' lookup, as we are about to delete it + // now. this->invalidJobsLookupGuard.lock(); this->invalidJobs.erase(job->id); this->invalidJobsLookupGuard.unlock(); @@ -860,7 +861,8 @@ class TieredSVSIndex : public VecSimTieredIndex { if (!labels_to_move.empty()) { auto svs_index = GetSVSIndex(); svs_index->setParallelism(std::min(availableThreads, labels_to_move.size())); - assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); + assert(labels_to_move.size() == + vectors_to_move.size() / this->frontendIndex->getDim()); // The backend may already have been initialized while this job was queued if (svs_index->ready()) { svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), @@ -885,7 +887,8 @@ class TieredSVSIndex : public VecSimTieredIndex { this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); int deleted = this->frontendIndex->deleteVectorById(label, id); if (deleted && id != this->frontendIndex->indexSize()) { - // If the vector removal caused a swap with the last id, update the relevant insert job. + // If the vector removal caused a swap with the last id, update the relevant + // insert job. this->updateInsertJobInternalId(this->frontendIndex->indexSize(), id, last_vec_label); } @@ -893,8 +896,9 @@ class TieredSVSIndex : public VecSimTieredIndex { } } - assert(total_deleted == std::count_if(labels_to_move.begin(), labels_to_move.end(), - [](labelType label) { return label != SKIP_LABEL; }) && + assert(total_deleted == + std::count_if(labels_to_move.begin(), labels_to_move.end(), + [](labelType label) { return label != SKIP_LABEL; }) && "Deleted vectors count does not match the number of labels to delete"); } // release frontend index executeTracingCallback("UpdateJob::after_add_to_svs"); @@ -905,16 +909,14 @@ class TieredSVSIndex : public VecSimTieredIndex { const TieredIndexParams &tiered_index_params, std::shared_ptr allocator) : Base(svs_index, bf_index, tiered_index_params, allocator), - uncompletedJobs(this->allocator), - labelToInsertJobs(this->allocator), - invalidJobs(this->allocator), - currInvalidJobId(0) { + uncompletedJobs(this->allocator), labelToInsertJobs(this->allocator), + invalidJobs(this->allocator), currInvalidJobId(0) { const auto &tiered_svs_params = tiered_index_params.specificParams.tieredSVSParams; // If flatBufferLimit is not initialized (0), use the default update threshold. flat_buffer_bound = tiered_index_params.flatBufferLimit == 0 - ? SVS_VAMANA_DEFAULT_UPDATE_THRESHOLD - : tiered_index_params.flatBufferLimit; + ? SVS_VAMANA_DEFAULT_UPDATE_THRESHOLD + : tiered_index_params.flatBufferLimit; this->updateTriggerThreshold = tiered_svs_params.updateTriggerThreshold == 0 @@ -951,13 +953,15 @@ class TieredSVSIndex : public VecSimTieredIndex { // so make the minimal needed lock here. // Backend index initialization data have to be buffered for proper // compression/training. - if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { + if ((!svs_index->ready()) && + (!this->backendInitSubmited.load(std::memory_order_acquire))) { // If backend index size is 0, first collect vectors in frontend index // lock in scope to ensure that these will be released before // initSVSIndexWrapper() is called. { std::lock_guard lock(this->flatIndexGuard); - if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { + if (!this->frontendIndex->isMultiValue() && + this->frontendIndex->isLabelExists(label)) { deleteAndUpdateInitIds(label); } ids_to_init_.insert(this->frontendIndex->indexSize()); @@ -998,8 +1002,10 @@ class TieredSVSIndex : public VecSimTieredIndex { if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { // Add vector to the frontend index. std::lock_guard lock(this->flatIndexGuard); - if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { - if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { + if ((!svs_index->ready()) && + (!this->backendInitSubmited.load(std::memory_order_acquire))) { + if (!this->frontendIndex->isMultiValue() && + this->frontendIndex->isLabelExists(label)) { deleteAndUpdateInitIds(label); } ids_to_init_.insert(this->frontendIndex->indexSize()); @@ -1041,20 +1047,21 @@ class TieredSVSIndex : public VecSimTieredIndex { return ret; } - // Overwrite the vector and invalidate its only pending job (since we are not in MULTI). - // Label exists, but job doesn't. It means the label is used for initialization. - // Just create new job, this job will remove duplicate from the backend. + // Overwrite the vector and invalidate its only pending job (since we are not in + // MULTI). Label exists, but job doesn't. It means the label is used for + // initialization. Just create new job, this job will remove duplicate from the + // backend. auto *old_job = this->labelToInsertJobs.at(label).at(0); old_job->id = this->setAndSaveInvalidJob(old_job); this->labelToInsertJobs.erase(label); ret = 0; - // We are going to update the internal id that currently holds the vector associated with - // the given label. + // We are going to update the internal id that currently holds the vector associated + // with the given label. new_flat_id = dynamic_cast *>(this->frontendIndex) ->getIdOfLabel(label); - // If we are adding a new element (rather than updating an exiting one) we may need to - // increase index capacity. + // If we are adding a new element (rather than updating an exiting one) we may need + // to increase index capacity. } // If this label already exists, this will do overwrite. ret += this->frontendIndex->addVector(blob, label); @@ -1062,10 +1069,11 @@ class TieredSVSIndex : public VecSimTieredIndex { AsyncJob *new_insert_job = new (this->allocator) SVSInsertJob(this->allocator, label, new_flat_id, executeInsertJobWrapper, this); - // Save a pointer to the job, so that if the vector is overwritten, we'll have an indication. + // Save a pointer to the job, so that if the vector is overwritten, we'll have an + // indication. if (this->labelToInsertJobs.find(label) != this->labelToInsertJobs.end()) { - // There's already a pending insert job for this label, add another one (without overwrite, - // only possible in multi index) + // There's already a pending insert job for this label, add another one (without + // overwrite, only possible in multi index) assert(this->backendIndex->isMultiValue()); this->labelToInsertJobs.at(label).push_back((SVSInsertJob *)new_insert_job); } else { @@ -1080,8 +1088,8 @@ class TieredSVSIndex : public VecSimTieredIndex { // we submit the insert job. if (!this->backendIndex->isMultiValue()) { if (svs_index->ready()) { - // If we removed the previous vector from both svs and flat in the overwrite process, - // we still return 0 (not -1). + // If we removed the previous vector from both svs and flat in the overwrite + // process, we still return 0 (not -1). auto deleted = svs_index->deleteVector(label); if (deleted > 0) scheduleSVSIndexConsolidate(label); @@ -1137,14 +1145,17 @@ class TieredSVSIndex : public VecSimTieredIndex { if (this->frontendIndex->isLabelExists(label)) { this->flatIndexGuard.unlock_shared(); std::lock_guard flat_lock{this->flatIndexGuard}; - // Check again if the label exists, as it may have been removed while we released the lock. + // Check again if the label exists, as it may have been removed while we released the + // lock. if (this->frontendIndex->isLabelExists(label)) { auto deleting_ids = this->frontendIndex->getElementIds(label); - if (deleting_ids.size() == 0) return 0; + if (deleting_ids.size() == 0) + return 0; // assert if all elements of deleting_ids are unique - assert(std::set(deleting_ids.begin(), deleting_ids.end()).size() == deleting_ids.size() && - "deleting_ids should contain unique ids"); + assert(std::set(deleting_ids.begin(), deleting_ids.end()).size() == + deleting_ids.size() && + "deleting_ids should contain unique ids"); // If id is deleted, don't use it for initialization for (idType id : deleting_ids) { @@ -1391,11 +1402,7 @@ class TieredSVSIndex : public VecSimTieredIndex { scheduleSVSIndexGC(); } - void acquireSharedLocks() override { - this->flatIndexGuard.lock_shared(); - } + void acquireSharedLocks() override { this->flatIndexGuard.lock_shared(); } - void releaseSharedLocks() override { - this->flatIndexGuard.unlock_shared(); - } + void releaseSharedLocks() override { this->flatIndexGuard.unlock_shared(); } }; diff --git a/src/VecSim/algorithms/svs/svs_utils.h b/src/VecSim/algorithms/svs/svs_utils.h index 45860c9ba..670ea172c 100644 --- a/src/VecSim/algorithms/svs/svs_utils.h +++ b/src/VecSim/algorithms/svs/svs_utils.h @@ -218,7 +218,7 @@ inline std::pair isSVSQuantBitsSupported(VecSimSvsQuan template svs::concurrent::SegmentedBlocked> make_segmented_blocked_allocator_handle(const svs::data::BlockingParameters ¶meters, - Alloc alloc) { + Alloc alloc) { using handle_type = svs::AllocatorHandle; return svs::concurrent::SegmentedBlocked{ parameters, svs::make_allocator_handle(std::move(alloc))}; @@ -338,8 +338,8 @@ struct SVSGraphBuilder { // based on the data types, which we found to perform better through heuristic analysis. auto prefetch_parameters = svs::index::vamana::extensions::estimate_prefetch_parameters(data); - auto builder = svs::concurrent::VamanaBuilder( - graph, data, std::move(distance), parameters, threadpool, prefetch_parameters, logger); + auto builder = svs::concurrent::VamanaBuilder(graph, data, std::move(distance), parameters, + threadpool, prefetch_parameters, logger); // Specific to the Vamana algorithm: // It builds in two rounds, one with alpha=1 and the second time with the user/config diff --git a/tests/unit/test_svs.cpp b/tests/unit/test_svs.cpp index 3105372b2..953a95cc4 100644 --- a/tests/unit/test_svs.cpp +++ b/tests/unit/test_svs.cpp @@ -1583,7 +1583,8 @@ TYPED_TEST(SVSTest, test_svs_parameter_combinations_and_defaults) { // Test: All default parameters (zeros/unset) {"all_defaults", { - .dim = dim, .metric = VecSimMetric_L2, + .dim = dim, + .metric = VecSimMetric_L2, // All other parameters left as default (0/unset) }, {.quantBits = get<0>(svs_details::isSVSQuantBitsSupported(TypeParam::get_quant_bits())), @@ -1768,7 +1769,8 @@ TYPED_TEST(SVSTest, test_svs_parameter_consistency_across_metrics) { // Create index with default parameters for this metric SVSParams params = { - .dim = dim, .metric = metric, + .dim = dim, + .metric = metric, // All other parameters use defaults }; @@ -3369,7 +3371,9 @@ TEST(SVSTest, NumThreadsParamIgnored) { // Verify: creating an index without setting num_threads produces no warning. captured_log.clear(); SVSParams svs_params_default = { - .type = VecSimType_FLOAT32, .dim = 4, .metric = VecSimMetric_L2, + .type = VecSimType_FLOAT32, + .dim = 4, + .metric = VecSimMetric_L2, // num_threads left as 0 (default / unset) }; VecSimParams params_default{.algo = VecSimAlgo_SVS, diff --git a/tests/unit/test_svs_multi.cpp b/tests/unit/test_svs_multi.cpp index bca0ee705..0ed155ba0 100644 --- a/tests/unit/test_svs_multi.cpp +++ b/tests/unit/test_svs_multi.cpp @@ -23,7 +23,6 @@ } \ } - static size_t EstimateBlockSize(const SVSParams ¶ms, size_t block_size, size_t num_elements) { const size_t reverse_edges_per_slot = SVSGraphBuilder::reverse_edges_element_size(); const size_t reverse_edges_slots = svs::lib::SegmentedVector(num_elements).capacity(); diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index d424d8124..abea602f4 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -1282,7 +1282,8 @@ TYPED_TEST(SVSTieredIndexTestBasic, markedDeleted) { ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 0); // Move vectors to the backend - while (mock_thread_pool.jobQ.size() > 0) mock_thread_pool.thread_iteration(); + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), n); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->indexSize(), n); From 0b9d5524a4a9c8d16fb1cf5546893bcb9fafdfb4 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 10 Sep 2026 01:08:17 -0700 Subject: [PATCH 28/99] cland-format-18 --- tests/unit/test_svs.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_svs.cpp b/tests/unit/test_svs.cpp index 953a95cc4..3105372b2 100644 --- a/tests/unit/test_svs.cpp +++ b/tests/unit/test_svs.cpp @@ -1583,8 +1583,7 @@ TYPED_TEST(SVSTest, test_svs_parameter_combinations_and_defaults) { // Test: All default parameters (zeros/unset) {"all_defaults", { - .dim = dim, - .metric = VecSimMetric_L2, + .dim = dim, .metric = VecSimMetric_L2, // All other parameters left as default (0/unset) }, {.quantBits = get<0>(svs_details::isSVSQuantBitsSupported(TypeParam::get_quant_bits())), @@ -1769,8 +1768,7 @@ TYPED_TEST(SVSTest, test_svs_parameter_consistency_across_metrics) { // Create index with default parameters for this metric SVSParams params = { - .dim = dim, - .metric = metric, + .dim = dim, .metric = metric, // All other parameters use defaults }; @@ -3371,9 +3369,7 @@ TEST(SVSTest, NumThreadsParamIgnored) { // Verify: creating an index without setting num_threads produces no warning. captured_log.clear(); SVSParams svs_params_default = { - .type = VecSimType_FLOAT32, - .dim = 4, - .metric = VecSimMetric_L2, + .type = VecSimType_FLOAT32, .dim = 4, .metric = VecSimMetric_L2, // num_threads left as 0 (default / unset) }; VecSimParams params_default{.algo = VecSimAlgo_SVS, From 1e243b23bec06ee127c77fba530ec05db3a91f14 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 10 Sep 2026 06:09:35 -0700 Subject: [PATCH 29/99] num_marked_deleted tracks not compacted entries now --- src/VecSim/algorithms/svs/svs.h | 3 +-- tests/unit/test_svs_tiered.cpp | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index c84751cf5..b7c19a1ed 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -325,8 +325,7 @@ class SVSIndex : public VecSimIndexAbstract, fl if (!ready()) return; - size_t n_consolidated = impl_->consolidate(labels); - num_marked_deleted.fetch_sub(n_consolidated, std::memory_order_relaxed); + impl_->consolidate(labels); } int deleteVectorImpl(const labelType label) { diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index abea602f4..63c69efe1 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -3096,6 +3096,7 @@ TYPED_TEST(SVSTieredIndexTest, writeInPlaceMode) { if (!TypeParam::isMulti()) { TEST_DATA_T overwritten_vec[] = {1, 1, 1, 1}; tiered_index->addVector(overwritten_vec, vec_label); + expected_marked_deleted++; ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 3); ASSERT_EQ(tiered_index->indexSize(), 3); ASSERT_EQ(tiered_index->indexLabelCount(), 2); @@ -3106,6 +3107,7 @@ TYPED_TEST(SVSTieredIndexTest, writeInPlaceMode) { // Validate that the vector is deleted and consolidated synchronously (in-place mode), // so no marked-deleted entry remains. tiered_index->deleteVector(vec_label); + expected_marked_deleted++; ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), TypeParam::isMulti() ? 2 : 3); ASSERT_EQ(tiered_index->indexLabelCount(), 1); From c02526c4f8c374ef619c5f849a072e630f0ae5a1 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Fri, 11 Sep 2026 05:06:49 -0700 Subject: [PATCH 30/99] clean-up and fix --- src/VecSim/algorithms/svs/svs.h | 29 ++---- src/VecSim/algorithms/svs/svs_tiered.h | 139 ++++++++----------------- src/VecSim/algorithms/svs/svs_utils.h | 1 - tests/unit/test_svs_tiered.cpp | 2 +- 4 files changed, 52 insertions(+), 119 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index b7c19a1ed..00afacc95 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -93,8 +93,6 @@ class SVSIndex : public VecSimIndexAbstract, fl using graph_builder_t = SVSGraphBuilder; using graph_type = typename graph_builder_t::graph_type; - // The concurrent index mirrors the upstream template signature and member surface, so - // the only change needed here is the namespace. using impl_type = std::conditional_t< isMulti, svs::concurrent::MultiMutableVamanaIndex, @@ -321,6 +319,8 @@ class SVSIndex : public VecSimIndexAbstract, fl } void consolidate(const std::vector &labels) override { + // There is documentation for consolidate(): + // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.consolidate std::shared_lock lock(this->pimplGuard_); if (!ready()) return; @@ -369,10 +369,9 @@ class SVSIndex : public VecSimIndexAbstract, fl if (indexLabelCount() == 0) { std::lock_guard lock(this->pimplGuard_); if (indexLabelCountUnsafe() == 0) { - { - setUnready(); - this->impl_.reset(); - } + setUnready(); + this->impl_.reset(); + num_marked_deleted.store(0, std::memory_order_relaxed); return; } @@ -657,19 +656,14 @@ class SVSIndex : public VecSimIndexAbstract, fl if (!ready()) { return std::numeric_limits::quiet_NaN(); } - { - std::shared_lock lock(this->pimplGuard_); - if (!impl_->has_id(label)) { - return std::numeric_limits::quiet_NaN(); - } + std::shared_lock lock(this->pimplGuard_); + if (!impl_->has_id(label)) { + return std::numeric_limits::quiet_NaN(); } auto query_datum = std::span{static_cast(vector_data), this->dim}; - { - std::shared_lock lock(this->pimplGuard_); - auto dist = impl_->get_distance(label, query_datum); - return toVecSimDistance(dist); - } + auto dist = impl_->get_distance(label, query_datum); + return toVecSimDistance(dist); } VecSimQueryReply *topKQuery(const void *queryBlob, size_t k, @@ -840,9 +834,6 @@ class SVSIndex : public VecSimIndexAbstract, fl void runGC() override { if (ready()) { std::shared_lock lock(this->pimplGuard_); - // There is documentation for consolidate(): - // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.consolidate - // impl_->consolidate(); // There is documentation for compact(): // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.compact impl_->compact(); diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index fd8b17239..80dcae8ea 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -11,6 +11,7 @@ #include #include #include +#include #include /** @@ -19,6 +20,7 @@ struct SVSInsertJob : public AsyncJob { labelType label; idType id; + std::atomic executing{false}; SVSInsertJob(std::shared_ptr allocator, labelType label_, idType id_, JobCallback insertCb, VecSimIndex *index_) @@ -37,14 +39,6 @@ struct SVSConsolidateJob : public AsyncJob { : AsyncJob(allocator, SVS_CONSOLIDATE_JOB, insertCb, index_), labels(labels_) {} }; -/** - * Definition of a job that launches GC on SVS Index. - */ -struct SVSGCJob : public AsyncJob { - SVSGCJob(std::shared_ptr allocator, JobCallback insertCb, VecSimIndex *index_) - : AsyncJob(allocator, SVS_GC_JOB, insertCb, index_) {} -}; - /** * @class SVSMultiThreadJob * @brief Represents a multi-threaded asynchronous job for the SVS algorithm. @@ -258,23 +252,6 @@ class TieredSVSIndex : public VecSimTieredIndex { using backend_index_t = VecSimIndexAbstract; using svs_index_t = SVSIndexBase; - // swaps_journal is used by updateSVSIndex() to track vectors swap operations that were done in - // the Flat index during SVS index updating. - // The journal contains tuples of (label, oldId, newId). - // oldId is the index of the label in flat index before the swap. - // newId is the index of the label in flat index after the swap. - // if oldId == newId, it means that the vector was not moved in the Flat index, but was removed - // from the end of the Flat index (no id swaps occurred internally). - // if label == SKIP_LABEL, it means that the vector was not moved in the Flat index, but - // updated in-place (hence was already removed and no need to remove it again) - using swap_record = std::tuple; - constexpr static size_t SKIP_LABEL = std::numeric_limits::max(); - std::vector swaps_journal; - // deleted_labels_journal is used by initSVSIndex() to track vectors that were deleted from - // Flat index during SVS index updating. The journal contains the deleted labels. These labels - // are used to delete the same vectors from the SVS index at the end of the update. - std::vector deleted_labels_journal; - size_t trainingTriggerThreshold; size_t updateTriggerThreshold; size_t updateJobWaitTime; @@ -726,37 +703,9 @@ class TieredSVSIndex : public VecSimTieredIndex { } private: - static void applySwapsToLabelsArray(std::vector &labels, - const std::vector &swaps) { - // Enumerate journal and reflect swaps in the labels. - // The journal contains tuples of (label, oldId, newId). - // oldId is the index of the label in flat index before the swap. - // newId is the index of the label in flat index after the swap. - for (const auto &p : swaps) { - auto oldId = std::get<1>(p); - auto newId = std::get<2>(p); - - if (oldId == newId || oldId >= labels.size()) { - // If oldId == newId, it means that the vector was not moved in the Flat index, - // but was removed or updated in-place. - // If oldId is out of bounds - new vector was added and swapped meanwhile. - // In both cases, we should not touch flat index at the new position. - if (newId < labels.size()) { - labels[newId] = SKIP_LABEL; - } - continue; // Next swap record. - } - - // Real swap case. - // If oldId != newId and oldId is in bounds, it means that the tracked vector was moved - // So, move the label to new position and skip the old position from deletions. - // NOTE: labels[oldId] can be SKIP_LABEL, but we still need to move it to newId. - labels[newId] = labels[oldId]; - labels[oldId] = SKIP_LABEL; - } - } - - idType setAndSaveInvalidJob(AsyncJob *job) { + idType setAndSaveInvalidJob(SVSInsertJob *job) { + // wait until insertion finishes + while (job->executing.load(std::memory_order_acquire)) {std::this_thread::yield();} this->invalidJobsLookupGuard.lock(); job->isValid = false; idType curInvalidId = currInvalidJobId++; @@ -787,10 +736,13 @@ class TieredSVSIndex : public VecSimTieredIndex { size_t data_size = this->frontendIndex->getStoredDataSize(); auto blob_copy = this->getAllocator()->allocate_unique(data_size); memcpy(blob_copy.get(), this->frontendIndex->getDataByInternalId(job->id), data_size); + + job->executing.store(true, std::memory_order_release); this->flatIndexGuard.unlock_shared(); svs_index->addVector(blob_copy.get(), job->label); + job->executing.store(false, std::memory_order_release); // Remove the vector and the insert job from the flat buffer. this->flatIndexGuard.lock(); // The job might have been invalidated due to overwrite in the meantime. In this case, @@ -882,23 +834,19 @@ class TieredSVSIndex : public VecSimTieredIndex { idType id = *it; auto label = this->frontendIndex->getVectorLabel(id); // Delete the vector from the frontend index if not in-place updated. - if (label != SKIP_LABEL) { - labelType last_vec_label = - this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); - int deleted = this->frontendIndex->deleteVectorById(label, id); - if (deleted && id != this->frontendIndex->indexSize()) { - // If the vector removal caused a swap with the last id, update the relevant - // insert job. - this->updateInsertJobInternalId(this->frontendIndex->indexSize(), id, - last_vec_label); - } - total_deleted += deleted; + labelType last_vec_label = + this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); + int deleted = this->frontendIndex->deleteVectorById(label, id); + if (deleted && id != this->frontendIndex->indexSize()) { + // If the vector removal caused a swap with the last id, update the relevant + // insert job. + this->updateInsertJobInternalId(this->frontendIndex->indexSize(), id, + last_vec_label); } + total_deleted += deleted; } - assert(total_deleted == - std::count_if(labels_to_move.begin(), labels_to_move.end(), - [](labelType label) { return label != SKIP_LABEL; }) && + assert(total_deleted == labels_to_move.size() && "Deleted vectors count does not match the number of labels to delete"); } // release frontend index executeTracingCallback("UpdateJob::after_add_to_svs"); @@ -936,9 +884,20 @@ class TieredSVSIndex : public VecSimTieredIndex { this->updateJobWaitTime = tiered_svs_params.updateJobWaitTime == 0 ? SVS_DEFAULT_UPDATE_JOB_WAIT_TIME : tiered_svs_params.updateJobWaitTime; + } + + ~TieredSVSIndex() { + // Delete all the pending insert jobs. + for (auto &jobs : this->labelToInsertJobs) { + for (auto *job : jobs.second) { + delete job; + } + } - // Reserve space for the journal to avoid reallocation. - this->swaps_journal.reserve(this->trainingTriggerThreshold); + // Delete all the pending invalid jobs. + for (auto &it : this->invalidJobs) { + delete it.second; + } } int addVector(const void *blob, labelType label) override { @@ -948,14 +907,11 @@ class TieredSVSIndex : public VecSimTieredIndex { // In-Place mode - add vector syncronously to the backend index. if (this->getWriteMode() == VecSim_WriteInPlace) { - // It is ok to lock everything at once for in-place mode, - // but we will have to unlock averything before calling initSVSIndexWrapper() - // so make the minimal needed lock here. // Backend index initialization data have to be buffered for proper // compression/training. if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { - // If backend index size is 0, first collect vectors in frontend index + // First collect vectors in frontend index // lock in scope to ensure that these will be released before // initSVSIndexWrapper() is called. { @@ -1106,17 +1062,15 @@ class TieredSVSIndex : public VecSimTieredIndex { void deleteAndUpdateInitIds(labelType label) { auto deleting_ids = this->frontendIndex->getElementIds(label); - // Sort deleting_ids by id descending order - std::sort(deleting_ids.begin(), deleting_ids.end(), - [](const auto &a, const auto &b) { return a > b; }); + std::sort(deleting_ids.begin(), deleting_ids.end()); // Delete vector from the frontend index. auto updated_ids = this->frontendIndex->deleteVectorAndGetUpdatedIds(label); assert(std::all_of(updated_ids.begin(), updated_ids.end(), [&deleting_ids](const auto &pair) { - return std::find(deleting_ids.begin(), deleting_ids.end(), - pair.first) != deleting_ids.end(); + return std::binary_search(deleting_ids.begin(), deleting_ids.end(), + pair.first); }) && "updated_ids should be a subset of deleting_ids"); @@ -1233,24 +1187,13 @@ class TieredSVSIndex : public VecSimTieredIndex { .updateTriggerThreshold = this->updateTriggerThreshold, .updateJobWaitTime = this->updateJobWaitTime, }; - { - // Use try_lock to avoid blocking the main thread during long-running - // training operations. initSVSIndexWrapper holds updateJobMutex for - // the entire training duration (which can take 40-85s on slow machines). - // If the mutex is held, training is actively running, so we report - // indexUpdateScheduled = true (BACKGROUND_INDEXING = 1). - std::unique_lock lock(this->updateJobMutex, std::try_to_lock); - if (lock.owns_lock()) { - svsTieredInfo.indexUpdateScheduled = this->indexUpdateScheduled.test(); - } else { - // Mutex is held by initSVSIndexWrapper — training is in progress. - svsTieredInfo.indexUpdateScheduled = true; - } - } + + svsTieredInfo.indexUpdateScheduled = + (info.tieredInfo.frontendCommonInfo.indexSize > 0) && + this->backendInitSubmited.load(std::memory_order_acquire); info.tieredInfo.specificTieredBackendInfo.svsTieredInfo = svsTieredInfo; - // Background indexing is in progress whenever the flat buffer is non-empty: every vector - // buffered there has either a pending per-vector insert job or a batch-init job that will - // drain it into the backend. + + // Background indexing is in progress whenever the flat buffer is non-empty info.tieredInfo.backgroundIndexing = info.tieredInfo.frontendCommonInfo.indexSize > 0 ? VecSimBool_TRUE : VecSimBool_FALSE; return info; diff --git a/src/VecSim/algorithms/svs/svs_utils.h b/src/VecSim/algorithms/svs/svs_utils.h index 670ea172c..232240ae9 100644 --- a/src/VecSim/algorithms/svs/svs_utils.h +++ b/src/VecSim/algorithms/svs/svs_utils.h @@ -375,7 +375,6 @@ struct SVSGraphBuilder { } // Per-node size of the concurrent graph's reverse-edge index. - // static constexpr size_t reverse_edges_element_size() { using reverse_edges_type = typename graph_type::reverse_edges_type; return sizeof(typename reverse_edges_type::list_type) + sizeof(svs::concurrent::SpinLock); diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 63c69efe1..47d69004b 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -3303,7 +3303,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, runGCAPI) { auto jobs_before_gc = mock_thread_pool.jobQ.size(); // Run the GC API call, expect that we will clean up the SVS index. VecSimTieredIndex_GC(tiered_index); - // Expected that a single GC job was added to the queue. + // Expected that a multi-thread GC job was added to the queue. ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + mock_thread_pool.thread_pool_size); // Run GC twice. VecSimTieredIndex_GC(tiered_index); From 7595df0eb51b498c7d544ec602ee6c1894b09c01 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Fri, 11 Sep 2026 11:16:17 -0700 Subject: [PATCH 31/99] update SVS submodule reference --- deps/ScalableVectorSearch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch index 7786d43b9..389597b59 160000 --- a/deps/ScalableVectorSearch +++ b/deps/ScalableVectorSearch @@ -1 +1 @@ -Subproject commit 7786d43b98ac9769ad7668d0d4896143cfb2f167 +Subproject commit 389597b59cbfe87c6887da262673ef386261947e From 0d15167669328bbaf9355b0b28df9414c67ecec5 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Fri, 11 Sep 2026 12:22:25 -0700 Subject: [PATCH 32/99] Fix semantic fallout from the main merge Three things the merge could not resolve textually, because both sides changed code the other side never saw. test_svs_tiered.cpp: #1033's new getDataByLabelReadsSvsBackendWhenUncompressed calls scheduleSVSIndexUpdate(), renamed to scheduleSVSIndexInit() on this branch. Git merged the call site cleanly and the build broke. A rename is enough -- CreateIndexInstance already uses the same add / schedule / single thread_iteration shape against the init-job model, so one iteration does complete the ingest. The other two tests #1033 added need no change: one never reaches the backend, and the compressed-split-label one's premises still hold here (threshold 1 latches backendInitSubmited on the first add, and the second add is buffered because the flat index is below its bound). svs.h: #1033's appendStoredDataByLabel guards impl_ with a bare null check, which was sufficient on main -- main has no pimplGuard_. Here every other const reader of impl_ takes it shared, and markIndexUpdate takes it exclusively to reset impl_ once the last label is deleted. The spans append_datum copies from point into the dataset, so a getDataByLabel racing that delete walked a destroyed index; mainIndexGuard in the tiered caller does not cover this, since backend writes on this branch deliberately do not take it. Now takes pimplGuard_ shared, testing impl_ under the lock rather than before it -- neither caller holds the guard, so there is no recursive acquisition. Also updates a comment naming updateSVSIndexWrapper. Note for follow-up, not fixed here: the pre-existing readers (isLabelExists, getDistanceFrom_Unsafe) test ready() *outside* pimplGuard_ and dereference impl_ after acquiring it, so a reset landing in that window still null-derefs. That predates the merge and belongs with the other lock-free review items. Co-Authored-By: Claude Opus 5 --- src/VecSim/algorithms/svs/svs.h | 5 +++++ tests/unit/test_svs_tiered.cpp | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 574db33da..d52ec7be6 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -849,6 +849,11 @@ class SVSIndex : public VecSimIndexAbstract, fl template void appendStoredDataByLabel(labelType label, std::vector> &vectors_output) const { + // The spans handed to `append_datum` point straight into the dataset, so `impl_` has to + // stay alive for the whole walk: `markIndexUpdate` takes `pimplGuard_` exclusively to + // reset it once the last label is gone. Test it under the lock rather than before it -- a + // check outside the lock can pass and then block until after the reset. + std::shared_lock lock(this->pimplGuard_); if (!impl_) { return; } diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 81c5bdb0c..4b817d472 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -407,7 +407,7 @@ TYPED_TEST(SVSTieredIndexTest, TestDebugInfoThreadCountWriteInPlace) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); } // Verify: numThreads (pool size) unchanged, lastReservedThreads (parallelism) reflects - // we only used one thread (write-in-place calls updateSVSIndexWrapper with availableThreads=1). + // we only used one thread (write-in-place calls initSVSIndexWrapper with availableThreads=1). backendIndexInfo = tiered_index->GetBackendIndex()->debugInfo(); ASSERT_EQ(backendIndexInfo.svsInfo.numThreads, num_threads); ASSERT_EQ(backendIndexInfo.svsInfo.lastReservedThreads, 1); @@ -549,7 +549,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, getDataByLabelReadsSvsBackendWhenUncompresse ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 1); // Ingest it into the SVS backend. - tiered_index->scheduleSVSIndexUpdate(); + tiered_index->scheduleSVSIndexInit(); mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 1); From 2ab67cc436b266ae285930db319c55f6f9ac231e Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Fri, 11 Sep 2026 12:22:46 -0700 Subject: [PATCH 33/99] Satisfy check-format The spin-wait added in "clean-up and fix" kept its body on the same line as the `while`. Only violation in the tree under clang-format-18, the version CI installs via llvm.sh on ubuntu-24.04. Co-Authored-By: Claude Opus 5 --- src/VecSim/algorithms/svs/svs_tiered.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 80dcae8ea..1e1783cca 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -705,7 +705,9 @@ class TieredSVSIndex : public VecSimTieredIndex { private: idType setAndSaveInvalidJob(SVSInsertJob *job) { // wait until insertion finishes - while (job->executing.load(std::memory_order_acquire)) {std::this_thread::yield();} + while (job->executing.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } this->invalidJobsLookupGuard.lock(); job->isValid = false; idType curInvalidId = currInvalidJobId++; From 5bf9f6a7de9948348faadd40dc5887f746a0f5f9 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 14 Sep 2026 00:20:22 -0700 Subject: [PATCH 34/99] improve svs pimpl guarding for concurent execution --- src/VecSim/algorithms/svs/svs.h | 264 ++++++++---------- .../algorithms/svs/svs_batch_iterator.h | 13 +- .../algorithms/svs/svs_serializer_impl.h | 33 ++- 3 files changed, 148 insertions(+), 162 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 00afacc95..a9b608697 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include "svs/index/vamana/dynamic_index.h" @@ -118,15 +120,21 @@ class SVSIndex : public VecSimIndexAbstract, fl // SVS thread pool VecSimSVSThreadPool threadpool_; svs::logging::logger_ptr logger_; - // SVS Index implementation instance - std::unique_ptr impl_; + // SVS Index implementation instance. + // initImpl() needs at least one point to compute an entry point. + // so impl_ is null until the first insert + std::shared_ptr impl_; + mutable std::shared_mutex pimplGuard_; + // Serialize mutators of the impl_ pointer (exclusive) + // vs add_points() (shared) to avoid silent drops of additions + // Lock order: implMutationGuard_ -> pimplGuard_. + std::shared_mutex implMutationGuard_; - std::atomic impl_ready_{false}; - - void setReady() { this->impl_ready_.store(true, std::memory_order_release); } - - void setUnready() { this->impl_ready_.store(false, std::memory_order_release); } + std::shared_ptr getImpl() const { + std::shared_lock lock(this->pimplGuard_); + return this->impl_; + } static double toVecSimDistance(float v) { return svs_details::toVecSimDistance(v); } @@ -169,7 +177,7 @@ class SVSIndex : public VecSimIndexAbstract, fl // Create SVS index instance with initial data // Data should not be empty template - std::unique_ptr initImpl(const Dataset &points, + std::shared_ptr initImpl(const Dataset &points, std::span ids) const { svs::threads::ThreadPoolHandle threadpool_handle{VecSimSVSThreadPool{threadpool_}}; @@ -190,7 +198,7 @@ class SVSIndex : public VecSimIndexAbstract, fl this->blockSize, this->getAllocator(), logger_); // Create SVS MutableIndex instance - auto impl = std::make_unique(std::move(graph), std::move(data), entry_point, + auto impl = std::make_shared(std::move(graph), std::move(data), entry_point, std::move(distance), ids, threadpool_, logger_); // Set SVS MutableIndex build parameters to be used in future updates @@ -233,8 +241,8 @@ class SVSIndex : public VecSimIndexAbstract, fl // Handler to manage SVS implementation instance struct SVSImplHandler : public SVSIndexBase::ImplHandler { - std::unique_ptr impl; - SVSImplHandler(std::unique_ptr impl) : impl{std::move(impl)} {} + std::shared_ptr impl; + SVSImplHandler(std::shared_ptr impl) : impl{std::move(impl)} {} }; std::unique_ptr createImpl(const void *vectors_data, const labelType *labels, @@ -254,20 +262,18 @@ class SVSIndex : public VecSimIndexAbstract, fl } void setImpl(std::unique_ptr handler) override { - if (ready()) { - throw std::logic_error("SVSIndex::setImpl called on non-empty impl_"); - } - SVSImplHandler *svs_handler = dynamic_cast(handler.get()); if (!svs_handler) { throw std::logic_error("Failed to cast to SVSImplHandler"); } - { - std::lock_guard lock(this->pimplGuard_); - this->impl_ = std::move(svs_handler->impl); - if (this->impl_) - setReady(); + + std::lock_guard replace_lock(this->implMutationGuard_); + if (getImpl()) { + throw std::logic_error("SVSIndex::setImpl called on non-empty impl_"); } + + std::lock_guard lock(this->pimplGuard_); + this->impl_ = std::move(svs_handler->impl); } // Assuming parallelism was updated to reflect the number of available threads before this @@ -290,29 +296,27 @@ class SVSIndex : public VecSimIndexAbstract, fl auto points = svs::data::SimpleDataView{typed_vectors_data, n, this->dim}; if constexpr (!isMulti) { - if (ready()) { - // SVS index does not support overriding vectors with the same label - // so we have to delete them first if needed - deleted_num = deleteVectorsImpl(labels, n); - } + // SVS index does not support overriding vectors with the same label + // so we have to delete them first if needed + deleted_num = deleteVectorsImpl(labels, n); } - this->pimplGuard_.lock_shared(); - if (!ready()) { - this->pimplGuard_.unlock_shared(); - // SVS index instance cannot be empty, so we have to construct it at first rows - std::lock_guard lock(this->pimplGuard_); - if (!ready()) { - impl_ = initImpl(points, ids); - assert(this->impl_ != nullptr); - setReady(); + this->implMutationGuard_.lock_shared(); + if (auto impl = getImpl()) { + impl->add_points(points, ids, /*reuse_empty*/ false); + this->implMutationGuard_.unlock_shared(); + } else { + this->implMutationGuard_.unlock_shared(); + this->implMutationGuard_.lock(); + if (auto existing = getImpl()) { + existing->add_points(points, ids, /*reuse_empty*/ false); } else { - impl_->add_points(points, ids, /*reuse_empty*/ false); + auto built = initImpl(points, ids); + assert(built != nullptr); + std::lock_guard lock(this->pimplGuard_); + this->impl_ = std::move(built); } - } else { - // Add new points to existing SVS index - impl_->add_points(points, ids, /*reuse_empty*/ false); - this->pimplGuard_.unlock_shared(); + this->implMutationGuard_.unlock(); } return n - deleted_num; @@ -321,57 +325,54 @@ class SVSIndex : public VecSimIndexAbstract, fl void consolidate(const std::vector &labels) override { // There is documentation for consolidate(): // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.consolidate - std::shared_lock lock(this->pimplGuard_); - if (!ready()) - return; - - impl_->consolidate(labels); + if (auto impl = getImpl()) { + impl->consolidate(labels); + } } int deleteVectorImpl(const labelType label) { - if (indexLabelCount() == 0) { + auto impl = getImpl(); + if (!impl) { return 0; } - int deleted_num = 0; - { - std::shared_lock lock(this->pimplGuard_); - deleted_num = impl_->delete_entries(std::span{&label, 1}); - } - + const int deleted_num = impl->delete_entries(std::span{&label, 1}); this->markIndexUpdate(deleted_num); return deleted_num; } int deleteVectorsImpl(const labelType *labels, size_t n) { - if (indexLabelCount() == 0) { + auto impl = getImpl(); + if (!impl) { return 0; } - int deleted_num = 0; - { - std::shared_lock lock(this->pimplGuard_); - deleted_num = impl_->delete_entries(std::span{labels, n}); - } - - if (deleted_num > 0) + const int deleted_num = impl->delete_entries(std::span{labels, n}); + if (deleted_num > 0) { this->markIndexUpdate(deleted_num); - + } return deleted_num; } - // Count deletions and consolidate index if needed + // Count deletions and drop the index instance if it became empty void markIndexUpdate(size_t n = 1) { - if (!ready()) + auto impl = getImpl(); + if (!impl) { return; + } // SVS index instance should not be empty - if (indexLabelCount() == 0) { - std::lock_guard lock(this->pimplGuard_); - if (indexLabelCountUnsafe() == 0) { - setUnready(); - this->impl_.reset(); - + if (labelCountOf(*impl) == 0) { + std::lock_guard replace_lock(this->implMutationGuard_); + if (getImpl() != impl) { + // The instance was already droped + return; + } + if (labelCountOf(*impl) == 0) { + { + std::lock_guard lock(this->pimplGuard_); + this->impl_.reset(); + } num_marked_deleted.store(0, std::memory_order_relaxed); return; } @@ -380,6 +381,14 @@ class SVSIndex : public VecSimIndexAbstract, fl num_marked_deleted.fetch_add(n, std::memory_order_relaxed); } + static size_t labelCountOf(const impl_type &impl) { + if constexpr (isMulti) { + return impl.labelcount(); + } else { + return impl.size(); + } + } + bool isTwoLevelLVQ(const VecSimSvsQuantBits &qbits) { switch (qbits) { case VecSimSvsQuant_4x4: @@ -416,67 +425,29 @@ class SVSIndex : public VecSimIndexAbstract, fl ~SVSIndex() = default; - bool ready() const override { return this->impl_ready_.load(std::memory_order_acquire); } + bool ready() const override { return getImpl() != nullptr; } size_t indexSize() const override { return indexStorageSize(); } size_t indexStorageSize() const override { - std::shared_lock lock(this->pimplGuard_); - if (ready()) { - return impl_->view_data().size(); - } else { - return 0; - } + auto impl = getImpl(); + return impl ? impl->view_data().size() : 0; } size_t indexCapacity() const override { - std::shared_lock lock(this->pimplGuard_); - if (ready()) { - return storage_traits_t::storage_capacity(impl_->view_data()); - } else { - return 0; - } - } - - size_t indexLabelCountUnsafe() const { - if constexpr (isMulti) { - if (ready()) { - return impl_->labelcount(); - } else { - return 0; - } - } else { - if (ready()) { - return impl_->size(); - } else { - return 0; - } - } + auto impl = getImpl(); + return impl ? storage_traits_t::storage_capacity(impl->view_data()) : 0; } size_t indexLabelCount() const override { - if constexpr (isMulti) { - if (ready()) { - std::shared_lock lock(this->pimplGuard_); - return impl_->labelcount(); - } else { - return 0; - } - } else { - if (ready()) { - std::shared_lock lock(this->pimplGuard_); - return impl_->size(); - } else { - return 0; - } - } + auto impl = getImpl(); + return impl ? labelCountOf(*impl) : 0; } vecsim_stl::set getLabelsSet() const override { vecsim_stl::set labels(this->allocator); - if (ready()) { - std::shared_lock lock(this->pimplGuard_); - impl_->on_ids([&labels](size_t label) { labels.insert(label); }); + if (auto impl = getImpl()) { + impl->on_ids([&labels](size_t label) { labels.insert(label); }); } return labels; } @@ -633,12 +604,8 @@ class SVSIndex : public VecSimIndexAbstract, fl } bool isLabelExists(labelType label) const override { - if (ready()) { - std::shared_lock lock(this->pimplGuard_); - return impl_->has_id(label); - } else { - return false; - } + auto impl = getImpl(); + return impl ? impl->has_id(label) : false; } size_t getParallelism() const override { return threadpool_.getParallelism(); } @@ -653,16 +620,13 @@ class SVSIndex : public VecSimIndexAbstract, fl } double getDistanceFrom_Unsafe(labelType label, const void *vector_data) const override { - if (!ready()) { - return std::numeric_limits::quiet_NaN(); - } - std::shared_lock lock(this->pimplGuard_); - if (!impl_->has_id(label)) { + auto impl = getImpl(); + if (!impl || !impl->has_id(label)) { return std::numeric_limits::quiet_NaN(); } auto query_datum = std::span{static_cast(vector_data), this->dim}; - auto dist = impl_->get_distance(label, query_datum); + auto dist = impl->get_distance(label, query_datum); return toVecSimDistance(dist); } @@ -670,12 +634,17 @@ class SVSIndex : public VecSimIndexAbstract, fl VecSimQueryParams *queryParams) const override { auto rep = new VecSimQueryReply(this->allocator); this->lastMode = STANDARD_KNN; - if (k == 0 || this->indexLabelCount() == 0) { + auto impl = getImpl(); + if (k == 0 || !impl) { return rep; } { + const auto label_count = labelCountOf(*impl); + if (label_count == 0) { + return rep; + } // limit result size to index size - k = std::min(k, this->indexLabelCount()); + k = std::min(k, label_count); auto processed_query_ptr = this->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); @@ -683,14 +652,13 @@ class SVSIndex : public VecSimIndexAbstract, fl auto query = svs::data::ConstSimpleDataView{ static_cast(processed_query), 1, this->dim}; auto result = svs::QueryResult{query.size(), k}; - std::shared_lock lock(this->pimplGuard_); - auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams, + auto sp = svs_details::joinSearchParams(impl->get_search_parameters(), queryParams, is_two_level_lvq); auto timeoutCtx = queryParams ? queryParams->timeoutCtx : nullptr; auto cancel = [timeoutCtx]() { return VECSIM_TIMEOUT(timeoutCtx); }; - impl_->search(result.view(), query, sp, cancel); + impl->search(result.view(), query, sp, cancel); if (cancel()) { rep->code = VecSim_QueryReply_TimedOut; return rep; @@ -717,12 +685,11 @@ class SVSIndex : public VecSimIndexAbstract, fl VecSimQueryParams *queryParams) const override { auto rep = new VecSimQueryReply(this->allocator); this->lastMode = RANGE_QUERY; - if (radius == 0 || this->indexLabelCount() == 0) { + auto impl = getImpl(); + if (radius == 0 || !impl || labelCountOf(*impl) == 0) { return rep; } { - std::shared_lock lock(this->pimplGuard_); - auto timeoutCtx = queryParams ? queryParams->timeoutCtx : nullptr; auto cancel = [timeoutCtx]() { return VECSIM_TIMEOUT(timeoutCtx); }; @@ -733,7 +700,7 @@ class SVSIndex : public VecSimIndexAbstract, fl this->dim}; // Base search parameters for the SVS iterator schedule. - auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams, + auto sp = svs_details::joinSearchParams(impl->get_search_parameters(), queryParams, is_two_level_lvq); // SVS BatchIterator handles the search in batches // The batch size is set to the index search window size by default @@ -741,7 +708,7 @@ class SVSIndex : public VecSimIndexAbstract, fl // Create SVS BatchIterator for range search // Search result is cached in the iterator and can be accessed by the user - auto svs_it = impl_->make_batch_iterator(query); + auto svs_it = impl->make_batch_iterator(query); svs_it.next(batch_size, cancel); if (cancel()) { rep->code = VecSim_QueryReply_TimedOut; @@ -793,12 +760,14 @@ class SVSIndex : public VecSimIndexAbstract, fl // take ownership of the blob copy and pass it to the batch iterator. auto *queryBlobCopyPtr = queryBlobCopy.release(); // Ownership of queryBlobCopy moves to VecSimBatchIterator that will free it at the end. - if (indexLabelCount() == 0) { + auto impl = getImpl(); + if (!impl || labelCountOf(*impl) == 0) { return new (this->getAllocator()) NullSVS_BatchIterator(queryBlobCopyPtr, queryParams, this->getAllocator()); } else { return new (this->getAllocator()) SVS_BatchIterator( - queryBlobCopyPtr, impl_.get(), queryParams, this->getAllocator(), is_two_level_lvq); + queryBlobCopyPtr, std::move(impl), queryParams, this->getAllocator(), + is_two_level_lvq); } } @@ -832,11 +801,10 @@ class SVSIndex : public VecSimIndexAbstract, fl } void runGC() override { - if (ready()) { - std::shared_lock lock(this->pimplGuard_); + if (auto impl = getImpl()) { // There is documentation for compact(): // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.compact - impl_->compact(); + impl->compact(); } num_marked_deleted.store(0, std::memory_order_relaxed); } @@ -865,14 +833,18 @@ class SVSIndex : public VecSimIndexAbstract, fl } else { std::vector> vectors_output; + auto impl = getImpl(); + if (!impl) { + return vectors_output; + } if constexpr (isMulti) { // Multi-index case: get all vectors for this label - auto it = impl_->get_label_to_external_lookup().find(label); - if (it != impl_->get_label_to_external_lookup().end()) { + auto it = impl->get_label_to_external_lookup().find(label); + if (it != impl->get_label_to_external_lookup().end()) { const auto &external_ids = it->second; for (auto external_id : external_ids) { - auto indexed_span = impl_->get_parent_index().get_datum(external_id); + auto indexed_span = impl->get_parent_index().get_datum(external_id); // For uncompressed data, indexed_span should be a simple span const char *data_ptr = reinterpret_cast(indexed_span.data()); @@ -883,7 +855,7 @@ class SVSIndex : public VecSimIndexAbstract, fl } } else { // Single-index case - auto indexed_span = impl_->get_datum(label); + auto indexed_span = impl->get_datum(label); // For uncompressed data, indexed_span should be a simple span const char *data_ptr = reinterpret_cast(indexed_span.data()); diff --git a/src/VecSim/algorithms/svs/svs_batch_iterator.h b/src/VecSim/algorithms/svs/svs_batch_iterator.h index 116546150..a31c02397 100644 --- a/src/VecSim/algorithms/svs/svs_batch_iterator.h +++ b/src/VecSim/algorithms/svs/svs_batch_iterator.h @@ -29,6 +29,9 @@ class SVS_BatchIterator : public VecSimBatchIterator { using dist_type = typename Index::distance_type; size_t dim; + // Declared before impl_ so that impl_, which holds a raw back-pointer into the index, is + // destroyed first. + std::shared_ptr index_; impl_type impl_; typename impl_type::const_iterator curr_it; size_t batch_size; @@ -67,14 +70,16 @@ class SVS_BatchIterator : public VecSimBatchIterator { } public: - SVS_BatchIterator(void *query_vector, const Index *index, const VecSimQueryParams *queryParams, + SVS_BatchIterator(void *query_vector, std::shared_ptr index, + const VecSimQueryParams *queryParams, std::shared_ptr allocator, bool is_two_level_lvq) : VecSimBatchIterator{query_vector, queryParams ? queryParams->timeoutCtx : nullptr, std::move(allocator)}, - dim{index->dimensions()}, impl_{index->make_batch_iterator(std::span{ - static_cast(query_vector), dim})}, + dim{index->dimensions()}, index_{std::move(index)}, + impl_{index_->make_batch_iterator( + std::span{static_cast(query_vector), dim})}, curr_it{impl_.begin()} { - auto sp = svs_details::joinSearchParams(index->get_search_parameters(), queryParams, + auto sp = svs_details::joinSearchParams(index_->get_search_parameters(), queryParams, is_two_level_lvq); batch_size = queryParams && queryParams->batchSize ? queryParams->batchSize diff --git a/src/VecSim/algorithms/svs/svs_serializer_impl.h b/src/VecSim/algorithms/svs/svs_serializer_impl.h index 5e551ef81..33fc6ca53 100644 --- a/src/VecSim/algorithms/svs/svs_serializer_impl.h +++ b/src/VecSim/algorithms/svs/svs_serializer_impl.h @@ -76,7 +76,11 @@ template void SVSIndex::impl_save( const std::string &location) { - impl_->save(location + "/config", location + "/graph", location + "/data"); + auto impl = getImpl(); + if (!impl) { + throw std::runtime_error("SVSIndex::impl_save called on an empty index"); + } + impl->save(location + "/config", location + "/graph", location + "/data"); } // This function will load the serialized svs index from the given folder path @@ -92,6 +96,7 @@ void SVSIndex // Verify metadata compatibility, will throw runtime exception if not compatible compareMetadataFile(folder_path + "/metadata"); + std::shared_ptr loaded_impl; if constexpr (isMulti) { auto loaded = svs::concurrent::auto_multi_dynamic_assemble( folder_path + "/config", @@ -101,7 +106,7 @@ void SVSIndex this->getAllocator())), distance_f(), std::move(threadpool_handle), svs::concurrent::MultiMutableVamanaLoad::FROM_MULTI, logger_); - impl_ = std::make_unique(std::move(loaded)); + loaded_impl = std::make_shared(std::move(loaded)); } else { auto loaded = svs::concurrent::auto_dynamic_assemble( folder_path + "/config", @@ -110,9 +115,12 @@ void SVSIndex SVS_LAZY(storage_traits_t::load(folder_path + "/data", this->blockSize, this->dim, this->getAllocator())), distance_f(), std::move(threadpool_handle), false, logger_); - impl_ = std::make_unique(std::move(loaded)); + loaded_impl = std::make_shared(std::move(loaded)); } - setReady(); + + std::lock_guard replace_lock(this->implMutationGuard_); + std::lock_guard lock(this->pimplGuard_); + impl_ = std::move(loaded_impl); } template bool SVSIndex::checkIntegrity() const { - if (!impl_) { + auto impl = getImpl(); + if (!impl) { throw std::runtime_error( "SVSIndex integrity check failed: index implementation (impl_) is null."); } @@ -171,9 +180,9 @@ bool SVSIndex try { // SVS internal index integrity validation if constexpr (isMulti) { - impl_->get_parent_index().debug_check_invariants(true); + impl->get_parent_index().debug_check_invariants(true); } else { - impl_->debug_check_invariants(true); + impl->debug_check_invariants(true); } } // debug_check_invariants throws svs::lib::ANNException : public std::runtime_error in case of @@ -183,10 +192,10 @@ bool SVSIndex } try { - size_t index_size = impl_->size(); - size_t storage_size = impl_->view_data().size(); - size_t capacity = storage_traits_t::storage_capacity(impl_->view_data()); - size_t label_count = this->indexLabelCount(); + size_t index_size = impl->size(); + size_t storage_size = impl->view_data().size(); + size_t capacity = storage_traits_t::storage_capacity(impl->view_data()); + size_t label_count = labelCountOf(*impl); // Storage size must match index size if (storage_size != index_size) { @@ -204,7 +213,7 @@ bool SVSIndex bool label_validation_passed = true; try { - impl_->on_ids([&](size_t label) { labels_counted++; }); + impl->on_ids([&](size_t label) { labels_counted++; }); // Validate label count consistency label_validation_passed = (labels_counted == label_count); From b50cd0d6fee6d0cb88d9fe5fd74334e460958ca5 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 14 Sep 2026 01:32:34 -0700 Subject: [PATCH 35/99] format --- src/VecSim/algorithms/svs/svs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 7a87fcc45..cfb000957 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -125,7 +125,7 @@ class SVSIndex : public VecSimIndexAbstract, fl // initImpl() needs at least one point to compute an entry point. // so impl_ is null until the first insert std::shared_ptr impl_; - + mutable std::shared_mutex pimplGuard_; // Serialize mutators of the impl_ pointer (exclusive) // vs add_points() (shared) to avoid silent drops of additions From 17b848d7a3ec452ae67622638c252e1935a53fbb Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 14 Sep 2026 01:48:44 -0700 Subject: [PATCH 36/99] fix leaking consolidate jobs --- src/VecSim/algorithms/svs/svs_tiered.h | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 1e1783cca..543a7105d 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -276,6 +276,9 @@ class TieredSVSIndex : public VecSimTieredIndex { idType currInvalidJobId; // A unique arbitrary identifier for accessing invalid jobs std::mutex invalidJobsLookupGuard; + vecsim_stl::unordered_set pendingConsolidateJobs; + std::mutex consolidateJobsGuard; + size_t flat_buffer_bound; /// @@ -649,6 +652,10 @@ class TieredSVSIndex : public VecSimTieredIndex { auto svs_index = index->GetSVSIndex(); svs_index->setParallelism(1); svs_index->consolidate(consolidate_job->labels); + { + std::lock_guard jobs_lock(index->consolidateJobsGuard); + index->pendingConsolidateJobs.erase(job); + } delete job; } @@ -698,6 +705,10 @@ class TieredSVSIndex : public VecSimTieredIndex { AsyncJob *new_consolidate_job = new (this->allocator) SVSConsolidateJob(this->allocator, {label}, SVSIndexConsolidateWrapper, this); + { + std::lock_guard lock(this->consolidateJobsGuard); + this->pendingConsolidateJobs.insert(new_consolidate_job); + } // Insert job to the queue. this->submitSingleJob(new_consolidate_job); } @@ -860,7 +871,8 @@ class TieredSVSIndex : public VecSimTieredIndex { std::shared_ptr allocator) : Base(svs_index, bf_index, tiered_index_params, allocator), uncompletedJobs(this->allocator), labelToInsertJobs(this->allocator), - invalidJobs(this->allocator), currInvalidJobId(0) { + invalidJobs(this->allocator), currInvalidJobId(0), + pendingConsolidateJobs(this->allocator) { const auto &tiered_svs_params = tiered_index_params.specificParams.tieredSVSParams; // If flatBufferLimit is not initialized (0), use the default update threshold. @@ -900,6 +912,11 @@ class TieredSVSIndex : public VecSimTieredIndex { for (auto &it : this->invalidJobs) { delete it.second; } + + // Delete all the pending consolidate jobs + for (auto *job : this->pendingConsolidateJobs) { + delete job; + } } int addVector(const void *blob, labelType label) override { From a5cb99d4b8bd69e155cba307241069bfc839ebb9 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 14 Sep 2026 02:32:02 -0700 Subject: [PATCH 37/99] resubmit insert job if backend isn't ready --- src/VecSim/algorithms/svs/svs_tiered.h | 28 ++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 543a7105d..1a5f3d133 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -597,9 +597,17 @@ class TieredSVSIndex : public VecSimTieredIndex { static void executeInsertJobWrapper(AsyncJob *job) { auto *insert_job = static_cast(job); auto *job_index = static_cast *>(insert_job->index); - // prevent parallel execution with index initilizing job - std::shared_lock lock(job_index->updateJobMutex); - job_index->executeInsertJob(insert_job); + InsertJobOutcome outcome; + { + // prevent parallel execution with index initilizing job + std::shared_lock lock(job_index->updateJobMutex); + outcome = job_index->executeInsertJob(insert_job); + } + + if (outcome == InsertJobOutcome::Deferred) { + job_index->submitSingleJob(job); + return; + } delete job; } @@ -727,9 +735,10 @@ class TieredSVSIndex : public VecSimTieredIndex { return curInvalidId; } - void executeInsertJob(SVSInsertJob *job) { + enum class InsertJobOutcome { Completed, Deferred }; + + InsertJobOutcome executeInsertJob(SVSInsertJob *job) { auto svs_index = GetSVSIndex(); - assert(svs_index->ready()); // Note that accessing the job fields should occur with flat index guard held (here and // later). @@ -741,7 +750,13 @@ class TieredSVSIndex : public VecSimTieredIndex { this->invalidJobsLookupGuard.lock(); this->invalidJobs.erase(job->id); this->invalidJobsLookupGuard.unlock(); - return; + return InsertJobOutcome::Completed; + } + + // Backend is still initilizing. Resubmit the job + if (!svs_index->ready() && this->indexUpdateScheduled.test()) { + this->flatIndexGuard.unlock_shared(); + return InsertJobOutcome::Deferred; } // Copy the vector blob out of the flat buffer while holding flatIndexGuard, so we @@ -794,6 +809,7 @@ class TieredSVSIndex : public VecSimTieredIndex { this->invalidJobsLookupGuard.unlock(); } this->flatIndexGuard.unlock(); + return InsertJobOutcome::Completed; } void initSVSIndex(size_t availableThreads) { From e3c0cbf38e6ac55f2d1aefea0eb3f8c897f6de88 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Mon, 14 Sep 2026 05:34:05 -0700 Subject: [PATCH 38/99] use RAII lock to harmonize topKQuery and rangeQuery --- src/VecSim/algorithms/svs/svs_tiered.h | 85 ++------------------------ src/VecSim/vec_sim_tiered_index.h | 37 +++++++---- 2 files changed, 30 insertions(+), 92 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 1a5f3d133..75a54d706 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -568,6 +568,12 @@ class TieredSVSIndex : public VecSimTieredIndex { } #endif +protected: + std::shared_lock lockMainIndexForQuery() const override { + // returns a no-op lock because SVS does its locking internally + return std::shared_lock{}; + } + private: /** * @brief Init the SVS index in a thread-safe manner. @@ -1270,85 +1276,6 @@ class TieredSVSIndex : public VecSimTieredIndex { return infoIterator; } - VecSimQueryReply *topKQuery(const void *queryBlob, size_t k, - VecSimQueryParams *queryParams) const override { - // SVS handles its own internal locking for concurrent search + modification, - // so we don't need mainIndexGuard for backend queries. - this->flatIndexGuard.lock_shared(); - - if (this->frontendIndex->indexSize() == 0) { - this->flatIndexGuard.unlock_shared(); - - auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); - const void *processed_query = processed_query_ptr.get(); - return this->backendIndex->topKQuery(processed_query, k, queryParams); - } else { - auto flat_results = this->frontendIndex->topKQuery(queryBlob, k, queryParams); - this->flatIndexGuard.unlock_shared(); - - if (flat_results->code != VecSim_QueryReply_OK) { - assert(flat_results->results.empty()); - return flat_results; - } - - auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); - const void *processed_query = processed_query_ptr.get(); - auto main_results = this->backendIndex->topKQuery(processed_query, k, queryParams); - - if (main_results->code != VecSim_QueryReply_OK) { - VecSimQueryReply_Free(flat_results); - assert(main_results->results.empty()); - return main_results; - } - - return merge_result_lists(main_results, flat_results, k); - } - } - - VecSimQueryReply *rangeQuery(const void *queryBlob, double radius, - VecSimQueryParams *queryParams, - VecSimQueryReply_Order order) const override { - // SVS handles its own internal locking for concurrent search + modification, - // so we don't need mainIndexGuard for backend queries. - this->flatIndexGuard.lock_shared(); - - if (this->frontendIndex->indexSize() == 0) { - this->flatIndexGuard.unlock_shared(); - - auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); - const void *processed_query = processed_query_ptr.get(); - auto res = this->backendIndex->rangeQuery(processed_query, radius, queryParams); - sort_results(res, order); - return res; - } else { - auto flat_results = this->frontendIndex->rangeQuery(queryBlob, radius, queryParams); - this->flatIndexGuard.unlock_shared(); - - if (flat_results->code != VecSim_QueryReply_OK) { - return flat_results; - } - - auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); - const void *processed_query = processed_query_ptr.get(); - auto main_results = - this->backendIndex->rangeQuery(processed_query, radius, queryParams); - - if (BY_SCORE == order) { - sort_results_by_score_then_id(main_results); - sort_results_by_score_then_id(flat_results); - - auto code = main_results->code; - VecSimQueryReply *ret = merge_result_lists(main_results, flat_results, -1); - ret->code = code; - return ret; - } else { // BY_ID - concat_results(main_results, flat_results); - filter_results_by_id(main_results); - return main_results; - } - } - } - VecSimBatchIterator *newBatchIterator(const void *queryBlob, VecSimQueryParams *queryParams) const override { // The query blob will be processed and copied by the internal indexes's batch iterator. diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 1497ffba3..8b2df74a0 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -46,6 +46,11 @@ struct AsyncJob : public VecsimBaseObject { template class VecSimTieredIndex : public VecSimIndexInterface { protected: + // RAII acquisition of mainIndexGuard to allow different locking behavior in child classes + virtual std::shared_lock lockMainIndexForQuery() const { + return std::shared_lock(mainIndexGuard); + } + VecSimIndexAbstract *backendIndex; BruteForceIndex *frontendIndex; @@ -237,9 +242,8 @@ VecSimTieredIndex::topKQueryImp(const void *queryBlob, size_ // Simply query the main index and return the results while holding the lock. auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); - this->mainIndexGuard.lock_shared(); + auto mainLock = lockMainIndexForQuery(); auto res = this->backendIndex->topKQuery(processed_query, k, queryParams); - this->mainIndexGuard.unlock_shared(); return res; } else { @@ -256,10 +260,12 @@ VecSimTieredIndex::topKQueryImp(const void *queryBlob, size_ auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); - // Lock the main index and query it. - this->mainIndexGuard.lock_shared(); - auto main_results = this->backendIndex->topKQuery(processed_query, k, queryParams); - this->mainIndexGuard.unlock_shared(); + VecSimQueryReply *main_results; + { + // Lock the main index and query it. + auto mainLock = lockMainIndexForQuery(); + main_results = this->backendIndex->topKQuery(processed_query, k, queryParams); + } // If the query failed (currently only on timeout), return the error code. if (main_results->code != VecSim_QueryReply_OK) { @@ -302,10 +308,12 @@ VecSimTieredIndex::rangeQueryImp(const void *queryBlob, doub auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); - // Simply query the main index and return the results while holding the lock. - this->mainIndexGuard.lock_shared(); - auto res = this->backendIndex->rangeQuery(processed_query, radius, queryParams); - this->mainIndexGuard.unlock_shared(); + VecSimQueryReply *res; + { + auto mainLock = lockMainIndexForQuery(); + // Simply query the main index and return the results while holding the lock. + res = this->backendIndex->rangeQuery(processed_query, radius, queryParams); + } // We could have passed the order to the main index, but we can sort them here after // unlocking it instead. @@ -326,9 +334,12 @@ VecSimTieredIndex::rangeQueryImp(const void *queryBlob, doub auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); // Lock the main index and query it. - this->mainIndexGuard.lock_shared(); - auto main_results = this->backendIndex->rangeQuery(processed_query, radius, queryParams); - this->mainIndexGuard.unlock_shared(); + + VecSimQueryReply *main_results; + { + auto mainLock = lockMainIndexForQuery(); + main_results = this->backendIndex->rangeQuery(processed_query, radius, queryParams); + } // Merge the results and return, avoiding duplicates. // At this point, the return code of the FLAT index is OK, and the return code of the MAIN From 0ca3a584a32c56bbe30d60abf7ba84a170590c0e Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 14 Sep 2026 06:25:27 -0700 Subject: [PATCH 39/99] fix and optimize --- src/VecSim/algorithms/svs/svs.h | 17 +++++++---------- src/python_bindings/bindings.cpp | 4 +++- tests/flow/test_svs_tiered.py | 5 +++++ 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index cfb000957..40b63dae0 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -338,7 +338,7 @@ class SVSIndex : public VecSimIndexAbstract, fl } const int deleted_num = impl->delete_entries(std::span{&label, 1}); - this->markIndexUpdate(deleted_num); + this->markIndexUpdate(impl, deleted_num); return deleted_num; } @@ -350,18 +350,12 @@ class SVSIndex : public VecSimIndexAbstract, fl const int deleted_num = impl->delete_entries(std::span{labels, n}); if (deleted_num > 0) { - this->markIndexUpdate(deleted_num); + this->markIndexUpdate(impl, deleted_num); } return deleted_num; } - // Count deletions and drop the index instance if it became empty - void markIndexUpdate(size_t n = 1) { - auto impl = getImpl(); - if (!impl) { - return; - } - + void markIndexUpdate(const std::shared_ptr &impl, size_t n = 1) { // SVS index instance should not be empty if (labelCountOf(*impl) == 0) { std::lock_guard replace_lock(this->implMutationGuard_); @@ -426,7 +420,10 @@ class SVSIndex : public VecSimIndexAbstract, fl ~SVSIndex() = default; - bool ready() const override { return getImpl() != nullptr; } + bool ready() const override { + std::shared_lock lock(this->pimplGuard_); + return this->impl_ != nullptr; + } size_t indexSize() const override { return indexStorageSize(); } diff --git a/src/python_bindings/bindings.cpp b/src/python_bindings/bindings.cpp index b68c14653..5888f5d5c 100644 --- a/src/python_bindings/bindings.cpp +++ b/src/python_bindings/bindings.cpp @@ -214,7 +214,9 @@ class PyVecSimIndex { return PyBatchIterator(index, py_batch_ptr); } - void runGC() { VecSimTieredIndex_GC(index.get()); } + // Not VecSimTieredIndex_GC(): that is a no-op unless the index is tiered, while + // SVSIndex::runGC() compacts and rebuilds reverse edges on a plain index too. + void runGC() { index->runGC(); } py::object getVector(labelType label) { VecSimIndexBasicInfo info = index->basicInfo(); diff --git a/tests/flow/test_svs_tiered.py b/tests/flow/test_svs_tiered.py index 746bab43f..e0d8b7f8d 100644 --- a/tests/flow/test_svs_tiered.py +++ b/tests/flow/test_svs_tiered.py @@ -166,6 +166,9 @@ def create_tiered_index(test_logger, is_multi: bool, num_per_label=1, data_type= test_logger.info(f"Total time for inserting vectors to the tiered index and indexing them into SVS using {threads_num}" f" threads took {round_ms(tiered_index_time)} ms") + index.run_gc() + index.wait_for_index() + # Measure total memory of the tiered index. tiered_memory = bytes_to_mega(index.index_memory()) @@ -175,6 +178,8 @@ def create_tiered_index(test_logger, is_multi: bool, num_per_label=1, data_type= _, svs_index_time, _ = indices_ctx.populate_index(svs_index) test_logger.info(f"Insert {num_elements} vectors directly to SVS index (one by one) took {round_(svs_index_time)} s") + + svs_index.run_gc() svs_memory = bytes_to_mega(svs_index.index_memory()) test_logger.info(f"total memory of svs index = {svs_memory} MB") From c33365f82ddc761b18a1cbdd7f21b25cc21a9e00 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Mon, 14 Sep 2026 06:47:58 -0700 Subject: [PATCH 40/99] generalize SharedMutexLockable and harominze SVS/HNSW functions with multiple locks --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 36 +++++++++++++----------- src/VecSim/algorithms/svs/svs_tiered.h | 22 ++++++--------- src/VecSim/vec_sim_tiered_index.h | 28 ++++++++++++++---- 3 files changed, 51 insertions(+), 35 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index 11f6cf76f..bc25b355e 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -154,6 +154,25 @@ class TieredHNSWIndex : public VecSimTieredIndex { #include "VecSim/algorithms/hnsw/hnsw_tiered_tests_friends.h" #endif +protected: + // Adapts HNSW's own data-guard mutex to Lockable, for indexSize/indexCapacity locking. + class HnswDataGuardLockable : public Lockable { + HNSWIndex *hnsw; + + public: + explicit HnswDataGuardLockable(HNSWIndex *hnsw) : hnsw(hnsw) {} + void lock() const override { hnsw->lockSharedIndexDataGuard(); } + void unlock() const override { hnsw->unlockSharedIndexDataGuard(); } + }; + HnswDataGuardLockable hnswDataGuardLockable{this->getHNSWIndex()}; + + ScopedLocks lockIndexForSize() const override { + return ScopedLocks(this->flatIndexLockable, hnswDataGuardLockable); + } + + // Old behavior: indexCapacity() took no lock at all. + ScopedLocks lockIndexForCapacity() const override { return ScopedLocks(); } + public: class TieredHNSW_BatchIterator : public VecSimBatchIterator { private: @@ -217,8 +236,6 @@ class TieredHNSWIndex : public VecSimTieredIndex { size_t getNumMarkedDeleted() const override { return this->getHNSWIndex()->getNumMarkedDeleted(); } - size_t indexSize() const override; - size_t indexCapacity() const override; double getDistanceFrom_Unsafe(labelType label, const void *blob) const override; // Do nothing here, each tier (flat buffer and HNSW) should increase capacity for itself when // needed. @@ -738,21 +755,6 @@ TieredHNSWIndex::~TieredHNSWIndex() { } } -template -size_t TieredHNSWIndex::indexSize() const { - this->flatIndexGuard.lock_shared(); - this->getHNSWIndex()->lockSharedIndexDataGuard(); - size_t res = this->backendIndex->indexSize() + this->frontendIndex->indexSize(); - this->getHNSWIndex()->unlockSharedIndexDataGuard(); - this->flatIndexGuard.unlock_shared(); - return res; -} - -template -size_t TieredHNSWIndex::indexCapacity() const { - return this->backendIndex->indexCapacity() + this->frontendIndex->indexCapacity(); -} - // In the tiered index, we assume that the blobs are processed by the flat buffer // before being transferred to the HNSW index. // When inserting vectors directly into the HNSW index—such as in VecSim_WriteInPlace mode— or when diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 75a54d706..8db0d6676 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -569,9 +569,15 @@ class TieredSVSIndex : public VecSimTieredIndex { #endif protected: - std::shared_lock lockMainIndexForQuery() const override { - // returns a no-op lock because SVS does its locking internally - return std::shared_lock{}; + ScopedLocks lockMainIndexForQuery() const override { + // No-op: SVS does its query locking internally. + return ScopedLocks(); + } + + ScopedLocks lockIndexForSize() const override { return ScopedLocks(this->flatIndexLockable); } + + ScopedLocks lockIndexForCapacity() const override { + return ScopedLocks(this->flatIndexLockable); } private: @@ -1190,16 +1196,6 @@ class TieredSVSIndex : public VecSimTieredIndex { return this->GetSVSIndex()->getNumMarkedDeleted(); } - size_t indexSize() const override { - std::shared_lock flat_lock(this->flatIndexGuard); - return this->frontendIndex->indexSize() + this->backendIndex->indexSize(); - } - - size_t indexCapacity() const override { - std::shared_lock flat_lock(this->flatIndexGuard); - return this->frontendIndex->indexCapacity() + this->backendIndex->indexCapacity(); - } - double getDistanceFrom_Unsafe(labelType label, const void *blob) const override { // Try to get the distance from the flat buffer. // If the label doesn't exist, the distance will be NaN. diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 8b2df74a0..a9836e3ab 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -15,6 +15,7 @@ #include "VecSim/tombstone_interface.h" #include "VecSim/utils/query_result_utils.h" #include "VecSim/utils/alignment.h" +#include "VecSim/utils/scoped_locks.h" #include @@ -46,11 +47,6 @@ struct AsyncJob : public VecsimBaseObject { template class VecSimTieredIndex : public VecSimIndexInterface { protected: - // RAII acquisition of mainIndexGuard to allow different locking behavior in child classes - virtual std::shared_lock lockMainIndexForQuery() const { - return std::shared_lock(mainIndexGuard); - } - VecSimIndexAbstract *backendIndex; BruteForceIndex *frontendIndex; @@ -60,6 +56,18 @@ class VecSimTieredIndex : public VecSimIndexInterface { mutable std::shared_mutex flatIndexGuard; mutable std::shared_mutex mainIndexGuard; + SharedMutexLockable flatIndexLockable{flatIndexGuard}; + SharedMutexLockable mainIndexLockable{mainIndexGuard}; + + // Locking behavior for topKQuery/rangeQuery: real lock by default, overridden to a no-op + // by backends (e.g. SVS) that handle their own concurrency internally. + virtual ScopedLocks lockMainIndexForQuery() const { return ScopedLocks(mainIndexLockable); } + + // Locking behavior for indexSize/indexCapacity, which each backend defines to match + // whatever it actually needs to guard those counters with -- the two differ per backend. + virtual ScopedLocks lockIndexForSize() const = 0; + virtual ScopedLocks lockIndexForCapacity() const = 0; + void lockMainIndexGuard() const { mainIndexGuard.lock(); #ifdef BUILD_TESTS @@ -197,6 +205,16 @@ class VecSimTieredIndex : public VecSimIndexInterface { VecSimQueryParams *queryParams, VecSimQueryReply_Order order) const override; + size_t indexSize() const override { + auto locks = lockIndexForSize(); + return this->frontendIndex->indexSize() + this->backendIndex->indexSize(); + } + + size_t indexCapacity() const override { + auto locks = lockIndexForCapacity(); + return this->frontendIndex->indexCapacity() + this->backendIndex->indexCapacity(); + } + virtual inline uint64_t getAllocationSize() const override { return this->allocator->getAllocationSize() + this->backendIndex->getAllocationSize() + this->frontendIndex->getAllocationSize(); From 41ba83337894a0dd2f23132cc44ce25aa06338c8 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 14 Sep 2026 06:56:48 -0700 Subject: [PATCH 41/99] move droping of gc schdule flag --- src/VecSim/algorithms/svs/svs_tiered.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 1a5f3d133..ea262cbb8 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -643,13 +643,13 @@ class TieredSVSIndex : public VecSimTieredIndex { index->executeTracingCallback("GCJob::before_run_gc"); std::lock_guard lock(index->updateJobMutex); + // Release the scheduled flag to allow scheduling again + index->indexGCScheduled.clear(); + svs_index->setParallelism(std::min(availableThreads, index->backendIndex->indexSize())); // VecSimIndexAbstract::runGC() is protected static_cast(index->backendIndex)->runGC(); svs_index->setParallelism(1); - - // Release the scheduled flag to allow scheduling again - index->indexGCScheduled.clear(); } static void SVSIndexConsolidateWrapper(AsyncJob *job) { From ffd45d9441451d4c3b107b8183b3502d66ad365b Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Mon, 14 Sep 2026 06:57:37 -0700 Subject: [PATCH 42/99] cleanup comments --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 287 ++++++++++++----------- src/VecSim/vec_sim_tiered_index.h | 10 +- 2 files changed, 160 insertions(+), 137 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index bc25b355e..0b3f3d968 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -155,7 +155,7 @@ class TieredHNSWIndex : public VecSimTieredIndex { #endif protected: - // Adapts HNSW's own data-guard mutex to Lockable, for indexSize/indexCapacity locking. + // Adapts HNSW's own data-guard mutex to Lockable class HnswDataGuardLockable : public Lockable { HNSWIndex *hnsw; @@ -166,12 +166,16 @@ class TieredHNSWIndex : public VecSimTieredIndex { }; HnswDataGuardLockable hnswDataGuardLockable{this->getHNSWIndex()}; + ScopedLocks lockMainIndexForQuery() { return ScopedLocks(mainIndexLockable); } + ScopedLocks lockIndexForSize() const override { return ScopedLocks(this->flatIndexLockable, hnswDataGuardLockable); } - // Old behavior: indexCapacity() took no lock at all. - ScopedLocks lockIndexForCapacity() const override { return ScopedLocks(); } + ScopedLocks lockIndexForCapacity() const override { + // No-op: No locking required for capacity() + return ScopedLocks(); + } public: class TieredHNSW_BatchIterator : public VecSimBatchIterator { @@ -185,16 +189,15 @@ class TieredHNSWIndex : public VecSimTieredIndex { VecSimBatchIterator *flat_iterator; VecSimBatchIterator *hnsw_iterator; - // On single value indices, this set holds the IDs of the results that were returned from - // the flat buffer. - // On multi value indices, this set holds the IDs of all the results that were returned. - // The difference between the two cases is that on multi value indices, the same ID can - // appear in both indexes and results with different scores, and therefore we can't tell in - // advance when we expect a possibility of a duplicate. - // On single value indices, a duplicate may appear at the same batch (and we will handle it - // when merging the results) Or it may appear in a different batches, first from the flat - // buffer and then from the HNSW, in the cases where a better result if found later in HNSW - // because of the approximate nature of the algorithm. + // On single value indices, this set holds the IDs of the results that were returned + // from the flat buffer. On multi value indices, this set holds the IDs of all the + // results that were returned. The difference between the two cases is that on multi + // value indices, the same ID can appear in both indexes and results with different + // scores, and therefore we can't tell in advance when we expect a possibility of a + // duplicate. On single value indices, a duplicate may appear at the same batch (and we + // will handle it when merging the results) Or it may appear in a different batches, + // first from the flat buffer and then from the HNSW, in the cases where a better result + // if found later in HNSW because of the approximate nature of the algorithm. vecsim_stl::unordered_set returned_results_set; private: @@ -237,8 +240,8 @@ class TieredHNSWIndex : public VecSimTieredIndex { return this->getHNSWIndex()->getNumMarkedDeleted(); } double getDistanceFrom_Unsafe(labelType label, const void *blob) const override; - // Do nothing here, each tier (flat buffer and HNSW) should increase capacity for itself when - // needed. + // Do nothing here, each tier (flat buffer and HNSW) should increase capacity for itself + // when needed. VecSimIndexDebugInfo debugInfo() const override; VecSimIndexBasicInfo basicInfo() const override; VecSimIndexStatsInfo statisticInfo() const override; @@ -320,16 +323,16 @@ void TieredHNSWIndex::fixJobsAfterSwap( idToRepairJobs.insert({deleted_id, idToRepairJobs.at(prev_last_id)}); idToRepairJobs.erase(prev_last_id); } - // Update the swap jobs if the last id also needs a swap, otherwise just collect to deleted id - // to be removed from the swap jobs. + // Update the swap jobs if the last id also needs a swap, otherwise just collect to deleted + // id to be removed from the swap jobs. if (prev_last_id != deleted_id && idToSwapJob.find(prev_last_id) != idToSwapJob.end() && std::find(idsToRemove.begin(), idsToRemove.end(), prev_last_id) == idsToRemove.end()) { - // Update the curr_last_id pending swap job id after the removal that renamed curr_last_id - // with the deleted id. + // Update the curr_last_id pending swap job id after the removal that renamed + // curr_last_id with the deleted id. idsToRemove.push_back(prev_last_id); idToSwapJob.at(prev_last_id)->deleted_id = deleted_id; - // If id was deleted in-place and there is no swap job for it, this will create a new entry - // in idToSwapJob for the swapped id, otherwise it will update the existing entry. + // If id was deleted in-place and there is no swap job for it, this will create a new + // entry in idToSwapJob for the swapped id, otherwise it will update the existing entry. idToSwapJob[deleted_id] = idToSwapJob.at(prev_last_id); } else { idsToRemove.push_back(deleted_id); @@ -338,7 +341,8 @@ void TieredHNSWIndex::fixJobsAfterSwap( template void TieredHNSWIndex::invalidateRepairJobs(idType deleted_id) { - // Invalidate repair jobs for the disposed id (if exist), and update the associated swap jobs. + // Invalidate repair jobs for the disposed id (if exist), and update the associated swap + // jobs. if (idToRepairJobs.find(deleted_id) == idToRepairJobs.end()) { return; } @@ -453,9 +457,9 @@ int TieredHNSWIndex::deleteLabelFromHNSW(labelType label) { this->idToRepairJobsGuard.unlock(); if (incomingEdges.size() == 0) { - // No repair job will ever run for this element, so this is already the point at which - // it can be taken out of the graph (outside the repair jobs guard, as isolating takes - // the per-element links locks). + // No repair job will ever run for this element, so this is already the point at + // which it can be taken out of the graph (outside the repair jobs guard, as + // isolating takes the per-element links locks). this->isolateRepairedElement(id); } @@ -495,8 +499,8 @@ void TieredHNSWIndex::insertVectorToHNSW( const void *processed_storage_blob = processed_blobs.getStorageBlob(); const void *processed_for_index = processed_blobs.getQueryBlob(); - // Acquire the index data lock, so we know what is the exact index size at this time. Acquire - // the main r/w lock before to avoid deadlocks. + // Acquire the index data lock, so we know what is the exact index size at this time. + // Acquire the main r/w lock before to avoid deadlocks. this->mainIndexGuard.lock_shared(); hnsw_index->lockIndexDataGuard(); // Check if resizing is needed for HNSW index (requires write lock). @@ -507,11 +511,11 @@ void TieredHNSWIndex::insertVectorToHNSW( this->lockMainIndexGuard(); hnsw_index->lockIndexDataGuard(); - // Hold the index data lock while we store the new element. If the new node's max level is - // higher than the current one, hold the lock through the entire insertion to ensure that - // graph scans will not occur, as they will try access the entry point's neighbors. - // If an index resize is still needed, `storeNewElement` will perform it. This is OK since - // we hold the main index lock for exclusive access. + // Hold the index data lock while we store the new element. If the new node's max level + // is higher than the current one, hold the lock through the entire insertion to ensure + // that graph scans will not occur, as they will try access the entry point's neighbors. + // If an index resize is still needed, `storeNewElement` will perform it. This is OK + // since we hold the main index lock for exclusive access. auto state = hnsw_index->storeNewElement(label, processed_storage_blob); if constexpr (releaseFlatGuard) { this->flatIndexGuard.unlock_shared(); @@ -520,11 +524,13 @@ void TieredHNSWIndex::insertVectorToHNSW( // If we're still holding the index data guard, we cannot take the main index lock for // shared ownership as it may cause deadlocks, and we also cannot release the main index // lock between, since we cannot allow swap jobs to happen, as they will make the - // saved state invalid. Hence, we insert the vector with the current exclusive lock held. + // saved state invalid. Hence, we insert the vector with the current exclusive lock + // held. if (state.elementMaxLevel <= state.currMaxLevel) { hnsw_index->unlockIndexDataGuard(); } - // Take the vector from the flat buffer and insert it to HNSW (overwrite should not occur). + // Take the vector from the flat buffer and insert it to HNSW (overwrite should not + // occur). hnsw_index->indexVector(processed_for_index, label, state); if (state.elementMaxLevel > state.currMaxLevel) { hnsw_index->unlockIndexDataGuard(); @@ -532,11 +538,11 @@ void TieredHNSWIndex::insertVectorToHNSW( this->unlockMainIndexGuard(); } else { // Do the same as above except for changing the capacity, but with *shared* lock held: - // Hold the index data lock while we store the new element. If the new node's max level is - // higher than the current one, hold the lock through the entire insertion to ensure that - // graph scans will not occur, as they will try access the entry point's neighbors. - // At this point we are certain that the index has enough capacity for the new element, and - // this call will not resize the index. + // Hold the index data lock while we store the new element. If the new node's max level + // is higher than the current one, hold the lock through the entire insertion to ensure + // that graph scans will not occur, as they will try access the entry point's neighbors. + // At this point we are certain that the index has enough capacity for the new element, + // and this call will not resize the index. auto state = hnsw_index->storeNewElement(label, processed_storage_blob); if constexpr (releaseFlatGuard) { this->flatIndexGuard.unlock_shared(); @@ -545,7 +551,8 @@ void TieredHNSWIndex::insertVectorToHNSW( if (state.elementMaxLevel <= state.currMaxLevel) { hnsw_index->unlockIndexDataGuard(); } - // Take the vector from the flat buffer and insert it to HNSW (overwrite should not occur). + // Take the vector from the flat buffer and insert it to HNSW (overwrite should not + // occur). hnsw_index->indexVector(processed_for_index, label, state); if (state.elementMaxLevel > state.currMaxLevel) { hnsw_index->unlockIndexDataGuard(); @@ -591,7 +598,8 @@ int TieredHNSWIndex::deleteLabelFromHNSWInplace(labelType la /******************** Job's callbacks **********************************/ template void TieredHNSWIndex::executeInsertJob(HNSWInsertJob *job) { - // Note that accessing the job fields should occur with flat index guard held (here and later). + // Note that accessing the job fields should occur with flat index guard held (here and + // later). this->flatIndexGuard.lock_shared(); if (!job->isValid) { this->flatIndexGuard.unlock_shared(); @@ -630,20 +638,22 @@ void TieredHNSWIndex::executeInsertJob(HNSWInsertJob *job) { if (labelToInsertJobs.at(job->label).empty()) { labelToInsertJobs.erase(job->label); } - // Remove the vector from the flat buffer. This may cause the last vector id to swap with - // the deleted id. Hold the label for the last id, so we can later on update its + // Remove the vector from the flat buffer. This may cause the last vector id to swap + // with the deleted id. Hold the label for the last id, so we can later on update its // corresponding job id. Note that after calling deleteVectorById, the last id's label // shouldn't be available, since it is removed from the lookup. labelType last_vec_label = this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); int deleted = this->frontendIndex->deleteVectorById(job->label, job->id); if (deleted && job->id != this->frontendIndex->indexSize()) { - // If the vector removal caused a swap with the last id, update the relevant insert job. + // If the vector removal caused a swap with the last id, update the relevant insert + // job. this->updateInsertJobInternalId(this->frontendIndex->indexSize(), job->id, last_vec_label); } } else { - // Remove the current job from the invalid jobs' lookup, as we are about to delete it now. + // Remove the current job from the invalid jobs' lookup, as we are about to delete it + // now. this->invalidJobsLookupGuard.lock(); this->invalidJobs.erase(job->id); this->invalidJobsLookupGuard.unlock(); @@ -665,10 +675,10 @@ void TieredHNSWIndex::executeRepairJob(HNSWRepairJob *job) { } HNSWIndex *hnsw_index = this->getHNSWIndex(); - // Remove this job pointer from the repair jobs lookup BEFORE it has been executed. Had we done - // it after executing the repair job, we might have see that there is a pending repair job for - // this node id upon deleting another neighbor of this node, and we may avoid creating another - // repair job even though *it has already been executed*. + // Remove this job pointer from the repair jobs lookup BEFORE it has been executed. Had we + // done it after executing the repair job, we might have see that there is a pending repair + // job for this node id upon deleting another neighbor of this node, and we may avoid + // creating another repair job even though *it has already been executed*. this->idToRepairJobsGuard.lock(); auto &repair_jobs = this->idToRepairJobs.at(job->node_id); assert(repair_jobs.size() > 0); @@ -701,9 +711,10 @@ void TieredHNSWIndex::executeRepairJob(HNSWRepairJob *job) { } this->idToRepairJobsGuard.unlock(); - // These deleted elements have no pending repair job left, so nothing points to them anymore. - // Take them out of the graph entirely, leaving no edge in or out for the swap job to deal with. - // Done outside the repair jobs guard, as isolating takes the per-element links locks. + // These deleted elements have no pending repair job left, so nothing points to them + // anymore. Take them out of the graph entirely, leaving no edge in or out for the swap job + // to deal with. Done outside the repair jobs guard, as isolating takes the per-element + // links locks. for (idType deleted_id : fully_repaired_ids) { this->isolateRepairedElement(deleted_id); } @@ -757,9 +768,9 @@ TieredHNSWIndex::~TieredHNSWIndex() { // In the tiered index, we assume that the blobs are processed by the flat buffer // before being transferred to the HNSW index. -// When inserting vectors directly into the HNSW index—such as in VecSim_WriteInPlace mode— or when -// the flat buffer is full, we must manually preprocess the blob according to the **frontend** index -// parameters. +// When inserting vectors directly into the HNSW index—such as in VecSim_WriteInPlace mode— or +// when the flat buffer is full, we must manually preprocess the blob according to the +// **frontend** index parameters. template int TieredHNSWIndex::addVector(const void *blob, labelType label) { int ret = 1; @@ -767,7 +778,8 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l // writeMode is not protected since it is assumed to be called only from the "main thread" // (that is the thread that is exclusively calling add/delete vector). if (this->getWriteMode() == VecSim_WriteInPlace) { - // First, check if we need to overwrite the vector in-place for single (from both indexes). + // First, check if we need to overwrite the vector in-place for single (from both + // indexes). if (!this->backendIndex->isMultiValue()) { ret -= this->deleteVector(label); } @@ -787,16 +799,15 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l if (this->frontendIndex->indexSize() >= this->flatBufferLimit) { // Handle overwrite situation. if (!this->backendIndex->isMultiValue()) { - // This will do nothing (and return 0) if this label doesn't exist. Otherwise, it may - // remove vector from the flat buffer and/or the HNSW index. + // This will do nothing (and return 0) if this label doesn't exist. Otherwise, it + // may remove vector from the flat buffer and/or the HNSW index. ret -= this->deleteVector(label); } if (this->frontendIndex->indexSize() >= this->flatBufferLimit) { - // We didn't remove a vector from flat buffer due to overwrite, insert the new vector - // directly to HNSW. Since flat buffer guard was not held, no need to release it - // internally. - // Use the frontend parameters to manually prepare the blob for its transfer to the HNSW - // index. + // We didn't remove a vector from flat buffer due to overwrite, insert the new + // vector directly to HNSW. Since flat buffer guard was not held, no need to release + // it internally. Use the frontend parameters to manually prepare the blob for its + // transfer to the HNSW index. auto storage_blob = this->frontendIndex->preprocessForStorage(blob); this->insertVectorToHNSW(hnsw_index, label, storage_blob.get()); // Track direct insertion to HNSW (flat buffer was full) @@ -804,7 +815,8 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l return ret; } // Otherwise, we fall back to the "regular" insertion into the flat buffer - // (since it is not full anymore after removing the previous vector stored under the label). + // (since it is not full anymore after removing the previous vector stored under the + // label). } this->flatIndexGuard.lock(); idType new_flat_id = this->frontendIndex->indexSize(); @@ -814,8 +826,8 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l old_job->id = this->setAndSaveInvalidJob(old_job); this->labelToInsertJobs.erase(label); ret = 0; - // We are going to update the internal id that currently holds the vector associated with - // the given label. + // We are going to update the internal id that currently holds the vector associated + // with the given label. new_flat_id = dynamic_cast *>(this->frontendIndex) ->getIdOfLabel(label); @@ -827,10 +839,11 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l AsyncJob *new_insert_job = new (this->allocator) HNSWInsertJob(this->allocator, label, new_flat_id, executeInsertJobWrapper, this); - // Save a pointer to the job, so that if the vector is overwritten, we'll have an indication. + // Save a pointer to the job, so that if the vector is overwritten, we'll have an + // indication. if (this->labelToInsertJobs.find(label) != this->labelToInsertJobs.end()) { - // There's already a pending insert job for this label, add another one (without overwrite, - // only possible in multi index) + // There's already a pending insert job for this label, add another one (without + // overwrite, only possible in multi index) assert(this->backendIndex->isMultiValue()); this->labelToInsertJobs.at(label).push_back((HNSWInsertJob *)new_insert_job); } else { @@ -851,8 +864,8 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l // Apply ready swap jobs if number of deleted vectors reached the threshold (under exclusive // lock of the main index guard). // If swapJobs size is equal or larger than a threshold, go over the swap jobs and execute a - // batch of jobs for which all of its pending repair jobs were executed (otherwise finish and - // return). + // batch of jobs for which all of its pending repair jobs were executed (otherwise finish + // and return). if (readySwapJobs >= this->pendingSwapJobsThreshold) { this->executeReadySwapJobs(this->pendingSwapJobsThreshold); } @@ -869,7 +882,8 @@ int TieredHNSWIndex::deleteVector(labelType label) { if (this->frontendIndex->isLabelExists(label)) { this->flatIndexGuard.unlock_shared(); this->flatIndexGuard.lock(); - // Check again if the label exists, as it may have been removed while we released the lock. + // Check again if the label exists, as it may have been removed while we released the + // lock. if (this->frontendIndex->isLabelExists(label)) { // Invalidate the pending insert job(s) into HNSW associated with this label auto &insert_jobs = this->labelToInsertJobs.at(label); @@ -879,10 +893,10 @@ int TieredHNSWIndex::deleteVector(labelType label) { num_deleted_vectors += insert_jobs.size(); // Remove the pending insert job(s) from the labelToInsertJobs mapping. this->labelToInsertJobs.erase(label); - // Go over the every id that corresponds the label and remove it from the flat buffer. - // Every delete may cause a swap of the deleted id with the last id, and we return a - // mapping from id to the original id that resides in this id after the deletion(s) (see - // an example in this function implementation in MULTI index). + // Go over the every id that corresponds the label and remove it from the flat + // buffer. Every delete may cause a swap of the deleted id with the last id, and we + // return a mapping from id to the original id that resides in this id after the + // deletion(s) (see an example in this function implementation in MULTI index). auto updated_ids = this->frontendIndex->deleteVectorAndGetUpdatedIds(label); for (auto &it : updated_ids) { idType prev_id = it.second.first; @@ -895,11 +909,11 @@ int TieredHNSWIndex::deleteVector(labelType label) { this->flatIndexGuard.unlock_shared(); } - // Next, check if there vector(s) stored under the given label in HNSW and delete them as well. - // Note that we may remove the same vector that has been removed from the flat index, if it was - // being ingested at that time. - // writeMode is not protected since it is assumed to be called only from the "main thread" - // (that is the thread that is exclusively calling add/delete vector). + // Next, check if there vector(s) stored under the given label in HNSW and delete them as + // well. Note that we may remove the same vector that has been removed from the flat index, + // if it was being ingested at that time. writeMode is not protected since it is assumed to + // be called only from the "main thread" (that is the thread that is exclusively calling + // add/delete vector). if (this->getWriteMode() == VecSim_WriteAsync) { num_deleted_vectors += this->deleteLabelFromHNSW(label); // Apply ready swap jobs if number of deleted vectors reached the threshold @@ -919,20 +933,20 @@ int TieredHNSWIndex::deleteVector(labelType label) { /** * Move a label across every place the tiered index records one. Unlike add/delete this needs no - * jobs and no vector data movement - the swap/repair machinery is keyed purely on internal ids, so - * it is unaffected. + * jobs and no vector data movement - the swap/repair machinery is keyed purely on internal ids, + * so it is unaffected. * * A label can legitimately live in *both* tiers at once: `executeInsertJob` inserts into HNSW * before removing the vector from the flat buffer, so all four updates below are independently * guarded by their own existence check rather than treated as mutually exclusive. * - * Locking: `flatIndexGuard` exclusively (it guards the flat buffer's lookups, `labelToInsertJobs` - * and - per the note in `executeInsertJob` - the job fields themselves), then the main index guard - * exclusively. That is the flat-then-main order used by `insertVectorToHNSW` and `topKQueryImp`, - * so it cannot deadlock against them; repair and swap jobs only ever take the main guard. - * The main guard must be *exclusive* rather than shared: query threads read - * `getExternalLabel` under a shared main guard, and `ElementMetaData` is byte-packed, so its - * `label` field can be unaligned and its store is not atomic. + * Locking: `flatIndexGuard` exclusively (it guards the flat buffer's lookups, + * `labelToInsertJobs` and - per the note in `executeInsertJob` - the job fields themselves), + * then the main index guard exclusively. That is the flat-then-main order used by + * `insertVectorToHNSW` and `topKQueryImp`, so it cannot deadlock against them; repair and swap + * jobs only ever take the main guard. The main guard must be *exclusive* rather than shared: + * query threads read `getExternalLabel` under a shared main guard, and `ElementMetaData` is + * byte-packed, so its `label` field can be unaligned and its store is not atomic. */ template VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType old_label, @@ -946,9 +960,9 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o auto *hnsw_index = this->getHNSWIndex(); // A label can live in any subset of its three homes, so both questions are asked of all of - // them: it is present if any home holds it, and the target is free only if none does. Reject if - // the target is taken anywhere, so that a partially applied move is impossible - a free target - // in all three homes means none of the updates below can collide. + // them: it is present if any home holds it, and the target is free only if none does. + // Reject if the target is taken anywhere, so that a partially applied move is impossible - + // a free target in all three homes means none of the updates below can collide. const bool source_exists = this->frontendIndex->isLabelExists(old_label) || this->labelToInsertJobs.find(old_label) != this->labelToInsertJobs.end() || @@ -958,10 +972,11 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o this->labelToInsertJobs.find(new_label) != this->labelToInsertJobs.end() || hnsw_index->isLabelExists(new_label); - // Each home is asked to move the label only once it reported holding it, and the checks above - // ruled out every other rejection - the label is present, the target is free everywhere, and - // the labels differ - so while both guards are held a home that is asked can only answer OK. - // The asserts below state that; a refusal would mean the state changed underneath us. + // Each home is asked to move the label only once it reported holding it, and the checks + // above ruled out every other rejection - the label is present, the target is free + // everywhere, and the labels differ - so while both guards are held a home that is asked + // can only answer OK. The asserts below state that; a refusal would mean the state changed + // underneath us. if (source_exists && !target_taken) { if (this->frontendIndex->isLabelExists(old_label)) { const VecSimRelabelCode flat_ret = @@ -973,10 +988,10 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o UNUSED(flat_ret); } - // Re-key the pending insert jobs *and* rewrite each job's own copy of the label. Both must - // move together: `executeInsertJob` indexes into HNSW under `job->label` and then looks the - // job up with `labelToInsertJobs.at(job->label)`, so a half-applied move either indexes the - // vector under the stale label or throws out of the worker thread. + // Re-key the pending insert jobs *and* rewrite each job's own copy of the label. Both + // must move together: `executeInsertJob` indexes into HNSW under `job->label` and then + // looks the job up with `labelToInsertJobs.at(job->label)`, so a half-applied move + // either indexes the vector under the stale label or throws out of the worker thread. auto jobs_it = this->labelToInsertJobs.find(old_label); if (jobs_it != this->labelToInsertJobs.end()) { auto jobs = std::move(jobs_it->second); @@ -1002,7 +1017,8 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o this->flatIndexGuard.unlock(); // An absent source outranks an occupied target: a caller that resolves `NewLabelTaken` by - // freeing the target would otherwise drop an unrelated vector for a move with nothing to move. + // freeing the target would otherwise drop an unrelated vector for a move with nothing to + // move. if (!source_exists) { return VecSimRelabel_OldLabelMissing; } @@ -1015,25 +1031,30 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o // `getDistanceFrom` returns the minimum distance between the given blob and the vector with the // given label. If the label doesn't exist, the distance will be NaN. // Therefore, it's better to just call `getDistanceFrom` on both indexes and return the minimum -// instead of checking if the label exists in each index. We first try to get the distance from the -// flat buffer, as vectors in the buffer might move to the Main while we're "between" the locks. -// Behavior for single (regular) index: +// instead of checking if the label exists in each index. We first try to get the distance from +// the flat buffer, as vectors in the buffer might move to the Main while we're "between" the +// locks. Behavior for single (regular) index: // 1. label doesn't exist in both indexes - return NaN -// 2. label exists in one of the indexes only - return the distance from that index (which is valid) -// 3. label exists in both indexes - return the value from the flat buffer (which is valid and equal +// 2. label exists in one of the indexes only - return the distance from that index (which is +// valid) +// 3. label exists in both indexes - return the value from the flat buffer (which is valid and +// equal // to the value from the Main index), saving us from locking the Main index. // Behavior for multi index: // 1. label doesn't exist in both indexes - return NaN -// 2. label exists in one of the indexes only - return the distance from that index (which is valid) -// 3. label exists in both indexes - we may have some of the vectors with the same label in the flat +// 2. label exists in one of the indexes only - return the distance from that index (which is +// valid) +// 3. label exists in both indexes - we may have some of the vectors with the same label in the +// flat // buffer only and some in the Main index only (and maybe temporal duplications). // So, we get the distance from both indexes and return the minimum. -// IMPORTANT: this should be called when the *tiered index locks are locked for shared ownership*, -// along with HNSW index data guard lock. That is since the internal getDistanceFrom calls access -// the indexes' data, and it is not safe to run insert/delete operation in parallel. Also, we avoid -// acquiring the locks internally, since this is usually called for every vector individually, and -// the overhead of acquiring and releasing the locks is significant in that case. +// IMPORTANT: this should be called when the *tiered index locks are locked for shared +// ownership*, along with HNSW index data guard lock. That is since the internal getDistanceFrom +// calls access the indexes' data, and it is not safe to run insert/delete operation in +// parallel. Also, we avoid acquiring the locks internally, since this is usually called for +// every vector individually, and the overhead of acquiring and releasing the locks is +// significant in that case. template double TieredHNSWIndex::getDistanceFrom_Unsafe(labelType label, const void *blob) const { @@ -1041,8 +1062,8 @@ double TieredHNSWIndex::getDistanceFrom_Unsafe(labelType lab // If the label doesn't exist, the distance will be NaN. auto flat_dist = this->frontendIndex->getDistanceFrom_Unsafe(label, blob); - // Optimization. TODO: consider having different implementations for single and multi indexes, - // to avoid checking the index type on every query. + // Optimization. TODO: consider having different implementations for single and multi + // indexes, to avoid checking the index type on every query. if (!this->backendIndex->isMultiValue() && !std::isnan(flat_dist)) { // If the index is single value, and we got a valid distance from the flat buffer, // we can return the distance without querying the Main index. @@ -1074,8 +1095,8 @@ TieredHNSWIndex::TieredHNSW_BatchIterator::TieredHNSW_BatchI // Tiered batch iterator doesn't hold its own copy of the query vector. // Instead, each internal batch iterators (flat_iterator and hnsw_iterator) create their own // copies: flat_iterator copy is created during TieredHNSW_BatchIterator construction When - // TieredHNSW_BatchIterator::getNextResults() is called and hnsw_iterator is not initialized, it - // retrieves the blob from flat_iterator + // TieredHNSW_BatchIterator::getNextResults() is called and hnsw_iterator is not + // initialized, it retrieves the blob from flat_iterator : VecSimBatchIterator(nullptr, queryParams ? queryParams->timeoutCtx : nullptr, std::move(allocator)), index(index), flat_results(this->allocator), hnsw_results(this->allocator), @@ -1114,8 +1135,8 @@ VecSimQueryReply *TieredHNSWIndex::TieredHNSW_BatchIterator: auto hnsw_code = VecSim_QueryReply_OK; if (this->hnsw_iterator == UNINITIALIZED) { - // First call to getNextResults. The call to the BF iterator will include calculating all - // the distances and access the BF index. We take the lock on this call. + // First call to getNextResults. The call to the BF iterator will include calculating + // all the distances and access the BF index. We take the lock on this call. this->index->flatIndexGuard.lock_shared(); auto cur_flat_results = this->flat_iterator->getNextResults(n_res, BY_SCORE_THEN_ID); this->index->flatIndexGuard.unlock_shared(); @@ -1125,8 +1146,8 @@ VecSimQueryReply *TieredHNSWIndex::TieredHNSW_BatchIterator: } this->flat_results.swap(cur_flat_results->results); VecSimQueryReply_Free(cur_flat_results); - // We also take the lock on the main index on the first call to getNextResults, and we hold - // it until the iterator is depleted or freed. + // We also take the lock on the main index on the first call to getNextResults, and we + // hold it until the iterator is depleted or freed. this->index->mainIndexGuard.lock_shared(); this->hnsw_iterator = this->index->backendIndex->newBatchIterator( this->flat_iterator->getQueryBlob(), queryParams); @@ -1148,10 +1169,10 @@ VecSimQueryReply *TieredHNSWIndex::TieredHNSW_BatchIterator: VecSimQueryReply_Free(tail); if (!isMulti) { - // On single-value indexes, duplicates will never appear in the hnsw results before - // they appear in the flat results (at the same time or later if the approximation - // misses) so we don't need to try and filter the flat results (and recheck - // conditions). + // On single-value indexes, duplicates will never appear in the hnsw results + // before they appear in the flat results (at the same time or later if the + // approximation misses) so we don't need to try and filter the flat results + // (and recheck conditions). break; } else { // On multi-value indexes, the flat results may contain results that are already @@ -1165,9 +1186,9 @@ VecSimQueryReply *TieredHNSWIndex::TieredHNSW_BatchIterator: auto tail = this->hnsw_iterator->getNextResults(n_res - this->hnsw_results.size(), BY_SCORE_THEN_ID); hnsw_code = tail->code; // Set the hnsw_results code to the last `getNextResults` code. - // New batch may contain better results than the previous batch, so we need to merge. - // We don't expect duplications (hence the ), as the iterator guarantees that - // no result is returned twice. + // New batch may contain better results than the previous batch, so we need to + // merge. We don't expect duplications (hence the ), as the iterator + // guarantees that no result is returned twice. VecSimQueryResultContainer cur_hnsw_results(this->allocator); merge_results(cur_hnsw_results, this->hnsw_results, tail->results, n_res); VecSimQueryReply_Free(tail); @@ -1244,13 +1265,15 @@ TieredHNSWIndex::TieredHNSW_BatchIterator::compute_current_b auto [from_hnsw, from_flat] = p; if (!isMultiValue) { - // If we're on a single-value index, update the set of results returned from the FLAT index - // before popping them, to prevent them to be returned from the HNSW index in later batches. + // If we're on a single-value index, update the set of results returned from the FLAT + // index before popping them, to prevent them to be returned from the HNSW index in + // later batches. for (size_t i = 0; i < from_flat; ++i) { this->returned_results_set.insert(this->flat_results[i].id); } } else { - // If we're on a multi-value index, update the set of results returned (from `batch_res`) + // If we're on a multi-value index, update the set of results returned (from + // `batch_res`) for (size_t i = 0; i < batch_res->results.size(); ++i) { this->returned_results_set.insert(batch_res->results[i].id); } diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index a9836e3ab..4406f5e48 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -59,13 +59,13 @@ class VecSimTieredIndex : public VecSimIndexInterface { SharedMutexLockable flatIndexLockable{flatIndexGuard}; SharedMutexLockable mainIndexLockable{mainIndexGuard}; - // Locking behavior for topKQuery/rangeQuery: real lock by default, overridden to a no-op - // by backends (e.g. SVS) that handle their own concurrency internally. - virtual ScopedLocks lockMainIndexForQuery() const { return ScopedLocks(mainIndexLockable); } + // Locking behavior for topKQuery/rangeQuery + virtual ScopedLocks lockMainIndexForQuery() const; - // Locking behavior for indexSize/indexCapacity, which each backend defines to match - // whatever it actually needs to guard those counters with -- the two differ per backend. + // Locking behavior for indexSize() virtual ScopedLocks lockIndexForSize() const = 0; + + // Locking behavior for indexCapacity() virtual ScopedLocks lockIndexForCapacity() const = 0; void lockMainIndexGuard() const { From e0ccdaaf3bd3eed7cc7fbe5a219a9170b6740f3f Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Mon, 14 Sep 2026 07:16:18 -0700 Subject: [PATCH 43/99] fixup --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 4 +++- src/VecSim/vec_sim_tiered_index.h | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index 0b3f3d968..8322e870d 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -166,7 +166,9 @@ class TieredHNSWIndex : public VecSimTieredIndex { }; HnswDataGuardLockable hnswDataGuardLockable{this->getHNSWIndex()}; - ScopedLocks lockMainIndexForQuery() { return ScopedLocks(mainIndexLockable); } + ScopedLocks lockMainIndexForQuery() const override { + return ScopedLocks(this->mainIndexLockable); + } ScopedLocks lockIndexForSize() const override { return ScopedLocks(this->flatIndexLockable, hnswDataGuardLockable); diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 4406f5e48..628cb6a65 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -60,7 +60,7 @@ class VecSimTieredIndex : public VecSimIndexInterface { SharedMutexLockable mainIndexLockable{mainIndexGuard}; // Locking behavior for topKQuery/rangeQuery - virtual ScopedLocks lockMainIndexForQuery() const; + virtual ScopedLocks lockMainIndexForQuery() const = 0; // Locking behavior for indexSize() virtual ScopedLocks lockIndexForSize() const = 0; From 7b2b646294c9b7a26a839fc6e888d4788aab6c96 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Mon, 14 Sep 2026 07:23:38 -0700 Subject: [PATCH 44/99] fix comment formatting --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 277 +++++++++++------------ 1 file changed, 129 insertions(+), 148 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index 8322e870d..e30576169 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -191,15 +191,16 @@ class TieredHNSWIndex : public VecSimTieredIndex { VecSimBatchIterator *flat_iterator; VecSimBatchIterator *hnsw_iterator; - // On single value indices, this set holds the IDs of the results that were returned - // from the flat buffer. On multi value indices, this set holds the IDs of all the - // results that were returned. The difference between the two cases is that on multi - // value indices, the same ID can appear in both indexes and results with different - // scores, and therefore we can't tell in advance when we expect a possibility of a - // duplicate. On single value indices, a duplicate may appear at the same batch (and we - // will handle it when merging the results) Or it may appear in a different batches, - // first from the flat buffer and then from the HNSW, in the cases where a better result - // if found later in HNSW because of the approximate nature of the algorithm. + // On single value indices, this set holds the IDs of the results that were returned from + // the flat buffer. + // On multi value indices, this set holds the IDs of all the results that were returned. + // The difference between the two cases is that on multi value indices, the same ID can + // appear in both indexes and results with different scores, and therefore we can't tell in + // advance when we expect a possibility of a duplicate. + // On single value indices, a duplicate may appear at the same batch (and we will handle it + // when merging the results) Or it may appear in a different batches, first from the flat + // buffer and then from the HNSW, in the cases where a better result if found later in HNSW + // because of the approximate nature of the algorithm. vecsim_stl::unordered_set returned_results_set; private: @@ -242,8 +243,8 @@ class TieredHNSWIndex : public VecSimTieredIndex { return this->getHNSWIndex()->getNumMarkedDeleted(); } double getDistanceFrom_Unsafe(labelType label, const void *blob) const override; - // Do nothing here, each tier (flat buffer and HNSW) should increase capacity for itself - // when needed. + // Do nothing here, each tier (flat buffer and HNSW) should increase capacity for itself when + // needed. VecSimIndexDebugInfo debugInfo() const override; VecSimIndexBasicInfo basicInfo() const override; VecSimIndexStatsInfo statisticInfo() const override; @@ -325,16 +326,16 @@ void TieredHNSWIndex::fixJobsAfterSwap( idToRepairJobs.insert({deleted_id, idToRepairJobs.at(prev_last_id)}); idToRepairJobs.erase(prev_last_id); } - // Update the swap jobs if the last id also needs a swap, otherwise just collect to deleted - // id to be removed from the swap jobs. + // Update the swap jobs if the last id also needs a swap, otherwise just collect to deleted id + // to be removed from the swap jobs. if (prev_last_id != deleted_id && idToSwapJob.find(prev_last_id) != idToSwapJob.end() && std::find(idsToRemove.begin(), idsToRemove.end(), prev_last_id) == idsToRemove.end()) { - // Update the curr_last_id pending swap job id after the removal that renamed - // curr_last_id with the deleted id. + // Update the curr_last_id pending swap job id after the removal that renamed curr_last_id + // with the deleted id. idsToRemove.push_back(prev_last_id); idToSwapJob.at(prev_last_id)->deleted_id = deleted_id; - // If id was deleted in-place and there is no swap job for it, this will create a new - // entry in idToSwapJob for the swapped id, otherwise it will update the existing entry. + // If id was deleted in-place and there is no swap job for it, this will create a new entry + // in idToSwapJob for the swapped id, otherwise it will update the existing entry. idToSwapJob[deleted_id] = idToSwapJob.at(prev_last_id); } else { idsToRemove.push_back(deleted_id); @@ -343,8 +344,7 @@ void TieredHNSWIndex::fixJobsAfterSwap( template void TieredHNSWIndex::invalidateRepairJobs(idType deleted_id) { - // Invalidate repair jobs for the disposed id (if exist), and update the associated swap - // jobs. + // Invalidate repair jobs for the disposed id (if exist), and update the associated swap jobs. if (idToRepairJobs.find(deleted_id) == idToRepairJobs.end()) { return; } @@ -459,9 +459,9 @@ int TieredHNSWIndex::deleteLabelFromHNSW(labelType label) { this->idToRepairJobsGuard.unlock(); if (incomingEdges.size() == 0) { - // No repair job will ever run for this element, so this is already the point at - // which it can be taken out of the graph (outside the repair jobs guard, as - // isolating takes the per-element links locks). + // No repair job will ever run for this element, so this is already the point at which + // it can be taken out of the graph (outside the repair jobs guard, as isolating takes + // the per-element links locks). this->isolateRepairedElement(id); } @@ -501,8 +501,8 @@ void TieredHNSWIndex::insertVectorToHNSW( const void *processed_storage_blob = processed_blobs.getStorageBlob(); const void *processed_for_index = processed_blobs.getQueryBlob(); - // Acquire the index data lock, so we know what is the exact index size at this time. - // Acquire the main r/w lock before to avoid deadlocks. + // Acquire the index data lock, so we know what is the exact index size at this time. Acquire + // the main r/w lock before to avoid deadlocks. this->mainIndexGuard.lock_shared(); hnsw_index->lockIndexDataGuard(); // Check if resizing is needed for HNSW index (requires write lock). @@ -513,11 +513,11 @@ void TieredHNSWIndex::insertVectorToHNSW( this->lockMainIndexGuard(); hnsw_index->lockIndexDataGuard(); - // Hold the index data lock while we store the new element. If the new node's max level - // is higher than the current one, hold the lock through the entire insertion to ensure - // that graph scans will not occur, as they will try access the entry point's neighbors. - // If an index resize is still needed, `storeNewElement` will perform it. This is OK - // since we hold the main index lock for exclusive access. + // Hold the index data lock while we store the new element. If the new node's max level is + // higher than the current one, hold the lock through the entire insertion to ensure that + // graph scans will not occur, as they will try access the entry point's neighbors. + // If an index resize is still needed, `storeNewElement` will perform it. This is OK since + // we hold the main index lock for exclusive access. auto state = hnsw_index->storeNewElement(label, processed_storage_blob); if constexpr (releaseFlatGuard) { this->flatIndexGuard.unlock_shared(); @@ -526,13 +526,11 @@ void TieredHNSWIndex::insertVectorToHNSW( // If we're still holding the index data guard, we cannot take the main index lock for // shared ownership as it may cause deadlocks, and we also cannot release the main index // lock between, since we cannot allow swap jobs to happen, as they will make the - // saved state invalid. Hence, we insert the vector with the current exclusive lock - // held. + // saved state invalid. Hence, we insert the vector with the current exclusive lock held. if (state.elementMaxLevel <= state.currMaxLevel) { hnsw_index->unlockIndexDataGuard(); } - // Take the vector from the flat buffer and insert it to HNSW (overwrite should not - // occur). + // Take the vector from the flat buffer and insert it to HNSW (overwrite should not occur). hnsw_index->indexVector(processed_for_index, label, state); if (state.elementMaxLevel > state.currMaxLevel) { hnsw_index->unlockIndexDataGuard(); @@ -540,11 +538,11 @@ void TieredHNSWIndex::insertVectorToHNSW( this->unlockMainIndexGuard(); } else { // Do the same as above except for changing the capacity, but with *shared* lock held: - // Hold the index data lock while we store the new element. If the new node's max level - // is higher than the current one, hold the lock through the entire insertion to ensure - // that graph scans will not occur, as they will try access the entry point's neighbors. - // At this point we are certain that the index has enough capacity for the new element, - // and this call will not resize the index. + // Hold the index data lock while we store the new element. If the new node's max level is + // higher than the current one, hold the lock through the entire insertion to ensure that + // graph scans will not occur, as they will try access the entry point's neighbors. + // At this point we are certain that the index has enough capacity for the new element, and + // this call will not resize the index. auto state = hnsw_index->storeNewElement(label, processed_storage_blob); if constexpr (releaseFlatGuard) { this->flatIndexGuard.unlock_shared(); @@ -553,8 +551,7 @@ void TieredHNSWIndex::insertVectorToHNSW( if (state.elementMaxLevel <= state.currMaxLevel) { hnsw_index->unlockIndexDataGuard(); } - // Take the vector from the flat buffer and insert it to HNSW (overwrite should not - // occur). + // Take the vector from the flat buffer and insert it to HNSW (overwrite should not occur). hnsw_index->indexVector(processed_for_index, label, state); if (state.elementMaxLevel > state.currMaxLevel) { hnsw_index->unlockIndexDataGuard(); @@ -600,8 +597,7 @@ int TieredHNSWIndex::deleteLabelFromHNSWInplace(labelType la /******************** Job's callbacks **********************************/ template void TieredHNSWIndex::executeInsertJob(HNSWInsertJob *job) { - // Note that accessing the job fields should occur with flat index guard held (here and - // later). + // Note that accessing the job fields should occur with flat index guard held (here and later). this->flatIndexGuard.lock_shared(); if (!job->isValid) { this->flatIndexGuard.unlock_shared(); @@ -640,22 +636,20 @@ void TieredHNSWIndex::executeInsertJob(HNSWInsertJob *job) { if (labelToInsertJobs.at(job->label).empty()) { labelToInsertJobs.erase(job->label); } - // Remove the vector from the flat buffer. This may cause the last vector id to swap - // with the deleted id. Hold the label for the last id, so we can later on update its + // Remove the vector from the flat buffer. This may cause the last vector id to swap with + // the deleted id. Hold the label for the last id, so we can later on update its // corresponding job id. Note that after calling deleteVectorById, the last id's label // shouldn't be available, since it is removed from the lookup. labelType last_vec_label = this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); int deleted = this->frontendIndex->deleteVectorById(job->label, job->id); if (deleted && job->id != this->frontendIndex->indexSize()) { - // If the vector removal caused a swap with the last id, update the relevant insert - // job. + // If the vector removal caused a swap with the last id, update the relevant insert job. this->updateInsertJobInternalId(this->frontendIndex->indexSize(), job->id, last_vec_label); } } else { - // Remove the current job from the invalid jobs' lookup, as we are about to delete it - // now. + // Remove the current job from the invalid jobs' lookup, as we are about to delete it now. this->invalidJobsLookupGuard.lock(); this->invalidJobs.erase(job->id); this->invalidJobsLookupGuard.unlock(); @@ -677,10 +671,10 @@ void TieredHNSWIndex::executeRepairJob(HNSWRepairJob *job) { } HNSWIndex *hnsw_index = this->getHNSWIndex(); - // Remove this job pointer from the repair jobs lookup BEFORE it has been executed. Had we - // done it after executing the repair job, we might have see that there is a pending repair - // job for this node id upon deleting another neighbor of this node, and we may avoid - // creating another repair job even though *it has already been executed*. + // Remove this job pointer from the repair jobs lookup BEFORE it has been executed. Had we done + // it after executing the repair job, we might have see that there is a pending repair job for + // this node id upon deleting another neighbor of this node, and we may avoid creating another + // repair job even though *it has already been executed*. this->idToRepairJobsGuard.lock(); auto &repair_jobs = this->idToRepairJobs.at(job->node_id); assert(repair_jobs.size() > 0); @@ -713,10 +707,9 @@ void TieredHNSWIndex::executeRepairJob(HNSWRepairJob *job) { } this->idToRepairJobsGuard.unlock(); - // These deleted elements have no pending repair job left, so nothing points to them - // anymore. Take them out of the graph entirely, leaving no edge in or out for the swap job - // to deal with. Done outside the repair jobs guard, as isolating takes the per-element - // links locks. + // These deleted elements have no pending repair job left, so nothing points to them anymore. + // Take them out of the graph entirely, leaving no edge in or out for the swap job to deal with. + // Done outside the repair jobs guard, as isolating takes the per-element links locks. for (idType deleted_id : fully_repaired_ids) { this->isolateRepairedElement(deleted_id); } @@ -770,9 +763,9 @@ TieredHNSWIndex::~TieredHNSWIndex() { // In the tiered index, we assume that the blobs are processed by the flat buffer // before being transferred to the HNSW index. -// When inserting vectors directly into the HNSW index—such as in VecSim_WriteInPlace mode— or -// when the flat buffer is full, we must manually preprocess the blob according to the -// **frontend** index parameters. +// When inserting vectors directly into the HNSW index—such as in VecSim_WriteInPlace mode— or when +// the flat buffer is full, we must manually preprocess the blob according to the **frontend** index +// parameters. template int TieredHNSWIndex::addVector(const void *blob, labelType label) { int ret = 1; @@ -780,8 +773,7 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l // writeMode is not protected since it is assumed to be called only from the "main thread" // (that is the thread that is exclusively calling add/delete vector). if (this->getWriteMode() == VecSim_WriteInPlace) { - // First, check if we need to overwrite the vector in-place for single (from both - // indexes). + // First, check if we need to overwrite the vector in-place for single (from both indexes). if (!this->backendIndex->isMultiValue()) { ret -= this->deleteVector(label); } @@ -801,15 +793,16 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l if (this->frontendIndex->indexSize() >= this->flatBufferLimit) { // Handle overwrite situation. if (!this->backendIndex->isMultiValue()) { - // This will do nothing (and return 0) if this label doesn't exist. Otherwise, it - // may remove vector from the flat buffer and/or the HNSW index. + // This will do nothing (and return 0) if this label doesn't exist. Otherwise, it may + // remove vector from the flat buffer and/or the HNSW index. ret -= this->deleteVector(label); } if (this->frontendIndex->indexSize() >= this->flatBufferLimit) { - // We didn't remove a vector from flat buffer due to overwrite, insert the new - // vector directly to HNSW. Since flat buffer guard was not held, no need to release - // it internally. Use the frontend parameters to manually prepare the blob for its - // transfer to the HNSW index. + // We didn't remove a vector from flat buffer due to overwrite, insert the new vector + // directly to HNSW. Since flat buffer guard was not held, no need to release it + // internally. + // Use the frontend parameters to manually prepare the blob for its transfer to the HNSW + // index. auto storage_blob = this->frontendIndex->preprocessForStorage(blob); this->insertVectorToHNSW(hnsw_index, label, storage_blob.get()); // Track direct insertion to HNSW (flat buffer was full) @@ -817,8 +810,7 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l return ret; } // Otherwise, we fall back to the "regular" insertion into the flat buffer - // (since it is not full anymore after removing the previous vector stored under the - // label). + // (since it is not full anymore after removing the previous vector stored under the label). } this->flatIndexGuard.lock(); idType new_flat_id = this->frontendIndex->indexSize(); @@ -828,8 +820,8 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l old_job->id = this->setAndSaveInvalidJob(old_job); this->labelToInsertJobs.erase(label); ret = 0; - // We are going to update the internal id that currently holds the vector associated - // with the given label. + // We are going to update the internal id that currently holds the vector associated with + // the given label. new_flat_id = dynamic_cast *>(this->frontendIndex) ->getIdOfLabel(label); @@ -841,11 +833,10 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l AsyncJob *new_insert_job = new (this->allocator) HNSWInsertJob(this->allocator, label, new_flat_id, executeInsertJobWrapper, this); - // Save a pointer to the job, so that if the vector is overwritten, we'll have an - // indication. + // Save a pointer to the job, so that if the vector is overwritten, we'll have an indication. if (this->labelToInsertJobs.find(label) != this->labelToInsertJobs.end()) { - // There's already a pending insert job for this label, add another one (without - // overwrite, only possible in multi index) + // There's already a pending insert job for this label, add another one (without overwrite, + // only possible in multi index) assert(this->backendIndex->isMultiValue()); this->labelToInsertJobs.at(label).push_back((HNSWInsertJob *)new_insert_job); } else { @@ -866,8 +857,8 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l // Apply ready swap jobs if number of deleted vectors reached the threshold (under exclusive // lock of the main index guard). // If swapJobs size is equal or larger than a threshold, go over the swap jobs and execute a - // batch of jobs for which all of its pending repair jobs were executed (otherwise finish - // and return). + // batch of jobs for which all of its pending repair jobs were executed (otherwise finish and + // return). if (readySwapJobs >= this->pendingSwapJobsThreshold) { this->executeReadySwapJobs(this->pendingSwapJobsThreshold); } @@ -884,8 +875,7 @@ int TieredHNSWIndex::deleteVector(labelType label) { if (this->frontendIndex->isLabelExists(label)) { this->flatIndexGuard.unlock_shared(); this->flatIndexGuard.lock(); - // Check again if the label exists, as it may have been removed while we released the - // lock. + // Check again if the label exists, as it may have been removed while we released the lock. if (this->frontendIndex->isLabelExists(label)) { // Invalidate the pending insert job(s) into HNSW associated with this label auto &insert_jobs = this->labelToInsertJobs.at(label); @@ -895,10 +885,10 @@ int TieredHNSWIndex::deleteVector(labelType label) { num_deleted_vectors += insert_jobs.size(); // Remove the pending insert job(s) from the labelToInsertJobs mapping. this->labelToInsertJobs.erase(label); - // Go over the every id that corresponds the label and remove it from the flat - // buffer. Every delete may cause a swap of the deleted id with the last id, and we - // return a mapping from id to the original id that resides in this id after the - // deletion(s) (see an example in this function implementation in MULTI index). + // Go over the every id that corresponds the label and remove it from the flat buffer. + // Every delete may cause a swap of the deleted id with the last id, and we return a + // mapping from id to the original id that resides in this id after the deletion(s) (see + // an example in this function implementation in MULTI index). auto updated_ids = this->frontendIndex->deleteVectorAndGetUpdatedIds(label); for (auto &it : updated_ids) { idType prev_id = it.second.first; @@ -911,11 +901,11 @@ int TieredHNSWIndex::deleteVector(labelType label) { this->flatIndexGuard.unlock_shared(); } - // Next, check if there vector(s) stored under the given label in HNSW and delete them as - // well. Note that we may remove the same vector that has been removed from the flat index, - // if it was being ingested at that time. writeMode is not protected since it is assumed to - // be called only from the "main thread" (that is the thread that is exclusively calling - // add/delete vector). + // Next, check if there vector(s) stored under the given label in HNSW and delete them as well. + // Note that we may remove the same vector that has been removed from the flat index, if it was + // being ingested at that time. + // writeMode is not protected since it is assumed to be called only from the "main thread" + // (that is the thread that is exclusively calling add/delete vector). if (this->getWriteMode() == VecSim_WriteAsync) { num_deleted_vectors += this->deleteLabelFromHNSW(label); // Apply ready swap jobs if number of deleted vectors reached the threshold @@ -935,20 +925,20 @@ int TieredHNSWIndex::deleteVector(labelType label) { /** * Move a label across every place the tiered index records one. Unlike add/delete this needs no - * jobs and no vector data movement - the swap/repair machinery is keyed purely on internal ids, - * so it is unaffected. + * jobs and no vector data movement - the swap/repair machinery is keyed purely on internal ids, so + * it is unaffected. * * A label can legitimately live in *both* tiers at once: `executeInsertJob` inserts into HNSW * before removing the vector from the flat buffer, so all four updates below are independently * guarded by their own existence check rather than treated as mutually exclusive. * - * Locking: `flatIndexGuard` exclusively (it guards the flat buffer's lookups, - * `labelToInsertJobs` and - per the note in `executeInsertJob` - the job fields themselves), - * then the main index guard exclusively. That is the flat-then-main order used by - * `insertVectorToHNSW` and `topKQueryImp`, so it cannot deadlock against them; repair and swap - * jobs only ever take the main guard. The main guard must be *exclusive* rather than shared: - * query threads read `getExternalLabel` under a shared main guard, and `ElementMetaData` is - * byte-packed, so its `label` field can be unaligned and its store is not atomic. + * Locking: `flatIndexGuard` exclusively (it guards the flat buffer's lookups, `labelToInsertJobs` + * and - per the note in `executeInsertJob` - the job fields themselves), then the main index guard + * exclusively. That is the flat-then-main order used by `insertVectorToHNSW` and `topKQueryImp`, + * so it cannot deadlock against them; repair and swap jobs only ever take the main guard. + * The main guard must be *exclusive* rather than shared: query threads read + * `getExternalLabel` under a shared main guard, and `ElementMetaData` is byte-packed, so its + * `label` field can be unaligned and its store is not atomic. */ template VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType old_label, @@ -962,9 +952,9 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o auto *hnsw_index = this->getHNSWIndex(); // A label can live in any subset of its three homes, so both questions are asked of all of - // them: it is present if any home holds it, and the target is free only if none does. - // Reject if the target is taken anywhere, so that a partially applied move is impossible - - // a free target in all three homes means none of the updates below can collide. + // them: it is present if any home holds it, and the target is free only if none does. Reject if + // the target is taken anywhere, so that a partially applied move is impossible - a free target + // in all three homes means none of the updates below can collide. const bool source_exists = this->frontendIndex->isLabelExists(old_label) || this->labelToInsertJobs.find(old_label) != this->labelToInsertJobs.end() || @@ -974,11 +964,10 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o this->labelToInsertJobs.find(new_label) != this->labelToInsertJobs.end() || hnsw_index->isLabelExists(new_label); - // Each home is asked to move the label only once it reported holding it, and the checks - // above ruled out every other rejection - the label is present, the target is free - // everywhere, and the labels differ - so while both guards are held a home that is asked - // can only answer OK. The asserts below state that; a refusal would mean the state changed - // underneath us. + // Each home is asked to move the label only once it reported holding it, and the checks above + // ruled out every other rejection - the label is present, the target is free everywhere, and + // the labels differ - so while both guards are held a home that is asked can only answer OK. + // The asserts below state that; a refusal would mean the state changed underneath us. if (source_exists && !target_taken) { if (this->frontendIndex->isLabelExists(old_label)) { const VecSimRelabelCode flat_ret = @@ -990,10 +979,10 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o UNUSED(flat_ret); } - // Re-key the pending insert jobs *and* rewrite each job's own copy of the label. Both - // must move together: `executeInsertJob` indexes into HNSW under `job->label` and then - // looks the job up with `labelToInsertJobs.at(job->label)`, so a half-applied move - // either indexes the vector under the stale label or throws out of the worker thread. + // Re-key the pending insert jobs *and* rewrite each job's own copy of the label. Both must + // move together: `executeInsertJob` indexes into HNSW under `job->label` and then looks the + // job up with `labelToInsertJobs.at(job->label)`, so a half-applied move either indexes the + // vector under the stale label or throws out of the worker thread. auto jobs_it = this->labelToInsertJobs.find(old_label); if (jobs_it != this->labelToInsertJobs.end()) { auto jobs = std::move(jobs_it->second); @@ -1019,8 +1008,7 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o this->flatIndexGuard.unlock(); // An absent source outranks an occupied target: a caller that resolves `NewLabelTaken` by - // freeing the target would otherwise drop an unrelated vector for a move with nothing to - // move. + // freeing the target would otherwise drop an unrelated vector for a move with nothing to move. if (!source_exists) { return VecSimRelabel_OldLabelMissing; } @@ -1033,30 +1021,25 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o // `getDistanceFrom` returns the minimum distance between the given blob and the vector with the // given label. If the label doesn't exist, the distance will be NaN. // Therefore, it's better to just call `getDistanceFrom` on both indexes and return the minimum -// instead of checking if the label exists in each index. We first try to get the distance from -// the flat buffer, as vectors in the buffer might move to the Main while we're "between" the -// locks. Behavior for single (regular) index: +// instead of checking if the label exists in each index. We first try to get the distance from the +// flat buffer, as vectors in the buffer might move to the Main while we're "between" the locks. +// Behavior for single (regular) index: // 1. label doesn't exist in both indexes - return NaN -// 2. label exists in one of the indexes only - return the distance from that index (which is -// valid) -// 3. label exists in both indexes - return the value from the flat buffer (which is valid and -// equal +// 2. label exists in one of the indexes only - return the distance from that index (which is valid) +// 3. label exists in both indexes - return the value from the flat buffer (which is valid and equal // to the value from the Main index), saving us from locking the Main index. // Behavior for multi index: // 1. label doesn't exist in both indexes - return NaN -// 2. label exists in one of the indexes only - return the distance from that index (which is -// valid) -// 3. label exists in both indexes - we may have some of the vectors with the same label in the -// flat +// 2. label exists in one of the indexes only - return the distance from that index (which is valid) +// 3. label exists in both indexes - we may have some of the vectors with the same label in the flat // buffer only and some in the Main index only (and maybe temporal duplications). // So, we get the distance from both indexes and return the minimum. -// IMPORTANT: this should be called when the *tiered index locks are locked for shared -// ownership*, along with HNSW index data guard lock. That is since the internal getDistanceFrom -// calls access the indexes' data, and it is not safe to run insert/delete operation in -// parallel. Also, we avoid acquiring the locks internally, since this is usually called for -// every vector individually, and the overhead of acquiring and releasing the locks is -// significant in that case. +// IMPORTANT: this should be called when the *tiered index locks are locked for shared ownership*, +// along with HNSW index data guard lock. That is since the internal getDistanceFrom calls access +// the indexes' data, and it is not safe to run insert/delete operation in parallel. Also, we avoid +// acquiring the locks internally, since this is usually called for every vector individually, and +// the overhead of acquiring and releasing the locks is significant in that case. template double TieredHNSWIndex::getDistanceFrom_Unsafe(labelType label, const void *blob) const { @@ -1064,8 +1047,8 @@ double TieredHNSWIndex::getDistanceFrom_Unsafe(labelType lab // If the label doesn't exist, the distance will be NaN. auto flat_dist = this->frontendIndex->getDistanceFrom_Unsafe(label, blob); - // Optimization. TODO: consider having different implementations for single and multi - // indexes, to avoid checking the index type on every query. + // Optimization. TODO: consider having different implementations for single and multi indexes, + // to avoid checking the index type on every query. if (!this->backendIndex->isMultiValue() && !std::isnan(flat_dist)) { // If the index is single value, and we got a valid distance from the flat buffer, // we can return the distance without querying the Main index. @@ -1097,8 +1080,8 @@ TieredHNSWIndex::TieredHNSW_BatchIterator::TieredHNSW_BatchI // Tiered batch iterator doesn't hold its own copy of the query vector. // Instead, each internal batch iterators (flat_iterator and hnsw_iterator) create their own // copies: flat_iterator copy is created during TieredHNSW_BatchIterator construction When - // TieredHNSW_BatchIterator::getNextResults() is called and hnsw_iterator is not - // initialized, it retrieves the blob from flat_iterator + // TieredHNSW_BatchIterator::getNextResults() is called and hnsw_iterator is not initialized, it + // retrieves the blob from flat_iterator : VecSimBatchIterator(nullptr, queryParams ? queryParams->timeoutCtx : nullptr, std::move(allocator)), index(index), flat_results(this->allocator), hnsw_results(this->allocator), @@ -1137,8 +1120,8 @@ VecSimQueryReply *TieredHNSWIndex::TieredHNSW_BatchIterator: auto hnsw_code = VecSim_QueryReply_OK; if (this->hnsw_iterator == UNINITIALIZED) { - // First call to getNextResults. The call to the BF iterator will include calculating - // all the distances and access the BF index. We take the lock on this call. + // First call to getNextResults. The call to the BF iterator will include calculating all + // the distances and access the BF index. We take the lock on this call. this->index->flatIndexGuard.lock_shared(); auto cur_flat_results = this->flat_iterator->getNextResults(n_res, BY_SCORE_THEN_ID); this->index->flatIndexGuard.unlock_shared(); @@ -1148,8 +1131,8 @@ VecSimQueryReply *TieredHNSWIndex::TieredHNSW_BatchIterator: } this->flat_results.swap(cur_flat_results->results); VecSimQueryReply_Free(cur_flat_results); - // We also take the lock on the main index on the first call to getNextResults, and we - // hold it until the iterator is depleted or freed. + // We also take the lock on the main index on the first call to getNextResults, and we hold + // it until the iterator is depleted or freed. this->index->mainIndexGuard.lock_shared(); this->hnsw_iterator = this->index->backendIndex->newBatchIterator( this->flat_iterator->getQueryBlob(), queryParams); @@ -1171,10 +1154,10 @@ VecSimQueryReply *TieredHNSWIndex::TieredHNSW_BatchIterator: VecSimQueryReply_Free(tail); if (!isMulti) { - // On single-value indexes, duplicates will never appear in the hnsw results - // before they appear in the flat results (at the same time or later if the - // approximation misses) so we don't need to try and filter the flat results - // (and recheck conditions). + // On single-value indexes, duplicates will never appear in the hnsw results before + // they appear in the flat results (at the same time or later if the approximation + // misses) so we don't need to try and filter the flat results (and recheck + // conditions). break; } else { // On multi-value indexes, the flat results may contain results that are already @@ -1188,9 +1171,9 @@ VecSimQueryReply *TieredHNSWIndex::TieredHNSW_BatchIterator: auto tail = this->hnsw_iterator->getNextResults(n_res - this->hnsw_results.size(), BY_SCORE_THEN_ID); hnsw_code = tail->code; // Set the hnsw_results code to the last `getNextResults` code. - // New batch may contain better results than the previous batch, so we need to - // merge. We don't expect duplications (hence the ), as the iterator - // guarantees that no result is returned twice. + // New batch may contain better results than the previous batch, so we need to merge. + // We don't expect duplications (hence the ), as the iterator guarantees that + // no result is returned twice. VecSimQueryResultContainer cur_hnsw_results(this->allocator); merge_results(cur_hnsw_results, this->hnsw_results, tail->results, n_res); VecSimQueryReply_Free(tail); @@ -1267,15 +1250,13 @@ TieredHNSWIndex::TieredHNSW_BatchIterator::compute_current_b auto [from_hnsw, from_flat] = p; if (!isMultiValue) { - // If we're on a single-value index, update the set of results returned from the FLAT - // index before popping them, to prevent them to be returned from the HNSW index in - // later batches. + // If we're on a single-value index, update the set of results returned from the FLAT index + // before popping them, to prevent them to be returned from the HNSW index in later batches. for (size_t i = 0; i < from_flat; ++i) { this->returned_results_set.insert(this->flat_results[i].id); } } else { - // If we're on a multi-value index, update the set of results returned (from - // `batch_res`) + // If we're on a multi-value index, update the set of results returned (from `batch_res`) for (size_t i = 0; i < batch_res->results.size(); ++i) { this->returned_results_set.insert(batch_res->results[i].id); } From 5f5c0f2db2e48312c28537a9bc65a718034b3f3c Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Mon, 14 Sep 2026 08:49:19 -0700 Subject: [PATCH 45/99] add missing file --- src/VecSim/utils/scoped_locks.h | 58 +++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 src/VecSim/utils/scoped_locks.h diff --git a/src/VecSim/utils/scoped_locks.h b/src/VecSim/utils/scoped_locks.h new file mode 100644 index 000000000..234047a9b --- /dev/null +++ b/src/VecSim/utils/scoped_locks.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ + +#pragma once + +#include +#include + +// A lockable resource, so that different mutex-like primitives (a shared_mutex, HNSW's own +// data guard, or nothing at all) can be composed under one RAII type below. +class Lockable { +public: + virtual void lock() const = 0; + virtual void unlock() const = 0; + virtual ~Lockable() = default; +}; + +// Adapts a std::shared_mutex, locked/unlocked for shared (read) access, to Lockable. +class SharedMutexLockable : public Lockable { + std::shared_mutex &guard; + +public: + explicit SharedMutexLockable(std::shared_mutex &guard) : guard(guard) {} + void lock() const override { guard.lock_shared(); } + void unlock() const override { guard.unlock_shared(); } +}; + +// RAII acquisition of zero, one or two Lockables, so callers can plug in whatever locking +// (or none) they need without repeating the lock/unlock bookkeeping. +class ScopedLocks { + std::array locks{}; + size_t count = 0; + +public: + ScopedLocks() = default; + explicit ScopedLocks(const Lockable &a) : locks{&a, nullptr}, count(1) { a.lock(); } + ScopedLocks(const Lockable &a, const Lockable &b) : locks{&a, &b}, count(2) { + a.lock(); + b.lock(); + } + ScopedLocks(ScopedLocks &&other) noexcept : locks(other.locks), count(other.count) { + other.count = 0; + } + ScopedLocks(const ScopedLocks &) = delete; + ScopedLocks &operator=(ScopedLocks &&) = delete; + ~ScopedLocks() { + // Unlock in reverse acquisition order. + for (size_t i = count; i-- > 0;) { + locks[i]->unlock(); + } + } +}; From 7dc20e5dfed54b03355b738c874d31801b6925c6 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Mon, 14 Sep 2026 21:16:28 -0700 Subject: [PATCH 46/99] use SVS rc2 binaries and latest submodule --- cmake/svs.cmake | 6 +++--- deps/ScalableVectorSearch | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index f6aebd87a..c1c04863b 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -64,7 +64,7 @@ if(USE_SVS) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.31" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "21.0") if(GLIBCXX_VERSION VERSION_GREATER_EQUAL "11") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc1-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc2-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") else() message(STATUS "libstdc++ >= GCC 11 is required for Clang SVS binaries - disabling SVS_SHARED_LIB") set(SVS_SHARED_LIB OFF) @@ -76,9 +76,9 @@ if(USE_SVS) else() if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.28") if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "14.0") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc1-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc2-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") else() - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc1-reduced.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc2-reduced.tar.gz" CACHE STRING "SVS URL") endif() else() message(STATUS "GLIBC >= 2.26 is required for GCC SVS binaries - disabling SVS_SHARED_LIB") diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch index 389597b59..3ee890d6d 160000 --- a/deps/ScalableVectorSearch +++ b/deps/ScalableVectorSearch @@ -1 +1 @@ -Subproject commit 389597b59cbfe87c6887da262673ef386261947e +Subproject commit 3ee890d6d52102e979ebd4331823b6236ec17a56 From 9f445ac9bd86eadd79086d1ed07b8d68e778ad20 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 14 Sep 2026 23:22:52 -0700 Subject: [PATCH 47/99] fix iterator --- src/VecSim/algorithms/svs/svs_tiered.h | 60 +++++++------------------- 1 file changed, 15 insertions(+), 45 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index ea262cbb8..7e91a98d1 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -305,19 +305,14 @@ class TieredSVSIndex : public VecSimTieredIndex { VecSimBatchIterator *flat_iterator; VecSimBatchIterator *svs_iterator; - // On single value indices, this set holds the IDs of the results that were returned from - // the flat buffer. - // On multi value indices, this set holds the IDs of all the results that were returned. - // The difference between the two cases is that on multi value indices, the same ID can - // appear in both indexes and results with different scores, and therefore we can't tell in - // advance when we expect a possibility of a duplicate. - // On single value indices, a duplicate may appear at the same batch (and we will handle it - // when merging the results) Or it may appear in a different batches, first from the flat - // buffer and then from the SVS, in the cases where a better result if found later in SVS - // because of the approximate nature of the algorithm. + // IDs of all the results returned so far, so that neither tier can yield one twice. + // Both directions have to be covered: nothing freezes the backend for the iterator's + // lifetime, so an insert job can publish a label into SVS and only then drop it from the + // flat buffer, while the flat iterator still holds it in the snapshot it computed on its + // first call. Either tier can therefore surface a label first. vecsim_stl::unordered_set returned_results_set; - VecSimQueryReply *compute_current_batch(size_t n_res, bool isMultiValue) { + VecSimQueryReply *compute_current_batch(size_t n_res) { // Merge results // This call will update `svs_res` and `bf_res` to point to the end of the merged // results. @@ -327,33 +322,18 @@ class TieredSVSIndex : public VecSimTieredIndex { auto [from_svs, from_flat] = merge_results(batch_res->results, svs_results, flat_results, n_res); - if (!isMultiValue) { - // If we're on a single-value index, update the set of results returned from the - // FLAT index before popping them, to prevent them to be returned from the SVS index - // in later batches. - for (size_t i = 0; i < from_flat; ++i) { - this->returned_results_set.insert(this->flat_results[i].id); - } - } else { - // If we're on a multi-value index, update the set of results returned (from - // `batch_res`) - for (size_t i = 0; i < batch_res->results.size(); ++i) { - this->returned_results_set.insert(batch_res->results[i].id); - } + for (size_t i = 0; i < batch_res->results.size(); ++i) { + this->returned_results_set.insert(batch_res->results[i].id); } // Update results flat_results.erase(flat_results.begin(), flat_results.begin() + from_flat); svs_results.erase(svs_results.begin(), svs_results.begin() + from_svs); - // clean up the results - // On multi-value indexes, one (or both) results lists may contain results that are - // already returned form the other list (with a different score). We need to filter them - // out. - if (isMultiValue) { - filter_irrelevant_results(this->flat_results); - filter_irrelevant_results(this->svs_results); - } + // Either list may still hold a label just returned from the other one (with a different + // score), so drop those before the next batch. + filter_irrelevant_results(this->flat_results); + filter_irrelevant_results(this->svs_results); // Return current batch return batch_res; @@ -422,7 +402,6 @@ class TieredSVSIndex : public VecSimTieredIndex { VecSimQueryReply *getNextResults(size_t n_res, VecSimQueryReply_Order order) override { auto svs_code = VecSim_QueryReply_OK; - const bool isMulti = this->index->backendIndex->isMultiValue(); if (svs_iterator == nullptr) { // first call // First call to getNextResults. The call to the BF iterator will include // calculating all the distances and access the BF index. We take the lock on this @@ -453,17 +432,8 @@ class TieredSVSIndex : public VecSimTieredIndex { tail->results.end()); VecSimQueryReply_Free(tail); - if (!isMulti) { - // On single-value indexes, duplicates will never appear in the hnsw results - // before they appear in the flat results (at the same time or later if the - // approximation misses) so we don't need to try and filter the flat results - // (and recheck conditions). - break; - } else { - // On multi-value indexes, the flat results may contain results that are - // already returned from the hnsw index. We need to filter them out. - filter_irrelevant_results(this->flat_results); - } + // The flat results may contain labels already returned from the SVS index. + filter_irrelevant_results(this->flat_results); } while (svs_results.size() < n_res && svs_iterator != depleted() && @@ -489,7 +459,7 @@ class TieredSVSIndex : public VecSimTieredIndex { } VecSimQueryReply *batch; - batch = compute_current_batch(n_res, isMulti); + batch = compute_current_batch(n_res); if (order == BY_ID) { sort_results_by_id(batch); From d934af9c890ee99c0a7bfb4d02a0b22f49273599 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Tue, 15 Sep 2026 00:27:37 -0700 Subject: [PATCH 48/99] relabelVector: switch check to concurent; rewrite impl call logic --- cmake/svs.cmake | 2 +- src/VecSim/algorithms/svs/svs.h | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index d777bc208..b9aef3431 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -126,7 +126,7 @@ if(USE_SVS) # so whether it is available depends on which SVS this build ended up with rather than on the # platform. Detect it in the header instead of assuming: it is a method, not a file, so this # greps the header the LVQ check would have tested for existence. - set(SVS_DYNAMIC_INDEX_HEADER "svs/index/vamana/dynamic_index.h") + set(SVS_DYNAMIC_INDEX_HEADER "svs/concurrent/dynamic_index.h") set(SVS_HAS_REPLACE_EXTERNAL_ID 0) if(EXISTS "${svs_SOURCE_DIR}/include/${SVS_DYNAMIC_INDEX_HEADER}") file(READ "${svs_SOURCE_DIR}/include/${SVS_DYNAMIC_INDEX_HEADER}" SVS_DYNAMIC_INDEX_SRC) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 2dc449e06..e41f5525d 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -610,16 +610,19 @@ class SVSIndex : public VecSimIndexAbstract, fl if (old_label == new_label) { return VecSimRelabel_SameLabel; } - // `isLabelExists` also covers the index that never held a vector, where `impl_` has not - // been created yet and so trivially holds nothing. - if (!isLabelExists(old_label)) { + + std::shared_lock mutation_lock(this->implMutationGuard_); + auto impl = getImpl(); + + // A null impl_ is an index that never held a vector, so it holds neither label. + if (!impl || !impl->has_id(old_label)) { return VecSimRelabel_OldLabelMissing; } - if (isLabelExists(new_label)) { + if (impl->has_id(new_label)) { return VecSimRelabel_NewLabelTaken; } - impl_->replace_external_id(old_label, new_label); + impl->replace_external_id(old_label, new_label); return VecSimRelabel_OK; } #endif // HAVE_SVS_REPLACE_EXTERNAL_ID From 89cd9b7385b36764639523c4a3f99f571a209698 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 15 Sep 2026 03:30:47 -0700 Subject: [PATCH 49/99] harmonize svs and hnsw tiered job types --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 17 ++++------------- src/VecSim/algorithms/svs/svs_tiered.h | 15 +++++---------- src/VecSim/vec_sim_tiered_index.h | 22 ++++++++++++++++++++++ 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index 6d2aeac1c..4ff54f44e 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -16,23 +16,14 @@ #include "hnsw.h" #include "VecSim/index_factories/hnsw_factory.h" -/** - * Definition of a job that inserts a new vector from flat into HNSW Index. - */ -struct HNSWInsertJob : public AsyncJob { - labelType label; - idType id; - - HNSWInsertJob(std::shared_ptr allocator, labelType label_, idType id_, - JobCallback insertCb, VecSimIndex *index_) - : AsyncJob(allocator, HNSW_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} -}; +using HNSWInsertJob = TieredInsertJob; /** * Definition of a job that swaps last id with a deleted id in HNSW Index after delete operation. */ struct HNSWSwapJob : public VecsimBaseObject { idType deleted_id; + std::atomic_int pending_repair_jobs_counter; // number of repair jobs left to complete before this job // is ready to be executed (atomic counter). @@ -53,14 +44,14 @@ static const size_t MAX_PENDING_SWAP_JOBS_THRESHOLD = 100000; * Definition of a job that repairs a certain node's connection in HNSW Index after delete * operation. */ -struct HNSWRepairJob : public AsyncJob { +struct HNSWRepairJob : public TieredJob { idType node_id; unsigned short level; vecsim_stl::vector associatedSwapJobs; HNSWRepairJob(std::shared_ptr allocator, idType id_, unsigned short level_, JobCallback repairCb, VecSimIndex *index_, HNSWSwapJob *swapJob) - : AsyncJob(allocator, HNSW_REPAIR_NODE_CONNECTIONS_JOB, repairCb, index_), node_id(id_), + : TieredJob(allocator, repairCb, index_), node_id(id_), level(level_), // Insert the first swap job from which this repair job was created. associatedSwapJobs(1, swapJob, this->allocator) {} diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 83dea01e4..d97867662 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -14,29 +14,24 @@ #include #include -/** - * Definition of a job that inserts a new vector from flat into SVS Index. - */ -struct SVSInsertJob : public AsyncJob { - labelType label; - idType id; +struct SVSInsertJob : public TieredInsertJob { std::atomic executing{false}; SVSInsertJob(std::shared_ptr allocator, labelType label_, idType id_, JobCallback insertCb, VecSimIndex *index_) - : AsyncJob(allocator, SVS_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} + : TieredInsertJob(allocator, label_, id_, insertCb, index_) {} }; /** * Definition of a job that launches partial consolidation on SVS Index. */ -struct SVSConsolidateJob : public AsyncJob { +struct SVSConsolidateJob : public TieredJob { std::vector labels; SVSConsolidateJob(std::shared_ptr allocator, const std::vector &labels_, JobCallback insertCb, VecSimIndex *index_) - : AsyncJob(allocator, SVS_CONSOLIDATE_JOB, insertCb, index_), labels(labels_) {} + : TieredJob(allocator, insertCb, index_), labels(labels_) {} }; /** @@ -579,7 +574,7 @@ class TieredSVSIndex : public VecSimTieredIndex { static void executeInsertJobWrapper(AsyncJob *job) { auto *insert_job = static_cast(job); auto *job_index = static_cast *>(insert_job->index); - InsertJobOutcome outcome; + InsertJobOutcome outcome = InsertJobOutcome::Completed; { // prevent parallel execution with index initilizing job std::shared_lock lock(job_index->updateJobMutex); diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 9a2ebb2ce..19751dbbb 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -41,6 +41,28 @@ struct AsyncJob : public VecsimBaseObject { isValid(true) {} }; +/** + * Common base for asynchronous jobs submitted by tiered indexes. + */ +template +struct TieredJob : public AsyncJob { + TieredJob(std::shared_ptr allocator, JobCallback callback, VecSimIndex *index) + : AsyncJob(allocator, Type, callback, index) {} +}; + +/** + * A job that inserts a new vector from the flat buffer to the backend. + */ +template +struct TieredInsertJob : public TieredJob { + labelType label; + idType id; + + TieredInsertJob(std::shared_ptr allocator, labelType label_, idType id_, + JobCallback insertCb, VecSimIndex *index_) + : TieredJob(allocator, insertCb, index_), label(label_), id(id_) {} +}; + // All read operations (including KNN, range, batch iterators and get-distance-from) are guaranteed // to consider all vectors that were added to the index before the query was submitted. The results // may include vectors that were added after the query was submitted, with no guarantees. From 4768855fa3cbc7aabdff43911f862fde87fe58bb Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 15 Sep 2026 05:25:42 -0700 Subject: [PATCH 50/99] Revert "harmonize svs and hnsw tiered job types" This reverts commit aedb77c87e756e3e46e810d7d60c2f5ea53033a6. --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 17 +++++++++++++---- src/VecSim/algorithms/svs/svs_tiered.h | 15 ++++++++++----- src/VecSim/vec_sim_tiered_index.h | 22 ---------------------- 3 files changed, 23 insertions(+), 31 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index 4ff54f44e..6d2aeac1c 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -16,14 +16,23 @@ #include "hnsw.h" #include "VecSim/index_factories/hnsw_factory.h" -using HNSWInsertJob = TieredInsertJob; +/** + * Definition of a job that inserts a new vector from flat into HNSW Index. + */ +struct HNSWInsertJob : public AsyncJob { + labelType label; + idType id; + + HNSWInsertJob(std::shared_ptr allocator, labelType label_, idType id_, + JobCallback insertCb, VecSimIndex *index_) + : AsyncJob(allocator, HNSW_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} +}; /** * Definition of a job that swaps last id with a deleted id in HNSW Index after delete operation. */ struct HNSWSwapJob : public VecsimBaseObject { idType deleted_id; - std::atomic_int pending_repair_jobs_counter; // number of repair jobs left to complete before this job // is ready to be executed (atomic counter). @@ -44,14 +53,14 @@ static const size_t MAX_PENDING_SWAP_JOBS_THRESHOLD = 100000; * Definition of a job that repairs a certain node's connection in HNSW Index after delete * operation. */ -struct HNSWRepairJob : public TieredJob { +struct HNSWRepairJob : public AsyncJob { idType node_id; unsigned short level; vecsim_stl::vector associatedSwapJobs; HNSWRepairJob(std::shared_ptr allocator, idType id_, unsigned short level_, JobCallback repairCb, VecSimIndex *index_, HNSWSwapJob *swapJob) - : TieredJob(allocator, repairCb, index_), node_id(id_), + : AsyncJob(allocator, HNSW_REPAIR_NODE_CONNECTIONS_JOB, repairCb, index_), node_id(id_), level(level_), // Insert the first swap job from which this repair job was created. associatedSwapJobs(1, swapJob, this->allocator) {} diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index d97867662..83dea01e4 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -14,24 +14,29 @@ #include #include -struct SVSInsertJob : public TieredInsertJob { +/** + * Definition of a job that inserts a new vector from flat into SVS Index. + */ +struct SVSInsertJob : public AsyncJob { + labelType label; + idType id; std::atomic executing{false}; SVSInsertJob(std::shared_ptr allocator, labelType label_, idType id_, JobCallback insertCb, VecSimIndex *index_) - : TieredInsertJob(allocator, label_, id_, insertCb, index_) {} + : AsyncJob(allocator, SVS_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} }; /** * Definition of a job that launches partial consolidation on SVS Index. */ -struct SVSConsolidateJob : public TieredJob { +struct SVSConsolidateJob : public AsyncJob { std::vector labels; SVSConsolidateJob(std::shared_ptr allocator, const std::vector &labels_, JobCallback insertCb, VecSimIndex *index_) - : TieredJob(allocator, insertCb, index_), labels(labels_) {} + : AsyncJob(allocator, SVS_CONSOLIDATE_JOB, insertCb, index_), labels(labels_) {} }; /** @@ -574,7 +579,7 @@ class TieredSVSIndex : public VecSimTieredIndex { static void executeInsertJobWrapper(AsyncJob *job) { auto *insert_job = static_cast(job); auto *job_index = static_cast *>(insert_job->index); - InsertJobOutcome outcome = InsertJobOutcome::Completed; + InsertJobOutcome outcome; { // prevent parallel execution with index initilizing job std::shared_lock lock(job_index->updateJobMutex); diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 19751dbbb..9a2ebb2ce 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -41,28 +41,6 @@ struct AsyncJob : public VecsimBaseObject { isValid(true) {} }; -/** - * Common base for asynchronous jobs submitted by tiered indexes. - */ -template -struct TieredJob : public AsyncJob { - TieredJob(std::shared_ptr allocator, JobCallback callback, VecSimIndex *index) - : AsyncJob(allocator, Type, callback, index) {} -}; - -/** - * A job that inserts a new vector from the flat buffer to the backend. - */ -template -struct TieredInsertJob : public TieredJob { - labelType label; - idType id; - - TieredInsertJob(std::shared_ptr allocator, labelType label_, idType id_, - JobCallback insertCb, VecSimIndex *index_) - : TieredJob(allocator, insertCb, index_), label(label_), id(id_) {} -}; - // All read operations (including KNN, range, batch iterators and get-distance-from) are guaranteed // to consider all vectors that were added to the index before the query was submitted. The results // may include vectors that were added after the query was submitted, with no guarantees. From 6248c257c9360f0b6c67e8af120259c49b3bec65 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Tue, 15 Sep 2026 07:32:43 -0700 Subject: [PATCH 51/99] adopt relabelVector for concuren svs --- src/VecSim/algorithms/svs/svs.h | 15 +- src/VecSim/algorithms/svs/svs_tiered.h | 192 ++++++++++++++++++------- 2 files changed, 154 insertions(+), 53 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index e41f5525d..3a6989ae5 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -615,15 +615,20 @@ class SVSIndex : public VecSimIndexAbstract, fl auto impl = getImpl(); // A null impl_ is an index that never held a vector, so it holds neither label. - if (!impl || !impl->has_id(old_label)) { + if (!impl) { return VecSimRelabel_OldLabelMissing; } - if (impl->has_id(new_label)) { + + // SVS checks both ends under its own translator lock + switch (impl->replace_external_id(old_label, new_label)) { + case svs::concurrent::ReplaceExternalIdResult::Ok: + return VecSimRelabel_OK; + case svs::concurrent::ReplaceExternalIdResult::NewIdExists: return VecSimRelabel_NewLabelTaken; + case svs::concurrent::ReplaceExternalIdResult::OldIdMissing: + break; } - - impl->replace_external_id(old_label, new_label); - return VecSimRelabel_OK; + return VecSimRelabel_OldLabelMissing; } #endif // HAVE_SVS_REPLACE_EXTERNAL_ID diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 83dea01e4..4d0ebe2cb 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -32,6 +32,7 @@ struct SVSInsertJob : public AsyncJob { */ struct SVSConsolidateJob : public AsyncJob { std::vector labels; + std::atomic executing{false}; SVSConsolidateJob(std::shared_ptr allocator, const std::vector &labels_, JobCallback insertCb, @@ -276,8 +277,9 @@ class TieredSVSIndex : public VecSimTieredIndex { idType currInvalidJobId; // A unique arbitrary identifier for accessing invalid jobs std::mutex invalidJobsLookupGuard; - vecsim_stl::unordered_set pendingConsolidateJobs; - std::mutex consolidateJobsGuard; + vecsim_stl::unordered_map> + labelToConsolidateJobs; + mutable std::shared_mutex consolidateJobsGuard; size_t flat_buffer_bound; @@ -641,11 +643,21 @@ class TieredSVSIndex : public VecSimTieredIndex { std::shared_lock lock(index->updateJobMutex); auto svs_index = index->GetSVSIndex(); svs_index->setParallelism(1); - svs_index->consolidate(consolidate_job->labels); + + bool valid = false; { - std::lock_guard jobs_lock(index->consolidateJobsGuard); - index->pendingConsolidateJobs.erase(job); + std::shared_lock flat_lock(index->flatIndexGuard); + valid = consolidate_job->isValid; + if (valid) { + consolidate_job->executing.store(true, std::memory_order_release); + } + } + if (valid) { + svs_index->consolidate(consolidate_job->labels); + // Cleared before the registry lock is needed again, so a waiter cannot deadlock us. + consolidate_job->executing.store(false, std::memory_order_release); } + index->forgetConsolidateJob(consolidate_job); delete job; } @@ -692,23 +704,79 @@ class TieredSVSIndex : public VecSimTieredIndex { } void scheduleSVSIndexConsolidate(labelType label) { - AsyncJob *new_consolidate_job = new (this->allocator) + auto *new_consolidate_job = new (this->allocator) SVSConsolidateJob(this->allocator, {label}, SVSIndexConsolidateWrapper, this); { - std::lock_guard lock(this->consolidateJobsGuard); - this->pendingConsolidateJobs.insert(new_consolidate_job); + std::lock_guard lock(this->consolidateJobsGuard); + auto it = this->labelToConsolidateJobs.find(label); + if (it != this->labelToConsolidateJobs.end()) { + it->second.push_back(new_consolidate_job); + } else { + vecsim_stl::vector jobs(1, new_consolidate_job, + this->allocator); + this->labelToConsolidateJobs.insert({label, std::move(jobs)}); + } } // Insert job to the queue. this->submitSingleJob(new_consolidate_job); } private: - idType setAndSaveInvalidJob(SVSInsertJob *job) { - // wait until insertion finishes + void forgetConsolidateJob(SVSConsolidateJob *job) { + std::lock_guard lock(this->consolidateJobsGuard); + for (auto label : job->labels) { + auto it = this->labelToConsolidateJobs.find(label); + if (it == this->labelToConsolidateJobs.end()) { + continue; + } + auto &jobs = it->second; + jobs.erase(std::remove(jobs.begin(), jobs.end(), job), jobs.end()); + if (jobs.empty()) { + this->labelToConsolidateJobs.erase(it); + } + } + } + // Caller must hold flatIndexGuard exclusive + std::vector takeOverConsolidateOf(labelType label) { + std::vector taken_over; + vecsim_stl::vector running(this->allocator); + { + std::shared_lock lock(this->consolidateJobsGuard); + auto it = this->labelToConsolidateJobs.find(label); + if (it == this->labelToConsolidateJobs.end()) { + return taken_over; + } + for (auto *job : it->second) { + if (!job->isValid) { + continue; // already taken over by an earlier call + } + if (job->executing.load(std::memory_order_acquire)) { + running.push_back(job); + } else { + job->isValid = false; + taken_over.insert(taken_over.end(), job->labels.begin(), job->labels.end()); + } + } + } + for (auto *job : running) { + while (job->executing.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + } + return taken_over; + } + + // Wait until the job leaves its publish window + // Safe to call while holding flatIndexGuard exclusive + static void waitForInsertJob(SVSInsertJob *job) { while (job->executing.load(std::memory_order_acquire)) { std::this_thread::yield(); } + } + + idType setAndSaveInvalidJob(SVSInsertJob *job) { + waitForInsertJob(job); this->invalidJobsLookupGuard.lock(); job->isValid = false; idType curInvalidId = currInvalidJobId++; @@ -870,7 +938,7 @@ class TieredSVSIndex : public VecSimTieredIndex { : Base(svs_index, bf_index, tiered_index_params, allocator), uncompletedJobs(this->allocator), labelToInsertJobs(this->allocator), invalidJobs(this->allocator), currInvalidJobId(0), - pendingConsolidateJobs(this->allocator) { + labelToConsolidateJobs(this->allocator) { const auto &tiered_svs_params = tiered_index_params.specificParams.tieredSVSParams; // If flatBufferLimit is not initialized (0), use the default update threshold. @@ -912,8 +980,10 @@ class TieredSVSIndex : public VecSimTieredIndex { } // Delete all the pending consolidate jobs - for (auto *job : this->pendingConsolidateJobs) { - delete job; + for (auto &jobs : this->labelToConsolidateJobs) { + for (auto *job : jobs.second) { + delete job; + } } } @@ -1179,12 +1249,6 @@ class TieredSVSIndex : public VecSimTieredIndex { * Reports `Unsupported` when the backend holds the label and cannot move it, which is the * case when built against an SVS without `replace_external_id`. All-or-nothing: on any code * other than `VecSimRelabel_OK` both tiers are untouched. - * - * `updateJobMutex` is taken first, in the order `updateSVSIndex` takes its own locks. Holding - * it is what makes this correct rather than merely serialised: an update job snapshots the - * buffer's labels *by value* and afterwards reconciles only id swaps and deletions, so a - * rename landing inside its window would be invisible to it and the vector would reach the - * backend under the old label. */ VecSimRelabelCode relabelVector(labelType old_label, labelType new_label) override { if (old_label == new_label) { @@ -1192,39 +1256,71 @@ class TieredSVSIndex : public VecSimTieredIndex { } auto *svs_index = GetSVSIndex(); - // Taken together through `std::lock`'s back-off rather than one after another, because - // the orders in this class disagree: nearly everything acquires flat before main, but the - // in-place add path takes `mainIndexGuard` before `flatIndexGuard` while the backend is - // still empty. Acquiring in sequence would hold one guard while blocking on another and - // close a cycle with that path; acquiring them together cannot, whichever order the other - // side uses. Same idiom that path already uses for its own two locks. - std::scoped_lock lock(this->updateJobMutex, this->flatIndexGuard, this->mainIndexGuard); - - const bool in_flat = this->frontendIndex->isLabelExists(old_label); - const bool in_backend = svs_index->isLabelExists(old_label); - if (!in_flat && !in_backend) { - return VecSimRelabel_OldLabelMissing; + std::shared_lock lock(this->updateJobMutex); + + bool flat_holds_old = false; + std::vector taken_over; + { + std::lock_guard flat_lock{this->flatIndexGuard}; + + flat_holds_old = this->frontendIndex->isLabelExists(old_label); + if (!flat_holds_old && !svs_index->isLabelExists(old_label)) { + return VecSimRelabel_OldLabelMissing; + } + if (this->frontendIndex->isLabelExists(new_label) || + svs_index->isLabelExists(new_label)) { + return VecSimRelabel_NewLabelTaken; + } + + auto pending = this->labelToInsertJobs.find(old_label); + if (pending != this->labelToInsertJobs.end()) { + auto jobs = std::move(pending->second); + this->labelToInsertJobs.erase(pending); + for (auto *job : jobs) { + waitForInsertJob(job); + job->label = new_label; + } + this->labelToInsertJobs.emplace(new_label, std::move(jobs)); + } + + // isLabelExists() said new_label is free, but SVS keeps the translator entry of a + // soft-deleted label until its consolidate job runs, and would refuse the remap. + taken_over = takeOverConsolidateOf(new_label); + + if (flat_holds_old) { + const VecSimRelabelCode flat_ret = + this->frontendIndex->relabelVector(old_label, new_label); + assert(flat_ret == VecSimRelabel_OK && + "the buffer just reported holding this label"); + UNUSED(flat_ret); + } } - if (this->frontendIndex->isLabelExists(new_label) || svs_index->isLabelExists(new_label)) { - return VecSimRelabel_NewLabelTaken; + + // Do the taken-over job's, + // the remap below isn't refused by a leftover translator entry + if (!taken_over.empty()) { + svs_index->consolidate(taken_over); } - // The backend goes first because it is the tier that can refuse; refusing after the - // buffer had already moved would leave the label half applied. - if (in_backend) { - const VecSimRelabelCode backend_ret = - this->backendIndex->relabelVector(old_label, new_label); - if (backend_ret != VecSimRelabel_OK) { - return backend_ret; + const VecSimRelabelCode backend_ret = + this->backendIndex->relabelVector(old_label, new_label); + if (backend_ret == VecSimRelabel_NewLabelTaken) { + // Only reachable if new_label was added concurrently with this rename + std::lock_guard flat_lock{this->flatIndexGuard}; + auto pending = this->labelToInsertJobs.find(new_label); + if (pending != this->labelToInsertJobs.end()) { + auto jobs = std::move(pending->second); + this->labelToInsertJobs.erase(pending); + for (auto *job : jobs) { + waitForInsertJob(job); + job->label = old_label; + } + this->labelToInsertJobs.emplace(old_label, std::move(jobs)); } - } - if (in_flat) { - const VecSimRelabelCode flat_ret = - this->frontendIndex->relabelVector(old_label, new_label); -#ifdef BUILD_TESTS - assert(flat_ret == VecSimRelabel_OK && "the buffer just reported holding this label"); -#endif - UNUSED(flat_ret); + if (flat_holds_old) { + this->frontendIndex->relabelVector(new_label, old_label); + } + return VecSimRelabel_NewLabelTaken; } return VecSimRelabel_OK; } From b641f176c717501ee636107b3c00ce3141730f86 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 01:24:14 -0700 Subject: [PATCH 52/99] move hnsw/svs tierd common logic to the base class --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 120 +++-------------------- src/VecSim/algorithms/svs/svs_tiered.h | 110 +++------------------ src/VecSim/vec_sim_tiered_index.h | 103 ++++++++++++++++++- tests/unit/test_hnsw_tiered.cpp | 10 +- 4 files changed, 137 insertions(+), 206 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index 3132e9bc7..5a7760875 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -24,13 +24,10 @@ /** * Definition of a job that inserts a new vector from flat into HNSW Index. */ -struct HNSWInsertJob : public AsyncJob { - labelType label; - idType id; - +struct HNSWInsertJob : public TieredInsertJob { HNSWInsertJob(std::shared_ptr allocator, labelType label_, idType id_, JobCallback insertCb, VecSimIndex *index_) - : AsyncJob(allocator, HNSW_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} + : TieredInsertJob(allocator, HNSW_INSERT_VECTOR_JOB, label_, id_, insertCb, index_) {} }; /** @@ -81,17 +78,10 @@ template class TieredHNSWIndex : public VecSimTieredIndex { private: /// Mappings from id/label to associated jobs, for invalidating and update ids if necessary. - // In MULTI, we can have more than one insert job pending per label. // **This map is protected with the flat buffer lock** - vecsim_stl::unordered_map> labelToInsertJobs; vecsim_stl::unordered_map> idToRepairJobs; vecsim_stl::unordered_map idToSwapJob; - // A mapping to hold invalid jobs, so we can dispose them upon index deletion. - vecsim_stl::unordered_map invalidJobs; - idType currInvalidJobId; // A unique arbitrary identifier for accessing invalid jobs - std::mutex invalidJobsLookupGuard; - // This threshold is tested upon deleting a label from HNSW, and once the number of deleted // vectors reached this limit, we apply swap jobs *only for vectors that has no more pending // repair jobs*, and are ready to be removed from the graph. @@ -172,13 +162,6 @@ class TieredHNSWIndex : public VecSimTieredIndex { inline HNSWIndex *getHNSWIndex() const; - // Helper function for deleting a vector from the flat buffer (after it has already been - // ingested into HNSW or deleted). This includes removing the corresponding insert job from the - // label-to-insert-jobs lookup. Also, since deletion a vector triggers swapping of the - // internal last id with the deleted vector id, here we update the pending insert job(s) for the - // last id (if needed). This should be called while *flat lock is held* (exclusive lock). - void updateInsertJobInternalId(idType prev_id, idType new_id, labelType label); - // Helper function for performing in place mark delete of vector(s) associated with a label // and creating the appropriate repair jobs for the effected connections. This should be called // while *HNSW shared lock is held* (shared locked). @@ -197,11 +180,6 @@ class TieredHNSWIndex : public VecSimTieredIndex { void insertVectorToHNSW(HNSWIndex *hnsw_index, labelType label, const void *blob); - // Set an insert/repair job as invalid, put the job pointer in the invalid jobs lookup under - // the current available id, increase it and return it (while holding invalidJobsLookupGuard). - // Returns the id that the job was stored under (to be set in the job id field). - idType setAndSaveInvalidJob(AsyncJob *job); - // Handle deletion of vector inplace considering that async deletion might occurred beforehand. int deleteLabelFromHNSWInplace(labelType label); @@ -540,22 +518,6 @@ int TieredHNSWIndex::deleteLabelFromHNSW(labelType label) { return internal_ids.size(); } -template -void TieredHNSWIndex::updateInsertJobInternalId(idType prev_id, idType new_id, - labelType label) { - // Update the pending job id, due to a swap that was caused after the removal of new_id. - assert(new_id != INVALID_ID && prev_id != INVALID_ID); - auto it = this->labelToInsertJobs.find(label); - if (it != this->labelToInsertJobs.end()) { - // There is a pending job for the label of the swapped last id - update its id. - for (HNSWInsertJob *job_it : it->second) { - if (job_it->id == prev_id) { - job_it->id = new_id; - } - } - } -} - template template void TieredHNSWIndex::insertVectorToHNSW( @@ -625,16 +587,6 @@ void TieredHNSWIndex::insertVectorToHNSW( } } -template -idType TieredHNSWIndex::setAndSaveInvalidJob(AsyncJob *job) { - this->invalidJobsLookupGuard.lock(); - job->isValid = false; - idType curInvalidId = currInvalidJobId++; - this->invalidJobs.insert({curInvalidId, job}); - this->invalidJobsLookupGuard.unlock(); - return curInvalidId; -} - template int TieredHNSWIndex::deleteLabelFromHNSWInplace(labelType label) { auto *hnsw_index = this->getHNSWIndex(); @@ -692,40 +644,7 @@ void TieredHNSWIndex::executeInsertJob(HNSWInsertJob *job) { #endif // Remove the vector and the insert job from the flat buffer. - this->flatIndexGuard.lock(); - // The job might have been invalidated due to overwrite in the meantime. In this case, - // it was already deleted and the job has been evicted. Otherwise, we need to do it now. - if (job->isValid) { - // Remove the job pointer from the labelToInsertJobs mapping. - auto &jobs = labelToInsertJobs.at(job->label); - for (size_t i = 0; i < jobs.size(); i++) { - if (jobs[i]->id == job->id) { - jobs.erase(jobs.begin() + (long)i); - break; - } - } - if (labelToInsertJobs.at(job->label).empty()) { - labelToInsertJobs.erase(job->label); - } - // Remove the vector from the flat buffer. This may cause the last vector id to swap with - // the deleted id. Hold the label for the last id, so we can later on update its - // corresponding job id. Note that after calling deleteVectorById, the last id's label - // shouldn't be available, since it is removed from the lookup. - labelType last_vec_label = - this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); - int deleted = this->frontendIndex->deleteVectorById(job->label, job->id); - if (deleted && job->id != this->frontendIndex->indexSize()) { - // If the vector removal caused a swap with the last id, update the relevant insert job. - this->updateInsertJobInternalId(this->frontendIndex->indexSize(), job->id, - last_vec_label); - } - } else { - // Remove the current job from the invalid jobs' lookup, as we are about to delete it now. - this->invalidJobsLookupGuard.lock(); - this->invalidJobs.erase(job->id); - this->invalidJobsLookupGuard.unlock(); - } - this->flatIndexGuard.unlock(); + this->removeIngestedVectorFromFlat(job); } template @@ -796,9 +715,7 @@ TieredHNSWIndex::TieredHNSWIndex(HNSWIndex allocator) : VecSimTieredIndex(hnsw_index, bf_index, tiered_index_params, allocator), - labelToInsertJobs(this->allocator), idToRepairJobs(this->allocator), - idToSwapJob(this->allocator), invalidJobs(this->allocator), currInvalidJobId(0), - readySwapJobs(0), + idToRepairJobs(this->allocator), idToSwapJob(this->allocator), readySwapJobs(0), isQuantized(tiered_index_params.primaryIndexParams->algoParams.hnswParams.quantType != VecSimQuant_NONE) { const auto &hnsw_params = tiered_index_params.primaryIndexParams->algoParams.hnswParams; @@ -818,12 +735,6 @@ TieredHNSWIndex::TieredHNSWIndex(HNSWIndex TieredHNSWIndex::~TieredHNSWIndex() { - // Delete all the pending insert jobs. - for (auto &jobs : this->labelToInsertJobs) { - for (auto *job : jobs.second) { - delete job; - } - } // Delete all the pending repair jobs. for (auto &jobs : this->idToRepairJobs) { for (auto *job : jobs.second) { @@ -834,10 +745,6 @@ TieredHNSWIndex::~TieredHNSWIndex() { for (auto &it : this->idToSwapJob) { delete it.second; } - // Delete all the pending invalid jobs. - for (auto &it : this->invalidJobs) { - delete it.second; - } } // Training writes stay in FLAT in both write modes, even beyond flatBufferLimit. @@ -848,20 +755,21 @@ int TieredHNSWIndex::addVectorDuringAccumulation(const void const auto [result, count, should_finalize] = [&] { std::lock_guard flat_lock(this->flatIndexGuard); idType id = this->frontendIndex->indexSize(); - HNSWInsertJob *job = nullptr; + TieredInsertJob *job = nullptr; if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { id = static_cast *>(this->frontendIndex) ->getIdOfLabel(label); subtractFromSum( {this->frontendIndex->getDataByInternalId(id), this->frontendIndex->getDim()}); - job = labelToInsertJobs.at(label).front(); + job = this->labelToInsertJobs.at(label).front(); } const int result = this->frontendIndex->addVector(blob, label); addToSum({this->frontendIndex->getDataByInternalId(id), this->frontendIndex->getDim()}); if (!job) { job = new (this->allocator) HNSWInsertJob(this->allocator, label, id, executeInsertJobWrapper, this); - auto [it, inserted] = labelToInsertJobs.try_emplace(label, 1, job, this->allocator); + auto [it, inserted] = + this->labelToInsertJobs.try_emplace(label, 1, job, this->allocator); if (!inserted) { it->second.push_back(job); } @@ -880,8 +788,8 @@ int TieredHNSWIndex::addVectorDuringAccumulation(const void template int TieredHNSWIndex::deleteVectorDuringAccumulation(labelType label) { std::lock_guard flat_lock(this->flatIndexGuard); - auto it = labelToInsertJobs.find(label); - if (it == labelToInsertJobs.end()) { + auto it = this->labelToInsertJobs.find(label); + if (it == this->labelToInsertJobs.end()) { return 0; } const int removed = it->second.size(); @@ -890,10 +798,10 @@ int TieredHNSWIndex::deleteVectorDuringAccumulation(labelTyp {this->frontendIndex->getDataByInternalId(job->id), this->frontendIndex->getDim()}); delete job; // Never submitted, so no worker can still reference this job. } - labelToInsertJobs.erase(it); + this->labelToInsertJobs.erase(it); auto updated_ids = this->frontendIndex->deleteVectorAndGetUpdatedIds(label); for (const auto &[new_id, previous] : updated_ids) { - updateInsertJobInternalId(previous.first, new_id, previous.second); + this->updateInsertJobInternalId(previous.first, new_id, previous.second); } return removed; } @@ -920,7 +828,7 @@ void TieredHNSWIndex::finalizeQuantizationAndSubmitJobs(size jobs.reserve(count); { std::shared_lock flat_lock(this->flatIndexGuard); - for (const auto &[label, pending] : labelToInsertJobs) { + for (const auto &[label, pending] : this->labelToInsertJobs) { jobs.insert(jobs.end(), pending.begin(), pending.end()); } } @@ -1177,7 +1085,7 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o if (jobs_it != this->labelToInsertJobs.end()) { auto jobs = std::move(jobs_it->second); this->labelToInsertJobs.erase(jobs_it); - for (HNSWInsertJob *job : jobs) { + for (TieredInsertJob *job : jobs) { job->label = new_label; } this->labelToInsertJobs.emplace(new_label, std::move(jobs)); diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 4d0ebe2cb..0e49c4820 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -17,14 +17,12 @@ /** * Definition of a job that inserts a new vector from flat into SVS Index. */ -struct SVSInsertJob : public AsyncJob { - labelType label; - idType id; +struct SVSInsertJob : public TieredInsertJob { std::atomic executing{false}; SVSInsertJob(std::shared_ptr allocator, labelType label_, idType id_, JobCallback insertCb, VecSimIndex *index_) - : AsyncJob(allocator, SVS_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} + : TieredInsertJob(allocator, SVS_INSERT_VECTOR_JOB, label_, id_, insertCb, index_) {} }; /** @@ -271,12 +269,6 @@ class TieredSVSIndex : public VecSimTieredIndex { std::atomic backendInitSubmited{false}; std::unordered_set ids_to_init_; - vecsim_stl::unordered_map> labelToInsertJobs; - // A mapping to hold invalid jobs, so we can dispose them upon index deletion. - vecsim_stl::unordered_map invalidJobs; - idType currInvalidJobId; // A unique arbitrary identifier for accessing invalid jobs - std::mutex invalidJobsLookupGuard; - vecsim_stl::unordered_map> labelToConsolidateJobs; mutable std::shared_mutex consolidateJobsGuard; @@ -665,20 +657,6 @@ class TieredSVSIndex : public VecSimTieredIndex { public: #endif - void updateInsertJobInternalId(idType prev_id, idType new_id, labelType label) { - // Update the pending job id, due to a swap that was caused after the removal of new_id. - assert(new_id != INVALID_ID && prev_id != INVALID_ID); - auto it = this->labelToInsertJobs.find(label); - if (it != this->labelToInsertJobs.end()) { - // There is a pending job for the label of the swapped last id - update its id. - for (SVSInsertJob *job_it : it->second) { - if (job_it->id == prev_id) { - job_it->id = new_id; - } - } - } - } - void scheduleSVSIndexInit() { // do not schedule if scheduled already if (indexUpdateScheduled.test_and_set()) { @@ -760,29 +738,23 @@ class TieredSVSIndex : public VecSimTieredIndex { } } for (auto *job : running) { - while (job->executing.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } + waitForJob(job); } return taken_over; } // Wait until the job leaves its publish window // Safe to call while holding flatIndexGuard exclusive - static void waitForInsertJob(SVSInsertJob *job) { + template + static void waitForJob(Job *job) { while (job->executing.load(std::memory_order_acquire)) { std::this_thread::yield(); } } - idType setAndSaveInvalidJob(SVSInsertJob *job) { - waitForInsertJob(job); - this->invalidJobsLookupGuard.lock(); - job->isValid = false; - idType curInvalidId = currInvalidJobId++; - this->invalidJobs.insert({curInvalidId, job}); - this->invalidJobsLookupGuard.unlock(); - return curInvalidId; + idType setAndSaveInvalidJob(AsyncJob *job) override { + waitForJob(static_cast(job)); + return Base::setAndSaveInvalidJob(job); } enum class InsertJobOutcome { Completed, Deferred }; @@ -822,43 +794,7 @@ class TieredSVSIndex : public VecSimTieredIndex { job->executing.store(false, std::memory_order_release); // Remove the vector and the insert job from the flat buffer. - this->flatIndexGuard.lock(); - // The job might have been invalidated due to overwrite in the meantime. In this case, - // it was already deleted and the job has been evicted. Otherwise, we need to do it now. - if (job->isValid) { - // Remove the job pointer from the labelToInsertJobs mapping. - auto &jobs = labelToInsertJobs.at(job->label); - for (size_t i = 0; i < jobs.size(); i++) { - if (jobs[i]->id == job->id) { - jobs.erase(jobs.begin() + (long)i); - break; - } - } - if (labelToInsertJobs.at(job->label).empty()) { - labelToInsertJobs.erase(job->label); - } - // Remove the vector from the flat buffer. - // The flat buffer stores data in a contiguous - // array, so deleting an element may move the last element into the freed slot to keep - // ids dense. Capture the last id's label beforehand (after deletion it is no longer in - // the lookup) so we can later fix up its insert job. - labelType last_vec_label = - this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); - int deleted = this->frontendIndex->deleteVectorById(job->label, job->id); - if (deleted && job->id != this->frontendIndex->indexSize()) { - // If the vector removal caused a swap with the last id, update the relevant insert - // job. - this->updateInsertJobInternalId(this->frontendIndex->indexSize(), job->id, - last_vec_label); - } - } else { - // Remove the current job from the invalid jobs' lookup, as we are about to delete it - // now. - this->invalidJobsLookupGuard.lock(); - this->invalidJobs.erase(job->id); - this->invalidJobsLookupGuard.unlock(); - } - this->flatIndexGuard.unlock(); + this->removeIngestedVectorFromFlat(job); return InsertJobOutcome::Completed; } @@ -936,9 +872,7 @@ class TieredSVSIndex : public VecSimTieredIndex { const TieredIndexParams &tiered_index_params, std::shared_ptr allocator) : Base(svs_index, bf_index, tiered_index_params, allocator), - uncompletedJobs(this->allocator), labelToInsertJobs(this->allocator), - invalidJobs(this->allocator), currInvalidJobId(0), - labelToConsolidateJobs(this->allocator) { + uncompletedJobs(this->allocator), labelToConsolidateJobs(this->allocator) { const auto &tiered_svs_params = tiered_index_params.specificParams.tieredSVSParams; // If flatBufferLimit is not initialized (0), use the default update threshold. @@ -967,18 +901,6 @@ class TieredSVSIndex : public VecSimTieredIndex { } ~TieredSVSIndex() { - // Delete all the pending insert jobs. - for (auto &jobs : this->labelToInsertJobs) { - for (auto *job : jobs.second) { - delete job; - } - } - - // Delete all the pending invalid jobs. - for (auto &it : this->invalidJobs) { - delete it.second; - } - // Delete all the pending consolidate jobs for (auto &jobs : this->labelToConsolidateJobs) { for (auto *job : jobs.second) { @@ -1109,7 +1031,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // If this label already exists, this will do overwrite. ret += this->frontendIndex->addVector(blob, label); - AsyncJob *new_insert_job = new (this->allocator) + TieredInsertJob *new_insert_job = new (this->allocator) SVSInsertJob(this->allocator, label, new_flat_id, executeInsertJobWrapper, this); // Save a pointer to the job, so that if the vector is overwritten, we'll have an @@ -1118,10 +1040,10 @@ class TieredSVSIndex : public VecSimTieredIndex { // There's already a pending insert job for this label, add another one (without // overwrite, only possible in multi index) assert(this->backendIndex->isMultiValue()); - this->labelToInsertJobs.at(label).push_back((SVSInsertJob *)new_insert_job); + this->labelToInsertJobs.at(label).push_back(new_insert_job); } else { - vecsim_stl::vector new_jobs_vec(1, (SVSInsertJob *)new_insert_job, - this->allocator); + vecsim_stl::vector new_jobs_vec(1, new_insert_job, + this->allocator); this->labelToInsertJobs.insert({label, new_jobs_vec}); } this->flatIndexGuard.unlock(); @@ -1277,7 +1199,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto jobs = std::move(pending->second); this->labelToInsertJobs.erase(pending); for (auto *job : jobs) { - waitForInsertJob(job); + waitForJob(static_cast(job)); job->label = new_label; } this->labelToInsertJobs.emplace(new_label, std::move(jobs)); @@ -1312,7 +1234,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto jobs = std::move(pending->second); this->labelToInsertJobs.erase(pending); for (auto *job : jobs) { - waitForInsertJob(job); + waitForJob(static_cast(job)); job->label = old_label; } this->labelToInsertJobs.emplace(old_label, std::move(jobs)); diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index ce159ec46..84fee4338 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -43,6 +43,19 @@ struct AsyncJob : public VecsimBaseObject { isValid(true) {} }; +/** + * Definition of a job that inserts a new vector from flat into the backend index. + * Backend specific insert jobs derive from it to set their own job type. + */ +struct TieredInsertJob : public AsyncJob { + labelType label; + idType id; + + TieredInsertJob(std::shared_ptr allocator, JobType type, labelType label_, + idType id_, JobCallback insertCb, VecSimIndex *index_) + : AsyncJob(allocator, type, insertCb, index_), label(label_), id(id_) {} +}; + // All read operations (including KNN, range, batch iterators and get-distance-from) are guaranteed // to consider all vectors that were added to the index before the query was submitted. The results // may include vectors that were added after the query was submitted, with no guarantees. @@ -134,6 +147,83 @@ class VecSimTieredIndex : public VecSimIndexInterface { VecSimQueryParams *queryParams, VecSimQueryReply_Order order) const; +#ifdef BUILD_TESTS +public: +#endif + /// Mappings from id/label to associated jobs, for invalidating and update ids if necessary. + // In MULTI, we can have more than one insert job pending per label. + // **This map is protected with the flat buffer lock** + vecsim_stl::unordered_map> labelToInsertJobs; + + // Helper function for updating the pending insert job(s) of a label after the flat buffer + // swapped the vector's internal id + void updateInsertJobInternalId(idType prev_id, idType new_id, labelType label) { + // Update the pending job id, due to a swap that was caused after the removal of new_id. + assert(new_id != INVALID_ID && prev_id != INVALID_ID); + auto it = this->labelToInsertJobs.find(label); + if (it != this->labelToInsertJobs.end()) { + // There is a pending job for the label of the swapped last id - update its id. + for (TieredInsertJob *job_it : it->second) { + if (job_it->id == prev_id) { + job_it->id = new_id; + } + } + } + } + + // A mapping to hold invalid jobs, so we can dispose them upon index deletion. + vecsim_stl::unordered_map invalidJobs; + idType currInvalidJobId; // A unique arbitrary identifier for accessing invalid jobs + std::mutex invalidJobsLookupGuard; + + // Set an insert/repair job as invalid, put the job pointer in the invalid jobs lookup under + // the current available id, increase it and return it (while holding invalidJobsLookupGuard). + // Returns the id that the job was stored under (to be set in the job id field). + virtual idType setAndSaveInvalidJob(AsyncJob *job) { + std::lock_guard lock(this->invalidJobsLookupGuard); + job->isValid = false; + idType curInvalidId = currInvalidJobId++; + this->invalidJobs.insert({curInvalidId, job}); + return curInvalidId; + } + + // Remove a vector and its insert job from the flat buffer + void removeIngestedVectorFromFlat(TieredInsertJob *job) { + std::lock_guard flat_lock(this->flatIndexGuard); + // The job might have been invalidated due to overwrite in the meantime. In this case, + // it was already deleted and the job has been evicted. Otherwise, we need to do it now. + if (!job->isValid) { + // Remove the current job from the invalid jobs' lookup, as we are about to delete it + // now. + std::lock_guard invalid_jobs_lock(this->invalidJobsLookupGuard); + this->invalidJobs.erase(job->id); + return; + } + // Remove the job pointer from the labelToInsertJobs mapping. + auto &jobs = this->labelToInsertJobs.at(job->label); + for (size_t i = 0; i < jobs.size(); i++) { + if (jobs[i]->id == job->id) { + jobs.erase(jobs.begin() + (long)i); + break; + } + } + if (jobs.empty()) { + this->labelToInsertJobs.erase(job->label); + } + // Remove the vector from the flat buffer. This may cause the last vector id to swap with + // the deleted id. Hold the label for the last id, so we can later on update its + // corresponding job id. Note that after calling deleteVectorById, the last id's label + // shouldn't be available, since it is removed from the lookup. + labelType last_vec_label = + this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); + int deleted = this->frontendIndex->deleteVectorById(job->label, job->id); + if (deleted && job->id != this->frontendIndex->indexSize()) { + // If the vector removal caused a swap with the last id, update the relevant insert job. + this->updateInsertJobInternalId(this->frontendIndex->indexSize(), job->id, + last_vec_label); + } + } + public: /** * @brief Get the vector elements stored under a label, in insertion order. @@ -207,11 +297,22 @@ class VecSimTieredIndex : public VecSimIndexInterface { : VecSimIndexInterface(allocator), backendIndex(backendIndex_), frontendIndex(frontendIndex_), jobQueue(tieredParams.jobQueue), jobQueueCtx(tieredParams.jobQueueCtx), SubmitJobsToQueue(tieredParams.submitCb), - flatBufferLimit(tieredParams.flatBufferLimit) { + flatBufferLimit(tieredParams.flatBufferLimit), labelToInsertJobs(this->allocator), + invalidJobs(this->allocator), currInvalidJobId(0) { assert(backendIndex != nullptr); } virtual ~VecSimTieredIndex() { + // Delete all the pending insert jobs. + for (auto &jobs : this->labelToInsertJobs) { + for (auto *job : jobs.second) { + delete job; + } + } + // Delete all the pending invalid jobs. + for (auto &it : this->invalidJobs) { + delete it.second; + } VecSimIndex_Free(backendIndex); VecSimIndex_Free(frontendIndex); } diff --git a/tests/unit/test_hnsw_tiered.cpp b/tests/unit/test_hnsw_tiered.cpp index 9b3600cdf..29702c24c 100644 --- a/tests/unit/test_hnsw_tiered.cpp +++ b/tests/unit/test_hnsw_tiered.cpp @@ -124,7 +124,7 @@ TYPED_TEST(HNSWTieredIndexTest, CreateIndexInstance) { auto job = new (allocator) HNSWInsertJob(tiered_index->allocator, vector_label, 0, insert_to_index, tiered_index); - auto jobs_vec = vecsim_stl::vector(1, job, allocator); + auto jobs_vec = vecsim_stl::vector(1, job, allocator); tiered_index->labelToInsertJobs.insert({vector_label, jobs_vec}); // Wrap this job with an array and submit the jobs to the queue. @@ -339,10 +339,10 @@ TYPED_TEST(HNSWTieredIndexTest, addVector) { sizeof(void *) + sizeof(size_t); // Account for the memory that was allocated in the labelToInsertJobs map (approx.) expected_mem += - sizeof( - vecsim_stl::unordered_map>::value_type) + + sizeof(vecsim_stl::unordered_map>::value_type) + sizeof(void *) + sizeof(size_t); - // Account for the inner buffer of the std::vector in the map. + // Account for the inner buffer of the std::vector in the map. expected_mem += sizeof(void *) + sizeof(size_t); // Account for the insert job that was created. expected_mem += sizeof(HNSWInsertJob) + sizeof(size_t); @@ -4988,7 +4988,7 @@ TYPED_TEST(HNSWTieredIndexTestBasic, relabelVectorMulti) { } ASSERT_EQ(tiered_index->labelToInsertJobs.count(7), 0); ASSERT_EQ(tiered_index->labelToInsertJobs.at(70).size(), per_label); - for (HNSWInsertJob *job : tiered_index->labelToInsertJobs.at(70)) { + for (TieredInsertJob *job : tiered_index->labelToInsertJobs.at(70)) { ASSERT_EQ(job->label, 70); } From f36ff4b1dc7f89ccd0ec512ed835a2e4a2bc561a Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 01:32:18 -0700 Subject: [PATCH 53/99] format --- tests/unit/test_hnsw_tiered.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_hnsw_tiered.cpp b/tests/unit/test_hnsw_tiered.cpp index 29702c24c..46afdede6 100644 --- a/tests/unit/test_hnsw_tiered.cpp +++ b/tests/unit/test_hnsw_tiered.cpp @@ -340,7 +340,7 @@ TYPED_TEST(HNSWTieredIndexTest, addVector) { // Account for the memory that was allocated in the labelToInsertJobs map (approx.) expected_mem += sizeof(vecsim_stl::unordered_map>::value_type) + + vecsim_stl::vector>::value_type) + sizeof(void *) + sizeof(size_t); // Account for the inner buffer of the std::vector in the map. expected_mem += sizeof(void *) + sizeof(size_t); From 93a51513a7a9291c524fb494a5c6c9f61093b51e Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 16 Sep 2026 01:42:01 -0700 Subject: [PATCH 54/99] upstream getDistanceFrom_Unsafe --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 53 ++---------------------- src/VecSim/algorithms/svs/svs_tiered.h | 20 --------- src/VecSim/vec_sim_tiered_index.h | 45 ++++++++++++++++++++ 3 files changed, 49 insertions(+), 69 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index 5a7760875..9180b91ff 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -288,7 +288,6 @@ class TieredHNSWIndex : public VecSimTieredIndex { int deleteVector(labelType label) override; VecSimRelabelCode relabelVector(labelType old_label, labelType new_label) override; size_t getNumMarkedDeleted() const override { return getHNSWIndex()->getNumMarkedDeleted(); } - double getDistanceFrom_Unsafe(labelType label, const void *blob) const override; // Do nothing here, each tier (flat buffer and HNSW) should increase capacity for itself when // needed. VecSimIndexDebugInfo debugInfo() const override; @@ -342,16 +341,16 @@ class TieredHNSWIndex : public VecSimTieredIndex { /* Helper methods */ template void TieredHNSWIndex::executeInsertJobWrapper(AsyncJob *job) { - auto *insert_job = reinterpret_cast(job); - auto *job_index = reinterpret_cast *>(insert_job->index); + auto *insert_job = static_cast(job); + auto *job_index = static_cast *>(insert_job->index); job_index->executeInsertJob(insert_job); delete job; } template void TieredHNSWIndex::executeRepairJobWrapper(AsyncJob *job) { - auto *repair_job = reinterpret_cast(job); - auto *job_index = reinterpret_cast *>(repair_job->index); + auto *repair_job = static_cast(job); + auto *job_index = static_cast *>(repair_job->index); job_index->executeRepairJob(repair_job); delete job; } @@ -1116,50 +1115,6 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o return VecSimRelabel_OK; } -// `getDistanceFrom` returns the minimum distance between the given blob and the vector with the -// given label. If the label doesn't exist, the distance will be NaN. -// Therefore, it's better to just call `getDistanceFrom` on both indexes and return the minimum -// instead of checking if the label exists in each index. We first try to get the distance from the -// flat buffer, as vectors in the buffer might move to the Main while we're "between" the locks. -// Behavior for single (regular) index: -// 1. label doesn't exist in both indexes - return NaN -// 2. label exists in one of the indexes only - return the distance from that index (which is valid) -// 3. label exists in both indexes - return the value from the flat buffer (which is valid and equal -// to the value from the Main index), saving us from locking the Main index. -// Behavior for multi index: -// 1. label doesn't exist in both indexes - return NaN -// 2. label exists in one of the indexes only - return the distance from that index (which is valid) -// 3. label exists in both indexes - we may have some of the vectors with the same label in the flat -// buffer only and some in the Main index only (and maybe temporal duplications). -// So, we get the distance from both indexes and return the minimum. - -// IMPORTANT: this should be called when the *tiered index locks are locked for shared ownership*, -// along with HNSW index data guard lock. That is since the internal getDistanceFrom calls access -// the indexes' data, and it is not safe to run insert/delete operation in parallel. Also, we avoid -// acquiring the locks internally, since this is usually called for every vector individually, and -// the overhead of acquiring and releasing the locks is significant in that case. -template -double TieredHNSWIndex::getDistanceFrom_Unsafe(labelType label, - const void *blob) const { - // Try to get the distance from the flat buffer. - // If the label doesn't exist, the distance will be NaN. - auto flat_dist = this->frontendIndex->getDistanceFrom_Unsafe(label, blob); - - // Optimization. TODO: consider having different implementations for single and multi indexes, - // to avoid checking the index type on every query. - if (!this->backendIndex->isMultiValue() && !std::isnan(flat_dist)) { - // If the index is single value, and we got a valid distance from the flat buffer, - // we can return the distance without querying the Main index. - return flat_dist; - } - - // Try to get the distance from the Main index. - auto hnsw_dist = getHNSWIndex()->getDistanceFrom_Unsafe(label, blob); - - // Return the minimum distance that is not NaN. - return std::fmin(flat_dist, hnsw_dist); -} - //////////////////////////////////////////////////////////////////////////////////////////////////// // TieredHNSW_BatchIterator // //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 0e49c4820..67955d724 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -1253,26 +1253,6 @@ class TieredSVSIndex : public VecSimTieredIndex { return this->GetSVSIndex()->getNumMarkedDeleted(); } - double getDistanceFrom_Unsafe(labelType label, const void *blob) const override { - // Try to get the distance from the flat buffer. - // If the label doesn't exist, the distance will be NaN. - auto flat_dist = this->frontendIndex->getDistanceFrom_Unsafe(label, blob); - - // Optimization. TODO: consider having different implementations for single and multi - // indexes, to avoid checking the index type on every query. - if (!this->backendIndex->isMultiValue() && !std::isnan(flat_dist)) { - // If the index is single value, and we got a valid distance from the flat buffer, - // we can return the distance without querying the Main index. - return flat_dist; - } - - // Try to get the distance from the Main index. - auto svs_dist = this->backendIndex->getDistanceFrom_Unsafe(label, blob); - - // Return the minimum distance that is not NaN. - return std::fmin(flat_dist, svs_dist); - } - VecSimIndexDebugInfo debugInfo() const override { auto info = Base::debugInfo(); diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 84fee4338..d28b7c9f2 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -18,6 +18,7 @@ #include "VecSim/utils/scoped_locks.h" #include +#include #include #include @@ -291,6 +292,50 @@ class VecSimTieredIndex : public VecSimIndexInterface { } } + // `getDistanceFrom` returns the minimum distance between the given blob and the vector with + // the given label. If the label doesn't exist, the distance will be NaN. + // Therefore, it's better to just call `getDistanceFrom` on both indexes and return the minimum + // instead of checking if the label exists in each index. We first try to get the distance from + // the flat buffer, as vectors in the buffer might move to the backend while we're "between" + // the locks. + // Behavior for single (regular) index: + // 1. label doesn't exist in both indexes - return NaN + // 2. label exists in one of the indexes only - return the distance from that index (valid) + // 3. label exists in both indexes - return the value from the flat buffer (valid and equal to + // the value from the backend index), saving us from locking the backend index. + // Behavior for multi index: + // 1. label doesn't exist in both indexes - return NaN + // 2. label exists in one of the indexes only - return the distance from that index (valid) + // 3. label exists in both indexes - we may have some of the vectors with the same label in the + // flat buffer only and some in the backend index only (and maybe temporal duplications). So, + // we get the distance from both indexes and return the minimum. + // + // IMPORTANT: this should be called when the *tiered index locks are locked for shared + // ownership*, along with the backend index's own data guard lock if it has one. That is since + // the internal getDistanceFrom calls access the indexes' data, and it is not safe to run + // insert/delete operations in parallel. Also, we avoid acquiring the locks internally, since + // this is usually called for every vector individually, and the overhead of acquiring and + // releasing the locks is significant in that case. + double getDistanceFrom_Unsafe(labelType label, const void *blob) const override { + // Try to get the distance from the flat buffer. + // If the label doesn't exist, the distance will be NaN. + auto flat_dist = this->frontendIndex->getDistanceFrom_Unsafe(label, blob); + + // Optimization. TODO: consider having different implementations for single and multi + // indexes, to avoid checking the index type on every query. + if (!this->backendIndex->isMultiValue() && !std::isnan(flat_dist)) { + // If the index is single value, and we got a valid distance from the flat buffer, + // we can return the distance without querying the backend index. + return flat_dist; + } + + // Try to get the distance from the backend index. + auto backend_dist = this->backendIndex->getDistanceFrom_Unsafe(label, blob); + + // Return the minimum distance that is not NaN. + return std::fmin(flat_dist, backend_dist); + } + VecSimTieredIndex(VecSimIndexAbstract *backendIndex_, BruteForceIndex *frontendIndex_, TieredIndexParams tieredParams, std::shared_ptr allocator) From 1e43b2b84c4ada741fa6242a21490cc29ad84446 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 04:50:43 -0700 Subject: [PATCH 55/99] fix Full-buffer overwrite skips pending jobs --- src/VecSim/algorithms/svs/svs_tiered.h | 75 +++++++++++++------------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 0e49c4820..04e78a3d1 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -990,14 +990,12 @@ class TieredSVSIndex : public VecSimTieredIndex { this->flatIndexGuard.unlock_shared(); auto storage_blob = this->frontendIndex->preprocessForStorage(blob); - if (!this->backendIndex->isMultiValue()) { - auto deleted = svs_index->deleteVector(label); - if (deleted > 0) - scheduleSVSIndexConsolidate(label); - } + const int overwritten = + this->backendIndex->isMultiValue() ? 0 : this->deleteVector(label); std::shared_lock lock(updateJobMutex); - return ret = svs_index->addVector(storage_blob.get(), label); + ret = svs_index->addVector(storage_blob.get(), label); + return std::max(ret - overwritten, 0); } else { this->flatIndexGuard.unlock_shared(); this->flatIndexGuard.lock(); @@ -1101,6 +1099,40 @@ class TieredSVSIndex : public VecSimTieredIndex { } } + int removeLabelFromFlat(labelType label) { + if (!this->frontendIndex->isLabelExists(label)) { + return 0; + } + auto deleting_ids = this->frontendIndex->getElementIds(label); + if (deleting_ids.size() == 0) { + return 0; + } + + // assert if all elements of deleting_ids are unique + assert(std::set(deleting_ids.begin(), deleting_ids.end()).size() == deleting_ids.size() && + "deleting_ids should contain unique ids"); + + // If id is deleted, don't use it for initialization + if (!ids_to_init_.empty()) { + for (idType id : deleting_ids) { + ids_to_init_.erase(id); + } + } + + if (this->labelToInsertJobs.count(label) > 0) { + // Invalidate the pending insert job(s) into SVS associated with this label + auto &insert_jobs = this->labelToInsertJobs.at(label); + for (auto *job : insert_jobs) { + job->id = this->setAndSaveInvalidJob(job); + } + // Remove the pending insert job(s) from the labelToInsertJobs mapping. + this->labelToInsertJobs.erase(label); + } + + deleteAndUpdateInitIds(label); + return static_cast(deleting_ids.size()); + } + int deleteVector(labelType label) override { int ret = 0; @@ -1108,36 +1140,7 @@ class TieredSVSIndex : public VecSimTieredIndex { if (this->frontendIndex->isLabelExists(label)) { this->flatIndexGuard.unlock_shared(); std::lock_guard flat_lock{this->flatIndexGuard}; - // Check again if the label exists, as it may have been removed while we released the - // lock. - if (this->frontendIndex->isLabelExists(label)) { - auto deleting_ids = this->frontendIndex->getElementIds(label); - if (deleting_ids.size() == 0) - return 0; - - // assert if all elements of deleting_ids are unique - assert(std::set(deleting_ids.begin(), deleting_ids.end()).size() == - deleting_ids.size() && - "deleting_ids should contain unique ids"); - - // If id is deleted, don't use it for initialization - for (idType id : deleting_ids) { - ids_to_init_.erase(id); - } - - if (this->labelToInsertJobs.count(label) > 0) { - // Invalidate the pending insert job(s) into SVS associated with this label - auto &insert_jobs = this->labelToInsertJobs.at(label); - for (auto *job : insert_jobs) { - job->id = this->setAndSaveInvalidJob(job); - } - // Remove the pending insert job(s) from the labelToInsertJobs mapping. - this->labelToInsertJobs.erase(label); - } - - deleteAndUpdateInitIds(label); - ret += deleting_ids.size(); - } + ret += removeLabelFromFlat(label); } else { this->flatIndexGuard.unlock_shared(); } From 99314d21a7356bd7aed305c8b89a5482d3c36686 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 05:00:19 -0700 Subject: [PATCH 56/99] fix Init overwrite reports a new insert --- src/VecSim/algorithms/svs/svs_tiered.h | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 04e78a3d1..ceec1a33e 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -925,12 +925,13 @@ class TieredSVSIndex : public VecSimTieredIndex { // initSVSIndexWrapper() is called. { std::lock_guard lock(this->flatIndexGuard); + int deleted = 0; if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { - deleteAndUpdateInitIds(label); + deleted = deleteAndUpdateInitIds(label); } ids_to_init_.insert(this->frontendIndex->indexSize()); - ret = this->frontendIndex->addVector(blob, label); + ret = std::max(this->frontendIndex->addVector(blob, label) - deleted, 0); // If frontend size exceeds the update job threshold, ... frontend_index_size = this->frontendIndex->indexSize(); } @@ -969,13 +970,14 @@ class TieredSVSIndex : public VecSimTieredIndex { std::lock_guard lock(this->flatIndexGuard); if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { + int deleted = 0; if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { - deleteAndUpdateInitIds(label); + deleted = deleteAndUpdateInitIds(label); } ids_to_init_.insert(this->frontendIndex->indexSize()); const auto ft_ret = this->frontendIndex->addVector(blob, label); - ret = std::max(ret + ft_ret, 0); + ret = std::max(ret + ft_ret - deleted, 0); if (this->frontendIndex->indexSize() >= this->trainingTriggerThreshold) { this->backendInitSubmited.store(true, std::memory_order_release); @@ -1066,7 +1068,8 @@ class TieredSVSIndex : public VecSimTieredIndex { } } - void deleteAndUpdateInitIds(labelType label) { + // Returns the number of vectors removed from the frontend index. + int deleteAndUpdateInitIds(labelType label) { auto deleting_ids = this->frontendIndex->getElementIds(label); std::sort(deleting_ids.begin(), deleting_ids.end()); @@ -1097,6 +1100,8 @@ class TieredSVSIndex : public VecSimTieredIndex { for (idType new_id : new_ids_to_init) { ids_to_init_.insert(new_id); } + + return static_cast(deleting_ids.size()); } int removeLabelFromFlat(labelType label) { From 73b23be38045092b81cd1dcda53b599c3f5047fa Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 06:28:02 -0700 Subject: [PATCH 57/99] fix Empty index skips compression retraining --- src/VecSim/algorithms/svs/svs_tiered.h | 163 ++++++++++++++++--------- src/VecSim/vec_sim_tiered_index.h | 32 +++-- 2 files changed, 125 insertions(+), 70 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index ceec1a33e..d11b3ea29 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -266,7 +267,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // The reason of following container just to properly destroy jobs which not executed yet SVSMultiThreadJob::JobsRegistry uncompletedJobs; - std::atomic backendInitSubmited{false}; + // frontend ids holding training data for the backend (re)initialization std::unordered_set ids_to_init_; vecsim_stl::unordered_map> @@ -550,9 +551,9 @@ class TieredSVSIndex : public VecSimTieredIndex { * * This static wrapper function performs the following actions: * - Acquires a lock on the index's updateJobMutex to prevent concurrent updates. - * - Clears the indexUpdateScheduled flag to allow future scheduling. * - Configures the number of threads for the underlying SVS index update operation. * - Calls the initSVSIndex method to perform the actual index update. + * - Clears the indexUpdateScheduled flag to allow future scheduling. * * @param idx Pointer to the VecSimIndex to be updated. * @param availableThreads The number of threads available for the update operation. Current @@ -564,9 +565,11 @@ class TieredSVSIndex : public VecSimTieredIndex { assert(index); // prevent parallel updates std::lock_guard lock(index->updateJobMutex); - // Release the scheduled flag to allow scheduling again - index->indexUpdateScheduled.clear(); - // Update the SVS index + // flag stays set while init runs: !ready() + !flag means "no one will init the backend" + struct ClearOnExit { + std::atomic_flag &flag; + ~ClearOnExit() { flag.clear(std::memory_order_release); } + } clear_on_exit{index->indexUpdateScheduled}; index->initSVSIndex(availableThreads); } @@ -775,10 +778,13 @@ class TieredSVSIndex : public VecSimTieredIndex { return InsertJobOutcome::Completed; } - // Backend is still initilizing. Resubmit the job - if (!svs_index->ready() && this->indexUpdateScheduled.test()) { + // a job never inits the backend: one point cannot train the compression + if (!svs_index->ready()) { this->flatIndexGuard.unlock_shared(); - return InsertJobOutcome::Deferred; + // init pending -> wait for it. no init pending -> back to the training buffer + return this->indexUpdateScheduled.test(std::memory_order_acquire) + ? InsertJobOutcome::Deferred + : adoptJobIntoInitBuffer(job); } // Copy the vector blob out of the flat buffer while holding flatIndexGuard, so we @@ -912,39 +918,20 @@ class TieredSVSIndex : public VecSimTieredIndex { int addVector(const void *blob, labelType label) override { int ret = 0; auto svs_index = GetSVSIndex(); - size_t frontend_index_size = 0; // In-Place mode - add vector syncronously to the backend index. if (this->getWriteMode() == VecSim_WriteInPlace) { // Backend index initialization data have to be buffered for proper // compression/training. - if ((!svs_index->ready()) && - (!this->backendInitSubmited.load(std::memory_order_acquire))) { - // First collect vectors in frontend index - // lock in scope to ensure that these will be released before - // initSVSIndexWrapper() is called. - { - std::lock_guard lock(this->flatIndexGuard); - int deleted = 0; - if (!this->frontendIndex->isMultiValue() && - this->frontendIndex->isLabelExists(label)) { - deleted = deleteAndUpdateInitIds(label); - } - ids_to_init_.insert(this->frontendIndex->indexSize()); - ret = std::max(this->frontendIndex->addVector(blob, label) - deleted, 0); - // If frontend size exceeds the update job threshold, ... - frontend_index_size = this->frontendIndex->indexSize(); - } - // ... move vectors to the backend index. - if (frontend_index_size >= this->trainingTriggerThreshold) { - // initSVSIndexWrapper() accures it's own locks - // initialize the SVS index synchonously using current thread only - initSVSIndexWrapper(this, 1); + if (!svs_index->ready()) { + if (auto buffered = tryBufferForTraining(blob, label, /*in_place=*/true)) { + return *buffered; } - return ret; - } else { - // backend index is initialized - we can add the vector directly - auto storage_blob = this->frontendIndex->preprocessForStorage(blob); + } + // backend index is initialized - we can add the vector directly + auto storage_blob = this->frontendIndex->preprocessForStorage(blob); + int deleted = 0; + { // prevent update job from running in parallel and lock any access to the backend // index // Only updateJobMutex is needed here, not mainIndexGuard: the concurrent @@ -953,37 +940,30 @@ class TieredSVSIndex : public VecSimTieredIndex { // Defensive: ensure single-threaded operation for write-in-place mode. // parallelism_ defaults to 1, so this is a no-op in the normal case. svs_index->setParallelism(1); - int deleted = 0; if (!this->backendIndex->isMultiValue()) { deleted = svs_index->deleteVector(label); if (deleted > 0) svs_index->consolidate({label}); } - return this->backendIndex->addVector(storage_blob.get(), label) - deleted; + if (svs_index->ready()) { + return this->backendIndex->addVector(storage_blob.get(), label) - deleted; + } } + // the delete above dropped the last backend vector: adding now would refit the + // compression to this single point + if (auto buffered = tryBufferForTraining(blob, label, /*in_place=*/true)) { + return std::max(*buffered - deleted, 0); + } + // a queued init job rebuilt the backend meanwhile + std::lock_guard lock(this->updateJobMutex); + return std::max(this->backendIndex->addVector(storage_blob.get(), label) - deleted, 0); } assert(this->getWriteMode() != VecSim_WriteInPlace && "InPlace mode returns early"); - // Async mode - add vector to the frontend index and schedule an update job if needed. - if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { - // Add vector to the frontend index. - std::lock_guard lock(this->flatIndexGuard); - if ((!svs_index->ready()) && - (!this->backendInitSubmited.load(std::memory_order_acquire))) { - int deleted = 0; - if (!this->frontendIndex->isMultiValue() && - this->frontendIndex->isLabelExists(label)) { - deleted = deleteAndUpdateInitIds(label); - } - ids_to_init_.insert(this->frontendIndex->indexSize()); - const auto ft_ret = this->frontendIndex->addVector(blob, label); - ret = std::max(ret + ft_ret - deleted, 0); - - if (this->frontendIndex->indexSize() >= this->trainingTriggerThreshold) { - this->backendInitSubmited.store(true, std::memory_order_release); - scheduleSVSIndexInit(); - } - return ret; + // Async mode - buffer training data until the backend can be inited from a batch. + if (!svs_index->ready() && !this->indexUpdateScheduled.test(std::memory_order_acquire)) { + if (auto buffered = tryBufferForTraining(blob, label, /*in_place=*/false)) { + return *buffered; } } @@ -995,6 +975,13 @@ class TieredSVSIndex : public VecSimTieredIndex { const int overwritten = this->backendIndex->isMultiValue() ? 0 : this->deleteVector(label); + // the delete above may have dropped the last backend vector + if (!svs_index->ready()) { + if (auto buffered = tryBufferForTraining(blob, label, /*in_place=*/false)) { + return std::max(*buffered - overwritten, 0); + } + } + std::shared_lock lock(updateJobMutex); ret = svs_index->addVector(storage_blob.get(), label); return std::max(ret - overwritten, 0); @@ -1068,6 +1055,14 @@ class TieredSVSIndex : public VecSimTieredIndex { } } + // training data is tracked by frontend id, so it has to follow the same swaps as insert jobs + void updateInsertJobInternalId(idType prev_id, idType new_id, labelType label) override { + if (ids_to_init_.erase(prev_id) > 0) { + ids_to_init_.insert(new_id); + } + Base::updateInsertJobInternalId(prev_id, new_id, label); + } + // Returns the number of vectors removed from the frontend index. int deleteAndUpdateInitIds(labelType label) { auto deleting_ids = this->frontendIndex->getElementIds(label); @@ -1138,6 +1133,58 @@ class TieredSVSIndex : public VecSimTieredIndex { return static_cast(deleting_ids.size()); } + // Buffer a vector in the frontend index as training data for the backend (re)initialization. + // Returns 1 for a new label, 0 for an overwrite, or nullopt if the backend turned ready while + // the guard was taken + std::optional tryBufferForTraining(const void *blob, labelType label, bool in_place) { + int ret = 0; + bool run_init = false; + { + std::lock_guard flat_lock{this->flatIndexGuard}; + // never buffer into a ready backend: ids_to_init_ would have no consumer + if (GetSVSIndex()->ready()) { + return std::nullopt; + } + int deleted = 0; + if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { + // once the backend has been dropped the buffer may also hold job-backed vectors, + // so the overwrite has to invalidate pending jobs + deleted = removeLabelFromFlat(label); + } + ids_to_init_.insert(this->frontendIndex->indexSize()); + ret = std::max(this->frontendIndex->addVector(blob, label) - deleted, 0); + run_init = ids_to_init_.size() >= this->trainingTriggerThreshold; + if (run_init && !in_place) { + scheduleSVSIndexInit(); + run_init = false; + } + } + if (run_init) { + initSVSIndexWrapper(this, 1); + } + return ret; + } + + // Move a pending job's vector from the insert-job path to the training batch. The vector stays + // in the frontend index, only its ownership changes. + InsertJobOutcome adoptJobIntoInitBuffer(SVSInsertJob *job) { + std::lock_guard flat_lock{this->flatIndexGuard}; + if (!job->isValid) { + std::lock_guard invalid_lock{this->invalidJobsLookupGuard}; + this->invalidJobs.erase(job->id); + return InsertJobOutcome::Completed; + } + if (GetSVSIndex()->ready() || this->indexUpdateScheduled.test(std::memory_order_acquire)) { + return InsertJobOutcome::Deferred; + } + ids_to_init_.insert(job->id); + this->detachInsertJob(job); + if (ids_to_init_.size() >= this->trainingTriggerThreshold) { + scheduleSVSIndexInit(); + } + return InsertJobOutcome::Completed; + } + int deleteVector(labelType label) override { int ret = 0; @@ -1292,7 +1339,7 @@ class TieredSVSIndex : public VecSimTieredIndex { svsTieredInfo.indexUpdateScheduled = (info.tieredInfo.frontendCommonInfo.indexSize > 0) && - this->backendInitSubmited.load(std::memory_order_acquire); + this->indexUpdateScheduled.test(std::memory_order_acquire); info.tieredInfo.specificTieredBackendInfo.svsTieredInfo = svsTieredInfo; // Background indexing is in progress whenever the flat buffer is non-empty diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 84fee4338..1dd5d99e2 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -157,7 +157,7 @@ class VecSimTieredIndex : public VecSimIndexInterface { // Helper function for updating the pending insert job(s) of a label after the flat buffer // swapped the vector's internal id - void updateInsertJobInternalId(idType prev_id, idType new_id, labelType label) { + virtual void updateInsertJobInternalId(idType prev_id, idType new_id, labelType label) { // Update the pending job id, due to a swap that was caused after the removal of new_id. assert(new_id != INVALID_ID && prev_id != INVALID_ID); auto it = this->labelToInsertJobs.find(label); @@ -187,6 +187,24 @@ class VecSimTieredIndex : public VecSimIndexInterface { return curInvalidId; } + // Remove the job pointer from the labelToInsertJobs mapping. Must hold flatIndexGuard. + void detachInsertJob(TieredInsertJob *job) { + auto it = this->labelToInsertJobs.find(job->label); + if (it == this->labelToInsertJobs.end()) { + return; + } + auto &jobs = it->second; + for (size_t i = 0; i < jobs.size(); i++) { + if (jobs[i] == job) { + jobs.erase(jobs.begin() + (long)i); + break; + } + } + if (jobs.empty()) { + this->labelToInsertJobs.erase(it); + } + } + // Remove a vector and its insert job from the flat buffer void removeIngestedVectorFromFlat(TieredInsertJob *job) { std::lock_guard flat_lock(this->flatIndexGuard); @@ -199,17 +217,7 @@ class VecSimTieredIndex : public VecSimIndexInterface { this->invalidJobs.erase(job->id); return; } - // Remove the job pointer from the labelToInsertJobs mapping. - auto &jobs = this->labelToInsertJobs.at(job->label); - for (size_t i = 0; i < jobs.size(); i++) { - if (jobs[i]->id == job->id) { - jobs.erase(jobs.begin() + (long)i); - break; - } - } - if (jobs.empty()) { - this->labelToInsertJobs.erase(job->label); - } + this->detachInsertJob(job); // Remove the vector from the flat buffer. This may cause the last vector id to swap with // the deleted id. Hold the label for the last id, so we can later on update its // corresponding job id. Note that after calling deleteVectorById, the last id's label From 4a989aa4decbadd3e7898119eec946483270232f Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 06:40:56 -0700 Subject: [PATCH 58/99] update comment for GC compacts without consolidating deletes --- src/VecSim/algorithms/svs/svs.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 3a6989ae5..917eeba8f 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -831,8 +831,8 @@ class SVSIndex : public VecSimIndexAbstract, fl void runGC() override { if (auto impl = getImpl()) { - // There is documentation for compact(): - // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.compact + // concurrent compact() consolidates every soft-deleted points first + // pending per-label consolidate jobs need not be drained - they become no-ops impl->compact(); } num_marked_deleted.store(0, std::memory_order_relaxed); From 56aeef90ef213b03a7fe1912bc060bb96900530e Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 07:39:33 -0700 Subject: [PATCH 59/99] fix Insert job can reinit dropped backend --- src/VecSim/algorithms/svs/svs.h | 19 ++++++++++++++++++- src/VecSim/algorithms/svs/svs_tiered.h | 8 +++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 917eeba8f..0aaba6aec 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -40,8 +40,14 @@ struct SVSIndexBase { SVSIndexBase() : num_marked_deleted{0} {}; virtual ~SVSIndexBase() = default; + // Returned by addVectorsIfInitialized() when nothing was added. + static constexpr int kNotInitialized = -1; + virtual int addVector(const void *vector_data, labelType label) = 0; virtual int addVectors(const void *vectors_data, const labelType *labels, size_t n) = 0; + // Add vectors into an already initialized instance. + virtual int addVectorsIfInitialized(const void *vectors_data, const labelType *labels, + size_t n) = 0; virtual int deleteVector(labelType label) = 0; virtual int deleteVectors(const labelType *labels, size_t n) = 0; virtual void consolidate(const std::vector &labels) = 0; @@ -283,7 +289,8 @@ class SVSIndex : public VecSimIndexAbstract, fl // for the operation. // Important NOTE: For single vector operations (n=1), parallelism should be 1. // For bulk operations (n>1), parallelism should reflect the number of available threads. - int addVectorsImpl(const void *vectors_data, const labelType *labels, size_t n) { + int addVectorsImpl(const void *vectors_data, const labelType *labels, size_t n, + bool allow_init = true) { if (n == 0) { return 0; } @@ -311,6 +318,9 @@ class SVSIndex : public VecSimIndexAbstract, fl this->implMutationGuard_.lock(); if (auto existing = getImpl()) { existing->add_points(points, ids, /*reuse_empty*/ false); + } else if (!allow_init) { + this->implMutationGuard_.unlock(); + return kNotInitialized; } else { auto built = initImpl(points, ids); assert(built != nullptr); @@ -595,6 +605,13 @@ class SVSIndex : public VecSimIndexAbstract, fl return addVectorsImpl(vectors_data, labels, n); } + int addVectorsIfInitialized(const void *vectors_data, const labelType *labels, + size_t n) override { + assert(!(n == 1 && getParallelism() > 1) && + "Can't use more than one thread to insert a single vector"); + return addVectorsImpl(vectors_data, labels, n, /*allow_init=*/false); + } + int deleteVector(labelType label) override { return deleteVectorImpl(label); } int deleteVectors(const labelType *labels, size_t n) override { diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 99e624463..48db63876 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -796,9 +796,15 @@ class TieredSVSIndex : public VecSimTieredIndex { job->executing.store(true, std::memory_order_release); this->flatIndexGuard.unlock_shared(); - svs_index->addVector(blob_copy.get(), job->label); + // if a concurrent deletion drops the instance + // initializing it from this single point would refit the compression + const int added = + svs_index->addVectorsIfInitialized(blob_copy.get(), &job->label, 1); job->executing.store(false, std::memory_order_release); + if (added == SVSIndexBase::kNotInitialized) { + return adoptJobIntoInitBuffer(job); + } // Remove the vector and the insert job from the flat buffer. this->removeIngestedVectorFromFlat(job); return InsertJobOutcome::Completed; From 00b292a4b7a725c86065b568fd940cd59caed1bc Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 07:52:36 -0700 Subject: [PATCH 60/99] modify comments --- src/VecSim/algorithms/svs/svs.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 0aaba6aec..dfddb7dc0 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -102,6 +102,8 @@ class SVSIndex : public VecSimIndexAbstract, fl using graph_builder_t = SVSGraphBuilder; using graph_type = typename graph_builder_t::graph_type; + // Note. svs::concurrent::* classes differ from the svs::index::vamana::* classes + // in their mutation contracts. using impl_type = std::conditional_t< isMulti, svs::concurrent::MultiMutableVamanaIndex, @@ -334,8 +336,6 @@ class SVSIndex : public VecSimIndexAbstract, fl } void consolidate(const std::vector &labels) override { - // There is documentation for consolidate(): - // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.consolidate if (auto impl = getImpl()) { impl->consolidate(labels); } From fd7c0d0ba07c010d52c474620a8e44f09eaf8d6e Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 08:58:15 -0700 Subject: [PATCH 61/99] fix Relabel races pending multi-value ingest --- src/VecSim/algorithms/svs/svs_tiered.h | 82 ++++++++++++++------------ 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 48db63876..376cda726 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -1024,21 +1024,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // If this label already exists, this will do overwrite. ret += this->frontendIndex->addVector(blob, label); - TieredInsertJob *new_insert_job = new (this->allocator) - SVSInsertJob(this->allocator, label, new_flat_id, executeInsertJobWrapper, this); - - // Save a pointer to the job, so that if the vector is overwritten, we'll have an - // indication. - if (this->labelToInsertJobs.find(label) != this->labelToInsertJobs.end()) { - // There's already a pending insert job for this label, add another one (without - // overwrite, only possible in multi index) - assert(this->backendIndex->isMultiValue()); - this->labelToInsertJobs.at(label).push_back(new_insert_job); - } else { - vecsim_stl::vector new_jobs_vec(1, new_insert_job, - this->allocator); - this->labelToInsertJobs.insert({label, new_jobs_vec}); - } + TieredInsertJob *new_insert_job = createInsertJob(label, new_flat_id); this->flatIndexGuard.unlock(); // Here, a worker might ingest the previous vector that was stored under "label" @@ -1139,6 +1125,24 @@ class TieredSVSIndex : public VecSimTieredIndex { return static_cast(deleting_ids.size()); } + // Create a pending insert job for a vector already buffered in the frontend index + // Caller must hold flatIndexGuard exclusively. + TieredInsertJob *createInsertJob(labelType label, idType flat_id) { + TieredInsertJob *job = new (this->allocator) + SVSInsertJob(this->allocator, label, flat_id, executeInsertJobWrapper, this); + auto it = this->labelToInsertJobs.find(label); + if (it != this->labelToInsertJobs.end()) { + // There's already a pending insert job for this label, add another one (without + // overwrite, only possible in multi index) + assert(this->backendIndex->isMultiValue()); + it->second.push_back(job); + } else { + vecsim_stl::vector new_jobs_vec(1, job, this->allocator); + this->labelToInsertJobs.insert({label, new_jobs_vec}); + } + return job; + } + // Buffer a vector in the frontend index as training data for the backend (re)initialization. // Returns 1 for a new label, 0 for an overwrite, or nullopt if the backend turned ready while // the guard was taken @@ -1242,6 +1246,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::shared_lock lock(this->updateJobMutex); bool flat_holds_old = false; + bool had_pending = false; std::vector taken_over; { std::lock_guard flat_lock{this->flatIndexGuard}; @@ -1255,15 +1260,14 @@ class TieredSVSIndex : public VecSimTieredIndex { return VecSimRelabel_NewLabelTaken; } + // invalid job and rebuild once the label is settled. auto pending = this->labelToInsertJobs.find(old_label); if (pending != this->labelToInsertJobs.end()) { - auto jobs = std::move(pending->second); - this->labelToInsertJobs.erase(pending); - for (auto *job : jobs) { - waitForJob(static_cast(job)); - job->label = new_label; + for (auto *job : pending->second) { + job->id = this->setAndSaveInvalidJob(job); } - this->labelToInsertJobs.emplace(new_label, std::move(jobs)); + this->labelToInsertJobs.erase(pending); + had_pending = true; } // isLabelExists() said new_label is free, but SVS keeps the translator entry of a @@ -1285,27 +1289,29 @@ class TieredSVSIndex : public VecSimTieredIndex { svs_index->consolidate(taken_over); } - const VecSimRelabelCode backend_ret = - this->backendIndex->relabelVector(old_label, new_label); - if (backend_ret == VecSimRelabel_NewLabelTaken) { - // Only reachable if new_label was added concurrently with this rename + const bool rollback = this->backendIndex->relabelVector(old_label, new_label) == + VecSimRelabel_NewLabelTaken; + const labelType final_label = rollback ? old_label : new_label; + + vecsim_stl::vector new_jobs(this->allocator); + if (rollback || had_pending) { std::lock_guard flat_lock{this->flatIndexGuard}; - auto pending = this->labelToInsertJobs.find(new_label); - if (pending != this->labelToInsertJobs.end()) { - auto jobs = std::move(pending->second); - this->labelToInsertJobs.erase(pending); - for (auto *job : jobs) { - waitForJob(static_cast(job)); - job->label = old_label; - } - this->labelToInsertJobs.emplace(old_label, std::move(jobs)); - } - if (flat_holds_old) { + if (rollback && flat_holds_old) { this->frontendIndex->relabelVector(new_label, old_label); } - return VecSimRelabel_NewLabelTaken; + // Rebuild the jobs invalidated above from the buffer itself + if (had_pending) { + for (idType id : this->frontendIndex->getElementIds(final_label)) { + if (ids_to_init_.count(id) == 0) { + new_jobs.push_back(createInsertJob(final_label, id)); + } + } + } + } + for (auto *job : new_jobs) { + this->submitSingleJob(job); } - return VecSimRelabel_OK; + return rollback ? VecSimRelabel_NewLabelTaken : VecSimRelabel_OK; } #endif // HAVE_SVS_REPLACE_EXTERNAL_ID From 03f803a5a25bbe8ae6b3aa98547b135127051ef9 Mon Sep 17 00:00:00 2001 From: ethanglaser <42726565+ethanglaser@users.noreply.github.com> Date: Wed, 16 Sep 2026 09:14:44 -0700 Subject: [PATCH 62/99] bump glibc version error message to 2.28 --- cmake/svs.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index b9aef3431..79baa152e 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -81,7 +81,7 @@ if(USE_SVS) set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc2-reduced.tar.gz" CACHE STRING "SVS URL") endif() else() - message(STATUS "GLIBC >= 2.26 is required for GCC SVS binaries - disabling SVS_SHARED_LIB") + message(STATUS "GLIBC >= 2.28 is required for GCC SVS binaries - disabling SVS_SHARED_LIB") set(SVS_SHARED_LIB OFF) endif() endif() From 75742396fce08dfe2ac8632e9d310b83ff72e22f Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 09:19:16 -0700 Subject: [PATCH 63/99] format --- src/VecSim/algorithms/svs/svs_tiered.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 376cda726..8509b6e91 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -798,8 +798,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // if a concurrent deletion drops the instance // initializing it from this single point would refit the compression - const int added = - svs_index->addVectorsIfInitialized(blob_copy.get(), &job->label, 1); + const int added = svs_index->addVectorsIfInitialized(blob_copy.get(), &job->label, 1); job->executing.store(false, std::memory_order_release); if (added == SVSIndexBase::kNotInitialized) { @@ -1289,8 +1288,8 @@ class TieredSVSIndex : public VecSimTieredIndex { svs_index->consolidate(taken_over); } - const bool rollback = this->backendIndex->relabelVector(old_label, new_label) == - VecSimRelabel_NewLabelTaken; + const bool rollback = + this->backendIndex->relabelVector(old_label, new_label) == VecSimRelabel_NewLabelTaken; const labelType final_label = rollback ? old_label : new_label; vecsim_stl::vector new_jobs(this->allocator); From 3a9d64c9b0ec114d1934de749f4f8e1efffc9d53 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 10:24:53 -0700 Subject: [PATCH 64/99] fix Relabel duplicates in-flight ingested vectors --- src/VecSim/algorithms/svs/svs_tiered.h | 88 +++++++++++++++++++------- 1 file changed, 64 insertions(+), 24 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 8509b6e91..3d7c1fa75 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -19,7 +19,11 @@ * Definition of a job that inserts a new vector from flat into SVS Index. */ struct SVSInsertJob : public TieredInsertJob { - std::atomic executing{false}; + // Pending -> Executing -> Done. Delayed means a relabel is remapping the label in the backend + // right now, and publishing would put the vector under a label that is not settled yet. + // Written under flatIndexGuard, except Executing/Done which waitForJob() reads unlocked. + enum class Status : uint8_t { Pending, Delayed, Executing, Done }; + std::atomic status{Status::Pending}; SVSInsertJob(std::shared_ptr allocator, labelType label_, idType id_, JobCallback insertCb, VecSimIndex *index_) @@ -748,8 +752,13 @@ class TieredSVSIndex : public VecSimTieredIndex { // Wait until the job leaves its publish window // Safe to call while holding flatIndexGuard exclusive - template - static void waitForJob(Job *job) { + static void waitForJob(SVSInsertJob *job) { + while (job->status.load(std::memory_order_acquire) == SVSInsertJob::Status::Executing) { + std::this_thread::yield(); + } + } + + static void waitForJob(SVSConsolidateJob *job) { while (job->executing.load(std::memory_order_acquire)) { std::this_thread::yield(); } @@ -778,6 +787,12 @@ class TieredSVSIndex : public VecSimTieredIndex { return InsertJobOutcome::Completed; } + if (job->status.load(std::memory_order_relaxed) == SVSInsertJob::Status::Delayed) { + this->flatIndexGuard.unlock_shared(); + // relabelVector() clears it once the backend remap settled the label + return InsertJobOutcome::Deferred; + } + // a job never inits the backend: one point cannot train the compression if (!svs_index->ready()) { this->flatIndexGuard.unlock_shared(); @@ -793,15 +808,17 @@ class TieredSVSIndex : public VecSimTieredIndex { auto blob_copy = this->getAllocator()->allocate_unique(data_size); memcpy(blob_copy.get(), this->frontendIndex->getDataByInternalId(job->id), data_size); - job->executing.store(true, std::memory_order_release); + job->status.store(SVSInsertJob::Status::Executing, std::memory_order_release); this->flatIndexGuard.unlock_shared(); // if a concurrent deletion drops the instance // initializing it from this single point would refit the compression const int added = svs_index->addVectorsIfInitialized(blob_copy.get(), &job->label, 1); + const bool published = added != SVSIndexBase::kNotInitialized; - job->executing.store(false, std::memory_order_release); - if (added == SVSIndexBase::kNotInitialized) { + job->status.store(published ? SVSInsertJob::Status::Done : SVSInsertJob::Status::Pending, + std::memory_order_release); + if (!published) { return adoptJobIntoInitBuffer(job); } // Remove the vector and the insert job from the flat buffer. @@ -1245,7 +1262,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::shared_lock lock(this->updateJobMutex); bool flat_holds_old = false; - bool had_pending = false; + bool delayed_any = false; std::vector taken_over; { std::lock_guard flat_lock{this->flatIndexGuard}; @@ -1259,14 +1276,25 @@ class TieredSVSIndex : public VecSimTieredIndex { return VecSimRelabel_NewLabelTaken; } - // invalid job and rebuild once the label is settled. auto pending = this->labelToInsertJobs.find(old_label); if (pending != this->labelToInsertJobs.end()) { - for (auto *job : pending->second) { - job->id = this->setAndSaveInvalidJob(job); - } + auto jobs = std::move(pending->second); this->labelToInsertJobs.erase(pending); - had_pending = true; + for (auto *job : jobs) { + auto *insert_job = static_cast(job); + // wait it out: a job that published before the remap is remapped with the + // backend, and still drops its own buffer copy under the new label + waitForJob(insert_job); + job->label = new_label; + if (insert_job->status.load(std::memory_order_relaxed) == + SVSInsertJob::Status::Pending) { + // did not publish yet - hold it until the remap settles the label + insert_job->status.store(SVSInsertJob::Status::Delayed, + std::memory_order_relaxed); + delayed_any = true; + } + } + this->labelToInsertJobs.emplace(new_label, std::move(jobs)); } // isLabelExists() said new_label is free, but SVS keeps the translator entry of a @@ -1292,24 +1320,36 @@ class TieredSVSIndex : public VecSimTieredIndex { this->backendIndex->relabelVector(old_label, new_label) == VecSimRelabel_NewLabelTaken; const labelType final_label = rollback ? old_label : new_label; - vecsim_stl::vector new_jobs(this->allocator); - if (rollback || had_pending) { + if (rollback || delayed_any) { std::lock_guard flat_lock{this->flatIndexGuard}; - if (rollback && flat_holds_old) { - this->frontendIndex->relabelVector(new_label, old_label); + if (rollback) { + auto pending = this->labelToInsertJobs.find(new_label); + if (pending != this->labelToInsertJobs.end()) { + auto jobs = std::move(pending->second); + this->labelToInsertJobs.erase(pending); + for (auto *job : jobs) { + job->label = old_label; + } + this->labelToInsertJobs.emplace(old_label, std::move(jobs)); + } + if (flat_holds_old) { + this->frontendIndex->relabelVector(new_label, old_label); + } } - // Rebuild the jobs invalidated above from the buffer itself - if (had_pending) { - for (idType id : this->frontendIndex->getElementIds(final_label)) { - if (ids_to_init_.count(id) == 0) { - new_jobs.push_back(createInsertJob(final_label, id)); + if (delayed_any) { + auto it = this->labelToInsertJobs.find(final_label); + if (it != this->labelToInsertJobs.end()) { + for (auto *job : it->second) { + auto *insert_job = static_cast(job); + if (insert_job->status.load(std::memory_order_relaxed) == + SVSInsertJob::Status::Delayed) { + insert_job->status.store(SVSInsertJob::Status::Pending, + std::memory_order_relaxed); + } } } } } - for (auto *job : new_jobs) { - this->submitSingleJob(job); - } return rollback ? VecSimRelabel_NewLabelTaken : VecSimRelabel_OK; } From 516a3242a531e23ccbb4b5f3e43c5b8827e0c822 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 23 Sep 2026 03:27:25 -0700 Subject: [PATCH 65/99] fix -Werror=sign-compare --- src/VecSim/algorithms/svs/svs_tiered.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 3d7c1fa75..5e4db5e1f 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -872,7 +872,7 @@ class TieredSVSIndex : public VecSimTieredIndex { } std::sort(ids_to_move.begin(), ids_to_move.end()); - [[maybe_unused]] int total_deleted = 0; + [[maybe_unused]] size_t total_deleted = 0; for (auto it = ids_to_move.rbegin(); it != ids_to_move.rend(); ++it) { idType id = *it; auto label = this->frontendIndex->getVectorLabel(id); From 462cbbe984ed74c6777ddb3190220d0b9146532f Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 23 Sep 2026 04:48:19 -0700 Subject: [PATCH 66/99] unify compute_current_batch --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 120 +++-------------------- src/VecSim/algorithms/svs/svs_tiered.h | 76 +++----------- src/VecSim/vec_sim_tiered_index.h | 73 ++++++++++++++ 3 files changed, 101 insertions(+), 168 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index dc4e83d70..8ff4c2787 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -218,35 +218,15 @@ class TieredHNSWIndex : public VecSimTieredIndex { } public: - class TieredHNSW_BatchIterator : public VecSimBatchIterator { + class TieredHNSW_BatchIterator : public TieredIndex_BatchIterator { private: const TieredHNSWIndex *index; std::shared_lock backend_index_lock; VecSimQueryParams *queryParams; - VecSimQueryResultContainer flat_results; - VecSimQueryResultContainer hnsw_results; - VecSimBatchIterator *flat_iterator; VecSimBatchIterator *hnsw_iterator; - // On single value indices, this set holds the IDs of the results that were returned from - // the flat buffer. - // On multi value indices, this set holds the IDs of all the results that were returned. - // The difference between the two cases is that on multi value indices, the same ID can - // appear in both indexes and results with different scores, and therefore we can't tell in - // advance when we expect a possibility of a duplicate. - // On single value indices, a duplicate may appear at the same batch (and we will handle it - // when merging the results) Or it may appear in a different batches, first from the flat - // buffer and then from the HNSW, in the cases where a better result if found later in HNSW - // because of the approximate nature of the algorithm. - vecsim_stl::unordered_set returned_results_set; - - private: - template - inline VecSimQueryReply *compute_current_batch(size_t n_res); - inline void filter_irrelevant_results(VecSimQueryResultContainer &); - public: TieredHNSW_BatchIterator(const void *query_vector, const TieredHNSWIndex *index, @@ -1157,11 +1137,10 @@ TieredHNSWIndex::TieredHNSW_BatchIterator::TieredHNSW_BatchI // copies: flat_iterator copy is created during TieredHNSW_BatchIterator construction When // TieredHNSW_BatchIterator::getNextResults() is called and hnsw_iterator is not initialized, it // retrieves the blob from flat_iterator - : VecSimBatchIterator(nullptr, queryParams ? queryParams->timeoutCtx : nullptr, - std::move(allocator)), + : TieredIndex_BatchIterator(nullptr, queryParams ? queryParams->timeoutCtx : nullptr, + std::move(allocator)), index(index), backend_index_lock(index->mainIndexGuard, std::defer_lock), - flat_results(this->allocator), hnsw_results(this->allocator), flat_iterator(UNINITIALIZED), - hnsw_iterator(UNINITIALIZED), returned_results_set(this->allocator) { + flat_iterator(UNINITIALIZED), hnsw_iterator(UNINITIALIZED) { { std::shared_lock flat_index_lock(this->index->flatIndexGuard); this->flat_iterator = @@ -1218,7 +1197,7 @@ VecSimQueryReply *TieredHNSWIndex::TieredHNSW_BatchIterator: this->flat_iterator->getQueryBlob(), queryParams); auto cur_hnsw_results = this->hnsw_iterator->getNextResults(n_res, BY_SCORE_THEN_ID); hnsw_code = cur_hnsw_results->code; - this->hnsw_results.swap(cur_hnsw_results->results); + this->backend_results.swap(cur_hnsw_results->results); VecSimQueryReply_Free(cur_hnsw_results); if (this->hnsw_iterator->isDepleted()) { delete this->hnsw_iterator; @@ -1246,19 +1225,19 @@ VecSimQueryReply *TieredHNSWIndex::TieredHNSW_BatchIterator: } } - while (this->hnsw_results.size() < n_res && this->hnsw_iterator != DEPLETED && + while (this->backend_results.size() < n_res && this->hnsw_iterator != DEPLETED && hnsw_code == VecSim_OK) { - auto tail = this->hnsw_iterator->getNextResults(n_res - this->hnsw_results.size(), + auto tail = this->hnsw_iterator->getNextResults(n_res - this->backend_results.size(), BY_SCORE_THEN_ID); hnsw_code = tail->code; // Set the hnsw_results code to the last `getNextResults` code. // New batch may contain better results than the previous batch, so we need to merge. // We don't expect duplications (hence the ), as the iterator guarantees that // no result is returned twice. VecSimQueryResultContainer cur_hnsw_results(this->allocator); - merge_results(cur_hnsw_results, this->hnsw_results, tail->results, n_res); + merge_results(cur_hnsw_results, this->backend_results, tail->results, n_res); VecSimQueryReply_Free(tail); - this->hnsw_results.swap(cur_hnsw_results); - filter_irrelevant_results(this->hnsw_results); + this->backend_results.swap(cur_hnsw_results); + filter_irrelevant_results(this->backend_results); if (this->hnsw_iterator->isDepleted()) { delete this->hnsw_iterator; this->hnsw_iterator = DEPLETED; @@ -1295,7 +1274,7 @@ VecSimQueryReply *TieredHNSWIndex::TieredHNSW_BatchIterator: template bool TieredHNSWIndex::TieredHNSW_BatchIterator::isDepleted() { return this->flat_results.empty() && this->flat_iterator->isDepleted() && - this->hnsw_results.empty() && this->hnsw_iterator == DEPLETED; + this->backend_results.empty() && this->hnsw_iterator == DEPLETED; } template @@ -1308,85 +1287,10 @@ void TieredHNSWIndex::TieredHNSW_BatchIterator::reset() { this->flat_iterator->reset(); this->hnsw_iterator = UNINITIALIZED; this->flat_results.clear(); - this->hnsw_results.clear(); + this->backend_results.clear(); returned_results_set.clear(); } -/****************** Helper Functions **************/ - -template -template -VecSimQueryReply * -TieredHNSWIndex::TieredHNSW_BatchIterator::compute_current_batch(size_t n_res) { - // Merge results - // This call will update `hnsw_res` and `bf_res` to point to the end of the merged results. - auto batch_res = new VecSimQueryReply(this->allocator); - std::pair p; - if (isMultiValue) { - p = merge_results(batch_res->results, this->hnsw_results, this->flat_results, n_res); - } else { - p = merge_results(batch_res->results, this->hnsw_results, this->flat_results, n_res); - } - auto [from_hnsw, from_flat] = p; - - if (!isMultiValue) { - // If we're on a single-value index, update the set of results returned from the FLAT index - // before popping them, to prevent them to be returned from the HNSW index in later batches. - for (size_t i = 0; i < from_flat; ++i) { - this->returned_results_set.insert(this->flat_results[i].id); - } - } else { - // If we're on a multi-value index, update the set of results returned (from `batch_res`) - for (size_t i = 0; i < batch_res->results.size(); ++i) { - this->returned_results_set.insert(batch_res->results[i].id); - } - } - - // Update results - this->flat_results.erase(this->flat_results.begin(), this->flat_results.begin() + from_flat); - this->hnsw_results.erase(this->hnsw_results.begin(), this->hnsw_results.begin() + from_hnsw); - - // clean up the results - // On multi-value indexes, one (or both) results lists may contain results that are already - // returned form the other list (with a different score). We need to filter them out. - if (isMultiValue) { - filter_irrelevant_results(this->flat_results); - filter_irrelevant_results(this->hnsw_results); - } - - // Return current batch - return batch_res; -} - -template -void TieredHNSWIndex::TieredHNSW_BatchIterator::filter_irrelevant_results( - VecSimQueryResultContainer &results) { - // Filter out results that were already returned. - auto it = results.begin(); - const auto end = results.end(); - // Skip results that not returned yet - while (it != end && this->returned_results_set.count(it->id) == 0) { - ++it; - } - // If none of the results were returned, return - if (it == end) { - return; - } - // Mark the current result as the first result to be filtered - auto cur_end = it; - ++it; - // "Append" all results that were not returned from the FLAT index - while (it != end) { - if (this->returned_results_set.count(it->id) == 0) { - *cur_end = *it; - ++cur_end; - } - ++it; - } - // Update number of results (pop the tail) - results.resize(cur_end - results.begin()); -} - template VecSimIndexStatsInfo TieredHNSWIndex::statisticInfo() const { auto stats = VecSimTieredIndex::statisticInfo(); diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 5e4db5e1f..f8e3632ed 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -285,7 +285,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // TieredSVS_BatchIterator // //////////////////////////////////////////////////////////////////////////////////////////////////// - class TieredSVS_BatchIterator : public VecSimBatchIterator { + class TieredSVS_BatchIterator : public TieredIndex_BatchIterator { // Defining spacial values for the svs_iterator field, to indicate if the iterator is // uninitialized or depleted when we don't have a valid iterator. static constexpr VecSimBatchIterator *depleted() { @@ -298,54 +298,9 @@ class TieredSVSIndex : public VecSimTieredIndex { const Index *index; VecSimQueryParams *queryParams; - VecSimQueryResultContainer flat_results; - VecSimQueryResultContainer svs_results; - VecSimBatchIterator *flat_iterator; VecSimBatchIterator *svs_iterator; - // IDs of all the results returned so far, so that neither tier can yield one twice. - // Both directions have to be covered: nothing freezes the backend for the iterator's - // lifetime, so an insert job can publish a label into SVS and only then drop it from the - // flat buffer, while the flat iterator still holds it in the snapshot it computed on its - // first call. Either tier can therefore surface a label first. - vecsim_stl::unordered_set returned_results_set; - - VecSimQueryReply *compute_current_batch(size_t n_res) { - // Merge results - // This call will update `svs_res` and `bf_res` to point to the end of the merged - // results. - auto batch_res = new VecSimQueryReply(allocator); - // VecSim and SVS distance computation is implemented differently, so we always have to - // merge results with set. - auto [from_svs, from_flat] = - merge_results(batch_res->results, svs_results, flat_results, n_res); - - for (size_t i = 0; i < batch_res->results.size(); ++i) { - this->returned_results_set.insert(batch_res->results[i].id); - } - - // Update results - flat_results.erase(flat_results.begin(), flat_results.begin() + from_flat); - svs_results.erase(svs_results.begin(), svs_results.begin() + from_svs); - - // Either list may still hold a label just returned from the other one (with a different - // score), so drop those before the next batch. - filter_irrelevant_results(this->flat_results); - filter_irrelevant_results(this->svs_results); - - // Return current batch - return batch_res; - } - - void filter_irrelevant_results(VecSimQueryResultContainer &results) { - // Filter out results that were already returned. - const auto it = std::remove_if(results.begin(), results.end(), [this](const auto &r) { - return returned_results_set.count(r.id) != 0; - }); - results.erase(it, results.end()); - } - void acquire_svs_iterator() { assert(svs_iterator == nullptr); svs_iterator = index->backendIndex->newBatchIterator( @@ -376,11 +331,11 @@ class TieredSVSIndex : public VecSimTieredIndex { // own copies: flat_iterator copy is created during TieredSVS_BatchIterator // construction When TieredSVS_BatchIterator::getNextResults() is called and // svs_iterator is not initialized, it retrieves the blob from flat_iterator - : VecSimBatchIterator(nullptr, queryParams ? queryParams->timeoutCtx : nullptr, - std::move(allocator)), - index(index), flat_results(this->allocator), svs_results(this->allocator), + : TieredIndex_BatchIterator(nullptr, queryParams ? queryParams->timeoutCtx : nullptr, + std::move(allocator)), + index(index), flat_iterator(index->frontendIndex->newBatchIterator(query_vector, queryParams)), - svs_iterator(nullptr), returned_results_set(this->allocator) { + svs_iterator(nullptr) { if (queryParams) { this->queryParams = (VecSimQueryParams *)this->allocator->allocate(sizeof(VecSimQueryParams)); @@ -420,7 +375,7 @@ class TieredSVSIndex : public VecSimTieredIndex { acquire_svs_iterator(); auto cur_svs_results = svs_iterator->getNextResults(n_res, BY_SCORE_THEN_ID); svs_code = cur_svs_results->code; - svs_results.swap(cur_svs_results->results); + backend_results.swap(cur_svs_results->results); VecSimQueryReply_Free(cur_svs_results); handle_svs_depletion(); } else { @@ -435,20 +390,20 @@ class TieredSVSIndex : public VecSimTieredIndex { filter_irrelevant_results(this->flat_results); } - while (svs_results.size() < n_res && svs_iterator != depleted() && + while (backend_results.size() < n_res && svs_iterator != depleted() && svs_code == VecSim_OK) { - auto tail = - svs_iterator->getNextResults(n_res - svs_results.size(), BY_SCORE_THEN_ID); + auto tail = svs_iterator->getNextResults(n_res - backend_results.size(), + BY_SCORE_THEN_ID); svs_code = tail->code; // Set the svs_results code to the last `getNextResults` code. // New batch may contain better results than the previous batch, so we need to // merge. We don't expect duplications (hence the ), as the iterator // guarantees that no result is returned twice. VecSimQueryResultContainer cur_svs_results(this->allocator); - merge_results(cur_svs_results, svs_results, tail->results, n_res); + merge_results(cur_svs_results, backend_results, tail->results, n_res); VecSimQueryReply_Free(tail); - svs_results.swap(cur_svs_results); - filter_irrelevant_results(svs_results); + backend_results.swap(cur_svs_results); + filter_irrelevant_results(backend_results); handle_svs_depletion(); } } @@ -458,7 +413,8 @@ class TieredSVSIndex : public VecSimTieredIndex { } VecSimQueryReply *batch; - batch = compute_current_batch(n_res); + // In concurent execution we can observe a vector duplication in backend and frontend + batch = compute_current_batch(n_res); if (order == BY_ID) { sort_results_by_id(batch); @@ -476,7 +432,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // The next call to `getNextResults` will return an empty batch, and then the iterators will // correctly report that they are depleted. bool isDepleted() override { - return flat_results.empty() && flat_iterator->isDepleted() && svs_results.empty() && + return flat_results.empty() && flat_iterator->isDepleted() && backend_results.empty() && svs_iterator == depleted(); } @@ -486,7 +442,7 @@ class TieredSVSIndex : public VecSimTieredIndex { flat_iterator->reset(); svs_iterator = nullptr; flat_results.clear(); - svs_results.clear(); + backend_results.clear(); returned_results_set.clear(); } }; diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 1e99e765b..cc87f5a0e 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -57,6 +57,79 @@ struct TieredInsertJob : public AsyncJob { : AsyncJob(allocator, type, insertCb, index_), label(label_), id(id_) {} }; +class TieredIndex_BatchIterator : public VecSimBatchIterator { +protected: + VecSimQueryResultContainer flat_results; + VecSimQueryResultContainer backend_results; + + // On single value indices, this set holds the IDs of the results that were returned from + // the flat buffer. + // On multi value indices, this set holds the IDs of all the results that were returned. + // The difference between the two cases is that on multi value indices, the same ID can + // appear in both indexes and results with different scores, and therefore we can't tell in + // advance when we expect a possibility of a duplicate. + vecsim_stl::unordered_set returned_results_set; + + TieredIndex_BatchIterator(void *query_vector, void *tctx, + std::shared_ptr allocator) + : VecSimBatchIterator(query_vector, tctx, std::move(allocator)), + flat_results(this->allocator), backend_results(this->allocator), + returned_results_set(this->allocator) {} + + template + VecSimQueryReply *compute_current_batch(size_t n_res) { + // Merge results + auto batch_res = new VecSimQueryReply(this->allocator); + std::pair p; + if (needsDedup) { + p = merge_results(batch_res->results, this->backend_results, this->flat_results, + n_res); + } else { + p = merge_results(batch_res->results, this->backend_results, this->flat_results, + n_res); + } + auto [from_backend, from_flat] = p; + + if (!needsDedup) { + // Update the set of results returned from the FLAT + // index before popping them. + for (size_t i = 0; i < from_flat; ++i) { + this->returned_results_set.insert(this->flat_results[i].id); + } + } else { + // Update the set of results returned (from `batch_res`) + for (size_t i = 0; i < batch_res->results.size(); ++i) { + this->returned_results_set.insert(batch_res->results[i].id); + } + } + + // Update results + this->flat_results.erase(this->flat_results.begin(), + this->flat_results.begin() + from_flat); + this->backend_results.erase(this->backend_results.begin(), + this->backend_results.begin() + from_backend); + + // clean up the results + // One (or both) results lists may contain results that are already + // returned form the other list (with a different score). We need to filter them out. + if (needsDedup) { + this->filter_irrelevant_results(this->flat_results); + this->filter_irrelevant_results(this->backend_results); + } + + // Return current batch + return batch_res; + } + + void filter_irrelevant_results(VecSimQueryResultContainer &results) { + // Filter out results that were already returned. + const auto it = std::remove_if(results.begin(), results.end(), [this](const auto &r) { + return returned_results_set.count(r.id) != 0; + }); + results.erase(it, results.end()); + } +}; + // All read operations (including KNN, range, batch iterators and get-distance-from) are guaranteed // to consider all vectors that were added to the index before the query was submitted. The results // may include vectors that were added after the query was submitted, with no guarantees. From 572efab9a1d87c8e0e71e57627e64b046aeb42d5 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 24 Sep 2026 07:18:07 -0700 Subject: [PATCH 67/99] svs pimpl is initalized once and never reset --- src/VecSim/algorithms/svs/svs.h | 126 +++++------------- .../algorithms/svs/svs_batch_iterator.h | 9 +- .../algorithms/svs/svs_serializer_impl.h | 10 +- src/VecSim/algorithms/svs/svs_tiered.h | 103 +++----------- tests/unit/test_svs.cpp | 36 +++-- tests/unit/test_svs_fp16.cpp | 34 ++++- tests/unit/test_svs_multi.cpp | 11 +- tests/unit/test_svs_tiered.cpp | 40 +++--- 8 files changed, 151 insertions(+), 218 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index dfddb7dc0..82257e745 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -40,14 +40,9 @@ struct SVSIndexBase { SVSIndexBase() : num_marked_deleted{0} {}; virtual ~SVSIndexBase() = default; - // Returned by addVectorsIfInitialized() when nothing was added. - static constexpr int kNotInitialized = -1; virtual int addVector(const void *vector_data, labelType label) = 0; virtual int addVectors(const void *vectors_data, const labelType *labels, size_t n) = 0; - // Add vectors into an already initialized instance. - virtual int addVectorsIfInitialized(const void *vectors_data, const labelType *labels, - size_t n) = 0; virtual int deleteVector(labelType label) = 0; virtual int deleteVectors(const labelType *labels, size_t n) = 0; virtual void consolidate(const std::vector &labels) = 0; @@ -132,18 +127,9 @@ class SVSIndex : public VecSimIndexAbstract, fl // SVS Index implementation instance. // initImpl() needs at least one point to compute an entry point. // so impl_ is null until the first insert - std::shared_ptr impl_; + std::atomic impl_{nullptr}; - mutable std::shared_mutex pimplGuard_; - // Serialize mutators of the impl_ pointer (exclusive) - // vs add_points() (shared) to avoid silent drops of additions - // Lock order: implMutationGuard_ -> pimplGuard_. - std::shared_mutex implMutationGuard_; - - std::shared_ptr getImpl() const { - std::shared_lock lock(this->pimplGuard_); - return this->impl_; - } + impl_type *getImpl() const { return this->impl_.load(std::memory_order_acquire); } static double toVecSimDistance(float v) { return svs_details::toVecSimDistance(v); } @@ -186,7 +172,7 @@ class SVSIndex : public VecSimIndexAbstract, fl // Create SVS index instance with initial data // Data should not be empty template - std::shared_ptr initImpl(const Dataset &points, + std::unique_ptr initImpl(const Dataset &points, std::span ids) const { svs::threads::ThreadPoolHandle threadpool_handle{VecSimSVSThreadPool{threadpool_}}; @@ -207,7 +193,7 @@ class SVSIndex : public VecSimIndexAbstract, fl this->blockSize, this->getAllocator(), logger_); // Create SVS MutableIndex instance - auto impl = std::make_shared(std::move(graph), std::move(data), entry_point, + auto impl = std::make_unique(std::move(graph), std::move(data), entry_point, std::move(distance), ids, threadpool_, logger_); // Set SVS MutableIndex build parameters to be used in future updates @@ -250,8 +236,8 @@ class SVSIndex : public VecSimIndexAbstract, fl // Handler to manage SVS implementation instance struct SVSImplHandler : public SVSIndexBase::ImplHandler { - std::shared_ptr impl; - SVSImplHandler(std::shared_ptr impl) : impl{std::move(impl)} {} + std::unique_ptr impl; + SVSImplHandler(std::unique_ptr impl) : impl{std::move(impl)} {} }; std::unique_ptr createImpl(const void *vectors_data, const labelType *labels, @@ -270,19 +256,21 @@ class SVSIndex : public VecSimIndexAbstract, fl return std::make_unique(initImpl(points, ids)); } + void storeImpl(impl_type *impl) { + this->impl_.store(impl, std::memory_order_release); + } + void setImpl(std::unique_ptr handler) override { SVSImplHandler *svs_handler = dynamic_cast(handler.get()); if (!svs_handler) { throw std::logic_error("Failed to cast to SVSImplHandler"); } - std::lock_guard replace_lock(this->implMutationGuard_); if (getImpl()) { throw std::logic_error("SVSIndex::setImpl called on non-empty impl_"); } - std::lock_guard lock(this->pimplGuard_); - this->impl_ = std::move(svs_handler->impl); + storeImpl(svs_handler->impl.release()); } // Assuming parallelism was updated to reflect the number of available threads before this @@ -291,8 +279,7 @@ class SVSIndex : public VecSimIndexAbstract, fl // for the operation. // Important NOTE: For single vector operations (n=1), parallelism should be 1. // For bulk operations (n>1), parallelism should reflect the number of available threads. - int addVectorsImpl(const void *vectors_data, const labelType *labels, size_t n, - bool allow_init = true) { + int addVectorsImpl(const void *vectors_data, const labelType *labels, size_t n) { if (n == 0) { return 0; } @@ -311,78 +298,46 @@ class SVSIndex : public VecSimIndexAbstract, fl deleted_num = deleteVectorsImpl(labels, n); } - this->implMutationGuard_.lock_shared(); if (auto impl = getImpl()) { + // Add new points to existing SVS index impl->add_points(points, ids, /*reuse_empty*/ false); - this->implMutationGuard_.unlock_shared(); } else { - this->implMutationGuard_.unlock_shared(); - this->implMutationGuard_.lock(); - if (auto existing = getImpl()) { - existing->add_points(points, ids, /*reuse_empty*/ false); - } else if (!allow_init) { - this->implMutationGuard_.unlock(); - return kNotInitialized; - } else { - auto built = initImpl(points, ids); - assert(built != nullptr); - std::lock_guard lock(this->pimplGuard_); - this->impl_ = std::move(built); - } - this->implMutationGuard_.unlock(); + // SVS index instance cannot be empty, so we have to construct it at first rows + storeImpl(initImpl(points, ids).release()); } return n - deleted_num; } void consolidate(const std::vector &labels) override { - if (auto impl = getImpl()) { + if (auto impl = getImpl(); impl && labelCountOf(*impl) > 0) { impl->consolidate(labels); } } int deleteVectorImpl(const labelType label) { - auto impl = getImpl(); - if (!impl) { + if (auto impl = getImpl()) { + const int deleted_num = impl->delete_entries(std::span{&label, 1}); + this->markIndexUpdate(deleted_num); + return deleted_num; + } else { return 0; } - - const int deleted_num = impl->delete_entries(std::span{&label, 1}); - this->markIndexUpdate(impl, deleted_num); - return deleted_num; } int deleteVectorsImpl(const labelType *labels, size_t n) { - auto impl = getImpl(); - if (!impl) { + if (auto impl = getImpl()) { + const int deleted_num = impl->delete_entries(std::span{labels, n}); + if (deleted_num > 0) { + this->markIndexUpdate(deleted_num); + } + return deleted_num; + } else { return 0; } - - const int deleted_num = impl->delete_entries(std::span{labels, n}); - if (deleted_num > 0) { - this->markIndexUpdate(impl, deleted_num); - } - return deleted_num; } - void markIndexUpdate(const std::shared_ptr &impl, size_t n = 1) { - // SVS index instance should not be empty - if (labelCountOf(*impl) == 0) { - std::lock_guard replace_lock(this->implMutationGuard_); - if (getImpl() != impl) { - // The instance was already droped - return; - } - if (labelCountOf(*impl) == 0) { - { - std::lock_guard lock(this->pimplGuard_); - this->impl_.reset(); - } - num_marked_deleted.store(0, std::memory_order_relaxed); - return; - } - } - + void markIndexUpdate(size_t n = 1) { num_marked_deleted.fetch_add(n, std::memory_order_relaxed); } @@ -419,7 +374,7 @@ class SVSIndex : public VecSimIndexAbstract, fl svs_details::getOrDefault(params.leanvec_dim, SVS_VAMANA_DEFAULT_LEANVEC_DIM)}, epsilon{svs_details::getOrDefault(params.epsilon, SVS_VAMANA_DEFAULT_EPSILON)}, is_two_level_lvq{isTwoLevelLVQ(params.quantBits)}, - threadpool_{this->allocator, this->logCallbackCtx}, impl_{nullptr} { + threadpool_{this->allocator, this->logCallbackCtx} { logger_ = makeLogger(); if (params.num_threads != 0) { this->log(VecSimCommonStrings::LOG_WARNING_STRING, @@ -428,12 +383,9 @@ class SVSIndex : public VecSimIndexAbstract, fl } } - ~SVSIndex() = default; + ~SVSIndex() { delete this->impl_.load(std::memory_order_relaxed); } - bool ready() const override { - std::shared_lock lock(this->pimplGuard_); - return this->impl_ != nullptr; - } + bool ready() const override { return getImpl() != nullptr; } size_t indexSize() const override { return indexStorageSize(); } @@ -605,13 +557,6 @@ class SVSIndex : public VecSimIndexAbstract, fl return addVectorsImpl(vectors_data, labels, n); } - int addVectorsIfInitialized(const void *vectors_data, const labelType *labels, - size_t n) override { - assert(!(n == 1 && getParallelism() > 1) && - "Can't use more than one thread to insert a single vector"); - return addVectorsImpl(vectors_data, labels, n, /*allow_init=*/false); - } - int deleteVector(labelType label) override { return deleteVectorImpl(label); } int deleteVectors(const labelType *labels, size_t n) override { @@ -628,7 +573,6 @@ class SVSIndex : public VecSimIndexAbstract, fl return VecSimRelabel_SameLabel; } - std::shared_lock mutation_lock(this->implMutationGuard_); auto impl = getImpl(); // A null impl_ is an index that never held a vector, so it holds neither label. @@ -812,8 +756,7 @@ class SVSIndex : public VecSimIndexAbstract, fl NullSVS_BatchIterator(queryBlobCopyPtr, queryParams, this->getAllocator()); } else { return new (this->getAllocator()) SVS_BatchIterator( - queryBlobCopyPtr, std::move(impl), queryParams, this->getAllocator(), - is_two_level_lvq); + queryBlobCopyPtr, impl, queryParams, this->getAllocator(), is_two_level_lvq); } } @@ -863,8 +806,7 @@ class SVSIndex : public VecSimIndexAbstract, fl void appendStoredDataByLabel(labelType label, std::vector> &vectors_output) const { // The spans handed to `append_datum` point straight into the dataset, so the instance has - // to stay alive for the whole walk. The snapshot's reference keeps it alive even if - // markIndexUpdate() drops impl_ meanwhile. + // to stay alive for the whole walk. auto impl = getImpl(); if (!impl) { return; diff --git a/src/VecSim/algorithms/svs/svs_batch_iterator.h b/src/VecSim/algorithms/svs/svs_batch_iterator.h index a31c02397..f40b38ece 100644 --- a/src/VecSim/algorithms/svs/svs_batch_iterator.h +++ b/src/VecSim/algorithms/svs/svs_batch_iterator.h @@ -29,9 +29,7 @@ class SVS_BatchIterator : public VecSimBatchIterator { using dist_type = typename Index::distance_type; size_t dim; - // Declared before impl_ so that impl_, which holds a raw back-pointer into the index, is - // destroyed first. - std::shared_ptr index_; + const Index *index_; impl_type impl_; typename impl_type::const_iterator curr_it; size_t batch_size; @@ -70,12 +68,11 @@ class SVS_BatchIterator : public VecSimBatchIterator { } public: - SVS_BatchIterator(void *query_vector, std::shared_ptr index, - const VecSimQueryParams *queryParams, + SVS_BatchIterator(void *query_vector, const Index *index, const VecSimQueryParams *queryParams, std::shared_ptr allocator, bool is_two_level_lvq) : VecSimBatchIterator{query_vector, queryParams ? queryParams->timeoutCtx : nullptr, std::move(allocator)}, - dim{index->dimensions()}, index_{std::move(index)}, + dim{index->dimensions()}, index_{index}, impl_{index_->make_batch_iterator( std::span{static_cast(query_vector), dim})}, curr_it{impl_.begin()} { diff --git a/src/VecSim/algorithms/svs/svs_serializer_impl.h b/src/VecSim/algorithms/svs/svs_serializer_impl.h index 33fc6ca53..565d904d9 100644 --- a/src/VecSim/algorithms/svs/svs_serializer_impl.h +++ b/src/VecSim/algorithms/svs/svs_serializer_impl.h @@ -96,7 +96,7 @@ void SVSIndex // Verify metadata compatibility, will throw runtime exception if not compatible compareMetadataFile(folder_path + "/metadata"); - std::shared_ptr loaded_impl; + std::unique_ptr loaded_impl; if constexpr (isMulti) { auto loaded = svs::concurrent::auto_multi_dynamic_assemble( folder_path + "/config", @@ -106,7 +106,7 @@ void SVSIndex this->getAllocator())), distance_f(), std::move(threadpool_handle), svs::concurrent::MultiMutableVamanaLoad::FROM_MULTI, logger_); - loaded_impl = std::make_shared(std::move(loaded)); + loaded_impl = std::make_unique(std::move(loaded)); } else { auto loaded = svs::concurrent::auto_dynamic_assemble( folder_path + "/config", @@ -115,12 +115,10 @@ void SVSIndex SVS_LAZY(storage_traits_t::load(folder_path + "/data", this->blockSize, this->dim, this->getAllocator())), distance_f(), std::move(threadpool_handle), false, logger_); - loaded_impl = std::make_shared(std::move(loaded)); + loaded_impl = std::make_unique(std::move(loaded)); } - std::lock_guard replace_lock(this->implMutationGuard_); - std::lock_guard lock(this->pimplGuard_); - impl_ = std::move(loaded_impl); + delete impl_.exchange(loaded_impl.release(), std::memory_order_release); } template { // The reason of following container just to properly destroy jobs which not executed yet SVSMultiThreadJob::JobsRegistry uncompletedJobs; - // frontend ids holding training data for the backend (re)initialization + // frontend ids holding training data for the backend initialization std::unordered_set ids_to_init_; vecsim_stl::unordered_map> @@ -525,11 +525,7 @@ class TieredSVSIndex : public VecSimTieredIndex { assert(index); // prevent parallel updates std::lock_guard lock(index->updateJobMutex); - // flag stays set while init runs: !ready() + !flag means "no one will init the backend" - struct ClearOnExit { - std::atomic_flag &flag; - ~ClearOnExit() { flag.clear(std::memory_order_release); } - } clear_on_exit{index->indexUpdateScheduled}; + // Update the SVS index index->initSVSIndex(availableThreads); } @@ -749,15 +745,6 @@ class TieredSVSIndex : public VecSimTieredIndex { return InsertJobOutcome::Deferred; } - // a job never inits the backend: one point cannot train the compression - if (!svs_index->ready()) { - this->flatIndexGuard.unlock_shared(); - // init pending -> wait for it. no init pending -> back to the training buffer - return this->indexUpdateScheduled.test(std::memory_order_acquire) - ? InsertJobOutcome::Deferred - : adoptJobIntoInitBuffer(job); - } - // Copy the vector blob out of the flat buffer while holding flatIndexGuard, so we // can release the flat lock before indexing into the SVS backend size_t data_size = this->frontendIndex->getStoredDataSize(); @@ -767,16 +754,9 @@ class TieredSVSIndex : public VecSimTieredIndex { job->status.store(SVSInsertJob::Status::Executing, std::memory_order_release); this->flatIndexGuard.unlock_shared(); - // if a concurrent deletion drops the instance - // initializing it from this single point would refit the compression - const int added = svs_index->addVectorsIfInitialized(blob_copy.get(), &job->label, 1); - const bool published = added != SVSIndexBase::kNotInitialized; + svs_index->addVector(blob_copy.get(), job->label); - job->status.store(published ? SVSInsertJob::Status::Done : SVSInsertJob::Status::Pending, - std::memory_order_release); - if (!published) { - return adoptJobIntoInitBuffer(job); - } + job->status.store(SVSInsertJob::Status::Done, std::memory_order_release); // Remove the vector and the insert job from the flat buffer. this->removeIngestedVectorFromFlat(job); return InsertJobOutcome::Completed; @@ -847,6 +827,11 @@ class TieredSVSIndex : public VecSimTieredIndex { assert(total_deleted == labels_to_move.size() && "Deleted vectors count does not match the number of labels to delete"); + + // Release the scheduled flag to allow scheduling again. + // Repeted sheduling is required if labels_to_move.empty() + // Under the guard, so a vector buffered after it schedules its own init. + indexUpdateScheduled.clear(); } // release frontend index executeTracingCallback("UpdateJob::after_add_to_svs"); } @@ -908,38 +893,23 @@ class TieredSVSIndex : public VecSimTieredIndex { } // backend index is initialized - we can add the vector directly auto storage_blob = this->frontendIndex->preprocessForStorage(blob); + // Only updateJobMutex is needed here + std::lock_guard lock(this->updateJobMutex); + // Defensive: ensure single-threaded operation for write-in-place mode. + // parallelism_ defaults to 1, so this is a no-op in the normal case. + svs_index->setParallelism(1); int deleted = 0; - { - // prevent update job from running in parallel and lock any access to the backend - // index - // Only updateJobMutex is needed here, not mainIndexGuard: the concurrent - // backend index serializes writes against concurrent readers itself. - std::lock_guard lock(this->updateJobMutex); - // Defensive: ensure single-threaded operation for write-in-place mode. - // parallelism_ defaults to 1, so this is a no-op in the normal case. - svs_index->setParallelism(1); - if (!this->backendIndex->isMultiValue()) { - deleted = svs_index->deleteVector(label); - if (deleted > 0) - svs_index->consolidate({label}); - } - if (svs_index->ready()) { - return this->backendIndex->addVector(storage_blob.get(), label) - deleted; - } - } - // the delete above dropped the last backend vector: adding now would refit the - // compression to this single point - if (auto buffered = tryBufferForTraining(blob, label, /*in_place=*/true)) { - return std::max(*buffered - deleted, 0); + if (!this->backendIndex->isMultiValue()) { + deleted = svs_index->deleteVector(label); + if (deleted > 0) + svs_index->consolidate({label}); } - // a queued init job rebuilt the backend meanwhile - std::lock_guard lock(this->updateJobMutex); - return std::max(this->backendIndex->addVector(storage_blob.get(), label) - deleted, 0); + return this->backendIndex->addVector(storage_blob.get(), label) - deleted; } assert(this->getWriteMode() != VecSim_WriteInPlace && "InPlace mode returns early"); // Async mode - buffer training data until the backend can be inited from a batch. - if (!svs_index->ready() && !this->indexUpdateScheduled.test(std::memory_order_acquire)) { + if (!svs_index->ready()) { if (auto buffered = tryBufferForTraining(blob, label, /*in_place=*/false)) { return *buffered; } @@ -953,13 +923,6 @@ class TieredSVSIndex : public VecSimTieredIndex { const int overwritten = this->backendIndex->isMultiValue() ? 0 : this->deleteVector(label); - // the delete above may have dropped the last backend vector - if (!svs_index->ready()) { - if (auto buffered = tryBufferForTraining(blob, label, /*in_place=*/false)) { - return std::max(*buffered - overwritten, 0); - } - } - std::shared_lock lock(updateJobMutex); ret = svs_index->addVector(storage_blob.get(), label); return std::max(ret - overwritten, 0); @@ -1115,7 +1078,7 @@ class TieredSVSIndex : public VecSimTieredIndex { return job; } - // Buffer a vector in the frontend index as training data for the backend (re)initialization. + // Buffer a vector in the frontend index as training data for the backend initialization. // Returns 1 for a new label, 0 for an overwrite, or nullopt if the backend turned ready while // the guard was taken std::optional tryBufferForTraining(const void *blob, labelType label, bool in_place) { @@ -1129,9 +1092,7 @@ class TieredSVSIndex : public VecSimTieredIndex { } int deleted = 0; if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { - // once the backend has been dropped the buffer may also hold job-backed vectors, - // so the overwrite has to invalidate pending jobs - deleted = removeLabelFromFlat(label); + deleted = deleteAndUpdateInitIds(label); } ids_to_init_.insert(this->frontendIndex->indexSize()); ret = std::max(this->frontendIndex->addVector(blob, label) - deleted, 0); @@ -1147,26 +1108,6 @@ class TieredSVSIndex : public VecSimTieredIndex { return ret; } - // Move a pending job's vector from the insert-job path to the training batch. The vector stays - // in the frontend index, only its ownership changes. - InsertJobOutcome adoptJobIntoInitBuffer(SVSInsertJob *job) { - std::lock_guard flat_lock{this->flatIndexGuard}; - if (!job->isValid) { - std::lock_guard invalid_lock{this->invalidJobsLookupGuard}; - this->invalidJobs.erase(job->id); - return InsertJobOutcome::Completed; - } - if (GetSVSIndex()->ready() || this->indexUpdateScheduled.test(std::memory_order_acquire)) { - return InsertJobOutcome::Deferred; - } - ids_to_init_.insert(job->id); - this->detachInsertJob(job); - if (ids_to_init_.size() >= this->trainingTriggerThreshold) { - scheduleSVSIndexInit(); - } - return InsertJobOutcome::Completed; - } - int deleteVector(labelType label) override { int ret = 0; diff --git a/tests/unit/test_svs.cpp b/tests/unit/test_svs.cpp index 164da83e7..c47708759 100644 --- a/tests/unit/test_svs.cpp +++ b/tests/unit/test_svs.cpp @@ -187,11 +187,22 @@ TYPED_TEST(SVSTest, svs_vector_update_test) { // Prepare new vector data and call addVector with the same id, different data. GenerateAndAddVector(index, dim, 1, 2.0); - // Index size shouldn't change. + // Now we have one deleted (not empty) slot + // and one live slot + EXPECT_EQ(VecSimIndex_IndexSize(index), 2); + ASSERT_EQ(svs_index->getNumMarkedDeleted(), 1); + + index->runGC(); + // GC consolidated and compacted deleted slot. EXPECT_EQ(VecSimIndex_IndexSize(index), 1); // Delete the last vector. VecSimIndex_DeleteVector(index, 1); + // Still have a single "deleted" slot + EXPECT_EQ(VecSimIndex_IndexSize(index), 1); + ASSERT_EQ(svs_index->getNumMarkedDeleted(), 1); + + index->runGC(); EXPECT_EQ(VecSimIndex_IndexSize(index), 0); ASSERT_EQ(svs_index->getNumMarkedDeleted(), 0); @@ -282,8 +293,11 @@ TYPED_TEST(SVSTest, svs_bulk_vectors_add_delete_test) { // Delete rest of the vectors // num_marked_deleted should reset. ASSERT_EQ(svs_index->deleteVectors(ids.data() + n - keep_num, keep_num), keep_num); - ASSERT_EQ(VecSimIndex_IndexSize(index), 0); + ASSERT_EQ(VecSimIndex_IndexSize(index), n); ASSERT_EQ(index->indexLabelCount(), 0); + ASSERT_EQ(svs_index->getNumMarkedDeleted(), n); + index->runGC(); + ASSERT_EQ(VecSimIndex_IndexSize(index), 0); ASSERT_EQ(svs_index->getNumMarkedDeleted(), 0); VecSimIndex_Free(index); } @@ -348,14 +362,13 @@ TYPED_TEST(SVSTest, two_stage_initialization_test) { impl = svs_index->createImpl(v.data(), ids.data(), n); EXPECT_THROW(svs_index->setImpl(std::move(impl)), std::logic_error); - // Delete rest of the vectors - index should be empty now and setImpl() should succeed. + // Delete rest of the vectors - index should be empty, but impl still alive. ASSERT_EQ(svs_index->deleteVectors(ids.data() + n - keep_num, keep_num), keep_num); - ASSERT_EQ(VecSimIndex_IndexSize(index), 0); - // Re-initialization should succeed. - impl = svs_index->createImpl(v.data(), ids.data(), n); - svs_index->setImpl(std::move(impl)); ASSERT_EQ(VecSimIndex_IndexSize(index), n); - runTopKSearchTest(index, query, k, verify_res, nullptr, BY_ID); + + // Re-initialization should fail again. + impl = svs_index->createImpl(v.data(), ids.data(), n); + EXPECT_THROW(svs_index->setImpl(std::move(impl)), std::logic_error); VecSimIndex_Free(index); } @@ -964,10 +977,11 @@ TYPED_TEST(SVSTest, svs_empty_index) { VecSimIndex_DeleteVector(index, 1); // Size equals 0. + index->runGC(); ASSERT_EQ(VecSimIndex_IndexSize(index), 0); - // The expected capacity should be 0 for empty index. - ASSERT_EQ(index->indexCapacity(), 0); + // Even an empty index holds the first block for nonquantized data + // ASSERT_EQ(index->indexCapacity(), 0); ASSERT_EQ(index->indexMetaDataCapacity(), index->indexCapacity()); // Try to remove it again. @@ -1215,6 +1229,7 @@ TYPED_TEST(SVSTest, test_dynamic_svs_info_iterator) { // Delete vector. VecSimIndex_DeleteVector(index, 0); + index->runGC(); info = VecSimIndex_DebugInfo(index); infoIter = VecSimIndex_DebugInfoIterator(index); ASSERT_EQ(0, info.commonInfo.indexSize); @@ -1426,6 +1441,7 @@ TYPED_TEST(SVSTest, svs_search_empty_index) { for (size_t i = 0; i < n; i++) { VecSimIndex_DeleteVector(index, i); } + index->runGC(); ASSERT_EQ(VecSimIndex_IndexSize(index), 0); // Again - we do not expect any results. diff --git a/tests/unit/test_svs_fp16.cpp b/tests/unit/test_svs_fp16.cpp index 63ccf1aab..08c989dc1 100644 --- a/tests/unit/test_svs_fp16.cpp +++ b/tests/unit/test_svs_fp16.cpp @@ -199,12 +199,24 @@ TYPED_TEST(FP16SVSTest, svs_vector_update_test) { // Prepare new vector data and call addVector with the same id, different data. this->GenerateAndAddVector(index, dim, 1, 2.0); - // Index size shouldn't change. + // Now we have one deleted (not empty) slot + // and one live slot + EXPECT_EQ(VecSimIndex_IndexSize(index), 2); + ASSERT_EQ(svs_index->getNumMarkedDeleted(), 1); + + index->runGC(); + // GC consolidated and compacted deleted slot. EXPECT_EQ(VecSimIndex_IndexSize(index), 1); // Delete the last vector. VecSimIndex_DeleteVector(index, 1); + // Still have a single "deleted" slot + EXPECT_EQ(VecSimIndex_IndexSize(index), 1); + ASSERT_EQ(svs_index->getNumMarkedDeleted(), 1); + + index->runGC(); EXPECT_EQ(VecSimIndex_IndexSize(index), 0); + ASSERT_EQ(svs_index->getNumMarkedDeleted(), 0); VecSimIndex_Free(index); } @@ -264,8 +276,11 @@ TYPED_TEST(FP16SVSTest, svs_bulk_vectors_add_delete_test) { // Delete rest of the vectors // num_marked_deleted should reset. ASSERT_EQ(svs_index->deleteVectors(ids.data() + n - keep_num, keep_num), keep_num); - ASSERT_EQ(VecSimIndex_IndexSize(index), 0); + ASSERT_EQ(VecSimIndex_IndexSize(index), n); ASSERT_EQ(index->indexLabelCount(), 0); + ASSERT_EQ(svs_index->getNumMarkedDeleted(), n); + index->runGC(); + ASSERT_EQ(VecSimIndex_IndexSize(index), 0); ASSERT_EQ(svs_index->getNumMarkedDeleted(), 0); VecSimIndex_Free(index); } @@ -760,10 +775,11 @@ TYPED_TEST(FP16SVSTest, svs_empty_index) { for (size_t i = 0; i < n; i++) { VecSimIndex_DeleteVector(index, i); } + index->runGC(); ASSERT_EQ(VecSimIndex_IndexSize(index), 0); - // The expected capacity should be 0 for empty index. - ASSERT_EQ(index->indexCapacity(), 0); + // Even an empty index holds the first block for nonquantized data + // ASSERT_EQ(index->indexCapacity(), 0); // Try to remove it again. VecSimIndex_DeleteVector(index, 1); @@ -1025,6 +1041,7 @@ TYPED_TEST(FP16SVSTest, test_dynamic_svs_info_iterator) { // Delete vector. VecSimIndex_DeleteVector(index, 0); + index->runGC(); info = VecSimIndex_DebugInfo(index); infoIter = VecSimIndex_DebugInfoIterator(index); ASSERT_EQ(0, info.commonInfo.indexSize); @@ -1805,6 +1822,7 @@ TYPED_TEST(FP16SVSMultiTest, vector_add_multiple_test) { // Deleting the label. All the vectors should be deleted. ASSERT_EQ(VecSimIndex_DeleteVector(index, label), n); + index->runGC(); ASSERT_EQ(VecSimIndex_IndexSize(index), 0); ASSERT_EQ(index->indexLabelCount(), 0); @@ -2042,6 +2060,7 @@ TYPED_TEST(FP16SVSMultiTest, reindexing_same_vector_different_id) { for (size_t i = 0; i < n; i++) { VecSimIndex_DeleteVector(index, i); } + index->runGC(); ASSERT_EQ(VecSimIndex_IndexSize(index), 0); // Reinsert the same vectors under different ids than before. @@ -2850,6 +2869,9 @@ TYPED_TEST(FP16SVSTieredIndexTest, deleteVector) { // Remove from main index. ASSERT_EQ(tiered_index->deleteVector(vec_label), 1); ASSERT_EQ(tiered_index->indexLabelCount(), 0); + VecSimTieredIndex_GC(tiered_index); + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->indexSize(), 0); // Re-insert a deleted label with a different vector. @@ -2906,6 +2928,10 @@ TYPED_TEST(FP16SVSTieredIndexTest, deleteVectorMulti) { ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 1); ASSERT_EQ(tiered_index->deleteVector(vec_label), 2); + + VecSimTieredIndex_GC(tiered_index); + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->indexSize(), 0); ASSERT_EQ(tiered_index->indexLabelCount(), 0); diff --git a/tests/unit/test_svs_multi.cpp b/tests/unit/test_svs_multi.cpp index 5e43b4b02..7fad50359 100644 --- a/tests/unit/test_svs_multi.cpp +++ b/tests/unit/test_svs_multi.cpp @@ -128,6 +128,7 @@ TYPED_TEST(SVSMultiTest, vector_add_multiple_test) { // Deleting the label. All the vectors should be deleted. ASSERT_EQ(VecSimIndex_DeleteVector(index, id), rep); + index->runGC(); ASSERT_EQ(VecSimIndex_IndexSize(index), 0); ASSERT_EQ(index->indexLabelCount(), 0); @@ -164,6 +165,7 @@ TYPED_TEST(SVSMultiTest, empty_index) { ASSERT_EQ(VecSimIndex_DeleteVector(index, 1), 3); // Size equals 0. + index->runGC(); ASSERT_EQ(VecSimIndex_IndexSize(index), 0); // Try to remove it again. @@ -431,6 +433,7 @@ TYPED_TEST(SVSMultiTest, reindexing_same_vector_different_id) { for (size_t i = 0; i < n; i++) { VecSimIndex_DeleteVector(index, i); } + index->runGC(); ASSERT_EQ(VecSimIndex_IndexSize(index), 0); // Reinsert the same vectors under different ids than before. @@ -616,6 +619,7 @@ TYPED_TEST(SVSMultiTest, search_empty_index) { } ASSERT_EQ(VecSimIndex_IndexSize(index), n); VecSimIndex_DeleteVector(index, 46); + index->runGC(); ASSERT_EQ(VecSimIndex_IndexSize(index), 0); // Again - we do not expect any results. @@ -786,16 +790,15 @@ TYPED_TEST(SVSMultiTest, emptyIndex) { // Try to remove it. ASSERT_EQ(VecSimIndex_DeleteVector(index, 1), 1); - // The capacity should change to be zero. - ASSERT_EQ(index->indexCapacity(), 0); + // Even an empty index holds the first block for nonquantized data + // ASSERT_EQ(index->indexCapacity(), 0); // Size equals 0. + index->runGC(); ASSERT_EQ(VecSimIndex_IndexSize(index), 0); // Try to remove it again. - // The capacity should remain unchanged, as we are trying to delete a label that doesn't exist. ASSERT_EQ(VecSimIndex_DeleteVector(index, 1), 0); - ASSERT_EQ(index->indexCapacity(), 0); // Nor the size. ASSERT_EQ(VecSimIndex_IndexSize(index), 0); diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 005411a67..977b659e8 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -1096,16 +1096,14 @@ TYPED_TEST(SVSTieredIndexTest, addVector) { ASSERT_LE(expected_mem, tiered_index->getAllocationSize()); if constexpr (TypeParam::isMulti()) { - // Add another vector under the same label. The backend init was already submitted - // by the first vector (thread_pool_size batch-init jobs), so this second vector takes - // the per-vector async path and schedules one additional SVSInsertJob of its own. + // Add another vector under the same label. VecSimIndex_AddVector(tiered_index, vector, vec_label); ASSERT_EQ(tiered_index->indexSize(), 2); ASSERT_EQ(tiered_index->indexLabelCount(), 1); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 2); - // The batch-init jobs plus the extra per-vector insert job. - ASSERT_EQ(mock_thread_pool.jobQ.size(), mock_thread_pool.thread_pool_size + 1); + // The batch-init jobs. + ASSERT_EQ(mock_thread_pool.jobQ.size(), mock_thread_pool.thread_pool_size); } } @@ -1618,6 +1616,10 @@ TYPED_TEST(SVSTieredIndexTest, deleteVector) { // Remove from main index. ASSERT_EQ(tiered_index->deleteVector(vec_label), 1); ASSERT_EQ(tiered_index->indexLabelCount(), 0); + VecSimTieredIndex_GC(tiered_index); + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); + ASSERT_EQ(tiered_index->indexSize(), 0); // Re-insert a deleted label with a different vector. @@ -1713,7 +1715,10 @@ TYPED_TEST(SVSTieredIndexTestBasic, markedDeleted) { VecSimIndex_DeleteVector(tiered_index, i); } - // Consolidate should be triggered and mark deleted count should be zeroed. + VecSimTieredIndex_GC(tiered_index); + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); + ASSERT_EQ(tiered_index->indexSize(), 0); ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 0); ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 0); @@ -1751,13 +1756,13 @@ TYPED_TEST(SVSTieredIndexTestBasic, deleteVectorMulti) { ASSERT_EQ(tiered_index->indexLabelCount(), 1); ASSERT_EQ(tiered_index->indexSize(), 2); ASSERT_EQ(tiered_index->deleteVector(vec_label), 2); + + VecSimTieredIndex_GC(tiered_index); + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->indexSize(), 0); ASSERT_EQ(tiered_index->indexLabelCount(), 0); - mock_thread_pool.thread_iteration(); - mock_thread_pool.thread_iteration(); - ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); - // Test deleting a label for which both of its vector's is in the flat index. GenerateAndAddVector(tiered_index, dim, vec_label, vec_label); GenerateAndAddVector(tiered_index, dim, vec_label, other_vec_val); @@ -3192,11 +3197,12 @@ TYPED_TEST(SVSTieredIndexTestBasic, overwriteVectorBasic) { ASSERT_EQ(tiered_index->addVector(overwritten_vec, 0), 0); ASSERT_EQ(tiered_index->indexLabelCount(), 1); // Overriding vector in tiered index should remove the vector from SVS to avoid duplicates. - ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 0); + ASSERT_EQ(tiered_index->GetBackendIndex()->indexLabelCount(), 0); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 1); ASSERT_EQ(tiered_index->getDistanceFrom_Unsafe(0, overwritten_vec), 0); // Ingest the updated vector to SVS. + VecSimTieredIndex_GC(tiered_index); while (mock_thread_pool.jobQ.size() > 0) mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 1); @@ -3381,6 +3387,10 @@ TYPED_TEST(SVSTieredIndexTest, testInfo) { } VecSimIndex_DeleteVector(tiered_index, 1); + VecSimTieredIndex_GC(tiered_index); + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); + info = tiered_index->debugInfo(); EXPECT_EQ(info.commonInfo.indexSize, 0); @@ -4000,8 +4010,8 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalSingle) { // Nothing remains in flat. EXPECT_EQ(tiered_index->GetBackendIndex()->indexLabelCount(), n - 2); - EXPECT_EQ(tiered_index->GetBackendIndex()->indexSize(), n - 2); - ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 0); + EXPECT_EQ(tiered_index->GetBackendIndex()->indexSize(), n + 3); + ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 5); // Flat buffer fully drained by the insert jobs. ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->indexSize(), tiered_index->GetBackendIndex()->indexSize() + @@ -4136,8 +4146,8 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalMulti) { // the flat buffer into the backend, so nothing remains in flat. EXPECT_EQ(tiered_index->GetBackendIndex()->indexLabelCount(), n - 2); - EXPECT_EQ(tiered_index->GetBackendIndex()->indexSize(), n - 1); - ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 0); + EXPECT_EQ(tiered_index->GetBackendIndex()->indexSize(), n + 3); + ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 4); // Flat buffer fully drained by the insert jobs. ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->indexSize(), tiered_index->GetBackendIndex()->indexSize() + From ab87a6e5b7d9cfc13e03592d272d0e92c7e6b47b Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 24 Sep 2026 07:18:57 -0700 Subject: [PATCH 68/99] format --- src/VecSim/algorithms/svs/svs.h | 4 +--- src/VecSim/algorithms/svs/svs_tiered.h | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 82257e745..9bdeedf92 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -256,9 +256,7 @@ class SVSIndex : public VecSimIndexAbstract, fl return std::make_unique(initImpl(points, ids)); } - void storeImpl(impl_type *impl) { - this->impl_.store(impl, std::memory_order_release); - } + void storeImpl(impl_type *impl) { this->impl_.store(impl, std::memory_order_release); } void setImpl(std::unique_ptr handler) override { SVSImplHandler *svs_handler = dynamic_cast(handler.get()); diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index af13134a3..691f09f07 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -829,7 +829,7 @@ class TieredSVSIndex : public VecSimTieredIndex { "Deleted vectors count does not match the number of labels to delete"); // Release the scheduled flag to allow scheduling again. - // Repeted sheduling is required if labels_to_move.empty() + // Repeted sheduling is required if labels_to_move.empty() // Under the guard, so a vector buffered after it schedules its own init. indexUpdateScheduled.clear(); } // release frontend index From 8f21456850f3e6dee082292cb2bed24b07550e22 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 24 Sep 2026 08:04:38 -0700 Subject: [PATCH 69/99] clean-up and unify --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 22 ++-------- src/VecSim/algorithms/svs/svs_tiered.h | 52 ++++-------------------- src/VecSim/vec_sim_tiered_index.h | 21 ++++++++++ 3 files changed, 33 insertions(+), 62 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index 8ff4c2787..353861c76 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -769,11 +769,7 @@ int TieredHNSWIndex::addVectorDuringAccumulation(const void if (!job) { job = new (this->allocator) HNSWInsertJob(this->allocator, label, id, executeInsertJobWrapper, this); - auto [it, inserted] = - this->labelToInsertJobs.try_emplace(label, 1, job, this->allocator); - if (!inserted) { - it->second.push_back(job); - } + this->registerInsertJob(label, job); } const size_t count = this->frontendIndex->indexSize(); const bool should_finalize = count >= sqAccumulationState->normalizationSetSize; @@ -916,16 +912,10 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l this->flatIndexGuard.lock(); idType new_flat_id = this->frontendIndex->indexSize(); if (this->frontendIndex->isLabelExists(label) && !this->frontendIndex->isMultiValue()) { - // Overwrite the vector and invalidate its only pending job (since we are not in MULTI). - auto *old_job = this->labelToInsertJobs.at(label).at(0); - old_job->id = this->setAndSaveInvalidJob(old_job); - this->labelToInsertJobs.erase(label); ret = 0; // We are going to update the internal id that currently holds the vector associated with // the given label. - new_flat_id = - dynamic_cast *>(this->frontendIndex) - ->getIdOfLabel(label); + new_flat_id = this->invalidatePendingInsertJob(label); // If we are adding a new element (rather than updating an exiting one) we may need to // increase index capacity. } @@ -934,13 +924,7 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l auto *new_insert_job = new (this->allocator) HNSWInsertJob(this->allocator, label, new_flat_id, executeInsertJobWrapper, this); - // Construct the job vector only for a new label; multi-value labels append to the existing one. - auto [it, inserted] = - this->labelToInsertJobs.try_emplace(label, 1, new_insert_job, this->allocator); - if (!inserted) { - assert(this->backendIndex->isMultiValue()); - it->second.push_back(new_insert_job); - } + this->registerInsertJob(label, new_insert_job); this->flatIndexGuard.unlock(); // Here, a worker might ingest the previous vector that was stored under "label" diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 691f09f07..66c643be6 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -794,17 +794,12 @@ class TieredSVSIndex : public VecSimTieredIndex { svs_index->setParallelism(std::min(availableThreads, labels_to_move.size())); assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); - // The backend may already have been initialized while this job was queued - if (svs_index->ready()) { - svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), - labels_to_move.size()); - svs_index->setParallelism(1); - } else { - auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), - labels_to_move.size()); - svs_index->setParallelism(1); - svs_index->setImpl(std::move(impl)); - } + assert(!svs_index->ready()); + + auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), + labels_to_move.size()); + svs_index->setParallelism(1); + svs_index->setImpl(std::move(impl)); } std::sort(ids_to_move.begin(), ids_to_move.end()); @@ -931,30 +926,10 @@ class TieredSVSIndex : public VecSimTieredIndex { this->flatIndexGuard.lock(); idType new_flat_id = this->frontendIndex->indexSize(); if (this->frontendIndex->isLabelExists(label) && !this->frontendIndex->isMultiValue()) { - if (this->labelToInsertJobs.count(label) == 0) { - // No pending insert job. - // Just replace vector in the frontend - // If this label already exists, this will do overwrite. - ret = this->frontendIndex->addVector(blob, label); - this->flatIndexGuard.unlock(); - return ret; - } - - // Overwrite the vector and invalidate its only pending job (since we are not in - // MULTI). Label exists, but job doesn't. It means the label is used for - // initialization. Just create new job, this job will remove duplicate from the - // backend. - auto *old_job = this->labelToInsertJobs.at(label).at(0); - old_job->id = this->setAndSaveInvalidJob(old_job); - this->labelToInsertJobs.erase(label); - ret = 0; // We are going to update the internal id that currently holds the vector associated // with the given label. - new_flat_id = - dynamic_cast *>(this->frontendIndex) - ->getIdOfLabel(label); - // If we are adding a new element (rather than updating an exiting one) we may need - // to increase index capacity. + new_flat_id = this->invalidatePendingInsertJob(label); + ret = 0; } // If this label already exists, this will do overwrite. ret += this->frontendIndex->addVector(blob, label); @@ -1065,16 +1040,7 @@ class TieredSVSIndex : public VecSimTieredIndex { TieredInsertJob *createInsertJob(labelType label, idType flat_id) { TieredInsertJob *job = new (this->allocator) SVSInsertJob(this->allocator, label, flat_id, executeInsertJobWrapper, this); - auto it = this->labelToInsertJobs.find(label); - if (it != this->labelToInsertJobs.end()) { - // There's already a pending insert job for this label, add another one (without - // overwrite, only possible in multi index) - assert(this->backendIndex->isMultiValue()); - it->second.push_back(job); - } else { - vecsim_stl::vector new_jobs_vec(1, job, this->allocator); - this->labelToInsertJobs.insert({label, new_jobs_vec}); - } + this->registerInsertJob(label, job); return job; } diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index cc87f5a0e..9cf574dc9 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -11,6 +11,7 @@ #include "vec_sim_index.h" #include "algorithms/brute_force/brute_force.h" +#include "algorithms/brute_force/brute_force_single.h" #include "VecSim/batch_iterator.h" #include "VecSim/tombstone_interface.h" #include "VecSim/utils/query_result_utils.h" @@ -279,6 +280,26 @@ class VecSimTieredIndex : public VecSimIndexInterface { } } + // A caller must hold flatIndexGuard. + idType invalidatePendingInsertJob(labelType label) { + auto *old_job = this->labelToInsertJobs.at(label).at(0); + old_job->id = this->setAndSaveInvalidJob(old_job); + this->labelToInsertJobs.erase(label); + return dynamic_cast *>(this->frontendIndex) + ->getIdOfLabel(label); + } + + // A caller must hold flatIndexGuard. + void registerInsertJob(labelType label, TieredInsertJob *job) { + // Construct the job vector only for a new label; multi-value labels append to the existing + // one. + auto [it, inserted] = this->labelToInsertJobs.try_emplace(label, 1, job, this->allocator); + if (!inserted) { + assert(this->backendIndex->isMultiValue()); + it->second.push_back(job); + } + } + // Remove a vector and its insert job from the flat buffer void removeIngestedVectorFromFlat(TieredInsertJob *job) { std::lock_guard flat_lock(this->flatIndexGuard); From d1e06ca072b80a1a0e2dd18476530ae65a44b6bb Mon Sep 17 00:00:00 2001 From: ethanglaser <42726565+ethanglaser@users.noreply.github.com> Date: Mon, 28 Sep 2026 11:46:38 -0700 Subject: [PATCH 70/99] Update to SVS rc3 binaries Co-authored-by: ethanglaser <42726565+ethanglaser@users.noreply.github.com> --- cmake/svs.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index 79baa152e..e028b4253 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -64,7 +64,7 @@ if(USE_SVS) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.31" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "21.0") if(GLIBCXX_VERSION VERSION_GREATER_EQUAL "11") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc2-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc3-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") else() message(STATUS "libstdc++ >= GCC 11 is required for Clang SVS binaries - disabling SVS_SHARED_LIB") set(SVS_SHARED_LIB OFF) @@ -76,9 +76,9 @@ if(USE_SVS) else() if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.28") if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "14.0") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc2-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc3-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") else() - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc2-reduced.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc3-reduced.tar.gz" CACHE STRING "SVS URL") endif() else() message(STATUS "GLIBC >= 2.28 is required for GCC SVS binaries - disabling SVS_SHARED_LIB") From 61baa1fff7e8068c865c5b7420770880101a809d Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Mon, 28 Sep 2026 11:52:49 -0700 Subject: [PATCH 71/99] update submodule --- deps/ScalableVectorSearch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch index 3ee890d6d..cbb74f289 160000 --- a/deps/ScalableVectorSearch +++ b/deps/ScalableVectorSearch @@ -1 +1 @@ -Subproject commit 3ee890d6d52102e979ebd4331823b6236ec17a56 +Subproject commit cbb74f28999e5f91e52e812e1f6188f965b08e13 From d3cdace7f763dce77ce0430735bfe4a28700a1c6 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Tue, 29 Sep 2026 04:15:15 -0700 Subject: [PATCH 72/99] harminize tests --- tests/unit/test_svs.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_svs.cpp b/tests/unit/test_svs.cpp index c24f10e1d..18803988c 100644 --- a/tests/unit/test_svs.cpp +++ b/tests/unit/test_svs.cpp @@ -2881,7 +2881,8 @@ TEST(SVSTest, quant_modes) { const size_t dim = 4; const size_t n = 100; - const size_t k = 10; + // Include both boundary labels, 45 and 55, which tie for query 50. + const size_t k = 11; for (auto quant_bits : {VecSimSvsQuant_NONE, VecSimSvsQuant_Scalar, VecSimSvsQuant_8, VecSimSvsQuant_4, VecSimSvsQuant_4x4, VecSimSvsQuant_4x8, @@ -2962,7 +2963,8 @@ TEST(SVSTest, save_load) { const size_t dim = 4; const size_t n = 100; - const size_t k = 10; + // Include both boundary labels, 45 and 55, in the single-value case. + const size_t k = 11; // Helper function to convert quant_bits to string for error messages auto quant_bits_to_string = [](VecSimSvsQuantBits quant_bits) -> std::string { @@ -3055,7 +3057,7 @@ TEST(SVSTest, save_load) { float query[] = {50, 50, 50, 50}; auto verify_res = [&](size_t id, double score, size_t idx) { EXPECT_DOUBLE_EQ(VecSimIndex_GetDistanceFrom_Unsafe(index, id, query), score); - // Both single and multi should return labels starting from 45 + // Single-value labels are 45..55; multi-value labels are 20..30. if (is_multi) { // For multi, that label of {50,50,50,50} is 25 size_t expected_label = (20 + idx); From 39d98bea15f72f722a2e9d52a15e7f804ff16c92 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Tue, 29 Sep 2026 06:50:34 -0700 Subject: [PATCH 73/99] update to SVS rc4 binaries --- cmake/svs.cmake | 6 +++--- deps/ScalableVectorSearch | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index e028b4253..7c2d28109 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -64,7 +64,7 @@ if(USE_SVS) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.31" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "21.0") if(GLIBCXX_VERSION VERSION_GREATER_EQUAL "11") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc3-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc4-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") else() message(STATUS "libstdc++ >= GCC 11 is required for Clang SVS binaries - disabling SVS_SHARED_LIB") set(SVS_SHARED_LIB OFF) @@ -76,9 +76,9 @@ if(USE_SVS) else() if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.28") if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "14.0") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc3-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc4-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") else() - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc3-reduced.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc4-reduced.tar.gz" CACHE STRING "SVS URL") endif() else() message(STATUS "GLIBC >= 2.28 is required for GCC SVS binaries - disabling SVS_SHARED_LIB") diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch index cbb74f289..c2854821d 160000 --- a/deps/ScalableVectorSearch +++ b/deps/ScalableVectorSearch @@ -1 +1 @@ -Subproject commit cbb74f28999e5f91e52e812e1f6188f965b08e13 +Subproject commit c2854821d226f072849c683857d23f107406b74c From 4ce4ad776bdeb3af943f1adeb428fa2d6012ff53 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Tue, 29 Sep 2026 07:47:26 -0700 Subject: [PATCH 74/99] guard tests --- tests/unit/test_svs.cpp | 13 ++++++++----- tests/unit/test_svs_multi.cpp | 11 +++++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/unit/test_svs.cpp b/tests/unit/test_svs.cpp index 18803988c..c536d9d36 100644 --- a/tests/unit/test_svs.cpp +++ b/tests/unit/test_svs.cpp @@ -3699,11 +3699,14 @@ TYPED_TEST(SVSTest, relabelVector) { ASSERT_EQ(VecSimIndex_GetDistanceFrom_Unsafe(index, 100, query), before); ASSERT_TRUE(std::isnan(VecSimIndex_GetDistanceFrom_Unsafe(index, 1, query))); - auto verify_res = [&](size_t id, double score, size_t rank) { - ASSERT_EQ(id, 100); - ASSERT_EQ(score, before); - }; - runTopKSearchTest(index, query, 1, verify_res); + // SVS Scalar quantization is not enough precise + if (!this->isFallbackToSQ()) { + auto verify_res = [&](size_t id, double score, size_t rank) { + ASSERT_EQ(id, 100); + ASSERT_EQ(score, before); + }; + runTopKSearchTest(index, query, 1, verify_res); + } VecSimIndex_Free(index); } diff --git a/tests/unit/test_svs_multi.cpp b/tests/unit/test_svs_multi.cpp index 8460e6ac2..16508c953 100644 --- a/tests/unit/test_svs_multi.cpp +++ b/tests/unit/test_svs_multi.cpp @@ -1322,10 +1322,13 @@ TYPED_TEST(SVSMultiTest, relabelVectorMulti) { ASSERT_FALSE(std::isnan(VecSimIndex_GetDistanceFrom_Unsafe(index, 8, neighbour))); // A search for one of the moved vectors reports the new label. - TEST_DATA_T query[dim]; - GenerateVector(query, dim, 0); - auto verify_res = [&](size_t id, double score, size_t rank) { ASSERT_EQ(id, 70); }; - runTopKSearchTest(index, query, 1, verify_res); + // SVS Scalar quantization is not enough precise + if (!this->isFallbackToSQ()) { + TEST_DATA_T query[dim]; + GenerateVector(query, dim, 0); + auto verify_res = [&](size_t id, double score, size_t rank) { ASSERT_EQ(id, 70); }; + runTopKSearchTest(index, query, 1, verify_res); + } VecSimIndex_Free(index); } From 368dbd683b8244fa60d1b541dcb1ae908357fc0a Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Tue, 29 Sep 2026 08:21:19 -0700 Subject: [PATCH 75/99] Revert "guard tests" This reverts commit af5c9ec440281412011dfa8255a343d97e8cb2ae. --- tests/unit/test_svs.cpp | 13 +++++-------- tests/unit/test_svs_multi.cpp | 11 ++++------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/tests/unit/test_svs.cpp b/tests/unit/test_svs.cpp index c536d9d36..18803988c 100644 --- a/tests/unit/test_svs.cpp +++ b/tests/unit/test_svs.cpp @@ -3699,14 +3699,11 @@ TYPED_TEST(SVSTest, relabelVector) { ASSERT_EQ(VecSimIndex_GetDistanceFrom_Unsafe(index, 100, query), before); ASSERT_TRUE(std::isnan(VecSimIndex_GetDistanceFrom_Unsafe(index, 1, query))); - // SVS Scalar quantization is not enough precise - if (!this->isFallbackToSQ()) { - auto verify_res = [&](size_t id, double score, size_t rank) { - ASSERT_EQ(id, 100); - ASSERT_EQ(score, before); - }; - runTopKSearchTest(index, query, 1, verify_res); - } + auto verify_res = [&](size_t id, double score, size_t rank) { + ASSERT_EQ(id, 100); + ASSERT_EQ(score, before); + }; + runTopKSearchTest(index, query, 1, verify_res); VecSimIndex_Free(index); } diff --git a/tests/unit/test_svs_multi.cpp b/tests/unit/test_svs_multi.cpp index 16508c953..8460e6ac2 100644 --- a/tests/unit/test_svs_multi.cpp +++ b/tests/unit/test_svs_multi.cpp @@ -1322,13 +1322,10 @@ TYPED_TEST(SVSMultiTest, relabelVectorMulti) { ASSERT_FALSE(std::isnan(VecSimIndex_GetDistanceFrom_Unsafe(index, 8, neighbour))); // A search for one of the moved vectors reports the new label. - // SVS Scalar quantization is not enough precise - if (!this->isFallbackToSQ()) { - TEST_DATA_T query[dim]; - GenerateVector(query, dim, 0); - auto verify_res = [&](size_t id, double score, size_t rank) { ASSERT_EQ(id, 70); }; - runTopKSearchTest(index, query, 1, verify_res); - } + TEST_DATA_T query[dim]; + GenerateVector(query, dim, 0); + auto verify_res = [&](size_t id, double score, size_t rank) { ASSERT_EQ(id, 70); }; + runTopKSearchTest(index, query, 1, verify_res); VecSimIndex_Free(index); } From 46e6da9b1ad41c63d57a7676f9cd0c554c21f62b Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Tue, 29 Sep 2026 10:30:02 -0700 Subject: [PATCH 76/99] update SVS to rc5 --- cmake/svs.cmake | 6 +++--- deps/ScalableVectorSearch | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index 7c2d28109..2491f2e1e 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -64,7 +64,7 @@ if(USE_SVS) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.31" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "21.0") if(GLIBCXX_VERSION VERSION_GREATER_EQUAL "11") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc4-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc5-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") else() message(STATUS "libstdc++ >= GCC 11 is required for Clang SVS binaries - disabling SVS_SHARED_LIB") set(SVS_SHARED_LIB OFF) @@ -76,9 +76,9 @@ if(USE_SVS) else() if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.28") if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "14.0") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc4-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc5-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") else() - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc4-reduced.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc5-reduced.tar.gz" CACHE STRING "SVS URL") endif() else() message(STATUS "GLIBC >= 2.28 is required for GCC SVS binaries - disabling SVS_SHARED_LIB") diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch index c2854821d..107071c99 160000 --- a/deps/ScalableVectorSearch +++ b/deps/ScalableVectorSearch @@ -1 +1 @@ -Subproject commit c2854821d226f072849c683857d23f107406b74c +Subproject commit 107071c99c5f3f111bc3f4d15412529832928368 From 0cc22dc131b6c439a8b20a90b540a483fdc52ae3 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 30 Sep 2026 00:51:43 -0700 Subject: [PATCH 77/99] add more tests --- tests/unit/test_svs_tiered.cpp | 378 +++++++++++++++++++++++++++++++++ 1 file changed, 378 insertions(+) diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 3947e9b08..077871c1f 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -660,6 +660,250 @@ TYPED_TEST(SVSTieredIndexTest, relabelVectorMovesTheLabelInBothWriteStates) { runTopKSearchTest(tiered_index, vector, 1, verify); } +// The test above never reaches `labelToInsertJobs`: with a training threshold of 1 the first +// vector is training data, and training data has no insert job. Draining once makes the backend +// ready, so the next add does get one. +TYPED_TEST(SVSTieredIndexTest, relabelVectorRemapsAPendingInsertJob) { + size_t dim = 4; + SVSParams params = {.type = TypeParam::get_index_type(), + .dim = dim, + .metric = VecSimMetric_L2, + .multi = TypeParam::isMulti(), + .quantBits = TypeParam::get_quant_bits()}; + VecSimParams svs_params = CreateParams(params); + auto mock_thread_pool = tieredIndexMock(); + auto *tiered_index = this->CreateTieredSVSIndex(svs_params, mock_thread_pool, 1, 1); + ASSERT_INDEX(tiered_index); + + mock_thread_pool.init_threads(); + GenerateAndAddVector(tiered_index, dim, 0, 0); + mock_thread_pool.thread_pool_join(); + ASSERT_TRUE(tiered_index->GetSVSIndex()->ready()); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); + + const size_t per_label = TypeParam::isMulti() ? 2 : 1; + for (size_t j = 0; j < per_label; j++) { + GenerateAndAddVector(tiered_index, dim, 7, 7 + j); + } + ASSERT_EQ(mock_thread_pool.jobQ.size(), per_label); + ASSERT_EQ(tiered_index->labelToInsertJobs.at(7).size(), per_label); + + ASSERT_EQ(VecSimIndex_RelabelVector(tiered_index, 7, 70), VecSimRelabel_OK); + + ASSERT_EQ(tiered_index->labelToInsertJobs.count(7), 0); + ASSERT_EQ(tiered_index->labelToInsertJobs.at(70).size(), per_label); + for (auto *job : tiered_index->labelToInsertJobs.at(70)) { + ASSERT_EQ(job->label, 70); + // Left Delayed, the ingest below would defer forever. + ASSERT_EQ(static_cast(job)->status.load(), SVSInsertJob::Status::Pending); + } + ASSERT_TRUE(tiered_index->GetFlatIndex()->isLabelExists(70)); + ASSERT_FALSE(tiered_index->GetFlatIndex()->isLabelExists(7)); + + mock_thread_pool.init_threads(); + mock_thread_pool.thread_pool_join(); + ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); + ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(70)); + ASSERT_FALSE(tiered_index->GetSVSIndex()->isLabelExists(7)); + ASSERT_EQ(tiered_index->indexLabelCount(), 2); +} + +// The delay window: relabelVector marks a pending job Delayed, releases flatIndexGuard, and +// clears Delayed only after the backend remap settled the label. A worker that runs the job +// inside that window must requeue it instead of publishing. The test sets Delayed itself +// rather than racing a relabel. +TYPED_TEST(SVSTieredIndexTest, relabelVectorDefersAnInsertJobInsideTheDelayWindow) { + size_t dim = 4; + SVSParams params = {.type = TypeParam::get_index_type(), + .dim = dim, + .metric = VecSimMetric_L2, + .multi = TypeParam::isMulti(), + .quantBits = TypeParam::get_quant_bits()}; + VecSimParams svs_params = CreateParams(params); + auto mock_thread_pool = tieredIndexMock(); + auto *tiered_index = this->CreateTieredSVSIndex(svs_params, mock_thread_pool, 1, 1); + ASSERT_INDEX(tiered_index); + + mock_thread_pool.init_threads(); + GenerateAndAddVector(tiered_index, dim, 0, 0); + mock_thread_pool.thread_pool_join(); + ASSERT_TRUE(tiered_index->GetSVSIndex()->ready()); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); + + GenerateAndAddVector(tiered_index, dim, 7, 7); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); + + auto *job = static_cast(mock_thread_pool.jobQ.front().job); + job->status.store(SVSInsertJob::Status::Delayed, std::memory_order_relaxed); + mock_thread_pool.thread_iteration(); + + ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 1); + ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 1); + ASSERT_FALSE(tiered_index->GetSVSIndex()->isLabelExists(7)); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); + ASSERT_EQ(mock_thread_pool.jobQ.front().job, job); + + job->status.store(SVSInsertJob::Status::Pending, std::memory_order_relaxed); + mock_thread_pool.thread_iteration(); + + ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); + ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(7)); + ASSERT_TRUE(tiered_index->labelToInsertJobs.empty()); + ASSERT_EQ(tiered_index->indexLabelCount(), 2); +} + +// isLabelExists() reports a soft-deleted label as free, but SVS keeps its translator entry +// until the label's consolidate job runs, and would refuse the remap. So relabelVector takes +// those jobs over and consolidates them itself. +TYPED_TEST(SVSTieredIndexTest, relabelVectorTakesOverPendingConsolidatesOfTheTarget) { + size_t dim = 4; + const size_t n = 3; + SVSParams params = {.type = TypeParam::get_index_type(), + .dim = dim, + .metric = VecSimMetric_L2, + .multi = TypeParam::isMulti(), + .quantBits = TypeParam::get_quant_bits()}; + VecSimParams svs_params = CreateParams(params); + auto mock_thread_pool = tieredIndexMock(); + auto *tiered_index = this->CreateTieredSVSIndex(svs_params, mock_thread_pool, 1, 1); + ASSERT_INDEX(tiered_index); + + mock_thread_pool.init_threads(); + for (size_t i = 0; i < n; i++) { + GenerateAndAddVector(tiered_index, dim, i, i); + } + mock_thread_pool.thread_pool_join(); + ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); + for (size_t i = 0; i < n; i++) { + ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(i)); + } + + ASSERT_EQ(VecSimIndex_DeleteVector(tiered_index, 1), 1); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); + ASSERT_EQ(mock_thread_pool.jobQ.front().job->jobType, SVS_CONSOLIDATE_JOB); + auto *first_job = static_cast(mock_thread_pool.jobQ.front().job); + + ASSERT_EQ(VecSimIndex_RelabelVector(tiered_index, 2, 1), VecSimRelabel_OK); + ASSERT_FALSE(first_job->isValid); + ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(1)); + ASSERT_FALSE(tiered_index->GetSVSIndex()->isLabelExists(2)); + + // The first job is still registered under label 1, so this one is appended to it. + ASSERT_EQ(VecSimIndex_DeleteVector(tiered_index, 1), 1); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 2); + + // The takeover now has to skip the job it already invalidated. + ASSERT_EQ(VecSimIndex_RelabelVector(tiered_index, 0, 1), VecSimRelabel_OK); + + mock_thread_pool.init_threads(); + mock_thread_pool.thread_pool_join(); + ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(1)); + ASSERT_FALSE(tiered_index->GetSVSIndex()->isLabelExists(0)); + ASSERT_EQ(tiered_index->indexLabelCount(), 1); +} + +// A consolidate job that is already inside consolidate() cannot be taken over, so the takeover +// waits for it to leave. The test holds the flag and does the consolidation the wrapper would, +// so the remap that follows really is unblocked. +TYPED_TEST(SVSTieredIndexTest, relabelVectorWaitsForAnExecutingConsolidate) { + size_t dim = 4; + SVSParams params = {.type = TypeParam::get_index_type(), + .dim = dim, + .metric = VecSimMetric_L2, + .multi = TypeParam::isMulti(), + .quantBits = TypeParam::get_quant_bits()}; + VecSimParams svs_params = CreateParams(params); + auto mock_thread_pool = tieredIndexMock(); + auto *tiered_index = this->CreateTieredSVSIndex(svs_params, mock_thread_pool, 1, 1); + ASSERT_INDEX(tiered_index); + + mock_thread_pool.init_threads(); + for (size_t i = 0; i < 2; i++) { + GenerateAndAddVector(tiered_index, dim, i, i); + } + mock_thread_pool.thread_pool_join(); + ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(1)); + + ASSERT_EQ(VecSimIndex_DeleteVector(tiered_index, 1), 1); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); + auto *consolidate_job = static_cast(mock_thread_pool.jobQ.front().job); + consolidate_job->executing.store(true, std::memory_order_release); + + std::atomic relabel_code{-1}; + std::thread relabeler( + [&]() { relabel_code = static_cast(VecSimIndex_RelabelVector(tiered_index, 0, 1)); }); + + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + // EXPECT, so the thread is always joined below. + EXPECT_EQ(relabel_code.load(), -1) << "the takeover did not wait for the running job"; + + tiered_index->GetSVSIndex()->consolidate({1}); + consolidate_job->executing.store(false, std::memory_order_release); + relabeler.join(); + + ASSERT_EQ(relabel_code.load(), static_cast(VecSimRelabel_OK)); + ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(1)); + ASSERT_FALSE(tiered_index->GetSVSIndex()->isLabelExists(0)); + + mock_thread_pool.init_threads(); + mock_thread_pool.thread_pool_join(); + ASSERT_EQ(tiered_index->indexLabelCount(), 1); +} + +// Rollback needs the backend to refuse the target, and to hold old_label live while the buffer +// still holds it with a registered job. Both are built by hand: +// - the target keeps a stale translator entry, because a backend-side delete leaves no +// consolidate job for the takeover to run. Single-value only - the multi backend checks live +// labels only, so a soft-deleted target never blocks it. +// - old_label is live in both tiers, the state an insert job is in after it published and +// before it dropped its buffer copy. Published here directly, then the job marked Done. +TYPED_TEST(SVSTieredIndexTestBasic, relabelVectorRollsBackWhenTheBackendRefusesTheTarget) { + size_t dim = 4; + SVSParams params = { + .type = TypeParam::get_index_type(), .dim = dim, .metric = VecSimMetric_L2, .multi = false}; + VecSimParams svs_params = CreateParams(params); + auto mock_thread_pool = tieredIndexMock(); + auto *tiered_index = this->CreateTieredSVSIndex(svs_params, mock_thread_pool, 1, 1); + ASSERT_INDEX(tiered_index); + + mock_thread_pool.init_threads(); + GenerateAndAddVector(tiered_index, dim, 0, 0); + mock_thread_pool.thread_pool_join(); + ASSERT_TRUE(tiered_index->GetSVSIndex()->ready()); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); + + GenerateAndAddVector(tiered_index->GetBackendIndex(), dim, 5, 5); + ASSERT_EQ(VecSimIndex_DeleteVector(tiered_index->GetBackendIndex(), 5), 1); + ASSERT_FALSE(tiered_index->GetSVSIndex()->isLabelExists(5)); + ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 1); + + GenerateAndAddVector(tiered_index, dim, 7, 7); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); + auto *job = static_cast(mock_thread_pool.jobQ.front().job); + + GenerateAndAddVector(tiered_index->GetBackendIndex(), dim, 7, 7); + job->status.store(SVSInsertJob::Status::Done, std::memory_order_relaxed); + ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(7)); + ASSERT_TRUE(tiered_index->GetFlatIndex()->isLabelExists(7)); + + ASSERT_EQ(VecSimIndex_RelabelVector(tiered_index, 7, 5), VecSimRelabel_NewLabelTaken); + + ASSERT_TRUE(tiered_index->GetFlatIndex()->isLabelExists(7)); + ASSERT_FALSE(tiered_index->GetFlatIndex()->isLabelExists(5)); + ASSERT_EQ(tiered_index->labelToInsertJobs.count(5), 0); + ASSERT_EQ(tiered_index->labelToInsertJobs.at(7).size(), 1); + ASSERT_EQ(job->label, 7); + ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(7)); + ASSERT_FALSE(tiered_index->GetSVSIndex()->isLabelExists(5)); + + mock_thread_pool.init_threads(); + mock_thread_pool.thread_pool_join(); + ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); + ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(7)); + ASSERT_FALSE(tiered_index->GetSVSIndex()->isLabelExists(5)); + ASSERT_EQ(tiered_index->indexLabelCount(), 2); +} + TYPED_TEST(SVSTieredIndexTest, relabelVectorRejectsOnATier) { size_t dim = 4; SVSParams params = {.type = TypeParam::get_index_type(), @@ -4727,6 +4971,140 @@ TYPED_TEST(SVSTieredIndexTestBasic, testDeletedJournalMulti) { ASSERT_NEAR(backend_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); } +// scheduleSVSIndexConsolidate() always registers the label, so a job with an unregistered one +// can only be built by hand. The callback is taken from a real job because the wrapper is +// private. +TYPED_TEST(SVSTieredIndexTestBasic, forgetConsolidateJobIgnoresAnUnregisteredLabel) { + const size_t dim = 4; + SVSParams params = { + .type = TypeParam::get_index_type(), .dim = dim, .metric = VecSimMetric_L2, .multi = false}; + VecSimParams svs_params = CreateParams(params); + auto mock_thread_pool = tieredIndexMock(); + auto *tiered_index = this->CreateTieredSVSIndex(svs_params, mock_thread_pool, 1, 1); + ASSERT_INDEX(tiered_index); + auto allocator = tiered_index->getAllocator(); + + mock_thread_pool.init_threads(); + for (size_t i = 0; i < 2; i++) { + GenerateAndAddVector(tiered_index, dim, i, i); + } + mock_thread_pool.thread_pool_join(); + + ASSERT_EQ(VecSimIndex_DeleteVector(tiered_index, 1), 1); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); + auto *registered_job = mock_thread_pool.jobQ.front().job; + ASSERT_EQ(registered_job->jobType, SVS_CONSOLIDATE_JOB); + + auto *orphan_job = + new (allocator) SVSConsolidateJob(allocator, {999}, registered_job->Execute, tiered_index); + tiered_index->submitSingleJob(orphan_job); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 2); + + mock_thread_pool.init_threads(); + mock_thread_pool.thread_pool_join(); + + ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(0)); + ASSERT_FALSE(tiered_index->GetSVSIndex()->isLabelExists(1)); + ASSERT_EQ(tiered_index->indexLabelCount(), 1); +} + +// Every add made while the backend is not ready becomes training data, so the init normally +// deletes the whole buffer and never swaps. Adding straight to the frontend leaves a vector the +// batch does not own, which the init has to swap down and keep. +TYPED_TEST(SVSTieredIndexTestBasic, initSVSIndexKeepsANonTrainingFlatVector) { + const size_t dim = 4; + const size_t n = 5; + const labelType kept_label = 99; + SVSParams params = { + .type = TypeParam::get_index_type(), .dim = dim, .metric = VecSimMetric_L2, .multi = false}; + VecSimParams svs_params = CreateParams(params); + auto mock_thread_pool = tieredIndexMock(); + auto *tiered_index = this->CreateTieredSVSIndex(svs_params, mock_thread_pool, 100, 100); + ASSERT_INDEX(tiered_index); + + for (size_t i = 0; i < n; i++) { + GenerateAndAddVector(tiered_index, dim, i, i); + } + ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); + ASSERT_FALSE(tiered_index->GetSVSIndex()->ready()); + + GenerateAndAddVector(tiered_index->GetFlatIndex(), dim, kept_label, kept_label); + ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), n + 1); + + tiered_index->scheduleSVSIndexInit(); + mock_thread_pool.init_threads(); + mock_thread_pool.thread_pool_join(); + + ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 1); + ASSERT_EQ(tiered_index->GetFlatIndex()->getVectorLabel(0), kept_label); + TEST_DATA_T expected_vector[dim]; + GenerateVector(expected_vector, dim, kept_label); + ASSERT_EQ(tiered_index->GetFlatIndex()->getDistanceFrom_Unsafe(kept_label, expected_vector), 0); + for (size_t i = 0; i < n; i++) { + ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(i)); + } + ASSERT_EQ(tiered_index->indexLabelCount(), n + 1); +} + +// ids_to_init_ is only non-empty while the backend is not ready, and insert jobs only exist once +// it is, so the swap bookkeeping of the two together is out of reach of addVector. The hand-made +// job below builds that state. Do not trigger an init after it: initSVSIndex asserts on a ready +// backend. +TYPED_TEST(SVSTieredIndexTestBasic, insertJobFollowsTrainingIdsThroughAFlatSwap) { + const size_t dim = 4; + const size_t n = 5; + const labelType ingested_label = 2; + SVSParams params = { + .type = TypeParam::get_index_type(), .dim = dim, .metric = VecSimMetric_L2, .multi = false}; + VecSimParams svs_params = CreateParams(params); + auto mock_thread_pool = tieredIndexMock(); + auto *tiered_index = this->CreateTieredSVSIndex(svs_params, mock_thread_pool, 100, 100); + ASSERT_INDEX(tiered_index); + + for (size_t i = 0; i < n; i++) { + GenerateAndAddVector(tiered_index, dim, i, i); + } + ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); + ASSERT_FALSE(tiered_index->GetSVSIndex()->ready()); + + // Label i sits at flat id i, so the last id gets swapped onto the ingested one. + tiered_index->submitSingleJob(tiered_index->createInsertJob(ingested_label, ingested_label)); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); + mock_thread_pool.thread_iteration(); + + ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), n - 1); + ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(ingested_label)); + ASSERT_TRUE(tiered_index->labelToInsertJobs.empty()); + + TEST_DATA_T expected_vector[dim]; + for (size_t i : {size_t{0}, size_t{1}, size_t{3}, size_t{4}}) { + GenerateVector(expected_vector, dim, i); + ASSERT_EQ(tiered_index->GetFlatIndex()->getDistanceFrom_Unsafe(i, expected_vector), 0) + << "label " << i; + } + ASSERT_EQ(tiered_index->indexLabelCount(), n); +} + +// deleteVector() drops its shared lock before taking the exclusive one, so the label can be gone +// by the time removeLabelFromFlat() runs. +TYPED_TEST(SVSTieredIndexTestBasic, removeLabelFromFlatIgnoresAnAbsentLabel) { + const size_t dim = 4; + SVSParams params = { + .type = TypeParam::get_index_type(), .dim = dim, .metric = VecSimMetric_L2, .multi = false}; + VecSimParams svs_params = CreateParams(params); + auto mock_thread_pool = tieredIndexMock(); + auto *tiered_index = this->CreateTieredSVSIndex(svs_params, mock_thread_pool, 100, 100); + ASSERT_INDEX(tiered_index); + + for (size_t i = 0; i < 2; i++) { + GenerateAndAddVector(tiered_index, dim, i, i); + } + + ASSERT_EQ(tiered_index->removeLabelFromFlat(999), 0); + ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 2); + ASSERT_EQ(tiered_index->indexLabelCount(), 2); +} + TEST(SVSTieredIndexTest, testThreadPool) { // Test VecSimSVSThreadPool with shared pool const size_t num_threads = 4; From 261b0f1e5d941bb8a94df3f23920488958ea3565 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Fri, 2 Oct 2026 08:06:33 -0700 Subject: [PATCH 78/99] SVS rc6 --- cmake/svs.cmake | 6 +++--- deps/ScalableVectorSearch | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index 2491f2e1e..9a0c080d1 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -64,7 +64,7 @@ if(USE_SVS) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.31" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "21.0") if(GLIBCXX_VERSION VERSION_GREATER_EQUAL "11") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc5-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc6-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") else() message(STATUS "libstdc++ >= GCC 11 is required for Clang SVS binaries - disabling SVS_SHARED_LIB") set(SVS_SHARED_LIB OFF) @@ -76,9 +76,9 @@ if(USE_SVS) else() if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.28") if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "14.0") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc5-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc6-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") else() - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc5-reduced.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc6-reduced.tar.gz" CACHE STRING "SVS URL") endif() else() message(STATUS "GLIBC >= 2.28 is required for GCC SVS binaries - disabling SVS_SHARED_LIB") diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch index 107071c99..aa35a5b2c 160000 --- a/deps/ScalableVectorSearch +++ b/deps/ScalableVectorSearch @@ -1 +1 @@ -Subproject commit 107071c99c5f3f111bc3f4d15412529832928368 +Subproject commit aa35a5b2ccc750fb68cb95ab6712e0567146432e From 732e5e5c7c823a11cdfe9f4270a82b514f6d99fd Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 5 Oct 2026 02:07:01 -0700 Subject: [PATCH 79/99] format --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index 5e3a6f1fc..8e580f420 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -952,8 +952,7 @@ TieredHNSWIndex::TieredHNSWIndex(HNSWIndex allocator) : VecSimTieredIndex(hnsw_index, bf_index, tiered_index_params, allocator), idToRepairJobs(this->allocator), idToSwapJob(this->allocator), readySwapJobs(0), - deferredReuseInsertJobs(this->allocator), - isQuantized(hnsw_index->usesQuantizedStorage()) { + deferredReuseInsertJobs(this->allocator), isQuantized(hnsw_index->usesQuantizedStorage()) { const size_t normalization_set_size = tiered_index_params.specificParams.tieredHnswParams.QuantNormalizationSetSize; if (isQuantized && normalization_set_size > 0) { From f14687a6d34e621af08c5352be76d1d26827ea42 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 5 Oct 2026 04:49:17 -0700 Subject: [PATCH 80/99] refactor switch-case in relabelVector --- src/VecSim/algorithms/svs/svs.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 5e2a9a233..0278fc9e9 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -657,9 +657,10 @@ class SVSIndex : public VecSimIndexAbstract, fl case svs::concurrent::ReplaceExternalIdResult::NewIdExists: return VecSimRelabel_NewLabelTaken; case svs::concurrent::ReplaceExternalIdResult::OldIdMissing: - break; + return VecSimRelabel_OldLabelMissing; + default: + throw std::logic_error("SVSIndex::relabelVector: unreachable SVS result"); } - return VecSimRelabel_OldLabelMissing; } #endif // HAVE_SVS_REPLACE_EXTERNAL_ID From 7578eee2105662da7e17df5be1992400629df33a Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 5 Oct 2026 05:15:59 -0700 Subject: [PATCH 81/99] remove unused index_ --- src/VecSim/algorithms/svs/svs_batch_iterator.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_batch_iterator.h b/src/VecSim/algorithms/svs/svs_batch_iterator.h index f40b38ece..34c41288b 100644 --- a/src/VecSim/algorithms/svs/svs_batch_iterator.h +++ b/src/VecSim/algorithms/svs/svs_batch_iterator.h @@ -29,7 +29,6 @@ class SVS_BatchIterator : public VecSimBatchIterator { using dist_type = typename Index::distance_type; size_t dim; - const Index *index_; impl_type impl_; typename impl_type::const_iterator curr_it; size_t batch_size; @@ -72,11 +71,11 @@ class SVS_BatchIterator : public VecSimBatchIterator { std::shared_ptr allocator, bool is_two_level_lvq) : VecSimBatchIterator{query_vector, queryParams ? queryParams->timeoutCtx : nullptr, std::move(allocator)}, - dim{index->dimensions()}, index_{index}, - impl_{index_->make_batch_iterator( + dim{index->dimensions()}, + impl_{index->make_batch_iterator( std::span{static_cast(query_vector), dim})}, curr_it{impl_.begin()} { - auto sp = svs_details::joinSearchParams(index_->get_search_parameters(), queryParams, + auto sp = svs_details::joinSearchParams(index->get_search_parameters(), queryParams, is_two_level_lvq); batch_size = queryParams && queryParams->batchSize ? queryParams->batchSize From a34064d89b9ad87fb62885f8abdf18c8bc05a87e Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 5 Oct 2026 08:28:52 -0700 Subject: [PATCH 82/99] reduce flatIndexGuard scope --- .../algorithms/svs/svs_batch_iterator.h | 5 +- src/VecSim/algorithms/svs/svs_tiered.h | 61 ++++++++----------- tests/unit/test_svs_tiered.cpp | 6 +- 3 files changed, 32 insertions(+), 40 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_batch_iterator.h b/src/VecSim/algorithms/svs/svs_batch_iterator.h index 34c41288b..116546150 100644 --- a/src/VecSim/algorithms/svs/svs_batch_iterator.h +++ b/src/VecSim/algorithms/svs/svs_batch_iterator.h @@ -71,9 +71,8 @@ class SVS_BatchIterator : public VecSimBatchIterator { std::shared_ptr allocator, bool is_two_level_lvq) : VecSimBatchIterator{query_vector, queryParams ? queryParams->timeoutCtx : nullptr, std::move(allocator)}, - dim{index->dimensions()}, - impl_{index->make_batch_iterator( - std::span{static_cast(query_vector), dim})}, + dim{index->dimensions()}, impl_{index->make_batch_iterator(std::span{ + static_cast(query_vector), dim})}, curr_it{impl_.begin()} { auto sp = svs_details::joinSearchParams(index->get_search_parameters(), queryParams, is_two_level_lvq); diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 36dee1fba..5cad5ef10 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -35,7 +35,10 @@ struct SVSInsertJob : public TieredInsertJob { */ struct SVSConsolidateJob : public AsyncJob { std::vector labels; - std::atomic executing{false}; + // Pending -> Executing -> Done, or Pending -> TakenOver when relabelVector() consolidates these + // labels itself. + enum class Status : uint8_t { Pending, TakenOver, Executing, Done }; + std::atomic status{Status::Pending}; SVSConsolidateJob(std::shared_ptr allocator, const std::vector &labels_, JobCallback insertCb, @@ -595,18 +598,14 @@ class TieredSVSIndex : public VecSimTieredIndex { auto svs_index = index->GetSVSIndex(); svs_index->setParallelism(1); - bool valid = false; - { - std::shared_lock flat_lock(index->flatIndexGuard); - valid = consolidate_job->isValid; - if (valid) { - consolidate_job->executing.store(true, std::memory_order_release); - } - } - if (valid) { + auto expected = SVSConsolidateJob::Status::Pending; + if (consolidate_job->status.compare_exchange_strong( + expected, SVSConsolidateJob::Status::Executing, std::memory_order_acq_rel)) { svs_index->consolidate(consolidate_job->labels); - // Cleared before the registry lock is needed again, so a waiter cannot deadlock us. - consolidate_job->executing.store(false, std::memory_order_release); + // Published before forgetConsolidateJob() needs the registry lock, so a waiter holding + // it shared is released before this job can be deleted. + consolidate_job->status.store(SVSConsolidateJob::Status::Done, + std::memory_order_release); } index->forgetConsolidateJob(consolidate_job); delete job; @@ -674,30 +673,22 @@ class TieredSVSIndex : public VecSimTieredIndex { } } } - // Caller must hold flatIndexGuard exclusive std::vector takeOverConsolidateOf(labelType label) { std::vector taken_over; - vecsim_stl::vector running(this->allocator); - { - std::shared_lock lock(this->consolidateJobsGuard); - auto it = this->labelToConsolidateJobs.find(label); - if (it == this->labelToConsolidateJobs.end()) { - return taken_over; - } - for (auto *job : it->second) { - if (!job->isValid) { - continue; // already taken over by an earlier call - } - if (job->executing.load(std::memory_order_acquire)) { - running.push_back(job); - } else { - job->isValid = false; - taken_over.insert(taken_over.end(), job->labels.begin(), job->labels.end()); - } - } + std::shared_lock lock(this->consolidateJobsGuard); + auto it = this->labelToConsolidateJobs.find(label); + if (it == this->labelToConsolidateJobs.end()) { + return taken_over; } - for (auto *job : running) { - waitForJob(job); + for (auto *job : it->second) { + auto expected = SVSConsolidateJob::Status::Pending; + if (job->status.compare_exchange_strong(expected, SVSConsolidateJob::Status::TakenOver, + std::memory_order_acq_rel)) { + taken_over.insert(taken_over.end(), job->labels.begin(), job->labels.end()); + } else if (expected == SVSConsolidateJob::Status::Executing) { + waitForJob(job); // its own consolidate covers these labels + } + // already TakenOver or Done } return taken_over; } @@ -710,8 +701,10 @@ class TieredSVSIndex : public VecSimTieredIndex { } } + // Safe to call while holding consolidateJobsGuard shared static void waitForJob(SVSConsolidateJob *job) { - while (job->executing.load(std::memory_order_acquire)) { + while (job->status.load(std::memory_order_acquire) == + SVSConsolidateJob::Status::Executing) { std::this_thread::yield(); } } diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index fea265a31..357da6462 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -815,7 +815,7 @@ TYPED_TEST(SVSTieredIndexTest, relabelVectorTakesOverPendingConsolidatesOfTheTar auto *first_job = static_cast(mock_thread_pool.jobQ.front().job); ASSERT_EQ(VecSimIndex_RelabelVector(tiered_index, 2, 1), VecSimRelabel_OK); - ASSERT_FALSE(first_job->isValid); + ASSERT_EQ(first_job->status.load(), SVSConsolidateJob::Status::TakenOver); ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(1)); ASSERT_FALSE(tiered_index->GetSVSIndex()->isLabelExists(2)); @@ -858,7 +858,7 @@ TYPED_TEST(SVSTieredIndexTest, relabelVectorWaitsForAnExecutingConsolidate) { ASSERT_EQ(VecSimIndex_DeleteVector(tiered_index, 1), 1); ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); auto *consolidate_job = static_cast(mock_thread_pool.jobQ.front().job); - consolidate_job->executing.store(true, std::memory_order_release); + consolidate_job->status.store(SVSConsolidateJob::Status::Executing, std::memory_order_release); std::atomic relabel_code{-1}; std::thread relabeler( @@ -869,7 +869,7 @@ TYPED_TEST(SVSTieredIndexTest, relabelVectorWaitsForAnExecutingConsolidate) { EXPECT_EQ(relabel_code.load(), -1) << "the takeover did not wait for the running job"; tiered_index->GetSVSIndex()->consolidate({1}); - consolidate_job->executing.store(false, std::memory_order_release); + consolidate_job->status.store(SVSConsolidateJob::Status::Done, std::memory_order_release); relabeler.join(); ASSERT_EQ(relabel_code.load(), static_cast(VecSimRelabel_OK)); From 5b9be335a1cca2bc765780a9332f11b3f09a56fc Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Tue, 6 Oct 2026 00:03:04 -0700 Subject: [PATCH 83/99] adopt set once for svs pimpl --- src/VecSim/algorithms/svs/svs.h | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 0278fc9e9..474a9d901 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -262,7 +262,13 @@ class SVSIndex : public VecSimIndexAbstract, fl return std::make_unique(initImpl(points, ids)); } - void storeImpl(impl_type *impl) { this->impl_.store(impl, std::memory_order_release); } + // if impl_ == nullptr, store impl, return true + // if impl_ != nullptr, do nothing, return false + bool CASImpl(impl_type *impl) { + impl_type *expected = nullptr; + return this->impl_.compare_exchange_strong(expected, impl, + std::memory_order_acq_rel); + } void setImpl(std::unique_ptr handler) override { SVSImplHandler *svs_handler = dynamic_cast(handler.get()); @@ -270,11 +276,10 @@ class SVSIndex : public VecSimIndexAbstract, fl throw std::logic_error("Failed to cast to SVSImplHandler"); } - if (getImpl()) { + if (!CASImpl(svs_handler->impl.get())) { throw std::logic_error("SVSIndex::setImpl called on non-empty impl_"); } - - storeImpl(svs_handler->impl.release()); + svs_handler->impl.release(); } // Assuming parallelism was updated to reflect the number of available threads before this @@ -326,7 +331,16 @@ class SVSIndex : public VecSimIndexAbstract, fl impl->add_points(points, ids, /*reuse_empty*/ false); } else { // SVS index instance cannot be empty, so we have to construct it at first rows - storeImpl(initImpl(points, ids).release()); + auto built = initImpl(points, ids); + if (CASImpl(built.get())) { + built.release(); + } else { + // Another writer published first. + if constexpr (!isMulti) { + deleted_num = deleteVectorsImpl(labels, n); + } + getImpl()->add_points(points, ids, /*reuse_empty*/ false); + } } return n - deleted_num; From 673174b5fd7cfaa4c6cc3ef247f5d0a78eeaf5f6 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 7 Oct 2026 04:53:11 -0700 Subject: [PATCH 84/99] fixes and additional tests --- src/VecSim/algorithms/svs/svs.h | 19 +- src/VecSim/algorithms/svs/svs_tiered.h | 51 ++- tests/unit/test_svs_tiered.cpp | 588 ++++++++++++++++++++++++- 3 files changed, 606 insertions(+), 52 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 474a9d901..54507e759 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -266,8 +266,7 @@ class SVSIndex : public VecSimIndexAbstract, fl // if impl_ != nullptr, do nothing, return false bool CASImpl(impl_type *impl) { impl_type *expected = nullptr; - return this->impl_.compare_exchange_strong(expected, impl, - std::memory_order_acq_rel); + return this->impl_.compare_exchange_strong(expected, impl, std::memory_order_acq_rel); } void setImpl(std::unique_ptr handler) override { @@ -745,6 +744,9 @@ class SVSIndex : public VecSimIndexAbstract, fl rep->results.reserve(n_neighbors); for (size_t i = 0; i < n_neighbors; i++) { + if (result.index(0, i) == svs::concurrent::no_external_id) { + continue; + } rep->results.push_back( VecSimQueryResult{result.index(0, i), toVecSimDistance(result.distance(0, i))}); } @@ -913,18 +915,9 @@ class SVSIndex : public VecSimIndexAbstract, fl }; if constexpr (isMulti) { - auto it = impl->get_label_to_external_lookup().find(label); - if (it == impl->get_label_to_external_lookup().end()) { - return; - } - for (auto external_id : it->second) { - append_datum(impl->get_parent_index().get_datum(external_id)); - } + impl->on_data(label, append_datum); } else { - if (!impl->has_id(label)) { - return; - } - append_datum(impl->get_datum(label)); + impl->on_datum(label, append_datum); } } diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 5cad5ef10..4fb5a3013 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -21,8 +21,9 @@ struct SVSInsertJob : public TieredInsertJob { // Pending -> Executing -> Done. Delayed means a relabel is remapping the label in the backend // right now, and publishing would put the vector under a label that is not settled yet. + // Parked means a worker consumed that submission; relabel must submit it again when settled. // Written under flatIndexGuard, except Executing/Done which waitForJob() reads unlocked. - enum class Status : uint8_t { Pending, Delayed, Executing, Done }; + enum class Status : uint8_t { Pending, Delayed, Parked, Executing, Done }; std::atomic status{Status::Pending}; SVSInsertJob(std::shared_ptr allocator, labelType label_, idType id_, @@ -373,8 +374,6 @@ class TieredSVSIndex : public VecSimTieredIndex { } flat_results.swap(cur_flat_results->results); VecSimQueryReply_Free(cur_flat_results); - // We also take the lock on the main index on the first call to getNextResults, and - // we hold it until the iterator is depleted or freed. acquire_svs_iterator(); auto cur_svs_results = svs_iterator->getNextResults(n_res, BY_SCORE_THEN_ID); svs_code = cur_svs_results->code; @@ -542,18 +541,16 @@ class TieredSVSIndex : public VecSimTieredIndex { outcome = job_index->executeInsertJob(insert_job); } - if (outcome == InsertJobOutcome::Deferred) { - job_index->submitSingleJob(job); - return; + if (outcome == InsertJobOutcome::Completed) { + delete job; } - delete job; } /** * @brief Run SVS index GC in a thread-safe manner. * * This static wrapper function performs the following actions: - * - Acquires a lock on the index's mainIndexGuard to ensure thread safety during the GC + * - Acquires a lock on the index's updateJobMutex to ensure thread safety during the GC * - Configures the number of threads for the underlying SVS index update operation. * - Calls the SVSIndex::runGC() method to perform the actual index update. * - Clears the indexGCScheduled flag to allow future scheduling. @@ -714,7 +711,7 @@ class TieredSVSIndex : public VecSimTieredIndex { return Base::setAndSaveInvalidJob(job); } - enum class InsertJobOutcome { Completed, Deferred }; + enum class InsertJobOutcome { Completed, Parked }; InsertJobOutcome executeInsertJob(SVSInsertJob *job) { auto svs_index = GetSVSIndex(); @@ -733,9 +730,10 @@ class TieredSVSIndex : public VecSimTieredIndex { } if (job->status.load(std::memory_order_relaxed) == SVSInsertJob::Status::Delayed) { + job->status.store(SVSInsertJob::Status::Parked, std::memory_order_relaxed); this->flatIndexGuard.unlock_shared(); - // relabelVector() clears it once the backend remap settled the label - return InsertJobOutcome::Deferred; + // Relabel owns resubmission. It may already have resumed the job after this unlock. + return InsertJobOutcome::Parked; } // Copy the vector blob out of the flat buffer while holding flatIndexGuard, so we @@ -904,7 +902,7 @@ class TieredSVSIndex : public VecSimTieredIndex { } this->flatIndexGuard.lock_shared(); - if (svs_index->ready() && (this->frontendIndex->indexSize() >= flat_buffer_bound)) { + if (this->frontendIndex->indexSize() >= flat_buffer_bound) { this->flatIndexGuard.unlock_shared(); auto storage_blob = this->frontendIndex->preprocessForStorage(blob); @@ -934,14 +932,12 @@ class TieredSVSIndex : public VecSimTieredIndex { // (in case of override in non-MULTI index) - so if it's there, we remove it // we submit the insert job. if (!this->backendIndex->isMultiValue()) { - if (svs_index->ready()) { - // If we removed the previous vector from both svs and flat in the overwrite - // process, we still return 0 (not -1). - auto deleted = svs_index->deleteVector(label); - if (deleted > 0) - scheduleSVSIndexConsolidate(label); - ret = std::max(ret - deleted, 0); - } + // If we removed the previous vector from both svs and flat in the overwrite + // process, we still return 0 (not -1). + auto deleted = svs_index->deleteVector(label); + if (deleted > 0) + scheduleSVSIndexConsolidate(label); + ret = std::max(ret - deleted, 0); } // Insert job to the queue and signal the workers' updater. @@ -1160,6 +1156,7 @@ class TieredSVSIndex : public VecSimTieredIndex { bool flat_holds_old = false; bool delayed_any = false; std::vector taken_over; + vecsim_stl::vector parked_jobs(this->allocator); { std::lock_guard flat_lock{this->flatIndexGuard}; @@ -1206,6 +1203,7 @@ class TieredSVSIndex : public VecSimTieredIndex { } } + executeTracingCallback("Relabel::before_backend_remap"); // Do the taken-over job's, // the remap below isn't refused by a leftover translator entry if (!taken_over.empty()) { @@ -1237,8 +1235,12 @@ class TieredSVSIndex : public VecSimTieredIndex { if (it != this->labelToInsertJobs.end()) { for (auto *job : it->second) { auto *insert_job = static_cast(job); - if (insert_job->status.load(std::memory_order_relaxed) == - SVSInsertJob::Status::Delayed) { + auto status = insert_job->status.load(std::memory_order_relaxed); + if (status == SVSInsertJob::Status::Parked) { + parked_jobs.push_back(job); + } + if (status == SVSInsertJob::Status::Delayed || + status == SVSInsertJob::Status::Parked) { insert_job->status.store(SVSInsertJob::Status::Pending, std::memory_order_relaxed); } @@ -1246,6 +1248,11 @@ class TieredSVSIndex : public VecSimTieredIndex { } } } + // The pool's submit callback may wait for workers blocked on either of our locks. + lock.unlock(); + if (!parked_jobs.empty()) { + this->submitJobs(parked_jobs); + } return rollback ? VecSimRelabel_NewLabelTaken : VecSimRelabel_OK; } diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 357da6462..c8dceed03 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -11,8 +11,16 @@ #include "mock_thread_pool.h" #if HAVE_SVS +#include +#include +#include +#include +#include +#include +#include #include #include +#include // For getAvailableCPUs(): #include @@ -739,10 +747,7 @@ TYPED_TEST(SVSTieredIndexTest, relabelVectorRemapsAPendingInsertJob) { ASSERT_EQ(tiered_index->indexLabelCount(), 2); } -// The delay window: relabelVector marks a pending job Delayed, releases flatIndexGuard, and -// clears Delayed only after the backend remap settled the label. A worker that runs the job -// inside that window must requeue it instead of publishing. The test sets Delayed itself -// rather than racing a relabel. +// The worker consumes the delayed submission; relabel must resume it exactly once. TYPED_TEST(SVSTieredIndexTest, relabelVectorDefersAnInsertJobInsideTheDelayWindow) { size_t dim = 4; SVSParams params = {.type = TypeParam::get_index_type(), @@ -765,8 +770,14 @@ TYPED_TEST(SVSTieredIndexTest, relabelVectorDefersAnInsertJobInsideTheDelayWindo ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); auto *job = static_cast(mock_thread_pool.jobQ.front().job); - job->status.store(SVSInsertJob::Status::Delayed, std::memory_order_relaxed); - mock_thread_pool.thread_iteration(); + tiered_index->registerTracingCallback("Relabel::before_backend_remap", [&]() { + std::thread worker([&]() { mock_thread_pool.thread_iteration(); }); + worker.join(); + EXPECT_TRUE(mock_thread_pool.jobQ.empty()); + EXPECT_EQ(job->status.load(), SVSInsertJob::Status::Parked); + EXPECT_FALSE(tiered_index->GetSVSIndex()->isLabelExists(70)); + }); + ASSERT_EQ(VecSimIndex_RelabelVector(tiered_index, 7, 70), VecSimRelabel_OK); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 1); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 1); @@ -774,11 +785,12 @@ TYPED_TEST(SVSTieredIndexTest, relabelVectorDefersAnInsertJobInsideTheDelayWindo ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); ASSERT_EQ(mock_thread_pool.jobQ.front().job, job); - job->status.store(SVSInsertJob::Status::Pending, std::memory_order_relaxed); + ASSERT_EQ(job->status.load(), SVSInsertJob::Status::Pending); + ASSERT_EQ(job->label, 70); mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); - ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(7)); + ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(70)); ASSERT_TRUE(tiered_index->labelToInsertJobs.empty()); ASSERT_EQ(tiered_index->indexLabelCount(), 2); } @@ -917,15 +929,30 @@ TYPED_TEST(SVSTieredIndexTestBasic, relabelVectorRollsBackWhenTheBackendRefusesT ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(7)); ASSERT_TRUE(tiered_index->GetFlatIndex()->isLabelExists(7)); - ASSERT_EQ(VecSimIndex_RelabelVector(tiered_index, 7, 5), VecSimRelabel_NewLabelTaken); - - ASSERT_TRUE(tiered_index->GetFlatIndex()->isLabelExists(7)); - ASSERT_FALSE(tiered_index->GetFlatIndex()->isLabelExists(5)); - ASSERT_EQ(tiered_index->labelToInsertJobs.count(5), 0); - ASSERT_EQ(tiered_index->labelToInsertJobs.at(7).size(), 1); - ASSERT_EQ(job->label, 7); - ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(7)); - ASSERT_FALSE(tiered_index->GetSVSIndex()->isLabelExists(5)); + // Keep the original completed-job case, then exercise rollback with a parked submission. + for (auto status : {SVSInsertJob::Status::Done, SVSInsertJob::Status::Pending}) { + job->status.store(status, std::memory_order_relaxed); + tiered_index->registerTracingCallback("Relabel::before_backend_remap", [&, status]() { + if (status == SVSInsertJob::Status::Pending) { + std::thread worker([&]() { mock_thread_pool.thread_iteration(); }); + worker.join(); + EXPECT_TRUE(mock_thread_pool.jobQ.empty()); + EXPECT_EQ(job->status.load(), SVSInsertJob::Status::Parked); + } + }); + ASSERT_EQ(VecSimIndex_RelabelVector(tiered_index, 7, 5), VecSimRelabel_NewLabelTaken); + + ASSERT_TRUE(tiered_index->GetFlatIndex()->isLabelExists(7)); + ASSERT_FALSE(tiered_index->GetFlatIndex()->isLabelExists(5)); + ASSERT_EQ(tiered_index->labelToInsertJobs.count(5), 0); + ASSERT_EQ(tiered_index->labelToInsertJobs.at(7).size(), 1); + ASSERT_EQ(job->label, 7); + ASSERT_EQ(job->status.load(), status); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); + ASSERT_EQ(mock_thread_pool.jobQ.front().job, job); + ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(7)); + ASSERT_FALSE(tiered_index->GetSVSIndex()->isLabelExists(5)); + } mock_thread_pool.init_threads(); mock_thread_pool.thread_pool_join(); @@ -4254,6 +4281,533 @@ TYPED_TEST(SVSTieredIndexTestBasic, preferAdHocOptimization) { ASSERT_NO_THROW(tiered_index->preferAdHocSearch(10, 5, false)); } +TYPED_TEST(SVSTieredIndexTest, runGCBetweenIteratorBatches) { + if (this->isFallbackToSQ()) { + GTEST_SKIP() << "Scalar fallback does not preserve the ordering checked by this test."; + } + constexpr size_t dim = 4; + constexpr size_t n_labels = 6; + for (auto mode : {VecSim_WriteAsync, VecSim_WriteInPlace}) { + SVSParams params = {.type = TypeParam::get_index_type(), + .dim = dim, + .metric = VecSimMetric_L2, + .multi = TypeParam::isMulti()}; + VecSimParams svs_params = CreateParams(params); + auto mock_thread_pool = tieredIndexMock(1); + auto *tiered_index = this->CreateTieredSVSIndex(svs_params, mock_thread_pool); + ASSERT_INDEX(tiered_index); + auto *backend = tiered_index->GetBackendIndex(); + for (size_t i = 0; i < n_labels; ++i) { + GenerateAndAddVector(backend, dim, i + 1000, i + 1); + } + + TEST_DATA_T query[dim] = {}; + VecSimQueryParams query_params{}; + query_params.batchSize = 3; + auto *iterator = VecSimBatchIterator_New(tiered_index, query, &query_params); + auto verify_first = [](size_t id, double score, size_t i) { + EXPECT_EQ(id, i + 1000); + if constexpr (TypeParam::get_quant_bits() == VecSimSvsQuant_NONE) { + EXPECT_DOUBLE_EQ(score, dim * double(i + 1) * double(i + 1)); + } + }; + runBatchIteratorSearchTest(iterator, 3, verify_first); + for (size_t i = 0; i < 3; ++i) { + backend->deleteVector(i + 1000); + } + + VecSim_SetWriteMode(mode); + std::promise gc_finished; + auto completion = gc_finished.get_future(); + std::thread gc([&]() { + VecSimTieredIndex_GC(tiered_index); + if (mode == VecSim_WriteAsync) { + mock_thread_pool.thread_iteration(); + } + gc_finished.set_value(); + }); + const bool completed = + completion.wait_for(std::chrono::seconds(5)) == std::future_status::ready; + // If the old lifetime lock is still present, release it on its owning thread + // so GC can finish and the regression fails without hanging the suite. + if (!completed) { + VecSimBatchIterator_Free(iterator); + iterator = nullptr; + } + gc.join(); + VecSim_SetWriteMode(VecSim_WriteAsync); + ASSERT_TRUE(completed) << "GC waited for an idle iterator to be freed"; + EXPECT_EQ(backend->indexSize(), 3); + EXPECT_TRUE(VecSimBatchIterator_HasNext(iterator)); + auto verify_remaining = [](size_t id, double score, size_t i) { + EXPECT_EQ(id, i + 1003); + if constexpr (TypeParam::get_quant_bits() == VecSimSvsQuant_NONE) { + EXPECT_DOUBLE_EQ(score, dim * double(i + 4) * double(i + 4)); + } + }; + runBatchIteratorSearchTest(iterator, 3, verify_remaining); + EXPECT_FALSE(VecSimBatchIterator_HasNext(iterator)); + VecSimBatchIterator_Free(iterator); + } +} + +namespace { + +constexpr size_t svs_gc_test_dim = 4; +constexpr size_t svs_gc_label_base = 1'000'000; +constexpr size_t svs_gc_label_stride = 17; +using SVSReferenceVectors = std::vector>>; +using SVSTestReply = std::unique_ptr; + +size_t svsGCLabel(size_t i) { return svs_gc_label_base + svs_gc_label_stride * i; } + +SVSReferenceVectors makeSVSReferenceVectors(size_t n, bool multi, bool distinct = false) { + SVSReferenceVectors reference(n); + for (size_t i = 0; i < n; ++i) { + for (size_t copy = 0; copy < (multi ? 2U : 1U); ++copy) { + std::vector vector(svs_gc_test_dim); + for (size_t d = 0; d < vector.size(); ++d) { + vector[d] = float(i + 1) + float(d) / 4 + (distinct ? float(copy) / 8 : 0); + } + reference[i].push_back(std::move(vector)); + } + } + return reference; +} + +void addSVSReferenceVectors(VecSimIndex *index, const SVSReferenceVectors &reference, + size_t begin, size_t end) { + for (size_t i = begin; i < end; ++i) { + for (const auto &vector : reference[i]) { + VecSimIndex_AddVector(index, vector.data(), svsGCLabel(i)); + } + } +} + +double svsReferenceDistance(const std::vector &vector, const std::vector &query) { + double distance = 0; + for (size_t d = 0; d < vector.size(); ++d) { + const double delta = double(vector[d]) - double(query[d]); + distance += delta * delta; + } + return distance; +} + +// All references are immutable. Concurrently deleted labels need only remain valid external +// labels; scores are checked for permanent survivors, whose vectors never change. +std::string checkSVSReply(const VecSimQueryReply &reply, const SVSReferenceVectors &reference, + const std::vector &query, size_t survivors_begin, + size_t &checked_survivors) { + if (reply.code != VecSim_QueryReply_OK) { + return "Query did not complete successfully"; + } + std::unordered_set labels; + for (const auto &result : reply.results) { + if (result.id < svs_gc_label_base || + (result.id - svs_gc_label_base) % svs_gc_label_stride != 0 || + (result.id - svs_gc_label_base) / svs_gc_label_stride >= reference.size()) { + return "Unknown external label: " + std::to_string(result.id); + } + if (!std::isfinite(result.score) || !labels.insert(result.id).second) { + return "Non-finite score or duplicate label: " + std::to_string(result.id); + } + const size_t i = (result.id - svs_gc_label_base) / svs_gc_label_stride; + if (i >= survivors_begin) { + const double expected = svsReferenceDistance(reference[i][0], query); + if (std::abs(result.score - expected) > 1e-5 * std::max(1.0, expected)) { + return "Wrong score for label " + std::to_string(result.id) + ": got " + + std::to_string(result.score) + ", expected " + std::to_string(expected); + } + ++checked_survivors; + } + } + return {}; +} + +std::string checkSVSCopies(TieredSVSIndex *index, const SVSReferenceVectors &reference, + size_t begin, size_t end) { + for (size_t i = begin; i < end; ++i) { + std::vector> copied; + index->getDataByLabel(svsGCLabel(i), copied); + std::sort(copied.begin(), copied.end()); + // Reference copies are generated in sorted order. Equality also checks multiplicity. + if (copied != reference[i]) { + return "Wrong vectors copied for label " + std::to_string(svsGCLabel(i)); + } + } + return {}; +} + +class SVSTestWriteMode { + VecSimWriteMode previous = TieredSVSIndex::getWriteMode(); + +public: + explicit SVSTestWriteMode(VecSimWriteMode mode) { VecSim_SetWriteMode(mode); } + ~SVSTestWriteMode() { VecSim_SetWriteMode(previous); } +}; + +// Reader checks run outside the synchronization mutex. Failures are reported after joining, +// and each mutation round waits for every reader to make progress without using sleeps. +class SVSTestReaders { + std::vector> checks; + std::vector threads; + std::vector completed; + std::mutex mutex; + std::condition_variable changed; + std::atomic stopped{false}; + bool started = false; + std::string failure; + +public: + explicit SVSTestReaders(std::vector> checks) + : checks(std::move(checks)), completed(this->checks.size(), 0) {} + + void start() { + for (size_t i = 0; i < checks.size(); ++i) { + threads.emplace_back([this, i] { + { + std::unique_lock lock(mutex); + changed.wait(lock, [&] { return started || stopped.load(); }); + } + while (!stopped.load()) { + auto error = checks[i](); + { + std::lock_guard lock(mutex); + ++completed[i]; + if (!error.empty()) { + if (failure.empty()) { + failure = "Reader " + std::to_string(i) + ": " + error; + } + stopped.store(true); + } + } + changed.notify_all(); + } + }); + } + { + std::lock_guard lock(mutex); + started = true; + } + changed.notify_all(); + } + + bool waitForProgress() { + std::unique_lock lock(mutex); + const auto before = completed; + const bool progressed = changed.wait_for(lock, std::chrono::seconds(5), [&] { + if (!failure.empty()) { + return true; + } + for (size_t i = 0; i < completed.size(); ++i) { + if (completed[i] == before[i]) { + return false; + } + } + return true; + }); + if (!progressed && failure.empty()) { + failure = "Timed out waiting for reader progress"; + stopped.store(true); + } + return progressed && failure.empty(); + } + + void finish() { + { + std::lock_guard lock(mutex); + stopped.store(true); + } + changed.notify_all(); + for (auto &thread : threads) { + if (thread.joinable()) { + thread.join(); + } + } + } + + const std::string &error() const { return failure; } // Read only after finish(). + ~SVSTestReaders() { finish(); } +}; + +} // namespace + +TYPED_TEST(SVSTieredIndexTestBasic, runGCBetweenIteratorBatchesWithStorageShrink) { + constexpr size_t n_labels = 96; + constexpr size_t batch_size = 16; + const auto reference = makeSVSReferenceVectors(n_labels, false); + const std::vector query(svs_gc_test_dim, 0); + for (auto mode : {VecSim_WriteAsync, VecSim_WriteInPlace}) { + for (bool delete_returned : {true, false}) { + SCOPED_TRACE(::testing::Message() << "mode=" << mode + << ", delete_returned=" << delete_returned); + SVSParams params = {.type = TypeParam::get_index_type(), + .dim = svs_gc_test_dim, + .metric = VecSimMetric_L2, + .blockSize = 8, + .search_window_size = n_labels, + .search_buffer_capacity = n_labels}; + auto svs_params = CreateParams(params); + tieredIndexMock mock_thread_pool(1); + auto *index = this->CreateTieredSVSIndex(svs_params, mock_thread_pool); + ASSERT_INDEX(index); + // Index creation resizes the thread pool, which switches write mode to async. + SVSTestWriteMode write_mode(mode); + auto *backend = index->GetBackendIndex(); + addSVSReferenceVectors(backend, reference, 0, n_labels); + ASSERT_EQ(index->GetFlatIndex()->indexSize(), 0); + + VecSimQueryParams query_params{}; + // Consume the entire native batch, leaving no prefetched external labels. + query_params.batchSize = batch_size; + std::unique_ptr iterator( + VecSimBatchIterator_New(index, query.data(), &query_params), + VecSimBatchIterator_Free); + std::unordered_set seen, deleted; + auto read_batch = [&] { + SVSTestReply reply(VecSimBatchIterator_Next(iterator.get(), batch_size, BY_SCORE), + VecSimQueryReply_Free); + size_t checked = 0; + EXPECT_EQ(checkSVSReply(*reply, reference, query, 0, checked), ""); + for (const auto &result : reply->results) { + EXPECT_EQ(deleted.count(result.id), 0) << result.id; + EXPECT_TRUE(seen.insert(result.id).second) << result.id; + } + return reply->results.size(); + }; + ASSERT_EQ(read_batch(), batch_size); + for (size_t round = 0; round < 2; ++round) { + const size_t begin = (delete_returned ? round : 2 * round + 1) * batch_size; + for (size_t i = begin; i < begin + batch_size; ++i) { + ASSERT_EQ(seen.count(svsGCLabel(i)), delete_returned ? 1 : 0); + // As in runGCBetweenIteratorBatches, defer consolidation to GC so the + // timeout below can unwind an old iterator holding a lifetime lock. + ASSERT_EQ(backend->deleteVector(svsGCLabel(i)), 1); + deleted.insert(svsGCLabel(i)); + } + const auto capacity_before = backend->indexCapacity(); + std::promise gc_finished; + auto completion = gc_finished.get_future(); + std::thread gc([&] { + VecSimTieredIndex_GC(index); + if (mode == VecSim_WriteAsync) { + mock_thread_pool.thread_iteration(); + } + gc_finished.set_value(); + }); + const bool completed = + completion.wait_for(std::chrono::seconds(5)) == std::future_status::ready; + if (!completed) { + iterator.reset(); + } + gc.join(); + ASSERT_TRUE(completed) << "GC waited for an idle iterator to be freed"; + EXPECT_EQ(index->GetSVSIndex()->indexStorageSize(), + n_labels - (round + 1) * batch_size); + EXPECT_LT(backend->indexCapacity(), capacity_before); + ASSERT_EQ(read_batch(), batch_size); + } + + auto drain_iterator = [&] { + // Bound the loop so failure to reach exhaustion cannot hang the test. + for (size_t batch = 0; + batch <= n_labels / batch_size && VecSimBatchIterator_HasNext(iterator.get()); + ++batch) { + const auto count = read_batch(); + EXPECT_TRUE(count > 0 || !VecSimBatchIterator_HasNext(iterator.get())); + } + EXPECT_FALSE(VecSimBatchIterator_HasNext(iterator.get())); + }; + drain_iterator(); + for (size_t i = 0; i < n_labels; ++i) { + EXPECT_EQ(seen.count(svsGCLabel(i)), + delete_returned || !deleted.count(svsGCLabel(i)) ? 1 : 0); + } + + VecSimBatchIterator_Reset(iterator.get()); + seen.clear(); + drain_iterator(); + EXPECT_EQ(seen.size(), n_labels - deleted.size()); + for (size_t i = 0; i < n_labels; ++i) { + EXPECT_EQ(seen.count(svsGCLabel(i)), deleted.count(svsGCLabel(i)) ? 0 : 1); + } + } + } +} + +TYPED_TEST(SVSTieredIndexTestBasic, parallelQueriesDuringGC) { + constexpr size_t n_labels = 256; + constexpr size_t rounds = 3; + constexpr size_t delete_per_round = 64; + constexpr size_t survivors_begin = rounds * delete_per_round; + SVSTestWriteMode write_mode(VecSim_WriteAsync); + for (bool multi : {false, true}) { + SCOPED_TRACE(::testing::Message() << "multi=" << multi); + const auto reference = makeSVSReferenceVectors(n_labels, multi); + const auto &query = reference.back()[0]; + SVSParams params = {.type = TypeParam::get_index_type(), + .dim = svs_gc_test_dim, + .metric = VecSimMetric_L2, + .multi = multi, + .blockSize = 16, + .search_window_size = n_labels}; + auto svs_params = CreateParams(params); + tieredIndexMock mock_thread_pool(1); + auto *index = this->CreateTieredSVSIndex(svs_params, mock_thread_pool); + ASSERT_INDEX(index); + addSVSReferenceVectors(index->GetBackendIndex(), reference, 0, n_labels); + ASSERT_EQ(index->GetFlatIndex()->indexSize(), 0); + + std::array checked_survivors{}; + auto query_once = [&](bool range) -> std::string { + SVSTestReply reply( + range ? VecSimIndex_RangeQuery(index, query.data(), 4096, nullptr, BY_SCORE) + : VecSimIndex_TopKQuery(index, query.data(), 10, nullptr, BY_SCORE), + VecSimQueryReply_Free); + if (reply->results.empty()) { + return "Empty reply for a query near permanent survivors"; + } + return checkSVSReply(*reply, reference, query, survivors_begin, + checked_survivors[range ? 1 : 0]); + }; + SVSTestReaders readers({[&] { return query_once(false); }, + [&] { return query_once(true); }}); + bool gc_reader_progress = false; + index->registerTracingCallback("GCJob::before_run_gc", [&] { + gc_reader_progress = readers.waitForProgress(); + }); + readers.start(); + bool progress = readers.waitForProgress(); + for (size_t round = 0; round < rounds && progress; ++round) { + const auto storage_before = index->GetSVSIndex()->indexStorageSize(); + const auto capacity_before = index->GetBackendIndex()->indexCapacity(); + for (size_t i = round * delete_per_round; i < (round + 1) * delete_per_round; ++i) { + EXPECT_EQ(index->deleteVector(svsGCLabel(i)), multi ? 2 : 1); + } + gc_reader_progress = false; + VecSimTieredIndex_GC(index); + // Also drain the consolidation jobs queued by deleteVector. + mock_thread_pool.init_threads(); + mock_thread_pool.thread_pool_join(); + progress = gc_reader_progress; + EXPECT_LT(index->GetSVSIndex()->indexStorageSize(), storage_before); + EXPECT_LT(index->GetBackendIndex()->indexCapacity(), capacity_before); + } + readers.finish(); + ASSERT_TRUE(progress) << readers.error(); + ASSERT_EQ(readers.error(), ""); + EXPECT_GT(checked_survivors[0], 0); + EXPECT_GT(checked_survivors[1], 0); + EXPECT_EQ(index->GetBackendIndex()->indexLabelCount(), n_labels - survivors_begin); + EXPECT_EQ(index->GetSVSIndex()->getNumMarkedDeleted(), 0); + EXPECT_TRUE(mock_thread_pool.jobQ.empty()); + } +} + +TYPED_TEST(SVSTieredIndexTestBasic, getDataByLabelDuringGC) { + constexpr size_t n_labels = 256; + constexpr size_t rounds = 3; + constexpr size_t delete_per_round = 64; + constexpr size_t monitored_begin = n_labels - 8; + SVSTestWriteMode write_mode(VecSim_WriteAsync); + for (bool multi : {false, true}) { + SCOPED_TRACE(::testing::Message() << "multi=" << multi); + const auto reference = makeSVSReferenceVectors(n_labels, multi, true); + SVSParams params = {.type = TypeParam::get_index_type(), + .dim = svs_gc_test_dim, + .metric = VecSimMetric_L2, + .multi = multi, + .blockSize = 16}; + auto svs_params = CreateParams(params); + tieredIndexMock mock_thread_pool(1); + auto *index = this->CreateTieredSVSIndex(svs_params, mock_thread_pool); + ASSERT_INDEX(index); + addSVSReferenceVectors(index->GetBackendIndex(), reference, 0, n_labels); + ASSERT_EQ(index->GetFlatIndex()->indexSize(), 0); + + SVSTestReaders readers( + {[&] { return checkSVSCopies(index, reference, monitored_begin, n_labels); }}); + bool gc_reader_progress = false; + index->registerTracingCallback("GCJob::before_run_gc", [&] { + gc_reader_progress = readers.waitForProgress(); + }); + readers.start(); + bool progress = readers.waitForProgress(); + for (size_t round = 0; round < rounds && progress; ++round) { + const auto capacity_before = index->GetBackendIndex()->indexCapacity(); + for (size_t i = round * delete_per_round; i < (round + 1) * delete_per_round; ++i) { + EXPECT_EQ(index->deleteVector(svsGCLabel(i)), multi ? 2 : 1); + } + gc_reader_progress = false; + VecSimTieredIndex_GC(index); + mock_thread_pool.init_threads(); + mock_thread_pool.thread_pool_join(); + progress = gc_reader_progress; + EXPECT_EQ(index->GetSVSIndex()->indexStorageSize(), + (n_labels - (round + 1) * delete_per_round) * (multi ? 2 : 1)); + EXPECT_LT(index->GetBackendIndex()->indexCapacity(), capacity_before); + } + readers.finish(); + ASSERT_TRUE(progress) << readers.error(); + ASSERT_EQ(readers.error(), ""); + EXPECT_EQ(checkSVSCopies(index, reference, monitored_begin, n_labels), ""); + EXPECT_EQ(index->GetSVSIndex()->getNumMarkedDeleted(), 0); + EXPECT_TRUE(mock_thread_pool.jobQ.empty()); + } +} + +TYPED_TEST(SVSTieredIndexTestBasic, getDataByLabelDuringIngestion) { + constexpr size_t initial_labels = 64; + constexpr size_t rounds = 4; + constexpr size_t labels_per_round = 64; + constexpr size_t n_labels = initial_labels + rounds * labels_per_round; + SVSTestWriteMode write_mode(VecSim_WriteAsync); + for (bool multi : {false, true}) { + SCOPED_TRACE(::testing::Message() << "multi=" << multi); + const auto reference = makeSVSReferenceVectors(n_labels, multi, true); + SVSParams params = {.type = TypeParam::get_index_type(), + .dim = svs_gc_test_dim, + .metric = VecSimMetric_L2, + .multi = multi, + .blockSize = 16}; + auto svs_params = CreateParams(params); + tieredIndexMock mock_thread_pool(1); + auto tiered_params = this->CreateTieredSVSParams(svs_params, mock_thread_pool); + // Keep each batch on the worker ingestion path instead of bypassing a full buffer. + tiered_params.flatBufferLimit = 2 * labels_per_round; + auto *index = this->CreateTieredSVSIndex(tiered_params, mock_thread_pool); + ASSERT_INDEX(index); + addSVSReferenceVectors(index->GetBackendIndex(), reference, 0, initial_labels); + ASSERT_TRUE(index->GetSVSIndex()->ready()); + + SVSTestReaders readers( + {[&] { return checkSVSCopies(index, reference, initial_labels - 8, initial_labels); }}); + readers.start(); + bool progress = readers.waitForProgress(); + for (size_t round = 0; round < rounds && progress; ++round) { + const size_t begin = initial_labels + round * labels_per_round; + const auto capacity_before = index->GetBackendIndex()->indexCapacity(); + addSVSReferenceVectors(index, reference, begin, begin + labels_per_round); + EXPECT_EQ(index->GetFlatIndex()->indexSize(), labels_per_round * (multi ? 2 : 1)); + EXPECT_EQ(index->GetBackendIndex()->indexLabelCount(), begin); + mock_thread_pool.init_threads(); + progress = readers.waitForProgress(); + mock_thread_pool.thread_pool_join(); + EXPECT_EQ(index->GetFlatIndex()->indexSize(), 0); + EXPECT_EQ(index->GetBackendIndex()->indexLabelCount(), begin + labels_per_round); + EXPECT_EQ(index->GetSVSIndex()->indexStorageSize(), + (begin + labels_per_round) * (multi ? 2 : 1)); + EXPECT_GT(index->GetBackendIndex()->indexCapacity(), capacity_before); + EXPECT_TRUE(mock_thread_pool.jobQ.empty()); + } + readers.finish(); + ASSERT_TRUE(progress) << readers.error(); + ASSERT_EQ(readers.error(), ""); + EXPECT_EQ(checkSVSCopies(index, reference, initial_labels - 8, initial_labels), ""); + EXPECT_EQ(index->GetBackendIndex()->indexLabelCount(), n_labels); + } +} + TYPED_TEST(SVSTieredIndexTestBasic, runGCAPI) { // Create TieredSVS index instance with a mock queue. size_t dim = 4; From a54595b404c225324e5d0e4b497ae4165774f261 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 7 Oct 2026 04:53:44 -0700 Subject: [PATCH 85/99] format --- tests/unit/test_svs_tiered.cpp | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index c8dceed03..c807bebc7 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -4375,8 +4375,8 @@ SVSReferenceVectors makeSVSReferenceVectors(size_t n, bool multi, bool distinct return reference; } -void addSVSReferenceVectors(VecSimIndex *index, const SVSReferenceVectors &reference, - size_t begin, size_t end) { +void addSVSReferenceVectors(VecSimIndex *index, const SVSReferenceVectors &reference, size_t begin, + size_t end) { for (size_t i = begin; i < end; ++i) { for (const auto &vector : reference[i]) { VecSimIndex_AddVector(index, vector.data(), svsGCLabel(i)); @@ -4539,8 +4539,8 @@ TYPED_TEST(SVSTieredIndexTestBasic, runGCBetweenIteratorBatchesWithStorageShrink const std::vector query(svs_gc_test_dim, 0); for (auto mode : {VecSim_WriteAsync, VecSim_WriteInPlace}) { for (bool delete_returned : {true, false}) { - SCOPED_TRACE(::testing::Message() << "mode=" << mode - << ", delete_returned=" << delete_returned); + SCOPED_TRACE(::testing::Message() + << "mode=" << mode << ", delete_returned=" << delete_returned); SVSParams params = {.type = TypeParam::get_index_type(), .dim = svs_gc_test_dim, .metric = VecSimMetric_L2, @@ -4670,12 +4670,11 @@ TYPED_TEST(SVSTieredIndexTestBasic, parallelQueriesDuringGC) { return checkSVSReply(*reply, reference, query, survivors_begin, checked_survivors[range ? 1 : 0]); }; - SVSTestReaders readers({[&] { return query_once(false); }, - [&] { return query_once(true); }}); + SVSTestReaders readers( + {[&] { return query_once(false); }, [&] { return query_once(true); }}); bool gc_reader_progress = false; - index->registerTracingCallback("GCJob::before_run_gc", [&] { - gc_reader_progress = readers.waitForProgress(); - }); + index->registerTracingCallback("GCJob::before_run_gc", + [&] { gc_reader_progress = readers.waitForProgress(); }); readers.start(); bool progress = readers.waitForProgress(); for (size_t round = 0; round < rounds && progress; ++round) { @@ -4728,9 +4727,8 @@ TYPED_TEST(SVSTieredIndexTestBasic, getDataByLabelDuringGC) { SVSTestReaders readers( {[&] { return checkSVSCopies(index, reference, monitored_begin, n_labels); }}); bool gc_reader_progress = false; - index->registerTracingCallback("GCJob::before_run_gc", [&] { - gc_reader_progress = readers.waitForProgress(); - }); + index->registerTracingCallback("GCJob::before_run_gc", + [&] { gc_reader_progress = readers.waitForProgress(); }); readers.start(); bool progress = readers.waitForProgress(); for (size_t round = 0; round < rounds && progress; ++round) { From a9616ac3e35f6f81e6c49f01ce4057983f048eae Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 7 Oct 2026 06:05:47 -0700 Subject: [PATCH 86/99] GC takes mainIndexGuard to protect baseclass methods --- src/VecSim/algorithms/svs/svs_tiered.h | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index e021630b7..73cf9b8e3 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -552,7 +552,8 @@ class TieredSVSIndex : public VecSimTieredIndex { * @brief Run SVS index GC in a thread-safe manner. * * This static wrapper function performs the following actions: - * - Acquires a lock on the index's updateJobMutex to ensure thread safety during the GC + * - Acquires updateJobMutex exclusively to wait for backend update jobs. + * - Acquires mainIndexGuard exclusively to exclude base-class readers during GC. * - Configures the number of threads for the underlying SVS index update operation. * - Calls the SVSIndex::runGC() method to perform the actual index update. * - Clears the indexGCScheduled flag to allow future scheduling. @@ -580,7 +581,10 @@ class TieredSVSIndex : public VecSimTieredIndex { return; } index->executeTracingCallback("GCJob::before_run_gc"); + // Take updateJobMutex first: workers may need flatIndexGuard while a base-class reader + // holds it and waits for mainIndexGuard. std::lock_guard lock(index->updateJobMutex); + std::lock_guard main_lock(index->mainIndexGuard); // Release the scheduled flag to allow scheduling again index->indexGCScheduled.clear(); @@ -613,8 +617,8 @@ class TieredSVSIndex : public VecSimTieredIndex { delete job; } - // Account for one backend GC run that started at `start`. Async GC timing includes - // the wait for updateJobMutex, which is part of what GC costs. + // Account for one backend GC run that started at `start`. The time includes waits for + // mainIndexGuard and, in async mode, updateJobMutex. void recordDefrag(std::chrono::steady_clock::time_point start) { const auto elapsedNs = std::chrono::duration_cast( std::chrono::steady_clock::now() - start) @@ -1367,6 +1371,7 @@ class TieredSVSIndex : public VecSimTieredIndex { "running synchronous GC for tiered SVS index in write-in-place mode"); // In write-in-place mode, we run GC synchronously. const auto start = std::chrono::steady_clock::now(); + std::lock_guard main_lock(this->mainIndexGuard); if (this->backendIndex->indexSize() == 0) { // No need to run GC on an empty index. return; From 38c48eb943c54d6393e947da69a1fd6387d8910f Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Wed, 7 Oct 2026 08:55:53 -0700 Subject: [PATCH 87/99] bump to SVS rc7 --- cmake/svs.cmake | 6 +++--- deps/ScalableVectorSearch | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index 9a0c080d1..e1335e16c 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -64,7 +64,7 @@ if(USE_SVS) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.31" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "21.0") if(GLIBCXX_VERSION VERSION_GREATER_EQUAL "11") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc6-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc7-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") else() message(STATUS "libstdc++ >= GCC 11 is required for Clang SVS binaries - disabling SVS_SHARED_LIB") set(SVS_SHARED_LIB OFF) @@ -76,9 +76,9 @@ if(USE_SVS) else() if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.28") if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "14.0") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc6-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc7-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") else() - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc6-reduced.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc7-reduced.tar.gz" CACHE STRING "SVS URL") endif() else() message(STATUS "GLIBC >= 2.28 is required for GCC SVS binaries - disabling SVS_SHARED_LIB") diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch index aa35a5b2c..42d8b6f25 160000 --- a/deps/ScalableVectorSearch +++ b/deps/ScalableVectorSearch @@ -1 +1 @@ -Subproject commit aa35a5b2ccc750fb68cb95ab6712e0567146432e +Subproject commit 42d8b6f25a4b677035bc0e5b2f410350f0317f13 From 315a89b187ab5e412667720409a946f794939f7e Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 12 Oct 2026 07:47:24 -0700 Subject: [PATCH 88/99] don't lock mainthread while training --- src/VecSim/algorithms/svs/svs_tiered.h | 75 ++++++++++++++++++-------- 1 file changed, 54 insertions(+), 21 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 73cf9b8e3..ee8fd2ec5 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -287,6 +287,9 @@ class TieredSVSIndex : public VecSimTieredIndex { size_t flat_buffer_bound; + bool initInProgress{false}; + std::vector initDeletedLabels; // Protected by flatIndexGuard. + /// //////////////////////////////////////////////////////////////////////////////////////////////////// // TieredSVS_BatchIterator // @@ -527,10 +530,14 @@ class TieredSVSIndex : public VecSimTieredIndex { assert(availableThreads > 0); auto index = static_cast *>(idx); assert(index); - // prevent parallel updates - std::lock_guard lock(index->updateJobMutex); - // Update the SVS index - index->initSVSIndex(availableThreads); + std::vector deleted_labels; + { + std::lock_guard lock(index->updateJobMutex); + deleted_labels = index->initSVSIndex(availableThreads); + } + for (labelType label : deleted_labels) { + index->scheduleSVSIndexConsolidate(label); + } } static void executeInsertJobWrapper(AsyncJob *job) { @@ -775,22 +782,27 @@ class TieredSVSIndex : public VecSimTieredIndex { return InsertJobOutcome::Completed; } - void initSVSIndex(size_t availableThreads) { + std::vector initSVSIndex(size_t availableThreads) { std::vector ids_to_move; std::vector labels_to_move; std::vector vectors_to_move; + std::vector deleted_labels; executeTracingCallback("UpdateJob::before_add_to_svs"); - { // lock frontendIndex from modifications - // The whole initialization is done under flatIndexGuard - std::lock_guard flat_lock{this->flatIndexGuard}; + { + std::shared_lock flat_lock{this->flatIndexGuard}; auto flat_index = this->GetFlatIndex(); const auto init_batch_size = ids_to_init_.size(); + if (init_batch_size == 0) { + indexUpdateScheduled.clear(); + return {}; + } const size_t dim = flat_index->getDim(); ids_to_move.reserve(init_batch_size); labels_to_move.reserve(init_batch_size); vectors_to_move.reserve(init_batch_size * dim); + initDeletedLabels.reserve(init_batch_size); for (idType id : ids_to_init_) { ids_to_move.push_back(id); @@ -798,8 +810,10 @@ class TieredSVSIndex : public VecSimTieredIndex { auto data = flat_index->getDataByInternalId(id); vectors_to_move.insert(vectors_to_move.end(), data, data + dim); } - ids_to_init_.clear(); - + // Keep tracking surviving snapshot IDs; new arrivals use insert jobs. + initInProgress = true; + } + { // Nothing to initialize from an empty batch, and setParallelism(0) is not a // valid request against the shared pool. if (!labels_to_move.empty()) { @@ -814,7 +828,16 @@ class TieredSVSIndex : public VecSimTieredIndex { svs_index->setParallelism(1); svs_index->setImpl(std::move(impl)); } - + } + { + std::lock_guard flat_lock{this->flatIndexGuard}; + deleted_labels.swap(initDeletedLabels); + if (!deleted_labels.empty()) { + GetSVSIndex()->deleteVectors(deleted_labels.data(), deleted_labels.size()); + } + // Deletions can swap surviving rows; cleanup needs their current IDs. + ids_to_move.assign(ids_to_init_.begin(), ids_to_init_.end()); + ids_to_init_.clear(); std::sort(ids_to_move.begin(), ids_to_move.end()); [[maybe_unused]] size_t total_deleted = 0; for (auto it = ids_to_move.rbegin(); it != ids_to_move.rend(); ++it) { @@ -833,15 +856,14 @@ class TieredSVSIndex : public VecSimTieredIndex { total_deleted += deleted; } - assert(total_deleted == labels_to_move.size() && + assert(total_deleted == ids_to_move.size() && "Deleted vectors count does not match the number of labels to delete"); - // Release the scheduled flag to allow scheduling again. - // Repeted sheduling is required if labels_to_move.empty() - // Under the guard, so a vector buffered after it schedules its own init. + initInProgress = false; indexUpdateScheduled.clear(); } // release frontend index executeTracingCallback("UpdateJob::after_add_to_svs"); + return deleted_labels; } public: @@ -941,19 +963,26 @@ class TieredSVSIndex : public VecSimTieredIndex { if (this->frontendIndex->isLabelExists(label) && !this->frontendIndex->isMultiValue()) { // We are going to update the internal id that currently holds the vector associated // with the given label. - new_flat_id = this->invalidatePendingInsertJob(label); - ret = 0; + if (initInProgress && !this->labelToInsertJobs.count(label)) { + // A captured row has no insert job; cancel it through the journal. + ret = -removeLabelFromFlat(label); + new_flat_id = this->frontendIndex->indexSize(); + } else { + new_flat_id = this->invalidatePendingInsertJob(label); + ret = 0; + } } // If this label already exists, this will do overwrite. ret += this->frontendIndex->addVector(blob, label); TieredInsertJob *new_insert_job = createInsertJob(label, new_flat_id); + const bool defer_delete = initInProgress; this->flatIndexGuard.unlock(); // Here, a worker might ingest the previous vector that was stored under "label" // (in case of override in non-MULTI index) - so if it's there, we remove it // we submit the insert job. - if (!this->backendIndex->isMultiValue()) { + if (!this->backendIndex->isMultiValue() && !defer_delete) { // If we removed the previous vector from both svs and flat in the overwrite // process, we still return 0 (not -1). auto deleted = svs_index->deleteVector(label); @@ -1026,11 +1055,15 @@ class TieredSVSIndex : public VecSimTieredIndex { "deleting_ids should contain unique ids"); // If id is deleted, don't use it for initialization + bool captured = false; if (!ids_to_init_.empty()) { for (idType id : deleting_ids) { - ids_to_init_.erase(id); + captured |= ids_to_init_.erase(id) > 0; } } + if (captured && initInProgress) { + initDeletedLabels.push_back(label); + } if (this->labelToInsertJobs.count(label) > 0) { // Invalidate the pending insert job(s) into SVS associated with this label @@ -1063,8 +1096,8 @@ class TieredSVSIndex : public VecSimTieredIndex { bool run_init = false; { std::lock_guard flat_lock{this->flatIndexGuard}; - // never buffer into a ready backend: ids_to_init_ would have no consumer - if (GetSVSIndex()->ready()) { + // Once the snapshot is taken, use the existing insertion path. + if (GetSVSIndex()->ready() || initInProgress) { return std::nullopt; } int deleted = 0; From 6f762692dcbe26e8ea87de987e6abf334d77ab32 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 15 Oct 2026 04:01:30 -0700 Subject: [PATCH 89/99] buffer added vectors during initSVSIndex --- src/VecSim/algorithms/svs/svs_tiered.h | 4 +- tests/unit/test_svs_tiered.cpp | 69 ++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index ee8fd2ec5..d9b7adfbb 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -813,6 +813,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // Keep tracking surviving snapshot IDs; new arrivals use insert jobs. initInProgress = true; } + executeTracingCallback("UpdateJob::after_snapshot"); { // Nothing to initialize from an empty batch, and setParallelism(0) is not a // valid request against the shared pool. @@ -946,7 +947,8 @@ class TieredSVSIndex : public VecSimTieredIndex { } this->flatIndexGuard.lock_shared(); - if (this->frontendIndex->indexSize() >= flat_buffer_bound) { + // Before the backend is ready, a direct insert would wait for the build. + if (svs_index->ready() && this->frontendIndex->indexSize() >= flat_buffer_bound) { this->flatIndexGuard.unlock_shared(); auto storage_blob = this->frontendIndex->preprocessForStorage(blob); diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index fb9761a5e..f45ce9d6b 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -1213,6 +1213,75 @@ TYPED_TEST(SVSTieredIndexTest, relabelVectorCannotLandInsideAnUpdateJobsWindow) #endif // HAVE_SVS_REPLACE_EXTERNAL_ID +TYPED_TEST(SVSTieredIndexTestBasic, addVectorDuringInitializationWithFullFlatBuffer) { + constexpr size_t dim = 4; + constexpr size_t n = 8; + for (bool multi : {false, true}) { + SCOPED_TRACE(::testing::Message() << "multi=" << multi); + SVSParams params = {.type = TypeParam::get_index_type(), + .dim = dim, + .metric = VecSimMetric_L2, + .multi = multi}; + auto svs_params = CreateParams(params); + tieredIndexMock mock_thread_pool(1); + auto tiered_params = this->CreateTieredSVSParams(svs_params, mock_thread_pool, n); + tiered_params.flatBufferLimit = n; + auto *index = this->CreateTieredSVSIndex(tiered_params, mock_thread_pool); + ASSERT_INDEX(index); + + for (size_t i = 0; i < n; ++i) { + GenerateAndAddVector(index, dim, i, i); + } + ASSERT_EQ(index->GetFlatIndex()->indexSize(), n); + ASSERT_FALSE(index->GetSVSIndex()->ready()); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); + + std::promise snapshot_taken; + auto snapshot_ready = snapshot_taken.get_future(); + std::promise additions_finished; + auto additions_ready = additions_finished.get_future(); + bool completed_during_init = false; + size_t buffered_during_init = 0; + // Pause with the snapshot fixed, flat unlocked, and updateJobMutex held exclusively. + index->registerTracingCallback("UpdateJob::after_snapshot", [&] { + snapshot_taken.set_value(); + completed_during_init = + additions_ready.wait_for(std::chrono::seconds(5)) == std::future_status::ready; + if (completed_during_init) { + buffered_during_init = index->GetFlatIndex()->indexSize(); + } + // On timeout, let init finish so a blocked add can return without hanging the test. + }); + mock_thread_pool.init_threads(); + const bool reached_snapshot = + snapshot_ready.wait_for(std::chrono::seconds(5)) == std::future_status::ready; + if (reached_snapshot) { + // Public writes stay on the test thread. The first add sees a full buffer; + // the second sees a buffer above the limit. Both must return before init resumes. + EXPECT_EQ(GenerateAndAddVector(index, dim, n, n), 1); + EXPECT_EQ(GenerateAndAddVector(index, dim, n + 1, n + 1), 1); + } + additions_finished.set_value(); + mock_thread_pool.thread_pool_join(); + + ASSERT_TRUE(reached_snapshot) << "Initialization did not reach the snapshot hook"; + ASSERT_TRUE(completed_during_init) << "addVector waited for initialization"; + EXPECT_EQ(buffered_during_init, n + 2); + EXPECT_TRUE(index->GetSVSIndex()->ready()); + EXPECT_EQ(index->GetFlatIndex()->indexSize(), 0); + EXPECT_EQ(index->GetBackendIndex()->indexSize(), n + 2); + EXPECT_EQ(index->GetBackendIndex()->indexLabelCount(), n + 2); + EXPECT_TRUE(mock_thread_pool.jobQ.empty()); + for (size_t i = 0; i < n + 2; ++i) { + std::vector> stored; + index->getDataByLabel(i, stored); + const std::vector> expected{ + std::vector(dim, static_cast(i))}; + EXPECT_EQ(stored, expected) << "label=" << i; + } + } +} + TYPED_TEST(SVSTieredIndexTest, TestDebugInfoThreadCount) { // Set thread_pool_size to 4 or actual number of available CPUs const auto num_threads = std::min(4U, getAvailableCPUs()); From 3b2f08e52c28aeddba375116f61b13e03bf044e8 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 15 Oct 2026 04:05:41 -0700 Subject: [PATCH 90/99] close delete-after-impl_publish window --- src/VecSim/algorithms/svs/svs.h | 11 +++ src/VecSim/algorithms/svs/svs_tiered.h | 17 +++- tests/unit/test_svs_tiered.cpp | 104 +++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 4 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 54507e759..d7b48e9e4 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -63,6 +63,7 @@ struct SVSIndexBase // Derived SVSIndex class should implement it struct ImplHandler { virtual ~ImplHandler() = default; + virtual void deleteVectors(const labelType *labels, size_t n) = 0; }; virtual std::unique_ptr createImpl(const void *vectors_data, const labelType *labels, size_t n) = 0; @@ -243,7 +244,14 @@ class SVSIndex : public VecSimIndexAbstract, fl // Handler to manage SVS implementation instance struct SVSImplHandler : public SVSIndexBase::ImplHandler { std::unique_ptr impl; + size_t num_deleted{0}; SVSImplHandler(std::unique_ptr impl) : impl{std::move(impl)} {} + + void deleteVectors(const labelType *labels, size_t n) override { + if (impl) { + num_deleted += impl->delete_entries(std::span{labels, n}); + } + } }; std::unique_ptr createImpl(const void *vectors_data, const labelType *labels, @@ -279,6 +287,9 @@ class SVSIndex : public VecSimIndexAbstract, fl throw std::logic_error("SVSIndex::setImpl called on non-empty impl_"); } svs_handler->impl.release(); + if (svs_handler->num_deleted > 0) { + this->markIndexUpdate(svs_handler->num_deleted); + } } // Assuming parallelism was updated to reflect the number of available threads before this diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index d9b7adfbb..3f4be8917 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -502,6 +502,9 @@ class TieredSVSIndex : public VecSimTieredIndex { protected: ScopedLocks lockMainIndexForQuery() const override { +#ifdef BUILD_TESTS + executeTracingCallback("Query::before_backend"); +#endif // No-op: SVS does its query locking internally. return ScopedLocks(); } @@ -787,6 +790,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::vector labels_to_move; std::vector vectors_to_move; std::vector deleted_labels; + std::unique_ptr impl; executeTracingCallback("UpdateJob::before_add_to_svs"); { @@ -824,17 +828,17 @@ class TieredSVSIndex : public VecSimTieredIndex { vectors_to_move.size() / this->frontendIndex->getDim()); assert(!svs_index->ready()); - auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), - labels_to_move.size()); + impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), + labels_to_move.size()); svs_index->setParallelism(1); - svs_index->setImpl(std::move(impl)); } } + executeTracingCallback("UpdateJob::before_publish"); { std::lock_guard flat_lock{this->flatIndexGuard}; deleted_labels.swap(initDeletedLabels); if (!deleted_labels.empty()) { - GetSVSIndex()->deleteVectors(deleted_labels.data(), deleted_labels.size()); + impl->deleteVectors(deleted_labels.data(), deleted_labels.size()); } // Deletions can swap surviving rows; cleanup needs their current IDs. ids_to_move.assign(ids_to_init_.begin(), ids_to_init_.end()); @@ -860,6 +864,11 @@ class TieredSVSIndex : public VecSimTieredIndex { assert(total_deleted == ids_to_move.size() && "Deleted vectors count does not match the number of labels to delete"); + // Publish only after applying writes made while the snapshot was built. + if (impl) { + GetSVSIndex()->setImpl(std::move(impl)); + executeTracingCallback("UpdateJob::after_publish"); + } initInProgress = false; indexUpdateScheduled.clear(); } // release frontend index diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index f45ce9d6b..4b1f0471d 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -1282,6 +1282,110 @@ TYPED_TEST(SVSTieredIndexTestBasic, addVectorDuringInitializationWithFullFlatBuf } } +// A search already occupies one worker, so init proceeds after its reservation timeout. +// The search reaches the backend while init is still paused immediately after publication. +TYPED_TEST(SVSTieredIndexTestBasic, deleteVectorBeforeInitializationPublishes) { + if (std::thread::hardware_concurrency() < 2) { + GTEST_SKIP() << "Requires two workers"; + } + constexpr size_t dim = 4; + constexpr size_t n = 8; + for (bool multi : {false, true}) { + SCOPED_TRACE(::testing::Message() << "multi=" << multi); + SVSParams params = {.type = TypeParam::get_index_type(), + .dim = dim, + .metric = VecSimMetric_L2, + .multi = multi}; + params.search_window_size = n; + auto svs_params = CreateParams(params); + tieredIndexMock mock_thread_pool(2); + auto *index = this->CreateTieredSVSIndex(svs_params, mock_thread_pool, + n + (multi ? 1 : 0)); + ASSERT_INDEX(index); + for (size_t i = 0; i < n - 1; ++i) { + GenerateAndAddVector(index, dim, i, i); + } + if (multi) { + GenerateAndAddVector(index, dim, 0, 20); + } + + std::promise worker_started, start_search, build_finished; + std::promise query_between_tiers, published, search_finished; + auto worker_ready = worker_started.get_future(); + auto start_ready = start_search.get_future(); + auto build_ready = build_finished.get_future(); + auto query_ready = query_between_tiers.get_future(); + auto publish_ready = published.get_future(); + auto search_ready = search_finished.get_future(); + const auto timeout = std::chrono::seconds(5); + bool query_reached_hook = false; + bool query_saw_publication = false; + bool search_completed_during_init = false; + index->registerTracingCallback("UpdateJob::before_publish", [&] { + EXPECT_FALSE(index->GetSVSIndex()->ready()); + build_finished.set_value(); + query_reached_hook = query_ready.wait_for(timeout) == std::future_status::ready; + }); + index->registerTracingCallback("Query::before_backend", [&] { + query_between_tiers.set_value(); + query_saw_publication = publish_ready.wait_for(timeout) == std::future_status::ready; + }); + index->registerTracingCallback("UpdateJob::after_publish", [&] { + published.set_value(); + search_completed_during_init = + search_ready.wait_for(timeout) == std::future_status::ready; + }); + + VecSimQueryReply *reply = nullptr; + auto search = [&](VecSimIndex *, size_t) { + worker_started.set_value(); + if (start_ready.wait_for(timeout) == std::future_status::ready) { + TEST_DATA_T query[dim] = {}; + reply = VecSimIndex_TopKQuery(index, query, n, nullptr, BY_ID); + } + search_finished.set_value(); + }; + SVSMultiThreadJob::JobsRegistry registry(index->getAllocator()); + auto jobs = SVSMultiThreadJob::createJobs(index->getAllocator(), HNSW_SEARCH_JOB, + search, index, 1, timeout, ®istry); + index->submitJobs(jobs); + mock_thread_pool.init_threads(); + const bool search_occupies_worker = + worker_ready.wait_for(timeout) == std::future_status::ready; + // Cross the training threshold only after the search worker has been occupied. + GenerateAndAddVector(index, dim, n - 1, n - 1); + const bool reached_build_hook = build_ready.wait_for(timeout) == std::future_status::ready; + if (reached_build_hook) { + EXPECT_EQ(VecSimIndex_DeleteVector(index, 0), multi ? 2 : 1); + EXPECT_EQ(VecSimIndex_DeleteVector(index, 0), 0); + } + start_search.set_value(); + // Init can still enqueue consolidation after the queue first becomes empty. + mock_thread_pool.thread_pool_wait(); + mock_thread_pool.thread_pool_join(); + + std::unique_ptr result( + reply, VecSimQueryReply_Free); + ASSERT_TRUE(search_occupies_worker); + ASSERT_TRUE(reached_build_hook); + ASSERT_TRUE(query_reached_hook); + ASSERT_TRUE(query_saw_publication); + ASSERT_TRUE(search_completed_during_init); + ASSERT_NE(reply, nullptr); + EXPECT_EQ(VecSimQueryReply_GetCode(reply), VecSim_QueryReply_OK); + ASSERT_EQ(VecSimQueryReply_Len(reply), n - 1); + for (size_t i = 0; i < n - 1; ++i) { + EXPECT_EQ(VecSimQueryResult_GetId(reply->results.data() + i), i + 1); + } + EXPECT_TRUE(index->GetSVSIndex()->ready()); + EXPECT_FALSE(index->GetSVSIndex()->isLabelExists(0)); + EXPECT_EQ(index->GetFlatIndex()->indexSize(), 0); + EXPECT_EQ(index->indexLabelCount(), n - 1); + EXPECT_EQ(index->getNumMarkedDeleted(), multi ? 2 : 1); + EXPECT_TRUE(mock_thread_pool.jobQ.empty()); + } +} + TYPED_TEST(SVSTieredIndexTest, TestDebugInfoThreadCount) { // Set thread_pool_size to 4 or actual number of available CPUs const auto num_threads = std::min(4U, getAvailableCPUs()); From 9521bcfb520fbec7515b30ba0cc42a1c7e179e1e Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 15 Oct 2026 04:06:55 -0700 Subject: [PATCH 91/99] update initDeletedLabel comment --- src/VecSim/algorithms/svs/svs_tiered.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 3f4be8917..93f9ecb79 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -287,8 +287,10 @@ class TieredSVSIndex : public VecSimTieredIndex { size_t flat_buffer_bound; + // flatIndexGuard: init (serialized by updateJobMutex) writes under a shared lock. + // Every other access to these two fields must hold the lock exclusively. bool initInProgress{false}; - std::vector initDeletedLabels; // Protected by flatIndexGuard. + std::vector initDeletedLabels; /// //////////////////////////////////////////////////////////////////////////////////////////////////// From 9092cb6179bbd0c4b37bc7adaa52ccdea46853a4 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 15 Oct 2026 04:07:33 -0700 Subject: [PATCH 92/99] update relabelVector comment --- src/VecSim/algorithms/svs/svs_tiered.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 93f9ecb79..4acc578e7 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -1212,6 +1212,9 @@ class TieredSVSIndex : public VecSimTieredIndex { * Reports `Unsupported` when the backend holds the label and cannot move it, which is the * case when built against an SVS without `replace_external_id`. All-or-nothing: on any code * other than `VecSimRelabel_OK` both tiers are untouched. + * + * Relabel is not atomic with concurrent queries. An overlapping query may return both the + * old and new labels, using two top-k positions for the moved label. */ VecSimRelabelCode relabelVector(labelType old_label, labelType new_label) override { if (old_label == new_label) { From 9ce1c6d19db1f9ff93fe3d15adb4f359243d09c1 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 15 Oct 2026 04:08:02 -0700 Subject: [PATCH 93/99] format --- tests/unit/test_svs_tiered.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 4b1f0471d..ada78e7e0 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -1299,8 +1299,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, deleteVectorBeforeInitializationPublishes) { params.search_window_size = n; auto svs_params = CreateParams(params); tieredIndexMock mock_thread_pool(2); - auto *index = this->CreateTieredSVSIndex(svs_params, mock_thread_pool, - n + (multi ? 1 : 0)); + auto *index = this->CreateTieredSVSIndex(svs_params, mock_thread_pool, n + (multi ? 1 : 0)); ASSERT_INDEX(index); for (size_t i = 0; i < n - 1; ++i) { GenerateAndAddVector(index, dim, i, i); @@ -1346,8 +1345,8 @@ TYPED_TEST(SVSTieredIndexTestBasic, deleteVectorBeforeInitializationPublishes) { search_finished.set_value(); }; SVSMultiThreadJob::JobsRegistry registry(index->getAllocator()); - auto jobs = SVSMultiThreadJob::createJobs(index->getAllocator(), HNSW_SEARCH_JOB, - search, index, 1, timeout, ®istry); + auto jobs = SVSMultiThreadJob::createJobs(index->getAllocator(), HNSW_SEARCH_JOB, search, + index, 1, timeout, ®istry); index->submitJobs(jobs); mock_thread_pool.init_threads(); const bool search_occupies_worker = From 46dfcd1db6a29997942e748208195917f192b34e Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 15 Oct 2026 04:33:35 -0700 Subject: [PATCH 94/99] add test --- src/VecSim/algorithms/svs/svs_tiered.h | 9 ++++ tests/unit/test_svs_tiered.cpp | 71 ++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 4acc578e7..4b830422b 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -620,6 +620,9 @@ class TieredSVSIndex : public VecSimTieredIndex { if (consolidate_job->status.compare_exchange_strong( expected, SVSConsolidateJob::Status::Executing, std::memory_order_acq_rel)) { svs_index->consolidate(consolidate_job->labels); +#ifdef BUILD_TESTS + index->executeTracingCallback("ConsolidateJob::before_done"); +#endif // Published before forgetConsolidateJob() needs the registry lock, so a waiter holding // it shared is released before this job can be deleted. consolidate_job->status.store(SVSConsolidateJob::Status::Done, @@ -627,6 +630,9 @@ class TieredSVSIndex : public VecSimTieredIndex { } index->forgetConsolidateJob(consolidate_job); delete job; +#ifdef BUILD_TESTS + index->executeTracingCallback("ConsolidateJob::after_delete"); +#endif } // Account for one backend GC run that started at `start`. The time includes waits for @@ -717,6 +723,9 @@ class TieredSVSIndex : public VecSimTieredIndex { std::memory_order_acq_rel)) { taken_over.insert(taken_over.end(), job->labels.begin(), job->labels.end()); } else if (expected == SVSConsolidateJob::Status::Executing) { +#ifdef BUILD_TESTS + executeTracingCallback("Relabel::before_wait_for_consolidate"); +#endif waitForJob(job); // its own consolidate covers these labels } // already TakenOver or Done diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index ada78e7e0..0c4fdb861 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -893,6 +893,77 @@ TYPED_TEST(SVSTieredIndexTest, relabelVectorWaitsForAnExecutingConsolidate) { ASSERT_EQ(tiered_index->indexLabelCount(), 1); } +// Use the real worker's Done -> unregister -> delete path while relabel holds its job pointer. +// Without the shared registry guard, deletion can precede waitForJob's next load (an ASan UAF). +TYPED_TEST(SVSTieredIndexTest, relabelVectorProtectsExecutingConsolidateJobLifetime) { + const size_t dim = 4; + SVSParams params = {.type = TypeParam::get_index_type(), + .dim = dim, + .metric = VecSimMetric_L2, + .multi = TypeParam::isMulti(), + .quantBits = TypeParam::get_quant_bits()}; + VecSimParams svs_params = CreateParams(params); + tieredIndexMock mock_thread_pool(1); + auto *tiered_index = this->CreateTieredSVSIndex(svs_params, mock_thread_pool, 2); + ASSERT_INDEX(tiered_index); + + GenerateAndAddVector(tiered_index, dim, 0, 0); + GenerateAndAddVector(tiered_index, dim, 1, 1); + mock_thread_pool.thread_iteration(); // initialize before starting the consolidate worker + ASSERT_TRUE(tiered_index->GetSVSIndex()->ready()); + ASSERT_EQ(VecSimIndex_DeleteVector(tiered_index, 1), 1); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); + ASSERT_EQ(mock_thread_pool.jobQ.front().job->jobType, SVS_CONSOLIDATE_JOB); + + std::promise worker_paused, relabel_waiting, job_deleted; + auto paused = worker_paused.get_future(); + auto waiting = relabel_waiting.get_future(); + auto deleted = job_deleted.get_future(); + const auto timeout = std::chrono::seconds(5); + bool worker_saw_waiter = false; + bool waiter_entered = false; + bool deleted_while_waiting = false; + tiered_index->registerTracingCallback("ConsolidateJob::before_done", [&] { + worker_paused.set_value(); + worker_saw_waiter = waiting.wait_for(timeout) == std::future_status::ready; + }); + tiered_index->registerTracingCallback("Relabel::before_wait_for_consolidate", [&] { + waiter_entered = true; + relabel_waiting.set_value(); + // Let the worker publish Done and attempt cleanup before waitForJob reads the job. + // The registry guard must prevent deletion until this callback and the wait return. + deleted_while_waiting = + deleted.wait_for(std::chrono::milliseconds(100)) == std::future_status::ready; + }); + tiered_index->registerTracingCallback("ConsolidateJob::after_delete", + [&] { job_deleted.set_value(); }); + + mock_thread_pool.init_threads(); + const bool reached_worker = paused.wait_for(timeout) == std::future_status::ready; + auto relabel_code = VecSimRelabel_OldLabelMissing; + if (reached_worker) { + // Public writes stay on the test thread; the worker alone changes the job's status. + relabel_code = VecSimIndex_RelabelVector(tiered_index, 0, 1); + } + if (!waiter_entered) { + relabel_waiting.set_value(); // release the worker if relabel never reached its wait + } + mock_thread_pool.thread_pool_wait(); + mock_thread_pool.thread_pool_join(); + + ASSERT_TRUE(reached_worker); + ASSERT_TRUE(worker_saw_waiter); + ASSERT_TRUE(waiter_entered); + EXPECT_FALSE(deleted_while_waiting); + EXPECT_EQ(deleted.wait_for(std::chrono::seconds(0)), std::future_status::ready); + EXPECT_EQ(relabel_code, VecSimRelabel_OK); + EXPECT_TRUE(mock_thread_pool.jobQ.empty()); + EXPECT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(1)); + EXPECT_FALSE(tiered_index->GetSVSIndex()->isLabelExists(0)); + EXPECT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); + EXPECT_EQ(tiered_index->indexLabelCount(), 1); +} + // Rollback needs the backend to refuse the target, and to hold old_label live while the buffer // still holds it with a registered job. Both are built by hand: // - the target keeps a stale translator entry, because a backend-side delete leaves no From 24cb70911340380320a036134f9ec91ab4bf695e Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Fri, 16 Oct 2026 02:04:27 -0700 Subject: [PATCH 95/99] don't create insert jobs while initilizing svs; add tests --- src/VecSim/algorithms/svs/svs_tiered.h | 28 ++++++-- tests/unit/test_svs_tiered.cpp | 94 ++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 5 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 4b830422b..a76028823 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -292,6 +292,11 @@ class TieredSVSIndex : public VecSimTieredIndex { bool initInProgress{false}; std::vector initDeletedLabels; + // Submission list only; labelToInsertJobs/invalidJobs retain ownership of these jobs. + // Access under flatIndexGuard exclusively. Submit after releasing updateJobMutex so + // inserts arriving during initialization do not occupy workers waiting for the build. + vecsim_stl::vector initInsertJobs; + /// //////////////////////////////////////////////////////////////////////////////////////////////////// // TieredSVS_BatchIterator // @@ -536,13 +541,20 @@ class TieredSVSIndex : public VecSimTieredIndex { auto index = static_cast *>(idx); assert(index); std::vector deleted_labels; + vecsim_stl::vector insert_jobs(index->allocator); { std::lock_guard lock(index->updateJobMutex); deleted_labels = index->initSVSIndex(availableThreads); + std::lock_guard flat_lock{index->flatIndexGuard}; + insert_jobs.swap(index->initInsertJobs); } for (labelType label : deleted_labels) { index->scheduleSVSIndexConsolidate(label); } + // Include invalidated jobs: their callbacks remove them from invalidJobs and delete them. + if (!insert_jobs.empty()) { + index->submitJobs(insert_jobs); + } } static void executeInsertJobWrapper(AsyncJob *job) { @@ -892,7 +904,8 @@ class TieredSVSIndex : public VecSimTieredIndex { const TieredIndexParams &tiered_index_params, std::shared_ptr allocator) : Base(svs_index, bf_index, tiered_index_params, allocator), - uncompletedJobs(this->allocator), labelToConsolidateJobs(this->allocator) { + uncompletedJobs(this->allocator), labelToConsolidateJobs(this->allocator), + initInsertJobs(this->allocator) { const auto &tiered_svs_params = tiered_index_params.specificParams.tieredSVSParams; // If flatBufferLimit is not initialized (0), use the default update threshold. @@ -998,13 +1011,16 @@ class TieredSVSIndex : public VecSimTieredIndex { ret += this->frontendIndex->addVector(blob, label); TieredInsertJob *new_insert_job = createInsertJob(label, new_flat_id); - const bool defer_delete = initInProgress; + const bool defer_until_initialized = initInProgress; + if (defer_until_initialized) { + initInsertJobs.push_back(new_insert_job); + } this->flatIndexGuard.unlock(); // Here, a worker might ingest the previous vector that was stored under "label" // (in case of override in non-MULTI index) - so if it's there, we remove it // we submit the insert job. - if (!this->backendIndex->isMultiValue() && !defer_delete) { + if (!this->backendIndex->isMultiValue() && !defer_until_initialized) { // If we removed the previous vector from both svs and flat in the overwrite // process, we still return 0 (not -1). auto deleted = svs_index->deleteVector(label); @@ -1013,8 +1029,10 @@ class TieredSVSIndex : public VecSimTieredIndex { ret = std::max(ret - deleted, 0); } - // Insert job to the queue and signal the workers' updater. - this->submitSingleJob(new_insert_job); + // Initialization submits its buffered jobs after releasing updateJobMutex. + if (!defer_until_initialized) { + this->submitSingleJob(new_insert_job); + } return ret; } } diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 0c4fdb861..fcf0b0a5c 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -474,6 +474,7 @@ TYPED_TEST(SVSTieredIndexTest, updateVectorsDuringUpdateJob) { ASSERT_EQ(tiered_index->updateVectors(i, replacements, new_per_label), VecSimUpdate_OK) << "label " << i; } + mock_thread_pool.thread_pool_wait(); mock_thread_pool.thread_pool_join(); auto *svs_index = tiered_index->GetSVSIndex(); @@ -589,6 +590,7 @@ TYPED_TEST(SVSTieredIndexTest, addRelabelAndUpdateTogetherUnderLoad) { GenerateVector(fresh, dim, 7000.0f + 10.0f * i); ASSERT_EQ(VecSimIndex_AddVector(tiered_index, fresh, fresh_base + i), 1); } + mock_thread_pool.thread_pool_wait(); mock_thread_pool.thread_pool_join(); auto *svs_index = tiered_index->GetSVSIndex(); @@ -815,6 +817,7 @@ TYPED_TEST(SVSTieredIndexTest, relabelVectorTakesOverPendingConsolidatesOfTheTar for (size_t i = 0; i < n; i++) { GenerateAndAddVector(tiered_index, dim, i, i); } + mock_thread_pool.thread_pool_wait(); mock_thread_pool.thread_pool_join(); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); for (size_t i = 0; i < n; i++) { @@ -864,6 +867,7 @@ TYPED_TEST(SVSTieredIndexTest, relabelVectorWaitsForAnExecutingConsolidate) { for (size_t i = 0; i < 2; i++) { GenerateAndAddVector(tiered_index, dim, i, i); } + mock_thread_pool.thread_pool_wait(); mock_thread_pool.thread_pool_join(); ASSERT_TRUE(tiered_index->GetSVSIndex()->isLabelExists(1)); @@ -1331,8 +1335,13 @@ TYPED_TEST(SVSTieredIndexTestBasic, addVectorDuringInitializationWithFullFlatBuf // the second sees a buffer above the limit. Both must return before init resumes. EXPECT_EQ(GenerateAndAddVector(index, dim, n, n), 1); EXPECT_EQ(GenerateAndAddVector(index, dim, n + 1, n + 1), 1); + // The only worker is paused in init. Neither insert may be submitted yet, + // where another worker could pick it up and block behind updateJobMutex. + std::lock_guard queue_lock{mock_thread_pool.queue_guard}; + EXPECT_TRUE(mock_thread_pool.jobQ.empty()); } additions_finished.set_value(); + mock_thread_pool.thread_pool_wait(); mock_thread_pool.thread_pool_join(); ASSERT_TRUE(reached_snapshot) << "Initialization did not reach the snapshot hook"; @@ -1353,6 +1362,85 @@ TYPED_TEST(SVSTieredIndexTestBasic, addVectorDuringInitializationWithFullFlatBuf } } +TYPED_TEST(SVSTieredIndexTestBasic, overwriteSnapshotLabelDuringInitialization) { + constexpr size_t dim = 4; + constexpr size_t n = 8; + constexpr labelType overwritten_label = 0; + for (size_t overwrite_count : {size_t{1}, size_t{2}}) { + SCOPED_TRACE(::testing::Message() << "overwrite_count=" << overwrite_count); + SVSParams params = {.type = TypeParam::get_index_type(), + .dim = dim, + .metric = VecSimMetric_L2, + .multi = false}; + auto svs_params = CreateParams(params); + tieredIndexMock mock_thread_pool(1); + auto tiered_params = this->CreateTieredSVSParams(svs_params, mock_thread_pool, n); + tiered_params.flatBufferLimit = n; + auto *index = this->CreateTieredSVSIndex(tiered_params, mock_thread_pool); + ASSERT_INDEX(index); + + for (size_t i = 0; i < n; ++i) { + GenerateAndAddVector(index, dim, i, i); + } + ASSERT_FALSE(index->GetSVSIndex()->ready()); + ASSERT_EQ(index->GetFlatIndex()->indexSize(), n); + ASSERT_EQ(index->labelToInsertJobs.count(overwritten_label), 0); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); + + std::promise snapshot_taken, overwrites_finished; + auto snapshot_ready = snapshot_taken.get_future(); + auto overwrites_ready = overwrites_finished.get_future(); + const auto timeout = std::chrono::seconds(5); + bool completed_during_init = false; + index->registerTracingCallback("UpdateJob::after_snapshot", [&] { + snapshot_taken.set_value(); + completed_during_init = + overwrites_ready.wait_for(timeout) == std::future_status::ready; + // Release init on timeout as well, so a regressed blocking add can return. + }); + + mock_thread_pool.init_threads(); + const bool reached_snapshot = + snapshot_ready.wait_for(timeout) == std::future_status::ready; + if (reached_snapshot) { + // The first overwrite cancels a captured row with no insert job. Removing + // label 0 swaps the last captured row down; init cleanup must later preserve + // the replacement and update its insert job's id as the buffer shrinks. + // A second overwrite also exercises invalidation of an unsubmitted job. + for (size_t i = 0; i < overwrite_count; ++i) { + EXPECT_EQ(GenerateAndAddVector(index, dim, overwritten_label, + n + i + 1), + 0); + } + } + overwrites_finished.set_value(); + mock_thread_pool.thread_pool_wait(); + mock_thread_pool.thread_pool_join(); + + ASSERT_TRUE(reached_snapshot) << "Initialization did not reach the snapshot hook"; + ASSERT_TRUE(completed_during_init) << "Overwrite waited for initialization"; + EXPECT_TRUE(index->GetSVSIndex()->ready()); + EXPECT_EQ(index->GetFlatIndex()->indexSize(), 0); + // Per-label consolidation does not compact the deleted snapshot row. + EXPECT_EQ(index->GetBackendIndex()->indexSize(), n + 1); + EXPECT_EQ(index->getNumMarkedDeleted(), 1); + EXPECT_EQ(index->GetBackendIndex()->indexLabelCount(), n); + EXPECT_TRUE(index->labelToInsertJobs.empty()); + EXPECT_TRUE(index->invalidJobs.empty()); + EXPECT_TRUE(mock_thread_pool.jobQ.empty()); + for (size_t i = 0; i < n; ++i) { + // Read the backend directly so a surviving flat row cannot hide a stale + // snapshot copy. Exact cardinality and values detect duplicates as well. + std::vector> stored; + index->GetBackendIndex()->getDataByLabel(i, stored); + const auto value = i == overwritten_label ? n + overwrite_count : i; + const std::vector> expected{ + std::vector(dim, static_cast(value))}; + EXPECT_EQ(stored, expected) << "label=" << i; + } + } +} + // A search already occupies one worker, so init proceeds after its reservation timeout. // The search reaches the backend while init is still paused immediately after publication. TYPED_TEST(SVSTieredIndexTestBasic, deleteVectorBeforeInitializationPublishes) { @@ -2100,6 +2188,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, insertJobAsync) { GenerateAndAddVector(tiered_index, dim, i, i / (TEST_DATA_T)n); } + mock_thread_pool.thread_pool_wait(); mock_thread_pool.thread_pool_join(); ASSERT_EQ(tiered_index->indexSize(), n); ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); @@ -2152,6 +2241,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, insertJobAsyncMulti) { } } + mock_thread_pool.thread_pool_wait(); mock_thread_pool.thread_pool_join(); EXPECT_EQ(mock_thread_pool.jobQ.size(), 0); EXPECT_EQ(tiered_index->indexLabelCount(), n / per_label); @@ -2872,6 +2962,7 @@ TYPED_TEST(SVSTieredIndexTest, parallelInsertSearch) { tiered_index->submitSingleJob(search_job); } + mock_thread_pool.thread_pool_wait(); mock_thread_pool.thread_pool_join(); // Verify that vectors were moved to SVS as expected @@ -4125,6 +4216,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, overwriteVectorAsync) { EXPECT_EQ(tiered_index->addVector(vector, label_to_overwrite), 0); } + mock_thread_pool.thread_pool_wait(); mock_thread_pool.thread_pool_join(); EXPECT_LE(tiered_index->GetFlatIndex()->indexSize(), updateThreshold); @@ -4426,6 +4518,7 @@ TYPED_TEST(SVSTieredIndexTest, switchWriteModes) { } VecSimIndex_AddVector(tiered_index, vector, i % n_labels + n_labels); } + mock_thread_pool.thread_pool_wait(); mock_thread_pool.thread_pool_join(); EXPECT_GE(tiered_index->GetBackendIndex()->indexSize(), 2 * n - this->getUpdateThreshold()); @@ -5825,6 +5918,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, forgetConsolidateJobIgnoresAnUnregisteredLab for (size_t i = 0; i < 2; i++) { GenerateAndAddVector(tiered_index, dim, i, i); } + mock_thread_pool.thread_pool_wait(); mock_thread_pool.thread_pool_join(); ASSERT_EQ(VecSimIndex_DeleteVector(tiered_index, 1), 1); From 548930af98db87676144d85fe0f47ae30a84d22a Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Fri, 16 Oct 2026 02:06:49 -0700 Subject: [PATCH 96/99] format --- tests/unit/test_svs_tiered.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index fcf0b0a5c..6aad95f20 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -1394,23 +1394,20 @@ TYPED_TEST(SVSTieredIndexTestBasic, overwriteSnapshotLabelDuringInitialization) bool completed_during_init = false; index->registerTracingCallback("UpdateJob::after_snapshot", [&] { snapshot_taken.set_value(); - completed_during_init = - overwrites_ready.wait_for(timeout) == std::future_status::ready; + completed_during_init = overwrites_ready.wait_for(timeout) == std::future_status::ready; // Release init on timeout as well, so a regressed blocking add can return. }); mock_thread_pool.init_threads(); - const bool reached_snapshot = - snapshot_ready.wait_for(timeout) == std::future_status::ready; + const bool reached_snapshot = snapshot_ready.wait_for(timeout) == std::future_status::ready; if (reached_snapshot) { // The first overwrite cancels a captured row with no insert job. Removing // label 0 swaps the last captured row down; init cleanup must later preserve // the replacement and update its insert job's id as the buffer shrinks. // A second overwrite also exercises invalidation of an unsubmitted job. for (size_t i = 0; i < overwrite_count; ++i) { - EXPECT_EQ(GenerateAndAddVector(index, dim, overwritten_label, - n + i + 1), - 0); + EXPECT_EQ( + GenerateAndAddVector(index, dim, overwritten_label, n + i + 1), 0); } } overwrites_finished.set_value(); From 7878b1ca86c421a63a70b00f80bbac49d840afdb Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Fri, 16 Oct 2026 11:12:15 -0700 Subject: [PATCH 97/99] SVS rc8 --- cmake/svs.cmake | 6 +++--- deps/ScalableVectorSearch | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index e1335e16c..56b72ce46 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -64,7 +64,7 @@ if(USE_SVS) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.31" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "21.0") if(GLIBCXX_VERSION VERSION_GREATER_EQUAL "11") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc7-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc8-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") else() message(STATUS "libstdc++ >= GCC 11 is required for Clang SVS binaries - disabling SVS_SHARED_LIB") set(SVS_SHARED_LIB OFF) @@ -76,9 +76,9 @@ if(USE_SVS) else() if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.28") if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "14.0") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc7-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc8-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") else() - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc7-reduced.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc8-reduced.tar.gz" CACHE STRING "SVS URL") endif() else() message(STATUS "GLIBC >= 2.28 is required for GCC SVS binaries - disabling SVS_SHARED_LIB") diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch index 42d8b6f25..d2c70b5bc 160000 --- a/deps/ScalableVectorSearch +++ b/deps/ScalableVectorSearch @@ -1 +1 @@ -Subproject commit 42d8b6f25a4b677035bc0e5b2f410350f0317f13 +Subproject commit d2c70b5bc0604108cee719098a5161571e35f4a5 From b455ebcb1859ed2fd808ed6af3250793d6ca7eef Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Fri, 16 Oct 2026 13:26:24 -0700 Subject: [PATCH 98/99] SVS rc9 --- cmake/svs.cmake | 6 +++--- deps/ScalableVectorSearch | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index 56b72ce46..e6d7e1c3b 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -64,7 +64,7 @@ if(USE_SVS) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.31" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "21.0") if(GLIBCXX_VERSION VERSION_GREATER_EQUAL "11") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc8-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc9-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") else() message(STATUS "libstdc++ >= GCC 11 is required for Clang SVS binaries - disabling SVS_SHARED_LIB") set(SVS_SHARED_LIB OFF) @@ -76,9 +76,9 @@ if(USE_SVS) else() if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.28") if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "14.0") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc8-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc9-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") else() - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc8-reduced.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc9-reduced.tar.gz" CACHE STRING "SVS URL") endif() else() message(STATUS "GLIBC >= 2.28 is required for GCC SVS binaries - disabling SVS_SHARED_LIB") diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch index d2c70b5bc..285992faf 160000 --- a/deps/ScalableVectorSearch +++ b/deps/ScalableVectorSearch @@ -1 +1 @@ -Subproject commit d2c70b5bc0604108cee719098a5161571e35f4a5 +Subproject commit 285992fafecff733fe17c8c02c3b56cb7f533889 From bf589d3f184854369e45d709b4209678d8cecae2 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Fri, 16 Oct 2026 15:14:59 -0700 Subject: [PATCH 99/99] bench fixes and SVS rc10 --- cmake/svs.cmake | 6 ++-- deps/ScalableVectorSearch | 2 +- src/VecSim/vec_sim_tiered_index.h | 18 ++++++++--- tests/unit/test_svs_tiered.cpp | 53 +++++++++++++++++++++++++++++++ tests/utils/mock_thread_pool.cpp | 3 ++ tests/utils/mock_thread_pool.h | 1 + 6 files changed, 74 insertions(+), 9 deletions(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index e6d7e1c3b..160e761be 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -64,7 +64,7 @@ if(USE_SVS) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.31" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "21.0") if(GLIBCXX_VERSION VERSION_GREATER_EQUAL "11") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc9-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc10-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") else() message(STATUS "libstdc++ >= GCC 11 is required for Clang SVS binaries - disabling SVS_SHARED_LIB") set(SVS_SHARED_LIB OFF) @@ -76,9 +76,9 @@ if(USE_SVS) else() if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.28") if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "14.0") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc9-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc10-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") else() - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc9-reduced.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc10-reduced.tar.gz" CACHE STRING "SVS URL") endif() else() message(STATUS "GLIBC >= 2.28 is required for GCC SVS binaries - disabling SVS_SHARED_LIB") diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch index 285992faf..1109f0cd2 160000 --- a/deps/ScalableVectorSearch +++ b/deps/ScalableVectorSearch @@ -1 +1 @@ -Subproject commit 285992fafecff733fe17c8c02c3b56cb7f533889 +Subproject commit 1109f0cd23f987753715cf4e635450f0d4462ec8 diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 9cf574dc9..e9218c6ae 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -18,6 +18,7 @@ #include "VecSim/utils/alignment.h" #include "VecSim/utils/scoped_locks.h" +#include #include #include #include @@ -30,6 +31,9 @@ #define TIERED_LOG this->backendIndex->log +// RediSearch's submit callback places an n_jobs-sized array on the worker's stack. +inline constexpr size_t TIERED_MAX_JOBS_PER_SUBMIT = 4096; + /** * Definition of generic job structure for asynchronous tiered index. */ @@ -182,12 +186,16 @@ class VecSimTieredIndex : public VecSimIndexInterface { } void submitJobs(vecsim_stl::vector &jobs) { - vecsim_stl::vector callbacks(jobs.size(), this->allocator); - for (size_t i = 0; i < jobs.size(); i++) { - callbacks[i] = jobs[i]->Execute; + vecsim_stl::vector callbacks(this->allocator); + for (size_t start = 0; start < jobs.size(); start += TIERED_MAX_JOBS_PER_SUBMIT) { + const size_t n = std::min(TIERED_MAX_JOBS_PER_SUBMIT, jobs.size() - start); + callbacks.assign(n, nullptr); + for (size_t i = 0; i < n; i++) { + callbacks[i] = jobs[start + i]->Execute; + } + this->SubmitJobsToQueue(this->jobQueue, this->jobQueueCtx, jobs.data() + start, + callbacks.data(), n); } - this->SubmitJobsToQueue(this->jobQueue, this->jobQueueCtx, jobs.data(), callbacks.data(), - jobs.size()); } /** diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 6aad95f20..228c1dcd6 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -1362,6 +1362,59 @@ TYPED_TEST(SVSTieredIndexTestBasic, addVectorDuringInitializationWithFullFlatBuf } } +// Inserts buffered during initialization are submitted in bounded batches: the real thread +// pool's submit callback stack-allocates per-call, so one huge batch overflows a worker stack. +TYPED_TEST(SVSTieredIndexTestBasic, buffersSubmittedInBoundedBatchesAfterInitialization) { + constexpr size_t dim = 4; + constexpr size_t n = 8; + constexpr size_t n_during_init = TIERED_MAX_JOBS_PER_SUBMIT + 100; + SVSParams params = { + .type = TypeParam::get_index_type(), .dim = dim, .metric = VecSimMetric_L2, .multi = false}; + auto svs_params = CreateParams(params); + tieredIndexMock mock_thread_pool(1); + auto tiered_params = this->CreateTieredSVSParams(svs_params, mock_thread_pool, n); + tiered_params.flatBufferLimit = n; + auto *index = this->CreateTieredSVSIndex(tiered_params, mock_thread_pool); + ASSERT_INDEX(index); + + for (size_t i = 0; i < n; ++i) { + GenerateAndAddVector(index, dim, i, i); + } + ASSERT_FALSE(index->GetSVSIndex()->ready()); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); + + std::promise snapshot_taken, additions_finished; + auto snapshot_ready = snapshot_taken.get_future(); + auto additions_ready = additions_finished.get_future(); + const auto timeout = std::chrono::seconds(30); + bool completed_during_init = false; + index->registerTracingCallback("UpdateJob::after_snapshot", [&] { + snapshot_taken.set_value(); + completed_during_init = additions_ready.wait_for(timeout) == std::future_status::ready; + // Release init on timeout as well, so a regressed blocking add can return. + }); + + mock_thread_pool.init_threads(); + const bool reached_snapshot = snapshot_ready.wait_for(timeout) == std::future_status::ready; + if (reached_snapshot) { + for (size_t i = 0; i < n_during_init; ++i) { + EXPECT_EQ(GenerateAndAddVector(index, dim, n + i, n + i), 1); + } + } + additions_finished.set_value(); + mock_thread_pool.thread_pool_wait(); + mock_thread_pool.thread_pool_join(); + + ASSERT_TRUE(reached_snapshot) << "Initialization did not reach the snapshot hook"; + ASSERT_TRUE(completed_during_init) << "addVector waited for initialization"; + EXPECT_LE(mock_thread_pool.max_submit_batch, TIERED_MAX_JOBS_PER_SUBMIT); + EXPECT_TRUE(index->GetSVSIndex()->ready()); + EXPECT_EQ(index->GetFlatIndex()->indexSize(), 0); + EXPECT_EQ(index->GetBackendIndex()->indexSize(), n + n_during_init); + EXPECT_EQ(index->GetBackendIndex()->indexLabelCount(), n + n_during_init); + EXPECT_TRUE(mock_thread_pool.jobQ.empty()); +} + TYPED_TEST(SVSTieredIndexTestBasic, overwriteSnapshotLabelDuringInitialization) { constexpr size_t dim = 4; constexpr size_t n = 8; diff --git a/tests/utils/mock_thread_pool.cpp b/tests/utils/mock_thread_pool.cpp index 53611f237..641661167 100644 --- a/tests/utils/mock_thread_pool.cpp +++ b/tests/utils/mock_thread_pool.cpp @@ -9,6 +9,8 @@ #include "mock_thread_pool.h" +#include + /* * Mock callbacks and synchronization primitives for testing async tiered index. * We use a simple std::queue to simulate the job queue. @@ -41,6 +43,7 @@ void tieredIndexMock::reset_ctx(IndexExtCtx *new_ctx) { int tieredIndexMock::submit_callback_internal(AsyncJob **jobs, JobCallback *CBs, size_t jobs_len) { { std::unique_lock lock(this->queue_guard); + this->max_submit_batch = std::max(this->max_submit_batch, jobs_len); for (size_t i = 0; i < jobs_len; i++) { // Wrap the job with a struct that contains a weak reference to the related index. auto owned_job = diff --git a/tests/utils/mock_thread_pool.h b/tests/utils/mock_thread_pool.h index 7b6f09974..2743225cf 100644 --- a/tests/utils/mock_thread_pool.h +++ b/tests/utils/mock_thread_pool.h @@ -107,6 +107,7 @@ class tieredIndexMock { std::mutex queue_guard; std::condition_variable queue_cond; JobQueue jobQ; + size_t max_submit_batch = 0; // Largest jobs_len seen by a single submit callback. bool run_thread; IndexExtCtx *ctx;