feat(reorder): add enbale_reorder as search parameter

Signed-off-by: LHT129 <tianlan.lht@antgroup.com>

Author: LHT129

src/algorithm/hgraph.cpp

@@ -2120,7 +2120,7 @@ HGraph::SearchWithRequest(const SearchRequest& request) const {
 
     this->pool_->ReturnOne(vt);
 
-    if (use_reorder_) {
+    if (use_reorder_ and params.enable_reorder) {
         this->reorder(raw_query, this->high_precise_codes_, search_result, k, nullptr, ctx);
     }
 

src/algorithm/hgraph_parameter.h

@@ -75,8 +75,7 @@ class HGraphSearchParameters : public IndexSearchParameter {
 
 public:
     int64_t ef_search{30};
-    float topk_factor{0.0F};
-    bool use_reorder{false};
+
     bool use_extra_info_filter{false};
 
 private:

src/algorithm/index_search_parameter.h

@@ -37,6 +37,14 @@ class IndexSearchParameter {
             timeout_ms = json[SEARCH_MAX_TIME_COST_MS].GetInt();
             enable_time_record = true;
         }
+
+        if (json.Contains(SEARCH_PARAM_FACTOR)) {
+            topk_factor = json[SEARCH_PARAM_FACTOR].GetFloat();
+        }
+
+        if (json.Contains(SEARCH_PARAM_ENABLE_REORDER)) {
+            enable_reorder = json[SEARCH_PARAM_ENABLE_REORDER].GetBool();
+        }
     }
 
 public:
@@ -45,5 +53,9 @@ class IndexSearchParameter {
     // for timeout
     double timeout_ms{std::numeric_limits<double>::max()};
     bool enable_time_record{false};
+
+    // for reorder
+    bool enable_reorder{true};
+    float topk_factor{0.0F};
 };
 }  // namespace vsag

src/algorithm/ivf.cpp

@@ -697,6 +697,7 @@ IVF::create_search_param(const std::string& parameters, const FilterPtr& filter)
     param.factor = search_param.topk_factor;
     param.first_order_scan_ratio = search_param.first_order_scan_ratio;
     param.parallel_search_thread_count = search_param.parallel_search_thread_count;
+    param.enable_reorder = search_param.enable_reorder;
     if (search_param.enable_time_record) {
         param.time_cost = std::make_shared<Timer>();
         param.time_cost->SetThreshold(search_param.timeout_ms);
@@ -942,7 +943,7 @@ IVF::SearchWithRequest(const SearchRequest& request) const {
     auto param = this->create_search_param(request.params_str_, request.filter_);
     param.search_mode = KNN_SEARCH;
     param.topk = request.topk_;
-    if (use_reorder_) {
+    if (use_reorder_ and param.enable_reorder) {
         param.topk = static_cast<int64_t>(param.factor * static_cast<float>(request.topk_));
     }
     auto query = request.query_;
@@ -956,10 +957,12 @@ IVF::SearchWithRequest(const SearchRequest& request) const {
             param.executors.emplace_back(executor);
         }
     }
+
     auto search_result = this->search<KNN_SEARCH>(query, param, ctx);
-    if (use_reorder_) {
+    if (use_reorder_ and param.enable_reorder) {
         return reorder(request.topk_, search_result, query->GetFloat32Vectors(), param, ctx);
     }
+
     auto count = static_cast<const int64_t>(search_result->Size());
     auto [dataset_results, dists, labels] = create_fast_dataset(count, allocator_);
     for (int64_t j = count - 1; j >= 0; --j) {

src/algorithm/ivf_parameter.cpp

@@ -108,11 +108,6 @@ IVFSearchParameters::FromJson(const std::string& json_string) {
                                IVF_SEARCH_PARAM_SCAN_BUCKETS_COUNT));
     obj.scan_buckets_count = params[INDEX_TYPE_IVF][IVF_SEARCH_PARAM_SCAN_BUCKETS_COUNT].GetInt();
 
-    // set obj.topk_factor
-    if (params[INDEX_TYPE_IVF].Contains(SEARCH_PARAM_FACTOR)) {
-        obj.topk_factor = params[INDEX_TYPE_IVF][SEARCH_PARAM_FACTOR].GetFloat();
-    }
-
     // set obj.first_order_scan_ratio
     if (params[INDEX_TYPE_IVF].Contains(GNO_IMI_SEARCH_PARAM_FIRST_ORDER_SCAN_RATIO)) {
         obj.first_order_scan_ratio =

src/algorithm/ivf_parameter.h

@@ -56,7 +56,6 @@ class IVFSearchParameters : public IndexSearchParameter {
 
 public:
     int64_t scan_buckets_count{30};
-    float topk_factor{2.0F};
     float first_order_scan_ratio{1.0F};
 
 private:

src/impl/inner_search_param.h

@@ -36,18 +36,22 @@ class InnerSearchParam {
 
 public:
     int64_t topk{0};
+    float factor{2.0F};
     float radius{0.0F};
-    InnerIdType ep{0};
-    uint64_t ef{10};
-    FilterPtr is_inner_id_allowed{nullptr};
-    float skip_ratio{0.8F};
+    bool enable_reorder{true};
     InnerSearchMode search_mode{KNN_SEARCH};
     int range_search_limit_size{-1};
+
     int64_t parallel_search_thread_count{1};
+    FilterPtr is_inner_id_allowed{nullptr};
+
+    // for HGraph
+    InnerIdType ep{0};
+    uint64_t ef{10};
+    float skip_ratio{0.8F};
 
-    // for ivf
+    // for IVF
     int scan_bucket_size{1};
-    float factor{2.0F};
     float first_order_scan_ratio{1.0F};
     std::vector<ExecutorPtr> executors;
 

src/inner_string_params.h

@@ -171,6 +171,7 @@ const char* const SPARSE_CODES = "sparse";
 
 const char* const IVF_SEARCH_PARAM_SCAN_BUCKETS_COUNT = "scan_buckets_count";
 const char* const SEARCH_PARAM_FACTOR = "factor";
+const char* const SEARCH_PARAM_ENABLE_REORDER = "enable_reorder";
 const char* const SEARCH_PARALLELISM = "parallelism";
 const char* const SEARCH_MAX_TIME_COST_MS = "timeout_ms";
 const char* const SPARSE_N_CANDIDATE = "n_candidate";

tests/test_hgraph.cpp

@@ -2134,6 +2134,61 @@ TEST_CASE("(Daily) HGraph Search Over Time", "[ft][hgraph][daily]") {
     TestHGraphSearchOverTime(test_index, resource);
 }
 
+static void
+TestHGraphSearchDisableReorder(const fixtures::HGraphTestIndexPtr& test_index,
+                               const fixtures::HGraphResourcePtr& resource) {
+    using namespace fixtures;
+    auto origin_size = vsag::Options::Instance().block_size_limit();
+    auto size = GENERATE(1024 * 1024 * 2);
+    constexpr const char* search_param_tmp = R"({{
+            "hgraph": {{
+                "ef_search": 200,
+                "enable_reorder": {}
+            }}
+        }})";
+
+    for (auto metric_type : resource->metric_types) {
+        for (auto dim : resource->dims) {
+            auto base_quantization_str = "sq4_uniform,fp32";
+            float recall_with_reorder = 0.95;
+            float recall_without_reorder = 0.75;
+            INFO(
+                fmt::format("metric_type: {}, dim: {}, base_quantization_str: {}, "
+                            "recall_with_reorder: {}, recall_without_reorder: {}",
+                            metric_type,
+                            dim,
+                            base_quantization_str,
+                            recall_with_reorder,
+                            recall_without_reorder));
+            vsag::Options::Instance().set_block_size_limit(size);
+            HGraphTestIndex::HGraphBuildParam build_param(metric_type, dim, base_quantization_str);
+            auto param = HGraphTestIndex::GenerateHGraphBuildParametersString(build_param);
+            auto index = TestIndex::TestFactory(test_index->name, param, true);
+            auto dataset =
+                HGraphTestIndex::pool.GetDatasetAndCreate(dim, resource->base_count, metric_type);
+            TestIndex::TestBuildIndex(index, dataset, true);
+            auto recall_result_with_reorder = TestIndex::TestKnnSearch(
+                index, dataset, fmt::format(search_param_tmp, true), recall_with_reorder, true);
+            auto recall_result_without_reorder = TestIndex::TestKnnSearch(
+                index, dataset, fmt::format(search_param_tmp, false), recall_without_reorder, true);
+            REQUIRE(recall_result_with_reorder > recall_result_without_reorder);
+            vsag::Options::Instance().set_block_size_limit(origin_size);
+        }
+    }
+}
+
+TEST_CASE("(PR) HGraph Search Disable Reorder", "[ft][hgraph][pr]") {
+    auto test_index = std::make_shared<fixtures::HGraphTestIndex>();
+    auto resource = test_index->GetResource(true);
+    TestHGraphSearchDisableReorder(test_index, resource);
+}
+
+TEST_CASE("(Daily) HGraph Search Disable Reorder", "[ft][hgraph][daily]") {
+    auto test_index = std::make_shared<fixtures::HGraphTestIndex>();
+    auto resource = test_index->GetResource(false);
+    TestHGraphSearchDisableReorder(test_index, resource);
+}
+
 static void
 TestHGraphDiskIOType(const fixtures::HGraphTestIndexPtr& test_index,
                      const fixtures::HGraphResourcePtr& resource) {

tests/test_index.cpp

@@ -435,14 +435,14 @@ TestIndex::TestKnnSearchCompare(const IndexPtr& index_weak,
     }
 }
 
-void
+float
 TestIndex::TestKnnSearch(const IndexPtr& index,
                          const TestDatasetPtr& dataset,
                          const std::string& search_param,
                          float expected_recall,
                          bool expected_success) {
     if (not index->CheckFeature(vsag::SUPPORT_KNN_SEARCH)) {
-        return;
+        return 0.0F;
     }
     auto queries = dataset->query_;
     auto query_count = queries->GetNumElements();
@@ -458,7 +458,7 @@ TestIndex::TestKnnSearch(const IndexPtr& index,
             if (res.has_value()) {
                 REQUIRE(res.value()->GetDim() == 0);
             }
-            return;
+            return 0.0F;
         } else {
             REQUIRE(res.has_value() == true);
         }
@@ -474,6 +474,7 @@ TestIndex::TestKnnSearch(const IndexPtr& index,
                          expected_recall * query_count));
     }
     REQUIRE(cur_recall > expected_recall * query_count * RECALL_THRESHOLD);
+    return cur_recall / static_cast<float>(query_count);
 }
 
 void