Test: Disable hnsw brute force env

CMakeLists.txt

@@ -105,6 +105,10 @@ find_package(nlohmann_json REQUIRED)
 find_package(glog REQUIRED)
 find_package(prometheus-cpp REQUIRED)
 find_package(fmt REQUIRED)
+if(NOT TARGET fmt::fmt)
+    add_library(fmt::fmt INTERFACE IMPORTED)
+    target_link_libraries(fmt::fmt INTERFACE fmt::fmt-header-only)
+endif()
 find_package(xxHash REQUIRED)
 include_directories(${xxHash_INCLUDE_DIRS})
 find_package(simde REQUIRED)

ci/pod/e2e-cpu.yaml

@@ -40,4 +40,4 @@ spec:
   volumes:
     - name: db-data
       persistentVolumeClaim:
-        claimName: db-data
+        claimName: db-data-pvc-ci-cluster

ci/pod/e2e-gpu.yaml

@@ -32,4 +32,4 @@ spec:
   volumes:
     - name: db-data
       persistentVolumeClaim:
-        claimName: db-data
+        claimName: db-data-pvc-ci-cluster

ci/pod/e2e-sse.yaml

@@ -41,4 +41,4 @@ spec:
   volumes:
     - name: db-data
       persistentVolumeClaim:
-        claimName: db-data
+        claimName: db-data-pvc-ci-cluster

cmake/libs/cardinal/v1/CMakeLists.txt

@@ -3,7 +3,7 @@ project(knowhere CXX C)
 
 # Use short SHA1 as version
 # v2.5 tag is used for cardinal v1
-set(CARDINAL_VERSION v2.5.103)
+set(CARDINAL_VERSION v2.5.104)
 set(CARDINAL_REPO_URL "https://github.com/zilliztech/cardinal.git")
 
 set(CARDINAL_ROOT  "${KNOWHERE_THRID_ROOT}/cardinalv1")

cmake/libs/cardinal/v2/CMakeLists.txt

@@ -3,7 +3,7 @@ project(knowhere CXX C)
 
 # Use short SHA1 as version
 # v2.6 tag is used for cardinal v2
-set(CARDINAL_VERSION v2.6.8)
+set(CARDINAL_VERSION v2.6.9)
 set(CARDINAL_REPO_URL "https://github.com/zilliztech/cardinal.git")
 
 set(CARDINAL_ROOT  "${KNOWHERE_THRID_ROOT}/cardinalv2")

cmake/libs/libmilvus-common.cmake

@@ -1,6 +1,6 @@
 
 set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES "")
-set( MILVUS-COMMON-VERSION c37f138 )
+set( MILVUS-COMMON-VERSION 9dc0923 )
 set( GIT_REPOSITORY  "https://github.com/zilliztech/milvus-common.git" )
 
 message(STATUS "milvus-common repo: ${GIT_REPOSITORY}")

python/build_portable_wheel.sh

@@ -73,6 +73,11 @@ check_deps() {
 
     command -v "$PYTHON" >/dev/null || { log_error "Python not found: $PYTHON"; exit 1; }
 
+    $PYTHON -c "import numpy" 2>/dev/null || {
+        log_error "numpy not found. Install with: pip3 install 'numpy<2'"
+        exit 1
+    }
+
     [ -f "$BUILD_DIR/libknowhere.so" ] || {
         log_error "libknowhere.so not found at $BUILD_DIR"
         log_error "Please build Knowhere C++ library first:"
@@ -145,15 +150,16 @@ repair_wheel() {
     log_info "Repairing wheel for $platform..." >&2
 
     # Export library paths from libknowhere RUNPATH
-    local lib_paths=$(readelf -d "$BUILD_DIR/libknowhere.so" | \
-                      grep -E 'RUNPATH|RPATH' | \
-                      sed 's/.*\[\(.*\)\]/\1/' | \
-                      tr ':' '\n' | grep -v '^$' | tr '\n' ':')
+    local lib_paths
+    lib_paths=$(readelf -d "$BUILD_DIR/libknowhere.so" | \
+                grep -E 'RUNPATH|RPATH' | \
+                sed 's/.*\[\(.*\)\]/\1/' | \
+                tr ':' '\n' | grep -v '^$' | tr '\n' ':')
 
     export LD_LIBRARY_PATH="${lib_paths}:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu"
 
-    # Run auditwheel repair with full output
-    if ! auditwheel repair "$wheel" -w dist/ --plat "$platform" 2>&1 | tee /dev/stderr; then
+    # Run auditwheel repair (all output to stderr so it doesn't leak into command substitution)
+    if ! auditwheel repair "$wheel" -w dist/ --plat "$platform" 2>&1 >&2; then
         log_error "auditwheel repair failed" >&2
         exit 1
     fi
@@ -237,11 +243,13 @@ main() {
     log_info "Target platform: $platform"
 
     # Build wheel
-    local wheel=$(build_wheel)
+    local wheel
+    wheel=$(build_wheel)
     log_info "Built: $(basename "$wheel")"
 
     # Repair wheel
-    local final_wheel=$(repair_wheel "$wheel" "$platform")
+    local final_wheel
+    final_wheel=$(repair_wheel "$wheel" "$platform")
     log_info "Final: $(basename "$final_wheel")"
 
     # Verify

python/setup.py

@@ -106,7 +106,9 @@ def get_readme():
     author_email="milvus-team@zilliz.com",
     license='Apache License 2.0',
     keywords="search nearest neighbors",
-    setup_requires=["numpy", "setuptools_scm"],
+    # numpy must be pre-installed (install_deps.sh handles this).
+    # Do NOT use setup_requires — it triggers the deprecated fetch_build_eggs
+    # which makes network calls to PyPI and fails intermittently in CI.
     #use_scm_version={'root': '..', 'local_scheme': 'no-local-version', 'version_scheme': 'release-branch-semver'},
     long_description=get_readme(),
     long_description_content_type="text/markdown",

src/index/hnsw/base_hnsw_config.h

@@ -31,8 +31,7 @@ class BaseHnswConfig : public BaseConfig {
     CFG_INT efConstruction;
     CFG_INT ef;
     CFG_INT overview_levels;
-    CFG_BOOL disable_fallback_brute_force;  // default is false, means we will use fallback brute force when hnsw search
-                                            // does not get enough topk results
+    CFG_BOOL disable_fallback_brute_force;  // per-query override (see also KNOWHERE_DISABLE_HNSW_BRUTE_FORCE env)
     KNOHWERE_DECLARE_CONFIG(BaseHnswConfig) {
         KNOWHERE_CONFIG_DECLARE_FIELD(M).description("hnsw M").set_default(30).set_range(2, 2048).for_train();
         KNOWHERE_CONFIG_DECLARE_FIELD(efConstruction)

src/index/hnsw/faiss_hnsw.cc

@@ -41,6 +41,7 @@
 #include "index/hnsw/impl/IndexConditionalWrapper.h"
 #include "index/hnsw/impl/IndexHNSWWrapper.h"
 #include "index/hnsw/impl/IndexWrapperCosine.h"
+#include "index/hnsw/impl/hnsw_brute_force_env.h"
 #include "index/refine/refine_utils.h"
 #include "io/memory_io.h"
 #include "knowhere/bitsetview_idselector.h"
@@ -1335,7 +1336,7 @@ class BaseFaissRegularIndexHNSWNode : public BaseFaissRegularIndexNode {
         // set up a bf wrapper as fallback
         std::unique_ptr<faiss::Index> bf_index_wrapper = nullptr;
         faiss::Index* bf_index_wrapper_ptr = nullptr;
-        if (!whether_bf_search.value_or(false)) {
+        if (!whether_bf_search.value_or(false) && !IsHnswBruteForceDisabledByEnv()) {
             std::tie(bf_index_wrapper, is_refined) =
                 create_conditional_hnsw_wrapper(indexes[index_id].get(), hnsw_cfg, true, whether_to_enable_refine);
             if (bf_index_wrapper == nullptr) {
@@ -1405,7 +1406,8 @@ class BaseFaissRegularIndexHNSWNode : public BaseFaissRegularIndexNode {
                             real_topk++;
                         }
                         if (real_topk < k && real_topk < bitset.size() - bitset.count() &&
-                            bf_index_wrapper_ptr != nullptr && !hnsw_cfg.disable_fallback_brute_force.value()) {
+                            bf_index_wrapper_ptr != nullptr && !IsHnswBruteForceDisabledByEnv() &&
+                            !hnsw_cfg.disable_fallback_brute_force.value()) {
                             LOG_KNOWHERE_WARNING_ << "required topk: " << k
                                                   << ", but the actual num of results got from hnsw: " << real_topk
                                                   << ", trigger brute force search as fallback for hnsw search";
@@ -1565,8 +1567,19 @@ class BaseFaissRegularIndexHNSWNode : public BaseFaissRegularIndexNode {
     expected<DataSetPtr>
     RangeSearch(const DataSetPtr dataset, std::unique_ptr<Config> cfg, const BitsetView& bitset_,
                 milvus::OpContext* op_context) const override {
-        // if support ann_iterator, use iterator-based range_search (IndexNode::RangeSearch)
-        if (is_ann_iterator_supported()) {
+        // Check brute-force threshold BEFORE iterator path.
+        // At high filter ratios (>=97%), brute force is much faster than graph traversal
+        // because the iterator visits too many filtered-out nodes.
+        if (is_ann_iterator_supported() && !this->indexes.empty() && indexes[0] != nullptr) {
+            const auto& hnsw_cfg_check = static_cast<const FaissHnswConfig&>(*cfg);
+            BitsetView bitset_check(bitset_);
+            auto whether_bf = WhetherPerformBruteForceRangeSearch(indexes[0].get(), hnsw_cfg_check, bitset_check);
+            if (!whether_bf.has_value() || !whether_bf.value()) {
+                // Not brute-force worthy: use iterator path
+                return IndexNode::RangeSearch(dataset, std::move(cfg), bitset_, op_context);
+            }
+            // Fall through to brute-force range search path below
+        } else if (is_ann_iterator_supported()) {
             return IndexNode::RangeSearch(dataset, std::move(cfg), bitset_, op_context);
         }
         if (this->indexes.empty()) {

src/index/hnsw/impl/IndexConditionalWrapper.cc

@@ -20,6 +20,7 @@
 #include "index/hnsw/impl/IndexBruteForceWrapper.h"
 #include "index/hnsw/impl/IndexHNSWWrapper.h"
 #include "index/hnsw/impl/IndexWrapperCosine.h"
+#include "index/hnsw/impl/hnsw_brute_force_env.h"
 #include "knowhere/utils.h"
 
 #if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT)
@@ -38,6 +39,10 @@ WhetherPerformBruteForceSearch(const faiss::Index* index, const BaseConfig& cfg,
         return std::nullopt;
     }
 
+    if (IsHnswBruteForceDisabledByEnv()) {
+        return false;
+    }
+
     // decide
     const auto k = cfg.k.value();
 
@@ -71,6 +76,10 @@ WhetherPerformBruteForceRangeSearch(const faiss::Index* index, const FaissHnswCo
         return std::nullopt;
     }
 
+    if (IsHnswBruteForceDisabledByEnv()) {
+        return false;
+    }
+
     // decide
     const auto ef = cfg.ef.value();
 

src/index/hnsw/impl/hnsw_brute_force_env.cc

@@ -0,0 +1,41 @@
+// Copyright (C) 2019-2024 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include "index/hnsw/impl/hnsw_brute_force_env.h"
+
+#include <strings.h>
+
+#include <cstdlib>
+
+namespace knowhere {
+
+namespace {
+
+constexpr const char* kDisableHnswBruteForceEnv = "KNOWHERE_DISABLE_HNSW_BRUTE_FORCE";
+
+bool
+ParseTruthyEnv(const char* value) {
+    if (value == nullptr || value[0] == '\0') {
+        return false;
+    }
+    return strcasecmp(value, "1") == 0 || strcasecmp(value, "true") == 0 || strcasecmp(value, "yes") == 0 ||
+           strcasecmp(value, "on") == 0;
+}
+
+}  // namespace
+
+bool
+IsHnswBruteForceDisabledByEnv() {
+    static const bool disabled = ParseTruthyEnv(std::getenv(kDisableHnswBruteForceEnv));
+    return disabled;
+}
+
+}  // namespace knowhere

src/index/hnsw/impl/hnsw_brute_force_env.h

@@ -0,0 +1,27 @@
+// Copyright (C) 2019-2024 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+namespace knowhere {
+
+// Env: KNOWHERE_DISABLE_HNSW_BRUTE_FORCE
+// Truthy values: 1, true, yes, on (case-insensitive).
+//
+// When enabled, sealed HNSW segments never use IndexBruteForceWrapper:
+//   - upfront BF for selective filters / large topk
+//   - post-HNSW fallback when graph search returns too few results
+//
+// Evaluated once per process on first call (intended for query-node pod env).
+bool
+IsHnswBruteForceDisabledByEnv();
+
+}  // namespace knowhere

src/simd/hook.h

@@ -159,6 +159,20 @@ disable_patch_for_fp32_bf16();
 void
 fvec_hook(std::string&);
 
+namespace cppcontrib {
+namespace knowhere {
+#if defined(__x86_64__)
+using faiss::cpu_support_avx2;
+using faiss::cpu_support_avx512;
+using faiss::cpu_support_f16c;
+using faiss::cpu_support_sse4_2;
+#endif
+#if defined(__aarch64__)
+using faiss::supports_sve;
+#endif
+}  // namespace knowhere
+}  // namespace cppcontrib
+
 }  // namespace faiss
 
 #endif /* HOOK_H */

thirdparty/faiss/faiss/utils/distances.cpp

@@ -1015,7 +1015,7 @@ void exhaustive_L2sqr_nearest_imp(
         float dis_buffer[ny_batch_size];
         for (size_t i = i0; i < i1; i++) {
             const float* x_i = x + i * d;
-            size_t nearest_idx = 0;
+            int64_t nearest_idx = -1;
             float min_dis = HUGE_VALF;
             // compute distances
             for (auto j = 0; j < ny; j += ny_batch_size) {