From 680dc19200ecee0fea19024d70d06789618f8a94 Mon Sep 17 00:00:00 2001 From: happenlee Date: Fri, 17 Jul 2026 00:15:51 +0800 Subject: [PATCH 1/6] [improvement](be) Optimize bit unpacking with PDEP ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: BitPacking decoded every full 32-value batch with scalar shifts and masks even on x86 CPUs that support BMI2 and AVX2. Add a PDEP-based unpacker for uint8_t, uint16_t, and uint32_t batches, use AVX2 widening for narrow values, and retain the existing scalar path for unsupported CPUs, architectures, types, and remainder values. ### Release note Improve bit-packed integer decoding performance on BMI2 and AVX2 capable x86 CPUs. ### Check List (For Author) - Test: Unit Test - ./run-be-ut.sh -j 48 --run --filter=BitPackingTest.* - Behavior changed: Yes, full 32-value bit-unpack batches use PDEP and AVX2 when supported; decoded results and fallback behavior are unchanged. - Does this need documentation: No --- be/src/util/bit_packing.inline.h | 21 ++++- be/src/util/pdep_unpack.h | 152 ++++++++++++++++++++++++++++++ be/test/util/bit_packing_test.cpp | 108 +++++++++++++++++++++ 3 files changed, 280 insertions(+), 1 deletion(-) create mode 100644 be/src/util/pdep_unpack.h create mode 100644 be/test/util/bit_packing_test.cpp diff --git a/be/src/util/bit_packing.inline.h b/be/src/util/bit_packing.inline.h index b4e0805d8aebe0..7ad260602341a7 100644 --- a/be/src/util/bit_packing.inline.h +++ b/be/src/util/bit_packing.inline.h @@ -20,6 +20,9 @@ #include #include "util/bit_packing.h" +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) +#include "util/pdep_unpack.h" +#endif namespace doris { inline int64_t BitPacking::NumValuesToUnpack(int bit_width, int64_t in_bytes, int64_t num_values) { @@ -83,8 +86,24 @@ std::pair BitPacking::UnpackValues(const uint8_t* __res const uint8_t* in_pos = in; OutType* out_pos = out; +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) + int64_t batches_read = 0; + if constexpr (PdepUnpack::is_supported_type()) { + if (PdepUnpack::is_supported()) { + for (; batches_read < batches_to_read; ++batches_read) { + PdepUnpack::unpack32(in_pos, out_pos); + in_pos += (BATCH_SIZE * BIT_WIDTH) / CHAR_BIT; + out_pos += BATCH_SIZE; + in_bytes -= (BATCH_SIZE * BIT_WIDTH) / CHAR_BIT; + } + } + } +#else + constexpr int64_t batches_read = 0; +#endif + // First unpack as many full batches as possible. - for (int64_t i = 0; i < batches_to_read; ++i) { + for (int64_t i = batches_read; i < batches_to_read; ++i) { in_pos = Unpack32Values(in_pos, in_bytes, out_pos); out_pos += BATCH_SIZE; in_bytes -= (BATCH_SIZE * BIT_WIDTH) / CHAR_BIT; diff --git a/be/src/util/pdep_unpack.h b/be/src/util/pdep_unpack.h new file mode 100644 index 00000000000000..d68104ce7001f1 --- /dev/null +++ b/be/src/util/pdep_unpack.h @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) + +#include + +#include +#include +#include +#include +#include +#include + +namespace doris { + +class PdepUnpack { +public: + static bool is_supported() { + return __builtin_cpu_supports("bmi2") && __builtin_cpu_supports("avx2"); + } + + template + static constexpr bool is_supported_type() { + return BIT_WIDTH > 0 && BIT_WIDTH <= std::numeric_limits::digits && + (std::is_same_v || std::is_same_v || + std::is_same_v); + } + + template + __attribute__((target("bmi2,avx2"))) static void unpack32(const uint8_t* input, T* output) { + static_assert(is_supported_type()); + if constexpr (std::is_same_v) { + unpack32_with_pdep(input, output); + } else if constexpr (std::is_same_v && BIT_WIDTH <= 8) { + for (int group = 0; group < 2; ++group) { + const int first_value = group * 16; + const uint8_t* group_input = input + group * 2 * BIT_WIDTH; + uint64_t expanded0 = _pdep_u64(load_packed_group<0, 8 * BIT_WIDTH>(group_input), + pdep_mask()); + uint64_t expanded1 = + _pdep_u64(load_packed_group<8 * BIT_WIDTH, 8 * BIT_WIDTH>(group_input), + pdep_mask()); + __m128i packed8 = _mm_set_epi64x(static_cast(expanded1), + static_cast(expanded0)); + __m256i unpacked16 = _mm256_cvtepu8_epi16(packed8); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(output + first_value), unpacked16); + } + } else if constexpr (std::is_same_v && BIT_WIDTH <= 8) { + for (int group = 0; group < 4; ++group) { + uint64_t expanded = + _pdep_u64(load_packed_group<0, 8 * BIT_WIDTH>(input + group * BIT_WIDTH), + pdep_mask()); + __m256i unpacked32 = _mm256_cvtepu8_epi32(_mm_cvtsi64_si128(expanded)); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(output + group * 8), unpacked32); + } + } else if constexpr (std::is_same_v && BIT_WIDTH <= 15) { + for (int group = 0; group < 4; ++group) { + const int first_value = group * 8; + const uint8_t* group_input = input + group * BIT_WIDTH; + uint64_t expanded0 = _pdep_u64(load_packed_group<0, 4 * BIT_WIDTH>(group_input), + pdep_mask()); + uint64_t expanded1 = + _pdep_u64(load_packed_group<4 * BIT_WIDTH, 4 * BIT_WIDTH>(group_input), + pdep_mask()); + __m128i packed16 = _mm_set_epi64x(static_cast(expanded1), + static_cast(expanded0)); + __m256i unpacked32 = _mm256_cvtepu16_epi32(packed16); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(output + first_value), unpacked32); + } + } else { + unpack32_with_pdep(input, output); + } + } + +private: + template + static constexpr uint64_t pdep_mask() { + constexpr int lane_bits = sizeof(T) * 8; + constexpr int lanes = 64 / lane_bits; + constexpr uint64_t value_mask = (1ULL << BIT_WIDTH) - 1; + uint64_t mask = 0; + for (int lane = 0; lane < lanes; ++lane) { + mask |= value_mask << (lane * lane_bits); + } + return mask; + } + + template + static uint64_t load_packed_group(const uint8_t* input) { + constexpr int byte_offset = BIT_OFFSET / 8; + constexpr int shift = BIT_OFFSET % 8; + constexpr int bytes_needed = (shift + PACKED_BITS + 7) / 8; + static_assert(PACKED_BITS <= 64); + static_assert(bytes_needed <= 9); + + uint64_t low = 0; + std::memcpy(&low, input + byte_offset, bytes_needed < 8 ? bytes_needed : 8); + if constexpr (bytes_needed <= 8) { + return low >> shift; + } else { + uint8_t high = input[byte_offset + 8]; + return static_cast((static_cast(high) << 64 | low) >> + shift); + } + } + + template + __attribute__((target("bmi2"))) static void unpack_group(const uint8_t* input, T* output) { + constexpr int lanes = 64 / (sizeof(T) * 8); + constexpr int first_value = GROUP * lanes; + constexpr int bit_offset = first_value * BIT_WIDTH; + constexpr int packed_bits = lanes * BIT_WIDTH; + uint64_t packed = load_packed_group(input); + uint64_t expanded = _pdep_u64(packed, pdep_mask()); + std::memcpy(output + first_value, &expanded, sizeof(expanded)); + } + + template + __attribute__((target("bmi2"))) static void unpack32_with_pdep_impl( + const uint8_t* input, T* output, std::index_sequence) { + (unpack_group(input, output), ...); + } + + template + __attribute__((target("bmi2"))) static void unpack32_with_pdep(const uint8_t* input, + T* output) { + constexpr int lanes = 64 / (sizeof(T) * 8); + unpack32_with_pdep_impl(input, output, + std::make_index_sequence<32 / lanes> {}); + } +}; + +} // namespace doris + +#endif diff --git a/be/test/util/bit_packing_test.cpp b/be/test/util/bit_packing_test.cpp new file mode 100644 index 00000000000000..9b260d242c8235 --- /dev/null +++ b/be/test/util/bit_packing_test.cpp @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include + +#include "util/bit_stream_utils.inline.h" +#include "util/faststring.h" + +namespace doris { +namespace { + +template +std::vector make_values(int bit_width, int num_values) { + const uint64_t mask = bit_width == std::numeric_limits::digits + ? std::numeric_limits::max() + : (1ULL << bit_width) - 1; + std::vector values(num_values); + for (int i = 0; i < num_values; ++i) { + values[i] = static_cast((0x9E3779B9ULL * i + 0x7F4A7C15ULL) & mask); + } + return values; +} + +template +void pack_values(const std::vector& values, int bit_width, faststring* packed) { + BitWriter writer(packed); + for (T value : values) { + writer.PutValue(value, bit_width); + } + writer.Flush(); +} + +template +void test_all_bit_widths() { + constexpr int num_values = 65; + for (int bit_width = 1; bit_width <= std::numeric_limits::digits; ++bit_width) { + std::vector expected = make_values(bit_width, num_values); + faststring packed; + pack_values(expected, bit_width, &packed); + std::vector actual(num_values); + + auto [end, values_read] = + BitPacking::UnpackValues(bit_width, reinterpret_cast(packed.data()), + packed.size(), num_values, actual.data()); + + EXPECT_EQ(values_read, num_values) << "bit_width=" << bit_width; + EXPECT_EQ(end, reinterpret_cast(packed.data()) + packed.size()) + << "bit_width=" << bit_width; + EXPECT_EQ(actual, expected) << "bit_width=" << bit_width; + } +} + +template +void test_truncated_input() { + constexpr int num_values = 65; + for (int bit_width = 1; bit_width <= std::numeric_limits::digits; ++bit_width) { + std::vector expected = make_values(bit_width, num_values); + faststring packed; + pack_values(expected, bit_width, &packed); + const int64_t input_bytes = packed.size() - 1; + const int64_t expected_values_read = input_bytes * 8 / bit_width; + const int64_t expected_bytes_read = (expected_values_read * bit_width + 7) / 8; + std::vector actual(num_values, std::numeric_limits::max()); + + auto [end, values_read] = + BitPacking::UnpackValues(bit_width, reinterpret_cast(packed.data()), + input_bytes, num_values, actual.data()); + + EXPECT_EQ(values_read, expected_values_read) << "bit_width=" << bit_width; + EXPECT_EQ(end, reinterpret_cast(packed.data()) + expected_bytes_read) + << "bit_width=" << bit_width; + EXPECT_TRUE(std::equal(expected.begin(), expected.begin() + expected_values_read, + actual.begin())) + << "bit_width=" << bit_width; + } +} + +TEST(BitPackingTest, PdepUnpackAllBitWidths) { + test_all_bit_widths(); + test_all_bit_widths(); + test_all_bit_widths(); +} + +TEST(BitPackingTest, PdepUnpackTruncatedInput) { + test_truncated_input(); + test_truncated_input(); + test_truncated_input(); +} + +} // namespace +} // namespace doris From f45049ce019c9120d7cca5b2778d0a99a3abab02 Mon Sep 17 00:00:00 2001 From: happenlee Date: Fri, 17 Jul 2026 00:45:57 +0800 Subject: [PATCH 2/6] [test](be) Add PDEP unpack benchmark ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Add a benchmark that compares scalar 32-value bit unpacking with the production PDEP and AVX2 implementation for uint8_t, uint16_t, and uint32_t across every supported bit width. Validate scalar and PDEP outputs before measuring either implementation. ### Release note None ### Check List (For Author) - Test: Manual test - ./build.sh --benchmark -j 48 - benchmark_test --benchmark_filter=BM_.*Unpack --benchmark_min_time=0.01s - build-support/run-clang-tidy.sh --build-dir be/build_RELEASE --files be/benchmark/benchmark_main.cpp - Behavior changed: No - Does this need documentation: No --- be/benchmark/benchmark_main.cpp | 1 + be/benchmark/benchmark_pdep_unpack.hpp | 181 +++++++++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 be/benchmark/benchmark_pdep_unpack.hpp diff --git a/be/benchmark/benchmark_main.cpp b/be/benchmark/benchmark_main.cpp index efb2930f8364fb..d68621982bd525 100644 --- a/be/benchmark/benchmark_main.cpp +++ b/be/benchmark/benchmark_main.cpp @@ -29,6 +29,7 @@ #include "benchmark_fmod.hpp" #include "benchmark_hll_merge.hpp" #include "benchmark_hybrid_set.hpp" +#include "benchmark_pdep_unpack.hpp" #include "benchmark_string.hpp" #include "benchmark_string_replace.hpp" #include "benchmark_zone_map_index.hpp" diff --git a/be/benchmark/benchmark_pdep_unpack.hpp b/be/benchmark/benchmark_pdep_unpack.hpp new file mode 100644 index 00000000000000..ca4513c8a605e8 --- /dev/null +++ b/be/benchmark/benchmark_pdep_unpack.hpp @@ -0,0 +1,181 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) + +#include + +#include +#include +#include + +#include "util/bit_stream_utils.inline.h" +#include "util/pdep_unpack.h" + +namespace doris { +namespace { + +constexpr int kPdepUnpackNumValues = 4096; +constexpr int kPdepUnpackBatchSize = 32; + +template +void scalar_unpack(const uint8_t* input, int num_values, T* output) { + constexpr int bytes_per_batch = kPdepUnpackBatchSize * BIT_WIDTH / 8; + for (int i = 0; i < num_values; i += kPdepUnpackBatchSize) { + BitPacking::Unpack32Values(input, bytes_per_batch, output + i); + input += bytes_per_batch; + } +} + +template +void pdep_unpack(const uint8_t* input, int num_values, T* output) { + for (int i = 0; i < num_values; i += kPdepUnpackBatchSize) { + PdepUnpack::unpack32(input, output + i); + input += kPdepUnpackBatchSize * BIT_WIDTH / 8; + } +} + +template +void unpack_if_supported(const uint8_t* input, int num_values, T* output) { + if constexpr (PdepUnpack::is_supported_type()) { + if constexpr (USE_PDEP) { + pdep_unpack(input, num_values, output); + } else { + scalar_unpack(input, num_values, output); + } + } else { + __builtin_unreachable(); + } +} + +template +void unpack(int bit_width, const uint8_t* input, int num_values, T* output) { +#define UNPACK_CASE(width) \ + case width: \ + unpack_if_supported(input, num_values, output); \ + return + switch (bit_width) { + UNPACK_CASE(1); + UNPACK_CASE(2); + UNPACK_CASE(3); + UNPACK_CASE(4); + UNPACK_CASE(5); + UNPACK_CASE(6); + UNPACK_CASE(7); + UNPACK_CASE(8); + UNPACK_CASE(9); + UNPACK_CASE(10); + UNPACK_CASE(11); + UNPACK_CASE(12); + UNPACK_CASE(13); + UNPACK_CASE(14); + UNPACK_CASE(15); + UNPACK_CASE(16); + UNPACK_CASE(17); + UNPACK_CASE(18); + UNPACK_CASE(19); + UNPACK_CASE(20); + UNPACK_CASE(21); + UNPACK_CASE(22); + UNPACK_CASE(23); + UNPACK_CASE(24); + UNPACK_CASE(25); + UNPACK_CASE(26); + UNPACK_CASE(27); + UNPACK_CASE(28); + UNPACK_CASE(29); + UNPACK_CASE(30); + UNPACK_CASE(31); + UNPACK_CASE(32); + default: + __builtin_unreachable(); + } +#undef UNPACK_CASE +} + +template +struct PdepUnpackBenchmarkData { + explicit PdepUnpackBenchmarkData(int bit_width) + : input(kPdepUnpackNumValues * bit_width / 8), + scalar_output(kPdepUnpackNumValues), + pdep_output(kPdepUnpackNumValues) { + std::mt19937_64 rng(0x17993); + for (auto& byte : input) { + byte = static_cast(rng()); + } + unpack(bit_width, input.data(), kPdepUnpackNumValues, scalar_output.data()); + unpack(bit_width, input.data(), kPdepUnpackNumValues, pdep_output.data()); + } + + std::vector input; + std::vector scalar_output; + std::vector pdep_output; +}; + +template +void BM_ScalarUnpack(benchmark::State& state) { + const int bit_width = static_cast(state.range(0)); + if (!PdepUnpack::is_supported()) { + state.SkipWithError("CPU does not support BMI2 and AVX2"); + return; + } + PdepUnpackBenchmarkData data(bit_width); + if (data.scalar_output != data.pdep_output) { + state.SkipWithError("PDEP+AVX2 output differs from scalar output"); + return; + } + for (auto _ : state) { + unpack(bit_width, data.input.data(), kPdepUnpackNumValues, + data.scalar_output.data()); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * kPdepUnpackNumValues); +} + +template +void BM_PdepUnpack(benchmark::State& state) { + const int bit_width = static_cast(state.range(0)); + if (!PdepUnpack::is_supported()) { + state.SkipWithError("CPU does not support BMI2 and AVX2"); + return; + } + PdepUnpackBenchmarkData data(bit_width); + if (data.scalar_output != data.pdep_output) { + state.SkipWithError("PDEP+AVX2 output differs from scalar output"); + return; + } + for (auto _ : state) { + unpack(bit_width, data.input.data(), kPdepUnpackNumValues, + data.pdep_output.data()); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * kPdepUnpackNumValues); +} + +BENCHMARK_TEMPLATE(BM_ScalarUnpack, uint8_t)->DenseRange(1, 8); +BENCHMARK_TEMPLATE(BM_PdepUnpack, uint8_t)->DenseRange(1, 8); +BENCHMARK_TEMPLATE(BM_ScalarUnpack, uint16_t)->DenseRange(1, 16); +BENCHMARK_TEMPLATE(BM_PdepUnpack, uint16_t)->DenseRange(1, 16); +BENCHMARK_TEMPLATE(BM_ScalarUnpack, uint32_t)->DenseRange(1, 32); +BENCHMARK_TEMPLATE(BM_PdepUnpack, uint32_t)->DenseRange(1, 32); + +} // namespace +} // namespace doris + +#endif From a43d68a37bbaf099d124ff0a1a5933b0f6eeff32 Mon Sep 17 00:00:00 2001 From: happenlee Date: Fri, 17 Jul 2026 01:36:58 +0800 Subject: [PATCH 3/6] [test](be) Cover multiple working sets in PDEP benchmark ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Parameterize the PDEP unpack benchmark with 4K, 256K, and 1M values, compare the scalar kernel, direct PDEP kernel, and the actual BitPacking entry point, and validate optimized output against the scalar reference before measuring. ### Release note None ### Check List (For Author) - Test: Manual test - Built benchmark_test and ran the uint32_t 256K/1M scalar, PDEP, and actual-path benchmark cases - Behavior changed: No - Does this need documentation: No --- be/benchmark/benchmark_pdep_unpack.hpp | 81 +++++++++++++++++--------- 1 file changed, 54 insertions(+), 27 deletions(-) diff --git a/be/benchmark/benchmark_pdep_unpack.hpp b/be/benchmark/benchmark_pdep_unpack.hpp index ca4513c8a605e8..9ad3794232a9dd 100644 --- a/be/benchmark/benchmark_pdep_unpack.hpp +++ b/be/benchmark/benchmark_pdep_unpack.hpp @@ -31,8 +31,10 @@ namespace doris { namespace { -constexpr int kPdepUnpackNumValues = 4096; constexpr int kPdepUnpackBatchSize = 32; +constexpr int kPdepUnpackL1NumValues = 1 << 12; +constexpr int kPdepUnpackL2NumValues = 1 << 18; +constexpr int kPdepUnpackLargeNumValues = 1 << 20; template void scalar_unpack(const uint8_t* input, int num_values, T* output) { @@ -111,16 +113,15 @@ void unpack(int bit_width, const uint8_t* input, int num_values, T* output) { template struct PdepUnpackBenchmarkData { - explicit PdepUnpackBenchmarkData(int bit_width) - : input(kPdepUnpackNumValues * bit_width / 8), - scalar_output(kPdepUnpackNumValues), - pdep_output(kPdepUnpackNumValues) { + PdepUnpackBenchmarkData(int bit_width, int num_values) + : input(num_values * bit_width / 8), + scalar_output(num_values), + pdep_output(num_values) { std::mt19937_64 rng(0x17993); for (auto& byte : input) { byte = static_cast(rng()); } - unpack(bit_width, input.data(), kPdepUnpackNumValues, scalar_output.data()); - unpack(bit_width, input.data(), kPdepUnpackNumValues, pdep_output.data()); + unpack(bit_width, input.data(), num_values, scalar_output.data()); } std::vector input; @@ -131,49 +132,75 @@ struct PdepUnpackBenchmarkData { template void BM_ScalarUnpack(benchmark::State& state) { const int bit_width = static_cast(state.range(0)); + const int num_values = static_cast(state.range(1)); + PdepUnpackBenchmarkData data(bit_width, num_values); + for (auto _ : state) { + unpack(bit_width, data.input.data(), num_values, data.scalar_output.data()); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * num_values); +} + +template +void BM_PdepUnpack(benchmark::State& state) { + const int bit_width = static_cast(state.range(0)); + const int num_values = static_cast(state.range(1)); if (!PdepUnpack::is_supported()) { state.SkipWithError("CPU does not support BMI2 and AVX2"); return; } - PdepUnpackBenchmarkData data(bit_width); + PdepUnpackBenchmarkData data(bit_width, num_values); + unpack(bit_width, data.input.data(), num_values, data.pdep_output.data()); if (data.scalar_output != data.pdep_output) { state.SkipWithError("PDEP+AVX2 output differs from scalar output"); return; } for (auto _ : state) { - unpack(bit_width, data.input.data(), kPdepUnpackNumValues, - data.scalar_output.data()); + unpack(bit_width, data.input.data(), num_values, data.pdep_output.data()); benchmark::ClobberMemory(); } - state.SetItemsProcessed(state.iterations() * kPdepUnpackNumValues); + state.SetItemsProcessed(state.iterations() * num_values); } template -void BM_PdepUnpack(benchmark::State& state) { +void BM_ActualUnpack(benchmark::State& state) { const int bit_width = static_cast(state.range(0)); - if (!PdepUnpack::is_supported()) { - state.SkipWithError("CPU does not support BMI2 and AVX2"); - return; - } - PdepUnpackBenchmarkData data(bit_width); + const int num_values = static_cast(state.range(1)); + PdepUnpackBenchmarkData data(bit_width, num_values); + auto result = BitPacking::UnpackValues(bit_width, data.input.data(), data.input.size(), + num_values, data.pdep_output.data()); if (data.scalar_output != data.pdep_output) { - state.SkipWithError("PDEP+AVX2 output differs from scalar output"); + state.SkipWithError("Actual output differs from scalar output"); return; } for (auto _ : state) { - unpack(bit_width, data.input.data(), kPdepUnpackNumValues, - data.pdep_output.data()); + result = BitPacking::UnpackValues(bit_width, data.input.data(), data.input.size(), + num_values, data.pdep_output.data()); + benchmark::DoNotOptimize(result); benchmark::ClobberMemory(); } - state.SetItemsProcessed(state.iterations() * kPdepUnpackNumValues); + state.SetItemsProcessed(state.iterations() * num_values); } -BENCHMARK_TEMPLATE(BM_ScalarUnpack, uint8_t)->DenseRange(1, 8); -BENCHMARK_TEMPLATE(BM_PdepUnpack, uint8_t)->DenseRange(1, 8); -BENCHMARK_TEMPLATE(BM_ScalarUnpack, uint16_t)->DenseRange(1, 16); -BENCHMARK_TEMPLATE(BM_PdepUnpack, uint16_t)->DenseRange(1, 16); -BENCHMARK_TEMPLATE(BM_ScalarUnpack, uint32_t)->DenseRange(1, 32); -BENCHMARK_TEMPLATE(BM_PdepUnpack, uint32_t)->DenseRange(1, 32); +#define REGISTER_PDEP_UNPACK_BENCHMARK(type, max_bit_width) \ + BENCHMARK_TEMPLATE(BM_ScalarUnpack, type) \ + ->ArgsProduct({benchmark::CreateDenseRange(1, max_bit_width, 1), \ + {kPdepUnpackL1NumValues, kPdepUnpackL2NumValues, \ + kPdepUnpackLargeNumValues}}); \ + BENCHMARK_TEMPLATE(BM_PdepUnpack, type) \ + ->ArgsProduct({benchmark::CreateDenseRange(1, max_bit_width, 1), \ + {kPdepUnpackL1NumValues, kPdepUnpackL2NumValues, \ + kPdepUnpackLargeNumValues}}); \ + BENCHMARK_TEMPLATE(BM_ActualUnpack, type) \ + ->ArgsProduct( \ + {benchmark::CreateDenseRange(1, max_bit_width, 1), \ + {kPdepUnpackL1NumValues, kPdepUnpackL2NumValues, kPdepUnpackLargeNumValues}}) + +REGISTER_PDEP_UNPACK_BENCHMARK(uint8_t, 8); +REGISTER_PDEP_UNPACK_BENCHMARK(uint16_t, 16); +REGISTER_PDEP_UNPACK_BENCHMARK(uint32_t, 32); + +#undef REGISTER_PDEP_UNPACK_BENCHMARK } // namespace } // namespace doris From 48c64e196344b78b411df4508db91bd9a2cac564 Mon Sep 17 00:00:00 2001 From: happenlee Date: Fri, 17 Jul 2026 02:35:12 +0800 Subject: [PATCH 4/6] [improvement](be) Limit PDEP unpack to narrow bit widths ### What problem does this PR solve? Issue Number: None Related PR: #65738 Problem Summary: The generic PDEP unpack implementation remains functionally valid for 16-32 bit values, but repeated Release benchmarks at 4K, 256K, and 1M values showed non-monotonic performance and repeatable regressions at several high widths. Keep the generic implementation available to the benchmark, while limiting the production dispatch to BIT_WIDTH < 16. Wider values conservatively retain the scalar implementation instead of using a CPU- and working-set-specific width allowlist. ### Release note Use the PDEP bit-unpack optimization only for bit widths below 16. ### Check List (For Author) - Test: Unit Test / Manual test - ./run-be-ut.sh -j 48 --run --filter=BitPackingTest.* - Release benchmark_test for uint32_t widths 16-32 at 4K, 256K, and 1M values - build-support/check-format.sh - build-support/run-clang-tidy.sh --build-dir be/build_Release --files be/benchmark/benchmark_main.cpp be/test/util/bit_packing_test.cpp - Behavior changed: Yes, bit widths 16 and above retain the scalar unpack path - Does this need documentation: No --- be/src/util/bit_packing.inline.h | 2 +- be/src/util/pdep_unpack.h | 14 ++++++++++++++ be/test/util/bit_packing_test.cpp | 8 ++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/be/src/util/bit_packing.inline.h b/be/src/util/bit_packing.inline.h index 7ad260602341a7..ce7aade4b3e42c 100644 --- a/be/src/util/bit_packing.inline.h +++ b/be/src/util/bit_packing.inline.h @@ -88,7 +88,7 @@ std::pair BitPacking::UnpackValues(const uint8_t* __res #if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) int64_t batches_read = 0; - if constexpr (PdepUnpack::is_supported_type()) { + if constexpr (PdepUnpack::should_use()) { if (PdepUnpack::is_supported()) { for (; batches_read < batches_to_read; ++batches_read) { PdepUnpack::unpack32(in_pos, out_pos); diff --git a/be/src/util/pdep_unpack.h b/be/src/util/pdep_unpack.h index d68104ce7001f1..f50c518deed949 100644 --- a/be/src/util/pdep_unpack.h +++ b/be/src/util/pdep_unpack.h @@ -43,6 +43,20 @@ class PdepUnpack { std::is_same_v); } + template + static constexpr bool should_use() { + // Keep the generic implementation available for benchmarking all supported widths, but + // only select PDEP in the production path for widths below 16. These widths can use the + // byte/word deposit layouts and the AVX2 widening specializations below. At 16 bits and + // above, unpack32() falls back to multiple generic 64-bit PDEP groups per batch and + // competes with efficient scalar specializations, including copy-like 16- and 32-bit + // cases. Benchmarks with L1-, L2-, and larger working sets show non-monotonic results and + // repeatable regressions for multiple high widths. Because the profitable high widths are + // CPU- and working-set-dependent, an irregular per-width allowlist would not be portable; + // use the scalar implementation conservatively instead. + return is_supported_type() && BIT_WIDTH < 16; + } + template __attribute__((target("bmi2,avx2"))) static void unpack32(const uint8_t* input, T* output) { static_assert(is_supported_type()); diff --git a/be/test/util/bit_packing_test.cpp b/be/test/util/bit_packing_test.cpp index 9b260d242c8235..bea1c237e175be 100644 --- a/be/test/util/bit_packing_test.cpp +++ b/be/test/util/bit_packing_test.cpp @@ -26,6 +26,14 @@ namespace doris { namespace { +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) +static_assert(PdepUnpack::should_use()); +static_assert(PdepUnpack::should_use()); +static_assert(!PdepUnpack::should_use()); +static_assert(!PdepUnpack::should_use()); +static_assert(!PdepUnpack::should_use()); +#endif + template std::vector make_values(int bit_width, int num_values) { const uint64_t mask = bit_width == std::numeric_limits::digits From cb53beb058be9595a9d8cb797180b7a5d3c42519 Mon Sep 17 00:00:00 2001 From: happenlee Date: Fri, 17 Jul 2026 10:57:27 +0800 Subject: [PATCH 5/6] [fix](be) Allow disabling BMI2 optimizations ### What problem does this PR solve? Issue Number: None Related PR: #65738 Problem Summary: PDEP is supported by AMD Zen+ and Zen 2, but those CPUs execute it as a slow microcoded instruction. Add a BE configuration switch so deployments on those CPUs can retain the scalar bit-unpacking path, while keeping BMI2 optimized paths enabled by default. The generic BMI2 name also allows future BMI2 instruction optimizations to share the same deployment-level switch. ### Release note Set enable_bmi2_optimizations=false on CPUs where BMI2 optimizations are slower than scalar implementations. ### Check List (For Author) - Test: Unit Test - Compiled config.cpp and BitPackingTest objects in ASAN-UT. - Added a unit test that verifies enable_bmi2_optimizations=false disables PDEP dispatch. - Behavior changed: Yes (adds a BE configuration switch; default behavior remains enabled) - Does this need documentation: No --- be/src/common/config.cpp | 4 ++++ be/src/common/config.h | 1 + be/src/util/pdep_unpack.h | 5 ++++- be/test/util/bit_packing_test.cpp | 9 +++++++++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index b2d51ecbb6d19a..32823fb06ffb1a 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -467,6 +467,10 @@ DEFINE_mDouble(sparse_column_compaction_threshold_percent, "0.05"); // Enable RLE batch Put optimization for compaction DEFINE_mBool(enable_rle_batch_put_optimization, "true"); +// Enable PDEP-based bit unpacking. Disable it on CPUs where PDEP is microcoded and slower than +// the scalar implementation, such as AMD Zen+ and Zen 2. +DEFINE_Bool(enable_bmi2_optimizations, "true"); + // If enabled, segments will be flushed column by column DEFINE_mBool(enable_vertical_segment_writer, "true"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 25c7a8ef89d1e1..55f11615bd19d9 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -536,6 +536,7 @@ DECLARE_mInt64(vertical_compaction_max_segment_size); DECLARE_mDouble(sparse_column_compaction_threshold_percent); // Enable RLE batch Put optimization for compaction DECLARE_mBool(enable_rle_batch_put_optimization); +DECLARE_Bool(enable_bmi2_optimizations); // If enabled, segments will be flushed column by column DECLARE_mBool(enable_vertical_segment_writer); diff --git a/be/src/util/pdep_unpack.h b/be/src/util/pdep_unpack.h index f50c518deed949..10403d94e4a158 100644 --- a/be/src/util/pdep_unpack.h +++ b/be/src/util/pdep_unpack.h @@ -28,12 +28,15 @@ #include #include +#include "common/config.h" + namespace doris { class PdepUnpack { public: static bool is_supported() { - return __builtin_cpu_supports("bmi2") && __builtin_cpu_supports("avx2"); + return config::enable_bmi2_optimizations && __builtin_cpu_supports("bmi2") && + __builtin_cpu_supports("avx2"); } template diff --git a/be/test/util/bit_packing_test.cpp b/be/test/util/bit_packing_test.cpp index bea1c237e175be..9bab326c85f553 100644 --- a/be/test/util/bit_packing_test.cpp +++ b/be/test/util/bit_packing_test.cpp @@ -112,5 +112,14 @@ TEST(BitPackingTest, PdepUnpackTruncatedInput) { test_truncated_input(); } +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) +TEST(BitPackingTest, DisableBmi2Optimizations) { + const bool was_enabled = config::enable_bmi2_optimizations; + config::enable_bmi2_optimizations = false; + EXPECT_FALSE(PdepUnpack::is_supported()); + config::enable_bmi2_optimizations = was_enabled; +} +#endif + } // namespace } // namespace doris From 790ad0cafb05c0a778c779794161892b8011846f Mon Sep 17 00:00:00 2001 From: happenlee Date: Fri, 17 Jul 2026 15:39:33 +0800 Subject: [PATCH 6/6] [fix](be) Skip PDEP probe for partial batches ### What problem does this PR solve? Issue Number: None Related PR: #65738 Problem Summary: BitPacking::UnpackValues called PdepUnpack::is_supported() even when fewer than 32 values were available. Parquet hybrid-RLE can send 8-, 16-, and 24-value literal tails through this path, so these scalar-only calls unnecessarily loaded the BMI2 configuration and probed BMI2/AVX2 support. Short-circuit the feature probe when batches_to_read is zero so partial batches continue directly to the unchanged scalar remainder decoder. ### Release note None ### Check List (For Author) - Test: Unit Test / Static Analysis - Compiled be/ut_build_ASAN/test/CMakeFiles/doris_be_test.dir/util/bit_packing_test.cpp.o - Attempted ./run-be-ut.sh -j 48 --run --filter=BitPackingTest.*; stopped because the local CMake state triggered an unrelated 2,259-target full rebuild - PATH=/mnt/disk6/common/ldb_toolchain_toucan/bin:$PATH build-support/check-format.sh - build-support/run-clang-tidy.sh --build-dir be/ut_build_ASAN --files be/src/util/bit_packing.inline.h - Behavior changed: No (partial batches keep the scalar decoder while avoiding unnecessary runtime feature dispatch) - Does this need documentation: No --- be/src/util/bit_packing.inline.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/util/bit_packing.inline.h b/be/src/util/bit_packing.inline.h index ce7aade4b3e42c..61e4caebf72ecc 100644 --- a/be/src/util/bit_packing.inline.h +++ b/be/src/util/bit_packing.inline.h @@ -89,7 +89,7 @@ std::pair BitPacking::UnpackValues(const uint8_t* __res #if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) int64_t batches_read = 0; if constexpr (PdepUnpack::should_use()) { - if (PdepUnpack::is_supported()) { + if (batches_to_read > 0 && PdepUnpack::is_supported()) { for (; batches_read < batches_to_read; ++batches_read) { PdepUnpack::unpack32(in_pos, out_pos); in_pos += (BATCH_SIZE * BIT_WIDTH) / CHAR_BIT;