Skip to content

[improvement](be) Optimize bit unpacking with PDEP #65738

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
HappenLee wants to merge 6 commits into
base: master
Choose a base branch
from
Open

[improvement](be) Optimize bit unpacking with PDEP #65738

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
680dc19
[improvement](be) Optimize bit unpacking with PDEP
HappenLee Jul 16, 2026
f45049c
[test](be) Add PDEP unpack benchmark
HappenLee Jul 16, 2026
a43d68a
[test](be) Cover multiple working sets in PDEP benchmark
HappenLee Jul 16, 2026
48c64e1
[improvement](be) Limit PDEP unpack to narrow bit widths
HappenLee Jul 16, 2026
cb53beb
[fix](be) Allow disabling BMI2 optimizations
HappenLee Jul 17, 2026
790ad0c
[fix](be) Skip PDEP probe for partial batches
HappenLee Jul 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions be/benchmark/benchmark_main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
#include "benchmark_fmod.hpp"
#include "benchmark_hll_merge.hpp"
#include "benchmark_hybrid_set.hpp"
#include "benchmark_pdep_unpack.hpp"
#include "benchmark_string.hpp"
#include "benchmark_string_replace.hpp"
#include "benchmark_zone_map_index.hpp"
Expand Down
208 changes: 208 additions & 0 deletions be/benchmark/benchmark_pdep_unpack.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,208 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#pragma once

#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__))

#include <benchmark/benchmark.h>

#include <cstdint>
#include <random>
#include <vector>

#include "util/bit_stream_utils.inline.h"
#include "util/pdep_unpack.h"

namespace doris {
namespace {

constexpr int kPdepUnpackBatchSize = 32;
constexpr int kPdepUnpackL1NumValues = 1 << 12;
constexpr int kPdepUnpackL2NumValues = 1 << 18;
constexpr int kPdepUnpackLargeNumValues = 1 << 20;

template <typename T, int BIT_WIDTH>
void scalar_unpack(const uint8_t* input, int num_values, T* output) {
constexpr int bytes_per_batch = kPdepUnpackBatchSize * BIT_WIDTH / 8;
for (int i = 0; i < num_values; i += kPdepUnpackBatchSize) {
BitPacking::Unpack32Values<T, BIT_WIDTH>(input, bytes_per_batch, output + i);
input += bytes_per_batch;
}
}

template <typename T, int BIT_WIDTH>
void pdep_unpack(const uint8_t* input, int num_values, T* output) {
for (int i = 0; i < num_values; i += kPdepUnpackBatchSize) {
PdepUnpack::unpack32<T, BIT_WIDTH>(input, output + i);
input += kPdepUnpackBatchSize * BIT_WIDTH / 8;
}
}

template <typename T, int BIT_WIDTH, bool USE_PDEP>
void unpack_if_supported(const uint8_t* input, int num_values, T* output) {
if constexpr (PdepUnpack::is_supported_type<T, BIT_WIDTH>()) {
if constexpr (USE_PDEP) {
pdep_unpack<T, BIT_WIDTH>(input, num_values, output);
} else {
scalar_unpack<T, BIT_WIDTH>(input, num_values, output);
}
} else {
__builtin_unreachable();
}
}

template <typename T, bool USE_PDEP>
void unpack(int bit_width, const uint8_t* input, int num_values, T* output) {
#define UNPACK_CASE(width) \
case width: \
unpack_if_supported<T, width, USE_PDEP>(input, num_values, output); \
return
switch (bit_width) {
UNPACK_CASE(1);
UNPACK_CASE(2);
UNPACK_CASE(3);
UNPACK_CASE(4);
UNPACK_CASE(5);
UNPACK_CASE(6);
UNPACK_CASE(7);
UNPACK_CASE(8);
UNPACK_CASE(9);
UNPACK_CASE(10);
UNPACK_CASE(11);
UNPACK_CASE(12);
UNPACK_CASE(13);
UNPACK_CASE(14);
UNPACK_CASE(15);
UNPACK_CASE(16);
UNPACK_CASE(17);
UNPACK_CASE(18);
UNPACK_CASE(19);
UNPACK_CASE(20);
UNPACK_CASE(21);
UNPACK_CASE(22);
UNPACK_CASE(23);
UNPACK_CASE(24);
UNPACK_CASE(25);
UNPACK_CASE(26);
UNPACK_CASE(27);
UNPACK_CASE(28);
UNPACK_CASE(29);
UNPACK_CASE(30);
UNPACK_CASE(31);
UNPACK_CASE(32);
default:
__builtin_unreachable();
}
#undef UNPACK_CASE
}

template <typename T>
struct PdepUnpackBenchmarkData {
PdepUnpackBenchmarkData(int bit_width, int num_values)
: input(num_values * bit_width / 8),
scalar_output(num_values),
pdep_output(num_values) {
std::mt19937_64 rng(0x17993);
for (auto& byte : input) {
byte = static_cast<uint8_t>(rng());
}
unpack<T, false>(bit_width, input.data(), num_values, scalar_output.data());
}

std::vector<uint8_t> input;
std::vector<T> scalar_output;
std::vector<T> pdep_output;
};

template <typename T>
void BM_ScalarUnpack(benchmark::State& state) {
const int bit_width = static_cast<int>(state.range(0));
const int num_values = static_cast<int>(state.range(1));
PdepUnpackBenchmarkData<T> data(bit_width, num_values);
for (auto _ : state) {
unpack<T, false>(bit_width, data.input.data(), num_values, data.scalar_output.data());
benchmark::ClobberMemory();
}
state.SetItemsProcessed(state.iterations() * num_values);
}

template <typename T>
void BM_PdepUnpack(benchmark::State& state) {
const int bit_width = static_cast<int>(state.range(0));
const int num_values = static_cast<int>(state.range(1));
if (!PdepUnpack::is_supported()) {

@github-actions github-actions Bot Jul 17, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Initialize config before running these benchmarks

DEFINE_Bool leaves its namespace-scope storage zero-initialized; Register only records the "true" default, and config::init() is what later assigns it. benchmark_main.cpp never calls config::init(), so this predicate is always false in the benchmark process: every direct PDEP case is skipped, while every BM_ActualUnpack case silently measures the scalar fallback. Because the config guard was added after the published runs, the current benchmark can no longer reproduce the evidence used to select production dispatch. Initialize registered defaults before RunSpecifiedBenchmarks() (or explicitly enable this flag for the direct/actual cases) and verify that the actual-path cases enter PDEP on capable hardware.

state.SkipWithError("CPU does not support BMI2 and AVX2");
return;
}
PdepUnpackBenchmarkData<T> data(bit_width, num_values);
unpack<T, true>(bit_width, data.input.data(), num_values, data.pdep_output.data());
if (data.scalar_output != data.pdep_output) {
state.SkipWithError("PDEP+AVX2 output differs from scalar output");
return;
}
for (auto _ : state) {
unpack<T, true>(bit_width, data.input.data(), num_values, data.pdep_output.data());
benchmark::ClobberMemory();
}
state.SetItemsProcessed(state.iterations() * num_values);
}

template <typename T>
void BM_ActualUnpack(benchmark::State& state) {
const int bit_width = static_cast<int>(state.range(0));
const int num_values = static_cast<int>(state.range(1));
PdepUnpackBenchmarkData<T> data(bit_width, num_values);
auto result = BitPacking::UnpackValues(bit_width, data.input.data(), data.input.size(),
num_values, data.pdep_output.data());
if (data.scalar_output != data.pdep_output) {
state.SkipWithError("Actual output differs from scalar output");
return;
}
for (auto _ : state) {
result = BitPacking::UnpackValues(bit_width, data.input.data(), data.input.size(),
num_values, data.pdep_output.data());
benchmark::DoNotOptimize(result);
benchmark::ClobberMemory();
}
state.SetItemsProcessed(state.iterations() * num_values);
}

#define REGISTER_PDEP_UNPACK_BENCHMARK(type, max_bit_width) \
BENCHMARK_TEMPLATE(BM_ScalarUnpack, type) \
->ArgsProduct({benchmark::CreateDenseRange(1, max_bit_width, 1), \
{kPdepUnpackL1NumValues, kPdepUnpackL2NumValues, \
kPdepUnpackLargeNumValues}}); \
BENCHMARK_TEMPLATE(BM_PdepUnpack, type) \
->ArgsProduct({benchmark::CreateDenseRange(1, max_bit_width, 1), \
{kPdepUnpackL1NumValues, kPdepUnpackL2NumValues, \
kPdepUnpackLargeNumValues}}); \
BENCHMARK_TEMPLATE(BM_ActualUnpack, type) \
->ArgsProduct( \
{benchmark::CreateDenseRange(1, max_bit_width, 1), \
{kPdepUnpackL1NumValues, kPdepUnpackL2NumValues, kPdepUnpackLargeNumValues}})

REGISTER_PDEP_UNPACK_BENCHMARK(uint8_t, 8);
REGISTER_PDEP_UNPACK_BENCHMARK(uint16_t, 16);
REGISTER_PDEP_UNPACK_BENCHMARK(uint32_t, 32);

#undef REGISTER_PDEP_UNPACK_BENCHMARK

} // namespace
} // namespace doris

#endif
4 changes: 4 additions & 0 deletions be/src/common/config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -467,6 +467,10 @@ DEFINE_mDouble(sparse_column_compaction_threshold_percent, "0.05");
// Enable RLE batch Put optimization for compaction
DEFINE_mBool(enable_rle_batch_put_optimization, "true");

// Enable PDEP-based bit unpacking. Disable it on CPUs where PDEP is microcoded and slower than
// the scalar implementation, such as AMD Zen+ and Zen 2.
DEFINE_Bool(enable_bmi2_optimizations, "true");

// If enabled, segments will be flushed column by column
DEFINE_mBool(enable_vertical_segment_writer, "true");

Expand Down
1 change: 1 addition & 0 deletions be/src/common/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -536,6 +536,7 @@ DECLARE_mInt64(vertical_compaction_max_segment_size);
DECLARE_mDouble(sparse_column_compaction_threshold_percent);
// Enable RLE batch Put optimization for compaction
DECLARE_mBool(enable_rle_batch_put_optimization);
DECLARE_Bool(enable_bmi2_optimizations);

// If enabled, segments will be flushed column by column
DECLARE_mBool(enable_vertical_segment_writer);
Expand Down
21 changes: 20 additions & 1 deletion be/src/util/bit_packing.inline.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@
#include <boost/preprocessor/repetition/repeat_from_to.hpp>

#include "util/bit_packing.h"
#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__))
#include "util/pdep_unpack.h"
#endif

namespace doris {
inline int64_t BitPacking::NumValuesToUnpack(int bit_width, int64_t in_bytes, int64_t num_values) {
Expand Down Expand Up @@ -83,8 +86,24 @@ std::pair<const uint8_t*, int64_t> BitPacking::UnpackValues(const uint8_t* __res
const uint8_t* in_pos = in;
OutType* out_pos = out;

#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__))
int64_t batches_read = 0;
if constexpr (PdepUnpack::should_use<OutType, BIT_WIDTH>()) {
if (batches_to_read > 0 && PdepUnpack::is_supported()) {

@github-actions github-actions Bot Jul 17, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Keep marginal one-batch calls on the scalar path

The new batches_to_read > 0 guard fixes the already-reported zero-batch probe, but exact 32-value calls are a separate normal production shape: RleBatchDecoder can send one batch through its rounded direct bypass, and FillLiteralBuffer() is fixed at 32 values. For uint32_t width 8, the PR reports only 0.224 us scalar versus 0.221 us PDEP over 4,096 values—3 ns across 128 kernels, or about 0.023 ns saved per kernel before this per-call config load, two feature tests, and branch. An exact-one-batch call therefore cannot amortize the new dispatch, while the benchmark starts at 4,096 values and never measures 32/64/128-value actual-entry calls. Add those cases and retain scalar for one/few batches at marginal widths (at least width 8) unless end-to-end results show a win.

for (; batches_read < batches_to_read; ++batches_read) {
PdepUnpack::unpack32<OutType, BIT_WIDTH>(in_pos, out_pos);
in_pos += (BATCH_SIZE * BIT_WIDTH) / CHAR_BIT;
out_pos += BATCH_SIZE;
in_bytes -= (BATCH_SIZE * BIT_WIDTH) / CHAR_BIT;
}
}
}
#else
constexpr int64_t batches_read = 0;
#endif

// First unpack as many full batches as possible.
for (int64_t i = 0; i < batches_to_read; ++i) {
for (int64_t i = batches_read; i < batches_to_read; ++i) {
in_pos = Unpack32Values<OutType, BIT_WIDTH>(in_pos, in_bytes, out_pos);
out_pos += BATCH_SIZE;
in_bytes -= (BATCH_SIZE * BIT_WIDTH) / CHAR_BIT;
Expand Down
Loading
Loading