diff --git a/be/src/storage/segment/segment_writer.cpp b/be/src/storage/segment/segment_writer.cpp index 6f4ecef1140d67..d6e1e162d1fa8f 100644 --- a/be/src/storage/segment/segment_writer.cpp +++ b/be/src/storage/segment/segment_writer.cpp @@ -196,7 +196,7 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co opts.need_zone_map = column.is_key() || schema->keys_type() != KeysType::AGG_KEYS; opts.need_bloom_filter = column.is_bf_column(); if (opts.need_bloom_filter) { - opts.bf_options.fpp = schema->has_bf_fpp() ? schema->bloom_filter_fpp() : 0.05; + opts.bf_options.fpp = schema->get_bloom_filter_fpp(column); } auto* tablet_index = schema->get_ngram_bf_index(column.unique_id()); if (tablet_index) { diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.cpp b/be/src/storage/segment/variant/variant_column_writer_impl.cpp index 1e8610f42d886f..321386b95f9fe4 100644 --- a/be/src/storage/segment/variant/variant_column_writer_impl.cpp +++ b/be/src/storage/segment/variant/variant_column_writer_impl.cpp @@ -102,6 +102,9 @@ Status _create_column_writer(uint32_t cid, const TabletColumn& column, } opt->need_zone_map = tablet_schema->keys_type() != KeysType::AGG_KEYS; opt->need_bloom_filter = column.is_bf_column(); + if (opt->need_bloom_filter) { + opt->bf_options.fpp = tablet_schema->get_bloom_filter_fpp(column); + } const auto& parent_index = tablet_schema->inverted_indexs(column.parent_unique_id()); // init inverted index diff --git a/be/src/storage/segment/vertical_segment_writer.cpp b/be/src/storage/segment/vertical_segment_writer.cpp index f5b60ebadb756c..f625fcae396399 100644 --- a/be/src/storage/segment/vertical_segment_writer.cpp +++ b/be/src/storage/segment/vertical_segment_writer.cpp @@ -204,8 +204,7 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo opts.need_zone_map = column.is_key() || tablet_schema->keys_type() != KeysType::AGG_KEYS; opts.need_bloom_filter = column.is_bf_column(); if (opts.need_bloom_filter) { - opts.bf_options.fpp = - tablet_schema->has_bf_fpp() ? tablet_schema->bloom_filter_fpp() : 0.05; + opts.bf_options.fpp = tablet_schema->get_bloom_filter_fpp(column); } auto* tablet_index = tablet_schema->get_ngram_bf_index(column.unique_id()); if (tablet_index) { diff --git a/be/src/storage/tablet/tablet_schema.cpp b/be/src/storage/tablet/tablet_schema.cpp index 3158b229837890..93012c55991686 100644 --- a/be/src/storage/tablet/tablet_schema.cpp +++ b/be/src/storage/tablet/tablet_schema.cpp @@ -27,11 +27,13 @@ #include #include // IWYU pragma: no_include +#include #include // IWYU pragma: keep #include #include #include +#include "common/check.h" #include "common/compiler_util.h" // IWYU pragma: keep #include "common/consts.h" #include "common/status.h" @@ -40,6 +42,7 @@ #include "core/data_type/data_type.h" #include "core/data_type/data_type_factory.hpp" #include "core/string_ref.h" +#include "core/types.h" #include "exec/common/hex.h" #include "exprs/aggregate/aggregate_function_simple_factory.h" #include "exprs/aggregate/aggregate_function_state_union.h" @@ -1868,6 +1871,30 @@ const TabletIndex* TabletSchema::get_ngram_bf_index(int32_t col_unique_id) const return nullptr; } +double TabletSchema::get_bloom_filter_fpp(int32_t col_unique_id) const { + const auto* bloom_filter_index = get_index(col_unique_id, IndexType::BLOOMFILTER, ""); + if (bloom_filter_index != nullptr) { + const auto& properties = bloom_filter_index->properties(); + auto iter = properties.find("bloom_filter_fpp"); + if (iter != properties.end()) { + StringParser::ParseResult parse_result = StringParser::PARSE_FAILURE; + auto index_level_fpp = StringParser::string_to_float( + iter->second.data(), iter->second.size(), &parse_result); + DORIS_CHECK(parse_result == StringParser::PARSE_SUCCESS) + << "failed to parse '" << iter->second << "' as double"; + return index_level_fpp; + } + return BLOOM_FILTER_DEFAULT_FPP; + } + return has_bf_fpp() ? bloom_filter_fpp() : BLOOM_FILTER_DEFAULT_FPP; +} + +double TabletSchema::get_bloom_filter_fpp(const TabletColumn& column) const { + const int32_t col_unique_id = + column.is_extracted_column() ? column.parent_unique_id() : column.unique_id(); + return get_bloom_filter_fpp(col_unique_id); +} + const TabletIndex* TabletSchema::get_index(int32_t col_unique_id, IndexType index_type, const std::string& suffix_path) const { IndexKey index_key(index_type, col_unique_id, suffix_path); diff --git a/be/src/storage/tablet/tablet_schema.h b/be/src/storage/tablet/tablet_schema.h index bf4598e2bb94fd..3885d02cc4307f 100644 --- a/be/src/storage/tablet/tablet_schema.h +++ b/be/src/storage/tablet/tablet_schema.h @@ -598,6 +598,8 @@ class TabletSchema : public MetadataAdder { bool has_ngram_bf_index(int32_t col_unique_id) const; const TabletIndex* get_ngram_bf_index(int32_t col_unique_id) const; + double get_bloom_filter_fpp(int32_t col_unique_id) const; + double get_bloom_filter_fpp(const TabletColumn& column) const; const TabletIndex* get_index(int32_t col_unique_id, IndexType index_type, const std::string& suffix_path) const; void update_indexes_from_thrift(const std::vector& indexes); diff --git a/be/test/storage/segment/bloom_filter_fpp_writer_test.cpp b/be/test/storage/segment/bloom_filter_fpp_writer_test.cpp new file mode 100644 index 00000000000000..e2c5c84c734e6b --- /dev/null +++ b/be/test/storage/segment/bloom_filter_fpp_writer_test.cpp @@ -0,0 +1,364 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include + +#include "common/config.h" +#include "core/block/block.h" +#include "io/fs/local_file_system.h" +#include "storage/olap_common.h" +#include "storage/rowset/rowset_writer_context.h" +#include "storage/segment/column_writer.h" +#include "storage/segment/encoding_info.h" +#include "storage/segment/segment_writer.h" +#include "storage/segment/vertical_segment_writer.h" +#include "storage/tablet/tablet_schema.h" +#include "util/debug_points.h" + +namespace doris::segment_v2 { + +namespace { + +constexpr std::string_view kTestDir = "./ut_dir/bloom_filter_fpp_writer_test"; +constexpr std::string_view kBloomFilterCreateDebugPoint = "BloomFilterIndexWriter::create"; + +class ScopedBloomFilterFppDebugPoint { +public: + explicit ScopedBloomFilterFppDebugPoint(double expected_fpp) + : _old_enable_debug_points(config::enable_debug_points) { + config::enable_debug_points = true; + DebugPoints::instance()->add_with_params(std::string(kBloomFilterCreateDebugPoint), + {{"fpp", std::to_string(expected_fpp)}}); + } + + ~ScopedBloomFilterFppDebugPoint() { + DebugPoints::instance()->remove(std::string(kBloomFilterCreateDebugPoint)); + config::enable_debug_points = _old_enable_debug_points; + } + +private: + bool _old_enable_debug_points; +}; + +struct ScalarBloomFilterTypeCase { + const char* name; + FieldType type; + int32_t length; + int32_t precision; + int32_t frac; +}; + +int32_t scalar_test_column_length(FieldType type) { + switch (type) { + case FieldType::OLAP_FIELD_TYPE_CHAR: + return 16; + case FieldType::OLAP_FIELD_TYPE_VARCHAR: + return 32; + case FieldType::OLAP_FIELD_TYPE_STRING: + return 64; + default: + return cast_set(field_type_size(type)); + } +} + +std::vector create_scalar_bloom_filter_type_cases() { + return { + {"bool", FieldType::OLAP_FIELD_TYPE_BOOL, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_BOOL), -1, -1}, + {"tinyint", FieldType::OLAP_FIELD_TYPE_TINYINT, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_TINYINT), -1, -1}, + {"smallint", FieldType::OLAP_FIELD_TYPE_SMALLINT, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_SMALLINT), -1, -1}, + {"int", FieldType::OLAP_FIELD_TYPE_INT, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_INT), -1, -1}, + {"unsigned_int", FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT), -1, -1}, + {"bigint", FieldType::OLAP_FIELD_TYPE_BIGINT, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_BIGINT), -1, -1}, + {"largeint", FieldType::OLAP_FIELD_TYPE_LARGEINT, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_LARGEINT), -1, -1}, + {"char", FieldType::OLAP_FIELD_TYPE_CHAR, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_CHAR), -1, -1}, + {"varchar", FieldType::OLAP_FIELD_TYPE_VARCHAR, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_VARCHAR), -1, -1}, + {"string", FieldType::OLAP_FIELD_TYPE_STRING, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_STRING), -1, -1}, + {"date", FieldType::OLAP_FIELD_TYPE_DATE, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_DATE), -1, -1}, + {"datetime", FieldType::OLAP_FIELD_TYPE_DATETIME, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_DATETIME), -1, -1}, + {"decimal_v2", FieldType::OLAP_FIELD_TYPE_DECIMAL, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_DECIMAL), 27, 9}, + {"datev2", FieldType::OLAP_FIELD_TYPE_DATEV2, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_DATEV2), -1, -1}, + {"datetimev2", FieldType::OLAP_FIELD_TYPE_DATETIMEV2, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_DATETIMEV2), -1, 6}, + {"timestamptz", FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ), -1, 6}, + {"decimal32", FieldType::OLAP_FIELD_TYPE_DECIMAL32, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_DECIMAL32), 9, 2}, + {"decimal64", FieldType::OLAP_FIELD_TYPE_DECIMAL64, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_DECIMAL64), 18, 4}, + {"decimal128i", FieldType::OLAP_FIELD_TYPE_DECIMAL128I, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_DECIMAL128I), 38, 6}, + {"decimal256", FieldType::OLAP_FIELD_TYPE_DECIMAL256, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_DECIMAL256), 76, 10}, + {"ipv4", FieldType::OLAP_FIELD_TYPE_IPV4, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_IPV4), -1, -1}, + {"ipv6", FieldType::OLAP_FIELD_TYPE_IPV6, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_IPV6), -1, -1}, + }; +} + +const std::vector kScalarBloomFilterTypeCases = + create_scalar_bloom_filter_type_cases(); + +std::vector create_segment_writer_compatible_type_cases() { + std::vector compatible_type_cases; + compatible_type_cases.reserve(kScalarBloomFilterTypeCases.size()); + for (const auto& type_case : kScalarBloomFilterTypeCases) { + // BF itself supports UNSIGNED_INT, but the current block/convertor stack used by + // SegmentWriter/VerticalSegmentWriter still does not materialize that type end-to-end. + // Keep its coverage at direct ColumnWriter level instead of failing these integration tests + // for an unrelated legacy limitation. + if (type_case.type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT) { + continue; + } + compatible_type_cases.push_back(type_case); + } + return compatible_type_cases; +} + +const std::vector kSegmentWriterCompatibleBloomFilterTypeCases = + create_segment_writer_compatible_type_cases(); + +Block create_single_row_default_block(const TabletSchemaSPtr& schema) { + Block block = schema->create_block(); + for (uint32_t cid = 0; cid < schema->num_columns(); ++cid) { + auto column = block.get_by_position(cid).column->assert_mutable(); + column->insert_default(); + block.replace_by_position(cid, std::move(column)); + } + return block; +} + +TabletIndexPB create_bloom_filter_index_pb(int64_t index_id, int32_t column_uid, double fpp) { + TabletIndexPB index_pb; + index_pb.set_index_id(index_id); + index_pb.set_index_name("idx_" + std::to_string(column_uid)); + index_pb.set_index_type(IndexType::BLOOMFILTER); + index_pb.add_col_unique_id(column_uid); + (*index_pb.mutable_properties())["bloom_filter_fpp"] = std::to_string(fpp); + return index_pb; +} + +io::FileWriterPtr create_file_writer(std::string_view file_name) { + io::FileWriterPtr file_writer; + auto path = std::string(kTestDir) + "/" + std::string(file_name); + auto st = io::global_local_filesystem()->create_file(path, &file_writer); + EXPECT_TRUE(st.ok()) << st; + return file_writer; +} + +} // namespace + +class BloomFilterFppWriterTest : public testing::Test { +protected: + void SetUp() override { + auto fs = io::global_local_filesystem(); + auto st = fs->delete_directory(kTestDir); + ASSERT_TRUE(st.ok() || st.is()) << st; + st = fs->create_directory(kTestDir); + ASSERT_TRUE(st.ok()) << st; + } + + void TearDown() override { + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(kTestDir).ok()); + DebugPoints::instance()->remove(std::string(kBloomFilterCreateDebugPoint)); + DebugPoints::instance()->clear(); + config::enable_debug_points = false; + } +}; + +class ScalarBloomFilterFppWriterTestBase : public BloomFilterFppWriterTest { +protected: + static TabletSchemaSPtr create_schema_with_scalar_bloom_filter_index( + const ScalarBloomFilterTypeCase& type_case, double fpp) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::AGG_KEYS); + + auto* key_column = schema_pb.add_column(); + key_column->set_unique_id(0); + key_column->set_name("k1"); + key_column->set_type("INT"); + key_column->set_is_key(true); + key_column->set_is_nullable(false); + key_column->set_length(4); + key_column->set_index_length(4); + key_column->set_aggregation("NONE"); + + auto* value_column = schema_pb.add_column(); + value_column->set_unique_id(1); + value_column->set_name(type_case.name); + value_column->set_type(TabletColumn::get_string_by_field_type(type_case.type)); + value_column->set_is_key(false); + value_column->set_is_nullable(true); + value_column->set_length(type_case.length); + // Use an AGG_KEYS value column so SegmentWriter won't create a zone map for it. + // That keeps this test focused on BF creation/fpp propagation for the full BF type set. + value_column->set_aggregation("REPLACE"); + value_column->set_is_bf_column(true); + if (type_case.precision >= 0) { + value_column->set_precision(type_case.precision); + } + if (type_case.frac >= 0) { + value_column->set_frac(type_case.frac); + } + + *schema_pb.add_index() = create_bloom_filter_index_pb(1, 1, fpp); + + auto schema = std::make_shared(); + schema->init_from_pb(schema_pb); + return schema; + } + + static TabletColumn create_scalar_bloom_filter_column( + const ScalarBloomFilterTypeCase& type_case, int32_t unique_id) { + TabletColumn column(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, type_case.type, + true, unique_id, type_case.length); + column.set_name(type_case.name); + column.set_is_bf_column(true); + if (type_case.precision >= 0) { + column.set_precision(type_case.precision); + } + if (type_case.frac >= 0) { + column.set_frac(type_case.frac); + } + return column; + } + + static ColumnMetaPB create_scalar_column_meta(const TabletColumn& column, uint32_t column_id) { + ColumnMetaPB meta; + meta.set_column_id(column_id); + meta.set_unique_id(column.unique_id()); + meta.set_type(static_cast(column.type())); + meta.set_length(column.length()); + meta.set_encoding(EncodingInfo::resolve_default_encoding( + TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2, column)); + meta.set_compression(CompressionTypePB::LZ4F); + meta.set_is_nullable(column.is_nullable()); + meta.set_precision(column.precision()); + meta.set_frac(column.frac()); + return meta; + } +}; + +class AllScalarBloomFilterFppWriterTest + : public ScalarBloomFilterFppWriterTestBase, + public testing::WithParamInterface {}; + +class SegmentWriterCompatibleBloomFilterFppWriterTest + : public ScalarBloomFilterFppWriterTestBase, + public testing::WithParamInterface {}; + +TEST_P(AllScalarBloomFilterFppWriterTest, column_writer_uses_bloom_filter_fpp) { + const auto& type_case = GetParam(); + constexpr double kExpectedFpp = 0.03; + + auto column = create_scalar_bloom_filter_column(type_case, 1); + auto meta = create_scalar_column_meta(column, 0); + + ColumnWriterOptions writer_opts; + writer_opts.meta = &meta; + writer_opts.need_bloom_filter = true; + writer_opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + writer_opts.bf_options.fpp = kExpectedFpp; + + auto file_writer = create_file_writer(std::string("column_writer_") + type_case.name + ".dat"); + std::unique_ptr writer; + ASSERT_TRUE(ColumnWriter::create(writer_opts, &column, file_writer.get(), &writer).ok()); + + // Cover the full scalar BF type matrix directly at ColumnWriter level, including + // UNSIGNED_INT which cannot currently flow through the segment writer block/convertor stack. + ScopedBloomFilterFppDebugPoint debug_point(kExpectedFpp); + auto st = writer->init(); + ASSERT_TRUE(st.ok()) << st; +} + +// SegmentWriter uses the same per-column fpp lookup regardless of scalar type, so this suite +// exercises the full integration path for every writer-compatible BF scalar type. +TEST_P(SegmentWriterCompatibleBloomFilterFppWriterTest, segment_writer_uses_index_level_fpp) { + const auto& type_case = GetParam(); + constexpr double kExpectedFpp = 0.03; + + auto schema = create_schema_with_scalar_bloom_filter_index(type_case, kExpectedFpp); + + SegmentWriterOptions opts; + auto file_writer = create_file_writer(std::string("segment_writer_") + type_case.name + ".dat"); + SegmentWriter writer(file_writer.get(), 0, schema, nullptr, nullptr, opts, nullptr); + + ScopedBloomFilterFppDebugPoint debug_point(kExpectedFpp); + auto st = writer.init(); + ASSERT_TRUE(st.ok()) << st; +} + +TEST_P(SegmentWriterCompatibleBloomFilterFppWriterTest, + vertical_segment_writer_uses_index_level_fpp) { + const auto& type_case = GetParam(); + constexpr double kExpectedFpp = 0.02; + + auto schema = create_schema_with_scalar_bloom_filter_index(type_case, kExpectedFpp); + Block block = create_single_row_default_block(schema); + + VerticalSegmentWriterOptions opts; + RowsetWriterContext rowset_ctx; + rowset_ctx.write_type = DataWriteType::TYPE_DEFAULT; + rowset_ctx.tablet_schema = schema; + opts.rowset_ctx = &rowset_ctx; + auto file_writer = + create_file_writer(std::string("vertical_segment_writer_") + type_case.name + ".dat"); + VerticalSegmentWriter writer(file_writer.get(), 0, schema, nullptr, nullptr, opts, nullptr); + + ASSERT_TRUE(writer.init().ok()); + ASSERT_TRUE(writer.batch_block(&block, 0, 1).ok()); + + // VerticalSegmentWriter only creates ColumnWriter instances during write_batch(), so use a + // real one-row block here to exercise the BF fpp propagation path in _create_column_writer(). + ScopedBloomFilterFppDebugPoint debug_point(kExpectedFpp); + auto st = writer.write_batch(); + ASSERT_TRUE(st.ok()) << st; +} + +INSTANTIATE_TEST_SUITE_P(AllScalarBloomFilterTypes, AllScalarBloomFilterFppWriterTest, + testing::ValuesIn(kScalarBloomFilterTypeCases), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +INSTANTIATE_TEST_SUITE_P(SegmentWriterCompatibleBloomFilterTypes, + SegmentWriterCompatibleBloomFilterFppWriterTest, + testing::ValuesIn(kSegmentWriterCompatibleBloomFilterTypeCases), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +} // namespace doris::segment_v2 diff --git a/be/test/storage/tablet/tablet_schema_test.cpp b/be/test/storage/tablet/tablet_schema_test.cpp index 6b29eea1d55333..347b7aa1a8146a 100644 --- a/be/test/storage/tablet/tablet_schema_test.cpp +++ b/be/test/storage/tablet/tablet_schema_test.cpp @@ -318,6 +318,184 @@ TEST_F(TabletSchemaTest, test_tablet_schema_update_indexes_from_thrift) { EXPECT_TRUE(found_varchar_idx); } +TEST_F(TabletSchemaTest, test_get_bloom_filter_fpp_prefers_index_property) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_bf_fpp(0.11); + + auto* value_column = schema_pb.add_column(); + value_column->set_unique_id(1001); + value_column->set_name("v1"); + value_column->set_type("INT"); + value_column->set_is_key(false); + value_column->set_is_nullable(true); + value_column->set_length(4); + value_column->set_aggregation("NONE"); + value_column->set_is_bf_column(true); + + auto* bf_index = schema_pb.add_index(); + bf_index->set_index_id(1); + bf_index->set_index_name("idx_v1"); + bf_index->set_index_type(IndexType::BLOOMFILTER); + bf_index->add_col_unique_id(1001); + (*bf_index->mutable_properties())["bloom_filter_fpp"] = "0.03"; + + TabletSchema schema; + schema.init_from_pb(schema_pb); + + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(1001), 0.03); + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(schema.column_by_uid(1001)), 0.03); +} + +TEST_F(TabletSchemaTest, test_get_bloom_filter_fpp_named_index_without_property_uses_default) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_bf_fpp(0.07); + + auto* value_column = schema_pb.add_column(); + value_column->set_unique_id(1002); + value_column->set_name("v2"); + value_column->set_type("INT"); + value_column->set_is_key(false); + value_column->set_is_nullable(true); + value_column->set_length(4); + value_column->set_aggregation("NONE"); + value_column->set_is_bf_column(true); + + auto* bf_index = schema_pb.add_index(); + bf_index->set_index_id(2); + bf_index->set_index_name("idx_v2"); + bf_index->set_index_type(IndexType::BLOOMFILTER); + bf_index->add_col_unique_id(1002); + + TabletSchema schema; + schema.init_from_pb(schema_pb); + + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(1002), BLOOM_FILTER_DEFAULT_FPP); + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(schema.column_by_uid(1002)), + BLOOM_FILTER_DEFAULT_FPP); +} + +TEST_F(TabletSchemaTest, + test_get_bloom_filter_fpp_named_index_without_property_ignores_table_level_fpp) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_bf_fpp(0.07); + + auto* legacy_bf_column = schema_pb.add_column(); + legacy_bf_column->set_unique_id(1003); + legacy_bf_column->set_name("legacy_bf_col"); + legacy_bf_column->set_type("INT"); + legacy_bf_column->set_is_key(false); + legacy_bf_column->set_is_nullable(true); + legacy_bf_column->set_length(4); + legacy_bf_column->set_aggregation("NONE"); + legacy_bf_column->set_is_bf_column(true); + + auto* named_bf_column = schema_pb.add_column(); + named_bf_column->set_unique_id(1004); + named_bf_column->set_name("named_bf_col"); + named_bf_column->set_type("INT"); + named_bf_column->set_is_key(false); + named_bf_column->set_is_nullable(true); + named_bf_column->set_length(4); + named_bf_column->set_aggregation("NONE"); + named_bf_column->set_is_bf_column(true); + + auto* bf_index = schema_pb.add_index(); + bf_index->set_index_id(4); + bf_index->set_index_name("idx_named_bf_col"); + bf_index->set_index_type(IndexType::BLOOMFILTER); + bf_index->add_col_unique_id(1004); + + TabletSchema schema; + schema.init_from_pb(schema_pb); + + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(1003), 0.07); + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(1004), BLOOM_FILTER_DEFAULT_FPP); +} + +TEST_F(TabletSchemaTest, test_get_bloom_filter_fpp_legacy_column_uses_table_property) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_bf_fpp(0.07); + + auto* value_column = schema_pb.add_column(); + value_column->set_unique_id(1004); + value_column->set_name("v4"); + value_column->set_type("INT"); + value_column->set_is_key(false); + value_column->set_is_nullable(true); + value_column->set_length(4); + value_column->set_aggregation("NONE"); + value_column->set_is_bf_column(true); + + TabletSchema schema; + schema.init_from_pb(schema_pb); + + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(1004), 0.07); + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(schema.column_by_uid(1004)), 0.07); +} + +TEST_F(TabletSchemaTest, test_get_bloom_filter_fpp_uses_default_without_table_property) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + + auto* value_column = schema_pb.add_column(); + value_column->set_unique_id(1003); + value_column->set_name("v3"); + value_column->set_type("INT"); + value_column->set_is_key(false); + value_column->set_is_nullable(true); + value_column->set_length(4); + value_column->set_aggregation("NONE"); + value_column->set_is_bf_column(true); + + TabletSchema schema; + schema.init_from_pb(schema_pb); + + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(1003), BLOOM_FILTER_DEFAULT_FPP); + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(schema.column_by_uid(1003)), + BLOOM_FILTER_DEFAULT_FPP); +} + +TEST_F(TabletSchemaTest, test_get_bloom_filter_fpp_for_extracted_column_uses_parent_index) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_bf_fpp(0.09); + + auto* variant_column = schema_pb.add_column(); + variant_column->set_unique_id(2001); + variant_column->set_name("v"); + variant_column->set_type("VARIANT"); + variant_column->set_is_key(false); + variant_column->set_is_nullable(true); + variant_column->set_length(4); + variant_column->set_aggregation("NONE"); + variant_column->set_is_bf_column(true); + variant_column->set_variant_max_subcolumns_count(8); + + auto* bf_index = schema_pb.add_index(); + bf_index->set_index_id(3); + bf_index->set_index_name("idx_v"); + bf_index->set_index_type(IndexType::BLOOMFILTER); + bf_index->add_col_unique_id(2001); + (*bf_index->mutable_properties())["bloom_filter_fpp"] = "0.02"; + + TabletSchema schema; + schema.init_from_pb(schema_pb); + + TabletColumn extracted; + extracted.set_name("v.a"); + extracted.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + extracted.set_parent_unique_id(2001); + extracted.set_path_info(PathInData("v.a")); + extracted.set_is_nullable(true); + + EXPECT_TRUE(extracted.is_extracted_column()); + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(extracted), 0.02); +} + TEST_F(TabletSchemaTest, test_tablet_schema_append_index) { TabletSchema schema; diff --git a/be/test/storage/variant/variant_column_writer_reader_test.cpp b/be/test/storage/variant/variant_column_writer_reader_test.cpp index 191189092bdb24..be3b66a3e08ea9 100644 --- a/be/test/storage/variant/variant_column_writer_reader_test.cpp +++ b/be/test/storage/variant/variant_column_writer_reader_test.cpp @@ -42,6 +42,7 @@ #include "storage/segment/variant/variant_doc_snpashot_compact_iterator.h" #include "storage/storage_engine.h" #include "testutil/variant_util.h" +#include "util/debug_points.h" using namespace doris; @@ -3152,6 +3153,101 @@ TEST_F(VariantColumnWriterReaderTest, test_storage_parse_kv_reduces_sparse_parse EXPECT_GT(kv_result.segment_file_size, static_cast(0)); } +TEST_F(VariantColumnWriterReaderTest, + test_extracted_subcolumn_writer_uses_parent_bloom_filter_fpp) { + constexpr int kRows = 2; + constexpr double kExpectedFpp = 0.013; + + auto old_enable_debug_points = config::enable_debug_points; + config::enable_debug_points = true; + DebugPoints::instance()->add_with_params("BloomFilterIndexWriter::create", + {{"fpp", std::to_string(kExpectedFpp)}}); + Defer defer([old_enable_debug_points]() { + DebugPoints::instance()->remove("BloomFilterIndexWriter::create"); + config::enable_debug_points = old_enable_debug_points; + }); + + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + construct_column(schema_pb.add_column(), 1, "VARIANT", "V1", + /*variant_max_subcolumns_count=*/2, + /*is_key=*/false, + /*is_nullable=*/false, + /*variant_sparse_hash_shard_count=*/0, + /*variant_enable_doc_mode=*/false); + auto* bf_index = schema_pb.add_index(); + bf_index->set_index_id(10009); + bf_index->set_index_name("idx_v1_bf"); + bf_index->set_index_type(IndexType::BLOOMFILTER); + bf_index->add_col_unique_id(1); + (*bf_index->mutable_properties())["bloom_filter_fpp"] = std::to_string(kExpectedFpp); + + _tablet_schema = std::make_shared(); + _tablet_schema->init_from_pb(schema_pb); + + TabletColumn parent_column = _tablet_schema->column(0); + TabletColumn extracted; + extracted.set_name(parent_column.name_lower_case() + ".hot"); + extracted.set_type(FieldType::OLAP_FIELD_TYPE_BIGINT); + extracted.set_parent_unique_id(parent_column.unique_id()); + extracted.set_path_info(PathInData(parent_column.name_lower_case() + ".hot")); + extracted.set_is_nullable(true); + extracted.set_is_bf_column(true); + _tablet_schema->append_column(extracted); + + init_tablet_from_current_schema(33007, TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); + + io::FileWriterPtr file_writer; + auto file_path = local_segment_path(_tablet->tablet_path(), "0", 0); + auto st = io::global_local_filesystem()->create_file(file_path, &file_writer); + ASSERT_TRUE(st.ok()) << st.msg(); + + SegmentFooterPB footer; + RowsetWriterContext rowset_ctx; + rowset_ctx.write_type = DataWriteType::TYPE_DIRECT; + rowset_ctx.tablet_schema = _tablet_schema; + + ColumnWriterOptions opts; + opts.meta = footer.add_columns(); + opts.compression_type = CompressionTypePB::LZ4; + opts.file_writer = file_writer.get(); + opts.footer = &footer; + opts.rowset_ctx = &rowset_ctx; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, parent_column, opts); + + std::unique_ptr writer; + ASSERT_TRUE(ColumnWriter::create(opts, &parent_column, file_writer.get(), &writer).ok()); + ASSERT_TRUE(writer->init().ok()); + + auto strings = ColumnString::create(); + const std::vector jsons = {R"({"hot":1,"cold":10})", R"({"hot":2,"cold":20})"}; + for (const auto& json : jsons) { + strings->insert_data(json.data(), json.size()); + } + + ParseConfig parse_cfg; + parse_cfg.deprecated_enable_flatten_nested = false; + parse_cfg.parse_to = ParseConfig::ParseTo::OnlyDocValueColumn; + auto variant_column = + ColumnVariant::create(parent_column.variant_max_subcolumns_count(), false); + variant_util::parse_json_to_variant(*variant_column, *strings, parse_cfg); + + auto variant_data = std::make_unique(); + variant_data->column_data = variant_column.get(); + variant_data->row_pos = 0; + const auto* data = reinterpret_cast(variant_data.get()); + ASSERT_TRUE(writer->append_data(&data, kRows).ok()); + + // `finish()` materializes the extracted BF-enabled subcolumn and creates its inner scalar + // writer. The debug point verifies that the writer receives the parent bloom filter index fpp. + ASSERT_TRUE(writer->finish().ok()); + ASSERT_TRUE(writer->write_data().ok()); + ASSERT_TRUE(writer->write_ordinal_index().ok()); + ASSERT_TRUE(writer->write_zone_map().ok()); + ASSERT_TRUE(file_writer->close().ok()); +} + TEST_F(VariantColumnWriterReaderTest, test_write_data_advanced) { // 1. create tablet_schema TabletSchemaPB schema_pb; diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Index.java index da427934220436..edc0fcc21773e0 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Index.java @@ -43,6 +43,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; /** * Internal representation of index, including index type, name, columns and comments. @@ -272,6 +273,31 @@ public List getColumnUniqueIds(List schema) { return columnUniqueIds; } + public boolean isBloomFilterIndex() { + return indexType == IndexType.BLOOMFILTER; + } + + public static Set extractBloomFilterColumns(Collection indexes) { + Set bloomFilterColumns = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + if (indexes == null) { + return bloomFilterColumns; + } + for (Index index : indexes) { + if (!index.isBloomFilterIndex()) { + continue; + } + bloomFilterColumns.addAll(index.getColumns()); + } + return bloomFilterColumns; + } + + public static boolean hasBloomFilterIndex(Collection indexes, String columnName) { + if (columnName == null) { + return false; + } + return extractBloomFilterColumns(indexes).contains(columnName); + } + public static void checkConflict(Collection indices, Set bloomFilters) throws AnalysisException { indices = indices == null ? Collections.emptyList() : indices; bloomFilters = bloomFilters == null ? Collections.emptySet() : bloomFilters; diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 233cead1bdce43..79f5338d609358 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -741,18 +741,18 @@ private boolean processDropColumn(DropColumnOp dropColumnOp, OlapTable olapTable } // drop bloom filter column - Set bfCols = olapTable.getCopiedBfColumns(); - if (bfCols != null) { - Set newBfCols = null; - for (String bfCol : bfCols) { - if (!bfCol.equalsIgnoreCase(dropColName)) { - if (newBfCols == null) { - newBfCols = Sets.newHashSet(); + Set legacyBloomFilterColumns = olapTable.getCopiedBfColumns(); + if (legacyBloomFilterColumns != null) { + Set updatedLegacyBloomFilterColumns = null; + for (String legacyBloomFilterColumn : legacyBloomFilterColumns) { + if (!legacyBloomFilterColumn.equalsIgnoreCase(dropColName)) { + if (updatedLegacyBloomFilterColumns == null) { + updatedLegacyBloomFilterColumns = Sets.newHashSet(); } - newBfCols.add(bfCol); + updatedLegacyBloomFilterColumns.add(legacyBloomFilterColumn); } } - olapTable.setBloomFilterInfo(newBfCols, olapTable.getBfFpp()); + olapTable.setBloomFilterInfo(updatedLegacyBloomFilterColumns, olapTable.getBfFpp()); } for (int i = 1; i < indexIds.size(); i++) { @@ -1699,6 +1699,7 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map bfColumns = null; double bfFpp = 0; try { @@ -1711,27 +1712,38 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map oriBfColumns = olapTable.getCopiedBfColumns(); + Set oriLegacyBfCols = olapTable.getCopiedBfColumns(); + Set oriNamedBfCols = Index.extractBloomFilterColumns(olapTable.getIndexes()); + if (oriNamedBfCols.isEmpty()) { + oriNamedBfCols = null; + } + Set namedBfCols = Index.extractBloomFilterColumns(indexes); + if (namedBfCols.isEmpty()) { + namedBfCols = null; + } + boolean hasNamedBfChange = !Objects.equals(oriNamedBfCols, namedBfCols); + if (hasBfColumnsProperty) { + checkLegacyBloomFilterColumnsManagedByNamedIndexes(oriLegacyBfCols, bfColumns, namedBfCols); + } double oriBfFpp = olapTable.getBfFpp(); + boolean originalHasLegacyBloomFilter = oriLegacyBfCols != null; if (bfColumns != null) { if (bfFpp == 0) { // columns: yes, fpp: no - if (bfColumns.equals(oriBfColumns)) { + if (bfColumns.equals(oriLegacyBfCols)) { throw new DdlException("Bloom filter index has no change"); } - - if (oriBfColumns == null) { + if (!originalHasLegacyBloomFilter) { bfFpp = FeConstants.default_bloom_filter_fpp; } else { bfFpp = oriBfFpp; } } else { // columns: yes, fpp: yes - if (bfColumns.equals(oriBfColumns) && bfFpp == oriBfFpp) { + if (bfColumns.equals(oriLegacyBfCols) && bfFpp == oriBfFpp) { throw new DdlException("Bloom filter index has no change"); } } - hasBfChange = true; } else { if (bfFpp == 0) { @@ -1742,25 +1754,35 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map conflictIndexes = new ArrayList<>(newIndexes); + conflictIndexes.add(alterIndex); + try { + Index.checkConflict(conflictIndexes, olapTable.getCopiedBfColumns()); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } newIndexes.add(alterIndex); return false; } @@ -3395,6 +3422,61 @@ private boolean processDropIndex(DropIndexOp dropIndexOp, OlapTable olapTable, L return false; } + /** + * Checks if legacy bloom filter columns are managed by named indexes. + */ + private void checkLegacyBloomFilterColumnsManagedByNamedIndexes( + Set oldLegacyBfColumns, Set newLegacyBfColumns, Set namedBfColumns) + throws DdlException { + if (namedBfColumns == null || namedBfColumns.isEmpty()) { + return; + } + + Set overlappingLegacyAndNamedBfColumns = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + if (oldLegacyBfColumns != null) { + for (String legacyBfColumn : oldLegacyBfColumns) { + if (containsIgnoreCase(namedBfColumns, legacyBfColumn)) { + overlappingLegacyAndNamedBfColumns.add(legacyBfColumn); + } + } + } + Preconditions.checkState(overlappingLegacyAndNamedBfColumns.isEmpty(), + "legacy bloom filter columns overlap named BLOOMFILTER columns: %s", + Joiner.on(",").join(overlappingLegacyAndNamedBfColumns)); + + // legacy bloom filter columns are not changed + if (Objects.equals(newLegacyBfColumns, oldLegacyBfColumns)) { + return; + } + + Set addedLegacyBfColumns = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + if (newLegacyBfColumns != null) { + addedLegacyBfColumns.addAll(newLegacyBfColumns); + } + if (oldLegacyBfColumns != null) { + addedLegacyBfColumns.removeAll(oldLegacyBfColumns); + } + for (String column : addedLegacyBfColumns) { + if (namedBfColumns.contains(column)) { + throw new DdlException("ALTER TABLE failed, expected to create bloom filter index on column " + + column + ", but this column is already defined by named BLOOMFILTER index"); + } + } + + } + + private boolean containsIgnoreCase(Collection values, String target) { + if (values == null || target == null) { + return false; + } + for (String value : values) { + if (target.equalsIgnoreCase(value)) { + return true; + } + } + return false; + } + @Override public void addAlterJobV2(AlterJobV2 alterJob) throws AnalysisException { super.addAlterJobV2(alterJob); @@ -3563,21 +3645,21 @@ public void modifyTableLightSchemaChange(String rawSql, Database db, OlapTable o } // for bloom filter, rebuild bloom filter info by table schema in replay if (isReplay) { - Set bfCols = olapTable.getCopiedBfColumns(); - if (bfCols != null) { + Set legacyBloomFilterColumns = olapTable.getCopiedBfColumns(); + if (legacyBloomFilterColumns != null) { List columns = olapTable.getBaseSchema(); - Set newBfCols = null; - for (String bfCol : bfCols) { + Set normalizedLegacyBloomFilterColumns = null; + for (String legacyBloomFilterColumn : legacyBloomFilterColumns) { for (Column column : columns) { - if (column.getName().equalsIgnoreCase(bfCol)) { - if (newBfCols == null) { - newBfCols = Sets.newHashSet(); + if (column.getName().equalsIgnoreCase(legacyBloomFilterColumn)) { + if (normalizedLegacyBloomFilterColumns == null) { + normalizedLegacyBloomFilterColumns = Sets.newHashSet(); } - newBfCols.add(column.getName()); + normalizedLegacyBloomFilterColumns.add(column.getName()); } } } - olapTable.setBloomFilterInfo(newBfCols, olapTable.getBfFpp()); + olapTable.setBloomFilterInfo(normalizedLegacyBloomFilterColumns, olapTable.getBfFpp()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index c044a8c258f804..fc05c4b5a798ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -297,7 +297,6 @@ protected void createShadowIndexReplica() throws AlterCancelException { long originIndexId = indexIdMap.get(shadowIdxId); int originSchemaHash = tbl.getSchemaHashByIndexId(originIndexId); KeysType originKeysType = tbl.getKeysTypeByIndexId(originIndexId); - List tabletIndexes = originIndexId == tbl.getBaseIndexId() ? indexes : null; for (Tablet shadowTablet : shadowIdx.getTablets()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToProtobuf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToProtobuf.java index d9c17012a64d28..33d223b39e5971 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToProtobuf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToProtobuf.java @@ -37,8 +37,9 @@ public static OlapFile.ColumnPB toPb(Column column, Set bfColumns, List< // when doing schema change, some modified column has a prefix in name. // this prefix is only used in FE, not visible to BE, so we should remove this prefix. String name = column.getName(); - builder.setName(name.startsWith(Column.SHADOW_NAME_PREFIX) - ? name.substring(Column.SHADOW_NAME_PREFIX.length()) : name); + String nonShadowColumnName = name.startsWith(Column.SHADOW_NAME_PREFIX) + ? name.substring(Column.SHADOW_NAME_PREFIX.length()) : name; + builder.setName(nonShadowColumnName); builder.setUniqueId(column.getUniqueId()); builder.setType(column.getDataType().toThrift().name()); @@ -88,7 +89,8 @@ public static OlapFile.ColumnPB toPb(Column column, Set bfColumns, List< builder.setIndexLength(column.getOlapColumnIndexSize()); } - if (bfColumns != null && bfColumns.contains(column.getName())) { + if ((bfColumns != null && bfColumns.contains(nonShadowColumnName)) + || Index.hasBloomFilterIndex(indexes, nonShadowColumnName)) { builder.setIsBfColumn(true); } else { builder.setIsBfColumn(false); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToThrift.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToThrift.java index 9ed324713b69b5..92ea0e714c8089 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToThrift.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToThrift.java @@ -30,8 +30,10 @@ public class ColumnToThrift { public static void setIndexFlag(TColumn tColumn, OlapTable olapTable) { + String columnName = tColumn.getColumnName(); Set bfColumns = olapTable.getCopiedBfColumns(); - if (bfColumns != null && bfColumns.contains(tColumn.getColumnName())) { + if ((bfColumns != null && bfColumns.contains(columnName)) + || Index.hasBloomFilterIndex(olapTable.getIndexes(), columnName)) { tColumn.setIsBloomFilterColumn(true); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index dc8cac76500b15..35d8009cce3751 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -3950,10 +3950,10 @@ private static void addOlapTablePropertyInfo(OlapTable olapTable, StringBuilder } // bloom filter - Set bfColumnNames = olapTable.getCopiedBfColumns(); - if (bfColumnNames != null) { + Set legacyBloomFilterColumnNames = olapTable.getCopiedBfColumns(); + if (legacyBloomFilterColumnNames != null) { sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_BF_COLUMNS).append("\" = \""); - sb.append(Joiner.on(", ").join(olapTable.getCopiedBfColumns())).append("\""); + sb.append(Joiner.on(", ").join(legacyBloomFilterColumnNames)).append("\""); } if (separatePartition) { @@ -6169,17 +6169,17 @@ private void renameColumn(Database db, OlapTable table, String colName, } // 6. modify bloom filter col - Set bfCols = table.getCopiedBfColumns(); - if (bfCols != null) { - Set newBfCols = new HashSet<>(); - for (String bfCol : bfCols) { - if (bfCol.equalsIgnoreCase(colName)) { - newBfCols.add(newColName); + Set legacyBloomFilterColumns = table.getCopiedBfColumns(); + if (legacyBloomFilterColumns != null) { + Set renamedLegacyBloomFilterColumns = new HashSet<>(); + for (String legacyBloomFilterColumn : legacyBloomFilterColumns) { + if (legacyBloomFilterColumn.equalsIgnoreCase(colName)) { + renamedLegacyBloomFilterColumns.add(newColName); } else { - newBfCols.add(bfCol); + renamedLegacyBloomFilterColumns.add(legacyBloomFilterColumn); } } - table.setBloomFilterInfo(newBfCols, table.getBfFpp()); + table.setBloomFilterInfo(renamedLegacyBloomFilterColumns, table.getBfFpp()); } table.rebuildFullSchema(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index bcd9e76e7c434c..99c306b295cebc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -200,7 +200,8 @@ public enum OlapTableState { @SerializedName(value = "tps", alternate = {"tempPartitions"}) private TempPartitions tempPartitions = new TempPartitions(); - // bloom filter columns + // Legacy bloom filter columns managed by table property bloom_filter_columns. + // Named BLOOMFILTER indexes are stored in `indexes`. @SerializedName(value = "bfc", alternate = {"bfColumns"}) private Set bfColumns; @@ -1973,8 +1974,9 @@ public String getSignature(int signatureVersion, List partNames) { } // bloom filter - if (bfColumns != null && !bfColumns.isEmpty()) { - for (String bfCol : bfColumns) { + Set legacyBloomFilterColumns = getCopiedBfColumns(); + if (legacyBloomFilterColumns != null && !legacyBloomFilterColumns.isEmpty()) { + for (String bfCol : legacyBloomFilterColumns) { sb.append(bfCol); } sb.append(bfFpp); @@ -2496,8 +2498,9 @@ public boolean equals(Object o) { && Objects.equals(partitionInfo, other.partitionInfo) && Objects.equals( idToPartition, other.idToPartition) && Objects.equals(nameToPartition, other.nameToPartition) && Objects.equals(tempPartitions, other.tempPartitions) - && Objects.equals(bfColumns, other.bfColumns) && Objects.equals(colocateGroup, - other.colocateGroup) && Objects.equals(sequenceType, other.sequenceType) + && Objects.equals(bfColumns, other.bfColumns) + && Objects.equals(colocateGroup, other.colocateGroup) + && Objects.equals(sequenceType, other.sequenceType) && Objects.equals(indexes, other.indexes) && Objects.equals(tableProperty, other.tableProperty); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index ee3091ec8e1751..2b394bd27e34e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -283,7 +283,9 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, schemaBuilder.setNumShortKeyColumns(shortKeyColumnCount); schemaBuilder.setNumRowsPerRowBlock(1024); schemaBuilder.setCompressKind(OlapCommon.CompressKind.COMPRESS_LZ4); - schemaBuilder.setBfFpp(bfFpp); + if (bfColumns != null) { + schemaBuilder.setBfFpp(bfFpp); + } int deleteSign = -1; int sequenceCol = -1; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java index 51dd6d0dc1af27..5be8fc10e4b680 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Index; import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.TableIf; @@ -29,6 +30,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import java.util.List; import java.util.Set; @@ -127,7 +129,14 @@ public ProcNodeInterface lookup(String idxIdStr) throws AnalysisException { if (schema == null) { throw new AnalysisException("Index " + idxId + " does not exist"); } - bfColumns = olapTable.getCopiedBfColumns(); + bfColumns = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + if (olapTable.getCopiedBfColumns() != null) { + bfColumns.addAll(olapTable.getCopiedBfColumns()); + } + bfColumns.addAll(Index.extractBloomFilterColumns(olapTable.getIndexes())); + if (bfColumns.isEmpty()) { + bfColumns = null; + } if (olapTable.hasVariantColumns() && SessionVariable.enableDescribeExtendVariantColumn()) { return new RemoteIndexSchemaProcDir(table, schema, bfColumns); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 6a43988ccfdf14..e3aad413c8ca6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2708,9 +2708,10 @@ private boolean createOlapTable(Database db, CreateTableInfo createTableInfo) th } bfFpp = PropertyAnalyzer.analyzeBloomFilterFpp(properties); - if (bfColumns != null && bfFpp == 0) { + boolean hasLegacyBf = bfColumns != null; + if (hasLegacyBf && bfFpp == 0) { bfFpp = FeConstants.default_bloom_filter_fpp; - } else if (bfColumns == null) { + } else if (!hasLegacyBf) { bfFpp = 0; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index c470bc789d9e78..7ad365c61fcd4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -6384,6 +6384,8 @@ public Command visitCreateIndex(CreateIndexContext ctx) { indexType = "INVERTED"; } else if (ctx.ANN() != null) { indexType = "ANN"; + } else if (ctx.BLOOMFILTER() != null) { + indexType = "BLOOMFILTER"; } String comment = ctx.STRING_LITERAL() == null ? "" : stripQuotes(ctx.STRING_LITERAL().getText()); IndexDefinition indexDefinition = new IndexDefinition(indexName, ifNotExists, indexCols, indexType, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DescribeCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DescribeCommand.java index 996a61c17cc61c..ce41cacb54d72a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DescribeCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DescribeCommand.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Index; import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.MysqlTable; import org.apache.doris.catalog.OlapTable; @@ -308,7 +309,14 @@ public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exc if (table instanceof OlapTable) { isOlapTable = true; OlapTable olapTable = (OlapTable) table; - Set bfColumns = olapTable.getCopiedBfColumns(); + Set bfColumns = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + if (olapTable.getCopiedBfColumns() != null) { + bfColumns.addAll(olapTable.getCopiedBfColumns()); + } + bfColumns.addAll(Index.extractBloomFilterColumns(olapTable.getIndexes())); + if (bfColumns.isEmpty()) { + bfColumns = null; + } Map> indexIdToSchema = olapTable.getIndexIdToSchema(); // indices order diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java index 8630d80b7dc0ab..efff2562ea6579 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java @@ -28,6 +28,7 @@ import org.apache.doris.catalog.info.IndexType; import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.common.Config; +import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DataType; @@ -84,6 +85,10 @@ public IndexDefinition(String name, boolean ifNotExists, List cols, Stri this.indexType = IndexType.INVERTED; break; } + case "BLOOMFILTER": { + this.indexType = IndexType.BLOOMFILTER; + break; + } case "NGRAM_BF": { this.indexType = IndexType.NGRAM_BF; break; @@ -139,7 +144,7 @@ public static boolean isSupportIdxType(DataType columnType) { } /** - * checkColumn + * Validate whether the index can be created on the given column definition. */ public void checkColumn(ColumnDefinition column, KeysType keysType, boolean enableUniqueKeyMergeOnWrite, @@ -187,8 +192,8 @@ public void checkColumn(ColumnDefinition column, KeysType keysType, (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V1 || invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.DEFAULT) && (Config.isCloudMode() || !Config.enable_inverted_index_v1_for_variant); - - if (colType.isVariantType() && notSupportInvertedIndexForVariant) { + // Bloom filter support variant type, so skip the check for bloom filter index. + if (indexType != IndexType.BLOOMFILTER && colType.isVariantType() && notSupportInvertedIndexForVariant) { throw new AnalysisException(colType + " is not supported in inverted index format V1," + "Please set properties(\"inverted_index_storage_format\"= \"v2\")," + "or upgrade to a newer version"); @@ -231,6 +236,12 @@ public void checkColumn(ColumnDefinition column, KeysType keysType, } catch (Exception ex) { throw new AnalysisException("invalid ngram bf index params:" + ex.getMessage(), ex); } + } else if (indexType == IndexType.BLOOMFILTER) { + Column bfColumn = new Column(indexColName, column.getType().toCatalogDataType()); + if (!bfColumn.isSupportBloomFilter()) { + throw new AnalysisException(colType + " is not supported in bloom filter index. " + + "invalid column: " + indexColName); + } } } else { throw new AnalysisException("Unsupported index type: " + indexType); @@ -294,8 +305,8 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V1 || invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.DEFAULT) && (Config.isCloudMode() || !Config.enable_inverted_index_v1_for_variant); - - if (colType.isVariantType() && notSupportInvertedIndexForVariant) { + // Bloom filter support variant type, so skip the check for bloom filter index. + if (indexType != IndexType.BLOOMFILTER && colType.isVariantType() && notSupportInvertedIndexForVariant) { throw new AnalysisException(colType + " is not supported in inverted index format V1," + "Please set properties(\"inverted_index_storage_format\"= \"v2\")," + "or upgrade to a newer version"); @@ -331,6 +342,11 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe parseAndValidateProperty(properties, NGRAM_SIZE_KEY, MIN_NGRAM_SIZE, MAX_NGRAM_SIZE); parseAndValidateProperty(properties, NGRAM_BF_SIZE_KEY, MIN_BF_SIZE, MAX_BF_SIZE); + } else if (indexType == IndexType.BLOOMFILTER) { + if (!column.isSupportBloomFilter()) { + throw new AnalysisException(colType + " is not supported in bloom filter index. " + + "invalid column: " + indexColName); + } } } else { throw new AnalysisException("Unsupported index type: " + indexType); @@ -360,7 +376,10 @@ public void validate() { } if (indexType == IndexType.BITMAP || indexType == IndexType.INVERTED - || indexType == IndexType.NGRAM_BF) { + || indexType == IndexType.BLOOMFILTER || indexType == IndexType.NGRAM_BF) { + if (indexType == IndexType.BLOOMFILTER) { + validateBloomFilterProperties(); + } if (cols == null || cols.size() != 1) { throw new AnalysisException( indexType.toString() + " index can only apply to a single column."); @@ -460,6 +479,22 @@ public Map getProperties() { return properties; } + private void validateBloomFilterProperties() { + if (properties.isEmpty()) { + return; + } + + if (properties.size() != 1 || !properties.containsKey(PropertyAnalyzer.PROPERTIES_BF_FPP)) { + throw new AnalysisException("BLOOMFILTER index only supports property bloom_filter_fpp"); + } + + try { + PropertyAnalyzer.analyzeBloomFilterFpp(new HashMap<>(properties)); + } catch (org.apache.doris.common.AnalysisException e) { + throw new AnalysisException(e.getMessage(), e); + } + } + public boolean isAnnIndex() { return (this.indexType == IndexType.ANN); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java index 32fe0946d8d782..0ec364ad03de2b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -320,11 +320,14 @@ public TCreateTabletReq toThrift() { tColumns = (List) tCols; } else { tColumns = new ArrayList<>(); + Set namedBfColumns = Index.extractBloomFilterColumns(indexes); for (int i = 0; i < columns.size(); i++) { Column column = columns.get(i); TColumn tColumn = ColumnToThrift.toThrift(column); + String nonShadowColumnName = column.getNonShadowName(); // is bloom filter column - if (bfColumns != null && bfColumns.contains(column.getName())) { + if ((bfColumns != null && bfColumns.contains(nonShadowColumnName)) + || namedBfColumns.contains(nonShadowColumnName)) { tColumn.setIsBloomFilterColumn(true); } // when doing schema change, some modified column has a prefix in name. diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java index 370a9cd90dc173..984be6367f3f9b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java @@ -21,16 +21,24 @@ import org.apache.doris.analysis.ResourceTypeEnum; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.CatalogTestUtil; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.EnvFactory; import org.apache.doris.catalog.FakeEditLog; import org.apache.doris.catalog.FakeEnv; +import org.apache.doris.catalog.Index; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.LocalTablet; +import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.OlapTable.OlapTableState; +import org.apache.doris.catalog.info.IndexType; import org.apache.doris.catalog.info.TableNameInfo; import org.apache.doris.cloud.catalog.CloudEnv; import org.apache.doris.cloud.catalog.CloudEnvFactory; +import org.apache.doris.cloud.catalog.CloudReplica; import org.apache.doris.cloud.datasource.CloudInternalCatalog; import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.cloud.proto.Cloud.MetaServiceCode; @@ -54,6 +62,7 @@ import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import org.apache.doris.thrift.TSortType; +import org.apache.doris.thrift.TStorageType; import org.apache.doris.thrift.TTaskType; import org.apache.doris.utframe.MockedMetaServerFactory; @@ -69,6 +78,7 @@ import org.mockito.Mockito; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -649,4 +659,119 @@ public void testCreateTokenizedInvertedIndex() throws Exception { Assert.assertEquals("true", table.getIndexes().get(0).getProperties().get("support_phrase")); Assert.assertEquals("true", table.getIndexes().get(0).getProperties().get("lower_case")); } + + @Test + public void testCreateShadowIndexReplicaForPartitionCopiesNamedIndexesOnlyForBaseShadowReplica() throws Exception { + CatalogTestUtil.createDupTable(db); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + org.apache.doris.catalog.Partition partition = table.getPartition(CatalogTestUtil.testPartitionId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + table.setDataSortInfo(dataSortInfo); + + long rollupIndexId = 41001L; + long rollupTabletId = 41002L; + long rollupReplicaId = 41003L; + String rollupName = "r1"; + List rollupSchema = Lists.newArrayList(new Column(table.getBaseSchema().get(0)), + new Column(table.getBaseSchema().get(2))); + partition.createRollupIndex(createCloudIndex(rollupIndexId, rollupTabletId, rollupReplicaId, + CatalogTestUtil.testBackendId1, db.getId(), table.getId(), partition.getId(), IndexState.NORMAL)); + table.setIndexMeta(rollupIndexId, rollupName, rollupSchema, 1, 42001, (short) 2, + TStorageType.COLUMN, KeysType.DUP_KEYS); + + List namedBloomFilterIndexes = Lists.newArrayList( + new Index(1L, "bf_v1", Lists.newArrayList("v1"), IndexType.BLOOMFILTER, null, "")); + table.setIndexes(namedBloomFilterIndexes); + + long shadowBaseIndexId = 51001L; + long shadowBaseTabletId = 51002L; + long shadowBaseReplicaId = 51003L; + long shadowRollupIndexId = 51011L; + long shadowRollupTabletId = 51012L; + long shadowRollupReplicaId = 51013L; + + CloudSchemaChangeJobV2 schemaChangeJob = new CloudSchemaChangeJobV2("", 1001L, db.getId(), + table.getId(), table.getName(), 60000L); + schemaChangeJob.setAlterIndexInfo(true, namedBloomFilterIndexes); + schemaChangeJob.setBloomFilterInfo(false, null, 0.02); + schemaChangeJob.addPartitionShadowIndex(partition.getId(), shadowBaseIndexId, + createCloudIndex(shadowBaseIndexId, shadowBaseTabletId, shadowBaseReplicaId, + CatalogTestUtil.testBackendId1, db.getId(), table.getId(), partition.getId(), + IndexState.SHADOW)); + schemaChangeJob.addIndexSchema(shadowBaseIndexId, table.getBaseIndexId(), + Column.SHADOW_NAME_PREFIX + table.getIndexNameById(table.getBaseIndexId()), 2, 52001, (short) 3, + createShadowSchema(table.getBaseSchema())); + + schemaChangeJob.addPartitionShadowIndex(partition.getId(), shadowRollupIndexId, + createCloudIndex(shadowRollupIndexId, shadowRollupTabletId, shadowRollupReplicaId, + CatalogTestUtil.testBackendId1, db.getId(), table.getId(), partition.getId(), + IndexState.SHADOW)); + schemaChangeJob.addIndexSchema(shadowRollupIndexId, rollupIndexId, + Column.SHADOW_NAME_PREFIX + rollupName, 2, 52002, (short) 2, createShadowSchema(rollupSchema)); + + List capturedRequests = new ArrayList<>(); + Mockito.doAnswer(invocation -> { + Cloud.CreateTabletsRequest request = invocation.getArgument(0); + capturedRequests.add(request); + return Cloud.CreateTabletsResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .build(); + }).when(mockProxy).createTablets(Mockito.any()); + + Method method = CloudSchemaChangeJobV2.class.getDeclaredMethod("createShadowIndexReplicaForPartition", + OlapTable.class); + method.setAccessible(true); + method.invoke(schemaChangeJob, table); + + // Only the base shadow index copies named index metadata. The rollup shadow index keeps + // the original schema-change behavior and does not receive named BF metadata or folded + // BF column flags from indexes. bfColumns is null so table-level bfFpp is not set; + // named indexes carry their own per-index FPP. + Assert.assertEquals(2, capturedRequests.size()); + Cloud.CreateTabletsRequest baseRequest = capturedRequests.stream() + .filter(request -> request.getTabletMetas(0).getIndexId() == shadowBaseIndexId) + .findFirst() + .orElseThrow(() -> new AssertionError("base shadow request not found")); + Cloud.CreateTabletsRequest rollupRequest = capturedRequests.stream() + .filter(request -> request.getTabletMetas(0).getIndexId() == shadowRollupIndexId) + .findFirst() + .orElseThrow(() -> new AssertionError("rollup shadow request not found")); + + Assert.assertEquals(1, baseRequest.getTabletMetas(0).getSchema().getIndexCount()); + Assert.assertEquals(0, rollupRequest.getTabletMetas(0).getSchema().getIndexCount()); + Assert.assertFalse(baseRequest.getTabletMetas(0).getSchema().hasBfFpp()); + Assert.assertFalse(rollupRequest.getTabletMetas(0).getSchema().hasBfFpp()); + Assert.assertEquals("k1", baseRequest.getTabletMetas(0).getSchema().getColumn(0).getName()); + Assert.assertEquals("k2", baseRequest.getTabletMetas(0).getSchema().getColumn(1).getName()); + Assert.assertEquals("v1", baseRequest.getTabletMetas(0).getSchema().getColumn(2).getName()); + Assert.assertFalse(baseRequest.getTabletMetas(0).getSchema().getColumn(0).getIsBfColumn()); + Assert.assertFalse(baseRequest.getTabletMetas(0).getSchema().getColumn(1).getIsBfColumn()); + Assert.assertTrue(baseRequest.getTabletMetas(0).getSchema().getColumn(2).getIsBfColumn()); + Assert.assertEquals("k1", rollupRequest.getTabletMetas(0).getSchema().getColumn(0).getName()); + Assert.assertEquals("v1", rollupRequest.getTabletMetas(0).getSchema().getColumn(1).getName()); + Assert.assertFalse(rollupRequest.getTabletMetas(0).getSchema().getColumn(0).getIsBfColumn()); + Assert.assertFalse(rollupRequest.getTabletMetas(0).getSchema().getColumn(1).getIsBfColumn()); + } + + private MaterializedIndex createCloudIndex(long indexId, long tabletId, long replicaId, long backendId, + long dbId, long tableId, long partitionId, IndexState indexState) { + MaterializedIndex index = new MaterializedIndex(indexId, indexState); + LocalTablet tablet = new LocalTablet(tabletId); + tablet.addReplica(new CloudReplica(replicaId, backendId, org.apache.doris.catalog.Replica.ReplicaState.NORMAL, + CatalogTestUtil.testStartVersion, 0, dbId, tableId, partitionId, indexId, 0), true); + index.addTablet(tablet, null, true); + return index; + } + + private List createShadowSchema(List originSchema) { + List shadowSchema = new ArrayList<>(originSchema.size()); + for (Column column : originSchema) { + Column shadowColumn = new Column(column); + shadowColumn.setName(Column.SHADOW_NAME_PREFIX + column.getName()); + shadowSchema.add(shadowColumn); + } + return shadowSchema; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java index 59d7cf0a9fc571..7202d4ad2be7c1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java @@ -17,6 +17,7 @@ package org.apache.doris.alter; +import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; @@ -25,9 +26,11 @@ import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.info.ColumnPosition; import org.apache.doris.catalog.info.IndexType; +import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.nereids.StatementContext; @@ -49,6 +52,7 @@ import org.mockito.Mockito; import java.lang.reflect.Method; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @@ -324,6 +328,119 @@ public void testWithRowBinlogSchemaChangeWithHistoricalValue() throws Exception Assert.assertFalse(cols.contains(Column.generateBeforeColName(Column.SEQUENCE_COL))); } + @Test + public void testCheckLegacyBloomFilterColumnsManagedByNamedIndexesPrivateHelper() { + SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); + + // No-op case: legacy/property-managed BF columns stay unchanged and do not overlap with + // named BF columns. + Deencapsulation.invoke(schemaChangeHandler, + "checkLegacyBloomFilterColumnsManagedByNamedIndexes", + Sets.newTreeSet(List.of("v1")), + Sets.newTreeSet(List.of("v1")), + Sets.newTreeSet(List.of("v2"))); + + // Adding a legacy BF definition on top of a named BF column must be rejected. + DdlException createConflict = Assertions.assertThrows(DdlException.class, + () -> Deencapsulation.invoke(schemaChangeHandler, + "checkLegacyBloomFilterColumnsManagedByNamedIndexes", + Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER), + Sets.newTreeSet(List.of("v1")), + Sets.newTreeSet(List.of("v1")))); + Assertions.assertTrue(createConflict.getMessage().contains( + "expected to create bloom filter index on column v1")); + + // Old legacy BF metadata overlapping named BF metadata is an internal invariant violation, + // not a user-facing ALTER error path. + IllegalStateException overlappingMetadata = Assertions.assertThrows(IllegalStateException.class, + () -> Deencapsulation.invoke(schemaChangeHandler, + "checkLegacyBloomFilterColumnsManagedByNamedIndexes", + Sets.newTreeSet(List.of("v1")), + Sets.newTreeSet(List.of("v1")), + Sets.newTreeSet(List.of("v1")))); + Assertions.assertTrue(overlappingMetadata.getMessage().contains( + "legacy bloom filter columns overlap named BLOOMFILTER columns: v1")); + } + + @Test + public void testModifyTableLightSchemaChangeReplayNormalizesLegacyBloomFilterColumns() throws Exception { + String tableName = "sc_replay_bf_case"; + dropTable("test." + tableName, false); + createTable("CREATE TABLE test." + tableName + " (\n" + + "k1 INT,\n" + + "v1 STRING\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1', 'light_schema_change' = 'true', " + + "'bloom_filter_columns' = 'k1');"); + + Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("test"); + OlapTable table = (OlapTable) db.getTableOrMetaException(tableName, Table.TableType.OLAP); + SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); + + LinkedList alteredSchema = table.getBaseSchema().stream() + .map(Column::new) + .collect(Collectors.toCollection(LinkedList::new)); + alteredSchema.add(new Column("v2", ScalarType.createType(PrimitiveType.INT), + false, AggregateType.NONE, "0", "")); + + Map> indexSchemaMap = Maps.newHashMap(); + indexSchemaMap.put(table.getBaseIndexId(), alteredSchema); + long replayJobId = Env.getCurrentEnv().getNextId(); + + table.writeLock(); + try { + table.setBloomFilterInfo(Sets.newHashSet("K1"), table.getBfFpp()); + schemaChangeHandler.modifyTableLightSchemaChange("", db, table, indexSchemaMap, table.getIndexes(), + null, false, replayJobId, true, Maps.newHashMap()); + + Assertions.assertNotNull(table.getColumn("v2")); + Assertions.assertEquals(Sets.newHashSet("k1"), table.getCopiedBfColumns()); + } finally { + table.writeUnlock(); + schemaChangeHandler.getAlterJobsV2().remove(replayJobId); + schemaChangeHandler.runnableSchemaChangeJobV2.remove(replayJobId); + } + } + + @Test + public void testCreateJobRejectsUnchangedLegacyBloomFilterColumnsAndFpp() throws Exception { + String tableName = "sc_bf_unchanged_cols_fpp"; + dropTable("test." + tableName, false); + createTable("CREATE TABLE test." + tableName + " (\n" + + "k1 INT,\n" + + "v1 STRING\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1', 'light_schema_change' = 'true', " + + "'bloom_filter_columns' = 'k1', 'bloom_filter_fpp' = '0.05');"); + + Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("test"); + OlapTable table = (OlapTable) db.getTableOrMetaException(tableName, Table.TableType.OLAP); + SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); + + Map> indexSchemaMap = Maps.newHashMap(); + indexSchemaMap.put(table.getBaseIndexId(), table.getBaseSchema().stream() + .map(Column::new) + .collect(Collectors.toCollection(LinkedList::new))); + + // This unit test invokes createJob() directly because SQL ALTER TABLE SET with both + // bloom_filter_columns and bloom_filter_fpp is rejected earlier by property validation. + // The goal here is to hit SchemaChangeHandler's "columns: yes, fpp: yes, nothing changed" + // branch and verify the exact no-change exception. + Map properties = Maps.newHashMap(); + properties.put("bloom_filter_columns", "k1"); + properties.put("bloom_filter_fpp", "0.05"); + + DdlException exception = Assertions.assertThrows(DdlException.class, + () -> Deencapsulation.invoke(schemaChangeHandler, "createJob", "", + db.getId(), table, indexSchemaMap, properties, table.getIndexes(), Maps.newHashMap())); + Assertions.assertTrue(exception.getMessage().contains("Bloom filter index has no change"), + exception.getMessage()); + } + @Test public void testWithRowBinlogOpNotSupported() throws Exception { // 1) MODIFY COLUMN not supported diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java index 424205dfb50d5d..ce4ab970220c19 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java @@ -28,7 +28,10 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.FakeEditLog; import org.apache.doris.catalog.FakeEnv; +import org.apache.doris.catalog.Index; import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.LocalReplica; +import org.apache.doris.catalog.LocalTablet; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.MaterializedIndex.IndexState; @@ -42,6 +45,7 @@ import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.Type; import org.apache.doris.catalog.info.ColumnPosition; +import org.apache.doris.catalog.info.IndexType; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeMetaVersion; @@ -56,10 +60,13 @@ import org.apache.doris.nereids.trees.plans.commands.info.ModifyTablePropertiesOp; import org.apache.doris.nereids.types.DataType; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.task.AgentBatchTask; import org.apache.doris.task.AgentTask; import org.apache.doris.task.AgentTaskExecutor; import org.apache.doris.task.AgentTaskQueue; +import org.apache.doris.task.CreateReplicaTask; import org.apache.doris.thrift.TStorageFormat; +import org.apache.doris.thrift.TStorageType; import org.apache.doris.thrift.TTaskType; import org.apache.doris.transaction.FakeTransactionIDGenerator; import org.apache.doris.transaction.GlobalTransactionMgr; @@ -87,6 +94,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; public class SchemaChangeJobV2Test { @@ -583,4 +591,134 @@ public void testAbnormalModifyTableDistributionType3() throws UserException { + "distribution type of aggregate keys table which has value columns with REPLACE type."); Env.getCurrentEnv().convertDistributionType(db, table); } + + @Test + public void testCreateShadowIndexReplicaCopiesNamedIndexesOnlyForBaseShadowReplica() throws Exception { + if (fakeEnv != null) { + fakeEnv.close(); + } + fakeEnv = new FakeEnv(); + if (fakeEditLog != null) { + fakeEditLog.close(); + } + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + + Database db = masterEnv.getInternalCatalog().getDbOrDdlException(CatalogTestUtil.testDbId1); + CatalogTestUtil.createDupTable(db); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + Partition partition = table.getPartition(CatalogTestUtil.testPartitionId2); + + long rollupIndexId = 40001L; + long rollupTabletId = 40002L; + long rollupReplicaId = 40003L; + String rollupName = "r1"; + List rollupSchema = Lists.newArrayList(new Column(table.getBaseSchema().get(0)), + new Column(table.getBaseSchema().get(2))); + partition.createRollupIndex(createLocalIndex(rollupIndexId, rollupTabletId, rollupReplicaId, + CatalogTestUtil.testBackendId1, IndexState.NORMAL)); + table.setIndexMeta(rollupIndexId, rollupName, rollupSchema, 1, 41001, (short) 2, + TStorageType.COLUMN, KeysType.DUP_KEYS); + + List namedBloomFilterIndexes = Lists.newArrayList( + new Index(1L, "bf_v1", Lists.newArrayList("v1"), IndexType.BLOOMFILTER, null, "")); + table.setIndexes(namedBloomFilterIndexes); + table.setState(OlapTableState.SCHEMA_CHANGE); + + long shadowBaseIndexId = 50001L; + long shadowBaseTabletId = 50002L; + long shadowBaseReplicaId = 50003L; + long shadowRollupIndexId = 50011L; + long shadowRollupTabletId = 50012L; + long shadowRollupReplicaId = 50013L; + + SchemaChangeJobV2 schemaChangeJob = new SchemaChangeJobV2("", 1000L, db.getId(), + table.getId(), table.getName(), 60000L); + schemaChangeJob.setAlterIndexInfo(true, namedBloomFilterIndexes); + schemaChangeJob.setBloomFilterInfo(false, null, 0.02); + schemaChangeJob.addPartitionShadowIndex(partition.getId(), shadowBaseIndexId, + createLocalIndex(shadowBaseIndexId, shadowBaseTabletId, shadowBaseReplicaId, + CatalogTestUtil.testBackendId1, IndexState.SHADOW)); + schemaChangeJob.addTabletIdMap(partition.getId(), shadowBaseIndexId, shadowBaseTabletId, + CatalogTestUtil.testTabletId2); + schemaChangeJob.addIndexSchema(shadowBaseIndexId, table.getBaseIndexId(), + Column.SHADOW_NAME_PREFIX + table.getIndexNameById(table.getBaseIndexId()), 2, 51001, (short) 3, + createShadowSchema(table.getBaseSchema())); + + schemaChangeJob.addPartitionShadowIndex(partition.getId(), shadowRollupIndexId, + createLocalIndex(shadowRollupIndexId, shadowRollupTabletId, shadowRollupReplicaId, + CatalogTestUtil.testBackendId1, IndexState.SHADOW)); + schemaChangeJob.addTabletIdMap(partition.getId(), shadowRollupIndexId, shadowRollupTabletId, rollupTabletId); + schemaChangeJob.addIndexSchema(shadowRollupIndexId, rollupIndexId, + Column.SHADOW_NAME_PREFIX + rollupName, 2, 51002, (short) 2, createShadowSchema(rollupSchema)); + + List submittedTasks = new ArrayList<>(); + boolean originalRunningUnitTest = FeConstants.runningUnitTest; + FeConstants.runningUnitTest = false; + mockedAgentTaskExecutor.when(() -> AgentTaskExecutor.submit(Mockito.any(AgentBatchTask.class))) + .thenAnswer(invocation -> { + AgentBatchTask batchTask = invocation.getArgument(0); + submittedTasks.addAll(batchTask.getAllTasks()); + for (AgentTask task : batchTask.getAllTasks()) { + CreateReplicaTask createReplicaTask = (CreateReplicaTask) task; + createReplicaTask.countDownLatch(createReplicaTask.getBackendId(), + createReplicaTask.getTabletId()); + } + return null; + }); + + try { + schemaChangeJob.createShadowIndexReplica(); + } finally { + FeConstants.runningUnitTest = originalRunningUnitTest; + AgentTaskQueue.clearAllTasks(); + } + + // Only base shadow indexes copy named index metadata. legacy/property-managed bloom + // filter columns are carried separately via bfColumns, so named BF does not get folded + // into the rollup shadow replica. + Assert.assertEquals(2, submittedTasks.size()); + CreateReplicaTask baseTask = (CreateReplicaTask) submittedTasks.stream() + .filter(task -> task.getIndexId() == shadowBaseIndexId) + .findFirst() + .orElseThrow(() -> new AssertionError("base shadow create task not found")); + CreateReplicaTask rollupTask = (CreateReplicaTask) submittedTasks.stream() + .filter(task -> task.getIndexId() == shadowRollupIndexId) + .findFirst() + .orElseThrow(() -> new AssertionError("rollup shadow create task not found")); + + @SuppressWarnings("unchecked") + List baseTaskIndexes = Deencapsulation.getField(baseTask, "indexes"); + @SuppressWarnings("unchecked") + List rollupTaskIndexes = Deencapsulation.getField(rollupTask, "indexes"); + @SuppressWarnings("unchecked") + Set baseTaskBfColumns = Deencapsulation.getField(baseTask, "bfColumns"); + @SuppressWarnings("unchecked") + Set rollupTaskBfColumns = Deencapsulation.getField(rollupTask, "bfColumns"); + + Assert.assertEquals(namedBloomFilterIndexes, baseTaskIndexes); + Assert.assertNull(rollupTaskIndexes); + Assert.assertNull(baseTaskBfColumns); + Assert.assertNull(rollupTaskBfColumns); + } + + private MaterializedIndex createLocalIndex(long indexId, long tabletId, long replicaId, long backendId, + IndexState indexState) { + MaterializedIndex index = new MaterializedIndex(indexId, indexState); + LocalTablet tablet = new LocalTablet(tabletId); + tablet.addReplica(new LocalReplica(replicaId, backendId, CatalogTestUtil.testStartVersion, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0), true); + index.addTablet(tablet, null, true); + return index; + } + + private List createShadowSchema(List originSchema) { + List shadowSchema = new ArrayList<>(originSchema.size()); + for (Column column : originSchema) { + Column shadowColumn = new Column(column); + shadowColumn.setName(Column.SHADOW_NAME_PREFIX + column.getName()); + shadowSchema.add(shadowColumn); + } + return shadowSchema; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnBloomFilterMaterializationTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnBloomFilterMaterializationTest.java new file mode 100644 index 00000000000000..60edc1dd925fee --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnBloomFilterMaterializationTest.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.catalog.info.IndexType; +import org.apache.doris.common.DdlException; +import org.apache.doris.proto.OlapFile; +import org.apache.doris.thrift.TColumn; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.Set; + +public class ColumnBloomFilterMaterializationTest { + + @Test + public void testSetIndexFlagMarksNamedBloomFilterOnShadowColumn() { + // The thrift column name is normalized to the non-shadow name. setIndexFlag() must + // still mark the column as bloom-filtered when FE metadata already exposes it in the + // copied BF column set. + OlapTable olapTable = new OlapTable(); + olapTable.setIndexes(Lists.newArrayList(new Index(1L, "bf_k1", Lists.newArrayList("k1"), + IndexType.BLOOMFILTER, null, ""))); + + Column shadowColumn = new Column(Column.SHADOW_NAME_PREFIX + "k1", ScalarType.createType(PrimitiveType.INT), + true, null, "1", ""); + TColumn tColumn = ColumnToThrift.toThrift(shadowColumn); + ColumnToThrift.setIndexFlag(tColumn, olapTable); + + Assert.assertEquals("k1", tColumn.getColumnName()); + Assert.assertTrue(tColumn.isIsBloomFilterColumn()); + } + + @Test + public void testSetIndexFlagFallsBackToNamedIndexMetadataWhenCopiedBfColumnsUnavailable() { + // This covers the fallback path in ColumnToThrift.setIndexFlag(): + // getCopiedBfColumns() does not report the column, so the named index list must + // still mark the thrift column correctly. + OlapTable olapTable = Mockito.mock(OlapTable.class); + Mockito.when(olapTable.getCopiedBfColumns()).thenReturn(null); + Mockito.when(olapTable.getIndexes()).thenReturn(Lists.newArrayList( + new Index(1L, "bf_k1", Lists.newArrayList("k1"), IndexType.BLOOMFILTER, null, ""))); + + Column shadowColumn = new Column(Column.SHADOW_NAME_PREFIX + "k1", ScalarType.createType(PrimitiveType.INT), + true, null, "1", ""); + TColumn tColumn = ColumnToThrift.toThrift(shadowColumn); + ColumnToThrift.setIndexFlag(tColumn, olapTable); + + Assert.assertEquals("k1", tColumn.getColumnName()); + Assert.assertTrue(tColumn.isIsBloomFilterColumn()); + } + + @Test + public void testColumnToProtobufMarksNamedBloomFilterOnShadowColumn() throws DdlException { + Column shadowColumn = new Column(Column.SHADOW_NAME_PREFIX + "k1", ScalarType.createType(PrimitiveType.INT), + true, null, "1", ""); + + OlapFile.ColumnPB columnPb = ColumnToProtobuf.toPb(shadowColumn, null, + Lists.newArrayList(new Index(1L, "bf_k1", Lists.newArrayList("k1"), + IndexType.BLOOMFILTER, null, ""))); + + Assert.assertEquals("k1", columnPb.getName()); + Assert.assertTrue(columnPb.getIsBfColumn()); + } + + @Test + public void testColumnToProtobufMarksLegacyBloomFilterWithoutNamedIndexes() throws DdlException { + // The protobuf builder should also work when only the effective BF column set is passed + // down and there is no named index metadata at all. + Column shadowColumn = new Column(Column.SHADOW_NAME_PREFIX + "k1", ScalarType.createType(PrimitiveType.INT), + true, null, "1", ""); + + OlapFile.ColumnPB columnPb = ColumnToProtobuf.toPb(shadowColumn, Sets.newHashSet("k1"), + Lists.newArrayList()); + + Assert.assertEquals("k1", columnPb.getName()); + Assert.assertTrue(columnPb.getIsBfColumn()); + } + + @Test + public void testOlapTableBloomFilterColumnGetters() { + OlapTable olapTable = new OlapTable(); + olapTable.setBloomFilterInfo(Sets.newHashSet("k1"), 0.05); + olapTable.setIndexes(Lists.newArrayList(new Index(1L, "bf_v1", Lists.newArrayList("v1"), + IndexType.BLOOMFILTER, null, ""))); + + Set legacyBloomFilterColumns = olapTable.getCopiedBfColumns(); + Set namedBloomFilterColumns = Index.extractBloomFilterColumns(olapTable.getIndexes()); + + Assert.assertEquals(Sets.newHashSet("k1"), legacyBloomFilterColumns); + Assert.assertEquals(Sets.newHashSet("v1"), namedBloomFilterColumns); + } + + @Test + public void testIndexBloomFilterHelpersIgnoreNonBloomFilterIndexesAndHandleNulls() { + Set emptyBloomFilterColumns = Index.extractBloomFilterColumns(null); + Assert.assertTrue(emptyBloomFilterColumns.isEmpty()); + Assert.assertFalse(Index.hasBloomFilterIndex(null, "k1")); + Assert.assertFalse(Index.hasBloomFilterIndex(Lists.newArrayList(), null)); + + Set bloomFilterColumns = Index.extractBloomFilterColumns(Lists.newArrayList( + new Index(1L, "bf_v1", Lists.newArrayList("v1"), IndexType.BLOOMFILTER, null, ""), + new Index(2L, "bitmap_k1", Lists.newArrayList("k1"), IndexType.BITMAP, null, ""), + new Index(3L, "bf_v2", Lists.newArrayList("V2"), IndexType.BLOOMFILTER, null, ""))); + + Assert.assertEquals(Sets.newHashSet("v1", "V2"), bloomFilterColumns); + Assert.assertTrue(Index.hasBloomFilterIndex(Lists.newArrayList( + new Index(1L, "bf_v2", Lists.newArrayList("V2"), IndexType.BLOOMFILTER, null, "")), "v2")); + Assert.assertFalse(Index.hasBloomFilterIndex(Lists.newArrayList( + new Index(1L, "bitmap_k1", Lists.newArrayList("k1"), IndexType.BITMAP, null, "")), "k1")); + } + +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableWithBloomFilterIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableWithBloomFilterIndexTest.java index 220b525ac448b2..514b7c675e05eb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableWithBloomFilterIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableWithBloomFilterIndexTest.java @@ -17,14 +17,20 @@ package org.apache.doris.catalog; +import org.apache.doris.alter.AlterJobV2; import org.apache.doris.common.DdlException; import org.apache.doris.common.ExceptionChecker; +import org.apache.doris.common.FeConstants; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.Lists; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.List; import java.util.UUID; +import java.util.concurrent.TimeUnit; public class CreateTableWithBloomFilterIndexTest extends TestWithFeService { private static String runningDir = "fe/mocked/CreateTableWithBloomFilterIndexTest/" @@ -35,6 +41,41 @@ protected void runBeforeAll() throws Exception { createDatabase("test"); } + private void expectFailureContains(String expectedMessage, ExceptionChecker.ThrowingRunnable runnable) { + Throwable throwable = Assertions.assertThrows(Throwable.class, runnable::run); + Assertions.assertTrue(throwable.getMessage().contains(expectedMessage), + "expected msg: " + expectedMessage + ", actual: " + throwable.getMessage()); + } + + private OlapTable getOlapTable(String tableName) throws Exception { + Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("test"); + return (OlapTable) db.getTableOrMetaException(tableName, Table.TableType.OLAP); + } + + private void waitForSchemaChangeDone(String tableName) throws Exception { + long deadlineMs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(30); + while (System.currentTimeMillis() < deadlineMs) { + OlapTable table = getOlapTable(tableName); + boolean allJobsFinished = true; + for (AlterJobV2 job : Env.getCurrentEnv().getSchemaChangeHandler().getAlterJobsV2().values()) { + if (job.getTableId() == table.getId() && !job.getJobState().isFinalState()) { + allJobsFinished = false; + break; + } + } + if (table.getState() == OlapTable.OlapTableState.NORMAL && allJobsFinished) { + return; + } + Thread.sleep(100); + } + Assertions.fail("schema change job did not finish for table " + tableName); + } + + private void alterTableSyncAndWait(String sql, String tableName) throws Exception { + alterTableSync(sql); + waitForSchemaChangeDone(tableName); + } + @Test public void testCreateTableWithTinyIntBloomFilterIndex() { ExceptionChecker.expectThrowsWithMsg(DdlException.class, @@ -902,4 +943,431 @@ public void testBloomFilterColumnsWithNullOrWhitespace() { + "\"replication_num\" = \"1\"\n" + ");")); } + + @Test + public void testCreateTableRejectsLegacyAndNamedBloomFilterOnSameColumn() { + expectFailureContains("k1 should have only one ngram bloom filter index or bloom filter index", + () -> createTable("CREATE TABLE test.tbl_bf_conflict_create (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_k1 (k1) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_columns\" = \"k1\",\n" + + "\"replication_num\" = \"1\"\n" + + ");")); + } + + @Test + public void testCreateTableWithNamedBloomFilterUsesDefaultFpp() throws Exception { + createTable("CREATE TABLE test.tbl_bf_named_default_fpp (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + OlapTable table = getOlapTable("tbl_bf_named_default_fpp"); + Assertions.assertNull(table.getCopiedBfColumns()); + Assertions.assertTrue(Index.extractBloomFilterColumns(table.getIndexes()).contains("v1")); + Assertions.assertEquals(0, table.getBfFpp(), 0); + Assertions.assertTrue(table.getIndexes().stream() + .filter(index -> "idx_v1".equalsIgnoreCase(index.getIndexName())) + .findFirst() + .orElseThrow(() -> new AssertionError("named bloom filter index not found")) + .getProperties() + .isEmpty()); + } + + @Test + public void testCreateTableWithNamedBloomFilterUsesIndexLevelFppProperty() throws Exception { + createTable("CREATE TABLE test.tbl_bf_named_index_level_fpp (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER PROPERTIES (\"bloom_filter_fpp\" = \"0.02\")\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + OlapTable table = getOlapTable("tbl_bf_named_index_level_fpp"); + Assertions.assertEquals(0, table.getBfFpp(), 0); + Assertions.assertEquals("0.02", table.getIndexes().stream() + .filter(index -> "idx_v1".equalsIgnoreCase(index.getIndexName())) + .findFirst() + .orElseThrow(() -> new AssertionError("named bloom filter index not found")) + .getProperties() + .get("bloom_filter_fpp")); + } + + @Test + public void testCreateTableWithNamedBloomFilterDoesNotUseTableLevelFpp() throws Exception { + createTable("CREATE TABLE test.tbl_bf_named_custom_fpp (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_fpp\" = \"0.01\",\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + OlapTable table = getOlapTable("tbl_bf_named_custom_fpp"); + Assertions.assertEquals(0, table.getBfFpp(), 0); + Assertions.assertTrue(Index.extractBloomFilterColumns(table.getIndexes()).contains("v1")); + Assertions.assertTrue(table.getIndexes().stream() + .filter(index -> "idx_v1".equalsIgnoreCase(index.getIndexName())) + .findFirst() + .orElseThrow(() -> new AssertionError("named bloom filter index not found")) + .getProperties() + .isEmpty()); + } + + @Test + public void testCreateIndexRejectsLegacyBloomFilterColumn() throws Exception { + createTable("CREATE TABLE test.tbl_bf_conflict_add_index (\n" + + "k1 INT,\n" + + "v1 STRING\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_columns\" = \"k1\",\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + expectFailureContains("k1 should have only one ngram bloom filter index or bloom filter index", + () -> alterTableSync("ALTER TABLE test.tbl_bf_conflict_add_index " + + "ADD INDEX idx_k1(k1) USING BLOOMFILTER")); + } + + @Test + public void testCreateFirstNamedBloomFilterIndexViaAlterUsesDefaultFpp() throws Exception { + createTable("CREATE TABLE test.tbl_bf_add_first_named (\n" + + "k1 INT,\n" + + "v1 STRING\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + alterTableSyncAndWait("ALTER TABLE test.tbl_bf_add_first_named " + + "ADD INDEX idx_v1(v1) USING BLOOMFILTER", "tbl_bf_add_first_named"); + + OlapTable table = getOlapTable("tbl_bf_add_first_named"); + Assertions.assertNull(table.getCopiedBfColumns()); + Assertions.assertEquals(0, table.getBfFpp(), 0); + Assertions.assertTrue(Index.extractBloomFilterColumns(table.getIndexes()).contains("v1")); + Assertions.assertTrue(table.getIndexes().stream() + .filter(index -> "idx_v1".equalsIgnoreCase(index.getIndexName())) + .findFirst() + .orElseThrow(() -> new AssertionError("named bloom filter index not found")) + .getProperties() + .isEmpty()); + } + + @Test + public void testCreateFirstNamedBloomFilterIndexViaAlterKeepsIndexLevelFppProperty() throws Exception { + createTable("CREATE TABLE test.tbl_bf_add_first_named_index_level_fpp (\n" + + "k1 INT,\n" + + "v1 STRING\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + alterTableSyncAndWait("ALTER TABLE test.tbl_bf_add_first_named_index_level_fpp " + + "ADD INDEX idx_v1(v1) USING BLOOMFILTER PROPERTIES (\"bloom_filter_fpp\" = \"0.02\")", + "tbl_bf_add_first_named_index_level_fpp"); + + OlapTable table = getOlapTable("tbl_bf_add_first_named_index_level_fpp"); + Assertions.assertEquals(0, table.getBfFpp(), 0); + Assertions.assertEquals("0.02", table.getIndexes().stream() + .filter(index -> "idx_v1".equalsIgnoreCase(index.getIndexName())) + .findFirst() + .orElseThrow(() -> new AssertionError("named bloom filter index not found")) + .getProperties() + .get("bloom_filter_fpp")); + } + + @Test + public void testAddSecondNamedBloomFilterIndexSucceeds() throws Exception { + createTable("CREATE TABLE test.tbl_bf_add_second_named (\n" + + "k1 INT,\n" + + "k2 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + alterTableSyncAndWait("ALTER TABLE test.tbl_bf_add_second_named " + + "ADD INDEX idx_k2(k2) USING BLOOMFILTER", "tbl_bf_add_second_named"); + + OlapTable table = getOlapTable("tbl_bf_add_second_named"); + Assertions.assertNull(table.getCopiedBfColumns()); + Assertions.assertEquals(0, table.getBfFpp(), 0); + Assertions.assertTrue(Index.extractBloomFilterColumns(table.getIndexes()).contains("v1")); + Assertions.assertTrue(Index.extractBloomFilterColumns(table.getIndexes()).contains("k2")); + Assertions.assertEquals(2, Index.extractBloomFilterColumns(table.getIndexes()).size()); + } + + @Test + public void testAlterTableSetBloomFilterColumnsRejectsNamedBloomFilterColumn() throws Exception { + createTable("CREATE TABLE test.tbl_bf_named_alter_conflict (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_columns\" = \"k1\",\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + expectFailureContains( + "ALTER TABLE failed, expected to create bloom filter index on column v1", + () -> alterTableSync("ALTER TABLE test.tbl_bf_named_alter_conflict " + + "SET (\"bloom_filter_columns\" = \"k1,v1\")")); + } + + @Test + public void testAlterTableSetBloomFilterColumnsRejectsNamedBloomFilterColumn2() throws Exception { + createTable("CREATE TABLE test.tbl_bf_named_alter_conflict2 (\n" + + "k1 INT,\n" + + "k2 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER,\n" + + "INDEX idx_k2 (k2) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_columns\" = \"k1\",\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + expectFailureContains( + "ALTER TABLE failed, expected to create bloom filter index on column k2", + () -> alterTableSync("ALTER TABLE test.tbl_bf_named_alter_conflict2 " + + "SET (\"bloom_filter_columns\" = \"k2\")")); + } + + @Test + public void testAlterTableSetBloomFilterFppOnNamedBloomFilterOnlyTableThrowsNoChange() throws Exception { + createTable("CREATE TABLE test.tbl_bf_named_alter_fpp (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + OlapTable table = getOlapTable("tbl_bf_named_alter_fpp"); + Assertions.assertNull(table.getCopiedBfColumns()); + Assertions.assertEquals(0, table.getBfFpp(), 0); + Assertions.assertTrue(Index.extractBloomFilterColumns(table.getIndexes()).contains("v1")); + expectFailureContains("Bloom filter index has no change", + () -> alterTableSync("ALTER TABLE test.tbl_bf_named_alter_fpp " + + "SET (\"bloom_filter_fpp\" = \"0.01\")")); + Assertions.assertTrue(getOlapTable("tbl_bf_named_alter_fpp").getIndexes().stream() + .filter(index -> "idx_v1".equalsIgnoreCase(index.getIndexName())) + .findFirst() + .orElseThrow(() -> new AssertionError("named bloom filter index not found")) + .getProperties() + .isEmpty()); + } + + @Test + public void testAlterTableSetSameBloomFilterFppOnNamedBloomFilterOnlyTableThrowsNoChange() throws Exception { + createTable("CREATE TABLE test.tbl_bf_named_same_fpp (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + expectFailureContains("Bloom filter index has no change", + () -> alterTableSync("ALTER TABLE test.tbl_bf_named_same_fpp " + + "SET (\"bloom_filter_fpp\" = \"" + FeConstants.default_bloom_filter_fpp + "\")")); + } + + @Test + public void testAlterTableSetBloomFilterFppWithoutAnyBloomFilterThrowsNoChange() throws Exception { + createTable("CREATE TABLE test.tbl_bf_no_index_alter_fpp (\n" + + "k1 INT,\n" + + "v1 STRING\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + expectFailureContains("Bloom filter index has no change", + () -> alterTableSync("ALTER TABLE test.tbl_bf_no_index_alter_fpp " + + "SET (\"bloom_filter_fpp\" = \"0.01\")")); + } + + @Test + public void testAlterTableSetSameLegacyBloomFilterColumnsThrowsNoChange() throws Exception { + createTable("CREATE TABLE test.tbl_bf_same_legacy_columns (\n" + + "k1 INT,\n" + + "v1 STRING\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_columns\" = \"k1\",\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + expectFailureContains("Bloom filter index has no change", + () -> alterTableSync("ALTER TABLE test.tbl_bf_same_legacy_columns " + + "SET (\"bloom_filter_columns\" = \"k1\")")); + } + + @Test + public void testDropIndexRejectsLegacyBloomFilterColumn() throws Exception { + createTable("CREATE TABLE test.tbl_bf_drop_legacy (\n" + + "k1 INT,\n" + + "v1 STRING\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_columns\" = \"k1\",\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + expectFailureContains("index k1 does not exist", + () -> alterTableSync("ALTER TABLE test.tbl_bf_drop_legacy DROP INDEX k1")); + } + + @Test + public void testDropLastNamedBloomFilterClearsBfFpp() throws Exception { + createTable("CREATE TABLE test.tbl_bf_drop_last_named (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + alterTableSyncAndWait("ALTER TABLE test.tbl_bf_drop_last_named DROP INDEX idx_v1", + "tbl_bf_drop_last_named"); + + OlapTable table = getOlapTable("tbl_bf_drop_last_named"); + Assertions.assertNull(table.getCopiedBfColumns()); + Assertions.assertTrue(Index.extractBloomFilterColumns(table.getIndexes()).isEmpty()); + Assertions.assertNull(table.getCopiedBfColumns()); + Assertions.assertEquals(0, table.getBfFpp(), 0); + Assertions.assertTrue(table.getIndexes().isEmpty()); + } + + @Test + public void testShowCreateTableKeepsNamedBloomFilterOutOfLegacyProperties() throws Exception { + createTable("CREATE TABLE test.tbl_bf_show_create (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_columns\" = \"k1\",\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("test"); + OlapTable table = (OlapTable) db.getTableOrMetaException("tbl_bf_show_create", Table.TableType.OLAP); + List createTableStmt = Lists.newArrayList(); + List addRollupStmt = Lists.newArrayList(); + Env.getDdlStmt(table, createTableStmt, null, addRollupStmt, false, true, -1L); + + String ddl = createTableStmt.get(0); + Assertions.assertTrue(ddl.contains("\"bloom_filter_columns\" = \"k1\"")); + Assertions.assertFalse(ddl.contains("\"bloom_filter_columns\" = \"k1, v1\"")); + Assertions.assertFalse(ddl.contains("\"bloom_filter_columns\" = \"v1\"")); + Assertions.assertTrue(ddl.contains("INDEX idx_v1 (`v1`) USING BLOOMFILTER")); + Assertions.assertTrue(table.getCopiedBfColumns().contains("k1")); + Assertions.assertTrue(Index.extractBloomFilterColumns(table.getIndexes()).contains("v1")); + Assertions.assertEquals(1, table.getCopiedBfColumns().size()); + } + + @Test + public void testRenameLegacyBloomFilterColumnUpdatesMetadata() throws Exception { + // Keep one untouched legacy BF column so renameColumn() has to rewrite the renamed entry + // and preserve the other one in the same metadata set. + createTable("CREATE TABLE test.tbl_bf_rename_legacy (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "v2 INT\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_columns\" = \"k1,v1\",\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + alterTableSyncAndWait("ALTER TABLE test.tbl_bf_rename_legacy RENAME COLUMN v1 v1_new", + "tbl_bf_rename_legacy"); + + OlapTable table = getOlapTable("tbl_bf_rename_legacy"); + Assertions.assertTrue(table.getCopiedBfColumns().contains("k1")); + Assertions.assertFalse(table.getCopiedBfColumns().contains("v1")); + Assertions.assertTrue(table.getCopiedBfColumns().contains("v1_new")); + } + + @Test + public void testDropLegacyBloomFilterColumnUpdatesMetadata() throws Exception { + createTable("CREATE TABLE test.tbl_bf_drop_legacy_column (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "v2 INT\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_columns\" = \"k1,v1\",\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + alterTableSyncAndWait("ALTER TABLE test.tbl_bf_drop_legacy_column DROP COLUMN v1", + "tbl_bf_drop_legacy_column"); + + OlapTable table = getOlapTable("tbl_bf_drop_legacy_column"); + Assertions.assertTrue(table.getCopiedBfColumns().contains("k1")); + Assertions.assertFalse(table.getCopiedBfColumns().contains("v1")); + Assertions.assertEquals(1, table.getCopiedBfColumns().size()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java index f7f7bb88beb8ab..436e001e5fed58 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java @@ -169,6 +169,35 @@ public void testResetPropertiesForRestore() { Assert.assertEquals((short) 3, olapTable.getDefaultReplicaAllocation().getTotalReplicaNum()); } + @Test + public void testNamedBloomFilterTableLevelFppDoesNotAffectSignature() throws IOException { + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS)) { + mockedEnv.when(Env::getCurrentEnvJournalVersion).thenReturn(FeConstants.meta_version); + + Database db = UnitTestUtil.createDb(11, 12, 13, 14, 15, 16, 17); + OlapTable olapTable = null; + for (Table table : db.getTables()) { + if (table.getType() == TableType.OLAP) { + olapTable = (OlapTable) table; + break; + } + } + Assert.assertNotNull(olapTable); + + olapTable.setIndexes(Lists.newArrayList(new Index(1L, "bf_v1", Lists.newArrayList("v1"), + IndexType.BLOOMFILTER, null, ""))); + + List partNames = Lists.newArrayList(olapTable.getPartitionNames()); + olapTable.setBloomFilterInfo(null, 0.01); + String signatureWithFpp001 = olapTable.getSignature(1, partNames); + + olapTable.setBloomFilterInfo(null, 0.02); + String signatureWithFpp002 = olapTable.getSignature(1, Lists.newArrayList(olapTable.getPartitionNames())); + + Assert.assertEquals(signatureWithFpp001, signatureWithFpp002); + } + } + @Test public void testResetPropertiesForRestoreInCloudMode() { // simulate a restoring table with properties that are unsupported in cloud mode diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogBloomFilterMaterializationTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogBloomFilterMaterializationTest.java new file mode 100644 index 00000000000000..bd9b83b97bc384 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogBloomFilterMaterializationTest.java @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cloud.datasource; + +import org.apache.doris.analysis.DataSortInfo; +import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Index; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.cloud.catalog.CloudReplica; +import org.apache.doris.cloud.catalog.CloudTablet; +import org.apache.doris.proto.OlapFile; +import org.apache.doris.proto.OlapFile.EncryptionAlgorithmPB; +import org.apache.doris.thrift.TCompressionType; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; +import org.apache.doris.thrift.TSortType; +import org.apache.doris.thrift.TStorageFormat; +import org.apache.doris.thrift.TTabletType; + +import com.google.common.collect.Lists; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class CloudInternalCatalogBloomFilterMaterializationTest { + + @Test + public void testCreateTabletMetaBuilderMaterializesNamedBloomFilter() throws Exception { + // Named-only BF indexes carry their own per-index FPP. The table-level bfFpp is only + // relevant for legacy bloom_filter_columns, so the cloud schema builder should NOT + // set it when bfColumns is null. + CloudInternalCatalog catalog = new CloudInternalCatalog(); + CloudTablet tablet = new CloudTablet(100L); + tablet.addReplica(new CloudReplica(101L, 1L, org.apache.doris.catalog.Replica.ReplicaState.NORMAL, + 1L, 1, 1L, 1L, 1L, 1L, 0L), true); + Column keyColumn = new Column("k1", ScalarType.createType(PrimitiveType.INT), true, null, "1", ""); + Column valueColumn = new Column("v1", ScalarType.createType(PrimitiveType.INT), false, + AggregateType.NONE, "1", ""); + + OlapFile.TabletMetaCloudPB tabletMeta = catalog.createTabletMetaBuilder(1L, 1L, 1L, tablet, + TTabletType.TABLET_TYPE_DISK, 1, KeysType.DUP_KEYS, (short) 1, null, 0, + Lists.newArrayList(new Index(1L, "bf_k1", Lists.newArrayList("k1"), + org.apache.doris.catalog.info.IndexType.BLOOMFILTER, + Map.of("bloom_filter_fpp", "0.02"), "")), + Lists.newArrayList(keyColumn, valueColumn), new DataSortInfo(TSortType.LEXICAL, 1), + TCompressionType.LZ4F, TStorageFormat.V2, "", false, false, "tbl", 0L, false, false, 1, "", + 0L, 0L, 0L, 0L, 0L, false, null, TInvertedIndexFileStorageFormat.V2, 16384L, false, null, + 65536L, EncryptionAlgorithmPB.PLAINTEXT, 262144L, false, Collections.emptyMap(), 5).build(); + + Assert.assertFalse(tabletMeta.getSchema().hasBfFpp()); + Assert.assertTrue(tabletMeta.getSchema().getColumn(0).getIsBfColumn()); + Assert.assertFalse(tabletMeta.getSchema().getColumn(1).getIsBfColumn()); + Assert.assertEquals("0.02", tabletMeta.getSchema().getIndex(0).getPropertiesMap().get("bloom_filter_fpp")); + } + + @Test + public void testCreateTabletMetaBuilderMaterializesNamedBloomFilterWithEmptyLegacySet() throws Exception { + // An empty legacy BF set should not change the named-BF behavior: the BF column still + // materializes, while the schema-level bfFpp remains whatever FE passed in. + CloudInternalCatalog catalog = new CloudInternalCatalog(); + CloudTablet tablet = new CloudTablet(100L); + tablet.addReplica(new CloudReplica(101L, 1L, org.apache.doris.catalog.Replica.ReplicaState.NORMAL, + 1L, 1, 1L, 1L, 1L, 1L, 0L), true); + Column keyColumn = new Column("k1", ScalarType.createType(PrimitiveType.INT), true, null, "1", ""); + + OlapFile.TabletMetaCloudPB tabletMeta = catalog.createTabletMetaBuilder(1L, 1L, 1L, tablet, + TTabletType.TABLET_TYPE_DISK, 1, KeysType.DUP_KEYS, (short) 1, new HashSet<>(), 0, + Lists.newArrayList(new Index(1L, "bf_k1", Lists.newArrayList("k1"), + org.apache.doris.catalog.info.IndexType.BLOOMFILTER, null, "")), + Lists.newArrayList(keyColumn), new DataSortInfo(TSortType.LEXICAL, 1), + TCompressionType.LZ4F, TStorageFormat.V2, "", false, false, "tbl", 0L, false, false, 1, "", + 0L, 0L, 0L, 0L, 0L, false, null, TInvertedIndexFileStorageFormat.V2, 16384L, false, null, + 65536L, EncryptionAlgorithmPB.PLAINTEXT, 262144L, false, Collections.emptyMap(), 5).build(); + + Assert.assertTrue(tabletMeta.getSchema().hasBfFpp()); + Assert.assertEquals(0, tabletMeta.getSchema().getBfFpp(), 0); + Assert.assertTrue(tabletMeta.getSchema().getColumn(0).getIsBfColumn()); + } + + @Test + public void testCreateTabletMetaBuilderDoesNotSetBfFppWithoutBloomFilter() throws Exception { + CloudInternalCatalog catalog = new CloudInternalCatalog(); + CloudTablet tablet = new CloudTablet(100L); + tablet.addReplica(new CloudReplica(101L, 1L, org.apache.doris.catalog.Replica.ReplicaState.NORMAL, + 1L, 1, 1L, 1L, 1L, 1L, 0L), true); + Column keyColumn = new Column("k1", ScalarType.createType(PrimitiveType.INT), true, null, "1", ""); + + OlapFile.TabletMetaCloudPB tabletMeta = catalog.createTabletMetaBuilder(1L, 1L, 1L, tablet, + TTabletType.TABLET_TYPE_DISK, 1, KeysType.DUP_KEYS, (short) 1, null, 0, Lists.newArrayList(), + Lists.newArrayList(keyColumn), new DataSortInfo(TSortType.LEXICAL, 1), + TCompressionType.LZ4F, TStorageFormat.V2, "", false, false, "tbl", 0L, false, false, 1, "", + 0L, 0L, 0L, 0L, 0L, false, null, TInvertedIndexFileStorageFormat.V2, 16384L, false, null, + 65536L, EncryptionAlgorithmPB.PLAINTEXT, 262144L, false, Collections.emptyMap(), 5).build(); + + Assert.assertFalse(tabletMeta.getSchema().hasBfFpp()); + Assert.assertFalse(tabletMeta.getSchema().getColumn(0).getIsBfColumn()); + } + + @Test + public void testCreateTabletMetaBuilderMaterializesLegacyBloomFilterWithExplicitFpp() throws Exception { + CloudInternalCatalog catalog = new CloudInternalCatalog(); + CloudTablet tablet = new CloudTablet(100L); + tablet.addReplica(new CloudReplica(101L, 1L, org.apache.doris.catalog.Replica.ReplicaState.NORMAL, + 1L, 1, 1L, 1L, 1L, 1L, 0L), true); + Column keyColumn = new Column("k1", ScalarType.createType(PrimitiveType.INT), true, null, "1", ""); + Column valueColumn = new Column("v1", ScalarType.createType(PrimitiveType.INT), false, + AggregateType.NONE, "1", ""); + Set legacyBloomFilterColumns = new HashSet<>(); + legacyBloomFilterColumns.add("k1"); + + OlapFile.TabletMetaCloudPB tabletMeta = catalog.createTabletMetaBuilder(1L, 1L, 1L, tablet, + TTabletType.TABLET_TYPE_DISK, 1, KeysType.DUP_KEYS, (short) 1, legacyBloomFilterColumns, 0.02, + Lists.newArrayList(), Lists.newArrayList(keyColumn, valueColumn), + new DataSortInfo(TSortType.LEXICAL, 1), TCompressionType.LZ4F, TStorageFormat.V2, + "", false, false, "tbl", 0L, false, false, 1, "", 0L, 0L, 0L, 0L, 0L, + false, null, TInvertedIndexFileStorageFormat.V2, 16384L, false, null, 65536L, + EncryptionAlgorithmPB.PLAINTEXT, 262144L, false, Collections.emptyMap(), 5).build(); + + Assert.assertTrue(tabletMeta.getSchema().hasBfFpp()); + Assert.assertEquals(0.02, tabletMeta.getSchema().getBfFpp(), 0); + Assert.assertTrue(tabletMeta.getSchema().getColumn(0).getIsBfColumn()); + Assert.assertFalse(tabletMeta.getSchema().getColumn(1).getIsBfColumn()); + } + + @Test + public void testCreateTabletMetaBuilderMaterializesShadowNamedBloomFilterWithIndexes() throws Exception { + // Shadow schema columns lose the FE-only shadow prefix when serialized, but the named BF + // index still matches on the non-shadow column name and keeps its index property. + CloudInternalCatalog catalog = new CloudInternalCatalog(); + CloudTablet tablet = new CloudTablet(100L); + tablet.addReplica(new CloudReplica(101L, 1L, org.apache.doris.catalog.Replica.ReplicaState.NORMAL, + 1L, 1, 1L, 1L, 1L, 1L, 0L), true); + Column shadowKeyColumn = new Column(Column.SHADOW_NAME_PREFIX + "k1", + ScalarType.createType(PrimitiveType.INT), true, null, "1", ""); + Column valueColumn = new Column("v1", ScalarType.createType(PrimitiveType.INT), false, + AggregateType.NONE, "1", ""); + OlapFile.TabletMetaCloudPB tabletMeta = catalog.createTabletMetaBuilder(1L, 2L, 1L, tablet, + TTabletType.TABLET_TYPE_DISK, 1, KeysType.DUP_KEYS, (short) 1, null, 0, + Lists.newArrayList(new Index(1L, "bf_k1", Lists.newArrayList("k1"), + org.apache.doris.catalog.info.IndexType.BLOOMFILTER, + Map.of("bloom_filter_fpp", "0.03"), "")), + Lists.newArrayList(shadowKeyColumn, valueColumn), + new DataSortInfo(TSortType.LEXICAL, 1), TCompressionType.LZ4F, TStorageFormat.V2, + "", false, true, "tbl", 0L, false, false, 1, "", 0L, 0L, 0L, 0L, 0L, + false, null, TInvertedIndexFileStorageFormat.V2, 16384L, false, null, 65536L, + EncryptionAlgorithmPB.PLAINTEXT, 262144L, false, Collections.emptyMap(), 5).build(); + + Assert.assertFalse(tabletMeta.getSchema().hasBfFpp()); + Assert.assertTrue(tabletMeta.getSchema().getColumn(0).getIsBfColumn()); + Assert.assertFalse(tabletMeta.getSchema().getColumn(1).getIsBfColumn()); + Assert.assertEquals("k1", tabletMeta.getSchema().getColumn(0).getName()); + Assert.assertEquals("0.03", tabletMeta.getSchema().getIndex(0).getPropertiesMap().get("bloom_filter_fpp")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java index 82c7066d4c9c5c..ccbbd19e0d2dff 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.StatementBase; import org.apache.doris.analysis.StmtType; import org.apache.doris.analysis.TableScanParams; +import org.apache.doris.catalog.info.IndexType; import org.apache.doris.common.Config; import org.apache.doris.common.Pair; import org.apache.doris.nereids.StatementContext; @@ -47,6 +48,7 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.commands.AlterTableCommand; import org.apache.doris.nereids.trees.plans.commands.CancelAlterTableCommand; import org.apache.doris.nereids.trees.plans.commands.CreateMaterializedViewCommand; import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; @@ -58,6 +60,9 @@ import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; import org.apache.doris.nereids.trees.plans.commands.ReplayCommand; import org.apache.doris.nereids.trees.plans.commands.UpdateCommand; +import org.apache.doris.nereids.trees.plans.commands.info.CreateIndexOp; +import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo; +import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition; import org.apache.doris.nereids.trees.plans.commands.merge.MergeIntoCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; @@ -1445,6 +1450,81 @@ public void testCtasWithoutAs() { Assertions.assertTrue(createTableCommand.getCtasQuery().isPresent()); } + @Test + public void testCreateIndexUsingBloomFilter() { + NereidsParser parser = new NereidsParser(); + String sql = "CREATE INDEX idx_content ON docs(content) USING BLOOMFILTER"; + Plan plan = parser.parseSingle(sql); + Assertions.assertInstanceOf(AlterTableCommand.class, plan); + AlterTableCommand alterTableCommand = (AlterTableCommand) plan; + Assertions.assertEquals(1, alterTableCommand.getOps().size()); + Assertions.assertInstanceOf(CreateIndexOp.class, alterTableCommand.getOps().get(0)); + CreateIndexOp createIndexOp = (CreateIndexOp) alterTableCommand.getOps().get(0); + IndexDefinition indexDefinition = createIndexOp.getIndexDef(); + Assertions.assertEquals("idx_content", indexDefinition.getIndexName()); + Assertions.assertEquals(Lists.newArrayList("content"), indexDefinition.getColumnNames()); + Assertions.assertEquals(IndexType.BLOOMFILTER, indexDefinition.getIndexType()); + Assertions.assertEquals("docs", createIndexOp.getTableName().getTbl()); + } + + @Test + public void testCreateIndexUsingBloomFilterWithPropertiesThrows() { + NereidsParser parser = new NereidsParser(); + String sql = "CREATE INDEX idx_content ON docs(content) USING BLOOMFILTER " + + "PROPERTIES (\"bloom_filter_fpp\" = \"0.05\")"; + Plan plan = parser.parseSingle(sql); + Assertions.assertInstanceOf(AlterTableCommand.class, plan); + AlterTableCommand alterTableCommand = (AlterTableCommand) plan; + Assertions.assertEquals(1, alterTableCommand.getOps().size()); + Assertions.assertInstanceOf(CreateIndexOp.class, alterTableCommand.getOps().get(0)); + CreateIndexOp createIndexOp = (CreateIndexOp) alterTableCommand.getOps().get(0); + IndexDefinition indexDefinition = createIndexOp.getIndexDef(); + Assertions.assertDoesNotThrow(indexDefinition::validate); + Assertions.assertEquals("0.05", indexDefinition.getProperties().get("bloom_filter_fpp")); + } + + @Test + public void testCreateTableInlineBloomFilterIndex() throws Exception { + NereidsParser parser = new NereidsParser(); + String sql = "CREATE TABLE docs (" + + "id BIGINT, " + + "content TEXT, " + + "INDEX idx_content (content) USING BLOOMFILTER COMMENT \"this is a simple bloomfilter index\"" + + ") DISTRIBUTED BY HASH(id) BUCKETS 1"; + Plan plan = parser.parseSingle(sql); + Assertions.assertInstanceOf(CreateTableCommand.class, plan); + CreateTableInfo createTableInfo = ((CreateTableCommand) plan).getCreateTableInfo(); + Field indexesField = CreateTableInfo.class.getDeclaredField("indexes"); + indexesField.setAccessible(true); + List indexDefinitions = (List) indexesField.get(createTableInfo); + Assertions.assertEquals(1, indexDefinitions.size()); + IndexDefinition indexDefinition = indexDefinitions.get(0); + Assertions.assertEquals("idx_content", indexDefinition.getIndexName()); + Assertions.assertEquals(Lists.newArrayList("content"), indexDefinition.getColumnNames()); + Assertions.assertEquals(IndexType.BLOOMFILTER, indexDefinition.getIndexType()); + Assertions.assertEquals("this is a simple bloomfilter index", indexDefinition.getComment()); + } + + @Test + public void testCreateTableInlineBloomFilterIndexWithFppProperty() throws Exception { + NereidsParser parser = new NereidsParser(); + String sql = "CREATE TABLE docs (" + + "id BIGINT, " + + "content TEXT, " + + "INDEX idx_content (content) USING BLOOMFILTER " + + "PROPERTIES (\"bloom_filter_fpp\" = \"0.05\")" + + ") DISTRIBUTED BY HASH(id) BUCKETS 1"; + Plan plan = parser.parseSingle(sql); + Assertions.assertInstanceOf(CreateTableCommand.class, plan); + CreateTableInfo createTableInfo = ((CreateTableCommand) plan).getCreateTableInfo(); + Field indexesField = CreateTableInfo.class.getDeclaredField("indexes"); + indexesField.setAccessible(true); + List indexDefinitions = (List) indexesField.get(createTableInfo); + Assertions.assertEquals(1, indexDefinitions.size()); + Assertions.assertDoesNotThrow(() -> indexDefinitions.get(0).validate()); + Assertions.assertEquals("0.05", indexDefinitions.get(0).getProperties().get("bloom_filter_fpp")); + } + @Test public void testCreateTableVariantNestedGroupPropertyIsAccepted() { NereidsParser parser = new NereidsParser(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java index 7b41ddc95cf840..02dc95394228e4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java @@ -18,12 +18,15 @@ package org.apache.doris.nereids.trees.plans.commands; import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.Type; import org.apache.doris.catalog.info.IndexType; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition; import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.FloatType; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.MapType; @@ -42,6 +45,16 @@ import java.util.Map; public class IndexDefinitionTest { + private IndexDefinition newBloomFilterIndexDefinition() { + return new IndexDefinition("bf_index", false, Lists.newArrayList("col1"), "BLOOMFILTER", null, "comment"); + } + + private Column newCatalogColumn(Type type, boolean isKey) { + Column column = new Column("col1", type); + column.setIsKey(isKey); + return column; + } + @Test void testVariantIndexFormatV1() throws AnalysisException { IndexDefinition def = new IndexDefinition("variant_index", false, Lists.newArrayList("col1"), "INVERTED", @@ -137,6 +150,174 @@ void testNgramBFIndex() throws AnalysisException { KeysType.DUP_KEYS, false, null); } + @Test + void testBloomFilterIndexTypeName() { + IndexDefinition def = new IndexDefinition("bf_index", false, Lists.newArrayList("col1"), "BLOOMFILTER", + null, "comment"); + Assertions.assertEquals(IndexType.BLOOMFILTER, def.getIndexType()); + } + + @Test + void testBloomFilterIndexSupportsBloomFilterFppProperty() { + Map properties = new HashMap<>(); + properties.put("bloom_filter_fpp", "0.05"); + + IndexDefinition def = new IndexDefinition("bf_index", false, Lists.newArrayList("col1"), "BLOOMFILTER", + properties, "comment"); + Assertions.assertDoesNotThrow(def::validate); + Assertions.assertEquals("0.05", def.getProperties().get("bloom_filter_fpp")); + } + + @Test + void testBloomFilterIndexRejectsNonBloomFilterFppProperty() { + Map properties = new HashMap<>(); + properties.put("foo", "bar"); + + IndexDefinition def = new IndexDefinition("bf_index", false, Lists.newArrayList("col1"), "BLOOMFILTER", + properties, "comment"); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, def::validate); + Assertions.assertEquals("BLOOMFILTER index only supports property bloom_filter_fpp", + exception.getMessage()); + } + + @Test + void testBloomFilterIndexRejectsInvalidBloomFilterFppProperty() { + Map properties = new HashMap<>(); + properties.put("bloom_filter_fpp", "0.1"); + + IndexDefinition def = new IndexDefinition("bf_index", false, Lists.newArrayList("col1"), "BLOOMFILTER", + properties, "comment"); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, def::validate); + Assertions.assertTrue(exception.getMessage().contains("Bloom filter fpp should in [1.0E-4, 0.05]")); + } + + @Test + void testBloomFilterIndexSupportsVariantColumnDefinition() { + // Baseline check for the new named BLOOMFILTER path on VARIANT columns. + IndexDefinition def = newBloomFilterIndexDefinition(); + Assertions.assertDoesNotThrow(() -> def.checkColumn( + new ColumnDefinition("col1", VariantType.INSTANCE, false, AggregateType.NONE, true, null, "comment"), + KeysType.DUP_KEYS, false, null)); + } + + @Test + void testBloomFilterIndexSupportsVariantCatalogColumn() { + IndexDefinition def = newBloomFilterIndexDefinition(); + Assertions.assertDoesNotThrow(() -> def.checkColumn( + newCatalogColumn(Type.VARIANT, true), KeysType.DUP_KEYS, false, null)); + } + + @Test + void testBloomFilterIndexSupportsVariantColumnDefinitionWithV1StorageFormat() { + // Inverted-index-specific VARIANT + V1 rejection must not leak into BLOOMFILTER. + IndexDefinition def = newBloomFilterIndexDefinition(); + Assertions.assertDoesNotThrow(() -> def.checkColumn( + new ColumnDefinition("col1", VariantType.INSTANCE, false, AggregateType.NONE, true, null, "comment"), + KeysType.DUP_KEYS, false, TInvertedIndexFileStorageFormat.V1)); + } + + @Test + void testBloomFilterIndexSupportsVariantColumnDefinitionWithDefaultStorageFormat() { + // DEFAULT may resolve to the old V1 rules elsewhere, so this regression test keeps + // BLOOMFILTER isolated from inverted-index-only validation. + IndexDefinition def = newBloomFilterIndexDefinition(); + Assertions.assertDoesNotThrow(() -> def.checkColumn( + new ColumnDefinition("col1", VariantType.INSTANCE, false, AggregateType.NONE, true, null, "comment"), + KeysType.DUP_KEYS, false, TInvertedIndexFileStorageFormat.DEFAULT)); + } + + @Test + void testBloomFilterIndexSupportsVariantCatalogColumnWithDefaultStorageFormat() { + IndexDefinition def = newBloomFilterIndexDefinition(); + Assertions.assertDoesNotThrow(() -> def.checkColumn( + newCatalogColumn(Type.VARIANT, true), KeysType.DUP_KEYS, false, + TInvertedIndexFileStorageFormat.DEFAULT)); + } + + @Test + void testBloomFilterIndexSupportsVariantCatalogColumnWithV1StorageFormat() { + IndexDefinition def = newBloomFilterIndexDefinition(); + Assertions.assertDoesNotThrow(() -> def.checkColumn( + newCatalogColumn(Type.VARIANT, true), KeysType.DUP_KEYS, false, + TInvertedIndexFileStorageFormat.V1)); + } + + @Test + void testBloomFilterIndexRejectsUnsupportedColumnDefinitionType() { + IndexDefinition def = newBloomFilterIndexDefinition(); + // BOOLEAN is accepted by neither legacy nor named bloom filters. + ColumnDefinition columnDef = new ColumnDefinition("col1", BooleanType.INSTANCE, false, AggregateType.NONE, + true, null, "comment"); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, () -> def.checkColumn( + columnDef, KeysType.DUP_KEYS, false, null)); + Assertions.assertEquals(columnDef.getType() + " is not supported in bloom filter index. invalid column: " + + columnDef.getName(), + exception.getMessage()); + } + + @Test + void testBloomFilterIndexSupportsValueColumnOnDupKeys() { + IndexDefinition def = newBloomFilterIndexDefinition(); + Assertions.assertDoesNotThrow(() -> def.checkColumn( + new ColumnDefinition("col1", StringType.INSTANCE, false, AggregateType.NONE, true, null, "comment"), + KeysType.DUP_KEYS, false, null)); + } + + @Test + void testBloomFilterIndexRejectsValueColumnOnAggKeysForColumnDefinition() { + IndexDefinition def = newBloomFilterIndexDefinition(); + // BLOOMFILTER follows the same key-column restriction as the existing secondary indexes. + ColumnDefinition columnDef = new ColumnDefinition("col1", StringType.INSTANCE, false, AggregateType.NONE, + true, null, "comment"); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, () -> def.checkColumn( + columnDef, KeysType.AGG_KEYS, false, null)); + Assertions.assertEquals("index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table" + + " or key columns of AGG_KEYS table. invalid index: " + def.getIndexName(), + exception.getMessage()); + } + + @Test + void testBloomFilterIndexSupportsValueColumnOnUniqueKeysForCatalogColumn() { + IndexDefinition def = newBloomFilterIndexDefinition(); + Assertions.assertDoesNotThrow(() -> def.checkColumn(newCatalogColumn(Type.STRING, false), + KeysType.UNIQUE_KEYS, false, null)); + } + + @Test + void testBloomFilterIndexRejectsUnsupportedCatalogColumnType() { + IndexDefinition def = newBloomFilterIndexDefinition(); + // Cover the catalog Column overload so both checkColumn(...) entry points stay aligned. + Column column = newCatalogColumn(Type.BOOLEAN, false); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> def.checkColumn(column, KeysType.DUP_KEYS, false, null)); + Assertions.assertEquals(column.getDataType() + " is not supported in bloom filter index. invalid column: " + + column.getName(), exception.getMessage()); + } + + @Test + void testBloomFilterIndexRejectsValueColumnOnAggKeysForCatalogColumn() { + IndexDefinition def = newBloomFilterIndexDefinition(); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> def.checkColumn(newCatalogColumn(Type.STRING, false), KeysType.AGG_KEYS, false, null)); + Assertions.assertEquals("index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table" + + " or key columns of AGG_KEYS table. invalid index: " + def.getIndexName(), + exception.getMessage()); + } + + @Test + void testBloomFilterIndexOnlySingleColumn() { + IndexDefinition def = new IndexDefinition("bf_index", false, Lists.newArrayList("col1", "col2"), + "BLOOMFILTER", null, "comment"); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, def::validate); + Assertions.assertEquals("BLOOMFILTER index can only apply to a single column.", exception.getMessage()); + } + + @Test + void testBloomFilterIndexToSqlUsesBloomFilterKeyword() { + IndexDefinition def = newBloomFilterIndexDefinition(); + Assertions.assertTrue(def.toSql().contains("USING BLOOMFILTER")); + } + @Test void testNgramBFIndexOnlySingleColumn() { IndexDefinition def = new IndexDefinition("ngram_bf_index", false, Lists.newArrayList("col1", "col2"), diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java index ededa002296bcb..96930f747e6e61 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java @@ -22,11 +22,13 @@ import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.BinlogConfig; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Index; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.info.IndexType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.MarkedCountDownLatch; import org.apache.doris.metric.MetricRepo; @@ -193,6 +195,103 @@ public void toThriftTest() throws Exception { Assert.assertNotNull(requestWithRowBinlog.getCreateTabletReq()); Assert.assertNotNull(requestWithRowBinlog.getCreateTabletReq().getRowBinlogSchema()); + List namedBloomFilterIndexes = Arrays.asList(new Index(1L, "bf_k1", Arrays.asList("k1"), + IndexType.BLOOMFILTER, Map.of("bloom_filter_fpp", "0.02"), "")); + AgentTask createWithNamedBloomFilter = new CreateReplicaTask(backendId1, dbId, tableId, partitionId, + indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, storageType, + TStorageMedium.SSD, columns, null, 0, latch, namedBloomFilterIndexes, false, + TTabletType.TABLET_TYPE_DISK, null, TCompressionType.LZ4F, false, "", false, false, "", 0, 0, 0, + 0, 0, false, null, null, new HashMap<>(), rowStorePageSize, false, storagePageSize, + TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize, new HashMap<>(), 5, null); + TAgentTaskRequest requestWithNamedBloomFilter = + (TAgentTaskRequest) toAgentTaskRequest.invoke(agentBatchTask, createWithNamedBloomFilter); + Assert.assertNotNull(requestWithNamedBloomFilter.getCreateTabletReq()); + Assert.assertTrue(requestWithNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getColumns().get(0).isIsBloomFilterColumn()); + Assert.assertFalse(requestWithNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getColumns().get(1).isSetIsBloomFilterColumn()); + // bfColumns is null, so table-level FPP is not set for named-only bloom filter indexes. + // Each named index carries its own FPP in its properties. + Assert.assertFalse(requestWithNamedBloomFilter.getCreateTabletReq().getTabletSchema().isSetBloomFilterFpp()); + Assert.assertTrue(requestWithNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getIndexes().get(0).getProperties().containsKey("bloom_filter_fpp")); + Assert.assertEquals("0.02", requestWithNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getIndexes().get(0).getProperties().get("bloom_filter_fpp")); + + Set legacyBloomFilterColumns = new HashSet<>(); + legacyBloomFilterColumns.add("k1"); + AgentTask createWithLegacyBloomFilter = new CreateReplicaTask(backendId1, dbId, tableId, partitionId, + indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, storageType, + TStorageMedium.SSD, columns, legacyBloomFilterColumns, 0.02, latch, null, false, + TTabletType.TABLET_TYPE_DISK, null, TCompressionType.LZ4F, false, "", false, false, "", 0, 0, 0, + 0, 0, false, null, null, new HashMap<>(), rowStorePageSize, false, storagePageSize, + TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize, new HashMap<>(), 5, null); + TAgentTaskRequest requestWithLegacyBloomFilter = + (TAgentTaskRequest) toAgentTaskRequest.invoke(agentBatchTask, createWithLegacyBloomFilter); + Assert.assertNotNull(requestWithLegacyBloomFilter.getCreateTabletReq()); + Assert.assertTrue(requestWithLegacyBloomFilter.getCreateTabletReq().getTabletSchema() + .getColumns().get(0).isIsBloomFilterColumn()); + Assert.assertEquals(0.02, + requestWithLegacyBloomFilter.getCreateTabletReq().getTabletSchema().getBloomFilterFpp(), 0); + + List shadowColumns = Arrays.asList( + new Column(Column.SHADOW_NAME_PREFIX + "k1", ScalarType.createType(PrimitiveType.INT), + false, null, "1", ""), + new Column("v1", ScalarType.createType(PrimitiveType.INT), false, AggregateType.SUM, "1", "")); + AgentTask createWithShadowNamedBloomFilter = new CreateReplicaTask(backendId1, dbId, tableId, partitionId, + indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, storageType, + TStorageMedium.SSD, shadowColumns, null, 0, latch, namedBloomFilterIndexes, false, + TTabletType.TABLET_TYPE_DISK, null, TCompressionType.LZ4F, false, "", false, false, "", 0, 0, 0, + 0, 0, false, null, null, new HashMap<>(), rowStorePageSize, false, storagePageSize, + TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize, new HashMap<>(), 5, null); + TAgentTaskRequest requestWithShadowNamedBloomFilter = + (TAgentTaskRequest) toAgentTaskRequest.invoke(agentBatchTask, createWithShadowNamedBloomFilter); + Assert.assertEquals("k1", requestWithShadowNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getColumns().get(0).getColumnName()); + Assert.assertTrue(requestWithShadowNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getColumns().get(0).isIsBloomFilterColumn()); + + AgentTask createWithFoldedNamedBloomFilter = new CreateReplicaTask(backendId1, dbId, tableId, partitionId, + indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, storageType, + TStorageMedium.SSD, shadowColumns, null, 0, latch, + Arrays.asList(new Index(2L, "bf_shadow_k1", Arrays.asList("k1"), + IndexType.BLOOMFILTER, Map.of("bloom_filter_fpp", "0.03"), "")), false, + TTabletType.TABLET_TYPE_DISK, null, TCompressionType.LZ4F, false, "", false, false, "", 0, 0, 0, + 0, 0, false, null, null, new HashMap<>(), rowStorePageSize, false, storagePageSize, + TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize, new HashMap<>(), 5, null); + TAgentTaskRequest requestWithFoldedNamedBloomFilter = + (TAgentTaskRequest) toAgentTaskRequest.invoke(agentBatchTask, createWithFoldedNamedBloomFilter); + Assert.assertEquals("k1", requestWithFoldedNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getColumns().get(0).getColumnName()); + Assert.assertTrue(requestWithFoldedNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getColumns().get(0).isIsBloomFilterColumn()); + // bfColumns is null, so table-level FPP is not set. Named indexes carry their own FPP. + Assert.assertFalse(requestWithFoldedNamedBloomFilter.getCreateTabletReq().getTabletSchema().isSetBloomFilterFpp()); + Assert.assertTrue(requestWithFoldedNamedBloomFilter.getCreateTabletReq().getTabletSchema().isSetIndexes()); + Assert.assertEquals("0.03", requestWithFoldedNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getIndexes().get(0).getProperties().get("bloom_filter_fpp")); + + Set emptyLegacyBloomFilterColumns = new HashSet<>(); + // When bfColumns is non-null (even empty), table-level FPP is set to whatever was + // passed in. Here bfFpp=0, so the schema-level FPP will be 0. Named indexes carry + // their own per-index FPP via index properties. + AgentTask createWithEmptyLegacySetAndNamedBloomFilter = new CreateReplicaTask(backendId1, dbId, tableId, + partitionId, indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, + storageType, TStorageMedium.SSD, columns, emptyLegacyBloomFilterColumns, 0, latch, + namedBloomFilterIndexes, false, TTabletType.TABLET_TYPE_DISK, null, TCompressionType.LZ4F, false, + "", false, false, "", 0, 0, 0, 0, 0, false, null, null, new HashMap<>(), rowStorePageSize, false, + storagePageSize, TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize, new HashMap<>(), 5, null); + TAgentTaskRequest requestWithEmptyLegacySetAndNamedBloomFilter = + (TAgentTaskRequest) toAgentTaskRequest.invoke(agentBatchTask, + createWithEmptyLegacySetAndNamedBloomFilter); + Assert.assertTrue(requestWithEmptyLegacySetAndNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getColumns().get(0).isIsBloomFilterColumn()); + Assert.assertTrue(requestWithEmptyLegacySetAndNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .isSetBloomFilterFpp()); + Assert.assertEquals(0.0, + requestWithEmptyLegacySetAndNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getBloomFilterFpp(), 0); + // drop TAgentTaskRequest request2 = (TAgentTaskRequest) toAgentTaskRequest.invoke(agentBatchTask, dropTask); Assert.assertEquals(TTaskType.DROP, request2.getTaskType()); diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 446685892295e4..615856e8d1929d 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -108,6 +108,7 @@ BITMAP_UNION: 'BITMAP_UNION'; BITOR: 'BITOR'; BITXOR: 'BITXOR'; BLOB: 'BLOB'; +BLOOMFILTER: 'BLOOMFILTER'; BOOLEAN: 'BOOLEAN'; BOTH: 'BOTH'; BRANCH: 'BRANCH'; diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index cd2391b1775f1e..abcfdf4e959200 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -236,7 +236,7 @@ supportedCreateStatement partitionSpec? #buildIndex | CREATE INDEX (IF NOT EXISTS)? name=identifier ON tableName=multipartIdentifier identifierList - (USING (NGRAM_BF | INVERTED | ANN))? + (USING indexType=(BLOOMFILTER | NGRAM_BF | INVERTED | ANN))? properties=propertyClause? (COMMENT STRING_LITERAL)? #createIndex | CREATE WORKLOAD POLICY (IF NOT EXISTS)? name=identifierOrText (CONDITIONS LEFT_PAREN workloadPolicyConditions RIGHT_PAREN)? @@ -1559,7 +1559,7 @@ indexDefs ; indexDef - : INDEX (ifNotExists=IF NOT EXISTS)? indexName=identifier cols=identifierList (USING indexType=(INVERTED | NGRAM_BF | ANN ))? (PROPERTIES LEFT_PAREN properties=propertyItemList RIGHT_PAREN)? (COMMENT comment=STRING_LITERAL)? + : INDEX (ifNotExists=IF NOT EXISTS)? indexName=identifier cols=identifierList (USING indexType=(BLOOMFILTER | INVERTED | NGRAM_BF | ANN ))? (PROPERTIES LEFT_PAREN properties=propertyItemList RIGHT_PAREN)? (COMMENT comment=STRING_LITERAL)? ; partitionsDef @@ -2121,6 +2121,7 @@ nonReserved | EXCLUDE | EXPIRED | EXTERNAL + | BLOOMFILTER | FAILED_LOGIN_ATTEMPTS | FAST | FEATURE diff --git a/regression-test/data/bloom_filter_p0/test_bloom_filter_named_index.out b/regression-test/data/bloom_filter_p0/test_bloom_filter_named_index.out new file mode 100644 index 00000000000000..5295bb00947fea --- /dev/null +++ b/regression-test/data/bloom_filter_p0/test_bloom_filter_named_index.out @@ -0,0 +1,109 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !named_inline_data -- +1 alpha 10 +2 beta 20 + +-- !desc_named_inline -- +k1 int Yes true \N BLOOM_FILTER +v1 text Yes false \N NONE,BLOOM_FILTER +v2 int Yes false \N NONE,BLOOM_FILTER + +-- !desc_all_named_inline -- +test_bloom_filter_named_inline DUP_KEYS k1 int int Yes true \N BLOOM_FILTER true + v1 text text Yes false \N NONE,BLOOM_FILTER true + v2 int int Yes false \N NONE,BLOOM_FILTER true + +-- !show_proc_named_inline_mixed -- +k1 int Yes true \N BLOOM_FILTER +v1 text Yes false \N NONE,BLOOM_FILTER +v2 int Yes false \N NONE,BLOOM_FILTER + +-- !show_index_named_inline -- +test_bloom_filter_named_inline idx_v1 v1 BLOOMFILTER +test_bloom_filter_named_inline idx_v2 v2 BLOOMFILTER + +-- !desc_named_inline_after_legacy_drop -- +k1 int Yes true \N +v1 text Yes false \N NONE,BLOOM_FILTER +v2 int Yes false \N NONE,BLOOM_FILTER + +-- !desc_named_add_on_legacy -- +k1 int Yes true \N BLOOM_FILTER +v1 text Yes false \N NONE +v2 int Yes false \N NONE,BLOOM_FILTER + +-- !show_index_named_add_on_legacy -- +test_bloom_filter_named_add_on_legacy idx_v2 v2 BLOOMFILTER + +-- !desc_named_drop_mixed_before -- +k1 int Yes true \N BLOOM_FILTER +v1 text Yes false \N NONE,BLOOM_FILTER +v2 int Yes false \N NONE + +-- !show_index_named_drop_mixed_before -- +test_bloom_filter_named_drop_mixed idx_v1 v1 BLOOMFILTER + +-- !desc_named_drop_mixed_after -- +k1 int Yes true \N BLOOM_FILTER +v1 text Yes false \N NONE +v2 int Yes false \N NONE + +-- !show_index_named_drop_mixed_after -- + +-- !desc_named_rename_legacy_mixed -- +k1 int Yes true \N +v1_new text Yes false \N NONE,BLOOM_FILTER +v2 int Yes false \N NONE,BLOOM_FILTER + +-- !show_index_named_rename_legacy_mixed -- +test_bloom_filter_named_rename_legacy_mixed idx_v2 v2 BLOOMFILTER + +-- !desc_named_drop_legacy_column_mixed -- +k1 int Yes true \N BLOOM_FILTER +v2 int Yes false \N NONE,BLOOM_FILTER + +-- !show_index_named_drop_legacy_column_mixed -- +test_bloom_filter_named_drop_legacy_column_mixed idx_v2 v2 BLOOMFILTER + +-- !named_rollup_data -- +1 10 alpha +2 20 beta + +-- !show_index_named_rollup -- +test_bloom_filter_named_rollup DUP_KEYS k1 int int Yes true \N true + v2 int int Yes true \N true + v1 text text Yes false \N NONE,BLOOM_FILTER true + +r1 DUP_KEYS k1 int int Yes true \N true + v1 text text Yes false \N NONE,BLOOM_FILTER true + +-- !show_index_named_rollup_after_drop -- +test_bloom_filter_named_rollup DUP_KEYS k1 int int Yes true \N true + v2 int int Yes true \N true + v1 text text Yes false \N NONE true + +r1 DUP_KEYS k1 int int Yes true \N true + v1 text text Yes false \N NONE true + +-- !named_create_index_data -- +1 one 101 +2 two 202 + +-- !desc_named_create_index -- +k1 int Yes true \N +v1 text Yes false \N NONE +v2 int Yes false \N NONE,BLOOM_FILTER + +-- !show_index_named_create_index -- +test_bloom_filter_named_create_index idx_v2 v2 BLOOMFILTER ("bloom_filter_fpp" = "0.02") + +-- !desc_named_create_index_after_drop -- +k1 int Yes true \N +v1 text Yes false \N NONE +v2 int Yes false \N NONE + +-- !show_index_named_create_index_after_drop -- + +-- !show_index_named_variant -- +test_bloom_filter_named_variant idx_v v BLOOMFILTER ("bloom_filter_fpp" = "0.03") + diff --git a/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy b/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy index d3b5a4791962d6..9ee82da0c9010b 100644 --- a/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy +++ b/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy @@ -221,6 +221,7 @@ suite("test_bloom_filter","nonConcurrent") { DISTRIBUTED BY HASH(`id`) BUCKETS 5 PROPERTIES ( "replication_num" = "1", + "disable_auto_compaction" = "true", "bloom_filter_columns" = "id", "bloom_filter_fpp" = "0.03" )""" @@ -234,11 +235,19 @@ suite("test_bloom_filter","nonConcurrent") { GetDebugPoint().disableDebugPointForAllBEs("BloomFilterIndexWriter::create"); } - sql """ALTER TABLE ${test_dynamic_fpp_tb} SET("bloom_filter_fpp" = "0.02")""" - wait_for_latest_op_on_table_finish(test_dynamic_fpp_tb, timeout) + try { + GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexWriter::create", + [fpp: "0.02", execute: "1", timeout: "120"]) + sql """ALTER TABLE ${test_dynamic_fpp_tb} SET("bloom_filter_fpp" = "0.02")""" + wait_for_latest_op_on_table_finish(test_dynamic_fpp_tb, timeout) + sql """ALTER TABLE ${test_dynamic_fpp_tb} SET("bloom_filter_fpp" = "0.03")""" + wait_for_latest_op_on_table_finish(test_dynamic_fpp_tb, timeout) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("BloomFilterIndexWriter::create"); + } try { - GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexWriter::create", [fpp: "0.02"]) + GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexWriter::create", [fpp: "0.03"]) sql """INSERT INTO ${test_dynamic_fpp_tb} VALUES (6, 'Grace'), (7, 'Henry'), diff --git a/regression-test/suites/bloom_filter_p0/test_bloom_filter_named_index.groovy b/regression-test/suites/bloom_filter_p0/test_bloom_filter_named_index.groovy new file mode 100644 index 00000000000000..f2a7952fcca539 --- /dev/null +++ b/regression-test/suites/bloom_filter_p0/test_bloom_filter_named_index.groovy @@ -0,0 +1,595 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_bloom_filter_named_index", "nonConcurrent") { + def getBaseIndexSchemaProcPath = { String tableName -> + def dbName = (sql """select database()""")[0][0] + def catalogId = get_catalog_id("internal") + def dbId = get_database_id("internal", dbName) + def tableId = get_table_id("internal", dbName, tableName) + def indexSchemaProc = """/catalogs/${catalogId}/${dbId}/${tableId}/index_schema""" + def indexRows = sql """show proc '${indexSchemaProc}'""" + def baseIndexId = null + for (int i = 0; i < indexRows.size(); i++) { + if (indexRows[i][1].equals(tableName)) { + baseIndexId = indexRows[i][0] + break + } + } + assert baseIndexId != null + return """${indexSchemaProc}/${baseIndexId}""" + } + + sql """DROP TABLE IF EXISTS test_bloom_filter_named_inline""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_create_index""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_add_on_legacy""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_drop_mixed""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_rename_legacy_mixed""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_drop_legacy_column_mixed""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_rollup""" + sql """DROP TABLE IF EXISTS test_bloom_filter_cov_inline_invalid_type""" + sql """DROP TABLE IF EXISTS test_bloom_filter_cov_legacy_same_columns""" + sql """DROP TABLE IF EXISTS test_bloom_filter_cov_legacy_same_fpp""" + sql """DROP TABLE IF EXISTS test_bloom_filter_cov_no_bf_fpp_only""" + sql """DROP TABLE IF EXISTS test_bloom_filter_cov_add_index_invalid_type""" + sql """DROP TABLE IF EXISTS test_bloom_filter_cov_add_index_with_properties""" + sql """DROP TABLE IF EXISTS test_bloom_filter_cov_empty_legacy_columns""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_variant""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_array_invalid""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_map_invalid""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_struct_invalid""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_duplicate_inline""" + sql """DROP TABLE IF EXISTS test_mixed_fpp_bloom_filter_tb""" + + test { + sql """ + CREATE TABLE test_bloom_filter_named_duplicate_inline ( + k1 INT, + v1 STRING, + INDEX idx_v1_a (v1) USING BLOOMFILTER, + INDEX idx_v1_b (v1) USING BLOOMFILTER + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + exception "column: v1 cannot have multiple indexes, index type: BLOOMFILTER" + } + + sql """ + CREATE TABLE test_bloom_filter_named_inline ( + k1 INT, + v1 STRING, + v2 INT, + INDEX idx_v1 (v1) USING BLOOMFILTER, + INDEX idx_v2 (v2) USING BLOOMFILTER + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """INSERT INTO test_bloom_filter_named_inline VALUES (1, 'alpha', 10), (2, 'beta', 20)""" + + sql """ALTER TABLE test_bloom_filter_named_inline SET ("bloom_filter_columns" = "k1")""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_inline' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + order_qt_named_inline_data """SELECT * FROM test_bloom_filter_named_inline ORDER BY k1""" + qt_desc_named_inline """DESC test_bloom_filter_named_inline""" + qt_desc_all_named_inline """DESC test_bloom_filter_named_inline ALL""" + qt_show_proc_named_inline_mixed """SHOW PROC '${getBaseIndexSchemaProcPath("test_bloom_filter_named_inline")}'""" + qt_show_index_named_inline """SHOW INDEX FROM test_bloom_filter_named_inline""" + + test { + sql """ALTER TABLE test_bloom_filter_named_inline SET ("bloom_filter_columns" = "k1,v1")""" + exception "ALTER TABLE failed, expected to create bloom filter index on column v1, but this column is already defined by named BLOOMFILTER index" + } + + test { + sql """DROP INDEX k1 ON test_bloom_filter_named_inline""" + exception "index k1 does not exist" + } + + test { + sql """CREATE INDEX idx_k1 ON test_bloom_filter_named_inline(k1) USING BLOOMFILTER""" + exception "k1 should have only one ngram bloom filter index or bloom filter index" + } + + test { + sql """CREATE INDEX idx_v1_dup ON test_bloom_filter_named_inline(v1) USING BLOOMFILTER""" + exception "BLOOMFILTER index for column (v1) with non-analyzed type already exists." + } + + sql """DROP INDEX IF EXISTS idx_missing ON test_bloom_filter_named_inline""" + + test { + sql """DROP INDEX idx_missing ON test_bloom_filter_named_inline""" + exception "index idx_missing does not exist" + } + + sql """ALTER TABLE test_bloom_filter_named_inline SET ("bloom_filter_columns" = "")""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_inline' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + qt_desc_named_inline_after_legacy_drop """DESC test_bloom_filter_named_inline""" + + sql """ + CREATE TABLE test_bloom_filter_named_add_on_legacy ( + k1 INT, + v1 STRING, + v2 INT + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "bloom_filter_columns" = "k1" + ) + """ + + sql """CREATE INDEX idx_v2 ON test_bloom_filter_named_add_on_legacy(v2) USING BLOOMFILTER""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_add_on_legacy' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + qt_desc_named_add_on_legacy """DESC test_bloom_filter_named_add_on_legacy""" + qt_show_index_named_add_on_legacy """SHOW INDEX FROM test_bloom_filter_named_add_on_legacy""" + + sql """ + CREATE TABLE test_bloom_filter_named_drop_mixed ( + k1 INT, + v1 STRING, + v2 INT, + INDEX idx_v1 (v1) USING BLOOMFILTER + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ALTER TABLE test_bloom_filter_named_drop_mixed SET ("bloom_filter_columns" = "k1")""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_drop_mixed' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + qt_desc_named_drop_mixed_before """DESC test_bloom_filter_named_drop_mixed""" + qt_show_index_named_drop_mixed_before """SHOW INDEX FROM test_bloom_filter_named_drop_mixed""" + + sql """DROP INDEX idx_v1 ON test_bloom_filter_named_drop_mixed""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_drop_mixed' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + qt_desc_named_drop_mixed_after """DESC test_bloom_filter_named_drop_mixed""" + qt_show_index_named_drop_mixed_after """SHOW INDEX FROM test_bloom_filter_named_drop_mixed""" + + sql """ + CREATE TABLE test_bloom_filter_named_rename_legacy_mixed ( + k1 INT, + v1 STRING, + v2 INT, + INDEX idx_v2 (v2) USING BLOOMFILTER + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "bloom_filter_columns" = "v1" + ) + """ + + sql """ALTER TABLE test_bloom_filter_named_rename_legacy_mixed RENAME COLUMN v1 v1_new""" + sql """SYNC""" + + qt_desc_named_rename_legacy_mixed """DESC test_bloom_filter_named_rename_legacy_mixed""" + qt_show_index_named_rename_legacy_mixed """SHOW INDEX FROM test_bloom_filter_named_rename_legacy_mixed""" + + sql """ + CREATE TABLE test_bloom_filter_named_drop_legacy_column_mixed ( + k1 INT, + v1 STRING, + v2 INT, + INDEX idx_v2 (v2) USING BLOOMFILTER + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "bloom_filter_columns" = "k1,v1" + ) + """ + + sql """ALTER TABLE test_bloom_filter_named_drop_legacy_column_mixed DROP COLUMN v1""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_drop_legacy_column_mixed' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + qt_desc_named_drop_legacy_column_mixed """DESC test_bloom_filter_named_drop_legacy_column_mixed""" + qt_show_index_named_drop_legacy_column_mixed """SHOW INDEX FROM test_bloom_filter_named_drop_legacy_column_mixed""" + + sql """ + CREATE TABLE test_bloom_filter_named_rollup ( + k1 INT, + v2 INT, + v1 STRING + ) ENGINE=OLAP + DUPLICATE KEY(k1, v2) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + ROLLUP ( + r1(k1, v1) + ) + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """INSERT INTO test_bloom_filter_named_rollup VALUES (1, 10, 'alpha'), (2, 20, 'beta')""" + + sql """CREATE INDEX idx_v1 ON test_bloom_filter_named_rollup(v1) USING BLOOMFILTER""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_rollup' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + order_qt_named_rollup_data """SELECT * FROM test_bloom_filter_named_rollup ORDER BY k1""" + qt_show_index_named_rollup """DESC test_bloom_filter_named_rollup ALL""" + + sql """DROP INDEX idx_v1 ON test_bloom_filter_named_rollup""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_rollup' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + qt_show_index_named_rollup_after_drop """DESC test_bloom_filter_named_rollup ALL""" + + sql """ + CREATE TABLE test_bloom_filter_named_create_index ( + k1 INT, + v1 STRING, + v2 INT + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ) + """ + + sql """INSERT INTO test_bloom_filter_named_create_index VALUES (1, 'one', 101), (2, 'two', 202)""" + + sql """CREATE INDEX idx_v2 ON test_bloom_filter_named_create_index(v2) + USING BLOOMFILTER PROPERTIES ("bloom_filter_fpp" = "0.02")""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_create_index' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + order_qt_named_create_index_data """SELECT * FROM test_bloom_filter_named_create_index ORDER BY k1""" + qt_desc_named_create_index """DESC test_bloom_filter_named_create_index""" + qt_show_index_named_create_index """SHOW INDEX FROM test_bloom_filter_named_create_index""" + + try { + GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexWriter::create", [fpp: "0.02"]) + sql """INSERT INTO test_bloom_filter_named_create_index VALUES (3, 'three', 303), (4, 'four', 404)""" + } finally { + GetDebugPoint().disableDebugPointForAllBEs("BloomFilterIndexWriter::create") + } + + test { + sql """ALTER TABLE test_bloom_filter_named_create_index SET ("bloom_filter_fpp" = "0.01")""" + exception "Bloom filter index has no change" + } + + sql """DROP INDEX idx_v2 ON test_bloom_filter_named_create_index""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_create_index' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + qt_desc_named_create_index_after_drop """DESC test_bloom_filter_named_create_index""" + qt_show_index_named_create_index_after_drop """SHOW INDEX FROM test_bloom_filter_named_create_index""" + + test { + sql """ + CREATE TABLE test_bloom_filter_cov_inline_invalid_type ( + k1 INT, + f1 FLOAT, + INDEX idx_f1 (f1) USING BLOOMFILTER + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + exception "FLOAT is not supported in bloom filter index. invalid column: f1" + } + + sql """ + CREATE TABLE test_bloom_filter_cov_add_index_invalid_type ( + k1 INT, + f1 FLOAT + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + test { + sql """ALTER TABLE test_bloom_filter_cov_add_index_invalid_type ADD INDEX idx_f1 (f1) USING BLOOMFILTER""" + exception "FLOAT is not supported in bloom filter index. invalid column: f1" + } + + sql """ + CREATE TABLE test_bloom_filter_cov_add_index_with_properties ( + k1 INT, + v1 STRING + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + test { + sql """ALTER TABLE test_bloom_filter_cov_add_index_with_properties + ADD INDEX idx_v1 (v1) USING BLOOMFILTER PROPERTIES ("foo" = "bar")""" + exception "BLOOMFILTER index only supports property bloom_filter_fpp" + } + + sql """ + CREATE TABLE test_bloom_filter_cov_legacy_same_columns ( + k1 INT, + v1 STRING + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "bloom_filter_columns" = "k1" + ) + """ + + test { + sql """ALTER TABLE test_bloom_filter_cov_legacy_same_columns SET ("bloom_filter_columns" = "k1")""" + exception "Bloom filter index has no change" + } + + sql """ + CREATE TABLE test_bloom_filter_cov_legacy_same_fpp ( + k1 INT, + v1 STRING + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "bloom_filter_columns" = "k1", + "bloom_filter_fpp" = "0.05" + ) + """ + + test { + sql """ALTER TABLE test_bloom_filter_cov_legacy_same_fpp + SET ("bloom_filter_columns" = "k1", "bloom_filter_fpp" = "0.05")""" + exception "Can only set one table property(without dynamic partition && binlog) at a time" + } + + sql """ + CREATE TABLE test_bloom_filter_cov_no_bf_fpp_only ( + k1 INT, + v1 STRING + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + test { + sql """ALTER TABLE test_bloom_filter_cov_no_bf_fpp_only SET ("bloom_filter_fpp" = "0.01")""" + exception "Bloom filter index has no change" + } + + test { + sql """ + CREATE TABLE test_bloom_filter_cov_empty_legacy_columns ( + k1 INT, + v1 STRING + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "bloom_filter_columns" = "" + ) + """ + exception "Unknown properties: {bloom_filter_columns=}" + } + + test { + sql """ + CREATE TABLE test_bloom_filter_named_array_invalid ( + k1 INT, + v ARRAY, + INDEX idx_v (v) USING BLOOMFILTER PROPERTIES ("bloom_filter_fpp" = "0.03") + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + exception "is not supported" + } + + test { + sql """ + CREATE TABLE test_bloom_filter_named_map_invalid ( + k1 INT, + v MAP, + INDEX idx_v (v) USING BLOOMFILTER PROPERTIES ("bloom_filter_fpp" = "0.03") + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + exception "is not supported" + } + + test { + sql """ + CREATE TABLE test_bloom_filter_named_struct_invalid ( + k1 INT, + v STRUCT, + INDEX idx_v (v) USING BLOOMFILTER PROPERTIES ("bloom_filter_fpp" = "0.03") + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + exception "is not supported" + } + + sql """ + CREATE TABLE test_bloom_filter_named_variant ( + k1 INT, + v VARIANT, + INDEX idx_v (v) USING BLOOMFILTER PROPERTIES ("bloom_filter_fpp" = "0.03") + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ) + """ + + sql """INSERT INTO test_bloom_filter_named_variant VALUES + (1, '{"a": 1, "b": "alpha", "c": {"d": 10}, "e": [1, 2]}'), + (2, '{"a": 2, "b": "beta", "c": {"d": 20}, "e": [3, 4]}')""" + qt_show_index_named_variant """SHOW INDEX FROM test_bloom_filter_named_variant""" + + try { + GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexWriter::create", [fpp: "0.03"]) + sql """INSERT INTO test_bloom_filter_named_variant VALUES + (3, '{"a": 3, "b": "gamma", "c": {"d": 30}, "e": [5, 6]}')""" + } finally { + GetDebugPoint().disableDebugPointForAllBEs("BloomFilterIndexWriter::create") + } + + // Test: table-level bloom_filter_fpp and index-level bloom_filter_fpp are independent. + // Legacy bloom_filter_columns use the table-level FPP, while named BLOOMFILTER indexes + // use their per-index FPP from PROPERTIES. We test each source in isolation — only one + // type of bloom filter is active during each INSERT, so the global debug point can + // assert the correct FPP for every bloom filter writer that fires. + def test_mixed_fpp_tb = "test_mixed_fpp_bloom_filter_tb" + sql """CREATE TABLE IF NOT EXISTS ${test_mixed_fpp_tb} ( + `k1` int(11) NOT NULL, + `v1` varchar(50) NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 5 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true", + "bloom_filter_columns" = "k1", + "bloom_filter_fpp" = "0.03" + )""" + + // + try { + GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexWriter::create", [fpp: "0.03"]) + sql """ INSERT INTO ${test_mixed_fpp_tb} VALUES (1, 'aaa'), (2, 'bbb'), (3, 'ccc') """ + } finally { + GetDebugPoint().disableDebugPointForAllBEs("BloomFilterIndexWriter::create"); + } + + // Step 2: remove legacy BF, create a named BLOOMFILTER index on v1 with its own FPP. + // Only the named index is active → index-level FPP (0.02) is used. + sql """ ALTER TABLE ${test_mixed_fpp_tb} SET ("bloom_filter_columns" = "") """ + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_mixed_fpp_bloom_filter_tb' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + sql """ CREATE INDEX idx_v1_bf ON ${test_mixed_fpp_tb} (v1) USING BLOOMFILTER + PROPERTIES("bloom_filter_fpp" = "0.02") """ + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_mixed_fpp_bloom_filter_tb' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexWriter::create", [fpp: "0.02"]) + sql """ INSERT INTO ${test_mixed_fpp_tb} VALUES (4, 'ddd'), (5, 'eee'), (6, 'fff') """ + } finally { + GetDebugPoint().disableDebugPointForAllBEs("BloomFilterIndexWriter::create"); + } + + // Step 3: drop named index, restore legacy BF on k1 → table-level FPP (0.03) again. + sql """ DROP INDEX idx_v1_bf ON ${test_mixed_fpp_tb} """ + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_mixed_fpp_bloom_filter_tb' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + sql """ ALTER TABLE ${test_mixed_fpp_tb} SET ("bloom_filter_columns" = "k1") """ + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_mixed_fpp_bloom_filter_tb' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + try { + GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexWriter::create", [fpp: "0.05"]) + sql """ INSERT INTO ${test_mixed_fpp_tb} VALUES (7, 'ggg'), (8, 'hhh'), (9, 'iii') """ + } finally { + GetDebugPoint().disableDebugPointForAllBEs("BloomFilterIndexWriter::create"); + } + + sql """ ALTER TABLE ${test_mixed_fpp_tb} SET ("bloom_filter_fpp" = "0.03") """ + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_mixed_fpp_bloom_filter_tb' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + try { + GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexWriter::create", [fpp: "0.03"]) + sql """ INSERT INTO ${test_mixed_fpp_tb} VALUES (10, 'jjj'), (11, 'kkk'), (9, 'lll') """ + } finally { + GetDebugPoint().disableDebugPointForAllBEs("BloomFilterIndexWriter::create"); + } +}