From 047a93300b1aa875d2ef9df54dd603ccb94ee1c4 Mon Sep 17 00:00:00 2001 From: hoshinojyunn Date: Sun, 21 Jun 2026 14:03:16 +0800 Subject: [PATCH] [feat](fe) Support named bloom filter indexes in DDL and schema change Assumption / semantic boundary: A named bloom filter index and a property-managed bloom filter index cannot be defined on the same column. The two metadata forms are managed separately, and users must use `DROP INDEX` for named bloom filter indexes and `ALTER TABLE SET("bloom_filter_columns" = ...)` for property-managed bloom filter indexes. Problem Summary: Previously Doris only supported bloom filter indexes through table properties such as `"bloom_filter_columns"` and `"bloom_filter_fpp"`. That made bloom filter indexes behave differently from other index types and prevented users from managing them with named `INDEX` / `CREATE INDEX` / `DROP INDEX` syntax. This change adds named `USING BLOOMFILTER` syntax in the Nereids parser and FE DDL pipeline, keeps the existing table-property bloom filter behavior for legacy metadata, and enforces that legacy bloom filter columns and named bloom filter indexes cannot be defined on the same column. The schema change path now distinguishes legacy bloom filter management from named bloom filter management. Named BF indexes using index-level `bloom_filter_fpp` properties while legacy bloom filter columns use table-level `bloom_filter_fpp` properties. The patch also fixes bloom filter materialization on shadow columns and ensures tablet metadata marks named bloom filter columns correctly on the BE side. FE unit tests and bloom filter regression tests are added to cover parser analysis, semantic validation, schema change checks, FE->BE task generation, and named bloom filter DDL behavior. --- be/src/storage/segment/segment_writer.cpp | 2 +- .../variant/variant_column_writer_impl.cpp | 3 + .../segment/vertical_segment_writer.cpp | 3 +- be/src/storage/tablet/tablet_schema.cpp | 27 + be/src/storage/tablet/tablet_schema.h | 2 + .../segment/bloom_filter_fpp_writer_test.cpp | 364 +++++++++++ be/test/storage/tablet/tablet_schema_test.cpp | 178 ++++++ .../variant_column_writer_reader_test.cpp | 96 +++ .../java/org/apache/doris/catalog/Index.java | 26 + .../doris/alter/SchemaChangeHandler.java | 172 +++-- .../apache/doris/alter/SchemaChangeJobV2.java | 1 - .../doris/catalog/ColumnToProtobuf.java | 8 +- .../apache/doris/catalog/ColumnToThrift.java | 4 +- .../java/org/apache/doris/catalog/Env.java | 22 +- .../org/apache/doris/catalog/OlapTable.java | 13 +- .../datasource/CloudInternalCatalog.java | 4 +- .../doris/common/proc/IndexInfoProcDir.java | 11 +- .../doris/datasource/InternalCatalog.java | 5 +- .../nereids/parser/LogicalPlanBuilder.java | 2 + .../trees/plans/commands/DescribeCommand.java | 10 +- .../plans/commands/info/IndexDefinition.java | 47 +- .../apache/doris/task/CreateReplicaTask.java | 5 +- .../apache/doris/alter/CloudIndexTest.java | 125 ++++ .../doris/alter/SchemaChangeHandlerTest.java | 117 ++++ .../doris/alter/SchemaChangeJobV2Test.java | 138 ++++ .../ColumnBloomFilterMaterializationTest.java | 132 ++++ .../CreateTableWithBloomFilterIndexTest.java | 468 ++++++++++++++ .../apache/doris/catalog/OlapTableTest.java | 29 + ...CatalogBloomFilterMaterializationTest.java | 175 ++++++ .../nereids/parser/NereidsParserTest.java | 80 +++ .../plans/commands/IndexDefinitionTest.java | 181 ++++++ .../org/apache/doris/task/AgentTaskTest.java | 99 +++ .../org/apache/doris/nereids/DorisLexer.g4 | 1 + .../org/apache/doris/nereids/DorisParser.g4 | 5 +- .../test_bloom_filter_named_index.out | 109 ++++ .../bloom_filter_p0/test_bloom_filter.groovy | 15 +- .../test_bloom_filter_named_index.groovy | 595 ++++++++++++++++++ 37 files changed, 3188 insertions(+), 86 deletions(-) create mode 100644 be/test/storage/segment/bloom_filter_fpp_writer_test.cpp create mode 100644 fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnBloomFilterMaterializationTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogBloomFilterMaterializationTest.java create mode 100644 regression-test/data/bloom_filter_p0/test_bloom_filter_named_index.out create mode 100644 regression-test/suites/bloom_filter_p0/test_bloom_filter_named_index.groovy diff --git a/be/src/storage/segment/segment_writer.cpp b/be/src/storage/segment/segment_writer.cpp index 6f4ecef1140d67..d6e1e162d1fa8f 100644 --- a/be/src/storage/segment/segment_writer.cpp +++ b/be/src/storage/segment/segment_writer.cpp @@ -196,7 +196,7 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co opts.need_zone_map = column.is_key() || schema->keys_type() != KeysType::AGG_KEYS; opts.need_bloom_filter = column.is_bf_column(); if (opts.need_bloom_filter) { - opts.bf_options.fpp = schema->has_bf_fpp() ? schema->bloom_filter_fpp() : 0.05; + opts.bf_options.fpp = schema->get_bloom_filter_fpp(column); } auto* tablet_index = schema->get_ngram_bf_index(column.unique_id()); if (tablet_index) { diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.cpp b/be/src/storage/segment/variant/variant_column_writer_impl.cpp index 1e8610f42d886f..321386b95f9fe4 100644 --- a/be/src/storage/segment/variant/variant_column_writer_impl.cpp +++ b/be/src/storage/segment/variant/variant_column_writer_impl.cpp @@ -102,6 +102,9 @@ Status _create_column_writer(uint32_t cid, const TabletColumn& column, } opt->need_zone_map = tablet_schema->keys_type() != KeysType::AGG_KEYS; opt->need_bloom_filter = column.is_bf_column(); + if (opt->need_bloom_filter) { + opt->bf_options.fpp = tablet_schema->get_bloom_filter_fpp(column); + } const auto& parent_index = tablet_schema->inverted_indexs(column.parent_unique_id()); // init inverted index diff --git a/be/src/storage/segment/vertical_segment_writer.cpp b/be/src/storage/segment/vertical_segment_writer.cpp index f5b60ebadb756c..f625fcae396399 100644 --- a/be/src/storage/segment/vertical_segment_writer.cpp +++ b/be/src/storage/segment/vertical_segment_writer.cpp @@ -204,8 +204,7 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo opts.need_zone_map = column.is_key() || tablet_schema->keys_type() != KeysType::AGG_KEYS; opts.need_bloom_filter = column.is_bf_column(); if (opts.need_bloom_filter) { - opts.bf_options.fpp = - tablet_schema->has_bf_fpp() ? tablet_schema->bloom_filter_fpp() : 0.05; + opts.bf_options.fpp = tablet_schema->get_bloom_filter_fpp(column); } auto* tablet_index = tablet_schema->get_ngram_bf_index(column.unique_id()); if (tablet_index) { diff --git a/be/src/storage/tablet/tablet_schema.cpp b/be/src/storage/tablet/tablet_schema.cpp index 3158b229837890..93012c55991686 100644 --- a/be/src/storage/tablet/tablet_schema.cpp +++ b/be/src/storage/tablet/tablet_schema.cpp @@ -27,11 +27,13 @@ #include #include // IWYU pragma: no_include +#include #include // IWYU pragma: keep #include #include #include +#include "common/check.h" #include "common/compiler_util.h" // IWYU pragma: keep #include "common/consts.h" #include "common/status.h" @@ -40,6 +42,7 @@ #include "core/data_type/data_type.h" #include "core/data_type/data_type_factory.hpp" #include "core/string_ref.h" +#include "core/types.h" #include "exec/common/hex.h" #include "exprs/aggregate/aggregate_function_simple_factory.h" #include "exprs/aggregate/aggregate_function_state_union.h" @@ -1868,6 +1871,30 @@ const TabletIndex* TabletSchema::get_ngram_bf_index(int32_t col_unique_id) const return nullptr; } +double TabletSchema::get_bloom_filter_fpp(int32_t col_unique_id) const { + const auto* bloom_filter_index = get_index(col_unique_id, IndexType::BLOOMFILTER, ""); + if (bloom_filter_index != nullptr) { + const auto& properties = bloom_filter_index->properties(); + auto iter = properties.find("bloom_filter_fpp"); + if (iter != properties.end()) { + StringParser::ParseResult parse_result = StringParser::PARSE_FAILURE; + auto index_level_fpp = StringParser::string_to_float( + iter->second.data(), iter->second.size(), &parse_result); + DORIS_CHECK(parse_result == StringParser::PARSE_SUCCESS) + << "failed to parse '" << iter->second << "' as double"; + return index_level_fpp; + } + return BLOOM_FILTER_DEFAULT_FPP; + } + return has_bf_fpp() ? bloom_filter_fpp() : BLOOM_FILTER_DEFAULT_FPP; +} + +double TabletSchema::get_bloom_filter_fpp(const TabletColumn& column) const { + const int32_t col_unique_id = + column.is_extracted_column() ? column.parent_unique_id() : column.unique_id(); + return get_bloom_filter_fpp(col_unique_id); +} + const TabletIndex* TabletSchema::get_index(int32_t col_unique_id, IndexType index_type, const std::string& suffix_path) const { IndexKey index_key(index_type, col_unique_id, suffix_path); diff --git a/be/src/storage/tablet/tablet_schema.h b/be/src/storage/tablet/tablet_schema.h index bf4598e2bb94fd..3885d02cc4307f 100644 --- a/be/src/storage/tablet/tablet_schema.h +++ b/be/src/storage/tablet/tablet_schema.h @@ -598,6 +598,8 @@ class TabletSchema : public MetadataAdder { bool has_ngram_bf_index(int32_t col_unique_id) const; const TabletIndex* get_ngram_bf_index(int32_t col_unique_id) const; + double get_bloom_filter_fpp(int32_t col_unique_id) const; + double get_bloom_filter_fpp(const TabletColumn& column) const; const TabletIndex* get_index(int32_t col_unique_id, IndexType index_type, const std::string& suffix_path) const; void update_indexes_from_thrift(const std::vector& indexes); diff --git a/be/test/storage/segment/bloom_filter_fpp_writer_test.cpp b/be/test/storage/segment/bloom_filter_fpp_writer_test.cpp new file mode 100644 index 00000000000000..e2c5c84c734e6b --- /dev/null +++ b/be/test/storage/segment/bloom_filter_fpp_writer_test.cpp @@ -0,0 +1,364 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include + +#include "common/config.h" +#include "core/block/block.h" +#include "io/fs/local_file_system.h" +#include "storage/olap_common.h" +#include "storage/rowset/rowset_writer_context.h" +#include "storage/segment/column_writer.h" +#include "storage/segment/encoding_info.h" +#include "storage/segment/segment_writer.h" +#include "storage/segment/vertical_segment_writer.h" +#include "storage/tablet/tablet_schema.h" +#include "util/debug_points.h" + +namespace doris::segment_v2 { + +namespace { + +constexpr std::string_view kTestDir = "./ut_dir/bloom_filter_fpp_writer_test"; +constexpr std::string_view kBloomFilterCreateDebugPoint = "BloomFilterIndexWriter::create"; + +class ScopedBloomFilterFppDebugPoint { +public: + explicit ScopedBloomFilterFppDebugPoint(double expected_fpp) + : _old_enable_debug_points(config::enable_debug_points) { + config::enable_debug_points = true; + DebugPoints::instance()->add_with_params(std::string(kBloomFilterCreateDebugPoint), + {{"fpp", std::to_string(expected_fpp)}}); + } + + ~ScopedBloomFilterFppDebugPoint() { + DebugPoints::instance()->remove(std::string(kBloomFilterCreateDebugPoint)); + config::enable_debug_points = _old_enable_debug_points; + } + +private: + bool _old_enable_debug_points; +}; + +struct ScalarBloomFilterTypeCase { + const char* name; + FieldType type; + int32_t length; + int32_t precision; + int32_t frac; +}; + +int32_t scalar_test_column_length(FieldType type) { + switch (type) { + case FieldType::OLAP_FIELD_TYPE_CHAR: + return 16; + case FieldType::OLAP_FIELD_TYPE_VARCHAR: + return 32; + case FieldType::OLAP_FIELD_TYPE_STRING: + return 64; + default: + return cast_set(field_type_size(type)); + } +} + +std::vector create_scalar_bloom_filter_type_cases() { + return { + {"bool", FieldType::OLAP_FIELD_TYPE_BOOL, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_BOOL), -1, -1}, + {"tinyint", FieldType::OLAP_FIELD_TYPE_TINYINT, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_TINYINT), -1, -1}, + {"smallint", FieldType::OLAP_FIELD_TYPE_SMALLINT, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_SMALLINT), -1, -1}, + {"int", FieldType::OLAP_FIELD_TYPE_INT, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_INT), -1, -1}, + {"unsigned_int", FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT), -1, -1}, + {"bigint", FieldType::OLAP_FIELD_TYPE_BIGINT, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_BIGINT), -1, -1}, + {"largeint", FieldType::OLAP_FIELD_TYPE_LARGEINT, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_LARGEINT), -1, -1}, + {"char", FieldType::OLAP_FIELD_TYPE_CHAR, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_CHAR), -1, -1}, + {"varchar", FieldType::OLAP_FIELD_TYPE_VARCHAR, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_VARCHAR), -1, -1}, + {"string", FieldType::OLAP_FIELD_TYPE_STRING, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_STRING), -1, -1}, + {"date", FieldType::OLAP_FIELD_TYPE_DATE, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_DATE), -1, -1}, + {"datetime", FieldType::OLAP_FIELD_TYPE_DATETIME, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_DATETIME), -1, -1}, + {"decimal_v2", FieldType::OLAP_FIELD_TYPE_DECIMAL, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_DECIMAL), 27, 9}, + {"datev2", FieldType::OLAP_FIELD_TYPE_DATEV2, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_DATEV2), -1, -1}, + {"datetimev2", FieldType::OLAP_FIELD_TYPE_DATETIMEV2, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_DATETIMEV2), -1, 6}, + {"timestamptz", FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ), -1, 6}, + {"decimal32", FieldType::OLAP_FIELD_TYPE_DECIMAL32, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_DECIMAL32), 9, 2}, + {"decimal64", FieldType::OLAP_FIELD_TYPE_DECIMAL64, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_DECIMAL64), 18, 4}, + {"decimal128i", FieldType::OLAP_FIELD_TYPE_DECIMAL128I, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_DECIMAL128I), 38, 6}, + {"decimal256", FieldType::OLAP_FIELD_TYPE_DECIMAL256, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_DECIMAL256), 76, 10}, + {"ipv4", FieldType::OLAP_FIELD_TYPE_IPV4, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_IPV4), -1, -1}, + {"ipv6", FieldType::OLAP_FIELD_TYPE_IPV6, + scalar_test_column_length(FieldType::OLAP_FIELD_TYPE_IPV6), -1, -1}, + }; +} + +const std::vector kScalarBloomFilterTypeCases = + create_scalar_bloom_filter_type_cases(); + +std::vector create_segment_writer_compatible_type_cases() { + std::vector compatible_type_cases; + compatible_type_cases.reserve(kScalarBloomFilterTypeCases.size()); + for (const auto& type_case : kScalarBloomFilterTypeCases) { + // BF itself supports UNSIGNED_INT, but the current block/convertor stack used by + // SegmentWriter/VerticalSegmentWriter still does not materialize that type end-to-end. + // Keep its coverage at direct ColumnWriter level instead of failing these integration tests + // for an unrelated legacy limitation. + if (type_case.type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT) { + continue; + } + compatible_type_cases.push_back(type_case); + } + return compatible_type_cases; +} + +const std::vector kSegmentWriterCompatibleBloomFilterTypeCases = + create_segment_writer_compatible_type_cases(); + +Block create_single_row_default_block(const TabletSchemaSPtr& schema) { + Block block = schema->create_block(); + for (uint32_t cid = 0; cid < schema->num_columns(); ++cid) { + auto column = block.get_by_position(cid).column->assert_mutable(); + column->insert_default(); + block.replace_by_position(cid, std::move(column)); + } + return block; +} + +TabletIndexPB create_bloom_filter_index_pb(int64_t index_id, int32_t column_uid, double fpp) { + TabletIndexPB index_pb; + index_pb.set_index_id(index_id); + index_pb.set_index_name("idx_" + std::to_string(column_uid)); + index_pb.set_index_type(IndexType::BLOOMFILTER); + index_pb.add_col_unique_id(column_uid); + (*index_pb.mutable_properties())["bloom_filter_fpp"] = std::to_string(fpp); + return index_pb; +} + +io::FileWriterPtr create_file_writer(std::string_view file_name) { + io::FileWriterPtr file_writer; + auto path = std::string(kTestDir) + "/" + std::string(file_name); + auto st = io::global_local_filesystem()->create_file(path, &file_writer); + EXPECT_TRUE(st.ok()) << st; + return file_writer; +} + +} // namespace + +class BloomFilterFppWriterTest : public testing::Test { +protected: + void SetUp() override { + auto fs = io::global_local_filesystem(); + auto st = fs->delete_directory(kTestDir); + ASSERT_TRUE(st.ok() || st.is()) << st; + st = fs->create_directory(kTestDir); + ASSERT_TRUE(st.ok()) << st; + } + + void TearDown() override { + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(kTestDir).ok()); + DebugPoints::instance()->remove(std::string(kBloomFilterCreateDebugPoint)); + DebugPoints::instance()->clear(); + config::enable_debug_points = false; + } +}; + +class ScalarBloomFilterFppWriterTestBase : public BloomFilterFppWriterTest { +protected: + static TabletSchemaSPtr create_schema_with_scalar_bloom_filter_index( + const ScalarBloomFilterTypeCase& type_case, double fpp) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::AGG_KEYS); + + auto* key_column = schema_pb.add_column(); + key_column->set_unique_id(0); + key_column->set_name("k1"); + key_column->set_type("INT"); + key_column->set_is_key(true); + key_column->set_is_nullable(false); + key_column->set_length(4); + key_column->set_index_length(4); + key_column->set_aggregation("NONE"); + + auto* value_column = schema_pb.add_column(); + value_column->set_unique_id(1); + value_column->set_name(type_case.name); + value_column->set_type(TabletColumn::get_string_by_field_type(type_case.type)); + value_column->set_is_key(false); + value_column->set_is_nullable(true); + value_column->set_length(type_case.length); + // Use an AGG_KEYS value column so SegmentWriter won't create a zone map for it. + // That keeps this test focused on BF creation/fpp propagation for the full BF type set. + value_column->set_aggregation("REPLACE"); + value_column->set_is_bf_column(true); + if (type_case.precision >= 0) { + value_column->set_precision(type_case.precision); + } + if (type_case.frac >= 0) { + value_column->set_frac(type_case.frac); + } + + *schema_pb.add_index() = create_bloom_filter_index_pb(1, 1, fpp); + + auto schema = std::make_shared(); + schema->init_from_pb(schema_pb); + return schema; + } + + static TabletColumn create_scalar_bloom_filter_column( + const ScalarBloomFilterTypeCase& type_case, int32_t unique_id) { + TabletColumn column(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, type_case.type, + true, unique_id, type_case.length); + column.set_name(type_case.name); + column.set_is_bf_column(true); + if (type_case.precision >= 0) { + column.set_precision(type_case.precision); + } + if (type_case.frac >= 0) { + column.set_frac(type_case.frac); + } + return column; + } + + static ColumnMetaPB create_scalar_column_meta(const TabletColumn& column, uint32_t column_id) { + ColumnMetaPB meta; + meta.set_column_id(column_id); + meta.set_unique_id(column.unique_id()); + meta.set_type(static_cast(column.type())); + meta.set_length(column.length()); + meta.set_encoding(EncodingInfo::resolve_default_encoding( + TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2, column)); + meta.set_compression(CompressionTypePB::LZ4F); + meta.set_is_nullable(column.is_nullable()); + meta.set_precision(column.precision()); + meta.set_frac(column.frac()); + return meta; + } +}; + +class AllScalarBloomFilterFppWriterTest + : public ScalarBloomFilterFppWriterTestBase, + public testing::WithParamInterface {}; + +class SegmentWriterCompatibleBloomFilterFppWriterTest + : public ScalarBloomFilterFppWriterTestBase, + public testing::WithParamInterface {}; + +TEST_P(AllScalarBloomFilterFppWriterTest, column_writer_uses_bloom_filter_fpp) { + const auto& type_case = GetParam(); + constexpr double kExpectedFpp = 0.03; + + auto column = create_scalar_bloom_filter_column(type_case, 1); + auto meta = create_scalar_column_meta(column, 0); + + ColumnWriterOptions writer_opts; + writer_opts.meta = &meta; + writer_opts.need_bloom_filter = true; + writer_opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + writer_opts.bf_options.fpp = kExpectedFpp; + + auto file_writer = create_file_writer(std::string("column_writer_") + type_case.name + ".dat"); + std::unique_ptr writer; + ASSERT_TRUE(ColumnWriter::create(writer_opts, &column, file_writer.get(), &writer).ok()); + + // Cover the full scalar BF type matrix directly at ColumnWriter level, including + // UNSIGNED_INT which cannot currently flow through the segment writer block/convertor stack. + ScopedBloomFilterFppDebugPoint debug_point(kExpectedFpp); + auto st = writer->init(); + ASSERT_TRUE(st.ok()) << st; +} + +// SegmentWriter uses the same per-column fpp lookup regardless of scalar type, so this suite +// exercises the full integration path for every writer-compatible BF scalar type. +TEST_P(SegmentWriterCompatibleBloomFilterFppWriterTest, segment_writer_uses_index_level_fpp) { + const auto& type_case = GetParam(); + constexpr double kExpectedFpp = 0.03; + + auto schema = create_schema_with_scalar_bloom_filter_index(type_case, kExpectedFpp); + + SegmentWriterOptions opts; + auto file_writer = create_file_writer(std::string("segment_writer_") + type_case.name + ".dat"); + SegmentWriter writer(file_writer.get(), 0, schema, nullptr, nullptr, opts, nullptr); + + ScopedBloomFilterFppDebugPoint debug_point(kExpectedFpp); + auto st = writer.init(); + ASSERT_TRUE(st.ok()) << st; +} + +TEST_P(SegmentWriterCompatibleBloomFilterFppWriterTest, + vertical_segment_writer_uses_index_level_fpp) { + const auto& type_case = GetParam(); + constexpr double kExpectedFpp = 0.02; + + auto schema = create_schema_with_scalar_bloom_filter_index(type_case, kExpectedFpp); + Block block = create_single_row_default_block(schema); + + VerticalSegmentWriterOptions opts; + RowsetWriterContext rowset_ctx; + rowset_ctx.write_type = DataWriteType::TYPE_DEFAULT; + rowset_ctx.tablet_schema = schema; + opts.rowset_ctx = &rowset_ctx; + auto file_writer = + create_file_writer(std::string("vertical_segment_writer_") + type_case.name + ".dat"); + VerticalSegmentWriter writer(file_writer.get(), 0, schema, nullptr, nullptr, opts, nullptr); + + ASSERT_TRUE(writer.init().ok()); + ASSERT_TRUE(writer.batch_block(&block, 0, 1).ok()); + + // VerticalSegmentWriter only creates ColumnWriter instances during write_batch(), so use a + // real one-row block here to exercise the BF fpp propagation path in _create_column_writer(). + ScopedBloomFilterFppDebugPoint debug_point(kExpectedFpp); + auto st = writer.write_batch(); + ASSERT_TRUE(st.ok()) << st; +} + +INSTANTIATE_TEST_SUITE_P(AllScalarBloomFilterTypes, AllScalarBloomFilterFppWriterTest, + testing::ValuesIn(kScalarBloomFilterTypeCases), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +INSTANTIATE_TEST_SUITE_P(SegmentWriterCompatibleBloomFilterTypes, + SegmentWriterCompatibleBloomFilterFppWriterTest, + testing::ValuesIn(kSegmentWriterCompatibleBloomFilterTypeCases), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +} // namespace doris::segment_v2 diff --git a/be/test/storage/tablet/tablet_schema_test.cpp b/be/test/storage/tablet/tablet_schema_test.cpp index 6b29eea1d55333..347b7aa1a8146a 100644 --- a/be/test/storage/tablet/tablet_schema_test.cpp +++ b/be/test/storage/tablet/tablet_schema_test.cpp @@ -318,6 +318,184 @@ TEST_F(TabletSchemaTest, test_tablet_schema_update_indexes_from_thrift) { EXPECT_TRUE(found_varchar_idx); } +TEST_F(TabletSchemaTest, test_get_bloom_filter_fpp_prefers_index_property) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_bf_fpp(0.11); + + auto* value_column = schema_pb.add_column(); + value_column->set_unique_id(1001); + value_column->set_name("v1"); + value_column->set_type("INT"); + value_column->set_is_key(false); + value_column->set_is_nullable(true); + value_column->set_length(4); + value_column->set_aggregation("NONE"); + value_column->set_is_bf_column(true); + + auto* bf_index = schema_pb.add_index(); + bf_index->set_index_id(1); + bf_index->set_index_name("idx_v1"); + bf_index->set_index_type(IndexType::BLOOMFILTER); + bf_index->add_col_unique_id(1001); + (*bf_index->mutable_properties())["bloom_filter_fpp"] = "0.03"; + + TabletSchema schema; + schema.init_from_pb(schema_pb); + + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(1001), 0.03); + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(schema.column_by_uid(1001)), 0.03); +} + +TEST_F(TabletSchemaTest, test_get_bloom_filter_fpp_named_index_without_property_uses_default) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_bf_fpp(0.07); + + auto* value_column = schema_pb.add_column(); + value_column->set_unique_id(1002); + value_column->set_name("v2"); + value_column->set_type("INT"); + value_column->set_is_key(false); + value_column->set_is_nullable(true); + value_column->set_length(4); + value_column->set_aggregation("NONE"); + value_column->set_is_bf_column(true); + + auto* bf_index = schema_pb.add_index(); + bf_index->set_index_id(2); + bf_index->set_index_name("idx_v2"); + bf_index->set_index_type(IndexType::BLOOMFILTER); + bf_index->add_col_unique_id(1002); + + TabletSchema schema; + schema.init_from_pb(schema_pb); + + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(1002), BLOOM_FILTER_DEFAULT_FPP); + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(schema.column_by_uid(1002)), + BLOOM_FILTER_DEFAULT_FPP); +} + +TEST_F(TabletSchemaTest, + test_get_bloom_filter_fpp_named_index_without_property_ignores_table_level_fpp) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_bf_fpp(0.07); + + auto* legacy_bf_column = schema_pb.add_column(); + legacy_bf_column->set_unique_id(1003); + legacy_bf_column->set_name("legacy_bf_col"); + legacy_bf_column->set_type("INT"); + legacy_bf_column->set_is_key(false); + legacy_bf_column->set_is_nullable(true); + legacy_bf_column->set_length(4); + legacy_bf_column->set_aggregation("NONE"); + legacy_bf_column->set_is_bf_column(true); + + auto* named_bf_column = schema_pb.add_column(); + named_bf_column->set_unique_id(1004); + named_bf_column->set_name("named_bf_col"); + named_bf_column->set_type("INT"); + named_bf_column->set_is_key(false); + named_bf_column->set_is_nullable(true); + named_bf_column->set_length(4); + named_bf_column->set_aggregation("NONE"); + named_bf_column->set_is_bf_column(true); + + auto* bf_index = schema_pb.add_index(); + bf_index->set_index_id(4); + bf_index->set_index_name("idx_named_bf_col"); + bf_index->set_index_type(IndexType::BLOOMFILTER); + bf_index->add_col_unique_id(1004); + + TabletSchema schema; + schema.init_from_pb(schema_pb); + + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(1003), 0.07); + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(1004), BLOOM_FILTER_DEFAULT_FPP); +} + +TEST_F(TabletSchemaTest, test_get_bloom_filter_fpp_legacy_column_uses_table_property) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_bf_fpp(0.07); + + auto* value_column = schema_pb.add_column(); + value_column->set_unique_id(1004); + value_column->set_name("v4"); + value_column->set_type("INT"); + value_column->set_is_key(false); + value_column->set_is_nullable(true); + value_column->set_length(4); + value_column->set_aggregation("NONE"); + value_column->set_is_bf_column(true); + + TabletSchema schema; + schema.init_from_pb(schema_pb); + + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(1004), 0.07); + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(schema.column_by_uid(1004)), 0.07); +} + +TEST_F(TabletSchemaTest, test_get_bloom_filter_fpp_uses_default_without_table_property) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + + auto* value_column = schema_pb.add_column(); + value_column->set_unique_id(1003); + value_column->set_name("v3"); + value_column->set_type("INT"); + value_column->set_is_key(false); + value_column->set_is_nullable(true); + value_column->set_length(4); + value_column->set_aggregation("NONE"); + value_column->set_is_bf_column(true); + + TabletSchema schema; + schema.init_from_pb(schema_pb); + + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(1003), BLOOM_FILTER_DEFAULT_FPP); + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(schema.column_by_uid(1003)), + BLOOM_FILTER_DEFAULT_FPP); +} + +TEST_F(TabletSchemaTest, test_get_bloom_filter_fpp_for_extracted_column_uses_parent_index) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_bf_fpp(0.09); + + auto* variant_column = schema_pb.add_column(); + variant_column->set_unique_id(2001); + variant_column->set_name("v"); + variant_column->set_type("VARIANT"); + variant_column->set_is_key(false); + variant_column->set_is_nullable(true); + variant_column->set_length(4); + variant_column->set_aggregation("NONE"); + variant_column->set_is_bf_column(true); + variant_column->set_variant_max_subcolumns_count(8); + + auto* bf_index = schema_pb.add_index(); + bf_index->set_index_id(3); + bf_index->set_index_name("idx_v"); + bf_index->set_index_type(IndexType::BLOOMFILTER); + bf_index->add_col_unique_id(2001); + (*bf_index->mutable_properties())["bloom_filter_fpp"] = "0.02"; + + TabletSchema schema; + schema.init_from_pb(schema_pb); + + TabletColumn extracted; + extracted.set_name("v.a"); + extracted.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + extracted.set_parent_unique_id(2001); + extracted.set_path_info(PathInData("v.a")); + extracted.set_is_nullable(true); + + EXPECT_TRUE(extracted.is_extracted_column()); + EXPECT_DOUBLE_EQ(schema.get_bloom_filter_fpp(extracted), 0.02); +} + TEST_F(TabletSchemaTest, test_tablet_schema_append_index) { TabletSchema schema; diff --git a/be/test/storage/variant/variant_column_writer_reader_test.cpp b/be/test/storage/variant/variant_column_writer_reader_test.cpp index 191189092bdb24..be3b66a3e08ea9 100644 --- a/be/test/storage/variant/variant_column_writer_reader_test.cpp +++ b/be/test/storage/variant/variant_column_writer_reader_test.cpp @@ -42,6 +42,7 @@ #include "storage/segment/variant/variant_doc_snpashot_compact_iterator.h" #include "storage/storage_engine.h" #include "testutil/variant_util.h" +#include "util/debug_points.h" using namespace doris; @@ -3152,6 +3153,101 @@ TEST_F(VariantColumnWriterReaderTest, test_storage_parse_kv_reduces_sparse_parse EXPECT_GT(kv_result.segment_file_size, static_cast(0)); } +TEST_F(VariantColumnWriterReaderTest, + test_extracted_subcolumn_writer_uses_parent_bloom_filter_fpp) { + constexpr int kRows = 2; + constexpr double kExpectedFpp = 0.013; + + auto old_enable_debug_points = config::enable_debug_points; + config::enable_debug_points = true; + DebugPoints::instance()->add_with_params("BloomFilterIndexWriter::create", + {{"fpp", std::to_string(kExpectedFpp)}}); + Defer defer([old_enable_debug_points]() { + DebugPoints::instance()->remove("BloomFilterIndexWriter::create"); + config::enable_debug_points = old_enable_debug_points; + }); + + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + construct_column(schema_pb.add_column(), 1, "VARIANT", "V1", + /*variant_max_subcolumns_count=*/2, + /*is_key=*/false, + /*is_nullable=*/false, + /*variant_sparse_hash_shard_count=*/0, + /*variant_enable_doc_mode=*/false); + auto* bf_index = schema_pb.add_index(); + bf_index->set_index_id(10009); + bf_index->set_index_name("idx_v1_bf"); + bf_index->set_index_type(IndexType::BLOOMFILTER); + bf_index->add_col_unique_id(1); + (*bf_index->mutable_properties())["bloom_filter_fpp"] = std::to_string(kExpectedFpp); + + _tablet_schema = std::make_shared(); + _tablet_schema->init_from_pb(schema_pb); + + TabletColumn parent_column = _tablet_schema->column(0); + TabletColumn extracted; + extracted.set_name(parent_column.name_lower_case() + ".hot"); + extracted.set_type(FieldType::OLAP_FIELD_TYPE_BIGINT); + extracted.set_parent_unique_id(parent_column.unique_id()); + extracted.set_path_info(PathInData(parent_column.name_lower_case() + ".hot")); + extracted.set_is_nullable(true); + extracted.set_is_bf_column(true); + _tablet_schema->append_column(extracted); + + init_tablet_from_current_schema(33007, TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); + + io::FileWriterPtr file_writer; + auto file_path = local_segment_path(_tablet->tablet_path(), "0", 0); + auto st = io::global_local_filesystem()->create_file(file_path, &file_writer); + ASSERT_TRUE(st.ok()) << st.msg(); + + SegmentFooterPB footer; + RowsetWriterContext rowset_ctx; + rowset_ctx.write_type = DataWriteType::TYPE_DIRECT; + rowset_ctx.tablet_schema = _tablet_schema; + + ColumnWriterOptions opts; + opts.meta = footer.add_columns(); + opts.compression_type = CompressionTypePB::LZ4; + opts.file_writer = file_writer.get(); + opts.footer = &footer; + opts.rowset_ctx = &rowset_ctx; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, parent_column, opts); + + std::unique_ptr writer; + ASSERT_TRUE(ColumnWriter::create(opts, &parent_column, file_writer.get(), &writer).ok()); + ASSERT_TRUE(writer->init().ok()); + + auto strings = ColumnString::create(); + const std::vector jsons = {R"({"hot":1,"cold":10})", R"({"hot":2,"cold":20})"}; + for (const auto& json : jsons) { + strings->insert_data(json.data(), json.size()); + } + + ParseConfig parse_cfg; + parse_cfg.deprecated_enable_flatten_nested = false; + parse_cfg.parse_to = ParseConfig::ParseTo::OnlyDocValueColumn; + auto variant_column = + ColumnVariant::create(parent_column.variant_max_subcolumns_count(), false); + variant_util::parse_json_to_variant(*variant_column, *strings, parse_cfg); + + auto variant_data = std::make_unique(); + variant_data->column_data = variant_column.get(); + variant_data->row_pos = 0; + const auto* data = reinterpret_cast(variant_data.get()); + ASSERT_TRUE(writer->append_data(&data, kRows).ok()); + + // `finish()` materializes the extracted BF-enabled subcolumn and creates its inner scalar + // writer. The debug point verifies that the writer receives the parent bloom filter index fpp. + ASSERT_TRUE(writer->finish().ok()); + ASSERT_TRUE(writer->write_data().ok()); + ASSERT_TRUE(writer->write_ordinal_index().ok()); + ASSERT_TRUE(writer->write_zone_map().ok()); + ASSERT_TRUE(file_writer->close().ok()); +} + TEST_F(VariantColumnWriterReaderTest, test_write_data_advanced) { // 1. create tablet_schema TabletSchemaPB schema_pb; diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Index.java index da427934220436..edc0fcc21773e0 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Index.java @@ -43,6 +43,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; /** * Internal representation of index, including index type, name, columns and comments. @@ -272,6 +273,31 @@ public List getColumnUniqueIds(List schema) { return columnUniqueIds; } + public boolean isBloomFilterIndex() { + return indexType == IndexType.BLOOMFILTER; + } + + public static Set extractBloomFilterColumns(Collection indexes) { + Set bloomFilterColumns = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + if (indexes == null) { + return bloomFilterColumns; + } + for (Index index : indexes) { + if (!index.isBloomFilterIndex()) { + continue; + } + bloomFilterColumns.addAll(index.getColumns()); + } + return bloomFilterColumns; + } + + public static boolean hasBloomFilterIndex(Collection indexes, String columnName) { + if (columnName == null) { + return false; + } + return extractBloomFilterColumns(indexes).contains(columnName); + } + public static void checkConflict(Collection indices, Set bloomFilters) throws AnalysisException { indices = indices == null ? Collections.emptyList() : indices; bloomFilters = bloomFilters == null ? Collections.emptySet() : bloomFilters; diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 233cead1bdce43..79f5338d609358 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -741,18 +741,18 @@ private boolean processDropColumn(DropColumnOp dropColumnOp, OlapTable olapTable } // drop bloom filter column - Set bfCols = olapTable.getCopiedBfColumns(); - if (bfCols != null) { - Set newBfCols = null; - for (String bfCol : bfCols) { - if (!bfCol.equalsIgnoreCase(dropColName)) { - if (newBfCols == null) { - newBfCols = Sets.newHashSet(); + Set legacyBloomFilterColumns = olapTable.getCopiedBfColumns(); + if (legacyBloomFilterColumns != null) { + Set updatedLegacyBloomFilterColumns = null; + for (String legacyBloomFilterColumn : legacyBloomFilterColumns) { + if (!legacyBloomFilterColumn.equalsIgnoreCase(dropColName)) { + if (updatedLegacyBloomFilterColumns == null) { + updatedLegacyBloomFilterColumns = Sets.newHashSet(); } - newBfCols.add(bfCol); + updatedLegacyBloomFilterColumns.add(legacyBloomFilterColumn); } } - olapTable.setBloomFilterInfo(newBfCols, olapTable.getBfFpp()); + olapTable.setBloomFilterInfo(updatedLegacyBloomFilterColumns, olapTable.getBfFpp()); } for (int i = 1; i < indexIds.size(); i++) { @@ -1699,6 +1699,7 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map bfColumns = null; double bfFpp = 0; try { @@ -1711,27 +1712,38 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map oriBfColumns = olapTable.getCopiedBfColumns(); + Set oriLegacyBfCols = olapTable.getCopiedBfColumns(); + Set oriNamedBfCols = Index.extractBloomFilterColumns(olapTable.getIndexes()); + if (oriNamedBfCols.isEmpty()) { + oriNamedBfCols = null; + } + Set namedBfCols = Index.extractBloomFilterColumns(indexes); + if (namedBfCols.isEmpty()) { + namedBfCols = null; + } + boolean hasNamedBfChange = !Objects.equals(oriNamedBfCols, namedBfCols); + if (hasBfColumnsProperty) { + checkLegacyBloomFilterColumnsManagedByNamedIndexes(oriLegacyBfCols, bfColumns, namedBfCols); + } double oriBfFpp = olapTable.getBfFpp(); + boolean originalHasLegacyBloomFilter = oriLegacyBfCols != null; if (bfColumns != null) { if (bfFpp == 0) { // columns: yes, fpp: no - if (bfColumns.equals(oriBfColumns)) { + if (bfColumns.equals(oriLegacyBfCols)) { throw new DdlException("Bloom filter index has no change"); } - - if (oriBfColumns == null) { + if (!originalHasLegacyBloomFilter) { bfFpp = FeConstants.default_bloom_filter_fpp; } else { bfFpp = oriBfFpp; } } else { // columns: yes, fpp: yes - if (bfColumns.equals(oriBfColumns) && bfFpp == oriBfFpp) { + if (bfColumns.equals(oriLegacyBfCols) && bfFpp == oriBfFpp) { throw new DdlException("Bloom filter index has no change"); } } - hasBfChange = true; } else { if (bfFpp == 0) { @@ -1742,25 +1754,35 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map conflictIndexes = new ArrayList<>(newIndexes); + conflictIndexes.add(alterIndex); + try { + Index.checkConflict(conflictIndexes, olapTable.getCopiedBfColumns()); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } newIndexes.add(alterIndex); return false; } @@ -3395,6 +3422,61 @@ private boolean processDropIndex(DropIndexOp dropIndexOp, OlapTable olapTable, L return false; } + /** + * Checks if legacy bloom filter columns are managed by named indexes. + */ + private void checkLegacyBloomFilterColumnsManagedByNamedIndexes( + Set oldLegacyBfColumns, Set newLegacyBfColumns, Set namedBfColumns) + throws DdlException { + if (namedBfColumns == null || namedBfColumns.isEmpty()) { + return; + } + + Set overlappingLegacyAndNamedBfColumns = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + if (oldLegacyBfColumns != null) { + for (String legacyBfColumn : oldLegacyBfColumns) { + if (containsIgnoreCase(namedBfColumns, legacyBfColumn)) { + overlappingLegacyAndNamedBfColumns.add(legacyBfColumn); + } + } + } + Preconditions.checkState(overlappingLegacyAndNamedBfColumns.isEmpty(), + "legacy bloom filter columns overlap named BLOOMFILTER columns: %s", + Joiner.on(",").join(overlappingLegacyAndNamedBfColumns)); + + // legacy bloom filter columns are not changed + if (Objects.equals(newLegacyBfColumns, oldLegacyBfColumns)) { + return; + } + + Set addedLegacyBfColumns = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + if (newLegacyBfColumns != null) { + addedLegacyBfColumns.addAll(newLegacyBfColumns); + } + if (oldLegacyBfColumns != null) { + addedLegacyBfColumns.removeAll(oldLegacyBfColumns); + } + for (String column : addedLegacyBfColumns) { + if (namedBfColumns.contains(column)) { + throw new DdlException("ALTER TABLE failed, expected to create bloom filter index on column " + + column + ", but this column is already defined by named BLOOMFILTER index"); + } + } + + } + + private boolean containsIgnoreCase(Collection values, String target) { + if (values == null || target == null) { + return false; + } + for (String value : values) { + if (target.equalsIgnoreCase(value)) { + return true; + } + } + return false; + } + @Override public void addAlterJobV2(AlterJobV2 alterJob) throws AnalysisException { super.addAlterJobV2(alterJob); @@ -3563,21 +3645,21 @@ public void modifyTableLightSchemaChange(String rawSql, Database db, OlapTable o } // for bloom filter, rebuild bloom filter info by table schema in replay if (isReplay) { - Set bfCols = olapTable.getCopiedBfColumns(); - if (bfCols != null) { + Set legacyBloomFilterColumns = olapTable.getCopiedBfColumns(); + if (legacyBloomFilterColumns != null) { List columns = olapTable.getBaseSchema(); - Set newBfCols = null; - for (String bfCol : bfCols) { + Set normalizedLegacyBloomFilterColumns = null; + for (String legacyBloomFilterColumn : legacyBloomFilterColumns) { for (Column column : columns) { - if (column.getName().equalsIgnoreCase(bfCol)) { - if (newBfCols == null) { - newBfCols = Sets.newHashSet(); + if (column.getName().equalsIgnoreCase(legacyBloomFilterColumn)) { + if (normalizedLegacyBloomFilterColumns == null) { + normalizedLegacyBloomFilterColumns = Sets.newHashSet(); } - newBfCols.add(column.getName()); + normalizedLegacyBloomFilterColumns.add(column.getName()); } } } - olapTable.setBloomFilterInfo(newBfCols, olapTable.getBfFpp()); + olapTable.setBloomFilterInfo(normalizedLegacyBloomFilterColumns, olapTable.getBfFpp()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index c044a8c258f804..fc05c4b5a798ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -297,7 +297,6 @@ protected void createShadowIndexReplica() throws AlterCancelException { long originIndexId = indexIdMap.get(shadowIdxId); int originSchemaHash = tbl.getSchemaHashByIndexId(originIndexId); KeysType originKeysType = tbl.getKeysTypeByIndexId(originIndexId); - List tabletIndexes = originIndexId == tbl.getBaseIndexId() ? indexes : null; for (Tablet shadowTablet : shadowIdx.getTablets()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToProtobuf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToProtobuf.java index d9c17012a64d28..33d223b39e5971 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToProtobuf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToProtobuf.java @@ -37,8 +37,9 @@ public static OlapFile.ColumnPB toPb(Column column, Set bfColumns, List< // when doing schema change, some modified column has a prefix in name. // this prefix is only used in FE, not visible to BE, so we should remove this prefix. String name = column.getName(); - builder.setName(name.startsWith(Column.SHADOW_NAME_PREFIX) - ? name.substring(Column.SHADOW_NAME_PREFIX.length()) : name); + String nonShadowColumnName = name.startsWith(Column.SHADOW_NAME_PREFIX) + ? name.substring(Column.SHADOW_NAME_PREFIX.length()) : name; + builder.setName(nonShadowColumnName); builder.setUniqueId(column.getUniqueId()); builder.setType(column.getDataType().toThrift().name()); @@ -88,7 +89,8 @@ public static OlapFile.ColumnPB toPb(Column column, Set bfColumns, List< builder.setIndexLength(column.getOlapColumnIndexSize()); } - if (bfColumns != null && bfColumns.contains(column.getName())) { + if ((bfColumns != null && bfColumns.contains(nonShadowColumnName)) + || Index.hasBloomFilterIndex(indexes, nonShadowColumnName)) { builder.setIsBfColumn(true); } else { builder.setIsBfColumn(false); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToThrift.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToThrift.java index 9ed324713b69b5..92ea0e714c8089 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToThrift.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToThrift.java @@ -30,8 +30,10 @@ public class ColumnToThrift { public static void setIndexFlag(TColumn tColumn, OlapTable olapTable) { + String columnName = tColumn.getColumnName(); Set bfColumns = olapTable.getCopiedBfColumns(); - if (bfColumns != null && bfColumns.contains(tColumn.getColumnName())) { + if ((bfColumns != null && bfColumns.contains(columnName)) + || Index.hasBloomFilterIndex(olapTable.getIndexes(), columnName)) { tColumn.setIsBloomFilterColumn(true); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index dc8cac76500b15..35d8009cce3751 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -3950,10 +3950,10 @@ private static void addOlapTablePropertyInfo(OlapTable olapTable, StringBuilder } // bloom filter - Set bfColumnNames = olapTable.getCopiedBfColumns(); - if (bfColumnNames != null) { + Set legacyBloomFilterColumnNames = olapTable.getCopiedBfColumns(); + if (legacyBloomFilterColumnNames != null) { sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_BF_COLUMNS).append("\" = \""); - sb.append(Joiner.on(", ").join(olapTable.getCopiedBfColumns())).append("\""); + sb.append(Joiner.on(", ").join(legacyBloomFilterColumnNames)).append("\""); } if (separatePartition) { @@ -6169,17 +6169,17 @@ private void renameColumn(Database db, OlapTable table, String colName, } // 6. modify bloom filter col - Set bfCols = table.getCopiedBfColumns(); - if (bfCols != null) { - Set newBfCols = new HashSet<>(); - for (String bfCol : bfCols) { - if (bfCol.equalsIgnoreCase(colName)) { - newBfCols.add(newColName); + Set legacyBloomFilterColumns = table.getCopiedBfColumns(); + if (legacyBloomFilterColumns != null) { + Set renamedLegacyBloomFilterColumns = new HashSet<>(); + for (String legacyBloomFilterColumn : legacyBloomFilterColumns) { + if (legacyBloomFilterColumn.equalsIgnoreCase(colName)) { + renamedLegacyBloomFilterColumns.add(newColName); } else { - newBfCols.add(bfCol); + renamedLegacyBloomFilterColumns.add(legacyBloomFilterColumn); } } - table.setBloomFilterInfo(newBfCols, table.getBfFpp()); + table.setBloomFilterInfo(renamedLegacyBloomFilterColumns, table.getBfFpp()); } table.rebuildFullSchema(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index bcd9e76e7c434c..99c306b295cebc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -200,7 +200,8 @@ public enum OlapTableState { @SerializedName(value = "tps", alternate = {"tempPartitions"}) private TempPartitions tempPartitions = new TempPartitions(); - // bloom filter columns + // Legacy bloom filter columns managed by table property bloom_filter_columns. + // Named BLOOMFILTER indexes are stored in `indexes`. @SerializedName(value = "bfc", alternate = {"bfColumns"}) private Set bfColumns; @@ -1973,8 +1974,9 @@ public String getSignature(int signatureVersion, List partNames) { } // bloom filter - if (bfColumns != null && !bfColumns.isEmpty()) { - for (String bfCol : bfColumns) { + Set legacyBloomFilterColumns = getCopiedBfColumns(); + if (legacyBloomFilterColumns != null && !legacyBloomFilterColumns.isEmpty()) { + for (String bfCol : legacyBloomFilterColumns) { sb.append(bfCol); } sb.append(bfFpp); @@ -2496,8 +2498,9 @@ public boolean equals(Object o) { && Objects.equals(partitionInfo, other.partitionInfo) && Objects.equals( idToPartition, other.idToPartition) && Objects.equals(nameToPartition, other.nameToPartition) && Objects.equals(tempPartitions, other.tempPartitions) - && Objects.equals(bfColumns, other.bfColumns) && Objects.equals(colocateGroup, - other.colocateGroup) && Objects.equals(sequenceType, other.sequenceType) + && Objects.equals(bfColumns, other.bfColumns) + && Objects.equals(colocateGroup, other.colocateGroup) + && Objects.equals(sequenceType, other.sequenceType) && Objects.equals(indexes, other.indexes) && Objects.equals(tableProperty, other.tableProperty); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index ee3091ec8e1751..2b394bd27e34e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -283,7 +283,9 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, schemaBuilder.setNumShortKeyColumns(shortKeyColumnCount); schemaBuilder.setNumRowsPerRowBlock(1024); schemaBuilder.setCompressKind(OlapCommon.CompressKind.COMPRESS_LZ4); - schemaBuilder.setBfFpp(bfFpp); + if (bfColumns != null) { + schemaBuilder.setBfFpp(bfFpp); + } int deleteSign = -1; int sequenceCol = -1; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java index 51dd6d0dc1af27..5be8fc10e4b680 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Index; import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.TableIf; @@ -29,6 +30,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import java.util.List; import java.util.Set; @@ -127,7 +129,14 @@ public ProcNodeInterface lookup(String idxIdStr) throws AnalysisException { if (schema == null) { throw new AnalysisException("Index " + idxId + " does not exist"); } - bfColumns = olapTable.getCopiedBfColumns(); + bfColumns = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + if (olapTable.getCopiedBfColumns() != null) { + bfColumns.addAll(olapTable.getCopiedBfColumns()); + } + bfColumns.addAll(Index.extractBloomFilterColumns(olapTable.getIndexes())); + if (bfColumns.isEmpty()) { + bfColumns = null; + } if (olapTable.hasVariantColumns() && SessionVariable.enableDescribeExtendVariantColumn()) { return new RemoteIndexSchemaProcDir(table, schema, bfColumns); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 6a43988ccfdf14..e3aad413c8ca6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2708,9 +2708,10 @@ private boolean createOlapTable(Database db, CreateTableInfo createTableInfo) th } bfFpp = PropertyAnalyzer.analyzeBloomFilterFpp(properties); - if (bfColumns != null && bfFpp == 0) { + boolean hasLegacyBf = bfColumns != null; + if (hasLegacyBf && bfFpp == 0) { bfFpp = FeConstants.default_bloom_filter_fpp; - } else if (bfColumns == null) { + } else if (!hasLegacyBf) { bfFpp = 0; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index c470bc789d9e78..7ad365c61fcd4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -6384,6 +6384,8 @@ public Command visitCreateIndex(CreateIndexContext ctx) { indexType = "INVERTED"; } else if (ctx.ANN() != null) { indexType = "ANN"; + } else if (ctx.BLOOMFILTER() != null) { + indexType = "BLOOMFILTER"; } String comment = ctx.STRING_LITERAL() == null ? "" : stripQuotes(ctx.STRING_LITERAL().getText()); IndexDefinition indexDefinition = new IndexDefinition(indexName, ifNotExists, indexCols, indexType, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DescribeCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DescribeCommand.java index 996a61c17cc61c..ce41cacb54d72a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DescribeCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DescribeCommand.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Index; import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.MysqlTable; import org.apache.doris.catalog.OlapTable; @@ -308,7 +309,14 @@ public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exc if (table instanceof OlapTable) { isOlapTable = true; OlapTable olapTable = (OlapTable) table; - Set bfColumns = olapTable.getCopiedBfColumns(); + Set bfColumns = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + if (olapTable.getCopiedBfColumns() != null) { + bfColumns.addAll(olapTable.getCopiedBfColumns()); + } + bfColumns.addAll(Index.extractBloomFilterColumns(olapTable.getIndexes())); + if (bfColumns.isEmpty()) { + bfColumns = null; + } Map> indexIdToSchema = olapTable.getIndexIdToSchema(); // indices order diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java index 8630d80b7dc0ab..efff2562ea6579 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java @@ -28,6 +28,7 @@ import org.apache.doris.catalog.info.IndexType; import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.common.Config; +import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DataType; @@ -84,6 +85,10 @@ public IndexDefinition(String name, boolean ifNotExists, List cols, Stri this.indexType = IndexType.INVERTED; break; } + case "BLOOMFILTER": { + this.indexType = IndexType.BLOOMFILTER; + break; + } case "NGRAM_BF": { this.indexType = IndexType.NGRAM_BF; break; @@ -139,7 +144,7 @@ public static boolean isSupportIdxType(DataType columnType) { } /** - * checkColumn + * Validate whether the index can be created on the given column definition. */ public void checkColumn(ColumnDefinition column, KeysType keysType, boolean enableUniqueKeyMergeOnWrite, @@ -187,8 +192,8 @@ public void checkColumn(ColumnDefinition column, KeysType keysType, (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V1 || invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.DEFAULT) && (Config.isCloudMode() || !Config.enable_inverted_index_v1_for_variant); - - if (colType.isVariantType() && notSupportInvertedIndexForVariant) { + // Bloom filter support variant type, so skip the check for bloom filter index. + if (indexType != IndexType.BLOOMFILTER && colType.isVariantType() && notSupportInvertedIndexForVariant) { throw new AnalysisException(colType + " is not supported in inverted index format V1," + "Please set properties(\"inverted_index_storage_format\"= \"v2\")," + "or upgrade to a newer version"); @@ -231,6 +236,12 @@ public void checkColumn(ColumnDefinition column, KeysType keysType, } catch (Exception ex) { throw new AnalysisException("invalid ngram bf index params:" + ex.getMessage(), ex); } + } else if (indexType == IndexType.BLOOMFILTER) { + Column bfColumn = new Column(indexColName, column.getType().toCatalogDataType()); + if (!bfColumn.isSupportBloomFilter()) { + throw new AnalysisException(colType + " is not supported in bloom filter index. " + + "invalid column: " + indexColName); + } } } else { throw new AnalysisException("Unsupported index type: " + indexType); @@ -294,8 +305,8 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V1 || invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.DEFAULT) && (Config.isCloudMode() || !Config.enable_inverted_index_v1_for_variant); - - if (colType.isVariantType() && notSupportInvertedIndexForVariant) { + // Bloom filter support variant type, so skip the check for bloom filter index. + if (indexType != IndexType.BLOOMFILTER && colType.isVariantType() && notSupportInvertedIndexForVariant) { throw new AnalysisException(colType + " is not supported in inverted index format V1," + "Please set properties(\"inverted_index_storage_format\"= \"v2\")," + "or upgrade to a newer version"); @@ -331,6 +342,11 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe parseAndValidateProperty(properties, NGRAM_SIZE_KEY, MIN_NGRAM_SIZE, MAX_NGRAM_SIZE); parseAndValidateProperty(properties, NGRAM_BF_SIZE_KEY, MIN_BF_SIZE, MAX_BF_SIZE); + } else if (indexType == IndexType.BLOOMFILTER) { + if (!column.isSupportBloomFilter()) { + throw new AnalysisException(colType + " is not supported in bloom filter index. " + + "invalid column: " + indexColName); + } } } else { throw new AnalysisException("Unsupported index type: " + indexType); @@ -360,7 +376,10 @@ public void validate() { } if (indexType == IndexType.BITMAP || indexType == IndexType.INVERTED - || indexType == IndexType.NGRAM_BF) { + || indexType == IndexType.BLOOMFILTER || indexType == IndexType.NGRAM_BF) { + if (indexType == IndexType.BLOOMFILTER) { + validateBloomFilterProperties(); + } if (cols == null || cols.size() != 1) { throw new AnalysisException( indexType.toString() + " index can only apply to a single column."); @@ -460,6 +479,22 @@ public Map getProperties() { return properties; } + private void validateBloomFilterProperties() { + if (properties.isEmpty()) { + return; + } + + if (properties.size() != 1 || !properties.containsKey(PropertyAnalyzer.PROPERTIES_BF_FPP)) { + throw new AnalysisException("BLOOMFILTER index only supports property bloom_filter_fpp"); + } + + try { + PropertyAnalyzer.analyzeBloomFilterFpp(new HashMap<>(properties)); + } catch (org.apache.doris.common.AnalysisException e) { + throw new AnalysisException(e.getMessage(), e); + } + } + public boolean isAnnIndex() { return (this.indexType == IndexType.ANN); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java index 32fe0946d8d782..0ec364ad03de2b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -320,11 +320,14 @@ public TCreateTabletReq toThrift() { tColumns = (List) tCols; } else { tColumns = new ArrayList<>(); + Set namedBfColumns = Index.extractBloomFilterColumns(indexes); for (int i = 0; i < columns.size(); i++) { Column column = columns.get(i); TColumn tColumn = ColumnToThrift.toThrift(column); + String nonShadowColumnName = column.getNonShadowName(); // is bloom filter column - if (bfColumns != null && bfColumns.contains(column.getName())) { + if ((bfColumns != null && bfColumns.contains(nonShadowColumnName)) + || namedBfColumns.contains(nonShadowColumnName)) { tColumn.setIsBloomFilterColumn(true); } // when doing schema change, some modified column has a prefix in name. diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java index 370a9cd90dc173..984be6367f3f9b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java @@ -21,16 +21,24 @@ import org.apache.doris.analysis.ResourceTypeEnum; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.CatalogTestUtil; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.EnvFactory; import org.apache.doris.catalog.FakeEditLog; import org.apache.doris.catalog.FakeEnv; +import org.apache.doris.catalog.Index; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.LocalTablet; +import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.OlapTable.OlapTableState; +import org.apache.doris.catalog.info.IndexType; import org.apache.doris.catalog.info.TableNameInfo; import org.apache.doris.cloud.catalog.CloudEnv; import org.apache.doris.cloud.catalog.CloudEnvFactory; +import org.apache.doris.cloud.catalog.CloudReplica; import org.apache.doris.cloud.datasource.CloudInternalCatalog; import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.cloud.proto.Cloud.MetaServiceCode; @@ -54,6 +62,7 @@ import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import org.apache.doris.thrift.TSortType; +import org.apache.doris.thrift.TStorageType; import org.apache.doris.thrift.TTaskType; import org.apache.doris.utframe.MockedMetaServerFactory; @@ -69,6 +78,7 @@ import org.mockito.Mockito; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -649,4 +659,119 @@ public void testCreateTokenizedInvertedIndex() throws Exception { Assert.assertEquals("true", table.getIndexes().get(0).getProperties().get("support_phrase")); Assert.assertEquals("true", table.getIndexes().get(0).getProperties().get("lower_case")); } + + @Test + public void testCreateShadowIndexReplicaForPartitionCopiesNamedIndexesOnlyForBaseShadowReplica() throws Exception { + CatalogTestUtil.createDupTable(db); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + org.apache.doris.catalog.Partition partition = table.getPartition(CatalogTestUtil.testPartitionId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + table.setDataSortInfo(dataSortInfo); + + long rollupIndexId = 41001L; + long rollupTabletId = 41002L; + long rollupReplicaId = 41003L; + String rollupName = "r1"; + List rollupSchema = Lists.newArrayList(new Column(table.getBaseSchema().get(0)), + new Column(table.getBaseSchema().get(2))); + partition.createRollupIndex(createCloudIndex(rollupIndexId, rollupTabletId, rollupReplicaId, + CatalogTestUtil.testBackendId1, db.getId(), table.getId(), partition.getId(), IndexState.NORMAL)); + table.setIndexMeta(rollupIndexId, rollupName, rollupSchema, 1, 42001, (short) 2, + TStorageType.COLUMN, KeysType.DUP_KEYS); + + List namedBloomFilterIndexes = Lists.newArrayList( + new Index(1L, "bf_v1", Lists.newArrayList("v1"), IndexType.BLOOMFILTER, null, "")); + table.setIndexes(namedBloomFilterIndexes); + + long shadowBaseIndexId = 51001L; + long shadowBaseTabletId = 51002L; + long shadowBaseReplicaId = 51003L; + long shadowRollupIndexId = 51011L; + long shadowRollupTabletId = 51012L; + long shadowRollupReplicaId = 51013L; + + CloudSchemaChangeJobV2 schemaChangeJob = new CloudSchemaChangeJobV2("", 1001L, db.getId(), + table.getId(), table.getName(), 60000L); + schemaChangeJob.setAlterIndexInfo(true, namedBloomFilterIndexes); + schemaChangeJob.setBloomFilterInfo(false, null, 0.02); + schemaChangeJob.addPartitionShadowIndex(partition.getId(), shadowBaseIndexId, + createCloudIndex(shadowBaseIndexId, shadowBaseTabletId, shadowBaseReplicaId, + CatalogTestUtil.testBackendId1, db.getId(), table.getId(), partition.getId(), + IndexState.SHADOW)); + schemaChangeJob.addIndexSchema(shadowBaseIndexId, table.getBaseIndexId(), + Column.SHADOW_NAME_PREFIX + table.getIndexNameById(table.getBaseIndexId()), 2, 52001, (short) 3, + createShadowSchema(table.getBaseSchema())); + + schemaChangeJob.addPartitionShadowIndex(partition.getId(), shadowRollupIndexId, + createCloudIndex(shadowRollupIndexId, shadowRollupTabletId, shadowRollupReplicaId, + CatalogTestUtil.testBackendId1, db.getId(), table.getId(), partition.getId(), + IndexState.SHADOW)); + schemaChangeJob.addIndexSchema(shadowRollupIndexId, rollupIndexId, + Column.SHADOW_NAME_PREFIX + rollupName, 2, 52002, (short) 2, createShadowSchema(rollupSchema)); + + List capturedRequests = new ArrayList<>(); + Mockito.doAnswer(invocation -> { + Cloud.CreateTabletsRequest request = invocation.getArgument(0); + capturedRequests.add(request); + return Cloud.CreateTabletsResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .build(); + }).when(mockProxy).createTablets(Mockito.any()); + + Method method = CloudSchemaChangeJobV2.class.getDeclaredMethod("createShadowIndexReplicaForPartition", + OlapTable.class); + method.setAccessible(true); + method.invoke(schemaChangeJob, table); + + // Only the base shadow index copies named index metadata. The rollup shadow index keeps + // the original schema-change behavior and does not receive named BF metadata or folded + // BF column flags from indexes. bfColumns is null so table-level bfFpp is not set; + // named indexes carry their own per-index FPP. + Assert.assertEquals(2, capturedRequests.size()); + Cloud.CreateTabletsRequest baseRequest = capturedRequests.stream() + .filter(request -> request.getTabletMetas(0).getIndexId() == shadowBaseIndexId) + .findFirst() + .orElseThrow(() -> new AssertionError("base shadow request not found")); + Cloud.CreateTabletsRequest rollupRequest = capturedRequests.stream() + .filter(request -> request.getTabletMetas(0).getIndexId() == shadowRollupIndexId) + .findFirst() + .orElseThrow(() -> new AssertionError("rollup shadow request not found")); + + Assert.assertEquals(1, baseRequest.getTabletMetas(0).getSchema().getIndexCount()); + Assert.assertEquals(0, rollupRequest.getTabletMetas(0).getSchema().getIndexCount()); + Assert.assertFalse(baseRequest.getTabletMetas(0).getSchema().hasBfFpp()); + Assert.assertFalse(rollupRequest.getTabletMetas(0).getSchema().hasBfFpp()); + Assert.assertEquals("k1", baseRequest.getTabletMetas(0).getSchema().getColumn(0).getName()); + Assert.assertEquals("k2", baseRequest.getTabletMetas(0).getSchema().getColumn(1).getName()); + Assert.assertEquals("v1", baseRequest.getTabletMetas(0).getSchema().getColumn(2).getName()); + Assert.assertFalse(baseRequest.getTabletMetas(0).getSchema().getColumn(0).getIsBfColumn()); + Assert.assertFalse(baseRequest.getTabletMetas(0).getSchema().getColumn(1).getIsBfColumn()); + Assert.assertTrue(baseRequest.getTabletMetas(0).getSchema().getColumn(2).getIsBfColumn()); + Assert.assertEquals("k1", rollupRequest.getTabletMetas(0).getSchema().getColumn(0).getName()); + Assert.assertEquals("v1", rollupRequest.getTabletMetas(0).getSchema().getColumn(1).getName()); + Assert.assertFalse(rollupRequest.getTabletMetas(0).getSchema().getColumn(0).getIsBfColumn()); + Assert.assertFalse(rollupRequest.getTabletMetas(0).getSchema().getColumn(1).getIsBfColumn()); + } + + private MaterializedIndex createCloudIndex(long indexId, long tabletId, long replicaId, long backendId, + long dbId, long tableId, long partitionId, IndexState indexState) { + MaterializedIndex index = new MaterializedIndex(indexId, indexState); + LocalTablet tablet = new LocalTablet(tabletId); + tablet.addReplica(new CloudReplica(replicaId, backendId, org.apache.doris.catalog.Replica.ReplicaState.NORMAL, + CatalogTestUtil.testStartVersion, 0, dbId, tableId, partitionId, indexId, 0), true); + index.addTablet(tablet, null, true); + return index; + } + + private List createShadowSchema(List originSchema) { + List shadowSchema = new ArrayList<>(originSchema.size()); + for (Column column : originSchema) { + Column shadowColumn = new Column(column); + shadowColumn.setName(Column.SHADOW_NAME_PREFIX + column.getName()); + shadowSchema.add(shadowColumn); + } + return shadowSchema; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java index 59d7cf0a9fc571..7202d4ad2be7c1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java @@ -17,6 +17,7 @@ package org.apache.doris.alter; +import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; @@ -25,9 +26,11 @@ import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.info.ColumnPosition; import org.apache.doris.catalog.info.IndexType; +import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.nereids.StatementContext; @@ -49,6 +52,7 @@ import org.mockito.Mockito; import java.lang.reflect.Method; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @@ -324,6 +328,119 @@ public void testWithRowBinlogSchemaChangeWithHistoricalValue() throws Exception Assert.assertFalse(cols.contains(Column.generateBeforeColName(Column.SEQUENCE_COL))); } + @Test + public void testCheckLegacyBloomFilterColumnsManagedByNamedIndexesPrivateHelper() { + SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); + + // No-op case: legacy/property-managed BF columns stay unchanged and do not overlap with + // named BF columns. + Deencapsulation.invoke(schemaChangeHandler, + "checkLegacyBloomFilterColumnsManagedByNamedIndexes", + Sets.newTreeSet(List.of("v1")), + Sets.newTreeSet(List.of("v1")), + Sets.newTreeSet(List.of("v2"))); + + // Adding a legacy BF definition on top of a named BF column must be rejected. + DdlException createConflict = Assertions.assertThrows(DdlException.class, + () -> Deencapsulation.invoke(schemaChangeHandler, + "checkLegacyBloomFilterColumnsManagedByNamedIndexes", + Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER), + Sets.newTreeSet(List.of("v1")), + Sets.newTreeSet(List.of("v1")))); + Assertions.assertTrue(createConflict.getMessage().contains( + "expected to create bloom filter index on column v1")); + + // Old legacy BF metadata overlapping named BF metadata is an internal invariant violation, + // not a user-facing ALTER error path. + IllegalStateException overlappingMetadata = Assertions.assertThrows(IllegalStateException.class, + () -> Deencapsulation.invoke(schemaChangeHandler, + "checkLegacyBloomFilterColumnsManagedByNamedIndexes", + Sets.newTreeSet(List.of("v1")), + Sets.newTreeSet(List.of("v1")), + Sets.newTreeSet(List.of("v1")))); + Assertions.assertTrue(overlappingMetadata.getMessage().contains( + "legacy bloom filter columns overlap named BLOOMFILTER columns: v1")); + } + + @Test + public void testModifyTableLightSchemaChangeReplayNormalizesLegacyBloomFilterColumns() throws Exception { + String tableName = "sc_replay_bf_case"; + dropTable("test." + tableName, false); + createTable("CREATE TABLE test." + tableName + " (\n" + + "k1 INT,\n" + + "v1 STRING\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1', 'light_schema_change' = 'true', " + + "'bloom_filter_columns' = 'k1');"); + + Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("test"); + OlapTable table = (OlapTable) db.getTableOrMetaException(tableName, Table.TableType.OLAP); + SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); + + LinkedList alteredSchema = table.getBaseSchema().stream() + .map(Column::new) + .collect(Collectors.toCollection(LinkedList::new)); + alteredSchema.add(new Column("v2", ScalarType.createType(PrimitiveType.INT), + false, AggregateType.NONE, "0", "")); + + Map> indexSchemaMap = Maps.newHashMap(); + indexSchemaMap.put(table.getBaseIndexId(), alteredSchema); + long replayJobId = Env.getCurrentEnv().getNextId(); + + table.writeLock(); + try { + table.setBloomFilterInfo(Sets.newHashSet("K1"), table.getBfFpp()); + schemaChangeHandler.modifyTableLightSchemaChange("", db, table, indexSchemaMap, table.getIndexes(), + null, false, replayJobId, true, Maps.newHashMap()); + + Assertions.assertNotNull(table.getColumn("v2")); + Assertions.assertEquals(Sets.newHashSet("k1"), table.getCopiedBfColumns()); + } finally { + table.writeUnlock(); + schemaChangeHandler.getAlterJobsV2().remove(replayJobId); + schemaChangeHandler.runnableSchemaChangeJobV2.remove(replayJobId); + } + } + + @Test + public void testCreateJobRejectsUnchangedLegacyBloomFilterColumnsAndFpp() throws Exception { + String tableName = "sc_bf_unchanged_cols_fpp"; + dropTable("test." + tableName, false); + createTable("CREATE TABLE test." + tableName + " (\n" + + "k1 INT,\n" + + "v1 STRING\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1', 'light_schema_change' = 'true', " + + "'bloom_filter_columns' = 'k1', 'bloom_filter_fpp' = '0.05');"); + + Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("test"); + OlapTable table = (OlapTable) db.getTableOrMetaException(tableName, Table.TableType.OLAP); + SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); + + Map> indexSchemaMap = Maps.newHashMap(); + indexSchemaMap.put(table.getBaseIndexId(), table.getBaseSchema().stream() + .map(Column::new) + .collect(Collectors.toCollection(LinkedList::new))); + + // This unit test invokes createJob() directly because SQL ALTER TABLE SET with both + // bloom_filter_columns and bloom_filter_fpp is rejected earlier by property validation. + // The goal here is to hit SchemaChangeHandler's "columns: yes, fpp: yes, nothing changed" + // branch and verify the exact no-change exception. + Map properties = Maps.newHashMap(); + properties.put("bloom_filter_columns", "k1"); + properties.put("bloom_filter_fpp", "0.05"); + + DdlException exception = Assertions.assertThrows(DdlException.class, + () -> Deencapsulation.invoke(schemaChangeHandler, "createJob", "", + db.getId(), table, indexSchemaMap, properties, table.getIndexes(), Maps.newHashMap())); + Assertions.assertTrue(exception.getMessage().contains("Bloom filter index has no change"), + exception.getMessage()); + } + @Test public void testWithRowBinlogOpNotSupported() throws Exception { // 1) MODIFY COLUMN not supported diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java index 424205dfb50d5d..ce4ab970220c19 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java @@ -28,7 +28,10 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.FakeEditLog; import org.apache.doris.catalog.FakeEnv; +import org.apache.doris.catalog.Index; import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.LocalReplica; +import org.apache.doris.catalog.LocalTablet; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.MaterializedIndex.IndexState; @@ -42,6 +45,7 @@ import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.Type; import org.apache.doris.catalog.info.ColumnPosition; +import org.apache.doris.catalog.info.IndexType; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeMetaVersion; @@ -56,10 +60,13 @@ import org.apache.doris.nereids.trees.plans.commands.info.ModifyTablePropertiesOp; import org.apache.doris.nereids.types.DataType; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.task.AgentBatchTask; import org.apache.doris.task.AgentTask; import org.apache.doris.task.AgentTaskExecutor; import org.apache.doris.task.AgentTaskQueue; +import org.apache.doris.task.CreateReplicaTask; import org.apache.doris.thrift.TStorageFormat; +import org.apache.doris.thrift.TStorageType; import org.apache.doris.thrift.TTaskType; import org.apache.doris.transaction.FakeTransactionIDGenerator; import org.apache.doris.transaction.GlobalTransactionMgr; @@ -87,6 +94,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; public class SchemaChangeJobV2Test { @@ -583,4 +591,134 @@ public void testAbnormalModifyTableDistributionType3() throws UserException { + "distribution type of aggregate keys table which has value columns with REPLACE type."); Env.getCurrentEnv().convertDistributionType(db, table); } + + @Test + public void testCreateShadowIndexReplicaCopiesNamedIndexesOnlyForBaseShadowReplica() throws Exception { + if (fakeEnv != null) { + fakeEnv.close(); + } + fakeEnv = new FakeEnv(); + if (fakeEditLog != null) { + fakeEditLog.close(); + } + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + + Database db = masterEnv.getInternalCatalog().getDbOrDdlException(CatalogTestUtil.testDbId1); + CatalogTestUtil.createDupTable(db); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + Partition partition = table.getPartition(CatalogTestUtil.testPartitionId2); + + long rollupIndexId = 40001L; + long rollupTabletId = 40002L; + long rollupReplicaId = 40003L; + String rollupName = "r1"; + List rollupSchema = Lists.newArrayList(new Column(table.getBaseSchema().get(0)), + new Column(table.getBaseSchema().get(2))); + partition.createRollupIndex(createLocalIndex(rollupIndexId, rollupTabletId, rollupReplicaId, + CatalogTestUtil.testBackendId1, IndexState.NORMAL)); + table.setIndexMeta(rollupIndexId, rollupName, rollupSchema, 1, 41001, (short) 2, + TStorageType.COLUMN, KeysType.DUP_KEYS); + + List namedBloomFilterIndexes = Lists.newArrayList( + new Index(1L, "bf_v1", Lists.newArrayList("v1"), IndexType.BLOOMFILTER, null, "")); + table.setIndexes(namedBloomFilterIndexes); + table.setState(OlapTableState.SCHEMA_CHANGE); + + long shadowBaseIndexId = 50001L; + long shadowBaseTabletId = 50002L; + long shadowBaseReplicaId = 50003L; + long shadowRollupIndexId = 50011L; + long shadowRollupTabletId = 50012L; + long shadowRollupReplicaId = 50013L; + + SchemaChangeJobV2 schemaChangeJob = new SchemaChangeJobV2("", 1000L, db.getId(), + table.getId(), table.getName(), 60000L); + schemaChangeJob.setAlterIndexInfo(true, namedBloomFilterIndexes); + schemaChangeJob.setBloomFilterInfo(false, null, 0.02); + schemaChangeJob.addPartitionShadowIndex(partition.getId(), shadowBaseIndexId, + createLocalIndex(shadowBaseIndexId, shadowBaseTabletId, shadowBaseReplicaId, + CatalogTestUtil.testBackendId1, IndexState.SHADOW)); + schemaChangeJob.addTabletIdMap(partition.getId(), shadowBaseIndexId, shadowBaseTabletId, + CatalogTestUtil.testTabletId2); + schemaChangeJob.addIndexSchema(shadowBaseIndexId, table.getBaseIndexId(), + Column.SHADOW_NAME_PREFIX + table.getIndexNameById(table.getBaseIndexId()), 2, 51001, (short) 3, + createShadowSchema(table.getBaseSchema())); + + schemaChangeJob.addPartitionShadowIndex(partition.getId(), shadowRollupIndexId, + createLocalIndex(shadowRollupIndexId, shadowRollupTabletId, shadowRollupReplicaId, + CatalogTestUtil.testBackendId1, IndexState.SHADOW)); + schemaChangeJob.addTabletIdMap(partition.getId(), shadowRollupIndexId, shadowRollupTabletId, rollupTabletId); + schemaChangeJob.addIndexSchema(shadowRollupIndexId, rollupIndexId, + Column.SHADOW_NAME_PREFIX + rollupName, 2, 51002, (short) 2, createShadowSchema(rollupSchema)); + + List submittedTasks = new ArrayList<>(); + boolean originalRunningUnitTest = FeConstants.runningUnitTest; + FeConstants.runningUnitTest = false; + mockedAgentTaskExecutor.when(() -> AgentTaskExecutor.submit(Mockito.any(AgentBatchTask.class))) + .thenAnswer(invocation -> { + AgentBatchTask batchTask = invocation.getArgument(0); + submittedTasks.addAll(batchTask.getAllTasks()); + for (AgentTask task : batchTask.getAllTasks()) { + CreateReplicaTask createReplicaTask = (CreateReplicaTask) task; + createReplicaTask.countDownLatch(createReplicaTask.getBackendId(), + createReplicaTask.getTabletId()); + } + return null; + }); + + try { + schemaChangeJob.createShadowIndexReplica(); + } finally { + FeConstants.runningUnitTest = originalRunningUnitTest; + AgentTaskQueue.clearAllTasks(); + } + + // Only base shadow indexes copy named index metadata. legacy/property-managed bloom + // filter columns are carried separately via bfColumns, so named BF does not get folded + // into the rollup shadow replica. + Assert.assertEquals(2, submittedTasks.size()); + CreateReplicaTask baseTask = (CreateReplicaTask) submittedTasks.stream() + .filter(task -> task.getIndexId() == shadowBaseIndexId) + .findFirst() + .orElseThrow(() -> new AssertionError("base shadow create task not found")); + CreateReplicaTask rollupTask = (CreateReplicaTask) submittedTasks.stream() + .filter(task -> task.getIndexId() == shadowRollupIndexId) + .findFirst() + .orElseThrow(() -> new AssertionError("rollup shadow create task not found")); + + @SuppressWarnings("unchecked") + List baseTaskIndexes = Deencapsulation.getField(baseTask, "indexes"); + @SuppressWarnings("unchecked") + List rollupTaskIndexes = Deencapsulation.getField(rollupTask, "indexes"); + @SuppressWarnings("unchecked") + Set baseTaskBfColumns = Deencapsulation.getField(baseTask, "bfColumns"); + @SuppressWarnings("unchecked") + Set rollupTaskBfColumns = Deencapsulation.getField(rollupTask, "bfColumns"); + + Assert.assertEquals(namedBloomFilterIndexes, baseTaskIndexes); + Assert.assertNull(rollupTaskIndexes); + Assert.assertNull(baseTaskBfColumns); + Assert.assertNull(rollupTaskBfColumns); + } + + private MaterializedIndex createLocalIndex(long indexId, long tabletId, long replicaId, long backendId, + IndexState indexState) { + MaterializedIndex index = new MaterializedIndex(indexId, indexState); + LocalTablet tablet = new LocalTablet(tabletId); + tablet.addReplica(new LocalReplica(replicaId, backendId, CatalogTestUtil.testStartVersion, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0), true); + index.addTablet(tablet, null, true); + return index; + } + + private List createShadowSchema(List originSchema) { + List shadowSchema = new ArrayList<>(originSchema.size()); + for (Column column : originSchema) { + Column shadowColumn = new Column(column); + shadowColumn.setName(Column.SHADOW_NAME_PREFIX + column.getName()); + shadowSchema.add(shadowColumn); + } + return shadowSchema; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnBloomFilterMaterializationTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnBloomFilterMaterializationTest.java new file mode 100644 index 00000000000000..60edc1dd925fee --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnBloomFilterMaterializationTest.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.catalog.info.IndexType; +import org.apache.doris.common.DdlException; +import org.apache.doris.proto.OlapFile; +import org.apache.doris.thrift.TColumn; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.Set; + +public class ColumnBloomFilterMaterializationTest { + + @Test + public void testSetIndexFlagMarksNamedBloomFilterOnShadowColumn() { + // The thrift column name is normalized to the non-shadow name. setIndexFlag() must + // still mark the column as bloom-filtered when FE metadata already exposes it in the + // copied BF column set. + OlapTable olapTable = new OlapTable(); + olapTable.setIndexes(Lists.newArrayList(new Index(1L, "bf_k1", Lists.newArrayList("k1"), + IndexType.BLOOMFILTER, null, ""))); + + Column shadowColumn = new Column(Column.SHADOW_NAME_PREFIX + "k1", ScalarType.createType(PrimitiveType.INT), + true, null, "1", ""); + TColumn tColumn = ColumnToThrift.toThrift(shadowColumn); + ColumnToThrift.setIndexFlag(tColumn, olapTable); + + Assert.assertEquals("k1", tColumn.getColumnName()); + Assert.assertTrue(tColumn.isIsBloomFilterColumn()); + } + + @Test + public void testSetIndexFlagFallsBackToNamedIndexMetadataWhenCopiedBfColumnsUnavailable() { + // This covers the fallback path in ColumnToThrift.setIndexFlag(): + // getCopiedBfColumns() does not report the column, so the named index list must + // still mark the thrift column correctly. + OlapTable olapTable = Mockito.mock(OlapTable.class); + Mockito.when(olapTable.getCopiedBfColumns()).thenReturn(null); + Mockito.when(olapTable.getIndexes()).thenReturn(Lists.newArrayList( + new Index(1L, "bf_k1", Lists.newArrayList("k1"), IndexType.BLOOMFILTER, null, ""))); + + Column shadowColumn = new Column(Column.SHADOW_NAME_PREFIX + "k1", ScalarType.createType(PrimitiveType.INT), + true, null, "1", ""); + TColumn tColumn = ColumnToThrift.toThrift(shadowColumn); + ColumnToThrift.setIndexFlag(tColumn, olapTable); + + Assert.assertEquals("k1", tColumn.getColumnName()); + Assert.assertTrue(tColumn.isIsBloomFilterColumn()); + } + + @Test + public void testColumnToProtobufMarksNamedBloomFilterOnShadowColumn() throws DdlException { + Column shadowColumn = new Column(Column.SHADOW_NAME_PREFIX + "k1", ScalarType.createType(PrimitiveType.INT), + true, null, "1", ""); + + OlapFile.ColumnPB columnPb = ColumnToProtobuf.toPb(shadowColumn, null, + Lists.newArrayList(new Index(1L, "bf_k1", Lists.newArrayList("k1"), + IndexType.BLOOMFILTER, null, ""))); + + Assert.assertEquals("k1", columnPb.getName()); + Assert.assertTrue(columnPb.getIsBfColumn()); + } + + @Test + public void testColumnToProtobufMarksLegacyBloomFilterWithoutNamedIndexes() throws DdlException { + // The protobuf builder should also work when only the effective BF column set is passed + // down and there is no named index metadata at all. + Column shadowColumn = new Column(Column.SHADOW_NAME_PREFIX + "k1", ScalarType.createType(PrimitiveType.INT), + true, null, "1", ""); + + OlapFile.ColumnPB columnPb = ColumnToProtobuf.toPb(shadowColumn, Sets.newHashSet("k1"), + Lists.newArrayList()); + + Assert.assertEquals("k1", columnPb.getName()); + Assert.assertTrue(columnPb.getIsBfColumn()); + } + + @Test + public void testOlapTableBloomFilterColumnGetters() { + OlapTable olapTable = new OlapTable(); + olapTable.setBloomFilterInfo(Sets.newHashSet("k1"), 0.05); + olapTable.setIndexes(Lists.newArrayList(new Index(1L, "bf_v1", Lists.newArrayList("v1"), + IndexType.BLOOMFILTER, null, ""))); + + Set legacyBloomFilterColumns = olapTable.getCopiedBfColumns(); + Set namedBloomFilterColumns = Index.extractBloomFilterColumns(olapTable.getIndexes()); + + Assert.assertEquals(Sets.newHashSet("k1"), legacyBloomFilterColumns); + Assert.assertEquals(Sets.newHashSet("v1"), namedBloomFilterColumns); + } + + @Test + public void testIndexBloomFilterHelpersIgnoreNonBloomFilterIndexesAndHandleNulls() { + Set emptyBloomFilterColumns = Index.extractBloomFilterColumns(null); + Assert.assertTrue(emptyBloomFilterColumns.isEmpty()); + Assert.assertFalse(Index.hasBloomFilterIndex(null, "k1")); + Assert.assertFalse(Index.hasBloomFilterIndex(Lists.newArrayList(), null)); + + Set bloomFilterColumns = Index.extractBloomFilterColumns(Lists.newArrayList( + new Index(1L, "bf_v1", Lists.newArrayList("v1"), IndexType.BLOOMFILTER, null, ""), + new Index(2L, "bitmap_k1", Lists.newArrayList("k1"), IndexType.BITMAP, null, ""), + new Index(3L, "bf_v2", Lists.newArrayList("V2"), IndexType.BLOOMFILTER, null, ""))); + + Assert.assertEquals(Sets.newHashSet("v1", "V2"), bloomFilterColumns); + Assert.assertTrue(Index.hasBloomFilterIndex(Lists.newArrayList( + new Index(1L, "bf_v2", Lists.newArrayList("V2"), IndexType.BLOOMFILTER, null, "")), "v2")); + Assert.assertFalse(Index.hasBloomFilterIndex(Lists.newArrayList( + new Index(1L, "bitmap_k1", Lists.newArrayList("k1"), IndexType.BITMAP, null, "")), "k1")); + } + +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableWithBloomFilterIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableWithBloomFilterIndexTest.java index 220b525ac448b2..514b7c675e05eb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableWithBloomFilterIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableWithBloomFilterIndexTest.java @@ -17,14 +17,20 @@ package org.apache.doris.catalog; +import org.apache.doris.alter.AlterJobV2; import org.apache.doris.common.DdlException; import org.apache.doris.common.ExceptionChecker; +import org.apache.doris.common.FeConstants; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.Lists; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.List; import java.util.UUID; +import java.util.concurrent.TimeUnit; public class CreateTableWithBloomFilterIndexTest extends TestWithFeService { private static String runningDir = "fe/mocked/CreateTableWithBloomFilterIndexTest/" @@ -35,6 +41,41 @@ protected void runBeforeAll() throws Exception { createDatabase("test"); } + private void expectFailureContains(String expectedMessage, ExceptionChecker.ThrowingRunnable runnable) { + Throwable throwable = Assertions.assertThrows(Throwable.class, runnable::run); + Assertions.assertTrue(throwable.getMessage().contains(expectedMessage), + "expected msg: " + expectedMessage + ", actual: " + throwable.getMessage()); + } + + private OlapTable getOlapTable(String tableName) throws Exception { + Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("test"); + return (OlapTable) db.getTableOrMetaException(tableName, Table.TableType.OLAP); + } + + private void waitForSchemaChangeDone(String tableName) throws Exception { + long deadlineMs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(30); + while (System.currentTimeMillis() < deadlineMs) { + OlapTable table = getOlapTable(tableName); + boolean allJobsFinished = true; + for (AlterJobV2 job : Env.getCurrentEnv().getSchemaChangeHandler().getAlterJobsV2().values()) { + if (job.getTableId() == table.getId() && !job.getJobState().isFinalState()) { + allJobsFinished = false; + break; + } + } + if (table.getState() == OlapTable.OlapTableState.NORMAL && allJobsFinished) { + return; + } + Thread.sleep(100); + } + Assertions.fail("schema change job did not finish for table " + tableName); + } + + private void alterTableSyncAndWait(String sql, String tableName) throws Exception { + alterTableSync(sql); + waitForSchemaChangeDone(tableName); + } + @Test public void testCreateTableWithTinyIntBloomFilterIndex() { ExceptionChecker.expectThrowsWithMsg(DdlException.class, @@ -902,4 +943,431 @@ public void testBloomFilterColumnsWithNullOrWhitespace() { + "\"replication_num\" = \"1\"\n" + ");")); } + + @Test + public void testCreateTableRejectsLegacyAndNamedBloomFilterOnSameColumn() { + expectFailureContains("k1 should have only one ngram bloom filter index or bloom filter index", + () -> createTable("CREATE TABLE test.tbl_bf_conflict_create (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_k1 (k1) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_columns\" = \"k1\",\n" + + "\"replication_num\" = \"1\"\n" + + ");")); + } + + @Test + public void testCreateTableWithNamedBloomFilterUsesDefaultFpp() throws Exception { + createTable("CREATE TABLE test.tbl_bf_named_default_fpp (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + OlapTable table = getOlapTable("tbl_bf_named_default_fpp"); + Assertions.assertNull(table.getCopiedBfColumns()); + Assertions.assertTrue(Index.extractBloomFilterColumns(table.getIndexes()).contains("v1")); + Assertions.assertEquals(0, table.getBfFpp(), 0); + Assertions.assertTrue(table.getIndexes().stream() + .filter(index -> "idx_v1".equalsIgnoreCase(index.getIndexName())) + .findFirst() + .orElseThrow(() -> new AssertionError("named bloom filter index not found")) + .getProperties() + .isEmpty()); + } + + @Test + public void testCreateTableWithNamedBloomFilterUsesIndexLevelFppProperty() throws Exception { + createTable("CREATE TABLE test.tbl_bf_named_index_level_fpp (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER PROPERTIES (\"bloom_filter_fpp\" = \"0.02\")\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + OlapTable table = getOlapTable("tbl_bf_named_index_level_fpp"); + Assertions.assertEquals(0, table.getBfFpp(), 0); + Assertions.assertEquals("0.02", table.getIndexes().stream() + .filter(index -> "idx_v1".equalsIgnoreCase(index.getIndexName())) + .findFirst() + .orElseThrow(() -> new AssertionError("named bloom filter index not found")) + .getProperties() + .get("bloom_filter_fpp")); + } + + @Test + public void testCreateTableWithNamedBloomFilterDoesNotUseTableLevelFpp() throws Exception { + createTable("CREATE TABLE test.tbl_bf_named_custom_fpp (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_fpp\" = \"0.01\",\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + OlapTable table = getOlapTable("tbl_bf_named_custom_fpp"); + Assertions.assertEquals(0, table.getBfFpp(), 0); + Assertions.assertTrue(Index.extractBloomFilterColumns(table.getIndexes()).contains("v1")); + Assertions.assertTrue(table.getIndexes().stream() + .filter(index -> "idx_v1".equalsIgnoreCase(index.getIndexName())) + .findFirst() + .orElseThrow(() -> new AssertionError("named bloom filter index not found")) + .getProperties() + .isEmpty()); + } + + @Test + public void testCreateIndexRejectsLegacyBloomFilterColumn() throws Exception { + createTable("CREATE TABLE test.tbl_bf_conflict_add_index (\n" + + "k1 INT,\n" + + "v1 STRING\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_columns\" = \"k1\",\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + expectFailureContains("k1 should have only one ngram bloom filter index or bloom filter index", + () -> alterTableSync("ALTER TABLE test.tbl_bf_conflict_add_index " + + "ADD INDEX idx_k1(k1) USING BLOOMFILTER")); + } + + @Test + public void testCreateFirstNamedBloomFilterIndexViaAlterUsesDefaultFpp() throws Exception { + createTable("CREATE TABLE test.tbl_bf_add_first_named (\n" + + "k1 INT,\n" + + "v1 STRING\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + alterTableSyncAndWait("ALTER TABLE test.tbl_bf_add_first_named " + + "ADD INDEX idx_v1(v1) USING BLOOMFILTER", "tbl_bf_add_first_named"); + + OlapTable table = getOlapTable("tbl_bf_add_first_named"); + Assertions.assertNull(table.getCopiedBfColumns()); + Assertions.assertEquals(0, table.getBfFpp(), 0); + Assertions.assertTrue(Index.extractBloomFilterColumns(table.getIndexes()).contains("v1")); + Assertions.assertTrue(table.getIndexes().stream() + .filter(index -> "idx_v1".equalsIgnoreCase(index.getIndexName())) + .findFirst() + .orElseThrow(() -> new AssertionError("named bloom filter index not found")) + .getProperties() + .isEmpty()); + } + + @Test + public void testCreateFirstNamedBloomFilterIndexViaAlterKeepsIndexLevelFppProperty() throws Exception { + createTable("CREATE TABLE test.tbl_bf_add_first_named_index_level_fpp (\n" + + "k1 INT,\n" + + "v1 STRING\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + alterTableSyncAndWait("ALTER TABLE test.tbl_bf_add_first_named_index_level_fpp " + + "ADD INDEX idx_v1(v1) USING BLOOMFILTER PROPERTIES (\"bloom_filter_fpp\" = \"0.02\")", + "tbl_bf_add_first_named_index_level_fpp"); + + OlapTable table = getOlapTable("tbl_bf_add_first_named_index_level_fpp"); + Assertions.assertEquals(0, table.getBfFpp(), 0); + Assertions.assertEquals("0.02", table.getIndexes().stream() + .filter(index -> "idx_v1".equalsIgnoreCase(index.getIndexName())) + .findFirst() + .orElseThrow(() -> new AssertionError("named bloom filter index not found")) + .getProperties() + .get("bloom_filter_fpp")); + } + + @Test + public void testAddSecondNamedBloomFilterIndexSucceeds() throws Exception { + createTable("CREATE TABLE test.tbl_bf_add_second_named (\n" + + "k1 INT,\n" + + "k2 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + alterTableSyncAndWait("ALTER TABLE test.tbl_bf_add_second_named " + + "ADD INDEX idx_k2(k2) USING BLOOMFILTER", "tbl_bf_add_second_named"); + + OlapTable table = getOlapTable("tbl_bf_add_second_named"); + Assertions.assertNull(table.getCopiedBfColumns()); + Assertions.assertEquals(0, table.getBfFpp(), 0); + Assertions.assertTrue(Index.extractBloomFilterColumns(table.getIndexes()).contains("v1")); + Assertions.assertTrue(Index.extractBloomFilterColumns(table.getIndexes()).contains("k2")); + Assertions.assertEquals(2, Index.extractBloomFilterColumns(table.getIndexes()).size()); + } + + @Test + public void testAlterTableSetBloomFilterColumnsRejectsNamedBloomFilterColumn() throws Exception { + createTable("CREATE TABLE test.tbl_bf_named_alter_conflict (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_columns\" = \"k1\",\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + expectFailureContains( + "ALTER TABLE failed, expected to create bloom filter index on column v1", + () -> alterTableSync("ALTER TABLE test.tbl_bf_named_alter_conflict " + + "SET (\"bloom_filter_columns\" = \"k1,v1\")")); + } + + @Test + public void testAlterTableSetBloomFilterColumnsRejectsNamedBloomFilterColumn2() throws Exception { + createTable("CREATE TABLE test.tbl_bf_named_alter_conflict2 (\n" + + "k1 INT,\n" + + "k2 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER,\n" + + "INDEX idx_k2 (k2) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_columns\" = \"k1\",\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + expectFailureContains( + "ALTER TABLE failed, expected to create bloom filter index on column k2", + () -> alterTableSync("ALTER TABLE test.tbl_bf_named_alter_conflict2 " + + "SET (\"bloom_filter_columns\" = \"k2\")")); + } + + @Test + public void testAlterTableSetBloomFilterFppOnNamedBloomFilterOnlyTableThrowsNoChange() throws Exception { + createTable("CREATE TABLE test.tbl_bf_named_alter_fpp (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + OlapTable table = getOlapTable("tbl_bf_named_alter_fpp"); + Assertions.assertNull(table.getCopiedBfColumns()); + Assertions.assertEquals(0, table.getBfFpp(), 0); + Assertions.assertTrue(Index.extractBloomFilterColumns(table.getIndexes()).contains("v1")); + expectFailureContains("Bloom filter index has no change", + () -> alterTableSync("ALTER TABLE test.tbl_bf_named_alter_fpp " + + "SET (\"bloom_filter_fpp\" = \"0.01\")")); + Assertions.assertTrue(getOlapTable("tbl_bf_named_alter_fpp").getIndexes().stream() + .filter(index -> "idx_v1".equalsIgnoreCase(index.getIndexName())) + .findFirst() + .orElseThrow(() -> new AssertionError("named bloom filter index not found")) + .getProperties() + .isEmpty()); + } + + @Test + public void testAlterTableSetSameBloomFilterFppOnNamedBloomFilterOnlyTableThrowsNoChange() throws Exception { + createTable("CREATE TABLE test.tbl_bf_named_same_fpp (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + expectFailureContains("Bloom filter index has no change", + () -> alterTableSync("ALTER TABLE test.tbl_bf_named_same_fpp " + + "SET (\"bloom_filter_fpp\" = \"" + FeConstants.default_bloom_filter_fpp + "\")")); + } + + @Test + public void testAlterTableSetBloomFilterFppWithoutAnyBloomFilterThrowsNoChange() throws Exception { + createTable("CREATE TABLE test.tbl_bf_no_index_alter_fpp (\n" + + "k1 INT,\n" + + "v1 STRING\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + expectFailureContains("Bloom filter index has no change", + () -> alterTableSync("ALTER TABLE test.tbl_bf_no_index_alter_fpp " + + "SET (\"bloom_filter_fpp\" = \"0.01\")")); + } + + @Test + public void testAlterTableSetSameLegacyBloomFilterColumnsThrowsNoChange() throws Exception { + createTable("CREATE TABLE test.tbl_bf_same_legacy_columns (\n" + + "k1 INT,\n" + + "v1 STRING\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_columns\" = \"k1\",\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + expectFailureContains("Bloom filter index has no change", + () -> alterTableSync("ALTER TABLE test.tbl_bf_same_legacy_columns " + + "SET (\"bloom_filter_columns\" = \"k1\")")); + } + + @Test + public void testDropIndexRejectsLegacyBloomFilterColumn() throws Exception { + createTable("CREATE TABLE test.tbl_bf_drop_legacy (\n" + + "k1 INT,\n" + + "v1 STRING\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_columns\" = \"k1\",\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + expectFailureContains("index k1 does not exist", + () -> alterTableSync("ALTER TABLE test.tbl_bf_drop_legacy DROP INDEX k1")); + } + + @Test + public void testDropLastNamedBloomFilterClearsBfFpp() throws Exception { + createTable("CREATE TABLE test.tbl_bf_drop_last_named (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + alterTableSyncAndWait("ALTER TABLE test.tbl_bf_drop_last_named DROP INDEX idx_v1", + "tbl_bf_drop_last_named"); + + OlapTable table = getOlapTable("tbl_bf_drop_last_named"); + Assertions.assertNull(table.getCopiedBfColumns()); + Assertions.assertTrue(Index.extractBloomFilterColumns(table.getIndexes()).isEmpty()); + Assertions.assertNull(table.getCopiedBfColumns()); + Assertions.assertEquals(0, table.getBfFpp(), 0); + Assertions.assertTrue(table.getIndexes().isEmpty()); + } + + @Test + public void testShowCreateTableKeepsNamedBloomFilterOutOfLegacyProperties() throws Exception { + createTable("CREATE TABLE test.tbl_bf_show_create (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "INDEX idx_v1 (v1) USING BLOOMFILTER\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_columns\" = \"k1\",\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("test"); + OlapTable table = (OlapTable) db.getTableOrMetaException("tbl_bf_show_create", Table.TableType.OLAP); + List createTableStmt = Lists.newArrayList(); + List addRollupStmt = Lists.newArrayList(); + Env.getDdlStmt(table, createTableStmt, null, addRollupStmt, false, true, -1L); + + String ddl = createTableStmt.get(0); + Assertions.assertTrue(ddl.contains("\"bloom_filter_columns\" = \"k1\"")); + Assertions.assertFalse(ddl.contains("\"bloom_filter_columns\" = \"k1, v1\"")); + Assertions.assertFalse(ddl.contains("\"bloom_filter_columns\" = \"v1\"")); + Assertions.assertTrue(ddl.contains("INDEX idx_v1 (`v1`) USING BLOOMFILTER")); + Assertions.assertTrue(table.getCopiedBfColumns().contains("k1")); + Assertions.assertTrue(Index.extractBloomFilterColumns(table.getIndexes()).contains("v1")); + Assertions.assertEquals(1, table.getCopiedBfColumns().size()); + } + + @Test + public void testRenameLegacyBloomFilterColumnUpdatesMetadata() throws Exception { + // Keep one untouched legacy BF column so renameColumn() has to rewrite the renamed entry + // and preserve the other one in the same metadata set. + createTable("CREATE TABLE test.tbl_bf_rename_legacy (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "v2 INT\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_columns\" = \"k1,v1\",\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + alterTableSyncAndWait("ALTER TABLE test.tbl_bf_rename_legacy RENAME COLUMN v1 v1_new", + "tbl_bf_rename_legacy"); + + OlapTable table = getOlapTable("tbl_bf_rename_legacy"); + Assertions.assertTrue(table.getCopiedBfColumns().contains("k1")); + Assertions.assertFalse(table.getCopiedBfColumns().contains("v1")); + Assertions.assertTrue(table.getCopiedBfColumns().contains("v1_new")); + } + + @Test + public void testDropLegacyBloomFilterColumnUpdatesMetadata() throws Exception { + createTable("CREATE TABLE test.tbl_bf_drop_legacy_column (\n" + + "k1 INT,\n" + + "v1 STRING,\n" + + "v2 INT\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"bloom_filter_columns\" = \"k1,v1\",\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + + alterTableSyncAndWait("ALTER TABLE test.tbl_bf_drop_legacy_column DROP COLUMN v1", + "tbl_bf_drop_legacy_column"); + + OlapTable table = getOlapTable("tbl_bf_drop_legacy_column"); + Assertions.assertTrue(table.getCopiedBfColumns().contains("k1")); + Assertions.assertFalse(table.getCopiedBfColumns().contains("v1")); + Assertions.assertEquals(1, table.getCopiedBfColumns().size()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java index f7f7bb88beb8ab..436e001e5fed58 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java @@ -169,6 +169,35 @@ public void testResetPropertiesForRestore() { Assert.assertEquals((short) 3, olapTable.getDefaultReplicaAllocation().getTotalReplicaNum()); } + @Test + public void testNamedBloomFilterTableLevelFppDoesNotAffectSignature() throws IOException { + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS)) { + mockedEnv.when(Env::getCurrentEnvJournalVersion).thenReturn(FeConstants.meta_version); + + Database db = UnitTestUtil.createDb(11, 12, 13, 14, 15, 16, 17); + OlapTable olapTable = null; + for (Table table : db.getTables()) { + if (table.getType() == TableType.OLAP) { + olapTable = (OlapTable) table; + break; + } + } + Assert.assertNotNull(olapTable); + + olapTable.setIndexes(Lists.newArrayList(new Index(1L, "bf_v1", Lists.newArrayList("v1"), + IndexType.BLOOMFILTER, null, ""))); + + List partNames = Lists.newArrayList(olapTable.getPartitionNames()); + olapTable.setBloomFilterInfo(null, 0.01); + String signatureWithFpp001 = olapTable.getSignature(1, partNames); + + olapTable.setBloomFilterInfo(null, 0.02); + String signatureWithFpp002 = olapTable.getSignature(1, Lists.newArrayList(olapTable.getPartitionNames())); + + Assert.assertEquals(signatureWithFpp001, signatureWithFpp002); + } + } + @Test public void testResetPropertiesForRestoreInCloudMode() { // simulate a restoring table with properties that are unsupported in cloud mode diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogBloomFilterMaterializationTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogBloomFilterMaterializationTest.java new file mode 100644 index 00000000000000..bd9b83b97bc384 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogBloomFilterMaterializationTest.java @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cloud.datasource; + +import org.apache.doris.analysis.DataSortInfo; +import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Index; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.cloud.catalog.CloudReplica; +import org.apache.doris.cloud.catalog.CloudTablet; +import org.apache.doris.proto.OlapFile; +import org.apache.doris.proto.OlapFile.EncryptionAlgorithmPB; +import org.apache.doris.thrift.TCompressionType; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; +import org.apache.doris.thrift.TSortType; +import org.apache.doris.thrift.TStorageFormat; +import org.apache.doris.thrift.TTabletType; + +import com.google.common.collect.Lists; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class CloudInternalCatalogBloomFilterMaterializationTest { + + @Test + public void testCreateTabletMetaBuilderMaterializesNamedBloomFilter() throws Exception { + // Named-only BF indexes carry their own per-index FPP. The table-level bfFpp is only + // relevant for legacy bloom_filter_columns, so the cloud schema builder should NOT + // set it when bfColumns is null. + CloudInternalCatalog catalog = new CloudInternalCatalog(); + CloudTablet tablet = new CloudTablet(100L); + tablet.addReplica(new CloudReplica(101L, 1L, org.apache.doris.catalog.Replica.ReplicaState.NORMAL, + 1L, 1, 1L, 1L, 1L, 1L, 0L), true); + Column keyColumn = new Column("k1", ScalarType.createType(PrimitiveType.INT), true, null, "1", ""); + Column valueColumn = new Column("v1", ScalarType.createType(PrimitiveType.INT), false, + AggregateType.NONE, "1", ""); + + OlapFile.TabletMetaCloudPB tabletMeta = catalog.createTabletMetaBuilder(1L, 1L, 1L, tablet, + TTabletType.TABLET_TYPE_DISK, 1, KeysType.DUP_KEYS, (short) 1, null, 0, + Lists.newArrayList(new Index(1L, "bf_k1", Lists.newArrayList("k1"), + org.apache.doris.catalog.info.IndexType.BLOOMFILTER, + Map.of("bloom_filter_fpp", "0.02"), "")), + Lists.newArrayList(keyColumn, valueColumn), new DataSortInfo(TSortType.LEXICAL, 1), + TCompressionType.LZ4F, TStorageFormat.V2, "", false, false, "tbl", 0L, false, false, 1, "", + 0L, 0L, 0L, 0L, 0L, false, null, TInvertedIndexFileStorageFormat.V2, 16384L, false, null, + 65536L, EncryptionAlgorithmPB.PLAINTEXT, 262144L, false, Collections.emptyMap(), 5).build(); + + Assert.assertFalse(tabletMeta.getSchema().hasBfFpp()); + Assert.assertTrue(tabletMeta.getSchema().getColumn(0).getIsBfColumn()); + Assert.assertFalse(tabletMeta.getSchema().getColumn(1).getIsBfColumn()); + Assert.assertEquals("0.02", tabletMeta.getSchema().getIndex(0).getPropertiesMap().get("bloom_filter_fpp")); + } + + @Test + public void testCreateTabletMetaBuilderMaterializesNamedBloomFilterWithEmptyLegacySet() throws Exception { + // An empty legacy BF set should not change the named-BF behavior: the BF column still + // materializes, while the schema-level bfFpp remains whatever FE passed in. + CloudInternalCatalog catalog = new CloudInternalCatalog(); + CloudTablet tablet = new CloudTablet(100L); + tablet.addReplica(new CloudReplica(101L, 1L, org.apache.doris.catalog.Replica.ReplicaState.NORMAL, + 1L, 1, 1L, 1L, 1L, 1L, 0L), true); + Column keyColumn = new Column("k1", ScalarType.createType(PrimitiveType.INT), true, null, "1", ""); + + OlapFile.TabletMetaCloudPB tabletMeta = catalog.createTabletMetaBuilder(1L, 1L, 1L, tablet, + TTabletType.TABLET_TYPE_DISK, 1, KeysType.DUP_KEYS, (short) 1, new HashSet<>(), 0, + Lists.newArrayList(new Index(1L, "bf_k1", Lists.newArrayList("k1"), + org.apache.doris.catalog.info.IndexType.BLOOMFILTER, null, "")), + Lists.newArrayList(keyColumn), new DataSortInfo(TSortType.LEXICAL, 1), + TCompressionType.LZ4F, TStorageFormat.V2, "", false, false, "tbl", 0L, false, false, 1, "", + 0L, 0L, 0L, 0L, 0L, false, null, TInvertedIndexFileStorageFormat.V2, 16384L, false, null, + 65536L, EncryptionAlgorithmPB.PLAINTEXT, 262144L, false, Collections.emptyMap(), 5).build(); + + Assert.assertTrue(tabletMeta.getSchema().hasBfFpp()); + Assert.assertEquals(0, tabletMeta.getSchema().getBfFpp(), 0); + Assert.assertTrue(tabletMeta.getSchema().getColumn(0).getIsBfColumn()); + } + + @Test + public void testCreateTabletMetaBuilderDoesNotSetBfFppWithoutBloomFilter() throws Exception { + CloudInternalCatalog catalog = new CloudInternalCatalog(); + CloudTablet tablet = new CloudTablet(100L); + tablet.addReplica(new CloudReplica(101L, 1L, org.apache.doris.catalog.Replica.ReplicaState.NORMAL, + 1L, 1, 1L, 1L, 1L, 1L, 0L), true); + Column keyColumn = new Column("k1", ScalarType.createType(PrimitiveType.INT), true, null, "1", ""); + + OlapFile.TabletMetaCloudPB tabletMeta = catalog.createTabletMetaBuilder(1L, 1L, 1L, tablet, + TTabletType.TABLET_TYPE_DISK, 1, KeysType.DUP_KEYS, (short) 1, null, 0, Lists.newArrayList(), + Lists.newArrayList(keyColumn), new DataSortInfo(TSortType.LEXICAL, 1), + TCompressionType.LZ4F, TStorageFormat.V2, "", false, false, "tbl", 0L, false, false, 1, "", + 0L, 0L, 0L, 0L, 0L, false, null, TInvertedIndexFileStorageFormat.V2, 16384L, false, null, + 65536L, EncryptionAlgorithmPB.PLAINTEXT, 262144L, false, Collections.emptyMap(), 5).build(); + + Assert.assertFalse(tabletMeta.getSchema().hasBfFpp()); + Assert.assertFalse(tabletMeta.getSchema().getColumn(0).getIsBfColumn()); + } + + @Test + public void testCreateTabletMetaBuilderMaterializesLegacyBloomFilterWithExplicitFpp() throws Exception { + CloudInternalCatalog catalog = new CloudInternalCatalog(); + CloudTablet tablet = new CloudTablet(100L); + tablet.addReplica(new CloudReplica(101L, 1L, org.apache.doris.catalog.Replica.ReplicaState.NORMAL, + 1L, 1, 1L, 1L, 1L, 1L, 0L), true); + Column keyColumn = new Column("k1", ScalarType.createType(PrimitiveType.INT), true, null, "1", ""); + Column valueColumn = new Column("v1", ScalarType.createType(PrimitiveType.INT), false, + AggregateType.NONE, "1", ""); + Set legacyBloomFilterColumns = new HashSet<>(); + legacyBloomFilterColumns.add("k1"); + + OlapFile.TabletMetaCloudPB tabletMeta = catalog.createTabletMetaBuilder(1L, 1L, 1L, tablet, + TTabletType.TABLET_TYPE_DISK, 1, KeysType.DUP_KEYS, (short) 1, legacyBloomFilterColumns, 0.02, + Lists.newArrayList(), Lists.newArrayList(keyColumn, valueColumn), + new DataSortInfo(TSortType.LEXICAL, 1), TCompressionType.LZ4F, TStorageFormat.V2, + "", false, false, "tbl", 0L, false, false, 1, "", 0L, 0L, 0L, 0L, 0L, + false, null, TInvertedIndexFileStorageFormat.V2, 16384L, false, null, 65536L, + EncryptionAlgorithmPB.PLAINTEXT, 262144L, false, Collections.emptyMap(), 5).build(); + + Assert.assertTrue(tabletMeta.getSchema().hasBfFpp()); + Assert.assertEquals(0.02, tabletMeta.getSchema().getBfFpp(), 0); + Assert.assertTrue(tabletMeta.getSchema().getColumn(0).getIsBfColumn()); + Assert.assertFalse(tabletMeta.getSchema().getColumn(1).getIsBfColumn()); + } + + @Test + public void testCreateTabletMetaBuilderMaterializesShadowNamedBloomFilterWithIndexes() throws Exception { + // Shadow schema columns lose the FE-only shadow prefix when serialized, but the named BF + // index still matches on the non-shadow column name and keeps its index property. + CloudInternalCatalog catalog = new CloudInternalCatalog(); + CloudTablet tablet = new CloudTablet(100L); + tablet.addReplica(new CloudReplica(101L, 1L, org.apache.doris.catalog.Replica.ReplicaState.NORMAL, + 1L, 1, 1L, 1L, 1L, 1L, 0L), true); + Column shadowKeyColumn = new Column(Column.SHADOW_NAME_PREFIX + "k1", + ScalarType.createType(PrimitiveType.INT), true, null, "1", ""); + Column valueColumn = new Column("v1", ScalarType.createType(PrimitiveType.INT), false, + AggregateType.NONE, "1", ""); + OlapFile.TabletMetaCloudPB tabletMeta = catalog.createTabletMetaBuilder(1L, 2L, 1L, tablet, + TTabletType.TABLET_TYPE_DISK, 1, KeysType.DUP_KEYS, (short) 1, null, 0, + Lists.newArrayList(new Index(1L, "bf_k1", Lists.newArrayList("k1"), + org.apache.doris.catalog.info.IndexType.BLOOMFILTER, + Map.of("bloom_filter_fpp", "0.03"), "")), + Lists.newArrayList(shadowKeyColumn, valueColumn), + new DataSortInfo(TSortType.LEXICAL, 1), TCompressionType.LZ4F, TStorageFormat.V2, + "", false, true, "tbl", 0L, false, false, 1, "", 0L, 0L, 0L, 0L, 0L, + false, null, TInvertedIndexFileStorageFormat.V2, 16384L, false, null, 65536L, + EncryptionAlgorithmPB.PLAINTEXT, 262144L, false, Collections.emptyMap(), 5).build(); + + Assert.assertFalse(tabletMeta.getSchema().hasBfFpp()); + Assert.assertTrue(tabletMeta.getSchema().getColumn(0).getIsBfColumn()); + Assert.assertFalse(tabletMeta.getSchema().getColumn(1).getIsBfColumn()); + Assert.assertEquals("k1", tabletMeta.getSchema().getColumn(0).getName()); + Assert.assertEquals("0.03", tabletMeta.getSchema().getIndex(0).getPropertiesMap().get("bloom_filter_fpp")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java index 82c7066d4c9c5c..ccbbd19e0d2dff 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.StatementBase; import org.apache.doris.analysis.StmtType; import org.apache.doris.analysis.TableScanParams; +import org.apache.doris.catalog.info.IndexType; import org.apache.doris.common.Config; import org.apache.doris.common.Pair; import org.apache.doris.nereids.StatementContext; @@ -47,6 +48,7 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.commands.AlterTableCommand; import org.apache.doris.nereids.trees.plans.commands.CancelAlterTableCommand; import org.apache.doris.nereids.trees.plans.commands.CreateMaterializedViewCommand; import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; @@ -58,6 +60,9 @@ import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; import org.apache.doris.nereids.trees.plans.commands.ReplayCommand; import org.apache.doris.nereids.trees.plans.commands.UpdateCommand; +import org.apache.doris.nereids.trees.plans.commands.info.CreateIndexOp; +import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo; +import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition; import org.apache.doris.nereids.trees.plans.commands.merge.MergeIntoCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; @@ -1445,6 +1450,81 @@ public void testCtasWithoutAs() { Assertions.assertTrue(createTableCommand.getCtasQuery().isPresent()); } + @Test + public void testCreateIndexUsingBloomFilter() { + NereidsParser parser = new NereidsParser(); + String sql = "CREATE INDEX idx_content ON docs(content) USING BLOOMFILTER"; + Plan plan = parser.parseSingle(sql); + Assertions.assertInstanceOf(AlterTableCommand.class, plan); + AlterTableCommand alterTableCommand = (AlterTableCommand) plan; + Assertions.assertEquals(1, alterTableCommand.getOps().size()); + Assertions.assertInstanceOf(CreateIndexOp.class, alterTableCommand.getOps().get(0)); + CreateIndexOp createIndexOp = (CreateIndexOp) alterTableCommand.getOps().get(0); + IndexDefinition indexDefinition = createIndexOp.getIndexDef(); + Assertions.assertEquals("idx_content", indexDefinition.getIndexName()); + Assertions.assertEquals(Lists.newArrayList("content"), indexDefinition.getColumnNames()); + Assertions.assertEquals(IndexType.BLOOMFILTER, indexDefinition.getIndexType()); + Assertions.assertEquals("docs", createIndexOp.getTableName().getTbl()); + } + + @Test + public void testCreateIndexUsingBloomFilterWithPropertiesThrows() { + NereidsParser parser = new NereidsParser(); + String sql = "CREATE INDEX idx_content ON docs(content) USING BLOOMFILTER " + + "PROPERTIES (\"bloom_filter_fpp\" = \"0.05\")"; + Plan plan = parser.parseSingle(sql); + Assertions.assertInstanceOf(AlterTableCommand.class, plan); + AlterTableCommand alterTableCommand = (AlterTableCommand) plan; + Assertions.assertEquals(1, alterTableCommand.getOps().size()); + Assertions.assertInstanceOf(CreateIndexOp.class, alterTableCommand.getOps().get(0)); + CreateIndexOp createIndexOp = (CreateIndexOp) alterTableCommand.getOps().get(0); + IndexDefinition indexDefinition = createIndexOp.getIndexDef(); + Assertions.assertDoesNotThrow(indexDefinition::validate); + Assertions.assertEquals("0.05", indexDefinition.getProperties().get("bloom_filter_fpp")); + } + + @Test + public void testCreateTableInlineBloomFilterIndex() throws Exception { + NereidsParser parser = new NereidsParser(); + String sql = "CREATE TABLE docs (" + + "id BIGINT, " + + "content TEXT, " + + "INDEX idx_content (content) USING BLOOMFILTER COMMENT \"this is a simple bloomfilter index\"" + + ") DISTRIBUTED BY HASH(id) BUCKETS 1"; + Plan plan = parser.parseSingle(sql); + Assertions.assertInstanceOf(CreateTableCommand.class, plan); + CreateTableInfo createTableInfo = ((CreateTableCommand) plan).getCreateTableInfo(); + Field indexesField = CreateTableInfo.class.getDeclaredField("indexes"); + indexesField.setAccessible(true); + List indexDefinitions = (List) indexesField.get(createTableInfo); + Assertions.assertEquals(1, indexDefinitions.size()); + IndexDefinition indexDefinition = indexDefinitions.get(0); + Assertions.assertEquals("idx_content", indexDefinition.getIndexName()); + Assertions.assertEquals(Lists.newArrayList("content"), indexDefinition.getColumnNames()); + Assertions.assertEquals(IndexType.BLOOMFILTER, indexDefinition.getIndexType()); + Assertions.assertEquals("this is a simple bloomfilter index", indexDefinition.getComment()); + } + + @Test + public void testCreateTableInlineBloomFilterIndexWithFppProperty() throws Exception { + NereidsParser parser = new NereidsParser(); + String sql = "CREATE TABLE docs (" + + "id BIGINT, " + + "content TEXT, " + + "INDEX idx_content (content) USING BLOOMFILTER " + + "PROPERTIES (\"bloom_filter_fpp\" = \"0.05\")" + + ") DISTRIBUTED BY HASH(id) BUCKETS 1"; + Plan plan = parser.parseSingle(sql); + Assertions.assertInstanceOf(CreateTableCommand.class, plan); + CreateTableInfo createTableInfo = ((CreateTableCommand) plan).getCreateTableInfo(); + Field indexesField = CreateTableInfo.class.getDeclaredField("indexes"); + indexesField.setAccessible(true); + List indexDefinitions = (List) indexesField.get(createTableInfo); + Assertions.assertEquals(1, indexDefinitions.size()); + Assertions.assertDoesNotThrow(() -> indexDefinitions.get(0).validate()); + Assertions.assertEquals("0.05", indexDefinitions.get(0).getProperties().get("bloom_filter_fpp")); + } + @Test public void testCreateTableVariantNestedGroupPropertyIsAccepted() { NereidsParser parser = new NereidsParser(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java index 7b41ddc95cf840..02dc95394228e4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java @@ -18,12 +18,15 @@ package org.apache.doris.nereids.trees.plans.commands; import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.Type; import org.apache.doris.catalog.info.IndexType; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition; import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.FloatType; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.MapType; @@ -42,6 +45,16 @@ import java.util.Map; public class IndexDefinitionTest { + private IndexDefinition newBloomFilterIndexDefinition() { + return new IndexDefinition("bf_index", false, Lists.newArrayList("col1"), "BLOOMFILTER", null, "comment"); + } + + private Column newCatalogColumn(Type type, boolean isKey) { + Column column = new Column("col1", type); + column.setIsKey(isKey); + return column; + } + @Test void testVariantIndexFormatV1() throws AnalysisException { IndexDefinition def = new IndexDefinition("variant_index", false, Lists.newArrayList("col1"), "INVERTED", @@ -137,6 +150,174 @@ void testNgramBFIndex() throws AnalysisException { KeysType.DUP_KEYS, false, null); } + @Test + void testBloomFilterIndexTypeName() { + IndexDefinition def = new IndexDefinition("bf_index", false, Lists.newArrayList("col1"), "BLOOMFILTER", + null, "comment"); + Assertions.assertEquals(IndexType.BLOOMFILTER, def.getIndexType()); + } + + @Test + void testBloomFilterIndexSupportsBloomFilterFppProperty() { + Map properties = new HashMap<>(); + properties.put("bloom_filter_fpp", "0.05"); + + IndexDefinition def = new IndexDefinition("bf_index", false, Lists.newArrayList("col1"), "BLOOMFILTER", + properties, "comment"); + Assertions.assertDoesNotThrow(def::validate); + Assertions.assertEquals("0.05", def.getProperties().get("bloom_filter_fpp")); + } + + @Test + void testBloomFilterIndexRejectsNonBloomFilterFppProperty() { + Map properties = new HashMap<>(); + properties.put("foo", "bar"); + + IndexDefinition def = new IndexDefinition("bf_index", false, Lists.newArrayList("col1"), "BLOOMFILTER", + properties, "comment"); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, def::validate); + Assertions.assertEquals("BLOOMFILTER index only supports property bloom_filter_fpp", + exception.getMessage()); + } + + @Test + void testBloomFilterIndexRejectsInvalidBloomFilterFppProperty() { + Map properties = new HashMap<>(); + properties.put("bloom_filter_fpp", "0.1"); + + IndexDefinition def = new IndexDefinition("bf_index", false, Lists.newArrayList("col1"), "BLOOMFILTER", + properties, "comment"); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, def::validate); + Assertions.assertTrue(exception.getMessage().contains("Bloom filter fpp should in [1.0E-4, 0.05]")); + } + + @Test + void testBloomFilterIndexSupportsVariantColumnDefinition() { + // Baseline check for the new named BLOOMFILTER path on VARIANT columns. + IndexDefinition def = newBloomFilterIndexDefinition(); + Assertions.assertDoesNotThrow(() -> def.checkColumn( + new ColumnDefinition("col1", VariantType.INSTANCE, false, AggregateType.NONE, true, null, "comment"), + KeysType.DUP_KEYS, false, null)); + } + + @Test + void testBloomFilterIndexSupportsVariantCatalogColumn() { + IndexDefinition def = newBloomFilterIndexDefinition(); + Assertions.assertDoesNotThrow(() -> def.checkColumn( + newCatalogColumn(Type.VARIANT, true), KeysType.DUP_KEYS, false, null)); + } + + @Test + void testBloomFilterIndexSupportsVariantColumnDefinitionWithV1StorageFormat() { + // Inverted-index-specific VARIANT + V1 rejection must not leak into BLOOMFILTER. + IndexDefinition def = newBloomFilterIndexDefinition(); + Assertions.assertDoesNotThrow(() -> def.checkColumn( + new ColumnDefinition("col1", VariantType.INSTANCE, false, AggregateType.NONE, true, null, "comment"), + KeysType.DUP_KEYS, false, TInvertedIndexFileStorageFormat.V1)); + } + + @Test + void testBloomFilterIndexSupportsVariantColumnDefinitionWithDefaultStorageFormat() { + // DEFAULT may resolve to the old V1 rules elsewhere, so this regression test keeps + // BLOOMFILTER isolated from inverted-index-only validation. + IndexDefinition def = newBloomFilterIndexDefinition(); + Assertions.assertDoesNotThrow(() -> def.checkColumn( + new ColumnDefinition("col1", VariantType.INSTANCE, false, AggregateType.NONE, true, null, "comment"), + KeysType.DUP_KEYS, false, TInvertedIndexFileStorageFormat.DEFAULT)); + } + + @Test + void testBloomFilterIndexSupportsVariantCatalogColumnWithDefaultStorageFormat() { + IndexDefinition def = newBloomFilterIndexDefinition(); + Assertions.assertDoesNotThrow(() -> def.checkColumn( + newCatalogColumn(Type.VARIANT, true), KeysType.DUP_KEYS, false, + TInvertedIndexFileStorageFormat.DEFAULT)); + } + + @Test + void testBloomFilterIndexSupportsVariantCatalogColumnWithV1StorageFormat() { + IndexDefinition def = newBloomFilterIndexDefinition(); + Assertions.assertDoesNotThrow(() -> def.checkColumn( + newCatalogColumn(Type.VARIANT, true), KeysType.DUP_KEYS, false, + TInvertedIndexFileStorageFormat.V1)); + } + + @Test + void testBloomFilterIndexRejectsUnsupportedColumnDefinitionType() { + IndexDefinition def = newBloomFilterIndexDefinition(); + // BOOLEAN is accepted by neither legacy nor named bloom filters. + ColumnDefinition columnDef = new ColumnDefinition("col1", BooleanType.INSTANCE, false, AggregateType.NONE, + true, null, "comment"); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, () -> def.checkColumn( + columnDef, KeysType.DUP_KEYS, false, null)); + Assertions.assertEquals(columnDef.getType() + " is not supported in bloom filter index. invalid column: " + + columnDef.getName(), + exception.getMessage()); + } + + @Test + void testBloomFilterIndexSupportsValueColumnOnDupKeys() { + IndexDefinition def = newBloomFilterIndexDefinition(); + Assertions.assertDoesNotThrow(() -> def.checkColumn( + new ColumnDefinition("col1", StringType.INSTANCE, false, AggregateType.NONE, true, null, "comment"), + KeysType.DUP_KEYS, false, null)); + } + + @Test + void testBloomFilterIndexRejectsValueColumnOnAggKeysForColumnDefinition() { + IndexDefinition def = newBloomFilterIndexDefinition(); + // BLOOMFILTER follows the same key-column restriction as the existing secondary indexes. + ColumnDefinition columnDef = new ColumnDefinition("col1", StringType.INSTANCE, false, AggregateType.NONE, + true, null, "comment"); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, () -> def.checkColumn( + columnDef, KeysType.AGG_KEYS, false, null)); + Assertions.assertEquals("index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table" + + " or key columns of AGG_KEYS table. invalid index: " + def.getIndexName(), + exception.getMessage()); + } + + @Test + void testBloomFilterIndexSupportsValueColumnOnUniqueKeysForCatalogColumn() { + IndexDefinition def = newBloomFilterIndexDefinition(); + Assertions.assertDoesNotThrow(() -> def.checkColumn(newCatalogColumn(Type.STRING, false), + KeysType.UNIQUE_KEYS, false, null)); + } + + @Test + void testBloomFilterIndexRejectsUnsupportedCatalogColumnType() { + IndexDefinition def = newBloomFilterIndexDefinition(); + // Cover the catalog Column overload so both checkColumn(...) entry points stay aligned. + Column column = newCatalogColumn(Type.BOOLEAN, false); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> def.checkColumn(column, KeysType.DUP_KEYS, false, null)); + Assertions.assertEquals(column.getDataType() + " is not supported in bloom filter index. invalid column: " + + column.getName(), exception.getMessage()); + } + + @Test + void testBloomFilterIndexRejectsValueColumnOnAggKeysForCatalogColumn() { + IndexDefinition def = newBloomFilterIndexDefinition(); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> def.checkColumn(newCatalogColumn(Type.STRING, false), KeysType.AGG_KEYS, false, null)); + Assertions.assertEquals("index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table" + + " or key columns of AGG_KEYS table. invalid index: " + def.getIndexName(), + exception.getMessage()); + } + + @Test + void testBloomFilterIndexOnlySingleColumn() { + IndexDefinition def = new IndexDefinition("bf_index", false, Lists.newArrayList("col1", "col2"), + "BLOOMFILTER", null, "comment"); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, def::validate); + Assertions.assertEquals("BLOOMFILTER index can only apply to a single column.", exception.getMessage()); + } + + @Test + void testBloomFilterIndexToSqlUsesBloomFilterKeyword() { + IndexDefinition def = newBloomFilterIndexDefinition(); + Assertions.assertTrue(def.toSql().contains("USING BLOOMFILTER")); + } + @Test void testNgramBFIndexOnlySingleColumn() { IndexDefinition def = new IndexDefinition("ngram_bf_index", false, Lists.newArrayList("col1", "col2"), diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java index ededa002296bcb..96930f747e6e61 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java @@ -22,11 +22,13 @@ import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.BinlogConfig; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Index; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.info.IndexType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.MarkedCountDownLatch; import org.apache.doris.metric.MetricRepo; @@ -193,6 +195,103 @@ public void toThriftTest() throws Exception { Assert.assertNotNull(requestWithRowBinlog.getCreateTabletReq()); Assert.assertNotNull(requestWithRowBinlog.getCreateTabletReq().getRowBinlogSchema()); + List namedBloomFilterIndexes = Arrays.asList(new Index(1L, "bf_k1", Arrays.asList("k1"), + IndexType.BLOOMFILTER, Map.of("bloom_filter_fpp", "0.02"), "")); + AgentTask createWithNamedBloomFilter = new CreateReplicaTask(backendId1, dbId, tableId, partitionId, + indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, storageType, + TStorageMedium.SSD, columns, null, 0, latch, namedBloomFilterIndexes, false, + TTabletType.TABLET_TYPE_DISK, null, TCompressionType.LZ4F, false, "", false, false, "", 0, 0, 0, + 0, 0, false, null, null, new HashMap<>(), rowStorePageSize, false, storagePageSize, + TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize, new HashMap<>(), 5, null); + TAgentTaskRequest requestWithNamedBloomFilter = + (TAgentTaskRequest) toAgentTaskRequest.invoke(agentBatchTask, createWithNamedBloomFilter); + Assert.assertNotNull(requestWithNamedBloomFilter.getCreateTabletReq()); + Assert.assertTrue(requestWithNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getColumns().get(0).isIsBloomFilterColumn()); + Assert.assertFalse(requestWithNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getColumns().get(1).isSetIsBloomFilterColumn()); + // bfColumns is null, so table-level FPP is not set for named-only bloom filter indexes. + // Each named index carries its own FPP in its properties. + Assert.assertFalse(requestWithNamedBloomFilter.getCreateTabletReq().getTabletSchema().isSetBloomFilterFpp()); + Assert.assertTrue(requestWithNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getIndexes().get(0).getProperties().containsKey("bloom_filter_fpp")); + Assert.assertEquals("0.02", requestWithNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getIndexes().get(0).getProperties().get("bloom_filter_fpp")); + + Set legacyBloomFilterColumns = new HashSet<>(); + legacyBloomFilterColumns.add("k1"); + AgentTask createWithLegacyBloomFilter = new CreateReplicaTask(backendId1, dbId, tableId, partitionId, + indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, storageType, + TStorageMedium.SSD, columns, legacyBloomFilterColumns, 0.02, latch, null, false, + TTabletType.TABLET_TYPE_DISK, null, TCompressionType.LZ4F, false, "", false, false, "", 0, 0, 0, + 0, 0, false, null, null, new HashMap<>(), rowStorePageSize, false, storagePageSize, + TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize, new HashMap<>(), 5, null); + TAgentTaskRequest requestWithLegacyBloomFilter = + (TAgentTaskRequest) toAgentTaskRequest.invoke(agentBatchTask, createWithLegacyBloomFilter); + Assert.assertNotNull(requestWithLegacyBloomFilter.getCreateTabletReq()); + Assert.assertTrue(requestWithLegacyBloomFilter.getCreateTabletReq().getTabletSchema() + .getColumns().get(0).isIsBloomFilterColumn()); + Assert.assertEquals(0.02, + requestWithLegacyBloomFilter.getCreateTabletReq().getTabletSchema().getBloomFilterFpp(), 0); + + List shadowColumns = Arrays.asList( + new Column(Column.SHADOW_NAME_PREFIX + "k1", ScalarType.createType(PrimitiveType.INT), + false, null, "1", ""), + new Column("v1", ScalarType.createType(PrimitiveType.INT), false, AggregateType.SUM, "1", "")); + AgentTask createWithShadowNamedBloomFilter = new CreateReplicaTask(backendId1, dbId, tableId, partitionId, + indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, storageType, + TStorageMedium.SSD, shadowColumns, null, 0, latch, namedBloomFilterIndexes, false, + TTabletType.TABLET_TYPE_DISK, null, TCompressionType.LZ4F, false, "", false, false, "", 0, 0, 0, + 0, 0, false, null, null, new HashMap<>(), rowStorePageSize, false, storagePageSize, + TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize, new HashMap<>(), 5, null); + TAgentTaskRequest requestWithShadowNamedBloomFilter = + (TAgentTaskRequest) toAgentTaskRequest.invoke(agentBatchTask, createWithShadowNamedBloomFilter); + Assert.assertEquals("k1", requestWithShadowNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getColumns().get(0).getColumnName()); + Assert.assertTrue(requestWithShadowNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getColumns().get(0).isIsBloomFilterColumn()); + + AgentTask createWithFoldedNamedBloomFilter = new CreateReplicaTask(backendId1, dbId, tableId, partitionId, + indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, storageType, + TStorageMedium.SSD, shadowColumns, null, 0, latch, + Arrays.asList(new Index(2L, "bf_shadow_k1", Arrays.asList("k1"), + IndexType.BLOOMFILTER, Map.of("bloom_filter_fpp", "0.03"), "")), false, + TTabletType.TABLET_TYPE_DISK, null, TCompressionType.LZ4F, false, "", false, false, "", 0, 0, 0, + 0, 0, false, null, null, new HashMap<>(), rowStorePageSize, false, storagePageSize, + TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize, new HashMap<>(), 5, null); + TAgentTaskRequest requestWithFoldedNamedBloomFilter = + (TAgentTaskRequest) toAgentTaskRequest.invoke(agentBatchTask, createWithFoldedNamedBloomFilter); + Assert.assertEquals("k1", requestWithFoldedNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getColumns().get(0).getColumnName()); + Assert.assertTrue(requestWithFoldedNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getColumns().get(0).isIsBloomFilterColumn()); + // bfColumns is null, so table-level FPP is not set. Named indexes carry their own FPP. + Assert.assertFalse(requestWithFoldedNamedBloomFilter.getCreateTabletReq().getTabletSchema().isSetBloomFilterFpp()); + Assert.assertTrue(requestWithFoldedNamedBloomFilter.getCreateTabletReq().getTabletSchema().isSetIndexes()); + Assert.assertEquals("0.03", requestWithFoldedNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getIndexes().get(0).getProperties().get("bloom_filter_fpp")); + + Set emptyLegacyBloomFilterColumns = new HashSet<>(); + // When bfColumns is non-null (even empty), table-level FPP is set to whatever was + // passed in. Here bfFpp=0, so the schema-level FPP will be 0. Named indexes carry + // their own per-index FPP via index properties. + AgentTask createWithEmptyLegacySetAndNamedBloomFilter = new CreateReplicaTask(backendId1, dbId, tableId, + partitionId, indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, + storageType, TStorageMedium.SSD, columns, emptyLegacyBloomFilterColumns, 0, latch, + namedBloomFilterIndexes, false, TTabletType.TABLET_TYPE_DISK, null, TCompressionType.LZ4F, false, + "", false, false, "", 0, 0, 0, 0, 0, false, null, null, new HashMap<>(), rowStorePageSize, false, + storagePageSize, TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize, new HashMap<>(), 5, null); + TAgentTaskRequest requestWithEmptyLegacySetAndNamedBloomFilter = + (TAgentTaskRequest) toAgentTaskRequest.invoke(agentBatchTask, + createWithEmptyLegacySetAndNamedBloomFilter); + Assert.assertTrue(requestWithEmptyLegacySetAndNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getColumns().get(0).isIsBloomFilterColumn()); + Assert.assertTrue(requestWithEmptyLegacySetAndNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .isSetBloomFilterFpp()); + Assert.assertEquals(0.0, + requestWithEmptyLegacySetAndNamedBloomFilter.getCreateTabletReq().getTabletSchema() + .getBloomFilterFpp(), 0); + // drop TAgentTaskRequest request2 = (TAgentTaskRequest) toAgentTaskRequest.invoke(agentBatchTask, dropTask); Assert.assertEquals(TTaskType.DROP, request2.getTaskType()); diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 446685892295e4..615856e8d1929d 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -108,6 +108,7 @@ BITMAP_UNION: 'BITMAP_UNION'; BITOR: 'BITOR'; BITXOR: 'BITXOR'; BLOB: 'BLOB'; +BLOOMFILTER: 'BLOOMFILTER'; BOOLEAN: 'BOOLEAN'; BOTH: 'BOTH'; BRANCH: 'BRANCH'; diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index cd2391b1775f1e..abcfdf4e959200 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -236,7 +236,7 @@ supportedCreateStatement partitionSpec? #buildIndex | CREATE INDEX (IF NOT EXISTS)? name=identifier ON tableName=multipartIdentifier identifierList - (USING (NGRAM_BF | INVERTED | ANN))? + (USING indexType=(BLOOMFILTER | NGRAM_BF | INVERTED | ANN))? properties=propertyClause? (COMMENT STRING_LITERAL)? #createIndex | CREATE WORKLOAD POLICY (IF NOT EXISTS)? name=identifierOrText (CONDITIONS LEFT_PAREN workloadPolicyConditions RIGHT_PAREN)? @@ -1559,7 +1559,7 @@ indexDefs ; indexDef - : INDEX (ifNotExists=IF NOT EXISTS)? indexName=identifier cols=identifierList (USING indexType=(INVERTED | NGRAM_BF | ANN ))? (PROPERTIES LEFT_PAREN properties=propertyItemList RIGHT_PAREN)? (COMMENT comment=STRING_LITERAL)? + : INDEX (ifNotExists=IF NOT EXISTS)? indexName=identifier cols=identifierList (USING indexType=(BLOOMFILTER | INVERTED | NGRAM_BF | ANN ))? (PROPERTIES LEFT_PAREN properties=propertyItemList RIGHT_PAREN)? (COMMENT comment=STRING_LITERAL)? ; partitionsDef @@ -2121,6 +2121,7 @@ nonReserved | EXCLUDE | EXPIRED | EXTERNAL + | BLOOMFILTER | FAILED_LOGIN_ATTEMPTS | FAST | FEATURE diff --git a/regression-test/data/bloom_filter_p0/test_bloom_filter_named_index.out b/regression-test/data/bloom_filter_p0/test_bloom_filter_named_index.out new file mode 100644 index 00000000000000..5295bb00947fea --- /dev/null +++ b/regression-test/data/bloom_filter_p0/test_bloom_filter_named_index.out @@ -0,0 +1,109 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !named_inline_data -- +1 alpha 10 +2 beta 20 + +-- !desc_named_inline -- +k1 int Yes true \N BLOOM_FILTER +v1 text Yes false \N NONE,BLOOM_FILTER +v2 int Yes false \N NONE,BLOOM_FILTER + +-- !desc_all_named_inline -- +test_bloom_filter_named_inline DUP_KEYS k1 int int Yes true \N BLOOM_FILTER true + v1 text text Yes false \N NONE,BLOOM_FILTER true + v2 int int Yes false \N NONE,BLOOM_FILTER true + +-- !show_proc_named_inline_mixed -- +k1 int Yes true \N BLOOM_FILTER +v1 text Yes false \N NONE,BLOOM_FILTER +v2 int Yes false \N NONE,BLOOM_FILTER + +-- !show_index_named_inline -- +test_bloom_filter_named_inline idx_v1 v1 BLOOMFILTER +test_bloom_filter_named_inline idx_v2 v2 BLOOMFILTER + +-- !desc_named_inline_after_legacy_drop -- +k1 int Yes true \N +v1 text Yes false \N NONE,BLOOM_FILTER +v2 int Yes false \N NONE,BLOOM_FILTER + +-- !desc_named_add_on_legacy -- +k1 int Yes true \N BLOOM_FILTER +v1 text Yes false \N NONE +v2 int Yes false \N NONE,BLOOM_FILTER + +-- !show_index_named_add_on_legacy -- +test_bloom_filter_named_add_on_legacy idx_v2 v2 BLOOMFILTER + +-- !desc_named_drop_mixed_before -- +k1 int Yes true \N BLOOM_FILTER +v1 text Yes false \N NONE,BLOOM_FILTER +v2 int Yes false \N NONE + +-- !show_index_named_drop_mixed_before -- +test_bloom_filter_named_drop_mixed idx_v1 v1 BLOOMFILTER + +-- !desc_named_drop_mixed_after -- +k1 int Yes true \N BLOOM_FILTER +v1 text Yes false \N NONE +v2 int Yes false \N NONE + +-- !show_index_named_drop_mixed_after -- + +-- !desc_named_rename_legacy_mixed -- +k1 int Yes true \N +v1_new text Yes false \N NONE,BLOOM_FILTER +v2 int Yes false \N NONE,BLOOM_FILTER + +-- !show_index_named_rename_legacy_mixed -- +test_bloom_filter_named_rename_legacy_mixed idx_v2 v2 BLOOMFILTER + +-- !desc_named_drop_legacy_column_mixed -- +k1 int Yes true \N BLOOM_FILTER +v2 int Yes false \N NONE,BLOOM_FILTER + +-- !show_index_named_drop_legacy_column_mixed -- +test_bloom_filter_named_drop_legacy_column_mixed idx_v2 v2 BLOOMFILTER + +-- !named_rollup_data -- +1 10 alpha +2 20 beta + +-- !show_index_named_rollup -- +test_bloom_filter_named_rollup DUP_KEYS k1 int int Yes true \N true + v2 int int Yes true \N true + v1 text text Yes false \N NONE,BLOOM_FILTER true + +r1 DUP_KEYS k1 int int Yes true \N true + v1 text text Yes false \N NONE,BLOOM_FILTER true + +-- !show_index_named_rollup_after_drop -- +test_bloom_filter_named_rollup DUP_KEYS k1 int int Yes true \N true + v2 int int Yes true \N true + v1 text text Yes false \N NONE true + +r1 DUP_KEYS k1 int int Yes true \N true + v1 text text Yes false \N NONE true + +-- !named_create_index_data -- +1 one 101 +2 two 202 + +-- !desc_named_create_index -- +k1 int Yes true \N +v1 text Yes false \N NONE +v2 int Yes false \N NONE,BLOOM_FILTER + +-- !show_index_named_create_index -- +test_bloom_filter_named_create_index idx_v2 v2 BLOOMFILTER ("bloom_filter_fpp" = "0.02") + +-- !desc_named_create_index_after_drop -- +k1 int Yes true \N +v1 text Yes false \N NONE +v2 int Yes false \N NONE + +-- !show_index_named_create_index_after_drop -- + +-- !show_index_named_variant -- +test_bloom_filter_named_variant idx_v v BLOOMFILTER ("bloom_filter_fpp" = "0.03") + diff --git a/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy b/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy index d3b5a4791962d6..9ee82da0c9010b 100644 --- a/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy +++ b/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy @@ -221,6 +221,7 @@ suite("test_bloom_filter","nonConcurrent") { DISTRIBUTED BY HASH(`id`) BUCKETS 5 PROPERTIES ( "replication_num" = "1", + "disable_auto_compaction" = "true", "bloom_filter_columns" = "id", "bloom_filter_fpp" = "0.03" )""" @@ -234,11 +235,19 @@ suite("test_bloom_filter","nonConcurrent") { GetDebugPoint().disableDebugPointForAllBEs("BloomFilterIndexWriter::create"); } - sql """ALTER TABLE ${test_dynamic_fpp_tb} SET("bloom_filter_fpp" = "0.02")""" - wait_for_latest_op_on_table_finish(test_dynamic_fpp_tb, timeout) + try { + GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexWriter::create", + [fpp: "0.02", execute: "1", timeout: "120"]) + sql """ALTER TABLE ${test_dynamic_fpp_tb} SET("bloom_filter_fpp" = "0.02")""" + wait_for_latest_op_on_table_finish(test_dynamic_fpp_tb, timeout) + sql """ALTER TABLE ${test_dynamic_fpp_tb} SET("bloom_filter_fpp" = "0.03")""" + wait_for_latest_op_on_table_finish(test_dynamic_fpp_tb, timeout) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("BloomFilterIndexWriter::create"); + } try { - GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexWriter::create", [fpp: "0.02"]) + GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexWriter::create", [fpp: "0.03"]) sql """INSERT INTO ${test_dynamic_fpp_tb} VALUES (6, 'Grace'), (7, 'Henry'), diff --git a/regression-test/suites/bloom_filter_p0/test_bloom_filter_named_index.groovy b/regression-test/suites/bloom_filter_p0/test_bloom_filter_named_index.groovy new file mode 100644 index 00000000000000..f2a7952fcca539 --- /dev/null +++ b/regression-test/suites/bloom_filter_p0/test_bloom_filter_named_index.groovy @@ -0,0 +1,595 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_bloom_filter_named_index", "nonConcurrent") { + def getBaseIndexSchemaProcPath = { String tableName -> + def dbName = (sql """select database()""")[0][0] + def catalogId = get_catalog_id("internal") + def dbId = get_database_id("internal", dbName) + def tableId = get_table_id("internal", dbName, tableName) + def indexSchemaProc = """/catalogs/${catalogId}/${dbId}/${tableId}/index_schema""" + def indexRows = sql """show proc '${indexSchemaProc}'""" + def baseIndexId = null + for (int i = 0; i < indexRows.size(); i++) { + if (indexRows[i][1].equals(tableName)) { + baseIndexId = indexRows[i][0] + break + } + } + assert baseIndexId != null + return """${indexSchemaProc}/${baseIndexId}""" + } + + sql """DROP TABLE IF EXISTS test_bloom_filter_named_inline""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_create_index""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_add_on_legacy""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_drop_mixed""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_rename_legacy_mixed""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_drop_legacy_column_mixed""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_rollup""" + sql """DROP TABLE IF EXISTS test_bloom_filter_cov_inline_invalid_type""" + sql """DROP TABLE IF EXISTS test_bloom_filter_cov_legacy_same_columns""" + sql """DROP TABLE IF EXISTS test_bloom_filter_cov_legacy_same_fpp""" + sql """DROP TABLE IF EXISTS test_bloom_filter_cov_no_bf_fpp_only""" + sql """DROP TABLE IF EXISTS test_bloom_filter_cov_add_index_invalid_type""" + sql """DROP TABLE IF EXISTS test_bloom_filter_cov_add_index_with_properties""" + sql """DROP TABLE IF EXISTS test_bloom_filter_cov_empty_legacy_columns""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_variant""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_array_invalid""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_map_invalid""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_struct_invalid""" + sql """DROP TABLE IF EXISTS test_bloom_filter_named_duplicate_inline""" + sql """DROP TABLE IF EXISTS test_mixed_fpp_bloom_filter_tb""" + + test { + sql """ + CREATE TABLE test_bloom_filter_named_duplicate_inline ( + k1 INT, + v1 STRING, + INDEX idx_v1_a (v1) USING BLOOMFILTER, + INDEX idx_v1_b (v1) USING BLOOMFILTER + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + exception "column: v1 cannot have multiple indexes, index type: BLOOMFILTER" + } + + sql """ + CREATE TABLE test_bloom_filter_named_inline ( + k1 INT, + v1 STRING, + v2 INT, + INDEX idx_v1 (v1) USING BLOOMFILTER, + INDEX idx_v2 (v2) USING BLOOMFILTER + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """INSERT INTO test_bloom_filter_named_inline VALUES (1, 'alpha', 10), (2, 'beta', 20)""" + + sql """ALTER TABLE test_bloom_filter_named_inline SET ("bloom_filter_columns" = "k1")""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_inline' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + order_qt_named_inline_data """SELECT * FROM test_bloom_filter_named_inline ORDER BY k1""" + qt_desc_named_inline """DESC test_bloom_filter_named_inline""" + qt_desc_all_named_inline """DESC test_bloom_filter_named_inline ALL""" + qt_show_proc_named_inline_mixed """SHOW PROC '${getBaseIndexSchemaProcPath("test_bloom_filter_named_inline")}'""" + qt_show_index_named_inline """SHOW INDEX FROM test_bloom_filter_named_inline""" + + test { + sql """ALTER TABLE test_bloom_filter_named_inline SET ("bloom_filter_columns" = "k1,v1")""" + exception "ALTER TABLE failed, expected to create bloom filter index on column v1, but this column is already defined by named BLOOMFILTER index" + } + + test { + sql """DROP INDEX k1 ON test_bloom_filter_named_inline""" + exception "index k1 does not exist" + } + + test { + sql """CREATE INDEX idx_k1 ON test_bloom_filter_named_inline(k1) USING BLOOMFILTER""" + exception "k1 should have only one ngram bloom filter index or bloom filter index" + } + + test { + sql """CREATE INDEX idx_v1_dup ON test_bloom_filter_named_inline(v1) USING BLOOMFILTER""" + exception "BLOOMFILTER index for column (v1) with non-analyzed type already exists." + } + + sql """DROP INDEX IF EXISTS idx_missing ON test_bloom_filter_named_inline""" + + test { + sql """DROP INDEX idx_missing ON test_bloom_filter_named_inline""" + exception "index idx_missing does not exist" + } + + sql """ALTER TABLE test_bloom_filter_named_inline SET ("bloom_filter_columns" = "")""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_inline' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + qt_desc_named_inline_after_legacy_drop """DESC test_bloom_filter_named_inline""" + + sql """ + CREATE TABLE test_bloom_filter_named_add_on_legacy ( + k1 INT, + v1 STRING, + v2 INT + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "bloom_filter_columns" = "k1" + ) + """ + + sql """CREATE INDEX idx_v2 ON test_bloom_filter_named_add_on_legacy(v2) USING BLOOMFILTER""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_add_on_legacy' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + qt_desc_named_add_on_legacy """DESC test_bloom_filter_named_add_on_legacy""" + qt_show_index_named_add_on_legacy """SHOW INDEX FROM test_bloom_filter_named_add_on_legacy""" + + sql """ + CREATE TABLE test_bloom_filter_named_drop_mixed ( + k1 INT, + v1 STRING, + v2 INT, + INDEX idx_v1 (v1) USING BLOOMFILTER + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ALTER TABLE test_bloom_filter_named_drop_mixed SET ("bloom_filter_columns" = "k1")""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_drop_mixed' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + qt_desc_named_drop_mixed_before """DESC test_bloom_filter_named_drop_mixed""" + qt_show_index_named_drop_mixed_before """SHOW INDEX FROM test_bloom_filter_named_drop_mixed""" + + sql """DROP INDEX idx_v1 ON test_bloom_filter_named_drop_mixed""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_drop_mixed' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + qt_desc_named_drop_mixed_after """DESC test_bloom_filter_named_drop_mixed""" + qt_show_index_named_drop_mixed_after """SHOW INDEX FROM test_bloom_filter_named_drop_mixed""" + + sql """ + CREATE TABLE test_bloom_filter_named_rename_legacy_mixed ( + k1 INT, + v1 STRING, + v2 INT, + INDEX idx_v2 (v2) USING BLOOMFILTER + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "bloom_filter_columns" = "v1" + ) + """ + + sql """ALTER TABLE test_bloom_filter_named_rename_legacy_mixed RENAME COLUMN v1 v1_new""" + sql """SYNC""" + + qt_desc_named_rename_legacy_mixed """DESC test_bloom_filter_named_rename_legacy_mixed""" + qt_show_index_named_rename_legacy_mixed """SHOW INDEX FROM test_bloom_filter_named_rename_legacy_mixed""" + + sql """ + CREATE TABLE test_bloom_filter_named_drop_legacy_column_mixed ( + k1 INT, + v1 STRING, + v2 INT, + INDEX idx_v2 (v2) USING BLOOMFILTER + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "bloom_filter_columns" = "k1,v1" + ) + """ + + sql """ALTER TABLE test_bloom_filter_named_drop_legacy_column_mixed DROP COLUMN v1""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_drop_legacy_column_mixed' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + qt_desc_named_drop_legacy_column_mixed """DESC test_bloom_filter_named_drop_legacy_column_mixed""" + qt_show_index_named_drop_legacy_column_mixed """SHOW INDEX FROM test_bloom_filter_named_drop_legacy_column_mixed""" + + sql """ + CREATE TABLE test_bloom_filter_named_rollup ( + k1 INT, + v2 INT, + v1 STRING + ) ENGINE=OLAP + DUPLICATE KEY(k1, v2) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + ROLLUP ( + r1(k1, v1) + ) + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """INSERT INTO test_bloom_filter_named_rollup VALUES (1, 10, 'alpha'), (2, 20, 'beta')""" + + sql """CREATE INDEX idx_v1 ON test_bloom_filter_named_rollup(v1) USING BLOOMFILTER""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_rollup' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + order_qt_named_rollup_data """SELECT * FROM test_bloom_filter_named_rollup ORDER BY k1""" + qt_show_index_named_rollup """DESC test_bloom_filter_named_rollup ALL""" + + sql """DROP INDEX idx_v1 ON test_bloom_filter_named_rollup""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_rollup' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + qt_show_index_named_rollup_after_drop """DESC test_bloom_filter_named_rollup ALL""" + + sql """ + CREATE TABLE test_bloom_filter_named_create_index ( + k1 INT, + v1 STRING, + v2 INT + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ) + """ + + sql """INSERT INTO test_bloom_filter_named_create_index VALUES (1, 'one', 101), (2, 'two', 202)""" + + sql """CREATE INDEX idx_v2 ON test_bloom_filter_named_create_index(v2) + USING BLOOMFILTER PROPERTIES ("bloom_filter_fpp" = "0.02")""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_create_index' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + order_qt_named_create_index_data """SELECT * FROM test_bloom_filter_named_create_index ORDER BY k1""" + qt_desc_named_create_index """DESC test_bloom_filter_named_create_index""" + qt_show_index_named_create_index """SHOW INDEX FROM test_bloom_filter_named_create_index""" + + try { + GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexWriter::create", [fpp: "0.02"]) + sql """INSERT INTO test_bloom_filter_named_create_index VALUES (3, 'three', 303), (4, 'four', 404)""" + } finally { + GetDebugPoint().disableDebugPointForAllBEs("BloomFilterIndexWriter::create") + } + + test { + sql """ALTER TABLE test_bloom_filter_named_create_index SET ("bloom_filter_fpp" = "0.01")""" + exception "Bloom filter index has no change" + } + + sql """DROP INDEX idx_v2 ON test_bloom_filter_named_create_index""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_bloom_filter_named_create_index' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + qt_desc_named_create_index_after_drop """DESC test_bloom_filter_named_create_index""" + qt_show_index_named_create_index_after_drop """SHOW INDEX FROM test_bloom_filter_named_create_index""" + + test { + sql """ + CREATE TABLE test_bloom_filter_cov_inline_invalid_type ( + k1 INT, + f1 FLOAT, + INDEX idx_f1 (f1) USING BLOOMFILTER + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + exception "FLOAT is not supported in bloom filter index. invalid column: f1" + } + + sql """ + CREATE TABLE test_bloom_filter_cov_add_index_invalid_type ( + k1 INT, + f1 FLOAT + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + test { + sql """ALTER TABLE test_bloom_filter_cov_add_index_invalid_type ADD INDEX idx_f1 (f1) USING BLOOMFILTER""" + exception "FLOAT is not supported in bloom filter index. invalid column: f1" + } + + sql """ + CREATE TABLE test_bloom_filter_cov_add_index_with_properties ( + k1 INT, + v1 STRING + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + test { + sql """ALTER TABLE test_bloom_filter_cov_add_index_with_properties + ADD INDEX idx_v1 (v1) USING BLOOMFILTER PROPERTIES ("foo" = "bar")""" + exception "BLOOMFILTER index only supports property bloom_filter_fpp" + } + + sql """ + CREATE TABLE test_bloom_filter_cov_legacy_same_columns ( + k1 INT, + v1 STRING + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "bloom_filter_columns" = "k1" + ) + """ + + test { + sql """ALTER TABLE test_bloom_filter_cov_legacy_same_columns SET ("bloom_filter_columns" = "k1")""" + exception "Bloom filter index has no change" + } + + sql """ + CREATE TABLE test_bloom_filter_cov_legacy_same_fpp ( + k1 INT, + v1 STRING + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "bloom_filter_columns" = "k1", + "bloom_filter_fpp" = "0.05" + ) + """ + + test { + sql """ALTER TABLE test_bloom_filter_cov_legacy_same_fpp + SET ("bloom_filter_columns" = "k1", "bloom_filter_fpp" = "0.05")""" + exception "Can only set one table property(without dynamic partition && binlog) at a time" + } + + sql """ + CREATE TABLE test_bloom_filter_cov_no_bf_fpp_only ( + k1 INT, + v1 STRING + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + test { + sql """ALTER TABLE test_bloom_filter_cov_no_bf_fpp_only SET ("bloom_filter_fpp" = "0.01")""" + exception "Bloom filter index has no change" + } + + test { + sql """ + CREATE TABLE test_bloom_filter_cov_empty_legacy_columns ( + k1 INT, + v1 STRING + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "bloom_filter_columns" = "" + ) + """ + exception "Unknown properties: {bloom_filter_columns=}" + } + + test { + sql """ + CREATE TABLE test_bloom_filter_named_array_invalid ( + k1 INT, + v ARRAY, + INDEX idx_v (v) USING BLOOMFILTER PROPERTIES ("bloom_filter_fpp" = "0.03") + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + exception "is not supported" + } + + test { + sql """ + CREATE TABLE test_bloom_filter_named_map_invalid ( + k1 INT, + v MAP, + INDEX idx_v (v) USING BLOOMFILTER PROPERTIES ("bloom_filter_fpp" = "0.03") + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + exception "is not supported" + } + + test { + sql """ + CREATE TABLE test_bloom_filter_named_struct_invalid ( + k1 INT, + v STRUCT, + INDEX idx_v (v) USING BLOOMFILTER PROPERTIES ("bloom_filter_fpp" = "0.03") + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + exception "is not supported" + } + + sql """ + CREATE TABLE test_bloom_filter_named_variant ( + k1 INT, + v VARIANT, + INDEX idx_v (v) USING BLOOMFILTER PROPERTIES ("bloom_filter_fpp" = "0.03") + ) ENGINE=OLAP + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ) + """ + + sql """INSERT INTO test_bloom_filter_named_variant VALUES + (1, '{"a": 1, "b": "alpha", "c": {"d": 10}, "e": [1, 2]}'), + (2, '{"a": 2, "b": "beta", "c": {"d": 20}, "e": [3, 4]}')""" + qt_show_index_named_variant """SHOW INDEX FROM test_bloom_filter_named_variant""" + + try { + GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexWriter::create", [fpp: "0.03"]) + sql """INSERT INTO test_bloom_filter_named_variant VALUES + (3, '{"a": 3, "b": "gamma", "c": {"d": 30}, "e": [5, 6]}')""" + } finally { + GetDebugPoint().disableDebugPointForAllBEs("BloomFilterIndexWriter::create") + } + + // Test: table-level bloom_filter_fpp and index-level bloom_filter_fpp are independent. + // Legacy bloom_filter_columns use the table-level FPP, while named BLOOMFILTER indexes + // use their per-index FPP from PROPERTIES. We test each source in isolation — only one + // type of bloom filter is active during each INSERT, so the global debug point can + // assert the correct FPP for every bloom filter writer that fires. + def test_mixed_fpp_tb = "test_mixed_fpp_bloom_filter_tb" + sql """CREATE TABLE IF NOT EXISTS ${test_mixed_fpp_tb} ( + `k1` int(11) NOT NULL, + `v1` varchar(50) NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 5 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true", + "bloom_filter_columns" = "k1", + "bloom_filter_fpp" = "0.03" + )""" + + // + try { + GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexWriter::create", [fpp: "0.03"]) + sql """ INSERT INTO ${test_mixed_fpp_tb} VALUES (1, 'aaa'), (2, 'bbb'), (3, 'ccc') """ + } finally { + GetDebugPoint().disableDebugPointForAllBEs("BloomFilterIndexWriter::create"); + } + + // Step 2: remove legacy BF, create a named BLOOMFILTER index on v1 with its own FPP. + // Only the named index is active → index-level FPP (0.02) is used. + sql """ ALTER TABLE ${test_mixed_fpp_tb} SET ("bloom_filter_columns" = "") """ + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_mixed_fpp_bloom_filter_tb' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + sql """ CREATE INDEX idx_v1_bf ON ${test_mixed_fpp_tb} (v1) USING BLOOMFILTER + PROPERTIES("bloom_filter_fpp" = "0.02") """ + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_mixed_fpp_bloom_filter_tb' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexWriter::create", [fpp: "0.02"]) + sql """ INSERT INTO ${test_mixed_fpp_tb} VALUES (4, 'ddd'), (5, 'eee'), (6, 'fff') """ + } finally { + GetDebugPoint().disableDebugPointForAllBEs("BloomFilterIndexWriter::create"); + } + + // Step 3: drop named index, restore legacy BF on k1 → table-level FPP (0.03) again. + sql """ DROP INDEX idx_v1_bf ON ${test_mixed_fpp_tb} """ + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_mixed_fpp_bloom_filter_tb' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + sql """ ALTER TABLE ${test_mixed_fpp_tb} SET ("bloom_filter_columns" = "k1") """ + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_mixed_fpp_bloom_filter_tb' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + try { + GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexWriter::create", [fpp: "0.05"]) + sql """ INSERT INTO ${test_mixed_fpp_tb} VALUES (7, 'ggg'), (8, 'hhh'), (9, 'iii') """ + } finally { + GetDebugPoint().disableDebugPointForAllBEs("BloomFilterIndexWriter::create"); + } + + sql """ ALTER TABLE ${test_mixed_fpp_tb} SET ("bloom_filter_fpp" = "0.03") """ + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_mixed_fpp_bloom_filter_tb' ORDER BY createtime DESC LIMIT 1""" + time 120 + } + try { + GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexWriter::create", [fpp: "0.03"]) + sql """ INSERT INTO ${test_mixed_fpp_tb} VALUES (10, 'jjj'), (11, 'kkk'), (9, 'lll') """ + } finally { + GetDebugPoint().disableDebugPointForAllBEs("BloomFilterIndexWriter::create"); + } +}