From f8030074255d0fcc29c14e162e794049fd69c632 Mon Sep 17 00:00:00 2001 From: hoshinojyunn Date: Fri, 17 Jul 2026 17:31:51 +0800 Subject: [PATCH] [feature](fe) Show inverted index storage format for partitions ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: SHOW PARTITIONS did not expose the configured inverted-index storage format or allow users to filter partitions by it. SHOW TABLET STORAGE FORMAT also omitted the inverted-index format and could not inspect Cloud tablets. Add the format column to partition output and filtering, return the tablet schema format from local and Cloud BEs, and include per-tablet and aggregate inverted-index format results. The Cloud regression now writes to every dynamic partition before validating the tablet output, because Cloud tablets are materialized on first write. ### Release note SHOW PARTITIONS now reports and filters by InvertedIndexStorageFormat. SHOW TABLET STORAGE FORMAT reports inverted-index storage formats for local and Cloud tablets. ### Check List (For Author) - Test: Regression test - source ~/.venv/bin/activate && export LOCAL_DORIS_PATH=/mnt/disk1/ganderun/partition_index_format_observability/tmp && ./run-regression-test.sh --run -d cloud_p0 -s test_cloud_show_inverted_index_storage_format -runMode=cloud - Behavior changed: Yes (SHOW PARTITIONS and SHOW TABLET STORAGE FORMAT expose inverted-index storage formats, including Cloud mode) - Does this need documentation: No --- be/src/cloud/cloud_backend_service.cpp | 25 +++ be/src/cloud/cloud_backend_service.h | 2 + be/src/storage/tablet/tablet_manager.cpp | 16 ++ .../doris/common/proc/PartitionsProcDir.java | 6 +- .../plans/commands/ShowPartitionsCommand.java | 13 +- .../ShowTabletStorageFormatCommand.java | 53 +++-- .../commands/ShowPartitionsCommandTest.java | 17 +- gensrc/thrift/BackendService.thrift | 3 + ...rtition_cloud_inverted_index_format.groovy | 186 ++++++++++++++++++ ...rtition_local_inverted_index_format.groovy | 167 ++++++++++++++++ 10 files changed, 459 insertions(+), 29 deletions(-) create mode 100644 regression-test/suites/cloud_p0/test_partition_cloud_inverted_index_format.groovy create mode 100644 regression-test/suites/query_p0/show/test_partition_local_inverted_index_format.groovy diff --git a/be/src/cloud/cloud_backend_service.cpp b/be/src/cloud/cloud_backend_service.cpp index 5f09f194e37485..f482d1579511cd 100644 --- a/be/src/cloud/cloud_backend_service.cpp +++ b/be/src/cloud/cloud_backend_service.cpp @@ -18,6 +18,7 @@ #include "cloud/cloud_backend_service.h" #include +#include #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" @@ -304,4 +305,28 @@ void CloudBackendService::get_stream_load_record(TStreamLoadRecordResult& result _engine.get_stream_load_recorder()); } +void CloudBackendService::check_storage_format(TCheckStorageFormatResult& result) { + for (const auto& tablet : _engine.tablet_mgr().get_all_tablet()) { + result.v2_tablets.push_back(tablet->tablet_id()); + switch (tablet->tablet_schema()->get_inverted_index_storage_format()) { + case InvertedIndexStorageFormatPB::V1: + result.inverted_index_v1_tablets.push_back(tablet->tablet_id()); + break; + case InvertedIndexStorageFormatPB::V2: + result.inverted_index_v2_tablets.push_back(tablet->tablet_id()); + break; + case InvertedIndexStorageFormatPB::V3: + result.inverted_index_v3_tablets.push_back(tablet->tablet_id()); + break; + default: + DCHECK(false) << "invalid inverted index storage format"; + } + } + result.__isset.v1_tablets = true; + result.__isset.v2_tablets = true; + result.__isset.inverted_index_v1_tablets = true; + result.__isset.inverted_index_v2_tablets = true; + result.__isset.inverted_index_v3_tablets = true; +} + } // namespace doris diff --git a/be/src/cloud/cloud_backend_service.h b/be/src/cloud/cloud_backend_service.h index 588cbd1f7ec9a0..601bcf6a50d79f 100644 --- a/be/src/cloud/cloud_backend_service.h +++ b/be/src/cloud/cloud_backend_service.h @@ -54,6 +54,8 @@ class CloudBackendService final : public BaseBackendService { void get_stream_load_record(TStreamLoadRecordResult& result, int64_t last_stream_record_time) override; + void check_storage_format(TCheckStorageFormatResult& result) override; + private: void _warm_up_cache(const TWarmUpCacheAsyncRequest& request); diff --git a/be/src/storage/tablet/tablet_manager.cpp b/be/src/storage/tablet/tablet_manager.cpp index c27215d3b914b8..22771998c74499 100644 --- a/be/src/storage/tablet/tablet_manager.cpp +++ b/be/src/storage/tablet/tablet_manager.cpp @@ -1739,11 +1739,27 @@ void TabletManager::get_all_tablets_storage_format(TCheckStorageFormatResult* re } else { result->v1_tablets.push_back(tablet->tablet_id()); } + switch (tablet->tablet_schema()->get_inverted_index_storage_format()) { + case InvertedIndexStorageFormatPB::V1: + result->inverted_index_v1_tablets.push_back(tablet->tablet_id()); + break; + case InvertedIndexStorageFormatPB::V2: + result->inverted_index_v2_tablets.push_back(tablet->tablet_id()); + break; + case InvertedIndexStorageFormatPB::V3: + result->inverted_index_v3_tablets.push_back(tablet->tablet_id()); + break; + default: + DCHECK(false) << "invalid inverted index storage format"; + } }; for_each_tablet(handler, filter_all_tablets); result->__isset.v1_tablets = true; result->__isset.v2_tablets = true; + result->__isset.inverted_index_v1_tablets = true; + result->__isset.inverted_index_v2_tablets = true; + result->__isset.inverted_index_v3_tablets = true; } std::set TabletManager::check_all_tablet_segment(bool repair) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java index 4bf00c0be67332..1c9fbb59840079 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java @@ -114,7 +114,7 @@ public class PartitionsProcDir implements ProcDirInterface { .add("Buckets").add("ReplicationNum").add("StorageMedium").add("CooldownTime").add("RemoteStoragePolicy") .add("LastConsistencyCheckTime").add("DataSize").add("IsInMemory").add("ReplicaAllocation") .add("IsMutable").add("SyncWithBaseTables").add("UnsyncTables").add("CommittedVersion") - .add("RowCount") + .add("RowCount").add("InvertedIndexStorageFormat") .build(); private Database db; @@ -651,6 +651,10 @@ private List, TRow>> getPartitionInfosInrernal() throws An partitionInfo.add(partition.getRowCount()); trow.addToColumnValue(new TCell().setLongVal(partition.getRowCount())); + String invertedIndexStorageFormat = olapTable.getInvertedIndexFileStorageFormat().name(); + partitionInfo.add(invertedIndexStorageFormat); + trow.addToColumnValue(new TCell().setStringVal(invertedIndexStorageFormat)); + partitionInfos.add(Pair.of(partitionInfo, trow)); } } finally { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionsCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionsCommand.java index a3b4fd438db14f..b5b6147aa40674 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionsCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionsCommand.java @@ -91,6 +91,7 @@ public class ShowPartitionsCommand extends ShowCommand { private static final String FILTER_BUCKETS = "Buckets"; private static final String FILTER_REPLICATION_NUM = "ReplicationNum"; private static final String FILTER_LAST_CONSISTENCY_CHECK_TIME = "LastConsistencyCheckTime"; + private static final String FILTER_INVERTED_INDEX_STORAGE_FORMAT = "InvertedIndexStorageFormat"; private final TableNameInfo tableName; private final Expression wildWhere; private final long limit; @@ -158,7 +159,8 @@ private void analyzeSubExpression(Expression subExpr) throws AnalysisException { // FILTER_LAST_CONSISTENCY_CHECK_TIME != 'abc' if (subExpr instanceof ComparisonPredicate) { - if (leftKey.equalsIgnoreCase(FILTER_PARTITION_NAME) || leftKey.equalsIgnoreCase(FILTER_STATE)) { + if (leftKey.equalsIgnoreCase(FILTER_PARTITION_NAME) || leftKey.equalsIgnoreCase(FILTER_STATE) + || leftKey.equalsIgnoreCase(FILTER_INVERTED_INDEX_STORAGE_FORMAT)) { if (!(subExpr instanceof EqualTo)) { throw new AnalysisException(String.format("Only operator =|like are supported for %s", leftKey)); } @@ -174,11 +176,14 @@ private void analyzeSubExpression(Expression subExpr) throws AnalysisException { } else if (!leftKey.equalsIgnoreCase(FILTER_PARTITION_ID) && !leftKey.equalsIgnoreCase(FILTER_BUCKETS) && !leftKey.equalsIgnoreCase(FILTER_REPLICATION_NUM)) { throw new AnalysisException("Only the columns of PartitionId/PartitionName/" - + "State/Buckets/ReplicationNum/LastConsistencyCheckTime are supported."); + + "State/Buckets/ReplicationNum/LastConsistencyCheckTime/" + + "InvertedIndexStorageFormat are supported."); } } else if (subExpr instanceof Like) { - if (!leftKey.equalsIgnoreCase(FILTER_PARTITION_NAME) && !leftKey.equalsIgnoreCase(FILTER_STATE)) { - throw new AnalysisException("Where clause : PartitionName|State like \"p20191012|NORMAL\""); + if (!leftKey.equalsIgnoreCase(FILTER_PARTITION_NAME) && !leftKey.equalsIgnoreCase(FILTER_STATE) + && !leftKey.equalsIgnoreCase(FILTER_INVERTED_INDEX_STORAGE_FORMAT)) { + throw new AnalysisException("Where clause : PartitionName|State|InvertedIndexStorageFormat like " + + "\"p20191012|NORMAL|V3\""); } } else { throw new AnalysisException("Only operator =|>=|<=|>|<|!=|like are supported."); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletStorageFormatCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletStorageFormatCommand.java index aa535e0f39dd15..df18e12f09873f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletStorageFormatCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletStorageFormatCommand.java @@ -21,7 +21,6 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.mysql.privilege.PrivPredicate; @@ -36,17 +35,17 @@ import org.apache.doris.thrift.TCheckStorageFormatResult; import com.google.common.collect.Lists; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.TreeSet; /** * show tablet storage format command */ public class ShowTabletStorageFormatCommand extends ShowCommand { - public static final Logger LOG = LogManager.getLogger(ShowTabletStorageFormatCommand.class); private final boolean verbose; /** @@ -65,11 +64,15 @@ public ShowResultSetMetaData getMetaData() { if (verbose) { builder.addColumn(new Column("BackendId", ScalarType.createVarchar(30))) .addColumn(new Column("TabletId", ScalarType.createVarchar(30))) - .addColumn(new Column("StorageFormat", ScalarType.createVarchar(30))); + .addColumn(new Column("StorageFormat", ScalarType.createVarchar(30))) + .addColumn(new Column("InvertedIndexStorageFormat", ScalarType.createVarchar(30))); } else { builder.addColumn(new Column("BackendId", ScalarType.createVarchar(30))) .addColumn(new Column("V1Count", ScalarType.createVarchar(30))) - .addColumn(new Column("V2Count", ScalarType.createVarchar(30))); + .addColumn(new Column("V2Count", ScalarType.createVarchar(30))) + .addColumn(new Column("InvertedIndexV1Count", ScalarType.createVarchar(30))) + .addColumn(new Column("InvertedIndexV2Count", ScalarType.createVarchar(30))) + .addColumn(new Column("InvertedIndexV3Count", ScalarType.createVarchar(30))); } return builder.build(); } @@ -89,19 +92,36 @@ public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exc if (result == null) { throw new AnalysisException("get tablet data from backend: " + be.getId() + "error."); } + List invertedIndexV1Tablets = result.isSetInvertedIndexV1Tablets() + ? result.getInvertedIndexV1Tablets() : Lists.newArrayList(); + List invertedIndexV2Tablets = result.isSetInvertedIndexV2Tablets() + ? result.getInvertedIndexV2Tablets() : Lists.newArrayList(); + List invertedIndexV3Tablets = result.isSetInvertedIndexV3Tablets() + ? result.getInvertedIndexV3Tablets() : Lists.newArrayList(); if (verbose) { + Map storageFormats = new HashMap<>(); for (long tabletId : result.getV1Tablets()) { - List row = new ArrayList<>(); - row.add(String.valueOf(be.getId())); - row.add(String.valueOf(tabletId)); - row.add("V1"); - resultRowSet.add(row); + storageFormats.put(tabletId, "V1"); } for (long tabletId : result.getV2Tablets()) { + storageFormats.put(tabletId, "V2"); + } + Map invertedIndexStorageFormats = new HashMap<>(); + for (long tabletId : invertedIndexV1Tablets) { + invertedIndexStorageFormats.put(tabletId, "V1"); + } + for (long tabletId : invertedIndexV2Tablets) { + invertedIndexStorageFormats.put(tabletId, "V2"); + } + for (long tabletId : invertedIndexV3Tablets) { + invertedIndexStorageFormats.put(tabletId, "V3"); + } + for (long tabletId : new TreeSet<>(storageFormats.keySet())) { List row = new ArrayList<>(); row.add(String.valueOf(be.getId())); row.add(String.valueOf(tabletId)); - row.add("V2"); + row.add(storageFormats.get(tabletId)); + row.add(invertedIndexStorageFormats.getOrDefault(tabletId, "UNKNOWN")); resultRowSet.add(row); } } else { @@ -109,6 +129,9 @@ public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exc row.add(String.valueOf(be.getId())); row.add(String.valueOf(result.getV1Tablets().size())); row.add(String.valueOf(result.getV2Tablets().size())); + row.add(String.valueOf(invertedIndexV1Tablets.size())); + row.add(String.valueOf(invertedIndexV2Tablets.size())); + row.add(String.valueOf(invertedIndexV3Tablets.size())); resultRowSet.add(row); } } @@ -120,10 +143,4 @@ public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exc public R accept(PlanVisitor visitor, C context) { return visitor.visitShowTabletStorageFormatCommand(this, context); } - - @Override - protected void checkSupportedInCloudMode(ConnectContext ctx) throws DdlException { - LOG.info("show tablet storage format not supported in cloud mode"); - throw new DdlException("Unsupported operation"); - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionsCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionsCommandTest.java index 8c10d1093f5425..c038e22ccc999d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionsCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionsCommandTest.java @@ -211,9 +211,14 @@ void testValidate() throws UserException { sp = new ShowPartitionsCommand(tbl, where13, null, -1, -1, false); sp.validate(connectContext); - Expression where14 = new EqualTo(new UnboundSlot(Lists.newArrayList("VisibleVersion")), - new IntegerLiteral(1)); + Expression where14 = new EqualTo(new UnboundSlot(Lists.newArrayList("InvertedIndexStorageFormat")), + new StringLiteral("V3")); sp = new ShowPartitionsCommand(tbl, where14, null, -1, -1, false); + sp.validate(connectContext); + + Expression where15 = new EqualTo(new UnboundSlot(Lists.newArrayList("VisibleVersion")), + new IntegerLiteral(1)); + sp = new ShowPartitionsCommand(tbl, where15, null, -1, -1, false); ShowPartitionsCommand finalSp = sp; Assertions.assertThrows(AnalysisException.class, () -> finalSp.validate(connectContext)); @@ -221,14 +226,14 @@ void testValidate() throws UserException { new StringLiteral("xxx")); Expression equalTo7 = new EqualTo(new UnboundSlot(Lists.newArrayList("PartitionName")), new StringLiteral("yyy")); - Expression where15 = new Or(equalTo6, equalTo7); - sp = new ShowPartitionsCommand(tbl, where15, null, -1, -1, false); + Expression where16 = new Or(equalTo6, equalTo7); + sp = new ShowPartitionsCommand(tbl, where16, null, -1, -1, false); ShowPartitionsCommand finalSp1 = sp; Assertions.assertThrows(AnalysisException.class, () -> finalSp1.validate(connectContext)); - Expression where16 = new GreaterThanEqual(new UnboundSlot(Lists.newArrayList("STATE")), + Expression where17 = new GreaterThanEqual(new UnboundSlot(Lists.newArrayList("STATE")), new IntegerLiteral(16)); - sp = new ShowPartitionsCommand(tbl, where16, null, -1, -1, false); + sp = new ShowPartitionsCommand(tbl, where17, null, -1, -1, false); ShowPartitionsCommand finalSp2 = sp; Assertions.assertThrows(AnalysisException.class, () -> finalSp2.validate(connectContext)); } diff --git a/gensrc/thrift/BackendService.thrift b/gensrc/thrift/BackendService.thrift index 3867cec245f12d..d0dd2c2f287ea8 100644 --- a/gensrc/thrift/BackendService.thrift +++ b/gensrc/thrift/BackendService.thrift @@ -143,6 +143,9 @@ struct TDiskTrashInfo { struct TCheckStorageFormatResult { 1: optional list v1_tablets; 2: optional list v2_tablets; + 3: optional list inverted_index_v1_tablets; + 4: optional list inverted_index_v2_tablets; + 5: optional list inverted_index_v3_tablets; } struct TWarmUpCacheAsyncRequest { diff --git a/regression-test/suites/cloud_p0/test_partition_cloud_inverted_index_format.groovy b/regression-test/suites/cloud_p0/test_partition_cloud_inverted_index_format.groovy new file mode 100644 index 00000000000000..baeca596cc1fba --- /dev/null +++ b/regression-test/suites/cloud_p0/test_partition_cloud_inverted_index_format.groovy @@ -0,0 +1,186 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions + +suite("test_cloud_show_inverted_index_storage_format", "p0, docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(1) + + docker(options) { + def assertPartitionFormats = { tableName, expectedPartitionCount, expectedFormat -> + def partitions = sql_return_maparray("SHOW PARTITIONS FROM ${tableName}") + assertEquals(expectedPartitionCount, partitions.size()) + assertTrue(partitions.every { it.InvertedIndexStorageFormat == expectedFormat }) + } + + def assertTabletFormats = { tableName, expectedFormat -> + def targetTabletIds = sql_return_maparray("SHOW TABLETS FROM ${tableName}") + .collect { it.TabletId } + assertFalse(targetTabletIds.isEmpty()) + + def verboseRows = sql_return_maparray "SHOW TABLET STORAGE FORMAT VERBOSE" + def targetRows = verboseRows.findAll { targetTabletIds.contains(it.TabletId) } + assertEquals(targetTabletIds.size(), targetRows.size()) + assertTrue(targetRows.every { it.StorageFormat == "V2" }) + assertTrue(targetRows.every { it.InvertedIndexStorageFormat == expectedFormat }) + return targetTabletIds + } + + sql "DROP TABLE IF EXISTS test_cloud_show_inverted_format_range" + sql """ + CREATE TABLE test_cloud_show_inverted_format_range ( + k INT, + v VARCHAR(20), + INDEX idx_v (v) USING INVERTED + ) + DUPLICATE KEY(k) + PARTITION BY RANGE(k) ( + PARTITION p1 VALUES [("0"), ("10")), + PARTITION p2 VALUES [("10"), ("20")), + PARTITION p3 VALUES [("20"), ("30")) + ) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V3" + ) + """ + sql """ + INSERT INTO test_cloud_show_inverted_format_range VALUES + (1, 'range-1a'), (2, 'range-1b'), (3, 'range-1c'), + (11, 'range-2a'), (12, 'range-2b'), (13, 'range-2c'), + (21, 'range-3a'), (22, 'range-3b'), (23, 'range-3c') + """ + + sql "DROP TABLE IF EXISTS test_cloud_show_inverted_format_list" + sql """ + CREATE TABLE test_cloud_show_inverted_format_list ( + k INT, + category VARCHAR(20), + INDEX idx_category (category) USING INVERTED + ) + DUPLICATE KEY(k, category) + PARTITION BY LIST(category) ( + PARTITION p_east VALUES IN ('east', 'north'), + PARTITION p_west VALUES IN ('west', 'south'), + PARTITION p_central VALUES IN ('central', 'overseas') + ) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V2" + ) + """ + sql """ + INSERT INTO test_cloud_show_inverted_format_list VALUES + (1, 'east'), (2, 'north'), (3, 'west'), + (4, 'south'), (5, 'central'), (6, 'overseas') + """ + + sql "DROP TABLE IF EXISTS test_cloud_show_inverted_format_auto" + sql """ + CREATE TABLE test_cloud_show_inverted_format_auto ( + k INT, + category VARCHAR(20), + INDEX idx_category (category) USING INVERTED + ) + DUPLICATE KEY(k, category) + AUTO PARTITION BY LIST(category) () + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V3" + ) + """ + sql """ + INSERT INTO test_cloud_show_inverted_format_auto VALUES + (1, 'book'), (2, 'book'), (3, 'music'), + (4, 'music'), (5, 'video'), (6, 'video') + """ + + sql "DROP TABLE IF EXISTS test_cloud_show_inverted_format_dynamic" + sql """ + CREATE TABLE test_cloud_show_inverted_format_dynamic ( + k INT, + dt DATE, + v VARCHAR(20), + INDEX idx_v (v) USING INVERTED + ) + DUPLICATE KEY(k, dt) + PARTITION BY RANGE(dt) () + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "DAY", + "dynamic_partition.start" = "-3", + "dynamic_partition.end" = "3", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "1", + "dynamic_partition.create_history_partition" = "true", + "inverted_index_storage_format" = "V3" + ) + """ + sql """ + INSERT INTO test_cloud_show_inverted_format_dynamic + SELECT 1, date_sub(curdate(), INTERVAL 3 DAY), 'dynamic-0' + UNION ALL SELECT 2, date_sub(curdate(), INTERVAL 2 DAY), 'dynamic-1a' + UNION ALL SELECT 3, date_sub(curdate(), INTERVAL 2 DAY), 'dynamic-1b' + UNION ALL SELECT 4, date_sub(curdate(), INTERVAL 1 DAY), 'dynamic-2a' + UNION ALL SELECT 5, date_sub(curdate(), INTERVAL 1 DAY), 'dynamic-2b' + UNION ALL SELECT 6, curdate(), 'dynamic-3a' + UNION ALL SELECT 7, curdate(), 'dynamic-3b' + UNION ALL SELECT 8, date_add(curdate(), INTERVAL 1 DAY), 'dynamic-4' + UNION ALL SELECT 9, date_add(curdate(), INTERVAL 2 DAY), 'dynamic-5' + UNION ALL SELECT 10, date_add(curdate(), INTERVAL 3 DAY), 'dynamic-6' + """ + sql "SYNC" + + def rangeCount = sql "SELECT COUNT(*) FROM test_cloud_show_inverted_format_range" + def listCount = sql "SELECT COUNT(*) FROM test_cloud_show_inverted_format_list" + def autoCount = sql "SELECT COUNT(*) FROM test_cloud_show_inverted_format_auto" + def dynamicCount = sql "SELECT COUNT(*) FROM test_cloud_show_inverted_format_dynamic" + assertEquals(9L, rangeCount[0][0]) + assertEquals(6L, listCount[0][0]) + assertEquals(6L, autoCount[0][0]) + assertEquals(10L, dynamicCount[0][0]) + + assertPartitionFormats("test_cloud_show_inverted_format_range", 3, "V3") + def rangeV3Partitions = sql_return_maparray """ + SHOW PARTITIONS FROM test_cloud_show_inverted_format_range + WHERE InvertedIndexStorageFormat = 'V3' + """ + assertEquals(3, rangeV3Partitions.size()) + + assertPartitionFormats("test_cloud_show_inverted_format_list", 3, "V2") + assertPartitionFormats("test_cloud_show_inverted_format_auto", 3, "V3") + assertPartitionFormats("test_cloud_show_inverted_format_dynamic", 7, "V3") + + def rangeV3Tablets = assertTabletFormats("test_cloud_show_inverted_format_range", "V3") + def listV2Tablets = assertTabletFormats("test_cloud_show_inverted_format_list", "V2") + def autoV3Tablets = assertTabletFormats("test_cloud_show_inverted_format_auto", "V3") + def dynamicV3Tablets = assertTabletFormats("test_cloud_show_inverted_format_dynamic", "V3") + + def summaryRows = sql_return_maparray "SHOW TABLET STORAGE FORMAT" + assertTrue(summaryRows.collect { it.InvertedIndexV2Count.toLong() }.sum() >= listV2Tablets.size()) + assertTrue(summaryRows.collect { it.InvertedIndexV3Count.toLong() }.sum() + >= rangeV3Tablets.size() + autoV3Tablets.size() + dynamicV3Tablets.size()) + } +} diff --git a/regression-test/suites/query_p0/show/test_partition_local_inverted_index_format.groovy b/regression-test/suites/query_p0/show/test_partition_local_inverted_index_format.groovy new file mode 100644 index 00000000000000..2a4a45fd5e2edd --- /dev/null +++ b/regression-test/suites/query_p0/show/test_partition_local_inverted_index_format.groovy @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_show_inverted_index_storage_format") { + if (isCloudMode()) { + return + } + + def assertPartitionFormats = { tableName, expectedPartitionCount, expectedFormat -> + def partitions = sql_return_maparray("SHOW PARTITIONS FROM ${tableName}") + assertEquals(expectedPartitionCount, partitions.size()) + assertTrue(partitions.every { it.InvertedIndexStorageFormat == expectedFormat }) + } + + def assertTabletFormats = { tableName, expectedFormat -> + def targetTabletIds = sql_return_maparray("SHOW TABLETS FROM ${tableName}") + .collect { it.TabletId } + assertFalse(targetTabletIds.isEmpty()) + + def verboseRows = sql_return_maparray "SHOW TABLET STORAGE FORMAT VERBOSE" + def targetRows = verboseRows.findAll { targetTabletIds.contains(it.TabletId) } + assertEquals(targetTabletIds.size(), targetRows.size()) + assertTrue(targetRows.every { it.InvertedIndexStorageFormat == expectedFormat }) + return targetTabletIds + } + + sql "DROP TABLE IF EXISTS test_show_inverted_format_range" + sql """ + CREATE TABLE test_show_inverted_format_range ( + k INT, + v VARCHAR(20), + INDEX idx_v (v) USING INVERTED + ) + DUPLICATE KEY(k) + PARTITION BY RANGE(k) ( + PARTITION p1 VALUES [("0"), ("10")), + PARTITION p2 VALUES [("10"), ("20")), + PARTITION p3 VALUES [("20"), ("30")) + ) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V3" + ) + """ + sql """ + INSERT INTO test_show_inverted_format_range VALUES + (1, 'range-1a'), (2, 'range-1b'), (3, 'range-1c'), + (11, 'range-2a'), (12, 'range-2b'), (13, 'range-2c'), + (21, 'range-3a'), (22, 'range-3b'), (23, 'range-3c') + """ + + sql "DROP TABLE IF EXISTS test_show_inverted_format_list" + sql """ + CREATE TABLE test_show_inverted_format_list ( + k INT, + category VARCHAR(20), + INDEX idx_category (category) USING INVERTED + ) + DUPLICATE KEY(k, category) + PARTITION BY LIST(category) ( + PARTITION p_east VALUES IN ('east', 'north'), + PARTITION p_west VALUES IN ('west', 'south'), + PARTITION p_central VALUES IN ('central', 'overseas') + ) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V2" + ) + """ + sql """ + INSERT INTO test_show_inverted_format_list VALUES + (1, 'east'), (2, 'north'), (3, 'west'), + (4, 'south'), (5, 'central'), (6, 'overseas') + """ + + sql "DROP TABLE IF EXISTS test_show_inverted_format_auto" + sql """ + CREATE TABLE test_show_inverted_format_auto ( + k INT, + category VARCHAR(20), + INDEX idx_category (category) USING INVERTED + ) + DUPLICATE KEY(k, category) + AUTO PARTITION BY LIST(category) () + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V3" + ) + """ + sql """ + INSERT INTO test_show_inverted_format_auto VALUES + (1, 'book'), (2, 'book'), (3, 'music'), + (4, 'music'), (5, 'video'), (6, 'video') + """ + + sql "DROP TABLE IF EXISTS test_show_inverted_format_dynamic" + sql """ + CREATE TABLE test_show_inverted_format_dynamic ( + k INT, + dt DATE, + v VARCHAR(20), + INDEX idx_v (v) USING INVERTED + ) + DUPLICATE KEY(k, dt) + PARTITION BY RANGE(dt) () + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "DAY", + "dynamic_partition.start" = "-3", + "dynamic_partition.end" = "3", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "1", + "dynamic_partition.create_history_partition" = "true", + "inverted_index_storage_format" = "V3" + ) + """ + sql """ + INSERT INTO test_show_inverted_format_dynamic + SELECT 1, date_sub(curdate(), INTERVAL 2 DAY), 'dynamic-1a' + UNION ALL SELECT 2, date_sub(curdate(), INTERVAL 2 DAY), 'dynamic-1b' + UNION ALL SELECT 3, date_sub(curdate(), INTERVAL 1 DAY), 'dynamic-2a' + UNION ALL SELECT 4, date_sub(curdate(), INTERVAL 1 DAY), 'dynamic-2b' + UNION ALL SELECT 5, curdate(), 'dynamic-3a' + UNION ALL SELECT 6, curdate(), 'dynamic-3b' + """ + sql "SYNC" + + assertPartitionFormats("test_show_inverted_format_range", 3, "V3") + def rangeV3Partitions = sql_return_maparray """ + SHOW PARTITIONS FROM test_show_inverted_format_range + WHERE InvertedIndexStorageFormat = 'V3' + """ + assertEquals(3, rangeV3Partitions.size()) + + assertPartitionFormats("test_show_inverted_format_list", 3, "V2") + assertPartitionFormats("test_show_inverted_format_auto", 3, "V3") + assertPartitionFormats("test_show_inverted_format_dynamic", 7, "V3") + + def rangeV3Tablets = assertTabletFormats("test_show_inverted_format_range", "V3") + def listV2Tablets = assertTabletFormats("test_show_inverted_format_list", "V2") + def autoV3Tablets = assertTabletFormats("test_show_inverted_format_auto", "V3") + def dynamicV3Tablets = assertTabletFormats("test_show_inverted_format_dynamic", "V3") + + def summaryRows = sql_return_maparray "SHOW TABLET STORAGE FORMAT" + assertTrue(summaryRows.collect { it.InvertedIndexV2Count.toLong() }.sum() >= listV2Tablets.size()) + assertTrue(summaryRows.collect { it.InvertedIndexV3Count.toLong() }.sum() + >= rangeV3Tablets.size() + autoV3Tablets.size() + dynamicV3Tablets.size()) +}