Skip to content

[feat](fe) Show inverted index storage format for partitions #65776

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hoshinojyunn wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[feat](fe) Show inverted index storage format for partitions #65776

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions be/src/cloud/cloud_backend_service.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include "cloud/cloud_backend_service.h"

#include <brpc/controller.h>
#include <gen_cpp/BackendService_types.h>

#include "cloud/cloud_storage_engine.h"
#include "cloud/cloud_tablet.h"
Expand Down Expand Up @@ -304,4 +305,28 @@ void CloudBackendService::get_stream_load_record(TStreamLoadRecordResult& result
_engine.get_stream_load_recorder());
}

void CloudBackendService::check_storage_format(TCheckStorageFormatResult& result) {
for (const auto& tablet : _engine.tablet_mgr().get_all_tablet()) {
result.v2_tablets.push_back(tablet->tablet_id());
switch (tablet->tablet_schema()->get_inverted_index_storage_format()) {
case InvertedIndexStorageFormatPB::V1:
result.inverted_index_v1_tablets.push_back(tablet->tablet_id());
break;
case InvertedIndexStorageFormatPB::V2:
result.inverted_index_v2_tablets.push_back(tablet->tablet_id());
break;
case InvertedIndexStorageFormatPB::V3:
result.inverted_index_v3_tablets.push_back(tablet->tablet_id());
break;
default:
DCHECK(false) << "invalid inverted index storage format";
}
}
result.__isset.v1_tablets = true;
result.__isset.v2_tablets = true;
result.__isset.inverted_index_v1_tablets = true;
result.__isset.inverted_index_v2_tablets = true;
result.__isset.inverted_index_v3_tablets = true;
}

} // namespace doris
2 changes: 2 additions & 0 deletions be/src/cloud/cloud_backend_service.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ class CloudBackendService final : public BaseBackendService {
void get_stream_load_record(TStreamLoadRecordResult& result,
int64_t last_stream_record_time) override;

void check_storage_format(TCheckStorageFormatResult& result) override;

private:
void _warm_up_cache(const TWarmUpCacheAsyncRequest& request);

Expand Down
16 changes: 16 additions & 0 deletions be/src/storage/tablet/tablet_manager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1739,11 +1739,27 @@ void TabletManager::get_all_tablets_storage_format(TCheckStorageFormatResult* re
} else {
result->v1_tablets.push_back(tablet->tablet_id());
}
switch (tablet->tablet_schema()->get_inverted_index_storage_format()) {
case InvertedIndexStorageFormatPB::V1:
result->inverted_index_v1_tablets.push_back(tablet->tablet_id());
break;
case InvertedIndexStorageFormatPB::V2:
result->inverted_index_v2_tablets.push_back(tablet->tablet_id());
break;
case InvertedIndexStorageFormatPB::V3:
result->inverted_index_v3_tablets.push_back(tablet->tablet_id());
break;
default:
DCHECK(false) << "invalid inverted index storage format";
}
};

for_each_tablet(handler, filter_all_tablets);
result->__isset.v1_tablets = true;
result->__isset.v2_tablets = true;
result->__isset.inverted_index_v1_tablets = true;
result->__isset.inverted_index_v2_tablets = true;
result->__isset.inverted_index_v3_tablets = true;
}

std::set<int64_t> TabletManager::check_all_tablet_segment(bool repair) {
Expand Down
6 changes: 5 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ public class PartitionsProcDir implements ProcDirInterface {
.add("Buckets").add("ReplicationNum").add("StorageMedium").add("CooldownTime").add("RemoteStoragePolicy")
.add("LastConsistencyCheckTime").add("DataSize").add("IsInMemory").add("ReplicaAllocation")
.add("IsMutable").add("SyncWithBaseTables").add("UnsyncTables").add("CommittedVersion")
.add("RowCount")
.add("RowCount").add("InvertedIndexStorageFormat")
.build();

private Database db;
Expand Down Expand Up @@ -651,6 +651,10 @@ private List<Pair<List<Comparable>, TRow>> getPartitionInfosInrernal() throws An
partitionInfo.add(partition.getRowCount());
trow.addToColumnValue(new TCell().setLongVal(partition.getRowCount()));

String invertedIndexStorageFormat = olapTable.getInvertedIndexFileStorageFormat().name();
partitionInfo.add(invertedIndexStorageFormat);
trow.addToColumnValue(new TCell().setStringVal(invertedIndexStorageFormat));

partitionInfos.add(Pair.of(partitionInfo, trow));
}
} finally {
Expand Down
13 changes: 9 additions & 4 deletions ...re/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionsCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ public class ShowPartitionsCommand extends ShowCommand {
private static final String FILTER_BUCKETS = "Buckets";
private static final String FILTER_REPLICATION_NUM = "ReplicationNum";
private static final String FILTER_LAST_CONSISTENCY_CHECK_TIME = "LastConsistencyCheckTime";
private static final String FILTER_INVERTED_INDEX_STORAGE_FORMAT = "InvertedIndexStorageFormat";
private final TableNameInfo tableName;
private final Expression wildWhere;
private final long limit;
Expand Down Expand Up @@ -158,7 +159,8 @@ private void analyzeSubExpression(Expression subExpr) throws AnalysisException {

// FILTER_LAST_CONSISTENCY_CHECK_TIME != 'abc'
if (subExpr instanceof ComparisonPredicate) {
if (leftKey.equalsIgnoreCase(FILTER_PARTITION_NAME) || leftKey.equalsIgnoreCase(FILTER_STATE)) {
if (leftKey.equalsIgnoreCase(FILTER_PARTITION_NAME) || leftKey.equalsIgnoreCase(FILTER_STATE)
|| leftKey.equalsIgnoreCase(FILTER_INVERTED_INDEX_STORAGE_FORMAT)) {
if (!(subExpr instanceof EqualTo)) {
throw new AnalysisException(String.format("Only operator =|like are supported for %s", leftKey));
}
Expand All @@ -174,11 +176,14 @@ private void analyzeSubExpression(Expression subExpr) throws AnalysisException {
} else if (!leftKey.equalsIgnoreCase(FILTER_PARTITION_ID) && !leftKey.equalsIgnoreCase(FILTER_BUCKETS)
&& !leftKey.equalsIgnoreCase(FILTER_REPLICATION_NUM)) {
throw new AnalysisException("Only the columns of PartitionId/PartitionName/"
+ "State/Buckets/ReplicationNum/LastConsistencyCheckTime are supported.");
+ "State/Buckets/ReplicationNum/LastConsistencyCheckTime/"
+ "InvertedIndexStorageFormat are supported.");
}
} else if (subExpr instanceof Like) {
if (!leftKey.equalsIgnoreCase(FILTER_PARTITION_NAME) && !leftKey.equalsIgnoreCase(FILTER_STATE)) {
throw new AnalysisException("Where clause : PartitionName|State like \"p20191012|NORMAL\"");
if (!leftKey.equalsIgnoreCase(FILTER_PARTITION_NAME) && !leftKey.equalsIgnoreCase(FILTER_STATE)
&& !leftKey.equalsIgnoreCase(FILTER_INVERTED_INDEX_STORAGE_FORMAT)) {
throw new AnalysisException("Where clause : PartitionName|State|InvertedIndexStorageFormat like "
+ "\"p20191012|NORMAL|V3\"");
}
} else {
throw new AnalysisException("Only operator =|>=|<=|>|<|!=|like are supported.");
Expand Down
53 changes: 35 additions & 18 deletions ...in/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletStorageFormatCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.mysql.privilege.PrivPredicate;
Expand All @@ -36,17 +35,17 @@
import org.apache.doris.thrift.TCheckStorageFormatResult;

import com.google.common.collect.Lists;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;

/**
* show tablet storage format command
*/
public class ShowTabletStorageFormatCommand extends ShowCommand {
public static final Logger LOG = LogManager.getLogger(ShowTabletStorageFormatCommand.class);
private final boolean verbose;

/**
Expand All @@ -65,11 +64,15 @@ public ShowResultSetMetaData getMetaData() {
if (verbose) {
builder.addColumn(new Column("BackendId", ScalarType.createVarchar(30)))
.addColumn(new Column("TabletId", ScalarType.createVarchar(30)))
.addColumn(new Column("StorageFormat", ScalarType.createVarchar(30)));
.addColumn(new Column("StorageFormat", ScalarType.createVarchar(30)))
.addColumn(new Column("InvertedIndexStorageFormat", ScalarType.createVarchar(30)));
} else {
builder.addColumn(new Column("BackendId", ScalarType.createVarchar(30)))
.addColumn(new Column("V1Count", ScalarType.createVarchar(30)))
.addColumn(new Column("V2Count", ScalarType.createVarchar(30)));
.addColumn(new Column("V2Count", ScalarType.createVarchar(30)))
.addColumn(new Column("InvertedIndexV1Count", ScalarType.createVarchar(30)))
.addColumn(new Column("InvertedIndexV2Count", ScalarType.createVarchar(30)))
.addColumn(new Column("InvertedIndexV3Count", ScalarType.createVarchar(30)));
}
return builder.build();
}
Expand All @@ -89,26 +92,46 @@ public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exc
if (result == null) {
throw new AnalysisException("get tablet data from backend: " + be.getId() + "error.");
}
List<Long> invertedIndexV1Tablets = result.isSetInvertedIndexV1Tablets()
? result.getInvertedIndexV1Tablets() : Lists.newArrayList();
List<Long> invertedIndexV2Tablets = result.isSetInvertedIndexV2Tablets()
? result.getInvertedIndexV2Tablets() : Lists.newArrayList();
List<Long> invertedIndexV3Tablets = result.isSetInvertedIndexV3Tablets()
? result.getInvertedIndexV3Tablets() : Lists.newArrayList();
if (verbose) {
Map<Long, String> storageFormats = new HashMap<>();
for (long tabletId : result.getV1Tablets()) {
List<String> row = new ArrayList<>();
row.add(String.valueOf(be.getId()));
row.add(String.valueOf(tabletId));
row.add("V1");
resultRowSet.add(row);
storageFormats.put(tabletId, "V1");
}
for (long tabletId : result.getV2Tablets()) {
storageFormats.put(tabletId, "V2");
}
Map<Long, String> invertedIndexStorageFormats = new HashMap<>();
for (long tabletId : invertedIndexV1Tablets) {
invertedIndexStorageFormats.put(tabletId, "V1");
}
for (long tabletId : invertedIndexV2Tablets) {
invertedIndexStorageFormats.put(tabletId, "V2");
}
for (long tabletId : invertedIndexV3Tablets) {
invertedIndexStorageFormats.put(tabletId, "V3");
}
for (long tabletId : new TreeSet<>(storageFormats.keySet())) {
List<String> row = new ArrayList<>();
row.add(String.valueOf(be.getId()));
row.add(String.valueOf(tabletId));
row.add("V2");
row.add(storageFormats.get(tabletId));
row.add(invertedIndexStorageFormats.getOrDefault(tabletId, "UNKNOWN"));
resultRowSet.add(row);
}
} else {
List<String> row = new ArrayList<>();
row.add(String.valueOf(be.getId()));
row.add(String.valueOf(result.getV1Tablets().size()));
row.add(String.valueOf(result.getV2Tablets().size()));
row.add(String.valueOf(invertedIndexV1Tablets.size()));
row.add(String.valueOf(invertedIndexV2Tablets.size()));
row.add(String.valueOf(invertedIndexV3Tablets.size()));
resultRowSet.add(row);
}
}
Expand All @@ -120,10 +143,4 @@ public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exc
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitShowTabletStorageFormatCommand(this, context);
}

@Override
protected void checkSupportedInCloudMode(ConnectContext ctx) throws DdlException {
LOG.info("show tablet storage format not supported in cloud mode");
throw new DdlException("Unsupported operation");
}
}
17 changes: 11 additions & 6 deletions ...rc/test/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionsCommandTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,24 +211,29 @@ void testValidate() throws UserException {
sp = new ShowPartitionsCommand(tbl, where13, null, -1, -1, false);
sp.validate(connectContext);

Expression where14 = new EqualTo(new UnboundSlot(Lists.newArrayList("VisibleVersion")),
new IntegerLiteral(1));
Expression where14 = new EqualTo(new UnboundSlot(Lists.newArrayList("InvertedIndexStorageFormat")),
new StringLiteral("V3"));
sp = new ShowPartitionsCommand(tbl, where14, null, -1, -1, false);
sp.validate(connectContext);

Expression where15 = new EqualTo(new UnboundSlot(Lists.newArrayList("VisibleVersion")),
new IntegerLiteral(1));
sp = new ShowPartitionsCommand(tbl, where15, null, -1, -1, false);
ShowPartitionsCommand finalSp = sp;
Assertions.assertThrows(AnalysisException.class, () -> finalSp.validate(connectContext));

Expression equalTo6 = new EqualTo(new UnboundSlot(Lists.newArrayList("STATE")),
new StringLiteral("xxx"));
Expression equalTo7 = new EqualTo(new UnboundSlot(Lists.newArrayList("PartitionName")),
new StringLiteral("yyy"));
Expression where15 = new Or(equalTo6, equalTo7);
sp = new ShowPartitionsCommand(tbl, where15, null, -1, -1, false);
Expression where16 = new Or(equalTo6, equalTo7);
sp = new ShowPartitionsCommand(tbl, where16, null, -1, -1, false);
ShowPartitionsCommand finalSp1 = sp;
Assertions.assertThrows(AnalysisException.class, () -> finalSp1.validate(connectContext));

Expression where16 = new GreaterThanEqual(new UnboundSlot(Lists.newArrayList("STATE")),
Expression where17 = new GreaterThanEqual(new UnboundSlot(Lists.newArrayList("STATE")),
new IntegerLiteral(16));
sp = new ShowPartitionsCommand(tbl, where16, null, -1, -1, false);
sp = new ShowPartitionsCommand(tbl, where17, null, -1, -1, false);
ShowPartitionsCommand finalSp2 = sp;
Assertions.assertThrows(AnalysisException.class, () -> finalSp2.validate(connectContext));
}
Expand Down
3 changes: 3 additions & 0 deletions gensrc/thrift/BackendService.thrift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,9 @@ struct TDiskTrashInfo {
struct TCheckStorageFormatResult {
1: optional list<i64> v1_tablets;
2: optional list<i64> v2_tablets;
3: optional list<i64> inverted_index_v1_tablets;
4: optional list<i64> inverted_index_v2_tablets;
5: optional list<i64> inverted_index_v3_tablets;
}

struct TWarmUpCacheAsyncRequest {
Expand Down
Loading
Loading