From fb5a55bd0dba94541d407b508fd74da3130dceef Mon Sep 17 00:00:00 2001 From: Refrain Date: Fri, 26 Jun 2026 11:51:32 +0800 Subject: [PATCH 01/19] [feature](fe) Add routine load target-table alter support ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Allow paused single-table Routine Load jobs to switch their target table with ALTER ROUTINE LOAD FOR [db.]job ON , while preserving existing progress and replaying the new table binding from edit log. ### Release note Support ALTER ROUTINE LOAD ... ON
to switch the target table for paused single-table routine load jobs. ### Check List (For Author) - Test: FE unit test - "/data/data3/huangruixin/include/src-master/apache-maven-3.9.9/bin/mvn -pl fe-core -am -DskipITs -Dcheckstyle.skip=true -DfailIfNoTests=false -Dtest=org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommandTest,org.apache.doris.persist.AlterRoutineLoadOperationLogTest,org.apache.doris.load.routineload.KafkaRoutineLoadJobTest test" - Kinesis unit tests skipped per user request - Behavior changed: Yes (new ALTER ROUTINE LOAD target-table switch behavior) - Does this need documentation: Yes (documented in /data/data3/huangruixin/docs/routine-load-alter-table-design.html) --- .../load/routineload/RoutineLoadJob.java | 25 +++ .../kafka/KafkaRoutineLoadJob.java | 8 +- .../kinesis/KinesisRoutineLoadJob.java | 8 +- .../nereids/parser/LogicalPlanBuilder.java | 7 +- .../commands/AlterRoutineLoadCommand.java | 73 ++++++- .../AlterRoutineLoadJobOperationLog.java | 12 ++ .../routineload/KafkaRoutineLoadJobTest.java | 19 ++ .../commands/AlterRoutineLoadCommandTest.java | 180 +++++++++++++++-- .../AlterRoutineLoadOperationLogTest.java | 63 ++++++ .../org/apache/doris/nereids/DorisParser.g4 | 11 +- .../test_routine_load_alter.groovy | 191 ++++++++++++++++++ 11 files changed, 572 insertions(+), 25 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 3223ff913594e1..e7c69da5a77488 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -468,6 +468,31 @@ protected void setRoutineLoadDesc(RoutineLoadDesc routineLoadDesc) { } } + protected RoutineLoadDesc buildRoutineLoadDescSnapshot() { + List columnsInfo = null; + if (columnDescs != null && !columnDescs.descs.isEmpty()) { + columnsInfo = new ArrayList<>(columnDescs.descs); + } + return new RoutineLoadDesc(columnSeparator, lineDelimiter, columnsInfo, precedingFilter, whereExpr, + partitionNamesInfo, deleteCondition, mergeType, sequenceCol); + } + + public void validateTargetTable(Database db, OlapTable targetTable) throws UserException { + if (isMultiTable) { + throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports single-table job"); + } + checkMeta(targetTable, buildRoutineLoadDescSnapshot()); + + targetTable.readLock(); + try { + NereidsStreamLoadPlanner planner = new NereidsStreamLoadPlanner(db, targetTable, + toNereidsRoutineLoadTaskInfo()); + planner.plan(new TUniqueId(0, 0)); + } finally { + targetTable.readUnlock(); + } + } + @Override public long getId() { return id; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java index dcb683e9e227dd..c66f5e2f1a5e70 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java @@ -757,9 +757,12 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti } modifyPropertiesInternal(jobProperties, dataSourceProperties); + if (command.hasTargetTable()) { + this.tableId = command.getTargetTableId(); + } AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, - jobProperties, dataSourceProperties); + jobProperties, dataSourceProperties, command.getTargetTableId()); Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); } finally { writeUnlock(); @@ -883,6 +886,9 @@ private void resetCloudProgress(Cloud.ResetRLProgressRequest.Builder builder) th public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { try { modifyPropertiesInternal(log.getJobProperties(), (KafkaDataSourceProperties) log.getDataSourceProperties()); + if (log.getTargetTableId() != 0) { + this.tableId = log.getTargetTableId(); + } } catch (UserException e) { // should not happen LOG.error("failed to replay modify kafka routine load job: {}", id, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index 0c05889929924e..a60f1b1f309d76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -689,9 +689,12 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti } modifyPropertiesInternal(jobProperties, dataSourceProperties); + if (command.hasTargetTable()) { + this.tableId = command.getTargetTableId(); + } AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, - jobProperties, dataSourceProperties); + jobProperties, dataSourceProperties, command.getTargetTableId()); Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); } finally { writeUnlock(); @@ -785,6 +788,9 @@ public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { try { modifyPropertiesInternal(log.getJobProperties(), (KinesisDataSourceProperties) log.getDataSourceProperties()); + if (log.getTargetTableId() != 0) { + this.tableId = log.getTargetTableId(); + } } catch (UserException e) { LOG.error("failed to replay modify kinesis routine load job: {}", id, e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index c470bc789d9e78..4116eab04c844a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -9284,6 +9284,10 @@ public LogicalPlan visitAlterRoutineLoad(DorisParser.AlterRoutineLoadContext ctx } LabelNameInfo labelNameInfo = new LabelNameInfo(dbName, jobName); + if (ctx.table != null) { + return new AlterRoutineLoadCommand(labelNameInfo, ctx.table.getText()); + } + Map properties = new HashMap<>(); if (ctx.properties != null) { properties.putAll(visitPropertyClause(ctx.properties)); @@ -9309,7 +9313,8 @@ public LogicalPlan visitAlterRoutineLoad(DorisParser.AlterRoutineLoadContext ctx } } - return new AlterRoutineLoadCommand(labelNameInfo, loadPropertyMap, properties, dataSourceMapProperties); + return new AlterRoutineLoadCommand(labelNameInfo, null, + loadPropertyMap, properties, dataSourceMapProperties); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java index 367480c5d934d9..dce80149607c60 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java @@ -21,12 +21,16 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.RandomDistributionInfo; import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.UserException; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; +import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; import org.apache.doris.datasource.property.fileformat.JsonFileFormatProperties; import org.apache.doris.load.RoutineLoadDesc; @@ -40,6 +44,7 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.mysql.privilege.PrivPredicate; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; @@ -85,10 +90,12 @@ public class AlterRoutineLoadCommand extends AlterCommand { .build(); private final LabelNameInfo labelNameInfo; + private final String targetTableName; private final Map loadPropertyMap; private RoutineLoadDesc routineLoadDesc; private final Map jobProperties; private final Map dataSourceMapProperties; + private long targetTableId; private boolean isPartialUpdate; // save analyzed job properties. @@ -100,6 +107,7 @@ public class AlterRoutineLoadCommand extends AlterCommand { * AlterRoutineLoadCommand */ public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, + String targetTableName, Map loadPropertyMap, Map jobProperties, Map dataSourceMapProperties) { @@ -108,6 +116,7 @@ public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, Objects.requireNonNull(jobProperties, "jobProperties is null"); Objects.requireNonNull(dataSourceMapProperties, "dataSourceMapProperties is null"); this.labelNameInfo = labelNameInfo; + this.targetTableName = targetTableName; this.loadPropertyMap = loadPropertyMap == null ? Maps.newHashMap() : loadPropertyMap; this.jobProperties = jobProperties; this.dataSourceMapProperties = dataSourceMapProperties; @@ -118,7 +127,11 @@ public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, Map jobProperties, Map dataSourceMapProperties) { - this(labelNameInfo, Maps.newHashMap(), jobProperties, dataSourceMapProperties); + this(labelNameInfo, null, Maps.newHashMap(), jobProperties, dataSourceMapProperties); + } + + public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, String targetTableName) { + this(labelNameInfo, targetTableName, Maps.newHashMap(), Maps.newHashMap(), Maps.newHashMap()); } public String getDbName() { @@ -133,6 +146,18 @@ public Map getAnalyzedJobProperties() { return analyzedJobProperties; } + public boolean hasTargetTable() { + return !StringUtil.isEmpty(targetTableName); + } + + public String getTargetTableName() { + return targetTableName; + } + + public long getTargetTableId() { + return targetTableId; + } + public boolean hasDataSourceProperty() { return MapUtils.isNotEmpty(dataSourceMapProperties); } @@ -161,18 +186,27 @@ public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { public void validate(ConnectContext ctx) throws UserException { labelNameInfo.validate(ctx); FeNameFormat.checkCommonName(NAME_TYPE, labelNameInfo.getLabel()); + if (hasTargetTable() && (MapUtils.isNotEmpty(loadPropertyMap) || MapUtils.isNotEmpty(jobProperties) + || MapUtils.isNotEmpty(dataSourceMapProperties))) { + throw new AnalysisException("ALTER ROUTINE LOAD target table change does not support other properties"); + } // check routine load job properties include desired concurrent number etc. checkJobProperties(); // check load properties RoutineLoadJob job = Env.getCurrentEnv().getRoutineLoadManager() .getJob(getDbName(), getJobName()); - this.routineLoadDesc = CreateRoutineLoadInfo.checkLoadProperties(ctx, loadPropertyMap, - job.getDbFullName(), job.getTableName(), job.isMultiTable(), job.getMergeType()); + if (MapUtils.isNotEmpty(loadPropertyMap)) { + this.routineLoadDesc = CreateRoutineLoadInfo.checkLoadProperties(ctx, loadPropertyMap, + job.getDbFullName(), job.getTableName(), job.isMultiTable(), job.getMergeType()); + } // check data source properties checkDataSourceProperties(); + if (hasTargetTable()) { + validateTargetTable(ctx, job); + } checkPartialUpdate(); if (analyzedJobProperties.isEmpty() && MapUtils.isEmpty(dataSourceMapProperties) - && routineLoadDesc == null) { + && MapUtils.isEmpty(loadPropertyMap) && !hasTargetTable()) { throw new AnalysisException("No properties are specified"); } } @@ -333,17 +367,44 @@ private void checkPartialUpdate() throws UserException { return; } RoutineLoadJob job = Env.getCurrentEnv().getRoutineLoadManager() - .getJob(getDbName(), getDbName()); + .getJob(getDbName(), getJobName()); if (job.isMultiTable()) { throw new AnalysisException("load by PARTIAL_COLUMNS is not supported in multi-table load."); } Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException(job.getDbFullName()); - Table table = db.getTableOrAnalysisException(job.getTableName()); + String tableName = hasTargetTable() ? targetTableName : job.getTableName(); + Table table = db.getTableOrAnalysisException(tableName); if (isPartialUpdate && !((OlapTable) table).getEnableUniqueKeyMergeOnWrite()) { throw new AnalysisException("load by PARTIAL_COLUMNS is only supported in unique table MoW"); } } + private void validateTargetTable(ConnectContext ctx, RoutineLoadJob job) throws UserException { + if (job.isMultiTable()) { + throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports single-table job"); + } + Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException(job.getDbFullName()); + Table table = db.getTableOrAnalysisException(targetTableName); + if (!(table instanceof OlapTable)) { + throw new AnalysisException("ALTER ROUTINE LOAD target table only supports OLAP table"); + } + OlapTable olapTable = (OlapTable) table; + if (olapTable.isTemporary()) { + throw new AnalysisException("Do not support load for temporary table " + olapTable.getDisplayName()); + } + if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ctx, InternalCatalog.INTERNAL_CATALOG_NAME, + job.getDbFullName(), targetTableName, PrivPredicate.LOAD)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "LOAD", + ctx.getQualifiedUser(), ctx.getRemoteIP(), job.getDbFullName() + ": " + targetTableName); + } + if (job.isLoadToSingleTablet() + && !(olapTable.getDefaultDistributionInfo() instanceof RandomDistributionInfo)) { + throw new AnalysisException("if load_to_single_tablet set to true, the olap table must be with random distribution"); + } + job.validateTargetTable(db, olapTable); + this.targetTableId = olapTable.getId(); + } + @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitAlterRoutineLoadCommand(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java index 4729882f7927fb..ae7c8ad214de7c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java @@ -37,12 +37,20 @@ public class AlterRoutineLoadJobOperationLog implements Writable { private Map jobProperties; @SerializedName(value = "dataSourceProperties") private AbstractDataSourceProperties dataSourceProperties; + @SerializedName(value = "targetTableId") + private long targetTableId; public AlterRoutineLoadJobOperationLog(long jobId, Map jobProperties, AbstractDataSourceProperties dataSourceProperties) { + this(jobId, jobProperties, dataSourceProperties, 0L); + } + + public AlterRoutineLoadJobOperationLog(long jobId, Map jobProperties, + AbstractDataSourceProperties dataSourceProperties, long targetTableId) { this.jobId = jobId; this.jobProperties = jobProperties; this.dataSourceProperties = dataSourceProperties; + this.targetTableId = targetTableId; } public long getJobId() { @@ -57,6 +65,10 @@ public AbstractDataSourceProperties getDataSourceProperties() { return dataSourceProperties; } + public long getTargetTableId() { + return targetTableId; + } + public static AlterRoutineLoadJobOperationLog read(DataInput in) throws IOException { String json = Text.readString(in); return GsonUtils.GSON.fromJson(json, AlterRoutineLoadJobOperationLog.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index 8b442012ff2c04..7a1d11513256c8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -44,6 +44,7 @@ import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo; import org.apache.doris.nereids.trees.plans.commands.load.LoadProperty; import org.apache.doris.nereids.trees.plans.commands.load.LoadSeparator; +import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TResourceInfo; @@ -415,6 +416,24 @@ public void testFromCreateStmt() throws UserException { } } + @Test + public void testReplayModifyPropertiesSwitchesTargetTableWithoutResettingProgress() { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, + 101L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Map partitionToOffset = Maps.newHashMap(); + partitionToOffset.put(1, 123L); + KafkaProgress progress = new KafkaProgress(partitionToOffset); + Deencapsulation.setField(routineLoadJob, "progress", progress); + + AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(1L, + Maps.newHashMap(), null, 202L); + routineLoadJob.replayModifyProperties(log); + + Assert.assertEquals(202L, routineLoadJob.getTableId()); + Assert.assertSame(progress, routineLoadJob.getProgress()); + Assert.assertEquals(Long.valueOf(123L), ((KafkaProgress) routineLoadJob.getProgress()).getOffsetByPartition(1)); + } + private CreateRoutineLoadInfo initCreateRoutineLoadInfo() { Map properties = Maps.newHashMap(); properties.put(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY, "2"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java index 0989bc127b046b..f52b269bfb79e6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java @@ -19,16 +19,23 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.RandomDistributionInfo; import org.apache.doris.catalog.Table; +import org.apache.doris.common.AnalysisException; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.RoutineLoadJob; import org.apache.doris.load.routineload.RoutineLoadManager; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.nereids.exceptions.ParseException; +import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.QueryState; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.thrift.TUniqueKeyUpdateMode; import com.google.common.collect.Maps; import org.junit.jupiter.api.AfterEach; @@ -41,8 +48,16 @@ import java.util.Map; public class AlterRoutineLoadCommandTest { + private static final NereidsParser PARSER = new NereidsParser(); + private Env env; private ConnectContext connectContext; + private AccessControllerManager accessManager; + private InternalCatalog catalog; + private Database db; + private OlapTable currentTable; + private RoutineLoadManager routineLoadManager; + private RoutineLoadJob routineLoadJob; private MockedStatic envMockedStatic; private MockedStatic ctxMockedStatic; @@ -50,26 +65,38 @@ public class AlterRoutineLoadCommandTest { public void setUp() throws Exception { env = Mockito.mock(Env.class); connectContext = Mockito.mock(ConnectContext.class); + accessManager = Mockito.mock(AccessControllerManager.class); + catalog = Mockito.mock(InternalCatalog.class); + db = Mockito.mock(Database.class); + currentTable = Mockito.mock(OlapTable.class); + routineLoadManager = Mockito.mock(RoutineLoadManager.class); + routineLoadJob = Mockito.mock(RoutineLoadJob.class); envMockedStatic = Mockito.mockStatic(Env.class); ctxMockedStatic = Mockito.mockStatic(ConnectContext.class); envMockedStatic.when(Env::getCurrentEnv).thenReturn(env); ctxMockedStatic.when(ConnectContext::get).thenReturn(connectContext); Mockito.when(connectContext.getSessionVariable()).thenReturn(new SessionVariable()); Mockito.when(connectContext.getState()).thenReturn(new QueryState()); - InternalCatalog catalog = Mockito.mock(InternalCatalog.class); - Database db = Mockito.mock(Database.class); - Table tbl = Mockito.mock(Table.class); + Mockito.when(connectContext.getQualifiedUser()).thenReturn("testUser"); + Mockito.when(connectContext.getRemoteIP()).thenReturn("127.0.0.1"); envMockedStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); Mockito.doReturn(db).when(catalog).getDbOrAnalysisException(Mockito.anyString()); - Mockito.doReturn(tbl).when(db).getTableOrAnalysisException(Mockito.anyString()); - Mockito.when(env.getRoutineLoadManager()).thenReturn(Mockito.mock(RoutineLoadManager.class)); - RoutineLoadManager rlm = env.getRoutineLoadManager(); - RoutineLoadJob rlJob = Mockito.mock(RoutineLoadJob.class); - Mockito.when(rlm.getJob(Mockito.anyString(), Mockito.anyString())).thenReturn(rlJob); - Mockito.when(rlJob.getDbFullName()).thenReturn("testDb"); - Mockito.when(rlJob.getTableName()).thenReturn("testTable"); - Mockito.when(rlJob.isMultiTable()).thenReturn(false); - Mockito.when(rlJob.getMergeType()).thenReturn(LoadTask.MergeType.APPEND); + Mockito.doReturn(currentTable).when(db).getTableOrAnalysisException(Mockito.anyString()); + Mockito.when(env.getRoutineLoadManager()).thenReturn(routineLoadManager); + Mockito.when(env.getAccessManager()).thenReturn(accessManager); + Mockito.when(accessManager.checkTblPriv(Mockito.any(ConnectContext.class), Mockito.anyString(), + Mockito.anyString(), Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(routineLoadManager.getJob(Mockito.anyString(), Mockito.anyString())).thenReturn(routineLoadJob); + Mockito.when(routineLoadJob.getDbFullName()).thenReturn("testDb"); + Mockito.when(routineLoadJob.getTableName()).thenReturn("testTable"); + Mockito.when(routineLoadJob.getDbId()).thenReturn(1000L); + Mockito.when(routineLoadJob.getTableId()).thenReturn(2000L); + Mockito.when(routineLoadJob.isMultiTable()).thenReturn(false); + Mockito.when(routineLoadJob.getMergeType()).thenReturn(LoadTask.MergeType.APPEND); + Mockito.when(routineLoadJob.isLoadToSingleTablet()).thenReturn(false); + Mockito.when(routineLoadJob.getUniqueKeyUpdateMode()).thenReturn(TUniqueKeyUpdateMode.UPSERT); + Mockito.when(currentTable.getType()).thenReturn(Table.TableType.OLAP); + Mockito.when(currentTable.isTemporary()).thenReturn(false); } @AfterEach @@ -86,6 +113,14 @@ private void runBefore() { Mockito.when(connectContext.isSkipAuth()).thenReturn(true); } + private void mockTargetTable(Table table) { + try { + Mockito.doReturn(table).when(db).getTableOrAnalysisException("testTable2"); + } catch (AnalysisException e) { + throw new RuntimeException(e); + } + } + @Test public void testValidate() { runBefore(); @@ -115,4 +150,125 @@ public void testValidate() { Assertions.assertTrue(command.getAnalyzedJobProperties().containsKey(CreateRoutineLoadInfo.STRICT_MODE)); Assertions.assertTrue(command.getAnalyzedJobProperties().containsKey(CreateRoutineLoadInfo.TIMEZONE)); } + + @Test + public void testParseAlterRoutineLoadOnTargetTable() { + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2"); + Assertions.assertEquals("testDb", command.getDbName()); + Assertions.assertEquals("label1", command.getJobName()); + Assertions.assertTrue(command.hasTargetTable()); + Assertions.assertEquals("testTable2", command.getTargetTableName()); + } + + @Test + public void testParseAlterRoutineLoadOnTargetTableRejectMixedProperties() { + Assertions.assertThrows(ParseException.class, () -> PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2 PROPERTIES(\"max_error_number\"=\"1\")")); + Assertions.assertThrows(ParseException.class, () -> PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2 FROM KAFKA(\"kafka_offsets\"=\"100\")")); + Assertions.assertThrows(ParseException.class, () -> PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2 COLUMNS(k1)")); + } + + @Test + public void testValidateTargetTableOnlyDoesNotRequireOtherProperties() throws Exception { + runBefore(); + mockTargetTable(currentTable); + + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2"); + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + Assertions.assertEquals("testTable2", command.getTargetTableName()); + } + + @Test + public void testValidateTargetTableRejectsMultiTableJob() throws Exception { + runBefore(); + Mockito.when(routineLoadJob.isMultiTable()).thenReturn(true); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2"); + + Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) + .getMessage().contains("single-table")); + } + + @Test + public void testValidateTargetTableRejectsWithoutLoadPrivilege() throws Exception { + runBefore(); + mockTargetTable(currentTable); + Mockito.when(accessManager.checkTblPriv(Mockito.any(ConnectContext.class), Mockito.anyString(), + Mockito.eq("testDb"), Mockito.eq("testTable2"), Mockito.any())).thenReturn(false); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2"); + + Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) + .getMessage().contains("LOAD")); + } + + @Test + public void testValidateTargetTableRejectsTemporaryTable() throws Exception { + runBefore(); + OlapTable tempTable = Mockito.mock(OlapTable.class); + Mockito.when(tempTable.getType()).thenReturn(Table.TableType.OLAP); + Mockito.when(tempTable.isTemporary()).thenReturn(true); + mockTargetTable(tempTable); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2"); + + Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) + .getMessage().contains("temporary table")); + } + + @Test + public void testValidateTargetTableRejectsLoadToSingleTabletWithoutRandomDistribution() throws Exception { + runBefore(); + OlapTable newTable = Mockito.mock(OlapTable.class); + Mockito.when(newTable.getType()).thenReturn(Table.TableType.OLAP); + Mockito.when(newTable.isTemporary()).thenReturn(false); + Mockito.when(newTable.getDefaultDistributionInfo()).thenReturn(null); + mockTargetTable(newTable); + Mockito.when(routineLoadJob.isLoadToSingleTablet()).thenReturn(true); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2"); + + Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) + .getMessage().contains("load_to_single_tablet")); + } + + @Test + public void testValidateTargetTableAllowsLoadToSingleTabletWithRandomDistribution() throws Exception { + runBefore(); + OlapTable newTable = Mockito.mock(OlapTable.class); + RandomDistributionInfo distributionInfo = Mockito.mock(RandomDistributionInfo.class); + Mockito.when(newTable.getType()).thenReturn(Table.TableType.OLAP); + Mockito.when(newTable.isTemporary()).thenReturn(false); + Mockito.when(newTable.getDefaultDistributionInfo()).thenReturn(distributionInfo); + mockTargetTable(newTable); + Mockito.when(routineLoadJob.isLoadToSingleTablet()).thenReturn(true); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2"); + + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + } + + @Test + public void testValidateTargetTablePassesTargetTableToJobValidation() throws Exception { + runBefore(); + OlapTable targetTable = Mockito.mock(OlapTable.class); + Mockito.when(targetTable.getType()).thenReturn(Table.TableType.OLAP); + Mockito.when(targetTable.isTemporary()).thenReturn(false); + Mockito.doReturn(targetTable).when(db).getTableOrAnalysisException("testTable2"); + Mockito.when(routineLoadJob.isLoadToSingleTablet()).thenReturn(false); + Mockito.doAnswer(invocation -> { + Assertions.assertSame(db, invocation.getArgument(0)); + Assertions.assertSame(targetTable, invocation.getArgument(1)); + return null; + }).when(routineLoadJob).validateTargetTable(Mockito.any(Database.class), Mockito.any(OlapTable.class)); + + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2"); + + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java index 8a1550d48f5d13..063a6067581c17 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java @@ -19,14 +19,18 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.common.io.Text; import org.apache.doris.load.routineload.kafka.KafkaConfiguration; import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; +import org.apache.doris.persist.gson.GsonUtils; import com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.File; @@ -86,4 +90,63 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User } + @Test + public void testSerializeAlterRoutineLoadOperationLogWithTargetTableId() throws Exception { + long jobId = 1000; + long targetTableId = 2001; + Map jobProperties = Maps.newHashMap(); + Map dataSourceProperties = Maps.newHashMap(); + dataSourceProperties.put("property.group.id", "mygroup"); + KafkaDataSourceProperties routineLoadDataSourceProperties = new KafkaDataSourceProperties( + dataSourceProperties); + routineLoadDataSourceProperties.setAlter(true); + routineLoadDataSourceProperties.setTimezone(TimeUtils.DEFAULT_TIME_ZONE); + routineLoadDataSourceProperties.analyze(); + + AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(jobId, jobProperties, + routineLoadDataSourceProperties, targetTableId); + + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(byteArrayOutputStream)) { + log.write(out); + } + + AlterRoutineLoadJobOperationLog readLog; + try (DataInputStream in = new DataInputStream( + new ByteArrayInputStream(byteArrayOutputStream.toByteArray()))) { + readLog = AlterRoutineLoadJobOperationLog.read(in); + } + + Assert.assertEquals(targetTableId, readLog.getTargetTableId()); + } + + @Test + public void testDeserializeAlterRoutineLoadOperationLogWithoutTargetTableId() throws Exception { + long jobId = 1000; + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY, "5"); + Map dataSourceProperties = Maps.newHashMap(); + dataSourceProperties.put("property.group.id", "mygroup"); + KafkaDataSourceProperties routineLoadDataSourceProperties = new KafkaDataSourceProperties( + dataSourceProperties); + routineLoadDataSourceProperties.setAlter(true); + routineLoadDataSourceProperties.setTimezone(TimeUtils.DEFAULT_TIME_ZONE); + routineLoadDataSourceProperties.analyze(); + AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(jobId, + jobProperties, routineLoadDataSourceProperties, 0L); + String legacyJson = GsonUtils.GSON.toJson(log).replace(",\"targetTableId\":0", ""); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(byteArrayOutputStream)) { + Text.writeString(out, legacyJson); + } + + AlterRoutineLoadJobOperationLog readLog; + try (DataInputStream in = new DataInputStream( + new ByteArrayInputStream(byteArrayOutputStream.toByteArray()))) { + readLog = AlterRoutineLoadJobOperationLog.read(in); + } + + Assert.assertEquals(0L, readLog.getTargetTableId()); + Assert.assertEquals("5", readLog.getJobProperties().get(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)); + } } diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index cd2391b1775f1e..d16c184be0a814 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -334,10 +334,13 @@ supportedAlterStatement | ALTER SYSTEM RENAME COMPUTE GROUP name=identifier newName=identifier #alterSystemRenameComputeGroup | ALTER RESOURCE name=identifierOrText properties=propertyClause? #alterResource | ALTER REPOSITORY name=identifier properties=propertyClause? #alterRepository - | ALTER ROUTINE LOAD FOR name=multipartIdentifier - (loadProperty (COMMA loadProperty)*)? - properties=propertyClause? - (FROM type=identifier LEFT_PAREN propertyItemList RIGHT_PAREN)? #alterRoutineLoad + | ALTER ROUTINE LOAD FOR name=multipartIdentifier + ( + ON table=identifier + | (loadProperty (COMMA loadProperty)*)? + properties=propertyClause? + (FROM type=identifier LEFT_PAREN propertyItemList RIGHT_PAREN)? + ) #alterRoutineLoad | ALTER COLOCATE GROUP name=multipartIdentifier SET LEFT_PAREN propertyItemList RIGHT_PAREN #alterColocateGroup | ALTER USER (IF EXISTS)? grantUserIdentify diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy index 32571b5e29abd8..f19ac5bb729904 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy @@ -16,6 +16,7 @@ // under the License. import org.apache.kafka.clients.admin.AdminClient +import org.apache.kafka.clients.admin.NewTopic import org.apache.kafka.clients.producer.KafkaProducer import org.apache.kafka.clients.producer.ProducerRecord import org.apache.kafka.clients.producer.ProducerConfig @@ -293,5 +294,195 @@ suite("test_routine_load_alter","p0") { sql "stop routine load for ${jobName}" sql "truncate table ${tableName}" } + + // test alter target table + def srcTableName = "test_routine_load_alter_src" + def dstTableName = "test_routine_load_alter_dst" + def alterTargetTopic = "test_routine_load_alter_target_table_${System.currentTimeMillis()}" + def alterTargetJob = "test_alter_target_table_${System.currentTimeMillis()}" + def alterTopicProducer = null + def alterTopicAdmin = null + try { + sql """ DROP TABLE IF EXISTS ${srcTableName} """ + sql """ DROP TABLE IF EXISTS ${dstTableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${srcTableName} ( + `k1` int(20) NULL, + `k2` string NULL, + `v1` date NULL, + `v2` string NULL, + `v3` datetime NULL, + `v4` string NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + sql """ + CREATE TABLE IF NOT EXISTS ${dstTableName} ( + `k1` int(20) NULL, + `k2` string NULL, + `v1` date NULL, + `v2` string NULL, + `v3` datetime NULL, + `v4` string NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + sql "sync" + + def topicProps = new Properties() + topicProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) + alterTopicAdmin = AdminClient.create(topicProps) + alterTopicAdmin.createTopics([new NewTopic(alterTargetTopic, 1, (short) 1)]).all().get() + + def producerProps = new Properties() + producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) + producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.StringSerializer") + producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.StringSerializer") + producerProps.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + producerProps.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + alterTopicProducer = new KafkaProducer<>(producerProps) + + def firstBatch = new File("""${context.file.parent}/data/${kafkaCsvTpoics[0]}.csv""").readLines() + firstBatch.each { line -> + alterTopicProducer.send(new ProducerRecord<>(alterTargetTopic, null, line)).get() + } + alterTopicProducer.flush() + + sql """ + CREATE ROUTINE LOAD ${alterTargetJob} ON ${srcTableName} + COLUMNS TERMINATED BY "," + PROPERTIES + ( + "max_batch_interval" = "5", + "max_batch_rows" = "300000", + "max_batch_size" = "209715200" + ) + FROM KAFKA + ( + "kafka_broker_list" = "${kafka_broker}", + "kafka_topic" = "${alterTargetTopic}", + "kafka_partitions" = "0", + "kafka_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + + count = 0 + while (true) { + def res = sql "select count(*) from ${srcTableName}" + def state = sql "show routine load for ${alterTargetJob}" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] >= 3) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(3, res[0][0]) + break + } + sleep(1000) + count++ + } + + sql "pause routine load for ${alterTargetJob}" + def showBeforeAlter = sql "show routine load for ${alterTargetJob}" + assertEquals(srcTableName, showBeforeAlter[0][6].toString()) + def progressBeforeAlter = showBeforeAlter[0][15].toString() + + sql "ALTER ROUTINE LOAD FOR ${alterTargetJob} ON ${dstTableName}" + + def showAfterAlter = sql "show routine load for ${alterTargetJob}" + assertEquals(dstTableName, showAfterAlter[0][6].toString()) + assertEquals(progressBeforeAlter, showAfterAlter[0][15].toString()) + + def secondBatch = [ + "4,eab,2023-07-16,def,2023-07-21:05:48:31,ghi", + "5,eab,2023-07-17,def,2023-07-22:05:48:31,ghi", + "6,eab,2023-07-18,def,2023-07-23:05:48:31,ghi" + ] + secondBatch.each { line -> + alterTopicProducer.send(new ProducerRecord<>(alterTargetTopic, null, line)).get() + } + alterTopicProducer.flush() + + sql "resume routine load for ${alterTargetJob}" + + count = 0 + def stableCount = 0 + while (true) { + def srcCount = sql "select count(*) from ${srcTableName}" + def dstCount = sql "select count(*) from ${dstTableName}" + long srcCountValue = (srcCount[0][0] as Number).longValue() + long dstCountValue = (dstCount[0][0] as Number).longValue() + log.info("src count: ${srcCountValue}".toString()) + log.info("dst count: ${dstCountValue}".toString()) + if (srcCountValue == 3 && dstCountValue == 3) { + stableCount++ + if (stableCount >= 5) { + sleep(2000) + def finalSrcCount = sql "select count(*) from ${srcTableName}" + def finalDstCount = sql "select count(*) from ${dstTableName}" + assertEquals(3L, (finalSrcCount[0][0] as Number).longValue()) + assertEquals(3L, (finalDstCount[0][0] as Number).longValue()) + break + } + } else { + stableCount = 0 + if (srcCountValue > 3 || dstCountValue > 3) { + assertEquals(3L, srcCountValue) + assertEquals(3L, dstCountValue) + } + } + if (count >= 120) { + log.error("routine load target table alter can not visible for long time") + assertEquals(3L, srcCountValue) + assertEquals(3L, dstCountValue) + break + } + sleep(1000) + count++ + } + + def srcRows = sql "select k1, k2 from ${srcTableName} order by k1" + def dstRows = sql "select k1, k2 from ${dstTableName} order by k1" + assertEquals(3, srcRows.size()) + assertEquals(3, dstRows.size()) + assertEquals(1, srcRows[0][0]) + assertEquals("eab", srcRows[0][1]) + assertEquals(2, srcRows[1][0]) + assertEquals("eab", srcRows[1][1]) + assertEquals(3, srcRows[2][0]) + assertEquals("eab", srcRows[2][1]) + assertEquals(4, dstRows[0][0]) + assertEquals("eab", dstRows[0][1]) + assertEquals(5, dstRows[1][0]) + assertEquals("eab", dstRows[1][1]) + assertEquals(6, dstRows[2][0]) + assertEquals("eab", dstRows[2][1]) + } finally { + try { + sql "stop routine load for ${alterTargetJob}" + } catch (Exception e) { + logger.warn("failed to stop alter target routine load: ${e.message}".toString()) + } + if (alterTopicProducer != null) { + alterTopicProducer.close() + } + if (alterTopicAdmin != null) { + alterTopicAdmin.close() + } + sql "truncate table ${srcTableName}" + sql "truncate table ${dstTableName}" + } } } From 03cf6269e260ad9bfbb2ce4fa13b26076317f650 Mon Sep 17 00:00:00 2001 From: Refrain Date: Fri, 26 Jun 2026 14:39:11 +0800 Subject: [PATCH 02/19] [fix](fe) Tighten routine load alter validation ### What problem does this PR solve? Issue Number: None Related PR: #64878 Problem Summary: Follow-up review found two issues in the new routine load target-table alter support. First, `AlterRoutineLoadCommand` had an import order regression that could fail FE checkstyle. Second, alter validation only rechecked `PARTIAL_COLUMNS=true` from the current command, which left the effective partial-update state under-validated when the existing job or a `unique_key_update_mode` change required merge-on-write semantics. This change restores import ordering, validates the effective unique key update mode against the destination table, and adds focused FE unit coverage for those cases. ### Release note Routine Load alter now rejects target-table or unique-key-update changes that are incompatible with partial update requirements. ### Check List (For Author) - Test: Unit Test - Behavior changed: Yes - Does this need documentation: No --- .../load/routineload/RoutineLoadJob.java | 3 +- .../commands/AlterRoutineLoadCommand.java | 24 ++++++++++--- .../commands/AlterRoutineLoadCommandTest.java | 34 +++++++++++++++++++ 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index e7c69da5a77488..407ce7ecd86fba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ExprToSqlVisitor; +import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.Separator; import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.analysis.UserIdentity; @@ -469,7 +470,7 @@ protected void setRoutineLoadDesc(RoutineLoadDesc routineLoadDesc) { } protected RoutineLoadDesc buildRoutineLoadDescSnapshot() { - List columnsInfo = null; + List columnsInfo = null; if (columnDescs != null && !columnDescs.descs.isEmpty()) { columnsInfo = new ArrayList<>(columnDescs.descs); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java index dce80149607c60..7d392e5d6d6fea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java @@ -37,6 +37,7 @@ import org.apache.doris.load.routineload.AbstractDataSourceProperties; import org.apache.doris.load.routineload.RoutineLoadDataSourcePropertyFactory; import org.apache.doris.load.routineload.RoutineLoadJob; +import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo; @@ -44,7 +45,7 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.StmtExecutor; -import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.thrift.TUniqueKeyUpdateMode; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; @@ -363,22 +364,35 @@ private void checkDataSourceProperties() throws UserException { } private void checkPartialUpdate() throws UserException { - if (!isPartialUpdate) { - return; - } RoutineLoadJob job = Env.getCurrentEnv().getRoutineLoadManager() .getJob(getDbName(), getJobName()); + TUniqueKeyUpdateMode uniqueKeyUpdateMode = getEffectiveUniqueKeyUpdateMode(job); + if (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { + return; + } if (job.isMultiTable()) { throw new AnalysisException("load by PARTIAL_COLUMNS is not supported in multi-table load."); } Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException(job.getDbFullName()); String tableName = hasTargetTable() ? targetTableName : job.getTableName(); Table table = db.getTableOrAnalysisException(tableName); - if (isPartialUpdate && !((OlapTable) table).getEnableUniqueKeyMergeOnWrite()) { + if (!((OlapTable) table).getEnableUniqueKeyMergeOnWrite()) { throw new AnalysisException("load by PARTIAL_COLUMNS is only supported in unique table MoW"); } } + private TUniqueKeyUpdateMode getEffectiveUniqueKeyUpdateMode(RoutineLoadJob job) { + if (analyzedJobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { + return TUniqueKeyUpdateMode.valueOf( + analyzedJobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)); + } + if (analyzedJobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS) && isPartialUpdate + && job.getUniqueKeyUpdateMode() == TUniqueKeyUpdateMode.UPSERT) { + return TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS; + } + return job.getUniqueKeyUpdateMode(); + } + private void validateTargetTable(ConnectContext ctx, RoutineLoadJob job) throws UserException { if (job.isMultiTable()) { throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports single-table job"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java index f52b269bfb79e6..93aa02e72b179f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java @@ -271,4 +271,38 @@ public void testValidateTargetTablePassesTargetTableToJobValidation() throws Exc Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); } + + @Test + public void testValidateTargetTableRejectsExistingFlexiblePartialUpdateOnNonMowTable() throws Exception { + runBefore(); + OlapTable targetTable = Mockito.mock(OlapTable.class); + Mockito.when(targetTable.getType()).thenReturn(Table.TableType.OLAP); + Mockito.when(targetTable.isTemporary()).thenReturn(false); + Mockito.when(targetTable.getEnableUniqueKeyMergeOnWrite()).thenReturn(false); + Mockito.doReturn(targetTable).when(db).getTableOrAnalysisException("testTable2"); + Mockito.when(routineLoadJob.getUniqueKeyUpdateMode()).thenReturn(TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS); + Mockito.doThrow(new AnalysisException("Only unique key merge on write support partial update")) + .when(routineLoadJob).validateTargetTable(Mockito.any(Database.class), Mockito.any(OlapTable.class)); + + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2"); + + Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) + .getMessage().contains("partial update")); + } + + @Test + public void testValidateRejectsUniqueKeyUpdateModeOnNonMowTable() { + runBefore(); + Mockito.when(routineLoadJob.getUniqueKeyUpdateMode()).thenReturn(TUniqueKeyUpdateMode.UPSERT); + Mockito.when(currentTable.getEnableUniqueKeyMergeOnWrite()).thenReturn(false); + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, "UPDATE_FIXED_COLUMNS"); + + AlterRoutineLoadCommand command = new AlterRoutineLoadCommand( + new LabelNameInfo("testDb", "label1"), jobProperties, Maps.newHashMap()); + + Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) + .getMessage().contains("PARTIAL_COLUMNS")); + } } From 9b2cf366a50563ec48449548b75778605826517a Mon Sep 17 00:00:00 2001 From: Refrain Date: Mon, 29 Jun 2026 09:40:17 +0800 Subject: [PATCH 03/19] [fix](fe) Fix routine load alter checkstyle issues ### What problem does this PR solve? Issue Number: None Related PR: #64878 Problem Summary: The routine load target-table alter change hit FE checkstyle in CI because one validation error message exceeded the line-length limit and a unit-test import order did not match the FE custom import ordering rule. This commit makes the minimal formatting-only fixes so the branch aligns with FE style checks. ### Release note None ### Check List (For Author) - Test: No need to test (formatting-only fix requested by reviewer; no local build or test run) - Behavior changed: No - Does this need documentation: No --- .../nereids/trees/plans/commands/AlterRoutineLoadCommand.java | 3 ++- .../apache/doris/persist/AlterRoutineLoadOperationLogTest.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java index 7d392e5d6d6fea..cd6e95f8c3c2a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java @@ -413,7 +413,8 @@ private void validateTargetTable(ConnectContext ctx, RoutineLoadJob job) throws } if (job.isLoadToSingleTablet() && !(olapTable.getDefaultDistributionInfo() instanceof RandomDistributionInfo)) { - throw new AnalysisException("if load_to_single_tablet set to true, the olap table must be with random distribution"); + throw new AnalysisException( + "if load_to_single_tablet set to true, the olap table must be with random distribution"); } job.validateTargetTable(db, olapTable); this.targetTableId = olapTable.getId(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java index 063a6067581c17..5aaf6423ad22b7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java @@ -18,8 +18,8 @@ package org.apache.doris.persist; import org.apache.doris.common.UserException; -import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.io.Text; +import org.apache.doris.common.util.TimeUtils; import org.apache.doris.load.routineload.kafka.KafkaConfiguration; import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; From 760890d5d7eef15765b1fc92f16783d7301870c2 Mon Sep 17 00:00:00 2001 From: Refrain Date: Mon, 29 Jun 2026 10:56:33 +0800 Subject: [PATCH 04/19] [fix](fe) Refine routine load alter target-table follow-up ### What problem does this PR solve? Issue Number: None Related PR: #64878 Problem Summary: The routine load target-table alter implementation had a single-use helper for constructing the validation load descriptor snapshot, and the parser branch for table-only alter did not mark the intended phase-one scope. This commit inlines the one-off snapshot construction at the validation call site and documents that the current parser branch only supports target table alteration before future support for combining target-table and property changes. ### Release note None ### Check List (For Author) - Test: No need to test (review follow-up only; no local build or test run) - Behavior changed: No - Does this need documentation: No --- .../doris/load/routineload/RoutineLoadJob.java | 16 ++++++---------- .../doris/nereids/parser/LogicalPlanBuilder.java | 2 ++ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 407ce7ecd86fba..0b8ca3c5b52f30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -469,20 +469,16 @@ protected void setRoutineLoadDesc(RoutineLoadDesc routineLoadDesc) { } } - protected RoutineLoadDesc buildRoutineLoadDescSnapshot() { - List columnsInfo = null; - if (columnDescs != null && !columnDescs.descs.isEmpty()) { - columnsInfo = new ArrayList<>(columnDescs.descs); - } - return new RoutineLoadDesc(columnSeparator, lineDelimiter, columnsInfo, precedingFilter, whereExpr, - partitionNamesInfo, deleteCondition, mergeType, sequenceCol); - } - public void validateTargetTable(Database db, OlapTable targetTable) throws UserException { if (isMultiTable) { throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports single-table job"); } - checkMeta(targetTable, buildRoutineLoadDescSnapshot()); + List columnsInfo = null; + if (columnDescs != null && !columnDescs.descs.isEmpty()) { + columnsInfo = new ArrayList<>(columnDescs.descs); + } + checkMeta(targetTable, new RoutineLoadDesc(columnSeparator, lineDelimiter, columnsInfo, precedingFilter, + whereExpr, partitionNamesInfo, deleteCondition, mergeType, sequenceCol)); targetTable.readLock(); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 4116eab04c844a..0b6f5f32c8a564 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -9284,6 +9284,8 @@ public LogicalPlan visitAlterRoutineLoad(DorisParser.AlterRoutineLoadContext ctx } LabelNameInfo labelNameInfo = new LabelNameInfo(dbName, jobName); + // TODO: Phase 1 only supports altering the target table. Phase 2 will allow altering + // the target table and routine load properties in the same statement. if (ctx.table != null) { return new AlterRoutineLoadCommand(labelNameInfo, ctx.table.getText()); } From 931b71726a52611851f489bdd831a78fb06e1ece Mon Sep 17 00:00:00 2001 From: Refrain Date: Mon, 29 Jun 2026 14:28:24 +0800 Subject: [PATCH 05/19] [fix](fe) Align routine load alter partial update mode ### What problem does this PR solve? Issue Number: None Related PR: #64878 Problem Summary: ALTER ROUTINE LOAD validation handled unique_key_update_mode and legacy partial_columns with different precedence from the mutation path. An ALTER containing unique_key_update_mode=UPSERT and partial_columns=true could pass validation on a non-MoW table, then be applied as UPDATE_FIXED_COLUMNS. Flexible partial-update ALTERs on non-MoW tables were also rejected by the generic PARTIAL_COLUMNS validation before reaching the flexible partial-update validation path. This change makes generic partial-column validation apply only to fixed partial update mode and makes the mutation path ignore legacy partial_columns when an explicit unique_key_update_mode is present. The routine load alter regression test also declares its polling counters as local variables to satisfy regression framework script checks. ### Release note None ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommandTest,org.apache.doris.load.routineload.KafkaRoutineLoadJobTest,org.apache.doris.load.routineload.KinesisRoutineLoadJobTest - Behavior changed: No - Does this need documentation: No --- .../load/routineload/RoutineLoadJob.java | 4 ++- .../kafka/KafkaRoutineLoadJob.java | 4 --- .../kinesis/KinesisRoutineLoadJob.java | 4 --- .../commands/AlterRoutineLoadCommand.java | 2 +- .../routineload/KafkaRoutineLoadJobTest.java | 19 ++++++++++++ .../KinesisRoutineLoadJobTest.java | 21 ++++++++++++++ .../commands/AlterRoutineLoadCommandTest.java | 29 +++++++++++++++++++ .../test_routine_load_alter.groovy | 6 ++-- 8 files changed, 76 insertions(+), 13 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 0b8ca3c5b52f30..2dd001b29e9eee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -2088,7 +2088,8 @@ protected void modifyCommonJobProperties(Map jobProperties) thro jobProperties.remove(CreateRoutineLoadInfo.MAX_BATCH_SIZE_PROPERTY)); } - if (jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { + boolean hasExplicitUniqueKeyUpdateMode = jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); + if (hasExplicitUniqueKeyUpdateMode) { String modeStr = jobProperties.remove(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); TUniqueKeyUpdateMode newMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode(modeStr); // Validate flexible partial update constraints when changing to UPDATE_FLEXIBLE_COLUMNS @@ -2099,6 +2100,7 @@ protected void modifyCommonJobProperties(Map jobProperties) thro this.isPartialUpdate = (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS); this.jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); + jobProperties.remove(CreateRoutineLoadInfo.PARTIAL_COLUMNS); } if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java index c66f5e2f1a5e70..f45a2295b1c0b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java @@ -73,7 +73,6 @@ import com.google.gson.annotations.SerializedName; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.BooleanUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -847,9 +846,6 @@ private void modifyPropertiesInternal(Map jobProperties, Map copiedJobProperties = Maps.newHashMap(jobProperties); modifyCommonJobProperties(copiedJobProperties); this.jobProperties.putAll(copiedJobProperties); - if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { - this.isPartialUpdate = BooleanUtils.toBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); - } if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { String policy = jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); if ("ERROR".equalsIgnoreCase(policy)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index a60f1b1f309d76..4f97857ff7a0a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -65,7 +65,6 @@ import com.google.gson.annotations.SerializedName; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.BooleanUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -767,9 +766,6 @@ private void modifyPropertiesInternal(Map jobProperties, Map copiedJobProperties = Maps.newHashMap(jobProperties); modifyCommonJobProperties(copiedJobProperties); this.jobProperties.putAll(copiedJobProperties); - if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { - this.isPartialUpdate = BooleanUtils.toBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); - } if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { String policy = jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); if ("ERROR".equalsIgnoreCase(policy)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java index cd6e95f8c3c2a6..6fad5299a477ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java @@ -367,7 +367,7 @@ private void checkPartialUpdate() throws UserException { RoutineLoadJob job = Env.getCurrentEnv().getRoutineLoadManager() .getJob(getDbName(), getJobName()); TUniqueKeyUpdateMode uniqueKeyUpdateMode = getEffectiveUniqueKeyUpdateMode(job); - if (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { + if (uniqueKeyUpdateMode != TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS) { return; } if (job.isMultiTable()) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index 7a1d11513256c8..069f57199550c7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -47,6 +47,7 @@ import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TResourceInfo; +import org.apache.doris.thrift.TUniqueKeyUpdateMode; import com.google.common.base.Joiner; import com.google.common.collect.Lists; @@ -198,6 +199,24 @@ public void testUpdateLagRefreshesLatestOffsetCache() throws UserException { } } + @Test + public void testModifyPropertiesHonorsExplicitUniqueKeyUpdateModePrecedence() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, + 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "uniqueKeyUpdateMode", TUniqueKeyUpdateMode.UPSERT); + Deencapsulation.setField(routineLoadJob, "isPartialUpdate", false); + + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, "UPSERT"); + jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, "true"); + + Deencapsulation.invoke(routineLoadJob, "modifyPropertiesInternal", jobProperties, + new KafkaDataSourceProperties(Maps.newHashMap())); + + Assert.assertEquals(TUniqueKeyUpdateMode.UPSERT, routineLoadJob.getUniqueKeyUpdateMode()); + Assert.assertFalse(routineLoadJob.isFixedPartialUpdate()); + } + @Test public void testUpdateLagRebuildsConvertedPropertiesAfterReplay() throws UserException { KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java index aa1dc052605f2b..67688ab2e790e7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java @@ -25,6 +25,8 @@ import org.apache.doris.load.routineload.kinesis.KinesisProgress; import org.apache.doris.load.routineload.kinesis.KinesisRoutineLoadJob; import org.apache.doris.load.routineload.kinesis.KinesisTaskInfo; +import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; +import org.apache.doris.thrift.TUniqueKeyUpdateMode; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -101,6 +103,25 @@ public void testGetStatisticContainsKinesisFields() { Assert.assertEquals(100L, ((Number) statistic.get("maxMillisBehindLatest")).longValue()); } + @Test + public void testModifyPropertiesHonorsExplicitUniqueKeyUpdateModePrecedence() throws Exception { + KinesisRoutineLoadJob routineLoadJob = + new KinesisRoutineLoadJob(1L, "kinesis_routine_load_job", 1L, + 1L, "ap-southeast-1", "stream-1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "uniqueKeyUpdateMode", TUniqueKeyUpdateMode.UPSERT); + Deencapsulation.setField(routineLoadJob, "isPartialUpdate", false); + + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, "UPSERT"); + jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, "true"); + + Deencapsulation.invoke(routineLoadJob, "modifyPropertiesInternal", jobProperties, + new KinesisDataSourceProperties(Maps.newHashMap())); + + Assert.assertEquals(TUniqueKeyUpdateMode.UPSERT, routineLoadJob.getUniqueKeyUpdateMode()); + Assert.assertFalse(routineLoadJob.isFixedPartialUpdate()); + } + @Test public void testHasMoreDataToConsumeShouldKeepPollingWhenLagCacheIsZero() throws Exception { KinesisRoutineLoadJob routineLoadJob = diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java index 93aa02e72b179f..f7fd80622ec1eb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java @@ -305,4 +305,33 @@ public void testValidateRejectsUniqueKeyUpdateModeOnNonMowTable() { Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) .getMessage().contains("PARTIAL_COLUMNS")); } + + @Test + public void testValidateAllowsExplicitUpsertToOverridePartialColumnsOnNonMowTable() { + runBefore(); + Mockito.when(routineLoadJob.getUniqueKeyUpdateMode()).thenReturn(TUniqueKeyUpdateMode.UPSERT); + Mockito.when(currentTable.getEnableUniqueKeyMergeOnWrite()).thenReturn(false); + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, "UPSERT"); + jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, "true"); + + AlterRoutineLoadCommand command = new AlterRoutineLoadCommand( + new LabelNameInfo("testDb", "label1"), jobProperties, Maps.newHashMap()); + + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + } + + @Test + public void testValidateAllowsFlexibleAlterToReachFlexibleValidation() { + runBefore(); + Mockito.when(routineLoadJob.getUniqueKeyUpdateMode()).thenReturn(TUniqueKeyUpdateMode.UPSERT); + Mockito.when(currentTable.getEnableUniqueKeyMergeOnWrite()).thenReturn(false); + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, "UPDATE_FLEXIBLE_COLUMNS"); + + AlterRoutineLoadCommand command = new AlterRoutineLoadCommand( + new LabelNameInfo("testDb", "label1"), jobProperties, Maps.newHashMap()); + + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + } } diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy index f19ac5bb729904..81817a60090e93 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy @@ -155,7 +155,7 @@ suite("test_routine_load_alter","p0") { } } - count = 0 + def count = 0 while (true) { def res = sql "select count(*) from ${tableName}" log.info("count: ${res[0][0]}".toString()) @@ -375,7 +375,7 @@ suite("test_routine_load_alter","p0") { """ sql "sync" - count = 0 + def count = 0 while (true) { def res = sql "select count(*) from ${srcTableName}" def state = sql "show routine load for ${alterTargetJob}" @@ -417,7 +417,7 @@ suite("test_routine_load_alter","p0") { sql "resume routine load for ${alterTargetJob}" - count = 0 + def count = 0 def stableCount = 0 while (true) { def srcCount = sql "select count(*) from ${srcTableName}" From 48f1b8c98a9525dcaf78df75897067e5faa65bd0 Mon Sep 17 00:00:00 2001 From: Refrain Date: Mon, 29 Jun 2026 15:54:22 +0800 Subject: [PATCH 06/19] [fix](fe) Fix routine load alter checkstyle line length ### What problem does this PR solve? Issue Number: close #xxx Related PR: #64878 Problem Summary: The PR pipeline failed in FE checkstyle because a RoutineLoadJob line added for altering unique key update mode exceeded the 120-character limit. Wrap the containsKey call without changing behavior so FE checkstyle can pass. ### Release note None ### Check List (For Author) - Test: Manual test - ./build.sh --fe -j60 - git diff --check -- fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java - Behavior changed: No - Does this need documentation: No --- .../java/org/apache/doris/load/routineload/RoutineLoadJob.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 2dd001b29e9eee..2452a2dde3c681 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -2088,7 +2088,8 @@ protected void modifyCommonJobProperties(Map jobProperties) thro jobProperties.remove(CreateRoutineLoadInfo.MAX_BATCH_SIZE_PROPERTY)); } - boolean hasExplicitUniqueKeyUpdateMode = jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); + boolean hasExplicitUniqueKeyUpdateMode = jobProperties.containsKey( + CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); if (hasExplicitUniqueKeyUpdateMode) { String modeStr = jobProperties.remove(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); TUniqueKeyUpdateMode newMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode(modeStr); From 0f1be9e1aa029625b1f257ceac9fe5f6776bc2fc Mon Sep 17 00:00:00 2001 From: Refrain Date: Mon, 29 Jun 2026 19:52:53 +0800 Subject: [PATCH 07/19] [fix](regression) Fix routine load alter test output checks ### What problem does this PR solve? Issue Number: None Related PR: #64878 Problem Summary: The routine load alter regression test checked deterministic target-table switch rows with direct Groovy assertions and reused local variable names in the same suite scope. CI reported the case as invalid, and the local framework rejected the duplicate variables during Groovy compilation. This records the deterministic source and destination row checks through qt_sql output checks, uses distinct polling counter names, and keeps Kafka topics isolated so repeated local runs do not consume retained messages. ### Release note None ### Check List (For Author) - Test: Regression test - ./run-regression-test.sh --run -d load_p0/routine_load -s test_routine_load_alter -c 'jdbc:mysql://127.0.0.1:49030/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round' -ha 127.0.0.1:48030 -conf enableKafkaTest=true -conf kafka_port=19194 -conf externalEnvIp=127.0.0.1 - Behavior changed: No - Does this need documentation: No --- .../routine_load/test_routine_load_alter.out | 9 ++ .../test_routine_load_alter.groovy | 87 +++++++++++-------- 2 files changed, 61 insertions(+), 35 deletions(-) diff --git a/regression-test/data/load_p0/routine_load/test_routine_load_alter.out b/regression-test/data/load_p0/routine_load/test_routine_load_alter.out index 427cdb2439463d..0764212aa7ae7d 100644 --- a/regression-test/data/load_p0/routine_load/test_routine_load_alter.out +++ b/regression-test/data/load_p0/routine_load/test_routine_load_alter.out @@ -10,3 +10,12 @@ 3 eab 2023-07-15 def 2023-07-20T05:48:31 "ghi" 3 eab 2023-07-15 def 2023-07-20T05:48:31 "ghi" +-- !sql_alter_target_src -- +1 eab +2 eab +3 eab + +-- !sql_alter_target_dst -- +4 eab +5 eab +6 eab diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy index 81817a60090e93..a27035e42f2cf3 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy @@ -16,14 +16,16 @@ // under the License. import org.apache.kafka.clients.admin.AdminClient +import org.apache.kafka.clients.admin.DeleteTopicsOptions import org.apache.kafka.clients.admin.NewTopic import org.apache.kafka.clients.producer.KafkaProducer import org.apache.kafka.clients.producer.ProducerRecord import org.apache.kafka.clients.producer.ProducerConfig suite("test_routine_load_alter","p0") { + def kafkaCsvDataFile = "test_routine_load_alter" def kafkaCsvTpoics = [ - "test_routine_load_alter", + "test_routine_load_alter_${System.currentTimeMillis()}".toString(), ] String enabled = context.config.otherConfigs.get("enableKafkaTest") String kafka_port = context.config.otherConfigs.get("kafka_port") @@ -39,6 +41,17 @@ suite("test_routine_load_alter","p0") { props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + def adminProps = new Properties() + adminProps.put("bootstrap.servers", "${kafka_broker}".toString()) + def mainTopicAdmin = AdminClient.create(adminProps) + try { + kafkaCsvTpoics.each { topic -> + mainTopicAdmin.createTopics([new NewTopic(topic.toString(), 1, (short) 1)]).all().get() + } + } finally { + mainTopicAdmin.close() + } + // check conenction def verifyKafkaConnection = { prod -> try { @@ -64,7 +77,7 @@ suite("test_routine_load_alter","p0") { logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { - def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text + def txt = new File("""${context.file.parent}/data/${kafkaCsvDataFile}.csv""").text def lines = txt.readLines() lines.each { line -> logger.info("=====${line}========") @@ -113,7 +126,7 @@ suite("test_routine_load_alter","p0") { """ sql "sync" - def count = 0 + def visibleCount = 0 while (true) { def res = sql "select count(*) from ${tableName}" def state = sql "show routine load for ${jobName}" @@ -123,13 +136,13 @@ suite("test_routine_load_alter","p0") { if (res[0][0] > 0) { break } - if (count >= 120) { + if (visibleCount >= 120) { log.error("routine load can not visible for long time") assertEquals(20, res[0][0]) break } sleep(5000) - count++ + visibleCount++ } qt_sql_before "select * from ${tableName} order by k1" @@ -146,7 +159,7 @@ suite("test_routine_load_alter","p0") { def producer = new KafkaProducer<>(props) for (String kafkaCsvTopic in kafkaCsvTpoics) { - def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text + def txt = new File("""${context.file.parent}/data/${kafkaCsvDataFile}.csv""").text def lines = txt.readLines() lines.each { line -> logger.info("=====${line}========") @@ -155,7 +168,7 @@ suite("test_routine_load_alter","p0") { } } - def count = 0 + def offsetVisibleCount = 0 while (true) { def res = sql "select count(*) from ${tableName}" log.info("count: ${res[0][0]}".toString()) @@ -166,13 +179,13 @@ suite("test_routine_load_alter","p0") { if (res[0][0] >= 6) { break } - if (count >= 120) { + if (offsetVisibleCount >= 120) { log.error("routine load can not visible for long time") assertEquals(20, res[0][0]) break } sleep(1000) - count++ + offsetVisibleCount++ } qt_sql_alter_after "select * from ${tableName} order by k1" } finally { @@ -199,7 +212,7 @@ suite("test_routine_load_alter","p0") { sql "ALTER ROUTINE LOAD FOR ${jobName} COLUMNS(k1, k2, v1, v2, v3, v4);" sql "ALTER ROUTINE LOAD FOR ${jobName} COLUMNS TERMINATED BY ',';" sql "resume routine load for ${jobName}" - def count = 0 + def columnVisibleCount = 0 while (true) { def res = sql "select count(*) from ${tableName}" log.info("count: ${res[0][0]}".toString()) @@ -211,13 +224,13 @@ suite("test_routine_load_alter","p0") { if (res[0][0] > 0) { break } - if (count >= 120) { + if (columnVisibleCount >= 120) { log.error("routine load can not visible for long time") assertEquals(20, res[0][0]) break } sleep(1000) - count++ + columnVisibleCount++ } def res = sql "select * from ${tableName} order by k1" log.info("res: ${res.size()}".toString()) @@ -295,6 +308,18 @@ suite("test_routine_load_alter","p0") { sql "truncate table ${tableName}" } + def mainTopicCleanupProps = new Properties() + mainTopicCleanupProps.put("bootstrap.servers", "${kafka_broker}".toString()) + def mainTopicCleanupAdmin = AdminClient.create(mainTopicCleanupProps) + try { + mainTopicCleanupAdmin.deleteTopics(kafkaCsvTpoics.collect { it.toString() }, + new DeleteTopicsOptions().timeoutMs(10000)).all().get() + } catch (Exception e) { + logger.warn("failed to delete kafka topics ${kafkaCsvTpoics}: ${e.message}".toString()) + } finally { + mainTopicCleanupAdmin.close() + } + // test alter target table def srcTableName = "test_routine_load_alter_src" def dstTableName = "test_routine_load_alter_dst" @@ -350,7 +375,7 @@ suite("test_routine_load_alter","p0") { producerProps.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") alterTopicProducer = new KafkaProducer<>(producerProps) - def firstBatch = new File("""${context.file.parent}/data/${kafkaCsvTpoics[0]}.csv""").readLines() + def firstBatch = new File("""${context.file.parent}/data/${kafkaCsvDataFile}.csv""").readLines() firstBatch.each { line -> alterTopicProducer.send(new ProducerRecord<>(alterTargetTopic, null, line)).get() } @@ -375,7 +400,7 @@ suite("test_routine_load_alter","p0") { """ sql "sync" - def count = 0 + def targetVisibleCount = 0 while (true) { def res = sql "select count(*) from ${srcTableName}" def state = sql "show routine load for ${alterTargetJob}" @@ -385,13 +410,13 @@ suite("test_routine_load_alter","p0") { if (res[0][0] >= 3) { break } - if (count >= 120) { + if (targetVisibleCount >= 120) { log.error("routine load can not visible for long time") assertEquals(3, res[0][0]) break } sleep(1000) - count++ + targetVisibleCount++ } sql "pause routine load for ${alterTargetJob}" @@ -417,7 +442,7 @@ suite("test_routine_load_alter","p0") { sql "resume routine load for ${alterTargetJob}" - def count = 0 + def targetAlterVisibleCount = 0 def stableCount = 0 while (true) { def srcCount = sql "select count(*) from ${srcTableName}" @@ -443,32 +468,18 @@ suite("test_routine_load_alter","p0") { assertEquals(3L, dstCountValue) } } - if (count >= 120) { + if (targetAlterVisibleCount >= 120) { log.error("routine load target table alter can not visible for long time") assertEquals(3L, srcCountValue) assertEquals(3L, dstCountValue) break } sleep(1000) - count++ + targetAlterVisibleCount++ } - def srcRows = sql "select k1, k2 from ${srcTableName} order by k1" - def dstRows = sql "select k1, k2 from ${dstTableName} order by k1" - assertEquals(3, srcRows.size()) - assertEquals(3, dstRows.size()) - assertEquals(1, srcRows[0][0]) - assertEquals("eab", srcRows[0][1]) - assertEquals(2, srcRows[1][0]) - assertEquals("eab", srcRows[1][1]) - assertEquals(3, srcRows[2][0]) - assertEquals("eab", srcRows[2][1]) - assertEquals(4, dstRows[0][0]) - assertEquals("eab", dstRows[0][1]) - assertEquals(5, dstRows[1][0]) - assertEquals("eab", dstRows[1][1]) - assertEquals(6, dstRows[2][0]) - assertEquals("eab", dstRows[2][1]) + qt_sql_alter_target_src "select k1, k2 from ${srcTableName} order by k1" + qt_sql_alter_target_dst "select k1, k2 from ${dstTableName} order by k1" } finally { try { sql "stop routine load for ${alterTargetJob}" @@ -479,6 +490,12 @@ suite("test_routine_load_alter","p0") { alterTopicProducer.close() } if (alterTopicAdmin != null) { + try { + alterTopicAdmin.deleteTopics([alterTargetTopic.toString()], + new DeleteTopicsOptions().timeoutMs(10000)).all().get() + } catch (Exception e) { + logger.warn("failed to delete kafka topic ${alterTargetTopic}: ${e.message}".toString()) + } alterTopicAdmin.close() } sql "truncate table ${srcTableName}" From 4a34f9f7b3fb56ff6cd7cf4aaa61295ea70c6dda Mon Sep 17 00:00:00 2001 From: Refrain Date: Wed, 15 Jul 2026 10:24:12 +0800 Subject: [PATCH 08/19] [feature](routine-load) Support composable target table alteration ### What problem does this PR solve? Issue Number: None Related PR: #64878 Problem Summary: ALTER ROUTINE LOAD previously used the ON target-table syntax and could not combine a target switch with supported job or data-source properties. Add the explicit SET TARGET TABLE syntax, preserve Doris property constraints, validate and apply target/job/source changes atomically, and keep unspecified Kafka and Kinesis default positions unchanged. ### Release note ALTER ROUTINE LOAD now uses SET TARGET TABLE = "table" and can combine a target table switch with supported PROPERTIES and FROM data-source properties. ### Check List (For Author) - Test: Regression test / Unit Test / Build - FE routine-load and parser unit tests - load_p0/routine_load/test_routine_load_alter regression test - ./build.sh --fe -j48 - Behavior changed: Yes. ALTER ROUTINE LOAD target-table syntax and composability changed as described above. - Does this need documentation: Yes (tracked by Related PR #64878) --- .../load/routineload/RoutineLoadJob.java | 104 +++++-- .../kafka/KafkaDataSourceProperties.java | 39 ++- .../kafka/KafkaRoutineLoadJob.java | 134 ++++++--- .../kinesis/KinesisDataSourceProperties.java | 8 +- .../kinesis/KinesisRoutineLoadJob.java | 9 +- .../load/NereidsRoutineLoadTaskInfo.java | 7 + .../nereids/parser/LogicalPlanBuilder.java | 32 +- .../commands/AlterRoutineLoadCommand.java | 72 ++--- .../routineload/KafkaRoutineLoadJobTest.java | 278 +++++++++++++++++- .../KinesisRoutineLoadJobTest.java | 87 ++++++ .../load/routineload/RoutineLoadJobTest.java | 31 ++ .../commands/AlterRoutineLoadCommandTest.java | 140 +++++++-- .../org/apache/doris/nereids/DorisLexer.g4 | 1 + .../org/apache/doris/nereids/DorisParser.g4 | 9 +- .../test_routine_load_alter.groovy | 11 +- 15 files changed, 817 insertions(+), 145 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 2452a2dde3c681..6cd8388e0013f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -470,6 +470,12 @@ protected void setRoutineLoadDesc(RoutineLoadDesc routineLoadDesc) { } public void validateTargetTable(Database db, OlapTable targetTable) throws UserException { + validateTargetTable(db, targetTable, Maps.newHashMap(), uniqueKeyUpdateMode); + } + + public void validateTargetTable(Database db, OlapTable targetTable, + Map alteredJobProperties, TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode) + throws UserException { if (isMultiTable) { throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports single-table job"); } @@ -479,17 +485,75 @@ public void validateTargetTable(Database db, OlapTable targetTable) throws UserE } checkMeta(targetTable, new RoutineLoadDesc(columnSeparator, lineDelimiter, columnsInfo, precedingFilter, whereExpr, partitionNamesInfo, deleteCondition, mergeType, sequenceCol)); + validateAlterJobProperties(targetTable, alteredJobProperties, effectiveUniqueKeyUpdateMode); targetTable.readLock(); try { - NereidsStreamLoadPlanner planner = new NereidsStreamLoadPlanner(db, targetTable, - toNereidsRoutineLoadTaskInfo()); + NereidsRoutineLoadTaskInfo taskInfo = toNereidsRoutineLoadTaskInfo(); + taskInfo.applyAlterPropertiesForValidation(alteredJobProperties, effectiveUniqueKeyUpdateMode); + NereidsStreamLoadPlanner planner = new NereidsStreamLoadPlanner(db, targetTable, taskInfo); planner.plan(new TUniqueId(0, 0)); } finally { targetTable.readUnlock(); } } + public void validateAlterJobProperties(OlapTable targetTable, Map alteredJobProperties, + TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode) throws UserException { + if (effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { + return; + } + if (isMultiTable) { + throw new AnalysisException("Partial update is not supported in multi-table load."); + } + if (effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS) { + if (!targetTable.getEnableUniqueKeyMergeOnWrite()) { + throw new AnalysisException("load by PARTIAL_COLUMNS is only supported in unique table MoW"); + } + return; + } + Preconditions.checkState(effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS, + effectiveUniqueKeyUpdateMode); + validateFlexiblePartialUpdateForAlter(targetTable, alteredJobProperties); + } + + public static TUniqueKeyUpdateMode getEffectiveUniqueKeyUpdateMode(TUniqueKeyUpdateMode currentUniqueKeyUpdateMode, + Map alteredJobProperties) { + if (alteredJobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { + return TUniqueKeyUpdateMode.valueOf( + alteredJobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)); + } + if (alteredJobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS) + && Boolean.parseBoolean(alteredJobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) + && currentUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { + return TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS; + } + return currentUniqueKeyUpdateMode; + } + + protected void validateAlterJobPropertiesForMutation(AlterRoutineLoadCommand command) throws UserException { + Map alteredJobProperties = command.getAnalyzedJobProperties(); + TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode = getEffectiveUniqueKeyUpdateMode( + uniqueKeyUpdateMode, alteredJobProperties); + if (effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { + return; + } + + Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId); + if (db == null) { + throw new DdlException("Database not found: " + dbId); + } + long effectiveTableId = command.hasTargetTable() ? command.getTargetTableId() : tableId; + Table table = db.getTableNullable(effectiveTableId); + if (table == null) { + throw new DdlException("Table not found: " + effectiveTableId); + } + if (!(table instanceof OlapTable)) { + throw new DdlException("Partial update is only supported for OLAP tables"); + } + validateAlterJobProperties((OlapTable) table, alteredJobProperties, effectiveUniqueKeyUpdateMode); + } + @Override public long getId() { return id; @@ -2093,10 +2157,6 @@ protected void modifyCommonJobProperties(Map jobProperties) thro if (hasExplicitUniqueKeyUpdateMode) { String modeStr = jobProperties.remove(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); TUniqueKeyUpdateMode newMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode(modeStr); - // Validate flexible partial update constraints when changing to UPDATE_FLEXIBLE_COLUMNS - if (newMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS) { - validateFlexiblePartialUpdateForAlter(); - } this.uniqueKeyUpdateMode = newMode; this.isPartialUpdate = (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS); this.jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); @@ -2120,43 +2180,27 @@ protected void modifyCommonJobProperties(Map jobProperties) thro /** * Validate flexible partial update constraints when altering routine load job. */ - private void validateFlexiblePartialUpdateForAlter() throws UserException { - // Multi-table load does not support flexible partial update - if (isMultiTable) { - throw new DdlException("Flexible partial update is not supported in multi-table load"); - } - - // Get the table to check table-level constraints - Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId); - if (db == null) { - throw new DdlException("Database not found: " + dbId); - } - Table table = db.getTableNullable(tableId); - if (table == null) { - throw new DdlException("Table not found: " + tableId); - } - if (!(table instanceof OlapTable)) { - throw new DdlException("Flexible partial update is only supported for OLAP tables"); - } - OlapTable olapTable = (OlapTable) table; - + private void validateFlexiblePartialUpdateForAlter(OlapTable targetTable, + Map alteredJobProperties) throws UserException { // Validate table-level constraints (MoW, skip_bitmap, light_schema_change, variant columns) - olapTable.validateForFlexiblePartialUpdate(); + targetTable.validateForFlexiblePartialUpdate(); + Map effectiveJobProperties = Maps.newHashMap(jobProperties); + effectiveJobProperties.putAll(alteredJobProperties); // Routine load specific validations // Must use JSON format - String format = this.jobProperties.getOrDefault(FileFormatProperties.PROP_FORMAT, "csv"); + String format = effectiveJobProperties.getOrDefault(FileFormatProperties.PROP_FORMAT, "csv"); if (!"json".equalsIgnoreCase(format)) { throw new DdlException("Flexible partial update only supports JSON format, but current job uses: " + format); } // Cannot use fuzzy_parse - if (Boolean.parseBoolean(this.jobProperties.getOrDefault( + if (Boolean.parseBoolean(effectiveJobProperties.getOrDefault( JsonFileFormatProperties.PROP_FUZZY_PARSE, "false"))) { throw new DdlException("Flexible partial update does not support fuzzy_parse"); } // Cannot use jsonpaths - String jsonPaths = getJsonPaths(); + String jsonPaths = effectiveJobProperties.get(JsonFileFormatProperties.PROP_JSON_PATHS); if (jsonPaths != null && !jsonPaths.isEmpty()) { throw new DdlException("Flexible partial update does not support jsonpaths"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaDataSourceProperties.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaDataSourceProperties.java index 9cab113563da61..de22c57a0096ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaDataSourceProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaDataSourceProperties.java @@ -105,6 +105,14 @@ public class KafkaDataSourceProperties extends AbstractDataSourceProperties { .add(KafkaConfiguration.KAFKA_TEXT_TABLE_NAME_FIELD_INDEX.getName()) .build(); + private static final ImmutableSet CREATE_ONLY_MULTI_TABLE_PROPERTIES_SET = + new ImmutableSet.Builder() + .add(KafkaConfiguration.KAFKA_TABLE_NAME_LOCATION.getName()) + .add(KafkaConfiguration.KAFKA_TABLE_NAME_FORMAT.getName()) + .add(KafkaConfiguration.KAFKA_TEXT_TABLE_NAME_FIELD_DELIMITER.getName()) + .add(KafkaConfiguration.KAFKA_TEXT_TABLE_NAME_FIELD_INDEX.getName()) + .build(); + public KafkaDataSourceProperties(Map dataSourceProperties, boolean multiLoad) { super(dataSourceProperties, multiLoad); } @@ -132,6 +140,14 @@ public void convertAndCheckDataSourceProperties() throws UserException { if (optional.isPresent()) { throw new AnalysisException(optional.get() + " is invalid kafka property or can not be set"); } + if (isAlter()) { + Optional createOnlyProperty = originalDataSourceProperties.keySet().stream() + .filter(CREATE_ONLY_MULTI_TABLE_PROPERTIES_SET::contains).findFirst(); + if (createOnlyProperty.isPresent()) { + throw new AnalysisException(createOnlyProperty.get() + " can only be set when creating " + + "a multi-table routine load job"); + } + } this.brokerList = KafkaConfiguration.KAFKA_BROKER_LIST.getParameterValue(originalDataSourceProperties .get(KafkaConfiguration.KAFKA_BROKER_LIST.getName())); @@ -166,9 +182,23 @@ public void convertAndCheckDataSourceProperties() throws UserException { //check offset List offsets = KafkaConfiguration.KAFKA_OFFSETS.getParameterValue(originalDataSourceProperties .get(KafkaConfiguration.KAFKA_OFFSETS.getName())); - String defaultOffsetString = originalDataSourceProperties + boolean hasCustomDefaultOffset = customKafkaProperties + .containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); + String defaultOffsetString = customKafkaProperties.get(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); + boolean hasDirectDefaultOffset = originalDataSourceProperties + .containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); + String directDefaultOffsetString = originalDataSourceProperties .get(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); - if (CollectionUtils.isNotEmpty(offsets) && StringUtils.isNotBlank(defaultOffsetString)) { + if (hasDirectDefaultOffset) { + if (hasCustomDefaultOffset) { + throw new AnalysisException("Only one of " + KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName() + + " and property." + KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName() + " can be set."); + } + defaultOffsetString = directDefaultOffsetString; + customKafkaProperties.put(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName(), defaultOffsetString); + } + boolean hasDefaultOffset = hasCustomDefaultOffset || hasDirectDefaultOffset; + if (CollectionUtils.isNotEmpty(offsets) && hasDefaultOffset) { throw new AnalysisException("Only one of " + KafkaConfiguration.KAFKA_OFFSETS.getName() + " and " + KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName() + " can be set."); } @@ -176,7 +206,7 @@ public void convertAndCheckDataSourceProperties() throws UserException { checkAndSetMultiLoadProperties(); } if (isAlter() && CollectionUtils.isNotEmpty(partitions) && CollectionUtils.isEmpty(offsets) - && StringUtils.isBlank(defaultOffsetString)) { + && !hasDefaultOffset) { // if this is an alter operation, the partition and (default)offset must be set together. throw new AnalysisException("Must set offset or default offset with partition property"); } @@ -184,6 +214,9 @@ public void convertAndCheckDataSourceProperties() throws UserException { this.isOffsetsForTimes = analyzeKafkaOffsetProperty(offsets); return; } + if (isAlter() && CollectionUtils.isEmpty(partitions) && !hasDefaultOffset) { + return; + } this.isOffsetsForTimes = analyzeKafkaDefaultOffsetProperty(); if (CollectionUtils.isNotEmpty(kafkaPartitionOffsets)) { defaultOffsetString = customKafkaProperties.get(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java index f45a2295b1c0b5..a78f88e8d06359 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java @@ -212,19 +212,25 @@ private void convertCustomProperties(boolean rebuild) throws DdlException { return; } - if (rebuild) { - convertedCustomProperties.clear(); - } + Pair, String> convertedProperties = buildConvertedCustomProperties( + customProperties, kafkaDefaultOffSet); + convertedCustomProperties.clear(); + convertedCustomProperties.putAll(convertedProperties.first); + kafkaDefaultOffSet = convertedProperties.second; + } - SmallFileMgr smallFileMgr = Env.getCurrentEnv().getSmallFileMgr(); - for (Map.Entry entry : customProperties.entrySet()) { + private Pair, String> buildConvertedCustomProperties( + Map sourceProperties, String currentDefaultOffset) throws DdlException { + Map convertedProperties = Maps.newHashMap(); + for (Map.Entry entry : sourceProperties.entrySet()) { if (entry.getValue().startsWith("FILE:")) { // convert FILE:file_name -> FILE:file_id:md5 String file = entry.getValue().substring(entry.getValue().indexOf(":") + 1); + SmallFileMgr smallFileMgr = Env.getCurrentEnv().getSmallFileMgr(); SmallFile smallFile = smallFileMgr.getSmallFile(dbId, KAFKA_FILE_CATALOG, file, true); - convertedCustomProperties.put(entry.getKey(), "FILE:" + smallFile.id + ":" + smallFile.md5); + convertedProperties.put(entry.getKey(), "FILE:" + smallFile.id + ":" + smallFile.md5); } else { - convertedCustomProperties.put(entry.getKey(), entry.getValue()); + convertedProperties.put(entry.getKey(), entry.getValue()); } } @@ -233,14 +239,14 @@ private void convertCustomProperties(boolean rebuild) throws DdlException { // KAFKA_DEFAULT_OFFSETS, and this attribute will be converted into a timestamp during the analyzing phase, // thus losing some information. So we use KAFKA_ORIGIN_DEFAULT_OFFSETS to store the original datetime // formatted KAFKA_DEFAULT_OFFSETS value - if (convertedCustomProperties.containsKey(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())) { - kafkaDefaultOffSet = convertedCustomProperties + String convertedDefaultOffset = currentDefaultOffset; + if (convertedProperties.containsKey(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())) { + convertedDefaultOffset = convertedProperties .remove(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName()); - return; - } - if (convertedCustomProperties.containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName())) { - kafkaDefaultOffSet = convertedCustomProperties.remove(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); + } else if (convertedProperties.containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName())) { + convertedDefaultOffset = convertedProperties.remove(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); } + return Pair.of(convertedProperties, convertedDefaultOffset); } @Override @@ -744,10 +750,6 @@ public Map getCustomProperties() { public void modifyProperties(AlterRoutineLoadCommand command) throws UserException { Map jobProperties = command.getAnalyzedJobProperties(); KafkaDataSourceProperties dataSourceProperties = (KafkaDataSourceProperties) command.getDataSourceProperties(); - if (null != dataSourceProperties) { - // if the partition offset is set by timestamp, convert it to real offset - convertOffset(dataSourceProperties); - } writeLock(); try { @@ -755,6 +757,7 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti throw new DdlException("Only supports modification of PAUSED jobs"); } + validateAlterJobPropertiesForMutation(command); modifyPropertiesInternal(jobProperties, dataSourceProperties); if (command.hasTargetTable()) { this.tableId = command.getTargetTableId(); @@ -768,16 +771,19 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti } } - private void convertOffset(KafkaDataSourceProperties dataSourceProperties) throws UserException { + private void convertOffset(KafkaDataSourceProperties dataSourceProperties, + KafkaDataSourceSnapshot dataSourceSnapshot) throws UserException { List> partitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); if (partitionOffsets.isEmpty()) { return; } List> newOffsets; if (dataSourceProperties.isOffsetsForTimes()) { - newOffsets = KafkaUtil.getOffsetsForTimes(brokerList, topic, convertedCustomProperties, partitionOffsets); + newOffsets = KafkaUtil.getOffsetsForTimes(dataSourceSnapshot.brokerList, dataSourceSnapshot.topic, + dataSourceSnapshot.convertedCustomProperties, partitionOffsets); } else { - newOffsets = KafkaUtil.getRealOffsets(brokerList, topic, convertedCustomProperties, partitionOffsets); + newOffsets = KafkaUtil.getRealOffsets(dataSourceSnapshot.brokerList, dataSourceSnapshot.topic, + dataSourceSnapshot.convertedCustomProperties, partitionOffsets); } dataSourceProperties.setKafkaPartitionOffsets(newOffsets); } @@ -785,26 +791,32 @@ private void convertOffset(KafkaDataSourceProperties dataSourceProperties) throw private void modifyPropertiesInternal(Map jobProperties, KafkaDataSourceProperties dataSourceProperties) throws UserException { + modifyPropertiesInternal(jobProperties, dataSourceProperties, false); + } + + private void modifyPropertiesInternal(Map jobProperties, + KafkaDataSourceProperties dataSourceProperties, + boolean isReplay) + throws UserException { if (null != dataSourceProperties) { + KafkaDataSourceSnapshot dataSourceSnapshot = stageDataSourceProperties(dataSourceProperties); List> kafkaPartitionOffsets = Lists.newArrayList(); - Map customKafkaProperties = Maps.newHashMap(); if (MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { kafkaPartitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); - customKafkaProperties = dataSourceProperties.getCustomKafkaProperties(); - } - - // convertCustomProperties and check partitions before reset progress to make modify operation atomic - if (!customKafkaProperties.isEmpty()) { - this.customProperties.putAll(customKafkaProperties); - convertCustomProperties(true); } - if (!kafkaPartitionOffsets.isEmpty()) { + if (!kafkaPartitionOffsets.isEmpty() + && (!dataSourceSnapshot.resetProgress || CollectionUtils.isNotEmpty(customKafkaPartitions))) { ((KafkaProgress) progress).checkPartitions(kafkaPartitionOffsets); } + if (!isReplay) { + convertOffset(dataSourceProperties, dataSourceSnapshot); + kafkaPartitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); + } - if (Config.isCloudMode()) { + if (Config.isCloudMode() && !isReplay + && (dataSourceSnapshot.resetProgress || !kafkaPartitionOffsets.isEmpty())) { Cloud.ResetRLProgressRequest.Builder builder = Cloud.ResetRLProgressRequest.newBuilder() .setRequestIp(FrontendOptions.getLocalHostAddressCached()); builder.setCloudUniqueId(Config.cloud_unique_id); @@ -824,10 +836,18 @@ private void modifyPropertiesInternal(Map jobProperties, resetCloudProgress(builder); } + if (dataSourceSnapshot.customPropertiesChanged) { + this.customProperties.clear(); + this.customProperties.putAll(dataSourceSnapshot.customProperties); + this.convertedCustomProperties.clear(); + this.convertedCustomProperties.putAll(dataSourceSnapshot.convertedCustomProperties); + this.kafkaDefaultOffSet = dataSourceSnapshot.kafkaDefaultOffset; + } + // It is necessary to reset the Kafka progress cache if topic change, // and should reset cache before modifying partition offset. - if (!Strings.isNullOrEmpty(dataSourceProperties.getTopic())) { - this.topic = dataSourceProperties.getTopic(); + if (dataSourceSnapshot.resetProgress) { + this.topic = dataSourceSnapshot.topic; this.progress = new KafkaProgress(); } @@ -839,7 +859,7 @@ private void modifyPropertiesInternal(Map jobProperties, // modify broker list if (!Strings.isNullOrEmpty(dataSourceProperties.getBrokerList())) { - this.brokerList = dataSourceProperties.getBrokerList(); + this.brokerList = dataSourceSnapshot.brokerList; } } if (!jobProperties.isEmpty()) { @@ -859,6 +879,51 @@ private void modifyPropertiesInternal(Map jobProperties, this.id, jobProperties, dataSourceProperties); } + private KafkaDataSourceSnapshot stageDataSourceProperties(KafkaDataSourceProperties dataSourceProperties) + throws DdlException { + Map stagedCustomProperties = Maps.newHashMap(customProperties); + Map stagedConvertedCustomProperties = Maps.newHashMap(convertedCustomProperties); + String stagedKafkaDefaultOffset = kafkaDefaultOffSet; + Map alteredCustomProperties = dataSourceProperties.getCustomKafkaProperties(); + boolean customPropertiesChanged = MapUtils.isNotEmpty(alteredCustomProperties); + if (customPropertiesChanged) { + stagedCustomProperties.putAll(alteredCustomProperties); + Pair, String> convertedProperties = buildConvertedCustomProperties( + stagedCustomProperties, stagedKafkaDefaultOffset); + stagedConvertedCustomProperties = convertedProperties.first; + stagedKafkaDefaultOffset = convertedProperties.second; + } + + boolean resetProgress = !Strings.isNullOrEmpty(dataSourceProperties.getTopic()); + String stagedTopic = resetProgress ? dataSourceProperties.getTopic() : topic; + String stagedBrokerList = Strings.isNullOrEmpty(dataSourceProperties.getBrokerList()) + ? brokerList : dataSourceProperties.getBrokerList(); + return new KafkaDataSourceSnapshot(stagedBrokerList, stagedTopic, stagedCustomProperties, + stagedConvertedCustomProperties, stagedKafkaDefaultOffset, customPropertiesChanged, resetProgress); + } + + private static class KafkaDataSourceSnapshot { + private final String brokerList; + private final String topic; + private final Map customProperties; + private final Map convertedCustomProperties; + private final String kafkaDefaultOffset; + private final boolean customPropertiesChanged; + private final boolean resetProgress; + + private KafkaDataSourceSnapshot(String brokerList, String topic, Map customProperties, + Map convertedCustomProperties, String kafkaDefaultOffset, + boolean customPropertiesChanged, boolean resetProgress) { + this.brokerList = brokerList; + this.topic = topic; + this.customProperties = customProperties; + this.convertedCustomProperties = convertedCustomProperties; + this.kafkaDefaultOffset = kafkaDefaultOffset; + this.customPropertiesChanged = customPropertiesChanged; + this.resetProgress = resetProgress; + } + } + private void resetCloudProgress(Cloud.ResetRLProgressRequest.Builder builder) throws DdlException { Cloud.ResetRLProgressResponse response; try { @@ -881,7 +946,8 @@ private void resetCloudProgress(Cloud.ResetRLProgressRequest.Builder builder) th @Override public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { try { - modifyPropertiesInternal(log.getJobProperties(), (KafkaDataSourceProperties) log.getDataSourceProperties()); + modifyPropertiesInternal(log.getJobProperties(), + (KafkaDataSourceProperties) log.getDataSourceProperties(), true); if (log.getTargetTableId() != 0) { this.tableId = log.getTargetTableId(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisDataSourceProperties.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisDataSourceProperties.java index edc6cd891fd739..047af58cf576ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisDataSourceProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisDataSourceProperties.java @@ -212,17 +212,18 @@ public void convertAndCheckDataSourceProperties() throws UserException { List positions = KinesisConfiguration.KINESIS_POSITIONS.getParameterValue( originalDataSourceProperties.get(KinesisConfiguration.KINESIS_POSITIONS.getName())); // Get default position from customKinesisProperties (already parsed from "property." prefix) + boolean hasDefaultPosition = customKinesisProperties.containsKey("kinesis_default_pos"); String defaultPositionString = customKinesisProperties.get("kinesis_default_pos"); // Validate that positions and default_position are not both set - if (CollectionUtils.isNotEmpty(positions) && StringUtils.isNotBlank(defaultPositionString)) { + if (CollectionUtils.isNotEmpty(positions) && hasDefaultPosition) { throw new AnalysisException("Only one of " + KinesisConfiguration.KINESIS_POSITIONS.getName() + " and property.kinesis_default_pos can be set."); } // For alter operation, shards and positions must be set together if (isAlter() && CollectionUtils.isNotEmpty(shards) && CollectionUtils.isEmpty(positions) - && StringUtils.isBlank(defaultPositionString)) { + && !hasDefaultPosition) { throw new AnalysisException("Must set position or default position with shard property"); } @@ -231,6 +232,9 @@ public void convertAndCheckDataSourceProperties() throws UserException { this.isPositionsForTimes = analyzeKinesisPositionProperty(positions); return; } + if (isAlter() && CollectionUtils.isEmpty(shards) && !hasDefaultPosition) { + return; + } this.isPositionsForTimes = analyzeKinesisDefaultPositionProperty(); if (CollectionUtils.isNotEmpty(kinesisShardPositions)) { setDefaultPositionForShards(this.kinesisShardPositions, defaultPositionString, this.isPositionsForTimes); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index 4f97857ff7a0a7..8aa16f2bc7bd97 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -687,6 +687,7 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti throw new DdlException("Only supports modification of PAUSED jobs"); } + validateAlterJobPropertiesForMutation(command); modifyPropertiesInternal(jobProperties, dataSourceProperties); if (command.hasTargetTable()) { this.tableId = command.getTargetTableId(); @@ -714,6 +715,10 @@ private void modifyPropertiesInternal(Map jobProperties, customKinesisProperties = dataSourceProperties.getCustomKinesisProperties(); hasExplicitShardPositions = !shardPositions.isEmpty(); } + resetProgress = !Strings.isNullOrEmpty(dataSourceProperties.getStream()); + if (hasExplicitShardPositions && !resetProgress) { + ((KinesisProgress) progress).checkShards(shardPositions); + } // Update custom properties if (!customKinesisProperties.isEmpty()) { @@ -724,7 +729,6 @@ private void modifyPropertiesInternal(Map jobProperties, // Modify stream if provided if (!Strings.isNullOrEmpty(dataSourceProperties.getStream())) { this.stream = dataSourceProperties.getStream(); - resetProgress = true; } // Modify region if provided @@ -755,9 +759,6 @@ private void modifyPropertiesInternal(Map jobProperties, } if (!shardPositions.isEmpty()) { - if (!resetProgress) { - ((KinesisProgress) progress).checkShards(shardPositions); - } ((KinesisProgress) progress).modifyPosition(shardPositions); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsRoutineLoadTaskInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsRoutineLoadTaskInfo.java index ef159cfb6f494a..c2ad889ca36681 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsRoutineLoadTaskInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsRoutineLoadTaskInfo.java @@ -121,6 +121,13 @@ public int calTimeoutSec() { return realTimeoutSec; } + /** Apply analyzed ALTER properties to this validation-only task. */ + public void applyAlterPropertiesForValidation(Map alteredJobProperties, + TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode) { + jobProperties.putAll(alteredJobProperties); + uniquekeyUpdateMode = effectiveUniqueKeyUpdateMode; + } + @Override public int getTimeout() { return this.timeoutSec; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 0b6f5f32c8a564..c4909e1e421285 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -3729,6 +3729,15 @@ public Literal visitIntegerLiteral(IntegerLiteralContext ctx) { @Override public Literal visitStringLiteral(StringLiteralContext ctx) { String txt = ctx.STRING_LITERAL().getText(); + String s = decodeStringLiteral(txt); + int strLength = Utils.containChinese(s) ? s.length() * StringLikeLiteral.CHINESE_CHAR_BYTE_LENGTH : s.length(); + if (strLength > ScalarType.MAX_VARCHAR_LENGTH) { + return new StringLiteral(s); + } + return new VarcharLiteral(s, strLength); + } + + private String decodeStringLiteral(String txt) { String s = txt.substring(1, txt.length() - 1); if (txt.charAt(0) == '\'') { // for single quote string, '' should be converted to ' @@ -3740,11 +3749,14 @@ public Literal visitStringLiteral(StringLiteralContext ctx) { if (!SqlModeHelper.hasNoBackSlashEscapes()) { s = LogicalPlanBuilderAssistant.escapeBackSlash(s); } - int strLength = Utils.containChinese(s) ? s.length() * StringLikeLiteral.CHINESE_CHAR_BYTE_LENGTH : s.length(); - if (strLength > ScalarType.MAX_VARCHAR_LENGTH) { - return new StringLiteral(s); + return s; + } + + private String decodeIdentifier(String txt) { + if (txt.charAt(0) == '`' && txt.charAt(txt.length() - 1) == '`') { + return txt.substring(1, txt.length() - 1).replace("``", "`"); } - return new VarcharLiteral(s, strLength); + return txt; } @Override @@ -9283,12 +9295,7 @@ public LogicalPlan visitAlterRoutineLoad(DorisParser.AlterRoutineLoadContext ctx throw new ParseException("only support [.]", ctx.name); } LabelNameInfo labelNameInfo = new LabelNameInfo(dbName, jobName); - - // TODO: Phase 1 only supports altering the target table. Phase 2 will allow altering - // the target table and routine load properties in the same statement. - if (ctx.table != null) { - return new AlterRoutineLoadCommand(labelNameInfo, ctx.table.getText()); - } + String targetTableName = ctx.targetTable == null ? null : decodeStringLiteral(ctx.targetTable.getText()); Map properties = new HashMap<>(); if (ctx.properties != null) { @@ -9315,8 +9322,9 @@ public LogicalPlan visitAlterRoutineLoad(DorisParser.AlterRoutineLoadContext ctx } } - return new AlterRoutineLoadCommand(labelNameInfo, null, - loadPropertyMap, properties, dataSourceMapProperties); + String dataSourceType = ctx.type == null ? null : decodeIdentifier(ctx.type.getText()); + return new AlterRoutineLoadCommand(labelNameInfo, targetTableName, + loadPropertyMap, properties, dataSourceType, dataSourceMapProperties); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java index 6fad5299a477ba..ea8cc19d80627f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java @@ -58,6 +58,7 @@ /** * ALTER ROUTINE LOAD db.label + * SET TARGET TABLE = "table" * PROPERTIES( * ... * ) @@ -95,6 +96,7 @@ public class AlterRoutineLoadCommand extends AlterCommand { private final Map loadPropertyMap; private RoutineLoadDesc routineLoadDesc; private final Map jobProperties; + private final String dataSourceType; private final Map dataSourceMapProperties; private long targetTableId; private boolean isPartialUpdate; @@ -111,6 +113,7 @@ public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, String targetTableName, Map loadPropertyMap, Map jobProperties, + String dataSourceType, Map dataSourceMapProperties) { super(PlanType.ALTER_ROUTINE_LOAD_COMMAND); Objects.requireNonNull(labelNameInfo, "labelNameInfo is null"); @@ -120,6 +123,7 @@ public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, this.targetTableName = targetTableName; this.loadPropertyMap = loadPropertyMap == null ? Maps.newHashMap() : loadPropertyMap; this.jobProperties = jobProperties; + this.dataSourceType = dataSourceType; this.dataSourceMapProperties = dataSourceMapProperties; this.isPartialUpdate = this.jobProperties.getOrDefault(CreateRoutineLoadInfo.PARTIAL_COLUMNS, "false") .equalsIgnoreCase("true"); @@ -128,11 +132,11 @@ public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, Map jobProperties, Map dataSourceMapProperties) { - this(labelNameInfo, null, Maps.newHashMap(), jobProperties, dataSourceMapProperties); + this(labelNameInfo, null, Maps.newHashMap(), jobProperties, null, dataSourceMapProperties); } public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, String targetTableName) { - this(labelNameInfo, targetTableName, Maps.newHashMap(), Maps.newHashMap(), Maps.newHashMap()); + this(labelNameInfo, targetTableName, Maps.newHashMap(), Maps.newHashMap(), null, Maps.newHashMap()); } public String getDbName() { @@ -148,7 +152,7 @@ public Map getAnalyzedJobProperties() { } public boolean hasTargetTable() { - return !StringUtil.isEmpty(targetTableName); + return targetTableName != null; } public String getTargetTableName() { @@ -187,10 +191,6 @@ public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { public void validate(ConnectContext ctx) throws UserException { labelNameInfo.validate(ctx); FeNameFormat.checkCommonName(NAME_TYPE, labelNameInfo.getLabel()); - if (hasTargetTable() && (MapUtils.isNotEmpty(loadPropertyMap) || MapUtils.isNotEmpty(jobProperties) - || MapUtils.isNotEmpty(dataSourceMapProperties))) { - throw new AnalysisException("ALTER ROUTINE LOAD target table change does not support other properties"); - } // check routine load job properties include desired concurrent number etc. checkJobProperties(); // check load properties @@ -201,11 +201,13 @@ public void validate(ConnectContext ctx) throws UserException { job.getDbFullName(), job.getTableName(), job.isMultiTable(), job.getMergeType()); } // check data source properties - checkDataSourceProperties(); + checkDataSourceProperties(job); + TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode = getEffectiveUniqueKeyUpdateMode(job); if (hasTargetTable()) { - validateTargetTable(ctx, job); + validateTargetTable(ctx, job, effectiveUniqueKeyUpdateMode); + } else { + validateAlterJobProperties(job, effectiveUniqueKeyUpdateMode); } - checkPartialUpdate(); if (analyzedJobProperties.isEmpty() && MapUtils.isEmpty(dataSourceMapProperties) && MapUtils.isEmpty(loadPropertyMap) && !hasTargetTable()) { throw new AnalysisException("No properties are specified"); @@ -350,50 +352,46 @@ private void checkJobProperties() throws UserException { } } - private void checkDataSourceProperties() throws UserException { + private void checkDataSourceProperties(RoutineLoadJob job) throws UserException { if (MapUtils.isEmpty(dataSourceMapProperties)) { return; } - RoutineLoadJob job = Env.getCurrentEnv().getRoutineLoadManager() - .getJob(getDbName(), getJobName()); + String effectiveDataSourceType = dataSourceType == null ? job.getDataSourceType().name() : dataSourceType; + if (!effectiveDataSourceType.equalsIgnoreCase(job.getDataSourceType().name())) { + throw new AnalysisException("The specified job type is not: " + effectiveDataSourceType); + } this.dataSourceProperties = RoutineLoadDataSourcePropertyFactory - .createDataSource(job.getDataSourceType().name(), dataSourceMapProperties, job.isMultiTable()); + .createDataSource(effectiveDataSourceType, dataSourceMapProperties, job.isMultiTable()); dataSourceProperties.setAlter(true); - dataSourceProperties.setTimezone(job.getTimezone()); + dataSourceProperties.setTimezone(analyzedJobProperties.getOrDefault( + CreateRoutineLoadInfo.TIMEZONE, job.getTimezone())); dataSourceProperties.analyze(); } - private void checkPartialUpdate() throws UserException { - RoutineLoadJob job = Env.getCurrentEnv().getRoutineLoadManager() - .getJob(getDbName(), getJobName()); - TUniqueKeyUpdateMode uniqueKeyUpdateMode = getEffectiveUniqueKeyUpdateMode(job); - if (uniqueKeyUpdateMode != TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS) { + private void validateAlterJobProperties(RoutineLoadJob job, + TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode) throws UserException { + if (effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { return; } if (job.isMultiTable()) { - throw new AnalysisException("load by PARTIAL_COLUMNS is not supported in multi-table load."); + throw new AnalysisException("Partial update is not supported in multi-table load."); } Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException(job.getDbFullName()); - String tableName = hasTargetTable() ? targetTableName : job.getTableName(); - Table table = db.getTableOrAnalysisException(tableName); - if (!((OlapTable) table).getEnableUniqueKeyMergeOnWrite()) { + Table table = db.getTableOrAnalysisException(job.getTableName()); + if (effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS + && !((OlapTable) table).getEnableUniqueKeyMergeOnWrite()) { throw new AnalysisException("load by PARTIAL_COLUMNS is only supported in unique table MoW"); } + job.validateAlterJobProperties((OlapTable) table, analyzedJobProperties, effectiveUniqueKeyUpdateMode); } private TUniqueKeyUpdateMode getEffectiveUniqueKeyUpdateMode(RoutineLoadJob job) { - if (analyzedJobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { - return TUniqueKeyUpdateMode.valueOf( - analyzedJobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)); - } - if (analyzedJobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS) && isPartialUpdate - && job.getUniqueKeyUpdateMode() == TUniqueKeyUpdateMode.UPSERT) { - return TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS; - } - return job.getUniqueKeyUpdateMode(); + return RoutineLoadJob.getEffectiveUniqueKeyUpdateMode( + job.getUniqueKeyUpdateMode(), analyzedJobProperties); } - private void validateTargetTable(ConnectContext ctx, RoutineLoadJob job) throws UserException { + private void validateTargetTable(ConnectContext ctx, RoutineLoadJob job, + TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode) throws UserException { if (job.isMultiTable()) { throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports single-table job"); } @@ -416,7 +414,11 @@ private void validateTargetTable(ConnectContext ctx, RoutineLoadJob job) throws throw new AnalysisException( "if load_to_single_tablet set to true, the olap table must be with random distribution"); } - job.validateTargetTable(db, olapTable); + if (effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS + && !olapTable.getEnableUniqueKeyMergeOnWrite()) { + throw new AnalysisException("load by PARTIAL_COLUMNS is only supported in unique table MoW"); + } + job.validateTargetTable(db, olapTable, analyzedJobProperties, effectiveUniqueKeyUpdateMode); this.targetTableId = olapTable.getId(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index 069f57199550c7..bae82063049c2e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -25,11 +25,15 @@ import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.info.PartitionNamesInfo; +import org.apache.doris.cloud.proto.Cloud; +import org.apache.doris.cloud.rpc.MetaServiceProxy; import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.common.util.SmallFileMgr; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.kafka.KafkaUtil; import org.apache.doris.load.RoutineLoadDesc; @@ -40,16 +44,19 @@ import org.apache.doris.load.routineload.kafka.KafkaRoutineLoadJob; import org.apache.doris.load.routineload.kafka.KafkaTaskInfo; import org.apache.doris.mysql.privilege.MockedAuth; +import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo; import org.apache.doris.nereids.trees.plans.commands.load.LoadProperty; import org.apache.doris.nereids.trees.plans.commands.load.LoadSeparator; import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; +import org.apache.doris.persist.EditLog; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TResourceInfo; import org.apache.doris.thrift.TUniqueKeyUpdateMode; import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.kafka.common.PartitionInfo; @@ -68,6 +75,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class KafkaRoutineLoadJobTest { private static final Logger LOG = LogManager.getLogger(KafkaRoutineLoadJobTest.class); @@ -217,6 +225,244 @@ public void testModifyPropertiesHonorsExplicitUniqueKeyUpdateModePrecedence() th Assert.assertFalse(routineLoadJob.isFixedPartialUpdate()); } + @Test + public void testModifyPropertiesRevalidatesWhileHoldingWriteLock() throws Exception { + KafkaRoutineLoadJob routineLoadJob = Mockito.spy(new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, + 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN)); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + + Mockito.doAnswer(invocation -> { + ReentrantReadWriteLock lock = Deencapsulation.getField(routineLoadJob, "lock"); + Assert.assertTrue(lock.isWriteLockedByCurrentThread()); + throw new DdlException("stop after validation"); + }).when(routineLoadJob).validateAlterJobPropertiesForMutation(command); + + try { + routineLoadJob.modifyProperties(command); + Assert.fail("Expected mutation-time validation to fail"); + } catch (DdlException e) { + Assert.assertTrue(e.getMessage().contains("stop after validation")); + } + } + + @Test + public void testModifyPropertiesKeepsKafkaPropertiesWhenFileConversionFails() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, + 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + Map originalCustomProperties = Maps.newHashMap(); + originalCustomProperties.put("client.id", "original-client"); + Map originalConvertedProperties = Maps.newHashMap(originalCustomProperties); + Deencapsulation.setField(routineLoadJob, "customProperties", originalCustomProperties); + Deencapsulation.setField(routineLoadJob, "convertedCustomProperties", originalConvertedProperties); + + Map alteredSourceProperties = Maps.newHashMap(); + alteredSourceProperties.put("property.ssl.ca.location", "FILE:missing.pem"); + KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties(alteredSourceProperties); + AlterRoutineLoadCommand command = mockAlterCommand(dataSourceProperties); + Env env = Mockito.mock(Env.class); + SmallFileMgr smallFileMgr = Mockito.mock(SmallFileMgr.class); + Mockito.when(env.getSmallFileMgr()).thenReturn(smallFileMgr); + Mockito.when(smallFileMgr.getSmallFile(1L, KafkaRoutineLoadJob.KAFKA_FILE_CATALOG, + "missing.pem", true)).thenThrow(new DdlException("missing file")); + + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + try { + routineLoadJob.modifyProperties(command); + Assert.fail("Expected missing small file to reject ALTER"); + } catch (DdlException e) { + Assert.assertTrue(e.getMessage().contains("missing file")); + } + } + + Assert.assertEquals(Maps.newHashMap(ImmutableMap.of("client.id", "original-client")), + Deencapsulation.getField(routineLoadJob, "customProperties")); + Assert.assertEquals(Maps.newHashMap(ImmutableMap.of("client.id", "original-client")), + routineLoadJob.getConvertedCustomProperties()); + } + + @Test + public void testModifyPropertiesKeepsKafkaPropertiesWhenPartitionValidationFails() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, + 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + Deencapsulation.setField(routineLoadJob, "progress", + new KafkaProgress(Maps.newHashMap(ImmutableMap.of(1, 10L)))); + Deencapsulation.setField(routineLoadJob, "customProperties", + Maps.newHashMap(ImmutableMap.of("client.id", "original-client"))); + Deencapsulation.setField(routineLoadJob, "convertedCustomProperties", + Maps.newHashMap(ImmutableMap.of("client.id", "original-client"))); + + Map alteredSourceProperties = Maps.newHashMap(); + alteredSourceProperties.put("property.client.id", "new-client"); + alteredSourceProperties.put(KafkaConfiguration.KAFKA_PARTITIONS.getName(), "2"); + alteredSourceProperties.put(KafkaConfiguration.KAFKA_OFFSETS.getName(), "100"); + KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties(alteredSourceProperties); + + try { + routineLoadJob.modifyProperties(mockAlterCommand(dataSourceProperties)); + Assert.fail("Expected unknown partition to reject ALTER"); + } catch (DdlException e) { + Assert.assertTrue(e.getMessage().contains("2")); + } + + Assert.assertEquals("original-client", + Deencapsulation.>getField(routineLoadJob, "customProperties").get("client.id")); + Assert.assertEquals("original-client", routineLoadJob.getConvertedCustomProperties().get("client.id")); + } + + @Test + public void testModifyTopicRejectsPartitionOutsidePinnedSet() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, + 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + KafkaProgress progress = new KafkaProgress(Maps.newHashMap(ImmutableMap.of(1, 10L))); + Deencapsulation.setField(routineLoadJob, "progress", progress); + Deencapsulation.setField(routineLoadJob, "customKafkaPartitions", Lists.newArrayList(1)); + + Map alteredSourceProperties = Maps.newHashMap(); + alteredSourceProperties.put(KafkaConfiguration.KAFKA_TOPIC.getName(), "topic2"); + alteredSourceProperties.put(KafkaConfiguration.KAFKA_PARTITIONS.getName(), "2"); + alteredSourceProperties.put(KafkaConfiguration.KAFKA_OFFSETS.getName(), "100"); + KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties(alteredSourceProperties); + + DdlException exception = Assert.assertThrows(DdlException.class, + () -> routineLoadJob.modifyProperties(mockAlterCommand(dataSourceProperties))); + Assert.assertTrue(exception.getMessage().contains("2")); + Assert.assertEquals("topic1", routineLoadJob.getTopic()); + Assert.assertSame(progress, routineLoadJob.getProgress()); + Assert.assertEquals(Lists.newArrayList(1), + Deencapsulation.getField(routineLoadJob, "customKafkaPartitions")); + } + + @Test + public void testModifyPropertiesResolvesOffsetsWithEffectiveKafkaConfiguration() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, + 1L, "old-broker:9092", "old-topic", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + Map alteredSourceProperties = Maps.newHashMap(); + alteredSourceProperties.put(KafkaConfiguration.KAFKA_BROKER_LIST.getName(), "new-broker:9092"); + alteredSourceProperties.put(KafkaConfiguration.KAFKA_TOPIC.getName(), "new-topic"); + alteredSourceProperties.put(KafkaConfiguration.KAFKA_PARTITIONS.getName(), "7"); + alteredSourceProperties.put(KafkaConfiguration.KAFKA_OFFSETS.getName(), "2026-01-01 00:00:00"); + alteredSourceProperties.put("property.client.id", "new-client"); + KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties(alteredSourceProperties); + AlterRoutineLoadCommand command = mockAlterCommand(dataSourceProperties); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getEditLog()).thenReturn(Mockito.mock(EditLog.class)); + + String originalCloudUniqueId = Config.cloud_unique_id; + String originalDeployMode = Config.deploy_mode; + Config.cloud_unique_id = ""; + Config.deploy_mode = ""; + try (MockedStatic envStatic = Mockito.mockStatic(Env.class); + MockedStatic kafkaUtilStatic = Mockito.mockStatic(KafkaUtil.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + kafkaUtilStatic.when(() -> KafkaUtil.getOffsetsForTimes(Mockito.eq("new-broker:9092"), + Mockito.eq("new-topic"), Mockito.anyMap(), Mockito.anyList())).thenAnswer(invocation -> { + Map effectiveCustomProperties = invocation.getArgument(2); + Assert.assertEquals("new-client", effectiveCustomProperties.get("client.id")); + return Lists.newArrayList(Pair.of(7, 700L)); + }); + + routineLoadJob.modifyProperties(command); + } finally { + Config.cloud_unique_id = originalCloudUniqueId; + Config.deploy_mode = originalDeployMode; + } + + Assert.assertEquals("new-broker:9092", routineLoadJob.getBrokerList()); + Assert.assertEquals("new-topic", routineLoadJob.getTopic()); + Assert.assertEquals(Long.valueOf(700L), ((KafkaProgress) routineLoadJob.getProgress()).getOffsetByPartition(7)); + Assert.assertEquals("new-client", routineLoadJob.getCustomProperties().get("property.client.id")); + } + + @Test + public void testModifyCustomKafkaPropertiesDoesNotResetCloudProgress() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, + 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + KafkaProgress progress = new KafkaProgress(Maps.newHashMap(ImmutableMap.of(1, 123L))); + Deencapsulation.setField(routineLoadJob, "progress", progress); + Deencapsulation.setField(routineLoadJob, "customProperties", Maps.newHashMap( + ImmutableMap.of(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName(), KafkaProgress.OFFSET_BEGINNING))); + Deencapsulation.setField(routineLoadJob, "kafkaDefaultOffSet", KafkaProgress.OFFSET_BEGINNING); + + KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties( + Maps.newHashMap(ImmutableMap.of("property.client.id", "new-client"))); + Assert.assertFalse(dataSourceProperties.getCustomKafkaProperties() + .containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName())); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getEditLog()).thenReturn(Mockito.mock(EditLog.class)); + MetaServiceProxy metaServiceProxy = Mockito.mock(MetaServiceProxy.class); + + String originalCloudUniqueId = Config.cloud_unique_id; + Config.cloud_unique_id = "test-cloud"; + try (MockedStatic envStatic = Mockito.mockStatic(Env.class); + MockedStatic metaServiceProxyStatic = Mockito.mockStatic(MetaServiceProxy.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + metaServiceProxyStatic.when(MetaServiceProxy::getInstance).thenReturn(metaServiceProxy); + + routineLoadJob.modifyProperties(mockAlterCommand(dataSourceProperties)); + } finally { + Config.cloud_unique_id = originalCloudUniqueId; + } + + Mockito.verifyNoInteractions(metaServiceProxy); + Assert.assertSame(progress, routineLoadJob.getProgress()); + Assert.assertEquals(Long.valueOf(123L), ((KafkaProgress) routineLoadJob.getProgress()).getOffsetByPartition(1)); + Assert.assertEquals(KafkaProgress.OFFSET_BEGINNING, + routineLoadJob.getCustomProperties().get("property.kafka_default_offsets")); + } + + @Test + public void testAlterRejectsEmptyKafkaDefaultOffset() { + Map alteredSourceProperties = Maps.newHashMap(); + alteredSourceProperties.put("property.kafka_default_offsets", ""); + + Assert.assertThrows(UserException.class, + () -> analyzedAlterDataSourceProperties(alteredSourceProperties)); + } + + @Test + public void testModifyKafkaTopicResetsCloudProgressWithoutOffsets() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, + 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + Deencapsulation.setField(routineLoadJob, "progress", + new KafkaProgress(Maps.newHashMap(ImmutableMap.of(1, 123L)))); + + KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties( + Maps.newHashMap(ImmutableMap.of(KafkaConfiguration.KAFKA_TOPIC.getName(), "topic2"))); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getEditLog()).thenReturn(Mockito.mock(EditLog.class)); + MetaServiceProxy metaServiceProxy = Mockito.mock(MetaServiceProxy.class); + Cloud.ResetRLProgressResponse response = Cloud.ResetRLProgressResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder().setCode(Cloud.MetaServiceCode.OK)) + .build(); + Mockito.when(metaServiceProxy.resetRLProgress(Mockito.any())).thenReturn(response); + + String originalCloudUniqueId = Config.cloud_unique_id; + Config.cloud_unique_id = "test-cloud"; + try (MockedStatic envStatic = Mockito.mockStatic(Env.class); + MockedStatic metaServiceProxyStatic = Mockito.mockStatic(MetaServiceProxy.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + metaServiceProxyStatic.when(MetaServiceProxy::getInstance).thenReturn(metaServiceProxy); + + routineLoadJob.modifyProperties(mockAlterCommand(dataSourceProperties)); + } finally { + Config.cloud_unique_id = originalCloudUniqueId; + } + + Mockito.verify(metaServiceProxy).resetRLProgress(Mockito.argThat( + request -> request.getPartitionToOffsetMap().isEmpty())); + Assert.assertEquals("topic2", routineLoadJob.getTopic()); + Assert.assertTrue(((KafkaProgress) routineLoadJob.getProgress()).getPartitionOffsetPairs(false).isEmpty()); + } + @Test public void testUpdateLagRebuildsConvertedPropertiesAfterReplay() throws UserException { KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, @@ -436,7 +682,7 @@ public void testFromCreateStmt() throws UserException { } @Test - public void testReplayModifyPropertiesSwitchesTargetTableWithoutResettingProgress() { + public void testReplayModifyPropertiesWithDataSourceSwitchesTargetTableInCloudMode() throws Exception { KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, 101L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); Map partitionToOffset = Maps.newHashMap(); @@ -444,13 +690,39 @@ public void testReplayModifyPropertiesSwitchesTargetTableWithoutResettingProgres KafkaProgress progress = new KafkaProgress(partitionToOffset); Deencapsulation.setField(routineLoadJob, "progress", progress); + KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties( + Maps.newHashMap(ImmutableMap.of("property.client.id", "replayed-client"))); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(1L, - Maps.newHashMap(), null, 202L); - routineLoadJob.replayModifyProperties(log); + Maps.newHashMap(), dataSourceProperties, 202L); + + String originalCloudUniqueId = Config.cloud_unique_id; + Config.cloud_unique_id = "replay-cloud"; + try { + routineLoadJob.replayModifyProperties(log); + } finally { + Config.cloud_unique_id = originalCloudUniqueId; + } Assert.assertEquals(202L, routineLoadJob.getTableId()); Assert.assertSame(progress, routineLoadJob.getProgress()); Assert.assertEquals(Long.valueOf(123L), ((KafkaProgress) routineLoadJob.getProgress()).getOffsetByPartition(1)); + Assert.assertEquals("replayed-client", routineLoadJob.getCustomProperties().get("property.client.id")); + } + + private KafkaDataSourceProperties analyzedAlterDataSourceProperties(Map properties) + throws UserException { + KafkaDataSourceProperties dataSourceProperties = new KafkaDataSourceProperties(properties); + dataSourceProperties.setAlter(true); + dataSourceProperties.setTimezone("UTC"); + dataSourceProperties.analyze(); + return dataSourceProperties; + } + + private AlterRoutineLoadCommand mockAlterCommand(KafkaDataSourceProperties dataSourceProperties) { + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + Mockito.when(command.getDataSourceProperties()).thenReturn(dataSourceProperties); + return command; } private CreateRoutineLoadInfo initCreateRoutineLoadInfo() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java index 67688ab2e790e7..fe1f06df02b678 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java @@ -18,13 +18,16 @@ package org.apache.doris.load.routineload; import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.load.routineload.kinesis.KinesisConfiguration; import org.apache.doris.load.routineload.kinesis.KinesisDataSourceProperties; import org.apache.doris.load.routineload.kinesis.KinesisProgress; import org.apache.doris.load.routineload.kinesis.KinesisRoutineLoadJob; import org.apache.doris.load.routineload.kinesis.KinesisTaskInfo; +import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; import org.apache.doris.thrift.TUniqueKeyUpdateMode; @@ -33,6 +36,7 @@ import com.google.gson.Gson; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.util.HashMap; import java.util.HashSet; @@ -40,6 +44,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class KinesisRoutineLoadJobTest { @@ -122,6 +127,28 @@ public void testModifyPropertiesHonorsExplicitUniqueKeyUpdateModePrecedence() th Assert.assertFalse(routineLoadJob.isFixedPartialUpdate()); } + @Test + public void testModifyPropertiesRevalidatesWhileHoldingWriteLock() throws Exception { + KinesisRoutineLoadJob routineLoadJob = Mockito.spy(new KinesisRoutineLoadJob(1L, + "kinesis_routine_load_job", 1L, 1L, "ap-southeast-1", "stream-1", UserIdentity.ADMIN)); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + + Mockito.doAnswer(invocation -> { + ReentrantReadWriteLock lock = Deencapsulation.getField(routineLoadJob, "lock"); + Assert.assertTrue(lock.isWriteLockedByCurrentThread()); + throw new DdlException("stop after validation"); + }).when(routineLoadJob).validateAlterJobPropertiesForMutation(command); + + try { + routineLoadJob.modifyProperties(command); + Assert.fail("Expected mutation-time validation to fail"); + } catch (DdlException e) { + Assert.assertTrue(e.getMessage().contains("stop after validation")); + } + } + @Test public void testHasMoreDataToConsumeShouldKeepPollingWhenLagCacheIsZero() throws Exception { KinesisRoutineLoadJob routineLoadJob = @@ -191,6 +218,10 @@ public void testModifyPropertiesShouldClearStaleCustomShardsWhenStreamChanges() Map oldLag = new HashMap<>(); oldLag.put("shard-old-0", 10L); Deencapsulation.setField(routineLoadJob, "cachedShardWithMillsBehindLatest", oldLag); + Deencapsulation.setField(routineLoadJob, "customProperties", + Maps.newHashMap(Map.of("kinesis_default_pos", KinesisDataSourceProperties.POSITION_TRIM_HORIZON))); + Deencapsulation.setField(routineLoadJob, "kinesisDefaultPosition", + KinesisDataSourceProperties.POSITION_TRIM_HORIZON); Map alterProps = new HashMap<>(); alterProps.put(KinesisConfiguration.KINESIS_STREAM.getName(), "stream-2"); @@ -198,6 +229,7 @@ public void testModifyPropertiesShouldClearStaleCustomShardsWhenStreamChanges() dataSourceProperties.setAlter(true); dataSourceProperties.setTimezone("Asia/Shanghai"); dataSourceProperties.analyze(); + Assert.assertFalse(dataSourceProperties.getCustomKinesisProperties().containsKey("kinesis_default_pos")); Deencapsulation.invoke(routineLoadJob, "modifyPropertiesInternal", new HashMap(), dataSourceProperties); @@ -215,6 +247,19 @@ public void testModifyPropertiesShouldClearStaleCustomShardsWhenStreamChanges() Assert.assertFalse(progress.hasShards()); Map cachedLag = Deencapsulation.getField(routineLoadJob, "cachedShardWithMillsBehindLatest"); Assert.assertTrue(cachedLag.isEmpty()); + Assert.assertEquals(KinesisDataSourceProperties.POSITION_TRIM_HORIZON, + Deencapsulation.getField(routineLoadJob, "kinesisDefaultPosition")); + } + + @Test + public void testAlterRejectsEmptyKinesisDefaultPosition() { + Map alterProperties = Maps.newHashMap(); + alterProperties.put(KinesisConfiguration.KINESIS_DEFAULT_POSITION.getName(), ""); + KinesisDataSourceProperties dataSourceProperties = new KinesisDataSourceProperties(alterProperties); + dataSourceProperties.setAlter(true); + dataSourceProperties.setTimezone("UTC"); + + Assert.assertThrows(AnalysisException.class, dataSourceProperties::analyze); } @Test @@ -250,6 +295,48 @@ public void testModifyPropertiesShouldReplaceCustomShardsWhenExplicitShardsProvi Assert.assertEquals("202", progress.getSequenceNumberByShard("shard-2")); } + @Test + public void testModifyPropertiesKeepsKinesisStateWhenShardValidationFails() throws Exception { + KinesisRoutineLoadJob routineLoadJob = new KinesisRoutineLoadJob(1L, + "kinesis_routine_load_job", 1L, 1L, "ap-southeast-1", "stream-1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + Deencapsulation.setField(routineLoadJob, "customKinesisShards", Lists.newArrayList("shard-0")); + Deencapsulation.setField(routineLoadJob, "progress", + new KinesisProgress(Maps.newHashMap(Map.of("shard-0", "10")))); + Deencapsulation.setField(routineLoadJob, "customProperties", + Maps.newHashMap(Map.of("max_connections", "5"))); + Deencapsulation.setField(routineLoadJob, "convertedCustomProperties", + Maps.newHashMap(Map.of("max_connections", "5"))); + + Map alterProperties = Maps.newHashMap(); + alterProperties.put(KinesisConfiguration.KINESIS_REGION.getName(), "us-east-1"); + alterProperties.put(KinesisConfiguration.KINESIS_SHARDS.getName(), "shard-missing"); + alterProperties.put(KinesisConfiguration.KINESIS_POSITIONS.getName(), "101"); + alterProperties.put("property.max_connections", "10"); + KinesisDataSourceProperties dataSourceProperties = new KinesisDataSourceProperties(alterProperties); + dataSourceProperties.setAlter(true); + dataSourceProperties.setTimezone("UTC"); + dataSourceProperties.analyze(); + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + Mockito.when(command.getDataSourceProperties()).thenReturn(dataSourceProperties); + + try { + routineLoadJob.modifyProperties(command); + Assert.fail("Expected unknown shard to reject ALTER"); + } catch (DdlException e) { + Assert.assertTrue(e.getMessage().contains("shard-missing")); + } + + Assert.assertEquals("ap-southeast-1", routineLoadJob.getRegion()); + Assert.assertEquals(Lists.newArrayList("shard-0"), + Deencapsulation.getField(routineLoadJob, "customKinesisShards")); + Assert.assertEquals("5", + Deencapsulation.>getField(routineLoadJob, "customProperties") + .get("max_connections")); + Assert.assertEquals("5", routineLoadJob.getConvertedCustomProperties().get("max_connections")); + } + @Test public void testShardRefreshShouldMoveRetiredParentToClosedUntilConsumed() throws Exception { KinesisRoutineLoadJob routineLoadJob = diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java index 8876c6a8aea9bc..b253d7d5ed3599 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.common.InternalErrorCode; import org.apache.doris.common.Pair; @@ -27,6 +28,8 @@ import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.kafka.KafkaUtil; +import org.apache.doris.datasource.property.fileformat.FileFormatProperties; +import org.apache.doris.datasource.property.fileformat.JsonFileFormatProperties; import org.apache.doris.load.routineload.kafka.KafkaProgress; import org.apache.doris.load.routineload.kafka.KafkaRoutineLoadJob; import org.apache.doris.load.routineload.kafka.KafkaTaskInfo; @@ -459,4 +462,32 @@ public void testUniqueKeyUpdateModeTakesPrecedenceOverPartialColumns() throws Ex Assert.assertFalse(isPartialUpdate); } + @Test + public void testValidateFlexiblePartialUpdateUsesAlteredProperties() throws Exception { + KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(); + OlapTable targetTable = Mockito.mock(OlapTable.class); + Map currentJobProperties = Maps.newHashMap(); + currentJobProperties.put(FileFormatProperties.PROP_FORMAT, "json"); + currentJobProperties.put(JsonFileFormatProperties.PROP_FUZZY_PARSE, "false"); + currentJobProperties.put(JsonFileFormatProperties.PROP_JSON_PATHS, "$.value"); + Deencapsulation.setField(job, "jobProperties", currentJobProperties); + + Map validAlterProperties = Maps.newHashMap(); + validAlterProperties.put(JsonFileFormatProperties.PROP_JSON_PATHS, ""); + job.validateAlterJobProperties(targetTable, validAlterProperties, + TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS); + + Map invalidAlterProperties = Maps.newHashMap(); + invalidAlterProperties.put(JsonFileFormatProperties.PROP_JSON_PATHS, ""); + invalidAlterProperties.put(JsonFileFormatProperties.PROP_FUZZY_PARSE, "true"); + try { + job.validateAlterJobProperties(targetTable, invalidAlterProperties, + TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS); + Assert.fail("Expected flexible partial update validation to reject fuzzy_parse"); + } catch (UserException e) { + Assert.assertTrue(e.getMessage().contains("fuzzy_parse")); + } + Mockito.verify(targetTable, Mockito.times(2)).validateForFlexiblePartialUpdate(); + } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java index f7fd80622ec1eb..afe162deeb0ff1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java @@ -25,8 +25,11 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.load.loadv2.LoadTask; +import org.apache.doris.load.routineload.LoadDataSourceType; import org.apache.doris.load.routineload.RoutineLoadJob; import org.apache.doris.load.routineload.RoutineLoadManager; +import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; +import org.apache.doris.load.routineload.kinesis.KinesisDataSourceProperties; import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.nereids.exceptions.ParseException; import org.apache.doris.nereids.parser.NereidsParser; @@ -93,6 +96,8 @@ public void setUp() throws Exception { Mockito.when(routineLoadJob.getTableId()).thenReturn(2000L); Mockito.when(routineLoadJob.isMultiTable()).thenReturn(false); Mockito.when(routineLoadJob.getMergeType()).thenReturn(LoadTask.MergeType.APPEND); + Mockito.when(routineLoadJob.getDataSourceType()).thenReturn(LoadDataSourceType.KAFKA); + Mockito.when(routineLoadJob.getTimezone()).thenReturn("UTC"); Mockito.when(routineLoadJob.isLoadToSingleTablet()).thenReturn(false); Mockito.when(routineLoadJob.getUniqueKeyUpdateMode()).thenReturn(TUniqueKeyUpdateMode.UPSERT); Mockito.when(currentTable.getType()).thenReturn(Table.TableType.OLAP); @@ -152,9 +157,21 @@ public void testValidate() { } @Test - public void testParseAlterRoutineLoadOnTargetTable() { + public void testLegacyConstructorInfersDataSourceType() { + runBefore(); + Map dataSourceProperties = Maps.newHashMap(); + dataSourceProperties.put("property.client.id", "alter-client"); + AlterRoutineLoadCommand command = new AlterRoutineLoadCommand( + new LabelNameInfo("testDb", "label1"), Maps.newHashMap(), dataSourceProperties); + + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + Assertions.assertInstanceOf(KafkaDataSourceProperties.class, command.getDataSourceProperties()); + } + + @Test + public void testParseAlterRoutineLoadSetTargetTable() { AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2"); + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); Assertions.assertEquals("testDb", command.getDbName()); Assertions.assertEquals("label1", command.getJobName()); Assertions.assertTrue(command.hasTargetTable()); @@ -162,13 +179,91 @@ public void testParseAlterRoutineLoadOnTargetTable() { } @Test - public void testParseAlterRoutineLoadOnTargetTableRejectMixedProperties() { + public void testParseAlterRoutineLoadDecodesTargetTableString() { + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"test\"\"Table2\""); + + Assertions.assertEquals("test\"Table2", command.getTargetTableName()); + } + + @Test + public void testParseAlterRoutineLoadSetTargetTableRejectsUnsupportedSyntax() { Assertions.assertThrows(ParseException.class, () -> PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2 PROPERTIES(\"max_error_number\"=\"1\")")); + "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2")); Assertions.assertThrows(ParseException.class, () -> PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2 FROM KAFKA(\"kafka_offsets\"=\"100\")")); + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = testTable2")); Assertions.assertThrows(ParseException.class, () -> PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2 COLUMNS(k1)")); + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" COLUMNS(k1)")); + } + + @Test + public void testValidateTargetTableWithJobAndDataSourceProperties() throws Exception { + runBefore(); + Mockito.when(currentTable.getId()).thenReturn(3000L); + mockTargetTable(currentTable); + + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" " + + "PROPERTIES(\"max_error_number\"=\"1\", \"timezone\"=\"Asia/Shanghai\") " + + "FROM `KAFKA`(\"property.client.id\"=\"target-switch\")"); + + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + Assertions.assertEquals("1", command.getAnalyzedJobProperties() + .get(CreateRoutineLoadInfo.MAX_ERROR_NUMBER_PROPERTY)); + Assertions.assertInstanceOf(KafkaDataSourceProperties.class, command.getDataSourceProperties()); + Assertions.assertEquals("target-switch", command.getDataSourceProperties() + .getOriginalDataSourceProperties().get("property.client.id")); + Assertions.assertEquals("Asia/Shanghai", command.getDataSourceProperties().getTimezone()); + Assertions.assertEquals(3000L, command.getTargetTableId()); + } + + @Test + public void testValidateTargetTableWithKinesisProperties() throws Exception { + runBefore(); + Mockito.when(currentTable.getId()).thenReturn(3000L); + Mockito.when(routineLoadJob.getDataSourceType()).thenReturn(LoadDataSourceType.KINESIS); + mockTargetTable(currentTable); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" " + + "FROM KINESIS(\"property.max_connections\"=\"10\")"); + + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + Assertions.assertInstanceOf(KinesisDataSourceProperties.class, command.getDataSourceProperties()); + Assertions.assertEquals(3000L, command.getTargetTableId()); + } + + @Test + public void testValidateTargetTableRejectsDataSourceTypeChange() throws Exception { + runBefore(); + mockTargetTable(currentTable); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" " + + "FROM KINESIS(\"kinesis_region\"=\"us-east-1\")"); + + Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, + () -> command.validate(connectContext)).getMessage().contains("KINESIS")); + } + + @Test + public void testValidateTargetTableRejectsInvalidProperties() throws Exception { + runBefore(); + mockTargetTable(currentTable); + AlterRoutineLoadCommand invalidJobProperty = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" " + + "PROPERTIES(\"format\"=\"json\")"); + AlterRoutineLoadCommand invalidDataSourceProperty = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" " + + "FROM KAFKA(\"invalid_key\"=\"value\")"); + AlterRoutineLoadCommand createOnlyDataSourceProperty = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" " + + "FROM KAFKA(\"kafka_table_name_location\"=\"key\")"); + + Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, + () -> invalidJobProperty.validate(connectContext)).getMessage().contains("invalid property")); + Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, + () -> invalidDataSourceProperty.validate(connectContext)).getMessage().contains("invalid kafka")); + Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, + () -> createOnlyDataSourceProperty.validate(connectContext)).getMessage().contains("only be set")); } @Test @@ -177,7 +272,7 @@ public void testValidateTargetTableOnlyDoesNotRequireOtherProperties() throws Ex mockTargetTable(currentTable); AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2"); + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); Assertions.assertEquals("testTable2", command.getTargetTableName()); } @@ -187,7 +282,7 @@ public void testValidateTargetTableRejectsMultiTableJob() throws Exception { runBefore(); Mockito.when(routineLoadJob.isMultiTable()).thenReturn(true); AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2"); + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) .getMessage().contains("single-table")); @@ -200,7 +295,7 @@ public void testValidateTargetTableRejectsWithoutLoadPrivilege() throws Exceptio Mockito.when(accessManager.checkTblPriv(Mockito.any(ConnectContext.class), Mockito.anyString(), Mockito.eq("testDb"), Mockito.eq("testTable2"), Mockito.any())).thenReturn(false); AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2"); + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) .getMessage().contains("LOAD")); @@ -214,7 +309,7 @@ public void testValidateTargetTableRejectsTemporaryTable() throws Exception { Mockito.when(tempTable.isTemporary()).thenReturn(true); mockTargetTable(tempTable); AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2"); + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) .getMessage().contains("temporary table")); @@ -230,7 +325,7 @@ public void testValidateTargetTableRejectsLoadToSingleTabletWithoutRandomDistrib mockTargetTable(newTable); Mockito.when(routineLoadJob.isLoadToSingleTablet()).thenReturn(true); AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2"); + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) .getMessage().contains("load_to_single_tablet")); @@ -247,7 +342,7 @@ public void testValidateTargetTableAllowsLoadToSingleTabletWithRandomDistributio mockTargetTable(newTable); Mockito.when(routineLoadJob.isLoadToSingleTablet()).thenReturn(true); AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2"); + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); } @@ -260,14 +355,22 @@ public void testValidateTargetTablePassesTargetTableToJobValidation() throws Exc Mockito.when(targetTable.isTemporary()).thenReturn(false); Mockito.doReturn(targetTable).when(db).getTableOrAnalysisException("testTable2"); Mockito.when(routineLoadJob.isLoadToSingleTablet()).thenReturn(false); + Mockito.when(routineLoadJob.getUniqueKeyUpdateMode()) + .thenReturn(TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS); Mockito.doAnswer(invocation -> { Assertions.assertSame(db, invocation.getArgument(0)); Assertions.assertSame(targetTable, invocation.getArgument(1)); + Map alteredJobProperties = invocation.getArgument(2); + Assertions.assertEquals("UPSERT", + alteredJobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)); + Assertions.assertEquals(TUniqueKeyUpdateMode.UPSERT, invocation.getArgument(3)); return null; - }).when(routineLoadJob).validateTargetTable(Mockito.any(Database.class), Mockito.any(OlapTable.class)); + }).when(routineLoadJob).validateTargetTable(Mockito.any(Database.class), Mockito.any(OlapTable.class), + Mockito.anyMap(), Mockito.any(TUniqueKeyUpdateMode.class)); AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2"); + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" " + + "PROPERTIES(\"unique_key_update_mode\"=\"UPSERT\")"); Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); } @@ -282,10 +385,11 @@ public void testValidateTargetTableRejectsExistingFlexiblePartialUpdateOnNonMowT Mockito.doReturn(targetTable).when(db).getTableOrAnalysisException("testTable2"); Mockito.when(routineLoadJob.getUniqueKeyUpdateMode()).thenReturn(TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS); Mockito.doThrow(new AnalysisException("Only unique key merge on write support partial update")) - .when(routineLoadJob).validateTargetTable(Mockito.any(Database.class), Mockito.any(OlapTable.class)); + .when(routineLoadJob).validateTargetTable(Mockito.any(Database.class), Mockito.any(OlapTable.class), + Mockito.anyMap(), Mockito.any(TUniqueKeyUpdateMode.class)); AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2"); + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) .getMessage().contains("partial update")); @@ -322,7 +426,7 @@ public void testValidateAllowsExplicitUpsertToOverridePartialColumnsOnNonMowTabl } @Test - public void testValidateAllowsFlexibleAlterToReachFlexibleValidation() { + public void testValidateAllowsFlexibleAlterToReachFlexibleValidation() throws Exception { runBefore(); Mockito.when(routineLoadJob.getUniqueKeyUpdateMode()).thenReturn(TUniqueKeyUpdateMode.UPSERT); Mockito.when(currentTable.getEnableUniqueKeyMergeOnWrite()).thenReturn(false); @@ -333,5 +437,7 @@ public void testValidateAllowsFlexibleAlterToReachFlexibleValidation() { new LabelNameInfo("testDb", "label1"), jobProperties, Maps.newHashMap()); Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + Mockito.verify(routineLoadJob).validateAlterJobProperties(Mockito.eq(currentTable), Mockito.anyMap(), + Mockito.eq(TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS)); } } diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 446685892295e4..407454f7b091d0 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -562,6 +562,7 @@ TABLESAMPLE: 'TABLESAMPLE'; TABLET: 'TABLET'; TABLETS: 'TABLETS'; TAG: 'TAG'; +TARGET: 'TARGET'; TASK: 'TASK'; TASKS: 'TASKS'; TDE: 'TDE'; diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index d16c184be0a814..a44467b2422554 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -336,11 +336,11 @@ supportedAlterStatement | ALTER REPOSITORY name=identifier properties=propertyClause? #alterRepository | ALTER ROUTINE LOAD FOR name=multipartIdentifier ( - ON table=identifier + SET TARGET TABLE EQ targetTable=STRING_LITERAL | (loadProperty (COMMA loadProperty)*)? - properties=propertyClause? - (FROM type=identifier LEFT_PAREN propertyItemList RIGHT_PAREN)? - ) #alterRoutineLoad + ) + properties=propertyClause? + (FROM type=identifier LEFT_PAREN propertyItemList RIGHT_PAREN)? #alterRoutineLoad | ALTER COLOCATE GROUP name=multipartIdentifier SET LEFT_PAREN propertyItemList RIGHT_PAREN #alterColocateGroup | ALTER USER (IF EXISTS)? grantUserIdentify @@ -2348,6 +2348,7 @@ nonReserved | SUM | TABLES | TAG + | TARGET | TASK | TASKS | TDE diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy index a27035e42f2cf3..6d9bf4aa2ec70d 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy @@ -424,11 +424,20 @@ suite("test_routine_load_alter","p0") { assertEquals(srcTableName, showBeforeAlter[0][6].toString()) def progressBeforeAlter = showBeforeAlter[0][15].toString() - sql "ALTER ROUTINE LOAD FOR ${alterTargetJob} ON ${dstTableName}" + sql """ + ALTER ROUTINE LOAD FOR ${alterTargetJob} + SET TARGET TABLE = "${dstTableName}" + PROPERTIES("max_error_number" = "10") + FROM KAFKA("property.client.id" = "target-switch") + """ def showAfterAlter = sql "show routine load for ${alterTargetJob}" assertEquals(dstTableName, showAfterAlter[0][6].toString()) assertEquals(progressBeforeAlter, showAfterAlter[0][15].toString()) + def alteredJobProperties = new groovy.json.JsonSlurper().parseText(showAfterAlter[0][11].toString()) + def alteredCustomProperties = new groovy.json.JsonSlurper().parseText(showAfterAlter[0][13].toString()) + assertEquals("10", alteredJobProperties.max_error_number.toString()) + assertEquals("target-switch", alteredCustomProperties["client.id"].toString()) def secondBatch = [ "4,eab,2023-07-16,def,2023-07-21:05:48:31,ghi", From 000c8389871358d515e624bcea7ddfceb317cf50 Mon Sep 17 00:00:00 2001 From: Refrain Date: Thu, 16 Jul 2026 13:59:31 +0800 Subject: [PATCH 09/19] [fix](fe) Fix routine load alter review issues ### What problem does this PR solve? Issue Number: None Related PR: #64878 Problem Summary: Routine load target-table alteration validated the target outside the mutation critical section, Kafka datetime default offsets could leak Doris-only properties to librdkafka, and SHOW could combine an old target table name with newly altered job state. Centralize ALTER mutation under the job and target metadata locks, repeat complete target validation before mutation, filter both internal Kafka default-offset properties, and read SHOW state under the job read lock. ### Release note Fix routine load target-table alteration consistency and Kafka datetime default-offset handling. ### Check List (For Author) - Test: Unit Test - `./run-fe-ut.sh --run org.apache.doris.load.routineload.KafkaRoutineLoadJobTest,org.apache.doris.load.routineload.KinesisRoutineLoadJobTest,org.apache.doris.load.routineload.RoutineLoadJobTest` - `./build.sh --fe -j48` - Behavior changed: Yes, target-table ALTER is validated atomically, Doris-only Kafka properties are filtered, and SHOW uses a consistent job snapshot - Does this need documentation: No --- .../load/routineload/RoutineLoadJob.java | 76 ++++++++++++++++-- .../kafka/KafkaRoutineLoadJob.java | 36 +++------ .../kinesis/KinesisRoutineLoadJob.java | 22 +----- .../routineload/KafkaRoutineLoadJobTest.java | 78 ++++++++++++++++++- .../load/routineload/RoutineLoadJobTest.java | 29 +++++++ 5 files changed, 187 insertions(+), 54 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 6cd8388e0013f0..ad4b9b92f9914b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -535,7 +535,7 @@ protected void validateAlterJobPropertiesForMutation(AlterRoutineLoadCommand com Map alteredJobProperties = command.getAnalyzedJobProperties(); TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode = getEffectiveUniqueKeyUpdateMode( uniqueKeyUpdateMode, alteredJobProperties); - if (effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { + if (!command.hasTargetTable() && effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { return; } @@ -549,7 +549,13 @@ protected void validateAlterJobPropertiesForMutation(AlterRoutineLoadCommand com throw new DdlException("Table not found: " + effectiveTableId); } if (!(table instanceof OlapTable)) { - throw new DdlException("Partial update is only supported for OLAP tables"); + throw new DdlException(command.hasTargetTable() + ? "Routine load target table must be an OLAP table" + : "Partial update is only supported for OLAP tables"); + } + if (command.hasTargetTable()) { + validateTargetTable(db, (OlapTable) table, alteredJobProperties, effectiveUniqueKeyUpdateMode); + return; } validateAlterJobProperties((OlapTable) table, alteredJobProperties, effectiveUniqueKeyUpdateMode); } @@ -1765,11 +1771,10 @@ public String getStateReason() { } public List getShowInfo() { - Optional database = Env.getCurrentInternalCatalog().getDb(dbId); - Optional
table = database.flatMap(db -> db.getTable(tableId)); - readLock(); try { + Optional database = Env.getCurrentInternalCatalog().getDb(dbId); + Optional
table = database.flatMap(db -> db.getTable(tableId)); List row = Lists.newArrayList(); row.add(String.valueOf(id)); row.add(name); @@ -1831,6 +1836,15 @@ public List> getTasksShowInfo() throws AnalysisException { } public String getShowCreateInfo() { + readLock(); + try { + return unprotectGetShowCreateInfo(); + } finally { + readUnlock(); + } + } + + private String unprotectGetShowCreateInfo() { Optional database = Env.getCurrentInternalCatalog().getDb(dbId); Optional
table = database.flatMap(db -> db.getTable(tableId)); StringBuilder sb = new StringBuilder(); @@ -2113,7 +2127,57 @@ public void gsonPostProcess() throws IOException { } } - public abstract void modifyProperties(AlterRoutineLoadCommand command) throws UserException; + public void modifyProperties(AlterRoutineLoadCommand command) throws UserException { + writeLock(); + try { + if (getState() != JobState.PAUSED) { + throw new DdlException("Only supports modification of PAUSED jobs"); + } + if (!command.hasTargetTable()) { + unprotectApplyAlter(command); + return; + } + + Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId); + if (db == null) { + throw new DdlException("Database not found: " + dbId); + } + db.readLock(); + try { + Table table = db.getTableNullable(command.getTargetTableId()); + if (table == null) { + throw new DdlException("Table not found: " + command.getTargetTableId()); + } + if (!(table instanceof OlapTable)) { + throw new DdlException("Routine load target table must be an OLAP table"); + } + table.readLock(); + try { + unprotectApplyAlter(command); + } finally { + table.readUnlock(); + } + } finally { + db.readUnlock(); + } + } finally { + writeUnlock(); + } + } + + private void unprotectApplyAlter(AlterRoutineLoadCommand command) throws UserException { + validateAlterJobPropertiesForMutation(command); + unprotectModifyProperties(command); + if (command.hasTargetTable()) { + this.tableId = command.getTargetTableId(); + } + + AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, + command.getAnalyzedJobProperties(), command.getDataSourceProperties(), command.getTargetTableId()); + Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); + } + + protected abstract void unprotectModifyProperties(AlterRoutineLoadCommand command) throws UserException; public abstract void replayModifyProperties(AlterRoutineLoadJobOperationLog log); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java index a78f88e8d06359..2adc6ff0a7be48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java @@ -239,13 +239,13 @@ private Pair, String> buildConvertedCustomProperties( // KAFKA_DEFAULT_OFFSETS, and this attribute will be converted into a timestamp during the analyzing phase, // thus losing some information. So we use KAFKA_ORIGIN_DEFAULT_OFFSETS to store the original datetime // formatted KAFKA_DEFAULT_OFFSETS value - String convertedDefaultOffset = currentDefaultOffset; - if (convertedProperties.containsKey(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())) { - convertedDefaultOffset = convertedProperties - .remove(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName()); - } else if (convertedProperties.containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName())) { - convertedDefaultOffset = convertedProperties.remove(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); - } + String originDefaultOffset = convertedProperties + .remove(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName()); + String analyzedDefaultOffset = convertedProperties + .remove(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); + String convertedDefaultOffset = originDefaultOffset != null + ? originDefaultOffset + : analyzedDefaultOffset != null ? analyzedDefaultOffset : currentDefaultOffset; return Pair.of(convertedProperties, convertedDefaultOffset); } @@ -747,28 +747,10 @@ public Map getCustomProperties() { } @Override - public void modifyProperties(AlterRoutineLoadCommand command) throws UserException { + protected void unprotectModifyProperties(AlterRoutineLoadCommand command) throws UserException { Map jobProperties = command.getAnalyzedJobProperties(); KafkaDataSourceProperties dataSourceProperties = (KafkaDataSourceProperties) command.getDataSourceProperties(); - - writeLock(); - try { - if (getState() != JobState.PAUSED) { - throw new DdlException("Only supports modification of PAUSED jobs"); - } - - validateAlterJobPropertiesForMutation(command); - modifyPropertiesInternal(jobProperties, dataSourceProperties); - if (command.hasTargetTable()) { - this.tableId = command.getTargetTableId(); - } - - AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, - jobProperties, dataSourceProperties, command.getTargetTableId()); - Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); - } finally { - writeUnlock(); - } + modifyPropertiesInternal(jobProperties, dataSourceProperties); } private void convertOffset(KafkaDataSourceProperties dataSourceProperties, diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index 8aa16f2bc7bd97..d8cf005480c7ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -676,29 +676,11 @@ public Map getCustomProperties() { } @Override - public void modifyProperties(AlterRoutineLoadCommand command) throws UserException { + protected void unprotectModifyProperties(AlterRoutineLoadCommand command) throws UserException { Map jobProperties = command.getAnalyzedJobProperties(); KinesisDataSourceProperties dataSourceProperties = (KinesisDataSourceProperties) command.getDataSourceProperties(); - - writeLock(); - try { - if (getState() != JobState.PAUSED) { - throw new DdlException("Only supports modification of PAUSED jobs"); - } - - validateAlterJobPropertiesForMutation(command); - modifyPropertiesInternal(jobProperties, dataSourceProperties); - if (command.hasTargetTable()) { - this.tableId = command.getTargetTableId(); - } - - AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, - jobProperties, dataSourceProperties, command.getTargetTableId()); - Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); - } finally { - writeUnlock(); - } + modifyPropertiesInternal(jobProperties, dataSourceProperties); } private void modifyPropertiesInternal(Map jobProperties, diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index bae82063049c2e..54311f7058dc3b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -75,6 +75,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantReadWriteLock; public class KafkaRoutineLoadJobTest { @@ -247,6 +248,76 @@ public void testModifyPropertiesRevalidatesWhileHoldingWriteLock() throws Except } } + @Test + public void testModifyTargetTableValidatesAndLogsUnderMetadataLocks() throws Exception { + KafkaRoutineLoadJob routineLoadJob = Mockito.spy(new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, + 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN)); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + Deencapsulation.setField(routineLoadJob, "uniqueKeyUpdateMode", TUniqueKeyUpdateMode.UPSERT); + + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.hasTargetTable()).thenReturn(true); + Mockito.when(command.getTargetTableId()).thenReturn(2L); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + + Env env = Mockito.mock(Env.class); + EditLog editLog = Mockito.mock(EditLog.class); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Database database = Mockito.mock(Database.class); + OlapTable targetTable = Mockito.mock(OlapTable.class); + Mockito.when(env.getEditLog()).thenReturn(editLog); + Mockito.when(catalog.getDbNullable(1L)).thenReturn(database); + Mockito.when(database.getTableNullable(2L)).thenReturn(targetTable); + + AtomicBoolean databaseLocked = new AtomicBoolean(false); + AtomicBoolean tableLocked = new AtomicBoolean(false); + AtomicBoolean targetValidated = new AtomicBoolean(false); + Mockito.doAnswer(invocation -> { + databaseLocked.set(true); + return null; + }).when(database).readLock(); + Mockito.doAnswer(invocation -> { + databaseLocked.set(false); + return null; + }).when(database).readUnlock(); + Mockito.doAnswer(invocation -> { + Assert.assertTrue(databaseLocked.get()); + tableLocked.set(true); + return null; + }).when(targetTable).readLock(); + Mockito.doAnswer(invocation -> { + tableLocked.set(false); + return null; + }).when(targetTable).readUnlock(); + Mockito.doAnswer(invocation -> { + ReentrantReadWriteLock lock = Deencapsulation.getField(routineLoadJob, "lock"); + Assert.assertTrue(lock.isWriteLockedByCurrentThread()); + Assert.assertTrue(databaseLocked.get()); + Assert.assertTrue(tableLocked.get()); + targetValidated.set(true); + return null; + }).when(routineLoadJob).validateTargetTable(Mockito.eq(database), Mockito.eq(targetTable), + Mockito.anyMap(), Mockito.eq(TUniqueKeyUpdateMode.UPSERT)); + Mockito.doAnswer(invocation -> { + Assert.assertTrue(databaseLocked.get()); + Assert.assertTrue(tableLocked.get()); + return null; + }).when(editLog).logAlterRoutineLoadJob(Mockito.any(AlterRoutineLoadJobOperationLog.class)); + + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + + routineLoadJob.modifyProperties(command); + } + + Assert.assertTrue(targetValidated.get()); + Assert.assertFalse(databaseLocked.get()); + Assert.assertFalse(tableLocked.get()); + Assert.assertEquals(2L, routineLoadJob.getTableId()); + Mockito.verify(editLog).logAlterRoutineLoadJob(Mockito.any(AlterRoutineLoadJobOperationLog.class)); + } + @Test public void testModifyPropertiesKeepsKafkaPropertiesWhenFileConversionFails() throws Exception { KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, @@ -347,7 +418,8 @@ public void testModifyPropertiesResolvesOffsetsWithEffectiveKafkaConfiguration() alteredSourceProperties.put(KafkaConfiguration.KAFKA_BROKER_LIST.getName(), "new-broker:9092"); alteredSourceProperties.put(KafkaConfiguration.KAFKA_TOPIC.getName(), "new-topic"); alteredSourceProperties.put(KafkaConfiguration.KAFKA_PARTITIONS.getName(), "7"); - alteredSourceProperties.put(KafkaConfiguration.KAFKA_OFFSETS.getName(), "2026-01-01 00:00:00"); + alteredSourceProperties.put("property." + KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName(), + "2026-01-01 00:00:00"); alteredSourceProperties.put("property.client.id", "new-client"); KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties(alteredSourceProperties); AlterRoutineLoadCommand command = mockAlterCommand(dataSourceProperties); @@ -365,6 +437,10 @@ public void testModifyPropertiesResolvesOffsetsWithEffectiveKafkaConfiguration() Mockito.eq("new-topic"), Mockito.anyMap(), Mockito.anyList())).thenAnswer(invocation -> { Map effectiveCustomProperties = invocation.getArgument(2); Assert.assertEquals("new-client", effectiveCustomProperties.get("client.id")); + Assert.assertFalse(effectiveCustomProperties + .containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName())); + Assert.assertFalse(effectiveCustomProperties + .containsKey(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())); return Lists.newArrayList(Pair.of(7, 700L)); }); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java index b253d7d5ed3599..8799e3312f3f27 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java @@ -54,6 +54,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class RoutineLoadJobTest { @Test @@ -339,6 +341,33 @@ public void testGetShowCreateInfo() throws UserException { Assert.assertEquals(expect, showCreateInfo); } + @Test + public void testShowTableLookupUsesJobReadLock() { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(111L, "test_load", 1L, + 11L, "localhost:9092", "test_topic", UserIdentity.ADMIN); + ReentrantReadWriteLock lock = Deencapsulation.getField(routineLoadJob, "lock"); + + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Database database = Mockito.mock(Database.class); + OlapTable table = Mockito.mock(OlapTable.class); + Mockito.when(catalog.getDb(1L)).thenReturn(Optional.of(database)); + Mockito.when(database.getFullName()).thenReturn("test_db"); + Mockito.when(database.getTable(11L)).thenAnswer(invocation -> { + Assert.assertTrue(lock.getReadHoldCount() > 0); + return Optional.of((Table) table); + }); + Mockito.when(table.getName()).thenReturn("test_table"); + + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + + Assert.assertEquals("test_table", routineLoadJob.getShowInfo().get(6)); + Assert.assertTrue(routineLoadJob.getShowCreateInfo().contains(" ON test_table\n")); + } + + Mockito.verify(database, Mockito.times(2)).getTable(11L); + } + @Test public void testParseUniqueKeyUpdateMode() { // Test valid mode strings From cfb7382610e4568eadb9c9515b3830449010743d Mon Sep 17 00:00:00 2001 From: Refrain Date: Thu, 16 Jul 2026 14:28:36 +0800 Subject: [PATCH 10/19] [fix](fe) Restrict routine load target alteration to Kafka ### What problem does this PR solve? Issue Number: N/A Related PR: #64878 Problem Summary: ALTER ROUTINE LOAD target-table switching is supported only for Kafka jobs. Remove the redundant Kinesis implementation and tests, restore the existing Kinesis ALTER behavior, and reject target-table changes for non-Kafka jobs during command validation. ### Release note ALTER ROUTINE LOAD SET TARGET TABLE supports Kafka Routine Load jobs only. ### Check List (For Author) - Test: Not run (per request) - Behavior changed: Yes. Kinesis jobs now explicitly reject target-table changes. - Does this need documentation: Yes. Document the Kafka-only limitation. --- .../kinesis/KinesisDataSourceProperties.java | 8 +- .../kinesis/KinesisRoutineLoadJob.java | 33 ++++-- .../commands/AlterRoutineLoadCommand.java | 4 + .../KinesisRoutineLoadJobTest.java | 108 ------------------ .../commands/AlterRoutineLoadCommandTest.java | 13 +-- 5 files changed, 34 insertions(+), 132 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisDataSourceProperties.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisDataSourceProperties.java index 047af58cf576ad..edc6cd891fd739 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisDataSourceProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisDataSourceProperties.java @@ -212,18 +212,17 @@ public void convertAndCheckDataSourceProperties() throws UserException { List positions = KinesisConfiguration.KINESIS_POSITIONS.getParameterValue( originalDataSourceProperties.get(KinesisConfiguration.KINESIS_POSITIONS.getName())); // Get default position from customKinesisProperties (already parsed from "property." prefix) - boolean hasDefaultPosition = customKinesisProperties.containsKey("kinesis_default_pos"); String defaultPositionString = customKinesisProperties.get("kinesis_default_pos"); // Validate that positions and default_position are not both set - if (CollectionUtils.isNotEmpty(positions) && hasDefaultPosition) { + if (CollectionUtils.isNotEmpty(positions) && StringUtils.isNotBlank(defaultPositionString)) { throw new AnalysisException("Only one of " + KinesisConfiguration.KINESIS_POSITIONS.getName() + " and property.kinesis_default_pos can be set."); } // For alter operation, shards and positions must be set together if (isAlter() && CollectionUtils.isNotEmpty(shards) && CollectionUtils.isEmpty(positions) - && !hasDefaultPosition) { + && StringUtils.isBlank(defaultPositionString)) { throw new AnalysisException("Must set position or default position with shard property"); } @@ -232,9 +231,6 @@ public void convertAndCheckDataSourceProperties() throws UserException { this.isPositionsForTimes = analyzeKinesisPositionProperty(positions); return; } - if (isAlter() && CollectionUtils.isEmpty(shards) && !hasDefaultPosition) { - return; - } this.isPositionsForTimes = analyzeKinesisDefaultPositionProperty(); if (CollectionUtils.isNotEmpty(kinesisShardPositions)) { setDefaultPositionForShards(this.kinesisShardPositions, defaultPositionString, this.isPositionsForTimes); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index c479b550a2e4ec..7cebc3f5165b49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -65,6 +65,7 @@ import com.google.gson.annotations.SerializedName; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.BooleanUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -674,11 +675,25 @@ private Map getMaskedCustomProperties(String keyPrefix) { } @Override - protected void unprotectModifyProperties(AlterRoutineLoadCommand command) throws UserException { + public void modifyProperties(AlterRoutineLoadCommand command) throws UserException { Map jobProperties = command.getAnalyzedJobProperties(); KinesisDataSourceProperties dataSourceProperties = (KinesisDataSourceProperties) command.getDataSourceProperties(); - modifyPropertiesInternal(jobProperties, dataSourceProperties); + + writeLock(); + try { + if (getState() != JobState.PAUSED) { + throw new DdlException("Only supports modification of PAUSED jobs"); + } + + modifyPropertiesInternal(jobProperties, dataSourceProperties); + + AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, + jobProperties, dataSourceProperties); + Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); + } finally { + writeUnlock(); + } } private void modifyPropertiesInternal(Map jobProperties, @@ -695,10 +710,6 @@ private void modifyPropertiesInternal(Map jobProperties, customKinesisProperties = dataSourceProperties.getCustomKinesisProperties(); hasExplicitShardPositions = !shardPositions.isEmpty(); } - resetProgress = !Strings.isNullOrEmpty(dataSourceProperties.getStream()); - if (hasExplicitShardPositions && !resetProgress) { - ((KinesisProgress) progress).checkShards(shardPositions); - } // Update custom properties if (!customKinesisProperties.isEmpty()) { @@ -709,6 +720,7 @@ private void modifyPropertiesInternal(Map jobProperties, // Modify stream if provided if (!Strings.isNullOrEmpty(dataSourceProperties.getStream())) { this.stream = dataSourceProperties.getStream(); + resetProgress = true; } // Modify region if provided @@ -739,6 +751,9 @@ private void modifyPropertiesInternal(Map jobProperties, } if (!shardPositions.isEmpty()) { + if (!resetProgress) { + ((KinesisProgress) progress).checkShards(shardPositions); + } ((KinesisProgress) progress).modifyPosition(shardPositions); } } @@ -747,6 +762,9 @@ private void modifyPropertiesInternal(Map jobProperties, Map copiedJobProperties = Maps.newHashMap(jobProperties); modifyCommonJobProperties(copiedJobProperties); this.jobProperties.putAll(copiedJobProperties); + if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { + this.isPartialUpdate = BooleanUtils.toBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); + } if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { String policy = jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); if ("ERROR".equalsIgnoreCase(policy)) { @@ -765,9 +783,6 @@ public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { try { modifyPropertiesInternal(log.getJobProperties(), (KinesisDataSourceProperties) log.getDataSourceProperties()); - if (log.getTargetTableId() != 0) { - this.tableId = log.getTargetTableId(); - } } catch (UserException e) { LOG.error("failed to replay modify kinesis routine load job: {}", id, e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java index ea8cc19d80627f..7ee00f33c93636 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java @@ -35,6 +35,7 @@ import org.apache.doris.datasource.property.fileformat.JsonFileFormatProperties; import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.routineload.AbstractDataSourceProperties; +import org.apache.doris.load.routineload.LoadDataSourceType; import org.apache.doris.load.routineload.RoutineLoadDataSourcePropertyFactory; import org.apache.doris.load.routineload.RoutineLoadJob; import org.apache.doris.mysql.privilege.PrivPredicate; @@ -392,6 +393,9 @@ private TUniqueKeyUpdateMode getEffectiveUniqueKeyUpdateMode(RoutineLoadJob job) private void validateTargetTable(ConnectContext ctx, RoutineLoadJob job, TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode) throws UserException { + if (job.getDataSourceType() != LoadDataSourceType.KAFKA) { + throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports Kafka jobs"); + } if (job.isMultiTable()) { throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports single-table job"); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java index f382c162015fcb..65aebd0084e729 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java @@ -18,25 +18,19 @@ package org.apache.doris.load.routineload; import org.apache.doris.analysis.UserIdentity; -import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; -import org.apache.doris.common.DdlException; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.load.routineload.kinesis.KinesisConfiguration; import org.apache.doris.load.routineload.kinesis.KinesisDataSourceProperties; import org.apache.doris.load.routineload.kinesis.KinesisProgress; import org.apache.doris.load.routineload.kinesis.KinesisRoutineLoadJob; import org.apache.doris.load.routineload.kinesis.KinesisTaskInfo; -import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; -import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; -import org.apache.doris.thrift.TUniqueKeyUpdateMode; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.Gson; import org.junit.Assert; import org.junit.Test; -import org.mockito.Mockito; import java.util.HashMap; import java.util.HashSet; @@ -44,7 +38,6 @@ import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.concurrent.locks.ReentrantReadWriteLock; public class KinesisRoutineLoadJobTest { @@ -108,47 +101,6 @@ public void testGetStatisticContainsKinesisFields() { Assert.assertEquals(100L, ((Number) statistic.get("maxMillisBehindLatest")).longValue()); } - @Test - public void testModifyPropertiesHonorsExplicitUniqueKeyUpdateModePrecedence() throws Exception { - KinesisRoutineLoadJob routineLoadJob = - new KinesisRoutineLoadJob(1L, "kinesis_routine_load_job", 1L, - 1L, "ap-southeast-1", "stream-1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "uniqueKeyUpdateMode", TUniqueKeyUpdateMode.UPSERT); - Deencapsulation.setField(routineLoadJob, "isPartialUpdate", false); - - Map jobProperties = Maps.newHashMap(); - jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, "UPSERT"); - jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, "true"); - - Deencapsulation.invoke(routineLoadJob, "modifyPropertiesInternal", jobProperties, - new KinesisDataSourceProperties(Maps.newHashMap())); - - Assert.assertEquals(TUniqueKeyUpdateMode.UPSERT, routineLoadJob.getUniqueKeyUpdateMode()); - Assert.assertFalse(routineLoadJob.isFixedPartialUpdate()); - } - - @Test - public void testModifyPropertiesRevalidatesWhileHoldingWriteLock() throws Exception { - KinesisRoutineLoadJob routineLoadJob = Mockito.spy(new KinesisRoutineLoadJob(1L, - "kinesis_routine_load_job", 1L, 1L, "ap-southeast-1", "stream-1", UserIdentity.ADMIN)); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); - Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); - - Mockito.doAnswer(invocation -> { - ReentrantReadWriteLock lock = Deencapsulation.getField(routineLoadJob, "lock"); - Assert.assertTrue(lock.isWriteLockedByCurrentThread()); - throw new DdlException("stop after validation"); - }).when(routineLoadJob).validateAlterJobPropertiesForMutation(command); - - try { - routineLoadJob.modifyProperties(command); - Assert.fail("Expected mutation-time validation to fail"); - } catch (DdlException e) { - Assert.assertTrue(e.getMessage().contains("stop after validation")); - } - } - @Test public void testHasMoreDataToConsumeShouldKeepPollingWhenLagCacheIsZero() throws Exception { KinesisRoutineLoadJob routineLoadJob = @@ -218,10 +170,6 @@ public void testModifyPropertiesShouldClearStaleCustomShardsWhenStreamChanges() Map oldLag = new HashMap<>(); oldLag.put("shard-old-0", 10L); Deencapsulation.setField(routineLoadJob, "cachedShardWithMillsBehindLatest", oldLag); - Deencapsulation.setField(routineLoadJob, "customProperties", - Maps.newHashMap(Map.of("kinesis_default_pos", KinesisDataSourceProperties.POSITION_TRIM_HORIZON))); - Deencapsulation.setField(routineLoadJob, "kinesisDefaultPosition", - KinesisDataSourceProperties.POSITION_TRIM_HORIZON); Map alterProps = new HashMap<>(); alterProps.put(KinesisConfiguration.KINESIS_STREAM.getName(), "stream-2"); @@ -229,7 +177,6 @@ public void testModifyPropertiesShouldClearStaleCustomShardsWhenStreamChanges() dataSourceProperties.setAlter(true); dataSourceProperties.setTimezone("Asia/Shanghai"); dataSourceProperties.analyze(); - Assert.assertFalse(dataSourceProperties.getCustomKinesisProperties().containsKey("kinesis_default_pos")); Deencapsulation.invoke(routineLoadJob, "modifyPropertiesInternal", new HashMap(), dataSourceProperties); @@ -247,19 +194,6 @@ public void testModifyPropertiesShouldClearStaleCustomShardsWhenStreamChanges() Assert.assertFalse(progress.hasShards()); Map cachedLag = Deencapsulation.getField(routineLoadJob, "cachedShardWithMillsBehindLatest"); Assert.assertTrue(cachedLag.isEmpty()); - Assert.assertEquals(KinesisDataSourceProperties.POSITION_TRIM_HORIZON, - Deencapsulation.getField(routineLoadJob, "kinesisDefaultPosition")); - } - - @Test - public void testAlterRejectsEmptyKinesisDefaultPosition() { - Map alterProperties = Maps.newHashMap(); - alterProperties.put(KinesisConfiguration.KINESIS_DEFAULT_POSITION.getName(), ""); - KinesisDataSourceProperties dataSourceProperties = new KinesisDataSourceProperties(alterProperties); - dataSourceProperties.setAlter(true); - dataSourceProperties.setTimezone("UTC"); - - Assert.assertThrows(AnalysisException.class, dataSourceProperties::analyze); } @Test @@ -295,48 +229,6 @@ public void testModifyPropertiesShouldReplaceCustomShardsWhenExplicitShardsProvi Assert.assertEquals("202", progress.getSequenceNumberByShard("shard-2")); } - @Test - public void testModifyPropertiesKeepsKinesisStateWhenShardValidationFails() throws Exception { - KinesisRoutineLoadJob routineLoadJob = new KinesisRoutineLoadJob(1L, - "kinesis_routine_load_job", 1L, 1L, "ap-southeast-1", "stream-1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - Deencapsulation.setField(routineLoadJob, "customKinesisShards", Lists.newArrayList("shard-0")); - Deencapsulation.setField(routineLoadJob, "progress", - new KinesisProgress(Maps.newHashMap(Map.of("shard-0", "10")))); - Deencapsulation.setField(routineLoadJob, "customProperties", - Maps.newHashMap(Map.of("max_connections", "5"))); - Deencapsulation.setField(routineLoadJob, "convertedCustomProperties", - Maps.newHashMap(Map.of("max_connections", "5"))); - - Map alterProperties = Maps.newHashMap(); - alterProperties.put(KinesisConfiguration.KINESIS_REGION.getName(), "us-east-1"); - alterProperties.put(KinesisConfiguration.KINESIS_SHARDS.getName(), "shard-missing"); - alterProperties.put(KinesisConfiguration.KINESIS_POSITIONS.getName(), "101"); - alterProperties.put("property.max_connections", "10"); - KinesisDataSourceProperties dataSourceProperties = new KinesisDataSourceProperties(alterProperties); - dataSourceProperties.setAlter(true); - dataSourceProperties.setTimezone("UTC"); - dataSourceProperties.analyze(); - AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); - Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); - Mockito.when(command.getDataSourceProperties()).thenReturn(dataSourceProperties); - - try { - routineLoadJob.modifyProperties(command); - Assert.fail("Expected unknown shard to reject ALTER"); - } catch (DdlException e) { - Assert.assertTrue(e.getMessage().contains("shard-missing")); - } - - Assert.assertEquals("ap-southeast-1", routineLoadJob.getRegion()); - Assert.assertEquals(Lists.newArrayList("shard-0"), - Deencapsulation.getField(routineLoadJob, "customKinesisShards")); - Assert.assertEquals("5", - Deencapsulation.>getField(routineLoadJob, "customProperties") - .get("max_connections")); - Assert.assertEquals("5", routineLoadJob.getConvertedCustomProperties().get("max_connections")); - } - @Test public void testShardRefreshShouldMoveRetiredParentToClosedUntilConsumed() throws Exception { KinesisRoutineLoadJob routineLoadJob = diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java index afe162deeb0ff1..12131e632733e8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java @@ -29,7 +29,6 @@ import org.apache.doris.load.routineload.RoutineLoadJob; import org.apache.doris.load.routineload.RoutineLoadManager; import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; -import org.apache.doris.load.routineload.kinesis.KinesisDataSourceProperties; import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.nereids.exceptions.ParseException; import org.apache.doris.nereids.parser.NereidsParser; @@ -218,18 +217,14 @@ public void testValidateTargetTableWithJobAndDataSourceProperties() throws Excep } @Test - public void testValidateTargetTableWithKinesisProperties() throws Exception { + public void testValidateTargetTableRejectsKinesisJob() throws Exception { runBefore(); - Mockito.when(currentTable.getId()).thenReturn(3000L); Mockito.when(routineLoadJob.getDataSourceType()).thenReturn(LoadDataSourceType.KINESIS); - mockTargetTable(currentTable); AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" " - + "FROM KINESIS(\"property.max_connections\"=\"10\")"); + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - Assertions.assertInstanceOf(KinesisDataSourceProperties.class, command.getDataSourceProperties()); - Assertions.assertEquals(3000L, command.getTargetTableId()); + Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, + () -> command.validate(connectContext)).getMessage().contains("only supports Kafka jobs")); } @Test From bd1fad56ceaa3908902d68ad44a61c4329cb0c5c Mon Sep 17 00:00:00 2001 From: Refrain Date: Thu, 16 Jul 2026 15:31:04 +0800 Subject: [PATCH 11/19] [fix](fe) Restore Kinesis routine load alter hook ### What problem does this PR solve? Issue Number: N/A Related PR: #64878 Problem Summary: Restricting target-table alteration to Kafka restored the old Kinesis modifyProperties override, but RoutineLoadJob now requires subclasses to implement unprotectModifyProperties. This caused FE compilation to fail. Implement the required Kinesis hook by delegating to the existing property mutation path while keeping target-table alteration rejected for non-Kafka jobs. ### Release note None ### Check List (For Author) - Test: - Unit Test: Routine Load focused FE UT (71 tests) - FE build: ./build.sh --fe --clean -j48 - Behavior changed: No. Kinesis target-table alteration remains unsupported. - Does this need documentation: No --- .../kinesis/KinesisRoutineLoadJob.java | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index 7cebc3f5165b49..094c4c5f278eff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -675,25 +675,11 @@ private Map getMaskedCustomProperties(String keyPrefix) { } @Override - public void modifyProperties(AlterRoutineLoadCommand command) throws UserException { + protected void unprotectModifyProperties(AlterRoutineLoadCommand command) throws UserException { Map jobProperties = command.getAnalyzedJobProperties(); KinesisDataSourceProperties dataSourceProperties = (KinesisDataSourceProperties) command.getDataSourceProperties(); - - writeLock(); - try { - if (getState() != JobState.PAUSED) { - throw new DdlException("Only supports modification of PAUSED jobs"); - } - - modifyPropertiesInternal(jobProperties, dataSourceProperties); - - AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, - jobProperties, dataSourceProperties); - Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); - } finally { - writeUnlock(); - } + modifyPropertiesInternal(jobProperties, dataSourceProperties); } private void modifyPropertiesInternal(Map jobProperties, From e848c184d7f70cfe904b7a8e50a14a18871e82fc Mon Sep 17 00:00:00 2001 From: 0AyanamiRei <3244156674@qq.com> Date: Fri, 17 Jul 2026 11:08:47 +0800 Subject: [PATCH 12/19] [refactor](fe) Simplify routine load target table alteration ### What problem does this PR solve? Issue Number: None Related PR: #64878 Problem Summary: The target-table ALTER implementation accumulated dead overloads, duplicate partial-update checks, repeated target-table lookup and locking, and redundant test setup. Simplify the grammar and validation flow, reuse the already resolved target table during mutation-time validation, keep target binding and edit-log ordering under the existing metadata locks, and let common job-property handling own partial-update precedence. ### Release note Routine load ALTER consistently honors explicit unique_key_update_mode over legacy partial_columns properties. ### Check List (For Author) - Test: No need to test (compilation and tests were intentionally not run per request; static diff checks only) - Behavior changed: Yes. Explicit unique_key_update_mode remains authoritative for Kinesis ALTER when partial_columns is also present. - Does this need documentation: No --- .../load/routineload/RoutineLoadJob.java | 60 ++++++++----------- .../kinesis/KinesisRoutineLoadJob.java | 4 -- .../commands/AlterRoutineLoadCommand.java | 20 +------ .../routineload/KafkaRoutineLoadJobTest.java | 3 +- .../load/routineload/RoutineLoadJobTest.java | 12 ++++ .../commands/AlterRoutineLoadCommandTest.java | 35 ----------- .../AlterRoutineLoadOperationLogTest.java | 37 ++---------- .../org/apache/doris/nereids/DorisParser.g4 | 4 +- .../test_routine_load_alter.groovy | 2 - 9 files changed, 49 insertions(+), 128 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 9c03f48e16d8b9..6efa3725265259 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -19,7 +19,6 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ExprToSqlVisitor; -import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.Separator; import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.analysis.UserIdentity; @@ -471,35 +470,30 @@ protected void setRoutineLoadDesc(RoutineLoadDesc routineLoadDesc) { } } - public void validateTargetTable(Database db, OlapTable targetTable) throws UserException { - validateTargetTable(db, targetTable, Maps.newHashMap(), uniqueKeyUpdateMode); - } - public void validateTargetTable(Database db, OlapTable targetTable, Map alteredJobProperties, TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode) throws UserException { - if (isMultiTable) { - throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports single-table job"); - } - List columnsInfo = null; - if (columnDescs != null && !columnDescs.descs.isEmpty()) { - columnsInfo = new ArrayList<>(columnDescs.descs); - } - checkMeta(targetTable, new RoutineLoadDesc(columnSeparator, lineDelimiter, columnsInfo, precedingFilter, - whereExpr, partitionNamesInfo, deleteCondition, mergeType, sequenceCol)); - validateAlterJobProperties(targetTable, alteredJobProperties, effectiveUniqueKeyUpdateMode); - targetTable.readLock(); try { - NereidsRoutineLoadTaskInfo taskInfo = toNereidsRoutineLoadTaskInfo(); - taskInfo.applyAlterPropertiesForValidation(alteredJobProperties, effectiveUniqueKeyUpdateMode); - NereidsStreamLoadPlanner planner = new NereidsStreamLoadPlanner(db, targetTable, taskInfo); - planner.plan(new TUniqueId(0, 0)); + unprotectedValidateTargetTable(db, targetTable, alteredJobProperties, effectiveUniqueKeyUpdateMode); } finally { targetTable.readUnlock(); } } + protected void unprotectedValidateTargetTable(Database db, OlapTable targetTable, + Map alteredJobProperties, TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode) + throws UserException { + if (isMultiTable) { + throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports single-table job"); + } + validateAlterJobProperties(targetTable, alteredJobProperties, effectiveUniqueKeyUpdateMode); + NereidsRoutineLoadTaskInfo taskInfo = toNereidsRoutineLoadTaskInfo(); + taskInfo.applyAlterPropertiesForValidation(alteredJobProperties, effectiveUniqueKeyUpdateMode); + NereidsStreamLoadPlanner planner = new NereidsStreamLoadPlanner(db, targetTable, taskInfo); + planner.plan(new TUniqueId(0, 0)); + } + public void validateAlterJobProperties(OlapTable targetTable, Map alteredJobProperties, TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode) throws UserException { if (effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { @@ -537,7 +531,7 @@ protected void validateAlterJobPropertiesForMutation(AlterRoutineLoadCommand com Map alteredJobProperties = command.getAnalyzedJobProperties(); TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode = getEffectiveUniqueKeyUpdateMode( uniqueKeyUpdateMode, alteredJobProperties); - if (!command.hasTargetTable() && effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { + if (effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { return; } @@ -545,19 +539,12 @@ protected void validateAlterJobPropertiesForMutation(AlterRoutineLoadCommand com if (db == null) { throw new DdlException("Database not found: " + dbId); } - long effectiveTableId = command.hasTargetTable() ? command.getTargetTableId() : tableId; - Table table = db.getTableNullable(effectiveTableId); + Table table = db.getTableNullable(tableId); if (table == null) { - throw new DdlException("Table not found: " + effectiveTableId); + throw new DdlException("Table not found: " + tableId); } if (!(table instanceof OlapTable)) { - throw new DdlException(command.hasTargetTable() - ? "Routine load target table must be an OLAP table" - : "Partial update is only supported for OLAP tables"); - } - if (command.hasTargetTable()) { - validateTargetTable(db, (OlapTable) table, alteredJobProperties, effectiveUniqueKeyUpdateMode); - return; + throw new DdlException("Partial update is only supported for OLAP tables"); } validateAlterJobProperties((OlapTable) table, alteredJobProperties, effectiveUniqueKeyUpdateMode); } @@ -2147,6 +2134,7 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti throw new DdlException("Only supports modification of PAUSED jobs"); } if (!command.hasTargetTable()) { + validateAlterJobPropertiesForMutation(command); unprotectApplyAlter(command); return; } @@ -2166,6 +2154,12 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti } table.readLock(); try { + Map alteredJobProperties = command.getAnalyzedJobProperties(); + TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode = getEffectiveUniqueKeyUpdateMode( + uniqueKeyUpdateMode, alteredJobProperties); + unprotectedValidateTargetTable( + db, (OlapTable) table, alteredJobProperties, effectiveUniqueKeyUpdateMode); + // Keep target binding and its journal ordered with concurrent table DDL. unprotectApplyAlter(command); } finally { table.readUnlock(); @@ -2179,12 +2173,10 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti } private void unprotectApplyAlter(AlterRoutineLoadCommand command) throws UserException { - validateAlterJobPropertiesForMutation(command); unprotectModifyProperties(command); if (command.hasTargetTable()) { - this.tableId = command.getTargetTableId(); + tableId = command.getTargetTableId(); } - AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, command.getAnalyzedJobProperties(), command.getDataSourceProperties(), command.getTargetTableId()); Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index 094c4c5f278eff..5edf8e1ccfc107 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -65,7 +65,6 @@ import com.google.gson.annotations.SerializedName; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.BooleanUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -748,9 +747,6 @@ private void modifyPropertiesInternal(Map jobProperties, Map copiedJobProperties = Maps.newHashMap(jobProperties); modifyCommonJobProperties(copiedJobProperties); this.jobProperties.putAll(copiedJobProperties); - if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { - this.isPartialUpdate = BooleanUtils.toBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); - } if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { String policy = jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); if ("ERROR".equalsIgnoreCase(policy)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java index 7ee00f33c93636..9f464c4391e98e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java @@ -136,10 +136,6 @@ public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, this(labelNameInfo, null, Maps.newHashMap(), jobProperties, null, dataSourceMapProperties); } - public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, String targetTableName) { - this(labelNameInfo, targetTableName, Maps.newHashMap(), Maps.newHashMap(), null, Maps.newHashMap()); - } - public String getDbName() { return labelNameInfo.getDb(); } @@ -203,7 +199,8 @@ public void validate(ConnectContext ctx) throws UserException { } // check data source properties checkDataSourceProperties(job); - TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode = getEffectiveUniqueKeyUpdateMode(job); + TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode = RoutineLoadJob.getEffectiveUniqueKeyUpdateMode( + job.getUniqueKeyUpdateMode(), analyzedJobProperties); if (hasTargetTable()) { validateTargetTable(ctx, job, effectiveUniqueKeyUpdateMode); } else { @@ -379,18 +376,9 @@ private void validateAlterJobProperties(RoutineLoadJob job, } Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException(job.getDbFullName()); Table table = db.getTableOrAnalysisException(job.getTableName()); - if (effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS - && !((OlapTable) table).getEnableUniqueKeyMergeOnWrite()) { - throw new AnalysisException("load by PARTIAL_COLUMNS is only supported in unique table MoW"); - } job.validateAlterJobProperties((OlapTable) table, analyzedJobProperties, effectiveUniqueKeyUpdateMode); } - private TUniqueKeyUpdateMode getEffectiveUniqueKeyUpdateMode(RoutineLoadJob job) { - return RoutineLoadJob.getEffectiveUniqueKeyUpdateMode( - job.getUniqueKeyUpdateMode(), analyzedJobProperties); - } - private void validateTargetTable(ConnectContext ctx, RoutineLoadJob job, TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode) throws UserException { if (job.getDataSourceType() != LoadDataSourceType.KAFKA) { @@ -418,10 +406,6 @@ private void validateTargetTable(ConnectContext ctx, RoutineLoadJob job, throw new AnalysisException( "if load_to_single_tablet set to true, the olap table must be with random distribution"); } - if (effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS - && !olapTable.getEnableUniqueKeyMergeOnWrite()) { - throw new AnalysisException("load by PARTIAL_COLUMNS is only supported in unique table MoW"); - } job.validateTargetTable(db, olapTable, analyzedJobProperties, effectiveUniqueKeyUpdateMode); this.targetTableId = olapTable.getId(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index 24810f96f7c178..cfc5c922d8766d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -296,11 +296,12 @@ public void testModifyTargetTableValidatesAndLogsUnderMetadataLocks() throws Exc Assert.assertTrue(tableLocked.get()); targetValidated.set(true); return null; - }).when(routineLoadJob).validateTargetTable(Mockito.eq(database), Mockito.eq(targetTable), + }).when(routineLoadJob).unprotectedValidateTargetTable(Mockito.eq(database), Mockito.eq(targetTable), Mockito.anyMap(), Mockito.eq(TUniqueKeyUpdateMode.UPSERT)); Mockito.doAnswer(invocation -> { Assert.assertTrue(databaseLocked.get()); Assert.assertTrue(tableLocked.get()); + Assert.assertEquals(2L, routineLoadJob.getTableId()); return null; }).when(editLog).logAlterRoutineLoadJob(Mockito.any(AlterRoutineLoadJobOperationLog.class)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java index 7955246be77173..6d89b17a5d52eb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java @@ -530,6 +530,18 @@ public void testUniqueKeyUpdateModeTakesPrecedenceOverPartialColumns() throws Ex Assert.assertFalse(isPartialUpdate); } + @Test + public void testValidateFixedPartialUpdateRequiresMowTable() { + KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(); + OlapTable targetTable = Mockito.mock(OlapTable.class); + Mockito.when(targetTable.getEnableUniqueKeyMergeOnWrite()).thenReturn(false); + + UserException exception = Assert.assertThrows(UserException.class, + () -> job.validateAlterJobProperties(targetTable, Maps.newHashMap(), + TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS)); + Assert.assertTrue(exception.getMessage().contains("PARTIAL_COLUMNS")); + } + @Test public void testValidateFlexiblePartialUpdateUsesAlteredProperties() throws Exception { KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java index 12131e632733e8..3bc98c954106bb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java @@ -370,41 +370,6 @@ public void testValidateTargetTablePassesTargetTableToJobValidation() throws Exc Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); } - @Test - public void testValidateTargetTableRejectsExistingFlexiblePartialUpdateOnNonMowTable() throws Exception { - runBefore(); - OlapTable targetTable = Mockito.mock(OlapTable.class); - Mockito.when(targetTable.getType()).thenReturn(Table.TableType.OLAP); - Mockito.when(targetTable.isTemporary()).thenReturn(false); - Mockito.when(targetTable.getEnableUniqueKeyMergeOnWrite()).thenReturn(false); - Mockito.doReturn(targetTable).when(db).getTableOrAnalysisException("testTable2"); - Mockito.when(routineLoadJob.getUniqueKeyUpdateMode()).thenReturn(TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS); - Mockito.doThrow(new AnalysisException("Only unique key merge on write support partial update")) - .when(routineLoadJob).validateTargetTable(Mockito.any(Database.class), Mockito.any(OlapTable.class), - Mockito.anyMap(), Mockito.any(TUniqueKeyUpdateMode.class)); - - AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); - - Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) - .getMessage().contains("partial update")); - } - - @Test - public void testValidateRejectsUniqueKeyUpdateModeOnNonMowTable() { - runBefore(); - Mockito.when(routineLoadJob.getUniqueKeyUpdateMode()).thenReturn(TUniqueKeyUpdateMode.UPSERT); - Mockito.when(currentTable.getEnableUniqueKeyMergeOnWrite()).thenReturn(false); - Map jobProperties = Maps.newHashMap(); - jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, "UPDATE_FIXED_COLUMNS"); - - AlterRoutineLoadCommand command = new AlterRoutineLoadCommand( - new LabelNameInfo("testDb", "label1"), jobProperties, Maps.newHashMap()); - - Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) - .getMessage().contains("PARTIAL_COLUMNS")); - } - @Test public void testValidateAllowsExplicitUpsertToOverridePartialColumnsOnNonMowTable() { runBefore(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java index 5aaf6423ad22b7..912915ffbe20e8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java @@ -51,6 +51,7 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); long jobId = 1000; + long targetTableId = 2001; Map jobProperties = Maps.newHashMap(); jobProperties.put(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY, "5"); @@ -64,8 +65,10 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User routineLoadDataSourceProperties.setTimezone(TimeUtils.DEFAULT_TIME_ZONE); routineLoadDataSourceProperties.analyze(); + Assert.assertEquals(0L, new AlterRoutineLoadJobOperationLog( + jobId, jobProperties, routineLoadDataSourceProperties).getTargetTableId()); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(jobId, - jobProperties, routineLoadDataSourceProperties); + jobProperties, routineLoadDataSourceProperties, targetTableId); log.write(out); out.flush(); out.close(); @@ -85,41 +88,11 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User kafkaDataSourceProperties.getKafkaPartitionOffsets().get(0)); Assert.assertEquals(routineLoadDataSourceProperties.getKafkaPartitionOffsets().get(1), kafkaDataSourceProperties.getKafkaPartitionOffsets().get(1)); + Assert.assertEquals(targetTableId, log2.getTargetTableId()); in.close(); } - - @Test - public void testSerializeAlterRoutineLoadOperationLogWithTargetTableId() throws Exception { - long jobId = 1000; - long targetTableId = 2001; - Map jobProperties = Maps.newHashMap(); - Map dataSourceProperties = Maps.newHashMap(); - dataSourceProperties.put("property.group.id", "mygroup"); - KafkaDataSourceProperties routineLoadDataSourceProperties = new KafkaDataSourceProperties( - dataSourceProperties); - routineLoadDataSourceProperties.setAlter(true); - routineLoadDataSourceProperties.setTimezone(TimeUtils.DEFAULT_TIME_ZONE); - routineLoadDataSourceProperties.analyze(); - - AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(jobId, jobProperties, - routineLoadDataSourceProperties, targetTableId); - - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - try (DataOutputStream out = new DataOutputStream(byteArrayOutputStream)) { - log.write(out); - } - - AlterRoutineLoadJobOperationLog readLog; - try (DataInputStream in = new DataInputStream( - new ByteArrayInputStream(byteArrayOutputStream.toByteArray()))) { - readLog = AlterRoutineLoadJobOperationLog.read(in); - } - - Assert.assertEquals(targetTableId, readLog.getTargetTableId()); - } - @Test public void testDeserializeAlterRoutineLoadOperationLogWithoutTargetTableId() throws Exception { long jobId = 1000; diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index a44467b2422554..4acd29b5c036a5 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -337,8 +337,8 @@ supportedAlterStatement | ALTER ROUTINE LOAD FOR name=multipartIdentifier ( SET TARGET TABLE EQ targetTable=STRING_LITERAL - | (loadProperty (COMMA loadProperty)*)? - ) + | loadProperty (COMMA loadProperty)* + )? properties=propertyClause? (FROM type=identifier LEFT_PAREN propertyItemList RIGHT_PAREN)? #alterRoutineLoad | ALTER COLOCATE GROUP name=multipartIdentifier diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy index 6d9bf4aa2ec70d..18c23258ce7a4a 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy @@ -507,8 +507,6 @@ suite("test_routine_load_alter","p0") { } alterTopicAdmin.close() } - sql "truncate table ${srcTableName}" - sql "truncate table ${dstTableName}" } } } From 8b19f60c7a9c973cb565b6963e76cfdf836a24d2 Mon Sep 17 00:00:00 2001 From: 0AyanamiRei <3244156674@qq.com> Date: Fri, 17 Jul 2026 11:35:10 +0800 Subject: [PATCH 13/19] [fix](fe) Move routine load ALTER I/O outside metadata locks ### What problem does this PR solve? Issue Number: None Related PR: #64878 Problem Summary: Target-table ALTER held database and table read locks while Kafka offset resolution and Cloud progress reset performed external I/O. Split ALTER into a preparation phase serialized only by the job write lock and a final commit phase under metadata locks. The prepared operation resolves and snapshots Kafka state without changing the live job; final target revalidation, in-memory mutation, target binding, and edit-log persistence remain ordered in one metadata critical section. ### Release note None ### Check List (For Author) - Test: No need to test (compilation and tests were intentionally not run per request; static diff checks only) - Behavior changed: No. Only the lock scope and internal staging of ALTER mutations changed. - Does this need documentation: No --- .../load/routineload/RoutineLoadJob.java | 22 ++- .../kafka/KafkaRoutineLoadJob.java | 131 +++++++++++------- .../kinesis/KinesisRoutineLoadJob.java | 4 +- .../routineload/KafkaRoutineLoadJobTest.java | 55 +++++++- 4 files changed, 153 insertions(+), 59 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 6efa3725265259..b67aed00f8e665 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -2135,7 +2135,12 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti } if (!command.hasTargetTable()) { validateAlterJobPropertiesForMutation(command); - unprotectApplyAlter(command); + } + + // Data-source preparation may perform external I/O. Keep it outside DB/table metadata locks. + PreparedAlter preparedAlter = prepareAlter(command); + if (!command.hasTargetTable()) { + unprotectApplyAlter(command, preparedAlter); return; } @@ -2159,8 +2164,7 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti uniqueKeyUpdateMode, alteredJobProperties); unprotectedValidateTargetTable( db, (OlapTable) table, alteredJobProperties, effectiveUniqueKeyUpdateMode); - // Keep target binding and its journal ordered with concurrent table DDL. - unprotectApplyAlter(command); + unprotectApplyAlter(command, preparedAlter); } finally { table.readUnlock(); } @@ -2172,8 +2176,9 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti } } - private void unprotectApplyAlter(AlterRoutineLoadCommand command) throws UserException { - unprotectModifyProperties(command); + private void unprotectApplyAlter(AlterRoutineLoadCommand command, PreparedAlter preparedAlter) + throws UserException { + preparedAlter.apply(); if (command.hasTargetTable()) { tableId = command.getTargetTableId(); } @@ -2182,7 +2187,12 @@ private void unprotectApplyAlter(AlterRoutineLoadCommand command) throws UserExc Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); } - protected abstract void unprotectModifyProperties(AlterRoutineLoadCommand command) throws UserException; + @FunctionalInterface + protected interface PreparedAlter { + void apply() throws UserException; + } + + protected abstract PreparedAlter prepareAlter(AlterRoutineLoadCommand command) throws UserException; public abstract void replayModifyProperties(AlterRoutineLoadJobOperationLog log); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java index a89af6ab2206a8..2a982a9b3e66c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java @@ -772,76 +772,43 @@ public Map getCustomProperties() { } @Override - protected void unprotectModifyProperties(AlterRoutineLoadCommand command) throws UserException { + protected PreparedAlter prepareAlter(AlterRoutineLoadCommand command) throws UserException { Map jobProperties = command.getAnalyzedJobProperties(); KafkaDataSourceProperties dataSourceProperties = (KafkaDataSourceProperties) command.getDataSourceProperties(); - modifyPropertiesInternal(jobProperties, dataSourceProperties); + PreparedKafkaProperties preparedDataSourceProperties = prepareDataSourceProperties(dataSourceProperties, false); + return () -> modifyPropertiesInternal(jobProperties, dataSourceProperties, preparedDataSourceProperties); } - private void convertOffset(KafkaDataSourceProperties dataSourceProperties, + private List> resolveOffsets(KafkaDataSourceProperties dataSourceProperties, KafkaDataSourceSnapshot dataSourceSnapshot) throws UserException { List> partitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); if (partitionOffsets.isEmpty()) { - return; + return partitionOffsets; } - List> newOffsets; if (dataSourceProperties.isOffsetsForTimes()) { - newOffsets = KafkaUtil.getOffsetsForTimes(dataSourceSnapshot.brokerList, dataSourceSnapshot.topic, - dataSourceSnapshot.convertedCustomProperties, partitionOffsets); - } else { - newOffsets = KafkaUtil.getRealOffsets(dataSourceSnapshot.brokerList, dataSourceSnapshot.topic, + return KafkaUtil.getOffsetsForTimes(dataSourceSnapshot.brokerList, dataSourceSnapshot.topic, dataSourceSnapshot.convertedCustomProperties, partitionOffsets); } - dataSourceProperties.setKafkaPartitionOffsets(newOffsets); + return KafkaUtil.getRealOffsets(dataSourceSnapshot.brokerList, dataSourceSnapshot.topic, + dataSourceSnapshot.convertedCustomProperties, partitionOffsets); } private void modifyPropertiesInternal(Map jobProperties, KafkaDataSourceProperties dataSourceProperties) throws UserException { - modifyPropertiesInternal(jobProperties, dataSourceProperties, false); + PreparedKafkaProperties preparedDataSourceProperties = prepareDataSourceProperties(dataSourceProperties, true); + modifyPropertiesInternal(jobProperties, dataSourceProperties, preparedDataSourceProperties); } private void modifyPropertiesInternal(Map jobProperties, KafkaDataSourceProperties dataSourceProperties, - boolean isReplay) + PreparedKafkaProperties preparedDataSourceProperties) throws UserException { if (null != dataSourceProperties) { - KafkaDataSourceSnapshot dataSourceSnapshot = stageDataSourceProperties(dataSourceProperties); - List> kafkaPartitionOffsets = Lists.newArrayList(); - - if (MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { - kafkaPartitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); - } - - if (!kafkaPartitionOffsets.isEmpty() - && (!dataSourceSnapshot.resetProgress || CollectionUtils.isNotEmpty(customKafkaPartitions))) { - ((KafkaProgress) progress).checkPartitions(kafkaPartitionOffsets); - } - if (!isReplay) { - convertOffset(dataSourceProperties, dataSourceSnapshot); - kafkaPartitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); - } - - if (Config.isCloudMode() && !isReplay - && (dataSourceSnapshot.resetProgress || !kafkaPartitionOffsets.isEmpty())) { - Cloud.ResetRLProgressRequest.Builder builder = Cloud.ResetRLProgressRequest.newBuilder() - .setRequestIp(FrontendOptions.getLocalHostAddressCached()); - builder.setCloudUniqueId(Config.cloud_unique_id); - builder.setDbId(dbId); - builder.setJobId(id); - if (!kafkaPartitionOffsets.isEmpty()) { - Map partitionOffsetMap = new HashMap<>(); - for (Pair pair : kafkaPartitionOffsets) { - // The reason why the value recorded in MS in cloud mode needs to be subtracted by one is - // this value will be incremented - // when pulling MS persistent progress data and updating memory - // in routineLoadJob.updateCloudProgress(). - partitionOffsetMap.put(pair.first, pair.second - 1); - } - builder.putAllPartitionToOffset(partitionOffsetMap); - } - resetCloudProgress(builder); - } + Preconditions.checkNotNull(preparedDataSourceProperties); + KafkaDataSourceSnapshot dataSourceSnapshot = preparedDataSourceProperties.dataSourceSnapshot; + List> kafkaPartitionOffsets = preparedDataSourceProperties.kafkaPartitionOffsets; + dataSourceProperties.setKafkaPartitionOffsets(kafkaPartitionOffsets); if (dataSourceSnapshot.customPropertiesChanged) { this.customProperties.clear(); @@ -886,6 +853,61 @@ private void modifyPropertiesInternal(Map jobProperties, this.id, jobProperties, dataSourceProperties); } + private PreparedKafkaProperties prepareDataSourceProperties(KafkaDataSourceProperties dataSourceProperties, + boolean isReplay) throws UserException { + if (dataSourceProperties == null) { + return null; + } + + KafkaDataSourceSnapshot dataSourceSnapshot = stageDataSourceProperties(dataSourceProperties); + List> kafkaPartitionOffsets = Lists.newArrayList(); + if (MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { + kafkaPartitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); + } + if (!kafkaPartitionOffsets.isEmpty() + && (!dataSourceSnapshot.resetProgress || CollectionUtils.isNotEmpty(customKafkaPartitions))) { + ((KafkaProgress) progress).checkPartitions(kafkaPartitionOffsets); + } + + if (!isReplay) { + kafkaPartitionOffsets = resolveOffsets(dataSourceProperties, dataSourceSnapshot); + resetCloudProgressIfNeeded(dataSourceSnapshot, kafkaPartitionOffsets); + } + return new PreparedKafkaProperties(dataSourceSnapshot, copyPartitionOffsets(kafkaPartitionOffsets)); + } + + private List> copyPartitionOffsets(List> kafkaPartitionOffsets) { + List> copiedPartitionOffsets = Lists.newArrayListWithCapacity(kafkaPartitionOffsets.size()); + for (Pair partitionOffset : kafkaPartitionOffsets) { + copiedPartitionOffsets.add(Pair.of(partitionOffset.first, partitionOffset.second)); + } + return copiedPartitionOffsets; + } + + private void resetCloudProgressIfNeeded(KafkaDataSourceSnapshot dataSourceSnapshot, + List> kafkaPartitionOffsets) throws DdlException { + if (!Config.isCloudMode() || (!dataSourceSnapshot.resetProgress && kafkaPartitionOffsets.isEmpty())) { + return; + } + + Cloud.ResetRLProgressRequest.Builder builder = Cloud.ResetRLProgressRequest.newBuilder() + .setRequestIp(FrontendOptions.getLocalHostAddressCached()); + builder.setCloudUniqueId(Config.cloud_unique_id); + builder.setDbId(dbId); + builder.setJobId(id); + if (!kafkaPartitionOffsets.isEmpty()) { + Map partitionOffsetMap = new HashMap<>(); + for (Pair pair : kafkaPartitionOffsets) { + // The reason why the value recorded in MS in cloud mode needs to be subtracted by one is + // this value will be incremented when pulling MS persistent progress data and updating memory + // in routineLoadJob.updateCloudProgress(). + partitionOffsetMap.put(pair.first, pair.second - 1); + } + builder.putAllPartitionToOffset(partitionOffsetMap); + } + resetCloudProgress(builder); + } + private KafkaDataSourceSnapshot stageDataSourceProperties(KafkaDataSourceProperties dataSourceProperties) throws DdlException { Map stagedCustomProperties = Maps.newHashMap(customProperties); @@ -931,6 +953,17 @@ private KafkaDataSourceSnapshot(String brokerList, String topic, Map> kafkaPartitionOffsets; + + private PreparedKafkaProperties(KafkaDataSourceSnapshot dataSourceSnapshot, + List> kafkaPartitionOffsets) { + this.dataSourceSnapshot = dataSourceSnapshot; + this.kafkaPartitionOffsets = kafkaPartitionOffsets; + } + } + private void resetCloudProgress(Cloud.ResetRLProgressRequest.Builder builder) throws DdlException { Cloud.ResetRLProgressResponse response; try { @@ -954,7 +987,7 @@ private void resetCloudProgress(Cloud.ResetRLProgressRequest.Builder builder) th public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { try { modifyPropertiesInternal(log.getJobProperties(), - (KafkaDataSourceProperties) log.getDataSourceProperties(), true); + (KafkaDataSourceProperties) log.getDataSourceProperties()); if (log.getTargetTableId() != 0) { this.tableId = log.getTargetTableId(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index 5edf8e1ccfc107..e568a8915ab2b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -674,11 +674,11 @@ private Map getMaskedCustomProperties(String keyPrefix) { } @Override - protected void unprotectModifyProperties(AlterRoutineLoadCommand command) throws UserException { + protected PreparedAlter prepareAlter(AlterRoutineLoadCommand command) { Map jobProperties = command.getAnalyzedJobProperties(); KinesisDataSourceProperties dataSourceProperties = (KinesisDataSourceProperties) command.getDataSourceProperties(); - modifyPropertiesInternal(jobProperties, dataSourceProperties); + return () -> modifyPropertiesInternal(jobProperties, dataSourceProperties); } private void modifyPropertiesInternal(Map jobProperties, diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index cfc5c922d8766d..016f601a74f63a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -249,16 +249,25 @@ public void testModifyPropertiesRevalidatesWhileHoldingWriteLock() throws Except } @Test - public void testModifyTargetTableValidatesAndLogsUnderMetadataLocks() throws Exception { + public void testModifyTargetTablePreparesExternalIoBeforeMetadataLocks() throws Exception { KafkaRoutineLoadJob routineLoadJob = Mockito.spy(new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN)); Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); Deencapsulation.setField(routineLoadJob, "uniqueKeyUpdateMode", TUniqueKeyUpdateMode.UPSERT); + KafkaProgress originalProgress = new KafkaProgress(Maps.newHashMap(ImmutableMap.of(1, 10L))); + Deencapsulation.setField(routineLoadJob, "progress", originalProgress); + + Map alteredSourceProperties = Maps.newHashMap(); + alteredSourceProperties.put(KafkaConfiguration.KAFKA_TOPIC.getName(), "topic2"); + alteredSourceProperties.put(KafkaConfiguration.KAFKA_PARTITIONS.getName(), "1"); + alteredSourceProperties.put(KafkaConfiguration.KAFKA_OFFSETS.getName(), KafkaProgress.OFFSET_BEGINNING); + KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties(alteredSourceProperties); AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); Mockito.when(command.hasTargetTable()).thenReturn(true); Mockito.when(command.getTargetTableId()).thenReturn(2L); Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + Mockito.when(command.getDataSourceProperties()).thenReturn(dataSourceProperties); Env env = Mockito.mock(Env.class); EditLog editLog = Mockito.mock(EditLog.class); @@ -271,6 +280,8 @@ public void testModifyTargetTableValidatesAndLogsUnderMetadataLocks() throws Exc AtomicBoolean databaseLocked = new AtomicBoolean(false); AtomicBoolean tableLocked = new AtomicBoolean(false); + AtomicBoolean kafkaPrepared = new AtomicBoolean(false); + AtomicBoolean cloudPrepared = new AtomicBoolean(false); AtomicBoolean targetValidated = new AtomicBoolean(false); Mockito.doAnswer(invocation -> { databaseLocked.set(true); @@ -294,6 +305,11 @@ public void testModifyTargetTableValidatesAndLogsUnderMetadataLocks() throws Exc Assert.assertTrue(lock.isWriteLockedByCurrentThread()); Assert.assertTrue(databaseLocked.get()); Assert.assertTrue(tableLocked.get()); + Assert.assertTrue(kafkaPrepared.get()); + Assert.assertTrue(cloudPrepared.get()); + Assert.assertEquals(1L, routineLoadJob.getTableId()); + Assert.assertEquals("topic1", routineLoadJob.getTopic()); + Assert.assertSame(originalProgress, routineLoadJob.getProgress()); targetValidated.set(true); return null; }).when(routineLoadJob).unprotectedValidateTargetTable(Mockito.eq(database), Mockito.eq(targetTable), @@ -301,22 +317,57 @@ public void testModifyTargetTableValidatesAndLogsUnderMetadataLocks() throws Exc Mockito.doAnswer(invocation -> { Assert.assertTrue(databaseLocked.get()); Assert.assertTrue(tableLocked.get()); + Assert.assertTrue(targetValidated.get()); Assert.assertEquals(2L, routineLoadJob.getTableId()); + Assert.assertEquals("topic2", routineLoadJob.getTopic()); return null; }).when(editLog).logAlterRoutineLoadJob(Mockito.any(AlterRoutineLoadJobOperationLog.class)); - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + MetaServiceProxy metaServiceProxy = Mockito.mock(MetaServiceProxy.class); + Cloud.ResetRLProgressResponse response = Cloud.ResetRLProgressResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder().setCode(Cloud.MetaServiceCode.OK)) + .build(); + String originalCloudUniqueId = Config.cloud_unique_id; + Config.cloud_unique_id = "test-cloud"; + try (MockedStatic envStatic = Mockito.mockStatic(Env.class); + MockedStatic kafkaUtilStatic = Mockito.mockStatic(KafkaUtil.class); + MockedStatic metaServiceProxyStatic = Mockito.mockStatic(MetaServiceProxy.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + kafkaUtilStatic.when(() -> KafkaUtil.getRealOffsets(Mockito.eq("127.0.0.1:9020"), + Mockito.eq("topic2"), Mockito.anyMap(), Mockito.anyList())).thenAnswer(invocation -> { + ReentrantReadWriteLock lock = Deencapsulation.getField(routineLoadJob, "lock"); + Assert.assertTrue(lock.isWriteLockedByCurrentThread()); + Assert.assertFalse(databaseLocked.get()); + Assert.assertFalse(tableLocked.get()); + kafkaPrepared.set(true); + return Lists.newArrayList(Pair.of(1, 100L)); + }); + metaServiceProxyStatic.when(MetaServiceProxy::getInstance).thenReturn(metaServiceProxy); + Mockito.when(metaServiceProxy.resetRLProgress(Mockito.any())).thenAnswer(invocation -> { + ReentrantReadWriteLock lock = Deencapsulation.getField(routineLoadJob, "lock"); + Assert.assertTrue(lock.isWriteLockedByCurrentThread()); + Assert.assertFalse(databaseLocked.get()); + Assert.assertFalse(tableLocked.get()); + Assert.assertTrue(kafkaPrepared.get()); + cloudPrepared.set(true); + return response; + }); routineLoadJob.modifyProperties(command); + } finally { + Config.cloud_unique_id = originalCloudUniqueId; } + Assert.assertTrue(kafkaPrepared.get()); + Assert.assertTrue(cloudPrepared.get()); Assert.assertTrue(targetValidated.get()); Assert.assertFalse(databaseLocked.get()); Assert.assertFalse(tableLocked.get()); Assert.assertEquals(2L, routineLoadJob.getTableId()); + Assert.assertEquals("topic2", routineLoadJob.getTopic()); Mockito.verify(editLog).logAlterRoutineLoadJob(Mockito.any(AlterRoutineLoadJobOperationLog.class)); + Mockito.verify(metaServiceProxy).resetRLProgress(Mockito.any()); } @Test From e93f3004586b2e899c3123850c5499964622d66b Mon Sep 17 00:00:00 2001 From: Refrain Date: Fri, 17 Jul 2026 17:20:19 +0800 Subject: [PATCH 14/19] [fix](fe) Make routine load ALTER state durable ### What problem does this PR solve? Issue Number: None Related PR: #64878 Problem Summary: Kafka routine load ALTER could keep a stale datetime origin when a symbolic default offset was supplied, reset Cloud MetaService progress before final metadata validation and edit-log persistence, and replay resolved ALTER state against follower-local progress. Routine load descriptors were also applied outside the Kafka job lock and were absent from the ALTER journal. Restore the pre-existing Kinesis ALTER implementation, keep the new preparation flow Kafka-only, persist pending Cloud progress changes until the job resumes, make replay deterministic, and journal descriptor changes with the target-table mutation. ### Release note Fix Kafka routine load ALTER consistency for default offsets, Cloud progress, target-table changes, and replay. ### Check List (For Author) - Test: FE unit tests and FE build - `./run-fe-ut.sh --run org.apache.doris.load.routineload.KafkaRoutineLoadJobTest,org.apache.doris.load.routineload.KinesisRoutineLoadJobTest,org.apache.doris.load.routineload.RoutineLoadJobTest,org.apache.doris.load.routineload.RoutineLoadManagerTest,org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommandTest,org.apache.doris.persist.AlterRoutineLoadOperationLogTest` - `./run-fe-ut.sh --run org.apache.doris.load.routineload.KafkaRoutineLoadJobTest` - `./build.sh --fe -j48` - Behavior changed: Yes. Cloud progress changes are submitted when the paused Kafka job resumes, after ALTER validation and persistence; symbolic default offsets now replace an earlier datetime default correctly. - Does this need documentation: No --- .../load/routineload/RoutineLoadJob.java | 131 +++++++------- .../load/routineload/RoutineLoadManager.java | 4 +- .../kafka/KafkaRoutineLoadJob.java | 161 ++++++++++++++++-- .../kinesis/KinesisRoutineLoadJob.java | 22 ++- .../commands/AlterRoutineLoadCommand.java | 6 + .../AlterRoutineLoadJobOperationLog.java | 79 +++++++++ .../routineload/KafkaRoutineLoadJobTest.java | 126 ++++++++++++-- .../AlterRoutineLoadOperationLogTest.java | 9 +- 8 files changed, 438 insertions(+), 100 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index b67aed00f8e665..4f919340c3f2b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -1567,6 +1567,9 @@ public void updateState(JobState jobState, ErrorReason reason, boolean isReplay) protected void unprotectUpdateState(JobState jobState, ErrorReason reason, boolean isReplay) throws UserException { checkStateTransform(jobState); + if (jobState == JobState.NEED_SCHEDULE) { + beforeTransitionToNeedSchedule(isReplay); + } switch (jobState) { case RUNNING: executeRunning(); @@ -1600,6 +1603,9 @@ protected void unprotectUpdateState(JobState jobState, ErrorReason reason, boole } } + protected void beforeTransitionToNeedSchedule(boolean isReplay) throws UserException { + } + private void executeRunning() { state = JobState.RUNNING; } @@ -2127,79 +2133,27 @@ public void gsonPostProcess() throws IOException { } } - public void modifyProperties(AlterRoutineLoadCommand command) throws UserException { - writeLock(); - try { - if (getState() != JobState.PAUSED) { - throw new DdlException("Only supports modification of PAUSED jobs"); - } - if (!command.hasTargetTable()) { - validateAlterJobPropertiesForMutation(command); - } + public abstract void modifyProperties(AlterRoutineLoadCommand command) throws UserException; - // Data-source preparation may perform external I/O. Keep it outside DB/table metadata locks. - PreparedAlter preparedAlter = prepareAlter(command); - if (!command.hasTargetTable()) { - unprotectApplyAlter(command, preparedAlter); - return; - } - - Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId); - if (db == null) { - throw new DdlException("Database not found: " + dbId); - } - db.readLock(); - try { - Table table = db.getTableNullable(command.getTargetTableId()); - if (table == null) { - throw new DdlException("Table not found: " + command.getTargetTableId()); - } - if (!(table instanceof OlapTable)) { - throw new DdlException("Routine load target table must be an OLAP table"); - } - table.readLock(); - try { - Map alteredJobProperties = command.getAnalyzedJobProperties(); - TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode = getEffectiveUniqueKeyUpdateMode( - uniqueKeyUpdateMode, alteredJobProperties); - unprotectedValidateTargetTable( - db, (OlapTable) table, alteredJobProperties, effectiveUniqueKeyUpdateMode); - unprotectApplyAlter(command, preparedAlter); - } finally { - table.readUnlock(); - } - } finally { - db.readUnlock(); - } - } finally { - writeUnlock(); - } - } - - private void unprotectApplyAlter(AlterRoutineLoadCommand command, PreparedAlter preparedAlter) - throws UserException { - preparedAlter.apply(); - if (command.hasTargetTable()) { - tableId = command.getTargetTableId(); - } - AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, - command.getAnalyzedJobProperties(), command.getDataSourceProperties(), command.getTargetTableId()); - Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); - } - - @FunctionalInterface - protected interface PreparedAlter { - void apply() throws UserException; + public boolean appliesRoutineLoadDescAtomically() { + return false; } - protected abstract PreparedAlter prepareAlter(AlterRoutineLoadCommand command) throws UserException; - public abstract void replayModifyProperties(AlterRoutineLoadJobOperationLog log); public abstract NereidsRoutineLoadTaskInfo toNereidsRoutineLoadTaskInfo() throws UserException; // for ALTER ROUTINE LOAD protected void modifyCommonJobProperties(Map jobProperties) throws UserException { + modifyCommonJobProperties(jobProperties, false); + } + + protected void modifyCommonJobPropertiesForKafka(Map jobProperties) throws UserException { + modifyCommonJobProperties(jobProperties, true); + } + + private void modifyCommonJobProperties(Map jobProperties, + boolean explicitUniqueKeyUpdateModeTakesPrecedence) throws UserException { if (jobProperties.containsKey(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)) { this.desireTaskConcurrentNum = Integer.parseInt( jobProperties.remove(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)); @@ -2231,16 +2185,20 @@ protected void modifyCommonJobProperties(Map jobProperties) thro jobProperties.remove(CreateRoutineLoadInfo.MAX_BATCH_SIZE_PROPERTY)); } - boolean hasExplicitUniqueKeyUpdateMode = jobProperties.containsKey( - CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); - if (hasExplicitUniqueKeyUpdateMode) { + if (jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { String modeStr = jobProperties.remove(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); TUniqueKeyUpdateMode newMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode(modeStr); + if (!explicitUniqueKeyUpdateModeTakesPrecedence + && newMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS) { + validateFlexiblePartialUpdateForAlter(); + } this.uniqueKeyUpdateMode = newMode; this.isPartialUpdate = (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS); this.jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); - jobProperties.remove(CreateRoutineLoadInfo.PARTIAL_COLUMNS); + if (explicitUniqueKeyUpdateModeTakesPrecedence) { + jobProperties.remove(CreateRoutineLoadInfo.PARTIAL_COLUMNS); + } } if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { @@ -2256,6 +2214,43 @@ protected void modifyCommonJobProperties(Map jobProperties) thro } } + private void validateFlexiblePartialUpdateForAlter() throws UserException { + if (isMultiTable) { + throw new DdlException("Flexible partial update is not supported in multi-table load"); + } + + Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId); + if (db == null) { + throw new DdlException("Database not found: " + dbId); + } + Table table = db.getTableNullable(tableId); + if (table == null) { + throw new DdlException("Table not found: " + tableId); + } + if (!(table instanceof OlapTable)) { + throw new DdlException("Flexible partial update is only supported for OLAP tables"); + } + OlapTable olapTable = (OlapTable) table; + + olapTable.validateForFlexiblePartialUpdate(); + String format = this.jobProperties.getOrDefault(FileFormatProperties.PROP_FORMAT, "csv"); + if (!"json".equalsIgnoreCase(format)) { + throw new DdlException("Flexible partial update only supports JSON format, but current job uses: " + + format); + } + if (Boolean.parseBoolean(this.jobProperties.getOrDefault( + JsonFileFormatProperties.PROP_FUZZY_PARSE, "false"))) { + throw new DdlException("Flexible partial update does not support fuzzy_parse"); + } + String jsonPaths = getJsonPaths(); + if (jsonPaths != null && !jsonPaths.isEmpty()) { + throw new DdlException("Flexible partial update does not support jsonpaths"); + } + if (columnDescs != null && !columnDescs.descs.isEmpty()) { + throw new DdlException("Flexible partial update does not support COLUMNS specification"); + } + } + /** * Validate flexible partial update constraints when altering routine load job. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java index df5615016bfa54..0e1000f3dae370 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java @@ -943,7 +943,9 @@ public void alterRoutineLoadJob(AlterRoutineLoadCommand command) throws UserExce + command.getDataSourceProperties().getDataSourceType()); } job.modifyProperties(command); - job.setRoutineLoadDesc(command.getRoutineLoadDesc()); + if (!job.appliesRoutineLoadDescAtomically()) { + job.setRoutineLoadDesc(command.getRoutineLoadDesc()); + } } public void replayAlterRoutineLoadJob(AlterRoutineLoadJobOperationLog log) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java index 2a982a9b3e66c4..758cf1f621cd03 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java @@ -25,6 +25,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.RandomDistributionInfo; +import org.apache.doris.catalog.Table; import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.cloud.rpc.MetaServiceProxy; import org.apache.doris.common.Config; @@ -61,6 +62,7 @@ import org.apache.doris.service.FrontendOptions; import org.apache.doris.thrift.TFileCompressType; import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; +import org.apache.doris.thrift.TUniqueKeyUpdateMode; import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; @@ -109,6 +111,8 @@ public class KafkaRoutineLoadJob extends RoutineLoadJob { private String brokerList; @SerializedName("tp") private String topic; + @SerializedName("pendingCloudProgressReset") + private PendingCloudProgressReset pendingCloudProgressReset; // optional, user want to load partitions. @SerializedName("cskp") private List customKafkaPartitions = Lists.newArrayList(); @@ -438,11 +442,24 @@ protected void unprotectUpdateProgress() throws UserException { // For cloud mode, should update cloud progress from meta service, // then update progress with default offset from Kafka if necessary. if (Config.isCloudMode()) { + resetPendingCloudProgress(); updateCloudProgress(); } updateNewPartitionProgress(); } + @Override + protected void beforeTransitionToNeedSchedule(boolean isReplay) throws UserException { + if (pendingCloudProgressReset == null) { + return; + } + if (isReplay) { + pendingCloudProgressReset = null; + return; + } + resetPendingCloudProgress(); + } + @Override protected boolean refreshKafkaPartitions(boolean needAutoResume) throws UserException { // If user does not specify kafka partition, @@ -772,11 +789,80 @@ public Map getCustomProperties() { } @Override - protected PreparedAlter prepareAlter(AlterRoutineLoadCommand command) throws UserException { + public boolean appliesRoutineLoadDescAtomically() { + return true; + } + + @Override + public void modifyProperties(AlterRoutineLoadCommand command) throws UserException { Map jobProperties = command.getAnalyzedJobProperties(); KafkaDataSourceProperties dataSourceProperties = (KafkaDataSourceProperties) command.getDataSourceProperties(); - PreparedKafkaProperties preparedDataSourceProperties = prepareDataSourceProperties(dataSourceProperties, false); - return () -> modifyPropertiesInternal(jobProperties, dataSourceProperties, preparedDataSourceProperties); + + writeLock(); + try { + if (getState() != JobState.PAUSED) { + throw new DdlException("Only supports modification of PAUSED jobs"); + } + if (command.getRoutineLoadDesc() != null && command.getLoadPropertyTableId() != tableId) { + throw new DdlException("Routine load target table changed while validating load properties; " + + "please retry ALTER ROUTINE LOAD"); + } + if (!command.hasTargetTable()) { + validateAlterJobPropertiesForMutation(command); + } + + // Kafka offset resolution may perform external I/O. Keep it outside DB/table metadata locks. + PreparedKafkaProperties preparedDataSourceProperties = + prepareDataSourceProperties(dataSourceProperties, false); + if (!command.hasTargetTable()) { + unprotectApplyAlter(command, dataSourceProperties, preparedDataSourceProperties); + return; + } + + Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId); + if (db == null) { + throw new DdlException("Database not found: " + dbId); + } + db.readLock(); + try { + Table table = db.getTableNullable(command.getTargetTableId()); + if (table == null) { + throw new DdlException("Table not found: " + command.getTargetTableId()); + } + if (!(table instanceof OlapTable)) { + throw new DdlException("Routine load target table must be an OLAP table"); + } + table.readLock(); + try { + TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode = getEffectiveUniqueKeyUpdateMode( + uniqueKeyUpdateMode, jobProperties); + unprotectedValidateTargetTable( + db, (OlapTable) table, jobProperties, effectiveUniqueKeyUpdateMode); + unprotectApplyAlter(command, dataSourceProperties, preparedDataSourceProperties); + } finally { + table.readUnlock(); + } + } finally { + db.readUnlock(); + } + } finally { + writeUnlock(); + } + } + + private void unprotectApplyAlter(AlterRoutineLoadCommand command, + KafkaDataSourceProperties dataSourceProperties, + PreparedKafkaProperties preparedDataSourceProperties) throws UserException { + modifyPropertiesInternal(command.getAnalyzedJobProperties(), dataSourceProperties, + preparedDataSourceProperties); + if (command.hasTargetTable()) { + tableId = command.getTargetTableId(); + } + setRoutineLoadDesc(command.getRoutineLoadDesc()); + AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(id, + command.getAnalyzedJobProperties(), dataSourceProperties, command.getTargetTableId(), + command.getRoutineLoadDesc()); + Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); } private List> resolveOffsets(KafkaDataSourceProperties dataSourceProperties, @@ -831,6 +917,8 @@ private void modifyPropertiesInternal(Map jobProperties, ((KafkaProgress) progress).modifyOffset(kafkaPartitionOffsets); } + mergePendingCloudProgressReset(dataSourceSnapshot, kafkaPartitionOffsets); + // modify broker list if (!Strings.isNullOrEmpty(dataSourceProperties.getBrokerList())) { this.brokerList = dataSourceSnapshot.brokerList; @@ -838,7 +926,7 @@ private void modifyPropertiesInternal(Map jobProperties, } if (!jobProperties.isEmpty()) { Map copiedJobProperties = Maps.newHashMap(jobProperties); - modifyCommonJobProperties(copiedJobProperties); + modifyCommonJobPropertiesForKafka(copiedJobProperties); this.jobProperties.putAll(copiedJobProperties); if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { String policy = jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); @@ -864,14 +952,13 @@ private PreparedKafkaProperties prepareDataSourceProperties(KafkaDataSourcePrope if (MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { kafkaPartitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); } - if (!kafkaPartitionOffsets.isEmpty() + if (!isReplay && !kafkaPartitionOffsets.isEmpty() && (!dataSourceSnapshot.resetProgress || CollectionUtils.isNotEmpty(customKafkaPartitions))) { ((KafkaProgress) progress).checkPartitions(kafkaPartitionOffsets); } if (!isReplay) { kafkaPartitionOffsets = resolveOffsets(dataSourceProperties, dataSourceSnapshot); - resetCloudProgressIfNeeded(dataSourceSnapshot, kafkaPartitionOffsets); } return new PreparedKafkaProperties(dataSourceSnapshot, copyPartitionOffsets(kafkaPartitionOffsets)); } @@ -884,28 +971,48 @@ private List> copyPartitionOffsets(List> return copiedPartitionOffsets; } - private void resetCloudProgressIfNeeded(KafkaDataSourceSnapshot dataSourceSnapshot, - List> kafkaPartitionOffsets) throws DdlException { + private void mergePendingCloudProgressReset(KafkaDataSourceSnapshot dataSourceSnapshot, + List> kafkaPartitionOffsets) { if (!Config.isCloudMode() || (!dataSourceSnapshot.resetProgress && kafkaPartitionOffsets.isEmpty())) { return; } + Map partitionOffsetMap = new HashMap<>(); + for (Pair pair : kafkaPartitionOffsets) { + // MS stores the last consumed offset, while FE progress stores the next offset to consume. + partitionOffsetMap.put(pair.first, pair.second - 1); + } + PendingCloudProgressReset newReset = new PendingCloudProgressReset( + dataSourceSnapshot.resetProgress, partitionOffsetMap); + if (pendingCloudProgressReset == null || newReset.clearProgress) { + pendingCloudProgressReset = newReset; + } else { + pendingCloudProgressReset.partitionToOffset.putAll(newReset.partitionToOffset); + } + } + + private Cloud.ResetRLProgressRequest.Builder newResetCloudProgressRequest() { Cloud.ResetRLProgressRequest.Builder builder = Cloud.ResetRLProgressRequest.newBuilder() .setRequestIp(FrontendOptions.getLocalHostAddressCached()); builder.setCloudUniqueId(Config.cloud_unique_id); builder.setDbId(dbId); builder.setJobId(id); - if (!kafkaPartitionOffsets.isEmpty()) { - Map partitionOffsetMap = new HashMap<>(); - for (Pair pair : kafkaPartitionOffsets) { - // The reason why the value recorded in MS in cloud mode needs to be subtracted by one is - // this value will be incremented when pulling MS persistent progress data and updating memory - // in routineLoadJob.updateCloudProgress(). - partitionOffsetMap.put(pair.first, pair.second - 1); - } - builder.putAllPartitionToOffset(partitionOffsetMap); + return builder; + } + + private void resetPendingCloudProgress() throws DdlException { + if (pendingCloudProgressReset == null) { + return; + } + if (pendingCloudProgressReset.clearProgress) { + resetCloudProgress(newResetCloudProgressRequest()); } - resetCloudProgress(builder); + if (!pendingCloudProgressReset.partitionToOffset.isEmpty()) { + Cloud.ResetRLProgressRequest.Builder builder = newResetCloudProgressRequest(); + builder.putAllPartitionToOffset(pendingCloudProgressReset.partitionToOffset); + resetCloudProgress(builder); + } + pendingCloudProgressReset = null; } private KafkaDataSourceSnapshot stageDataSourceProperties(KafkaDataSourceProperties dataSourceProperties) @@ -916,6 +1023,11 @@ private KafkaDataSourceSnapshot stageDataSourceProperties(KafkaDataSourcePropert Map alteredCustomProperties = dataSourceProperties.getCustomKafkaProperties(); boolean customPropertiesChanged = MapUtils.isNotEmpty(alteredCustomProperties); if (customPropertiesChanged) { + if (alteredCustomProperties.containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()) + && !alteredCustomProperties.containsKey( + KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())) { + stagedCustomProperties.remove(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName()); + } stagedCustomProperties.putAll(alteredCustomProperties); Pair, String> convertedProperties = buildConvertedCustomProperties( stagedCustomProperties, stagedKafkaDefaultOffset); @@ -964,6 +1076,18 @@ private PreparedKafkaProperties(KafkaDataSourceSnapshot dataSourceSnapshot, } } + private static class PendingCloudProgressReset { + @SerializedName("clearProgress") + private boolean clearProgress; + @SerializedName("partitionToOffset") + private Map partitionToOffset = new HashMap<>(); + + private PendingCloudProgressReset(boolean clearProgress, Map partitionToOffset) { + this.clearProgress = clearProgress; + this.partitionToOffset.putAll(partitionToOffset); + } + } + private void resetCloudProgress(Cloud.ResetRLProgressRequest.Builder builder) throws DdlException { Cloud.ResetRLProgressResponse response; try { @@ -991,6 +1115,7 @@ public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { if (log.getTargetTableId() != 0) { this.tableId = log.getTargetTableId(); } + setRoutineLoadDesc(log.getRoutineLoadDesc()); } catch (UserException e) { // should not happen LOG.error("failed to replay modify kafka routine load job: {}", id, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index e568a8915ab2b3..7cebc3f5165b49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -65,6 +65,7 @@ import com.google.gson.annotations.SerializedName; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.BooleanUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -674,11 +675,25 @@ private Map getMaskedCustomProperties(String keyPrefix) { } @Override - protected PreparedAlter prepareAlter(AlterRoutineLoadCommand command) { + public void modifyProperties(AlterRoutineLoadCommand command) throws UserException { Map jobProperties = command.getAnalyzedJobProperties(); KinesisDataSourceProperties dataSourceProperties = (KinesisDataSourceProperties) command.getDataSourceProperties(); - return () -> modifyPropertiesInternal(jobProperties, dataSourceProperties); + + writeLock(); + try { + if (getState() != JobState.PAUSED) { + throw new DdlException("Only supports modification of PAUSED jobs"); + } + + modifyPropertiesInternal(jobProperties, dataSourceProperties); + + AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, + jobProperties, dataSourceProperties); + Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); + } finally { + writeUnlock(); + } } private void modifyPropertiesInternal(Map jobProperties, @@ -747,6 +762,9 @@ private void modifyPropertiesInternal(Map jobProperties, Map copiedJobProperties = Maps.newHashMap(jobProperties); modifyCommonJobProperties(copiedJobProperties); this.jobProperties.putAll(copiedJobProperties); + if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { + this.isPartialUpdate = BooleanUtils.toBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); + } if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { String policy = jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); if ("ERROR".equalsIgnoreCase(policy)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java index 9f464c4391e98e..175bf4d5fc8ca7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java @@ -100,6 +100,7 @@ public class AlterRoutineLoadCommand extends AlterCommand { private final String dataSourceType; private final Map dataSourceMapProperties; private long targetTableId; + private long loadPropertyTableId; private boolean isPartialUpdate; // save analyzed job properties. @@ -160,6 +161,10 @@ public long getTargetTableId() { return targetTableId; } + public long getLoadPropertyTableId() { + return loadPropertyTableId; + } + public boolean hasDataSourceProperty() { return MapUtils.isNotEmpty(dataSourceMapProperties); } @@ -194,6 +199,7 @@ public void validate(ConnectContext ctx) throws UserException { RoutineLoadJob job = Env.getCurrentEnv().getRoutineLoadManager() .getJob(getDbName(), getJobName()); if (MapUtils.isNotEmpty(loadPropertyMap)) { + this.loadPropertyTableId = job.getTableId(); this.routineLoadDesc = CreateRoutineLoadInfo.checkLoadProperties(ctx, loadPropertyMap, job.getDbFullName(), job.getTableName(), job.isMultiTable(), job.getMergeType()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java index ae7c8ad214de7c..f69129bd74097f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java @@ -17,8 +17,14 @@ package org.apache.doris.persist; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.ImportColumnDesc; +import org.apache.doris.analysis.Separator; +import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.load.RoutineLoadDesc; +import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.AbstractDataSourceProperties; import org.apache.doris.persist.gson.GsonUtils; @@ -27,6 +33,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.List; import java.util.Map; public class AlterRoutineLoadJobOperationLog implements Writable { @@ -39,6 +46,8 @@ public class AlterRoutineLoadJobOperationLog implements Writable { private AbstractDataSourceProperties dataSourceProperties; @SerializedName(value = "targetTableId") private long targetTableId; + @SerializedName(value = "routineLoadDesc") + private RoutineLoadDescSnapshot routineLoadDesc; public AlterRoutineLoadJobOperationLog(long jobId, Map jobProperties, AbstractDataSourceProperties dataSourceProperties) { @@ -47,10 +56,17 @@ public AlterRoutineLoadJobOperationLog(long jobId, Map jobProper public AlterRoutineLoadJobOperationLog(long jobId, Map jobProperties, AbstractDataSourceProperties dataSourceProperties, long targetTableId) { + this(jobId, jobProperties, dataSourceProperties, targetTableId, null); + } + + public AlterRoutineLoadJobOperationLog(long jobId, Map jobProperties, + AbstractDataSourceProperties dataSourceProperties, long targetTableId, + RoutineLoadDesc routineLoadDesc) { this.jobId = jobId; this.jobProperties = jobProperties; this.dataSourceProperties = dataSourceProperties; this.targetTableId = targetTableId; + this.routineLoadDesc = routineLoadDesc == null ? null : new RoutineLoadDescSnapshot(routineLoadDesc); } public long getJobId() { @@ -69,6 +85,69 @@ public long getTargetTableId() { return targetTableId; } + public RoutineLoadDesc getRoutineLoadDesc() { + return routineLoadDesc == null ? null : routineLoadDesc.toRoutineLoadDesc(); + } + + private static class RoutineLoadDescSnapshot { + @SerializedName("columnSeparator") + private SeparatorSnapshot columnSeparator; + @SerializedName("lineDelimiter") + private SeparatorSnapshot lineDelimiter; + @SerializedName("columnsInfo") + private List columnsInfo; + @SerializedName("precedingFilter") + private Expr precedingFilter; + @SerializedName("filter") + private Expr filter; + @SerializedName("partitionNamesInfo") + private PartitionNamesInfo partitionNamesInfo; + @SerializedName("deleteCondition") + private Expr deleteCondition; + @SerializedName("mergeType") + private LoadTask.MergeType mergeType; + @SerializedName("sequenceColName") + private String sequenceColName; + + private RoutineLoadDescSnapshot(RoutineLoadDesc routineLoadDesc) { + this.columnSeparator = SeparatorSnapshot.fromSeparator(routineLoadDesc.getColumnSeparator()); + this.lineDelimiter = SeparatorSnapshot.fromSeparator(routineLoadDesc.getLineDelimiter()); + this.columnsInfo = routineLoadDesc.getColumnsInfo(); + this.precedingFilter = routineLoadDesc.getPrecedingFilter(); + this.filter = routineLoadDesc.getFilter(); + this.partitionNamesInfo = routineLoadDesc.getPartitionNamesInfo(); + this.deleteCondition = routineLoadDesc.getDeleteCondition(); + this.mergeType = routineLoadDesc.getMergeType(); + this.sequenceColName = routineLoadDesc.getSequenceColName(); + } + + private RoutineLoadDesc toRoutineLoadDesc() { + return new RoutineLoadDesc(SeparatorSnapshot.toSeparator(columnSeparator), + SeparatorSnapshot.toSeparator(lineDelimiter), columnsInfo, precedingFilter, filter, + partitionNamesInfo, deleteCondition, mergeType, sequenceColName); + } + } + + private static class SeparatorSnapshot { + @SerializedName("separator") + private String separator; + @SerializedName("oriSeparator") + private String oriSeparator; + + private SeparatorSnapshot(Separator separator) { + this.separator = separator.getSeparator(); + this.oriSeparator = separator.getOriSeparator(); + } + + private static SeparatorSnapshot fromSeparator(Separator separator) { + return separator == null ? null : new SeparatorSnapshot(separator); + } + + private static Separator toSeparator(SeparatorSnapshot snapshot) { + return snapshot == null ? null : new Separator(snapshot.separator, snapshot.oriSeparator); + } + } + public static AlterRoutineLoadJobOperationLog read(DataInput in) throws IOException { String json = Text.readString(in); return GsonUtils.GSON.fromJson(json, AlterRoutineLoadJobOperationLog.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index 016f601a74f63a..cc19866a80e90e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -51,6 +51,7 @@ import org.apache.doris.nereids.trees.plans.commands.load.LoadSeparator; import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.persist.EditLog; +import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TResourceInfo; import org.apache.doris.thrift.TUniqueKeyUpdateMode; @@ -249,7 +250,24 @@ public void testModifyPropertiesRevalidatesWhileHoldingWriteLock() throws Except } @Test - public void testModifyTargetTablePreparesExternalIoBeforeMetadataLocks() throws Exception { + public void testModifyPropertiesRejectsLoadDescValidatedAgainstStaleTable() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, + 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + Mockito.when(command.getRoutineLoadDesc()).thenReturn(Mockito.mock(RoutineLoadDesc.class)); + Mockito.when(command.getLoadPropertyTableId()).thenReturn(2L); + + DdlException exception = Assert.assertThrows(DdlException.class, + () -> routineLoadJob.modifyProperties(command)); + + Assert.assertTrue(exception.getMessage().contains("please retry")); + Assert.assertEquals(1L, routineLoadJob.getTableId()); + } + + @Test + public void testModifyTargetTableDefersCloudIoUntilResume() throws Exception { KafkaRoutineLoadJob routineLoadJob = Mockito.spy(new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN)); Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); @@ -281,7 +299,7 @@ public void testModifyTargetTablePreparesExternalIoBeforeMetadataLocks() throws AtomicBoolean databaseLocked = new AtomicBoolean(false); AtomicBoolean tableLocked = new AtomicBoolean(false); AtomicBoolean kafkaPrepared = new AtomicBoolean(false); - AtomicBoolean cloudPrepared = new AtomicBoolean(false); + AtomicBoolean cloudProgressReset = new AtomicBoolean(false); AtomicBoolean targetValidated = new AtomicBoolean(false); Mockito.doAnswer(invocation -> { databaseLocked.set(true); @@ -306,7 +324,7 @@ public void testModifyTargetTablePreparesExternalIoBeforeMetadataLocks() throws Assert.assertTrue(databaseLocked.get()); Assert.assertTrue(tableLocked.get()); Assert.assertTrue(kafkaPrepared.get()); - Assert.assertTrue(cloudPrepared.get()); + Assert.assertFalse(cloudProgressReset.get()); Assert.assertEquals(1L, routineLoadJob.getTableId()); Assert.assertEquals("topic1", routineLoadJob.getTopic()); Assert.assertSame(originalProgress, routineLoadJob.getProgress()); @@ -318,6 +336,7 @@ public void testModifyTargetTablePreparesExternalIoBeforeMetadataLocks() throws Assert.assertTrue(databaseLocked.get()); Assert.assertTrue(tableLocked.get()); Assert.assertTrue(targetValidated.get()); + Assert.assertFalse(cloudProgressReset.get()); Assert.assertEquals(2L, routineLoadJob.getTableId()); Assert.assertEquals("topic2", routineLoadJob.getTopic()); return null; @@ -350,24 +369,30 @@ public void testModifyTargetTablePreparesExternalIoBeforeMetadataLocks() throws Assert.assertFalse(databaseLocked.get()); Assert.assertFalse(tableLocked.get()); Assert.assertTrue(kafkaPrepared.get()); - cloudPrepared.set(true); + cloudProgressReset.set(true); return response; }); routineLoadJob.modifyProperties(command); + String persistedJob = GsonUtils.GSON.toJson(routineLoadJob); + Assert.assertTrue(persistedJob.contains("\"pendingCloudProgressReset\"")); + Assert.assertTrue(persistedJob.contains("\"clearProgress\":true")); + Assert.assertTrue(persistedJob.contains("\"1\":99")); + Mockito.verifyNoInteractions(metaServiceProxy); + routineLoadJob.updateState(RoutineLoadJob.JobState.NEED_SCHEDULE, null, false); } finally { Config.cloud_unique_id = originalCloudUniqueId; } Assert.assertTrue(kafkaPrepared.get()); - Assert.assertTrue(cloudPrepared.get()); + Assert.assertTrue(cloudProgressReset.get()); Assert.assertTrue(targetValidated.get()); Assert.assertFalse(databaseLocked.get()); Assert.assertFalse(tableLocked.get()); Assert.assertEquals(2L, routineLoadJob.getTableId()); Assert.assertEquals("topic2", routineLoadJob.getTopic()); Mockito.verify(editLog).logAlterRoutineLoadJob(Mockito.any(AlterRoutineLoadJobOperationLog.class)); - Mockito.verify(metaServiceProxy).resetRLProgress(Mockito.any()); + Mockito.verify(metaServiceProxy, Mockito.times(2)).resetRLProgress(Mockito.any()); } @Test @@ -546,6 +571,37 @@ public void testModifyCustomKafkaPropertiesDoesNotResetCloudProgress() throws Ex routineLoadJob.getCustomProperties().get("property.kafka_default_offsets")); } + @Test + public void testSymbolicDefaultOffsetOverridesPreviousDatetimeOrigin() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, + 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + Map originalCustomProperties = Maps.newHashMap(); + originalCustomProperties.put(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName(), "1767225600000"); + originalCustomProperties.put(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName(), + "2026-01-01 00:00:00"); + Deencapsulation.setField(routineLoadJob, "customProperties", originalCustomProperties); + Deencapsulation.setField(routineLoadJob, "kafkaDefaultOffSet", "2026-01-01 00:00:00"); + + KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties( + Maps.newHashMap(ImmutableMap.of("property." + + KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName(), KafkaProgress.OFFSET_BEGINNING))); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getEditLog()).thenReturn(Mockito.mock(EditLog.class)); + + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + routineLoadJob.modifyProperties(mockAlterCommand(dataSourceProperties)); + } + + Map customProperties = Deencapsulation.getField(routineLoadJob, "customProperties"); + Assert.assertEquals(KafkaProgress.OFFSET_BEGINNING, + Deencapsulation.getField(routineLoadJob, "kafkaDefaultOffSet")); + Assert.assertEquals(KafkaProgress.OFFSET_BEGINNING, + customProperties.get(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName())); + Assert.assertFalse(customProperties.containsKey(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())); + } + @Test public void testAlterRejectsEmptyKafkaDefaultOffset() { Map alteredSourceProperties = Maps.newHashMap(); @@ -581,6 +637,8 @@ public void testModifyKafkaTopicResetsCloudProgressWithoutOffsets() throws Excep metaServiceProxyStatic.when(MetaServiceProxy::getInstance).thenReturn(metaServiceProxy); routineLoadJob.modifyProperties(mockAlterCommand(dataSourceProperties)); + Mockito.verifyNoInteractions(metaServiceProxy); + routineLoadJob.updateState(RoutineLoadJob.JobState.NEED_SCHEDULE, null, false); } finally { Config.cloud_unique_id = originalCloudUniqueId; } @@ -591,6 +649,40 @@ public void testModifyKafkaTopicResetsCloudProgressWithoutOffsets() throws Excep Assert.assertTrue(((KafkaProgress) routineLoadJob.getProgress()).getPartitionOffsetPairs(false).isEmpty()); } + @Test + public void testCloudProgressResetFailureKeepsJobPausedAndPending() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, + 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties( + Maps.newHashMap(ImmutableMap.of(KafkaConfiguration.KAFKA_TOPIC.getName(), "topic2"))); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getEditLog()).thenReturn(Mockito.mock(EditLog.class)); + MetaServiceProxy metaServiceProxy = Mockito.mock(MetaServiceProxy.class); + Cloud.ResetRLProgressResponse response = Cloud.ResetRLProgressResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(Cloud.MetaServiceCode.INVALID_ARGUMENT).setMsg("reset failed")) + .build(); + Mockito.when(metaServiceProxy.resetRLProgress(Mockito.any())).thenReturn(response); + + String originalCloudUniqueId = Config.cloud_unique_id; + Config.cloud_unique_id = "test-cloud"; + try (MockedStatic envStatic = Mockito.mockStatic(Env.class); + MockedStatic metaServiceProxyStatic = Mockito.mockStatic(MetaServiceProxy.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + metaServiceProxyStatic.when(MetaServiceProxy::getInstance).thenReturn(metaServiceProxy); + + routineLoadJob.modifyProperties(mockAlterCommand(dataSourceProperties)); + Assert.assertThrows(DdlException.class, + () -> routineLoadJob.updateState(RoutineLoadJob.JobState.NEED_SCHEDULE, null, false)); + } finally { + Config.cloud_unique_id = originalCloudUniqueId; + } + + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, routineLoadJob.getState()); + Assert.assertNotNull(Deencapsulation.getField(routineLoadJob, "pendingCloudProgressReset")); + } + @Test public void testUpdateLagRebuildsConvertedPropertiesAfterReplay() throws UserException { KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, @@ -898,16 +990,26 @@ public void testReplayModifyPropertiesWithDataSourceSwitchesTargetTableInCloudMo partitionToOffset.put(1, 123L); KafkaProgress progress = new KafkaProgress(partitionToOffset); Deencapsulation.setField(routineLoadJob, "progress", progress); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties( - Maps.newHashMap(ImmutableMap.of("property.client.id", "replayed-client"))); + Map replayedProperties = Maps.newHashMap(); + replayedProperties.put("property.client.id", "replayed-client"); + replayedProperties.put(KafkaConfiguration.KAFKA_PARTITIONS.getName(), "2"); + replayedProperties.put(KafkaConfiguration.KAFKA_OFFSETS.getName(), "200"); + KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties(replayedProperties); + RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc(new Separator("|"), null, null, + null, null, null, null, LoadTask.MergeType.APPEND, null); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(1L, - Maps.newHashMap(), dataSourceProperties, 202L); + Maps.newHashMap(), dataSourceProperties, 202L, routineLoadDesc); + MetaServiceProxy metaServiceProxy = Mockito.mock(MetaServiceProxy.class); String originalCloudUniqueId = Config.cloud_unique_id; Config.cloud_unique_id = "replay-cloud"; - try { + try (MockedStatic metaServiceProxyStatic = Mockito.mockStatic(MetaServiceProxy.class)) { + metaServiceProxyStatic.when(MetaServiceProxy::getInstance).thenReturn(metaServiceProxy); routineLoadJob.replayModifyProperties(log); + Assert.assertNotNull(Deencapsulation.getField(routineLoadJob, "pendingCloudProgressReset")); + routineLoadJob.updateState(RoutineLoadJob.JobState.NEED_SCHEDULE, null, true); } finally { Config.cloud_unique_id = originalCloudUniqueId; } @@ -915,7 +1017,11 @@ public void testReplayModifyPropertiesWithDataSourceSwitchesTargetTableInCloudMo Assert.assertEquals(202L, routineLoadJob.getTableId()); Assert.assertSame(progress, routineLoadJob.getProgress()); Assert.assertEquals(Long.valueOf(123L), ((KafkaProgress) routineLoadJob.getProgress()).getOffsetByPartition(1)); + Assert.assertEquals(Long.valueOf(200L), ((KafkaProgress) routineLoadJob.getProgress()).getOffsetByPartition(2)); Assert.assertEquals("replayed-client", routineLoadJob.getCustomProperties().get("property.client.id")); + Assert.assertEquals("|", routineLoadJob.getColumnSeparator().getOriSeparator()); + Assert.assertNull(Deencapsulation.getField(routineLoadJob, "pendingCloudProgressReset")); + Mockito.verifyNoInteractions(metaServiceProxy); } private KafkaDataSourceProperties analyzedAlterDataSourceProperties(Map properties) diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java index 912915ffbe20e8..54973e0ef70abc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java @@ -17,9 +17,12 @@ package org.apache.doris.persist; +import org.apache.doris.analysis.Separator; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.load.RoutineLoadDesc; +import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.kafka.KafkaConfiguration; import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; @@ -67,8 +70,10 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User Assert.assertEquals(0L, new AlterRoutineLoadJobOperationLog( jobId, jobProperties, routineLoadDataSourceProperties).getTargetTableId()); + RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc(new Separator("|"), null, null, + null, null, null, null, LoadTask.MergeType.APPEND, null); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(jobId, - jobProperties, routineLoadDataSourceProperties, targetTableId); + jobProperties, routineLoadDataSourceProperties, targetTableId, routineLoadDesc); log.write(out); out.flush(); out.close(); @@ -89,6 +94,8 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User Assert.assertEquals(routineLoadDataSourceProperties.getKafkaPartitionOffsets().get(1), kafkaDataSourceProperties.getKafkaPartitionOffsets().get(1)); Assert.assertEquals(targetTableId, log2.getTargetTableId()); + Assert.assertEquals("|", log2.getRoutineLoadDesc().getColumnSeparator().getOriSeparator()); + Assert.assertEquals(LoadTask.MergeType.APPEND, log2.getRoutineLoadDesc().getMergeType()); in.close(); } From 0f81dc705e7cc4a57871f36350632a8b09c114a5 Mon Sep 17 00:00:00 2001 From: Refrain Date: Fri, 17 Jul 2026 18:36:45 +0800 Subject: [PATCH 15/19] [improvement](fe) Reduce routine load ALTER lock scope ### What problem does this PR solve? Issue Number: None Related PR: #64878 Problem Summary: Kafka routine load target-table ALTER revalidated the target while holding database and table metadata locks across the final in-memory mutation and edit-log write. The target has already been validated during command analysis, and each routine load task resolves the persisted table ID and replans under the table read lock before execution. Remove the duplicate mutation-time validation and metadata locks so ALTER commit is serialized only by the routine load job write lock. ### Release note None ### Check List (For Author) - Test: FE unit tests and FE build - `./run-fe-ut.sh --run org.apache.doris.load.routineload.KafkaRoutineLoadJobTest` - `./run-fe-ut.sh --run org.apache.doris.load.routineload.KafkaRoutineLoadJobTest,org.apache.doris.load.routineload.KinesisRoutineLoadJobTest,org.apache.doris.load.routineload.RoutineLoadJobTest,org.apache.doris.load.routineload.RoutineLoadManagerTest,org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommandTest,org.apache.doris.persist.AlterRoutineLoadOperationLogTest` - `./build.sh --fe -j48` - Behavior changed: Yes. Target-table ALTER no longer repeats metadata validation during commit; task planning remains the authoritative runtime validation. - Does this need documentation: No --- .../kafka/KafkaRoutineLoadJob.java | 35 +------------ .../routineload/KafkaRoutineLoadJobTest.java | 51 ++----------------- 2 files changed, 4 insertions(+), 82 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java index 758cf1f621cd03..40f4c4889ea83d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java @@ -25,7 +25,6 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.RandomDistributionInfo; -import org.apache.doris.catalog.Table; import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.cloud.rpc.MetaServiceProxy; import org.apache.doris.common.Config; @@ -62,7 +61,6 @@ import org.apache.doris.service.FrontendOptions; import org.apache.doris.thrift.TFileCompressType; import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; -import org.apache.doris.thrift.TUniqueKeyUpdateMode; import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; @@ -795,7 +793,6 @@ public boolean appliesRoutineLoadDescAtomically() { @Override public void modifyProperties(AlterRoutineLoadCommand command) throws UserException { - Map jobProperties = command.getAnalyzedJobProperties(); KafkaDataSourceProperties dataSourceProperties = (KafkaDataSourceProperties) command.getDataSourceProperties(); writeLock(); @@ -814,37 +811,7 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti // Kafka offset resolution may perform external I/O. Keep it outside DB/table metadata locks. PreparedKafkaProperties preparedDataSourceProperties = prepareDataSourceProperties(dataSourceProperties, false); - if (!command.hasTargetTable()) { - unprotectApplyAlter(command, dataSourceProperties, preparedDataSourceProperties); - return; - } - - Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId); - if (db == null) { - throw new DdlException("Database not found: " + dbId); - } - db.readLock(); - try { - Table table = db.getTableNullable(command.getTargetTableId()); - if (table == null) { - throw new DdlException("Table not found: " + command.getTargetTableId()); - } - if (!(table instanceof OlapTable)) { - throw new DdlException("Routine load target table must be an OLAP table"); - } - table.readLock(); - try { - TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode = getEffectiveUniqueKeyUpdateMode( - uniqueKeyUpdateMode, jobProperties); - unprotectedValidateTargetTable( - db, (OlapTable) table, jobProperties, effectiveUniqueKeyUpdateMode); - unprotectApplyAlter(command, dataSourceProperties, preparedDataSourceProperties); - } finally { - table.readUnlock(); - } - } finally { - db.readUnlock(); - } + unprotectApplyAlter(command, dataSourceProperties, preparedDataSourceProperties); } finally { writeUnlock(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index cc19866a80e90e..52f251401d7119 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -267,7 +267,7 @@ public void testModifyPropertiesRejectsLoadDescValidatedAgainstStaleTable() thro } @Test - public void testModifyTargetTableDefersCloudIoUntilResume() throws Exception { + public void testModifyTargetTableUsesOnlyJobLockAndDefersCloudIoUntilResume() throws Exception { KafkaRoutineLoadJob routineLoadJob = Mockito.spy(new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN)); Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); @@ -289,54 +289,15 @@ public void testModifyTargetTableDefersCloudIoUntilResume() throws Exception { Env env = Mockito.mock(Env.class); EditLog editLog = Mockito.mock(EditLog.class); - InternalCatalog catalog = Mockito.mock(InternalCatalog.class); - Database database = Mockito.mock(Database.class); - OlapTable targetTable = Mockito.mock(OlapTable.class); Mockito.when(env.getEditLog()).thenReturn(editLog); - Mockito.when(catalog.getDbNullable(1L)).thenReturn(database); - Mockito.when(database.getTableNullable(2L)).thenReturn(targetTable); - AtomicBoolean databaseLocked = new AtomicBoolean(false); - AtomicBoolean tableLocked = new AtomicBoolean(false); AtomicBoolean kafkaPrepared = new AtomicBoolean(false); AtomicBoolean cloudProgressReset = new AtomicBoolean(false); - AtomicBoolean targetValidated = new AtomicBoolean(false); - Mockito.doAnswer(invocation -> { - databaseLocked.set(true); - return null; - }).when(database).readLock(); - Mockito.doAnswer(invocation -> { - databaseLocked.set(false); - return null; - }).when(database).readUnlock(); - Mockito.doAnswer(invocation -> { - Assert.assertTrue(databaseLocked.get()); - tableLocked.set(true); - return null; - }).when(targetTable).readLock(); - Mockito.doAnswer(invocation -> { - tableLocked.set(false); - return null; - }).when(targetTable).readUnlock(); Mockito.doAnswer(invocation -> { ReentrantReadWriteLock lock = Deencapsulation.getField(routineLoadJob, "lock"); Assert.assertTrue(lock.isWriteLockedByCurrentThread()); - Assert.assertTrue(databaseLocked.get()); - Assert.assertTrue(tableLocked.get()); Assert.assertTrue(kafkaPrepared.get()); Assert.assertFalse(cloudProgressReset.get()); - Assert.assertEquals(1L, routineLoadJob.getTableId()); - Assert.assertEquals("topic1", routineLoadJob.getTopic()); - Assert.assertSame(originalProgress, routineLoadJob.getProgress()); - targetValidated.set(true); - return null; - }).when(routineLoadJob).unprotectedValidateTargetTable(Mockito.eq(database), Mockito.eq(targetTable), - Mockito.anyMap(), Mockito.eq(TUniqueKeyUpdateMode.UPSERT)); - Mockito.doAnswer(invocation -> { - Assert.assertTrue(databaseLocked.get()); - Assert.assertTrue(tableLocked.get()); - Assert.assertTrue(targetValidated.get()); - Assert.assertFalse(cloudProgressReset.get()); Assert.assertEquals(2L, routineLoadJob.getTableId()); Assert.assertEquals("topic2", routineLoadJob.getTopic()); return null; @@ -352,13 +313,10 @@ public void testModifyTargetTableDefersCloudIoUntilResume() throws Exception { MockedStatic kafkaUtilStatic = Mockito.mockStatic(KafkaUtil.class); MockedStatic metaServiceProxyStatic = Mockito.mockStatic(MetaServiceProxy.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); - envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); kafkaUtilStatic.when(() -> KafkaUtil.getRealOffsets(Mockito.eq("127.0.0.1:9020"), Mockito.eq("topic2"), Mockito.anyMap(), Mockito.anyList())).thenAnswer(invocation -> { ReentrantReadWriteLock lock = Deencapsulation.getField(routineLoadJob, "lock"); Assert.assertTrue(lock.isWriteLockedByCurrentThread()); - Assert.assertFalse(databaseLocked.get()); - Assert.assertFalse(tableLocked.get()); kafkaPrepared.set(true); return Lists.newArrayList(Pair.of(1, 100L)); }); @@ -366,8 +324,6 @@ public void testModifyTargetTableDefersCloudIoUntilResume() throws Exception { Mockito.when(metaServiceProxy.resetRLProgress(Mockito.any())).thenAnswer(invocation -> { ReentrantReadWriteLock lock = Deencapsulation.getField(routineLoadJob, "lock"); Assert.assertTrue(lock.isWriteLockedByCurrentThread()); - Assert.assertFalse(databaseLocked.get()); - Assert.assertFalse(tableLocked.get()); Assert.assertTrue(kafkaPrepared.get()); cloudProgressReset.set(true); return response; @@ -386,11 +342,10 @@ public void testModifyTargetTableDefersCloudIoUntilResume() throws Exception { Assert.assertTrue(kafkaPrepared.get()); Assert.assertTrue(cloudProgressReset.get()); - Assert.assertTrue(targetValidated.get()); - Assert.assertFalse(databaseLocked.get()); - Assert.assertFalse(tableLocked.get()); Assert.assertEquals(2L, routineLoadJob.getTableId()); Assert.assertEquals("topic2", routineLoadJob.getTopic()); + Mockito.verify(routineLoadJob, Mockito.never()).unprotectedValidateTargetTable( + Mockito.any(), Mockito.any(), Mockito.anyMap(), Mockito.any()); Mockito.verify(editLog).logAlterRoutineLoadJob(Mockito.any(AlterRoutineLoadJobOperationLog.class)); Mockito.verify(metaServiceProxy, Mockito.times(2)).resetRLProgress(Mockito.any()); } From ba918d060522378b6a55a067185a3a6ea73d82d8 Mon Sep 17 00:00:00 2001 From: 0AyanamiRei <3244156674@qq.com> Date: Sun, 19 Jul 2026 18:39:42 +0800 Subject: [PATCH 16/19] [fix](fe) Authorize routine load target before metadata lookup ### What problem does this PR solve? Issue Number: None Related PR: #64878 Problem Summary: ALTER ROUTINE LOAD target-table validation resolved target metadata before authorizing the current job and requested target. A limited-RBAC user could distinguish missing, non-OLAP, or temporary target tables through different validation errors. Authorize the current job first and check the target LOAD privilege by name before resolving database or table metadata. ### Release note Unauthorized ALTER ROUTINE LOAD target-table requests no longer reveal target table metadata. ### Check List (For Author) - Test: FE unit tests added but not run per user request - Behavior changed: Yes. Unauthorized requests now fail before target metadata resolution. - Does this need documentation: No --- .../commands/AlterRoutineLoadCommand.java | 15 +++--- .../commands/AlterRoutineLoadCommandTest.java | 51 +++++++++++++++++-- 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java index 175bf4d5fc8ca7..4f87d00938e7b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java @@ -197,7 +197,7 @@ public void validate(ConnectContext ctx) throws UserException { checkJobProperties(); // check load properties RoutineLoadJob job = Env.getCurrentEnv().getRoutineLoadManager() - .getJob(getDbName(), getJobName()); + .checkPrivAndGetJob(getDbName(), getJobName()); if (MapUtils.isNotEmpty(loadPropertyMap)) { this.loadPropertyTableId = job.getTableId(); this.routineLoadDesc = CreateRoutineLoadInfo.checkLoadProperties(ctx, loadPropertyMap, @@ -393,7 +393,13 @@ private void validateTargetTable(ConnectContext ctx, RoutineLoadJob job, if (job.isMultiTable()) { throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports single-table job"); } - Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException(job.getDbFullName()); + String dbFullName = job.getDbFullName(); + if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ctx, InternalCatalog.INTERNAL_CATALOG_NAME, + dbFullName, targetTableName, PrivPredicate.LOAD)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "LOAD", + ctx.getQualifiedUser(), ctx.getRemoteIP(), dbFullName + ": " + targetTableName); + } + Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException(dbFullName); Table table = db.getTableOrAnalysisException(targetTableName); if (!(table instanceof OlapTable)) { throw new AnalysisException("ALTER ROUTINE LOAD target table only supports OLAP table"); @@ -402,11 +408,6 @@ private void validateTargetTable(ConnectContext ctx, RoutineLoadJob job, if (olapTable.isTemporary()) { throw new AnalysisException("Do not support load for temporary table " + olapTable.getDisplayName()); } - if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ctx, InternalCatalog.INTERNAL_CATALOG_NAME, - job.getDbFullName(), targetTableName, PrivPredicate.LOAD)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "LOAD", - ctx.getQualifiedUser(), ctx.getRemoteIP(), job.getDbFullName() + ": " + targetTableName); - } if (job.isLoadToSingleTablet() && !(olapTable.getDefaultDistributionInfo() instanceof RandomDistributionInfo)) { throw new AnalysisException( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java index 3bc98c954106bb..17923dc02d96e7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java @@ -30,6 +30,7 @@ import org.apache.doris.load.routineload.RoutineLoadManager; import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.exceptions.ParseException; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; @@ -44,6 +45,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.InOrder; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -89,6 +91,8 @@ public void setUp() throws Exception { Mockito.when(accessManager.checkTblPriv(Mockito.any(ConnectContext.class), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.any())).thenReturn(true); Mockito.when(routineLoadManager.getJob(Mockito.anyString(), Mockito.anyString())).thenReturn(routineLoadJob); + Mockito.when(routineLoadManager.checkPrivAndGetJob(Mockito.anyString(), Mockito.anyString())) + .thenReturn(routineLoadJob); Mockito.when(routineLoadJob.getDbFullName()).thenReturn("testDb"); Mockito.when(routineLoadJob.getTableName()).thenReturn("testTable"); Mockito.when(routineLoadJob.getDbId()).thenReturn(1000L); @@ -284,16 +288,57 @@ public void testValidateTargetTableRejectsMultiTableJob() throws Exception { } @Test - public void testValidateTargetTableRejectsWithoutLoadPrivilege() throws Exception { + public void testValidateUnauthorizedTargetDoesNotResolveMetadata() throws Exception { runBefore(); - mockTargetTable(currentTable); Mockito.when(accessManager.checkTblPriv(Mockito.any(ConnectContext.class), Mockito.anyString(), - Mockito.eq("testDb"), Mockito.eq("testTable2"), Mockito.any())).thenReturn(false); + Mockito.eq("testDb"), Mockito.eq("testTable2"), Mockito.eq(PrivPredicate.LOAD))).thenReturn(false); AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) .getMessage().contains("LOAD")); + Mockito.verify(routineLoadManager).checkPrivAndGetJob("testDb", "label1"); + Mockito.verify(accessManager).checkTblPriv(connectContext, InternalCatalog.INTERNAL_CATALOG_NAME, + "testDb", "testTable2", PrivPredicate.LOAD); + Mockito.verify(catalog, Mockito.never()).getDbOrAnalysisException(Mockito.anyString()); + Mockito.verify(db, Mockito.never()).getTableOrAnalysisException(Mockito.anyString()); + Mockito.verify(routineLoadJob, Mockito.never()).validateTargetTable( + Mockito.any(Database.class), Mockito.any(OlapTable.class), Mockito.anyMap(), + Mockito.any(TUniqueKeyUpdateMode.class)); + } + + @Test + public void testValidateTargetTableRejectsUnauthorizedJobBeforeTargetLookup() throws Exception { + runBefore(); + Mockito.when(routineLoadManager.checkPrivAndGetJob("testDb", "label1")) + .thenThrow(new AnalysisException("access denied")); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); + + Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) + .getMessage().contains("access denied")); + Mockito.verify(accessManager, Mockito.never()).checkTblPriv( + Mockito.any(ConnectContext.class), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), + Mockito.any()); + Mockito.verify(catalog, Mockito.never()).getDbOrAnalysisException(Mockito.anyString()); + Mockito.verify(db, Mockito.never()).getTableOrAnalysisException(Mockito.anyString()); + } + + @Test + public void testValidateTargetTableAuthorizesBeforeMetadataLookup() throws Exception { + runBefore(); + mockTargetTable(currentTable); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); + + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + + InOrder inOrder = Mockito.inOrder(routineLoadManager, accessManager, catalog, db); + inOrder.verify(routineLoadManager).checkPrivAndGetJob("testDb", "label1"); + inOrder.verify(accessManager).checkTblPriv(connectContext, InternalCatalog.INTERNAL_CATALOG_NAME, + "testDb", "testTable2", PrivPredicate.LOAD); + inOrder.verify(catalog).getDbOrAnalysisException("testDb"); + inOrder.verify(db).getTableOrAnalysisException("testTable2"); } @Test From a756d7c8ca4379f1e75c58840d7627a5e165525d Mon Sep 17 00:00:00 2001 From: Refrain Date: Mon, 20 Jul 2026 00:56:32 +0800 Subject: [PATCH 17/19] [refactor](fe) Narrow routine load target-table changes ### What problem does this PR solve? Issue Number: N/A Related PR: #64878 Problem Summary: Keep ALTER ROUTINE LOAD target-table switching composable with existing job and Kafka/Kinesis data-source properties while removing unrelated Kafka ALTER atomicity, Cloud progress staging, partial-update compatibility, and RoutineLoadDesc persistence changes from this PR. ### Release note ALTER ROUTINE LOAD supports SET TARGET TABLE together with supported job and Kafka/Kinesis data-source properties. ### Check List (For Author) - Test: - Unit Test: Focused Routine Load FE UT, 34 tests passed - Build: ./build.sh --fe -j48 - Regression test: Added Kafka end-to-end coverage; not run locally because an isolated FE/BE/Kafka cluster was not started - Behavior changed: Yes. Paused single-table Kafka and Kinesis Routine Load jobs can switch target tables while preserving progress. - Does this need documentation: Yes. apache/doris-website#3988 --- .../load/routineload/RoutineLoadJob.java | 164 +---- .../load/routineload/RoutineLoadManager.java | 4 +- .../kafka/KafkaDataSourceProperties.java | 39 +- .../kafka/KafkaRoutineLoadJob.java | 329 +++------- .../kinesis/KinesisRoutineLoadJob.java | 8 +- .../load/NereidsRoutineLoadTaskInfo.java | 7 - .../nereids/parser/LogicalPlanBuilder.java | 10 +- .../commands/AlterRoutineLoadCommand.java | 63 +- .../AlterRoutineLoadJobOperationLog.java | 79 --- .../routineload/KafkaRoutineLoadJobTest.java | 572 +++--------------- .../KinesisRoutineLoadJobTest.java | 66 ++ .../load/routineload/RoutineLoadJobTest.java | 72 --- .../commands/AlterRoutineLoadCommandTest.java | 256 ++------ .../AlterRoutineLoadOperationLogTest.java | 50 +- .../test_routine_load_alter.groovy | 199 ++---- 15 files changed, 422 insertions(+), 1496 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 4f919340c3f2b7..7053e696488bda 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ExprToSqlVisitor; +import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.Separator; import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.analysis.UserIdentity; @@ -470,83 +471,25 @@ protected void setRoutineLoadDesc(RoutineLoadDesc routineLoadDesc) { } } - public void validateTargetTable(Database db, OlapTable targetTable, - Map alteredJobProperties, TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode) - throws UserException { - targetTable.readLock(); - try { - unprotectedValidateTargetTable(db, targetTable, alteredJobProperties, effectiveUniqueKeyUpdateMode); - } finally { - targetTable.readUnlock(); - } - } - - protected void unprotectedValidateTargetTable(Database db, OlapTable targetTable, - Map alteredJobProperties, TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode) - throws UserException { + public void validateTargetTable(Database db, OlapTable targetTable) throws UserException { if (isMultiTable) { throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports single-table job"); } - validateAlterJobProperties(targetTable, alteredJobProperties, effectiveUniqueKeyUpdateMode); - NereidsRoutineLoadTaskInfo taskInfo = toNereidsRoutineLoadTaskInfo(); - taskInfo.applyAlterPropertiesForValidation(alteredJobProperties, effectiveUniqueKeyUpdateMode); - NereidsStreamLoadPlanner planner = new NereidsStreamLoadPlanner(db, targetTable, taskInfo); - planner.plan(new TUniqueId(0, 0)); - } - - public void validateAlterJobProperties(OlapTable targetTable, Map alteredJobProperties, - TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode) throws UserException { - if (effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { - return; - } - if (isMultiTable) { - throw new AnalysisException("Partial update is not supported in multi-table load."); - } - if (effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS) { - if (!targetTable.getEnableUniqueKeyMergeOnWrite()) { - throw new AnalysisException("load by PARTIAL_COLUMNS is only supported in unique table MoW"); - } - return; - } - Preconditions.checkState(effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS, - effectiveUniqueKeyUpdateMode); - validateFlexiblePartialUpdateForAlter(targetTable, alteredJobProperties); - } - - public static TUniqueKeyUpdateMode getEffectiveUniqueKeyUpdateMode(TUniqueKeyUpdateMode currentUniqueKeyUpdateMode, - Map alteredJobProperties) { - if (alteredJobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { - return TUniqueKeyUpdateMode.valueOf( - alteredJobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)); - } - if (alteredJobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS) - && Boolean.parseBoolean(alteredJobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) - && currentUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { - return TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS; - } - return currentUniqueKeyUpdateMode; - } - - protected void validateAlterJobPropertiesForMutation(AlterRoutineLoadCommand command) throws UserException { - Map alteredJobProperties = command.getAnalyzedJobProperties(); - TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode = getEffectiveUniqueKeyUpdateMode( - uniqueKeyUpdateMode, alteredJobProperties); - if (effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { - return; + List columnsInfo = null; + if (columnDescs != null && !columnDescs.descs.isEmpty()) { + columnsInfo = new ArrayList<>(columnDescs.descs); } + checkMeta(targetTable, new RoutineLoadDesc(columnSeparator, lineDelimiter, columnsInfo, + precedingFilter, whereExpr, partitionNamesInfo, deleteCondition, mergeType, sequenceCol)); - Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId); - if (db == null) { - throw new DdlException("Database not found: " + dbId); - } - Table table = db.getTableNullable(tableId); - if (table == null) { - throw new DdlException("Table not found: " + tableId); - } - if (!(table instanceof OlapTable)) { - throw new DdlException("Partial update is only supported for OLAP tables"); + targetTable.readLock(); + try { + NereidsStreamLoadPlanner planner = new NereidsStreamLoadPlanner(db, targetTable, + toNereidsRoutineLoadTaskInfo()); + planner.plan(new TUniqueId(0, 0)); + } finally { + targetTable.readUnlock(); } - validateAlterJobProperties((OlapTable) table, alteredJobProperties, effectiveUniqueKeyUpdateMode); } @Override @@ -1567,9 +1510,6 @@ public void updateState(JobState jobState, ErrorReason reason, boolean isReplay) protected void unprotectUpdateState(JobState jobState, ErrorReason reason, boolean isReplay) throws UserException { checkStateTransform(jobState); - if (jobState == JobState.NEED_SCHEDULE) { - beforeTransitionToNeedSchedule(isReplay); - } switch (jobState) { case RUNNING: executeRunning(); @@ -1603,9 +1543,6 @@ protected void unprotectUpdateState(JobState jobState, ErrorReason reason, boole } } - protected void beforeTransitionToNeedSchedule(boolean isReplay) throws UserException { - } - private void executeRunning() { state = JobState.RUNNING; } @@ -1774,10 +1711,11 @@ public String getStateReason() { } public List getShowInfo() { + Optional database = Env.getCurrentInternalCatalog().getDb(dbId); + Optional
table = database.flatMap(db -> db.getTable(tableId)); + readLock(); try { - Optional database = Env.getCurrentInternalCatalog().getDb(dbId); - Optional
table = database.flatMap(db -> db.getTable(tableId)); List row = Lists.newArrayList(); row.add(String.valueOf(id)); row.add(name); @@ -1840,15 +1778,6 @@ public List> getTasksShowInfo() throws AnalysisException { } public String getShowCreateInfo() { - readLock(); - try { - return unprotectGetShowCreateInfo(); - } finally { - readUnlock(); - } - } - - private String unprotectGetShowCreateInfo() { Optional database = Env.getCurrentInternalCatalog().getDb(dbId); Optional
table = database.flatMap(db -> db.getTable(tableId)); StringBuilder sb = new StringBuilder(); @@ -2135,25 +2064,12 @@ public void gsonPostProcess() throws IOException { public abstract void modifyProperties(AlterRoutineLoadCommand command) throws UserException; - public boolean appliesRoutineLoadDescAtomically() { - return false; - } - public abstract void replayModifyProperties(AlterRoutineLoadJobOperationLog log); public abstract NereidsRoutineLoadTaskInfo toNereidsRoutineLoadTaskInfo() throws UserException; // for ALTER ROUTINE LOAD protected void modifyCommonJobProperties(Map jobProperties) throws UserException { - modifyCommonJobProperties(jobProperties, false); - } - - protected void modifyCommonJobPropertiesForKafka(Map jobProperties) throws UserException { - modifyCommonJobProperties(jobProperties, true); - } - - private void modifyCommonJobProperties(Map jobProperties, - boolean explicitUniqueKeyUpdateModeTakesPrecedence) throws UserException { if (jobProperties.containsKey(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)) { this.desireTaskConcurrentNum = Integer.parseInt( jobProperties.remove(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)); @@ -2188,17 +2104,14 @@ private void modifyCommonJobProperties(Map jobProperties, if (jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { String modeStr = jobProperties.remove(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); TUniqueKeyUpdateMode newMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode(modeStr); - if (!explicitUniqueKeyUpdateModeTakesPrecedence - && newMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS) { + // Validate flexible partial update constraints when changing to UPDATE_FLEXIBLE_COLUMNS + if (newMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS) { validateFlexiblePartialUpdateForAlter(); } this.uniqueKeyUpdateMode = newMode; this.isPartialUpdate = (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS); this.jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); - if (explicitUniqueKeyUpdateModeTakesPrecedence) { - jobProperties.remove(CreateRoutineLoadInfo.PARTIAL_COLUMNS); - } } if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { @@ -2214,11 +2127,16 @@ private void modifyCommonJobProperties(Map jobProperties, } } + /** + * Validate flexible partial update constraints when altering routine load job. + */ private void validateFlexiblePartialUpdateForAlter() throws UserException { + // Multi-table load does not support flexible partial update if (isMultiTable) { throw new DdlException("Flexible partial update is not supported in multi-table load"); } + // Get the table to check table-level constraints Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId); if (db == null) { throw new DdlException("Database not found: " + dbId); @@ -2232,49 +2150,23 @@ private void validateFlexiblePartialUpdateForAlter() throws UserException { } OlapTable olapTable = (OlapTable) table; - olapTable.validateForFlexiblePartialUpdate(); - String format = this.jobProperties.getOrDefault(FileFormatProperties.PROP_FORMAT, "csv"); - if (!"json".equalsIgnoreCase(format)) { - throw new DdlException("Flexible partial update only supports JSON format, but current job uses: " - + format); - } - if (Boolean.parseBoolean(this.jobProperties.getOrDefault( - JsonFileFormatProperties.PROP_FUZZY_PARSE, "false"))) { - throw new DdlException("Flexible partial update does not support fuzzy_parse"); - } - String jsonPaths = getJsonPaths(); - if (jsonPaths != null && !jsonPaths.isEmpty()) { - throw new DdlException("Flexible partial update does not support jsonpaths"); - } - if (columnDescs != null && !columnDescs.descs.isEmpty()) { - throw new DdlException("Flexible partial update does not support COLUMNS specification"); - } - } - - /** - * Validate flexible partial update constraints when altering routine load job. - */ - private void validateFlexiblePartialUpdateForAlter(OlapTable targetTable, - Map alteredJobProperties) throws UserException { // Validate table-level constraints (MoW, skip_bitmap, light_schema_change, variant columns) - targetTable.validateForFlexiblePartialUpdate(); - Map effectiveJobProperties = Maps.newHashMap(jobProperties); - effectiveJobProperties.putAll(alteredJobProperties); + olapTable.validateForFlexiblePartialUpdate(); // Routine load specific validations // Must use JSON format - String format = effectiveJobProperties.getOrDefault(FileFormatProperties.PROP_FORMAT, "csv"); + String format = this.jobProperties.getOrDefault(FileFormatProperties.PROP_FORMAT, "csv"); if (!"json".equalsIgnoreCase(format)) { throw new DdlException("Flexible partial update only supports JSON format, but current job uses: " + format); } // Cannot use fuzzy_parse - if (Boolean.parseBoolean(effectiveJobProperties.getOrDefault( + if (Boolean.parseBoolean(this.jobProperties.getOrDefault( JsonFileFormatProperties.PROP_FUZZY_PARSE, "false"))) { throw new DdlException("Flexible partial update does not support fuzzy_parse"); } // Cannot use jsonpaths - String jsonPaths = effectiveJobProperties.get(JsonFileFormatProperties.PROP_JSON_PATHS); + String jsonPaths = getJsonPaths(); if (jsonPaths != null && !jsonPaths.isEmpty()) { throw new DdlException("Flexible partial update does not support jsonpaths"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java index 0e1000f3dae370..df5615016bfa54 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java @@ -943,9 +943,7 @@ public void alterRoutineLoadJob(AlterRoutineLoadCommand command) throws UserExce + command.getDataSourceProperties().getDataSourceType()); } job.modifyProperties(command); - if (!job.appliesRoutineLoadDescAtomically()) { - job.setRoutineLoadDesc(command.getRoutineLoadDesc()); - } + job.setRoutineLoadDesc(command.getRoutineLoadDesc()); } public void replayAlterRoutineLoadJob(AlterRoutineLoadJobOperationLog log) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaDataSourceProperties.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaDataSourceProperties.java index de22c57a0096ed..9cab113563da61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaDataSourceProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaDataSourceProperties.java @@ -105,14 +105,6 @@ public class KafkaDataSourceProperties extends AbstractDataSourceProperties { .add(KafkaConfiguration.KAFKA_TEXT_TABLE_NAME_FIELD_INDEX.getName()) .build(); - private static final ImmutableSet CREATE_ONLY_MULTI_TABLE_PROPERTIES_SET = - new ImmutableSet.Builder() - .add(KafkaConfiguration.KAFKA_TABLE_NAME_LOCATION.getName()) - .add(KafkaConfiguration.KAFKA_TABLE_NAME_FORMAT.getName()) - .add(KafkaConfiguration.KAFKA_TEXT_TABLE_NAME_FIELD_DELIMITER.getName()) - .add(KafkaConfiguration.KAFKA_TEXT_TABLE_NAME_FIELD_INDEX.getName()) - .build(); - public KafkaDataSourceProperties(Map dataSourceProperties, boolean multiLoad) { super(dataSourceProperties, multiLoad); } @@ -140,14 +132,6 @@ public void convertAndCheckDataSourceProperties() throws UserException { if (optional.isPresent()) { throw new AnalysisException(optional.get() + " is invalid kafka property or can not be set"); } - if (isAlter()) { - Optional createOnlyProperty = originalDataSourceProperties.keySet().stream() - .filter(CREATE_ONLY_MULTI_TABLE_PROPERTIES_SET::contains).findFirst(); - if (createOnlyProperty.isPresent()) { - throw new AnalysisException(createOnlyProperty.get() + " can only be set when creating " - + "a multi-table routine load job"); - } - } this.brokerList = KafkaConfiguration.KAFKA_BROKER_LIST.getParameterValue(originalDataSourceProperties .get(KafkaConfiguration.KAFKA_BROKER_LIST.getName())); @@ -182,23 +166,9 @@ public void convertAndCheckDataSourceProperties() throws UserException { //check offset List offsets = KafkaConfiguration.KAFKA_OFFSETS.getParameterValue(originalDataSourceProperties .get(KafkaConfiguration.KAFKA_OFFSETS.getName())); - boolean hasCustomDefaultOffset = customKafkaProperties - .containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); - String defaultOffsetString = customKafkaProperties.get(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); - boolean hasDirectDefaultOffset = originalDataSourceProperties - .containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); - String directDefaultOffsetString = originalDataSourceProperties + String defaultOffsetString = originalDataSourceProperties .get(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); - if (hasDirectDefaultOffset) { - if (hasCustomDefaultOffset) { - throw new AnalysisException("Only one of " + KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName() - + " and property." + KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName() + " can be set."); - } - defaultOffsetString = directDefaultOffsetString; - customKafkaProperties.put(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName(), defaultOffsetString); - } - boolean hasDefaultOffset = hasCustomDefaultOffset || hasDirectDefaultOffset; - if (CollectionUtils.isNotEmpty(offsets) && hasDefaultOffset) { + if (CollectionUtils.isNotEmpty(offsets) && StringUtils.isNotBlank(defaultOffsetString)) { throw new AnalysisException("Only one of " + KafkaConfiguration.KAFKA_OFFSETS.getName() + " and " + KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName() + " can be set."); } @@ -206,7 +176,7 @@ public void convertAndCheckDataSourceProperties() throws UserException { checkAndSetMultiLoadProperties(); } if (isAlter() && CollectionUtils.isNotEmpty(partitions) && CollectionUtils.isEmpty(offsets) - && !hasDefaultOffset) { + && StringUtils.isBlank(defaultOffsetString)) { // if this is an alter operation, the partition and (default)offset must be set together. throw new AnalysisException("Must set offset or default offset with partition property"); } @@ -214,9 +184,6 @@ public void convertAndCheckDataSourceProperties() throws UserException { this.isOffsetsForTimes = analyzeKafkaOffsetProperty(offsets); return; } - if (isAlter() && CollectionUtils.isEmpty(partitions) && !hasDefaultOffset) { - return; - } this.isOffsetsForTimes = analyzeKafkaDefaultOffsetProperty(); if (CollectionUtils.isNotEmpty(kafkaPartitionOffsets)) { defaultOffsetString = customKafkaProperties.get(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java index 40f4c4889ea83d..b7f1ff0fc51af9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java @@ -74,6 +74,7 @@ import com.google.gson.annotations.SerializedName; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.BooleanUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -109,8 +110,6 @@ public class KafkaRoutineLoadJob extends RoutineLoadJob { private String brokerList; @SerializedName("tp") private String topic; - @SerializedName("pendingCloudProgressReset") - private PendingCloudProgressReset pendingCloudProgressReset; // optional, user want to load partitions. @SerializedName("cskp") private List customKafkaPartitions = Lists.newArrayList(); @@ -217,25 +216,19 @@ private void convertCustomProperties(boolean rebuild) throws DdlException { return; } - Pair, String> convertedProperties = buildConvertedCustomProperties( - customProperties, kafkaDefaultOffSet); - convertedCustomProperties.clear(); - convertedCustomProperties.putAll(convertedProperties.first); - kafkaDefaultOffSet = convertedProperties.second; - } + if (rebuild) { + convertedCustomProperties.clear(); + } - private Pair, String> buildConvertedCustomProperties( - Map sourceProperties, String currentDefaultOffset) throws DdlException { - Map convertedProperties = Maps.newHashMap(); - for (Map.Entry entry : sourceProperties.entrySet()) { + SmallFileMgr smallFileMgr = Env.getCurrentEnv().getSmallFileMgr(); + for (Map.Entry entry : customProperties.entrySet()) { if (entry.getValue().startsWith("FILE:")) { // convert FILE:file_name -> FILE:file_id:md5 String file = entry.getValue().substring(entry.getValue().indexOf(":") + 1); - SmallFileMgr smallFileMgr = Env.getCurrentEnv().getSmallFileMgr(); SmallFile smallFile = smallFileMgr.getSmallFile(dbId, KAFKA_FILE_CATALOG, file, true); - convertedProperties.put(entry.getKey(), "FILE:" + smallFile.id + ":" + smallFile.md5); + convertedCustomProperties.put(entry.getKey(), "FILE:" + smallFile.id + ":" + smallFile.md5); } else { - convertedProperties.put(entry.getKey(), entry.getValue()); + convertedCustomProperties.put(entry.getKey(), entry.getValue()); } } @@ -244,14 +237,14 @@ private Pair, String> buildConvertedCustomProperties( // KAFKA_DEFAULT_OFFSETS, and this attribute will be converted into a timestamp during the analyzing phase, // thus losing some information. So we use KAFKA_ORIGIN_DEFAULT_OFFSETS to store the original datetime // formatted KAFKA_DEFAULT_OFFSETS value - String originDefaultOffset = convertedProperties - .remove(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName()); - String analyzedDefaultOffset = convertedProperties - .remove(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); - String convertedDefaultOffset = originDefaultOffset != null - ? originDefaultOffset - : analyzedDefaultOffset != null ? analyzedDefaultOffset : currentDefaultOffset; - return Pair.of(convertedProperties, convertedDefaultOffset); + if (convertedCustomProperties.containsKey(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())) { + kafkaDefaultOffSet = convertedCustomProperties + .remove(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName()); + return; + } + if (convertedCustomProperties.containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName())) { + kafkaDefaultOffSet = convertedCustomProperties.remove(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); + } } @Override @@ -440,24 +433,11 @@ protected void unprotectUpdateProgress() throws UserException { // For cloud mode, should update cloud progress from meta service, // then update progress with default offset from Kafka if necessary. if (Config.isCloudMode()) { - resetPendingCloudProgress(); updateCloudProgress(); } updateNewPartitionProgress(); } - @Override - protected void beforeTransitionToNeedSchedule(boolean isReplay) throws UserException { - if (pendingCloudProgressReset == null) { - return; - } - if (isReplay) { - pendingCloudProgressReset = null; - return; - } - resetPendingCloudProgress(); - } - @Override protected boolean refreshKafkaPartitions(boolean needAutoResume) throws UserException { // If user does not specify kafka partition, @@ -786,95 +766,94 @@ public Map getCustomProperties() { return getMaskedCustomProperties("property."); } - @Override - public boolean appliesRoutineLoadDescAtomically() { - return true; - } - @Override public void modifyProperties(AlterRoutineLoadCommand command) throws UserException { + Map jobProperties = command.getAnalyzedJobProperties(); KafkaDataSourceProperties dataSourceProperties = (KafkaDataSourceProperties) command.getDataSourceProperties(); + if (null != dataSourceProperties) { + // if the partition offset is set by timestamp, convert it to real offset + convertOffset(dataSourceProperties); + } writeLock(); try { if (getState() != JobState.PAUSED) { throw new DdlException("Only supports modification of PAUSED jobs"); } - if (command.getRoutineLoadDesc() != null && command.getLoadPropertyTableId() != tableId) { - throw new DdlException("Routine load target table changed while validating load properties; " - + "please retry ALTER ROUTINE LOAD"); - } - if (!command.hasTargetTable()) { - validateAlterJobPropertiesForMutation(command); + + modifyPropertiesInternal(jobProperties, dataSourceProperties); + if (command.hasTargetTable()) { + tableId = command.getTargetTableId(); } - // Kafka offset resolution may perform external I/O. Keep it outside DB/table metadata locks. - PreparedKafkaProperties preparedDataSourceProperties = - prepareDataSourceProperties(dataSourceProperties, false); - unprotectApplyAlter(command, dataSourceProperties, preparedDataSourceProperties); + AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, + jobProperties, dataSourceProperties, command.getTargetTableId()); + Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); } finally { writeUnlock(); } } - private void unprotectApplyAlter(AlterRoutineLoadCommand command, - KafkaDataSourceProperties dataSourceProperties, - PreparedKafkaProperties preparedDataSourceProperties) throws UserException { - modifyPropertiesInternal(command.getAnalyzedJobProperties(), dataSourceProperties, - preparedDataSourceProperties); - if (command.hasTargetTable()) { - tableId = command.getTargetTableId(); - } - setRoutineLoadDesc(command.getRoutineLoadDesc()); - AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(id, - command.getAnalyzedJobProperties(), dataSourceProperties, command.getTargetTableId(), - command.getRoutineLoadDesc()); - Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); - } - - private List> resolveOffsets(KafkaDataSourceProperties dataSourceProperties, - KafkaDataSourceSnapshot dataSourceSnapshot) throws UserException { + private void convertOffset(KafkaDataSourceProperties dataSourceProperties) throws UserException { List> partitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); if (partitionOffsets.isEmpty()) { - return partitionOffsets; + return; } + List> newOffsets; if (dataSourceProperties.isOffsetsForTimes()) { - return KafkaUtil.getOffsetsForTimes(dataSourceSnapshot.brokerList, dataSourceSnapshot.topic, - dataSourceSnapshot.convertedCustomProperties, partitionOffsets); + newOffsets = KafkaUtil.getOffsetsForTimes(brokerList, topic, convertedCustomProperties, partitionOffsets); + } else { + newOffsets = KafkaUtil.getRealOffsets(brokerList, topic, convertedCustomProperties, partitionOffsets); } - return KafkaUtil.getRealOffsets(dataSourceSnapshot.brokerList, dataSourceSnapshot.topic, - dataSourceSnapshot.convertedCustomProperties, partitionOffsets); + dataSourceProperties.setKafkaPartitionOffsets(newOffsets); } private void modifyPropertiesInternal(Map jobProperties, KafkaDataSourceProperties dataSourceProperties) throws UserException { - PreparedKafkaProperties preparedDataSourceProperties = prepareDataSourceProperties(dataSourceProperties, true); - modifyPropertiesInternal(jobProperties, dataSourceProperties, preparedDataSourceProperties); - } - - private void modifyPropertiesInternal(Map jobProperties, - KafkaDataSourceProperties dataSourceProperties, - PreparedKafkaProperties preparedDataSourceProperties) - throws UserException { if (null != dataSourceProperties) { - Preconditions.checkNotNull(preparedDataSourceProperties); - KafkaDataSourceSnapshot dataSourceSnapshot = preparedDataSourceProperties.dataSourceSnapshot; - List> kafkaPartitionOffsets = preparedDataSourceProperties.kafkaPartitionOffsets; - dataSourceProperties.setKafkaPartitionOffsets(kafkaPartitionOffsets); - - if (dataSourceSnapshot.customPropertiesChanged) { - this.customProperties.clear(); - this.customProperties.putAll(dataSourceSnapshot.customProperties); - this.convertedCustomProperties.clear(); - this.convertedCustomProperties.putAll(dataSourceSnapshot.convertedCustomProperties); - this.kafkaDefaultOffSet = dataSourceSnapshot.kafkaDefaultOffset; + List> kafkaPartitionOffsets = Lists.newArrayList(); + Map customKafkaProperties = Maps.newHashMap(); + + if (MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { + kafkaPartitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); + customKafkaProperties = dataSourceProperties.getCustomKafkaProperties(); + } + + // convertCustomProperties and check partitions before reset progress to make modify operation atomic + if (!customKafkaProperties.isEmpty()) { + this.customProperties.putAll(customKafkaProperties); + convertCustomProperties(true); + } + + if (!kafkaPartitionOffsets.isEmpty()) { + ((KafkaProgress) progress).checkPartitions(kafkaPartitionOffsets); + } + + if (Config.isCloudMode()) { + Cloud.ResetRLProgressRequest.Builder builder = Cloud.ResetRLProgressRequest.newBuilder() + .setRequestIp(FrontendOptions.getLocalHostAddressCached()); + builder.setCloudUniqueId(Config.cloud_unique_id); + builder.setDbId(dbId); + builder.setJobId(id); + if (!kafkaPartitionOffsets.isEmpty()) { + Map partitionOffsetMap = new HashMap<>(); + for (Pair pair : kafkaPartitionOffsets) { + // The reason why the value recorded in MS in cloud mode needs to be subtracted by one is + // this value will be incremented + // when pulling MS persistent progress data and updating memory + // in routineLoadJob.updateCloudProgress(). + partitionOffsetMap.put(pair.first, pair.second - 1); + } + builder.putAllPartitionToOffset(partitionOffsetMap); + } + resetCloudProgress(builder); } // It is necessary to reset the Kafka progress cache if topic change, // and should reset cache before modifying partition offset. - if (dataSourceSnapshot.resetProgress) { - this.topic = dataSourceSnapshot.topic; + if (!Strings.isNullOrEmpty(dataSourceProperties.getTopic())) { + this.topic = dataSourceProperties.getTopic(); this.progress = new KafkaProgress(); } @@ -884,17 +863,18 @@ private void modifyPropertiesInternal(Map jobProperties, ((KafkaProgress) progress).modifyOffset(kafkaPartitionOffsets); } - mergePendingCloudProgressReset(dataSourceSnapshot, kafkaPartitionOffsets); - // modify broker list if (!Strings.isNullOrEmpty(dataSourceProperties.getBrokerList())) { - this.brokerList = dataSourceSnapshot.brokerList; + this.brokerList = dataSourceProperties.getBrokerList(); } } if (!jobProperties.isEmpty()) { Map copiedJobProperties = Maps.newHashMap(jobProperties); - modifyCommonJobPropertiesForKafka(copiedJobProperties); + modifyCommonJobProperties(copiedJobProperties); this.jobProperties.putAll(copiedJobProperties); + if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { + this.isPartialUpdate = BooleanUtils.toBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); + } if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { String policy = jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); if ("ERROR".equalsIgnoreCase(policy)) { @@ -908,153 +888,6 @@ private void modifyPropertiesInternal(Map jobProperties, this.id, jobProperties, dataSourceProperties); } - private PreparedKafkaProperties prepareDataSourceProperties(KafkaDataSourceProperties dataSourceProperties, - boolean isReplay) throws UserException { - if (dataSourceProperties == null) { - return null; - } - - KafkaDataSourceSnapshot dataSourceSnapshot = stageDataSourceProperties(dataSourceProperties); - List> kafkaPartitionOffsets = Lists.newArrayList(); - if (MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { - kafkaPartitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); - } - if (!isReplay && !kafkaPartitionOffsets.isEmpty() - && (!dataSourceSnapshot.resetProgress || CollectionUtils.isNotEmpty(customKafkaPartitions))) { - ((KafkaProgress) progress).checkPartitions(kafkaPartitionOffsets); - } - - if (!isReplay) { - kafkaPartitionOffsets = resolveOffsets(dataSourceProperties, dataSourceSnapshot); - } - return new PreparedKafkaProperties(dataSourceSnapshot, copyPartitionOffsets(kafkaPartitionOffsets)); - } - - private List> copyPartitionOffsets(List> kafkaPartitionOffsets) { - List> copiedPartitionOffsets = Lists.newArrayListWithCapacity(kafkaPartitionOffsets.size()); - for (Pair partitionOffset : kafkaPartitionOffsets) { - copiedPartitionOffsets.add(Pair.of(partitionOffset.first, partitionOffset.second)); - } - return copiedPartitionOffsets; - } - - private void mergePendingCloudProgressReset(KafkaDataSourceSnapshot dataSourceSnapshot, - List> kafkaPartitionOffsets) { - if (!Config.isCloudMode() || (!dataSourceSnapshot.resetProgress && kafkaPartitionOffsets.isEmpty())) { - return; - } - - Map partitionOffsetMap = new HashMap<>(); - for (Pair pair : kafkaPartitionOffsets) { - // MS stores the last consumed offset, while FE progress stores the next offset to consume. - partitionOffsetMap.put(pair.first, pair.second - 1); - } - PendingCloudProgressReset newReset = new PendingCloudProgressReset( - dataSourceSnapshot.resetProgress, partitionOffsetMap); - if (pendingCloudProgressReset == null || newReset.clearProgress) { - pendingCloudProgressReset = newReset; - } else { - pendingCloudProgressReset.partitionToOffset.putAll(newReset.partitionToOffset); - } - } - - private Cloud.ResetRLProgressRequest.Builder newResetCloudProgressRequest() { - Cloud.ResetRLProgressRequest.Builder builder = Cloud.ResetRLProgressRequest.newBuilder() - .setRequestIp(FrontendOptions.getLocalHostAddressCached()); - builder.setCloudUniqueId(Config.cloud_unique_id); - builder.setDbId(dbId); - builder.setJobId(id); - return builder; - } - - private void resetPendingCloudProgress() throws DdlException { - if (pendingCloudProgressReset == null) { - return; - } - if (pendingCloudProgressReset.clearProgress) { - resetCloudProgress(newResetCloudProgressRequest()); - } - if (!pendingCloudProgressReset.partitionToOffset.isEmpty()) { - Cloud.ResetRLProgressRequest.Builder builder = newResetCloudProgressRequest(); - builder.putAllPartitionToOffset(pendingCloudProgressReset.partitionToOffset); - resetCloudProgress(builder); - } - pendingCloudProgressReset = null; - } - - private KafkaDataSourceSnapshot stageDataSourceProperties(KafkaDataSourceProperties dataSourceProperties) - throws DdlException { - Map stagedCustomProperties = Maps.newHashMap(customProperties); - Map stagedConvertedCustomProperties = Maps.newHashMap(convertedCustomProperties); - String stagedKafkaDefaultOffset = kafkaDefaultOffSet; - Map alteredCustomProperties = dataSourceProperties.getCustomKafkaProperties(); - boolean customPropertiesChanged = MapUtils.isNotEmpty(alteredCustomProperties); - if (customPropertiesChanged) { - if (alteredCustomProperties.containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()) - && !alteredCustomProperties.containsKey( - KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())) { - stagedCustomProperties.remove(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName()); - } - stagedCustomProperties.putAll(alteredCustomProperties); - Pair, String> convertedProperties = buildConvertedCustomProperties( - stagedCustomProperties, stagedKafkaDefaultOffset); - stagedConvertedCustomProperties = convertedProperties.first; - stagedKafkaDefaultOffset = convertedProperties.second; - } - - boolean resetProgress = !Strings.isNullOrEmpty(dataSourceProperties.getTopic()); - String stagedTopic = resetProgress ? dataSourceProperties.getTopic() : topic; - String stagedBrokerList = Strings.isNullOrEmpty(dataSourceProperties.getBrokerList()) - ? brokerList : dataSourceProperties.getBrokerList(); - return new KafkaDataSourceSnapshot(stagedBrokerList, stagedTopic, stagedCustomProperties, - stagedConvertedCustomProperties, stagedKafkaDefaultOffset, customPropertiesChanged, resetProgress); - } - - private static class KafkaDataSourceSnapshot { - private final String brokerList; - private final String topic; - private final Map customProperties; - private final Map convertedCustomProperties; - private final String kafkaDefaultOffset; - private final boolean customPropertiesChanged; - private final boolean resetProgress; - - private KafkaDataSourceSnapshot(String brokerList, String topic, Map customProperties, - Map convertedCustomProperties, String kafkaDefaultOffset, - boolean customPropertiesChanged, boolean resetProgress) { - this.brokerList = brokerList; - this.topic = topic; - this.customProperties = customProperties; - this.convertedCustomProperties = convertedCustomProperties; - this.kafkaDefaultOffset = kafkaDefaultOffset; - this.customPropertiesChanged = customPropertiesChanged; - this.resetProgress = resetProgress; - } - } - - private static class PreparedKafkaProperties { - private final KafkaDataSourceSnapshot dataSourceSnapshot; - private final List> kafkaPartitionOffsets; - - private PreparedKafkaProperties(KafkaDataSourceSnapshot dataSourceSnapshot, - List> kafkaPartitionOffsets) { - this.dataSourceSnapshot = dataSourceSnapshot; - this.kafkaPartitionOffsets = kafkaPartitionOffsets; - } - } - - private static class PendingCloudProgressReset { - @SerializedName("clearProgress") - private boolean clearProgress; - @SerializedName("partitionToOffset") - private Map partitionToOffset = new HashMap<>(); - - private PendingCloudProgressReset(boolean clearProgress, Map partitionToOffset) { - this.clearProgress = clearProgress; - this.partitionToOffset.putAll(partitionToOffset); - } - } - private void resetCloudProgress(Cloud.ResetRLProgressRequest.Builder builder) throws DdlException { Cloud.ResetRLProgressResponse response; try { @@ -1077,12 +910,10 @@ private void resetCloudProgress(Cloud.ResetRLProgressRequest.Builder builder) th @Override public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { try { - modifyPropertiesInternal(log.getJobProperties(), - (KafkaDataSourceProperties) log.getDataSourceProperties()); + modifyPropertiesInternal(log.getJobProperties(), (KafkaDataSourceProperties) log.getDataSourceProperties()); if (log.getTargetTableId() != 0) { - this.tableId = log.getTargetTableId(); + tableId = log.getTargetTableId(); } - setRoutineLoadDesc(log.getRoutineLoadDesc()); } catch (UserException e) { // should not happen LOG.error("failed to replay modify kafka routine load job: {}", id, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index 7cebc3f5165b49..32208975eea95f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -687,9 +687,12 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti } modifyPropertiesInternal(jobProperties, dataSourceProperties); + if (command.hasTargetTable()) { + this.tableId = command.getTargetTableId(); + } AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, - jobProperties, dataSourceProperties); + jobProperties, dataSourceProperties, command.getTargetTableId()); Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); } finally { writeUnlock(); @@ -783,6 +786,9 @@ public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { try { modifyPropertiesInternal(log.getJobProperties(), (KinesisDataSourceProperties) log.getDataSourceProperties()); + if (log.getTargetTableId() != 0) { + this.tableId = log.getTargetTableId(); + } } catch (UserException e) { LOG.error("failed to replay modify kinesis routine load job: {}", id, e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsRoutineLoadTaskInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsRoutineLoadTaskInfo.java index c2ad889ca36681..ef159cfb6f494a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsRoutineLoadTaskInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsRoutineLoadTaskInfo.java @@ -121,13 +121,6 @@ public int calTimeoutSec() { return realTimeoutSec; } - /** Apply analyzed ALTER properties to this validation-only task. */ - public void applyAlterPropertiesForValidation(Map alteredJobProperties, - TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode) { - jobProperties.putAll(alteredJobProperties); - uniquekeyUpdateMode = effectiveUniqueKeyUpdateMode; - } - @Override public int getTimeout() { return this.timeoutSec; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index c4909e1e421285..84366af77e1c5a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -3752,13 +3752,6 @@ private String decodeStringLiteral(String txt) { return s; } - private String decodeIdentifier(String txt) { - if (txt.charAt(0) == '`' && txt.charAt(txt.length() - 1) == '`') { - return txt.substring(1, txt.length() - 1).replace("``", "`"); - } - return txt; - } - @Override public Literal visitVarbinaryLiteral(DorisParser.VarbinaryLiteralContext ctx) { String txt = ctx.VARBINARY_LITERAL().getText(); @@ -9322,9 +9315,8 @@ public LogicalPlan visitAlterRoutineLoad(DorisParser.AlterRoutineLoadContext ctx } } - String dataSourceType = ctx.type == null ? null : decodeIdentifier(ctx.type.getText()); return new AlterRoutineLoadCommand(labelNameInfo, targetTableName, - loadPropertyMap, properties, dataSourceType, dataSourceMapProperties); + loadPropertyMap, properties, dataSourceMapProperties); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java index 4f87d00938e7b1..8ad6392e329f28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java @@ -35,7 +35,6 @@ import org.apache.doris.datasource.property.fileformat.JsonFileFormatProperties; import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.routineload.AbstractDataSourceProperties; -import org.apache.doris.load.routineload.LoadDataSourceType; import org.apache.doris.load.routineload.RoutineLoadDataSourcePropertyFactory; import org.apache.doris.load.routineload.RoutineLoadJob; import org.apache.doris.mysql.privilege.PrivPredicate; @@ -46,7 +45,6 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.StmtExecutor; -import org.apache.doris.thrift.TUniqueKeyUpdateMode; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; @@ -97,10 +95,8 @@ public class AlterRoutineLoadCommand extends AlterCommand { private final Map loadPropertyMap; private RoutineLoadDesc routineLoadDesc; private final Map jobProperties; - private final String dataSourceType; private final Map dataSourceMapProperties; private long targetTableId; - private long loadPropertyTableId; private boolean isPartialUpdate; // save analyzed job properties. @@ -115,7 +111,6 @@ public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, String targetTableName, Map loadPropertyMap, Map jobProperties, - String dataSourceType, Map dataSourceMapProperties) { super(PlanType.ALTER_ROUTINE_LOAD_COMMAND); Objects.requireNonNull(labelNameInfo, "labelNameInfo is null"); @@ -125,7 +120,6 @@ public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, this.targetTableName = targetTableName; this.loadPropertyMap = loadPropertyMap == null ? Maps.newHashMap() : loadPropertyMap; this.jobProperties = jobProperties; - this.dataSourceType = dataSourceType; this.dataSourceMapProperties = dataSourceMapProperties; this.isPartialUpdate = this.jobProperties.getOrDefault(CreateRoutineLoadInfo.PARTIAL_COLUMNS, "false") .equalsIgnoreCase("true"); @@ -134,7 +128,7 @@ public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, Map jobProperties, Map dataSourceMapProperties) { - this(labelNameInfo, null, Maps.newHashMap(), jobProperties, null, dataSourceMapProperties); + this(labelNameInfo, null, Maps.newHashMap(), jobProperties, dataSourceMapProperties); } public String getDbName() { @@ -161,10 +155,6 @@ public long getTargetTableId() { return targetTableId; } - public long getLoadPropertyTableId() { - return loadPropertyTableId; - } - public boolean hasDataSourceProperty() { return MapUtils.isNotEmpty(dataSourceMapProperties); } @@ -196,24 +186,21 @@ public void validate(ConnectContext ctx) throws UserException { // check routine load job properties include desired concurrent number etc. checkJobProperties(); // check load properties - RoutineLoadJob job = Env.getCurrentEnv().getRoutineLoadManager() - .checkPrivAndGetJob(getDbName(), getJobName()); + RoutineLoadJob job = hasTargetTable() + ? Env.getCurrentEnv().getRoutineLoadManager().checkPrivAndGetJob(getDbName(), getJobName()) + : Env.getCurrentEnv().getRoutineLoadManager().getJob(getDbName(), getJobName()); if (MapUtils.isNotEmpty(loadPropertyMap)) { - this.loadPropertyTableId = job.getTableId(); this.routineLoadDesc = CreateRoutineLoadInfo.checkLoadProperties(ctx, loadPropertyMap, job.getDbFullName(), job.getTableName(), job.isMultiTable(), job.getMergeType()); } // check data source properties - checkDataSourceProperties(job); - TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode = RoutineLoadJob.getEffectiveUniqueKeyUpdateMode( - job.getUniqueKeyUpdateMode(), analyzedJobProperties); + checkDataSourceProperties(); + checkPartialUpdate(); if (hasTargetTable()) { - validateTargetTable(ctx, job, effectiveUniqueKeyUpdateMode); - } else { - validateAlterJobProperties(job, effectiveUniqueKeyUpdateMode); + validateTargetTable(ctx, job); } if (analyzedJobProperties.isEmpty() && MapUtils.isEmpty(dataSourceMapProperties) - && MapUtils.isEmpty(loadPropertyMap) && !hasTargetTable()) { + && routineLoadDesc == null && !hasTargetTable()) { throw new AnalysisException("No properties are specified"); } } @@ -356,40 +343,36 @@ private void checkJobProperties() throws UserException { } } - private void checkDataSourceProperties(RoutineLoadJob job) throws UserException { + private void checkDataSourceProperties() throws UserException { if (MapUtils.isEmpty(dataSourceMapProperties)) { return; } - String effectiveDataSourceType = dataSourceType == null ? job.getDataSourceType().name() : dataSourceType; - if (!effectiveDataSourceType.equalsIgnoreCase(job.getDataSourceType().name())) { - throw new AnalysisException("The specified job type is not: " + effectiveDataSourceType); - } + RoutineLoadJob job = Env.getCurrentEnv().getRoutineLoadManager() + .getJob(getDbName(), getJobName()); this.dataSourceProperties = RoutineLoadDataSourcePropertyFactory - .createDataSource(effectiveDataSourceType, dataSourceMapProperties, job.isMultiTable()); + .createDataSource(job.getDataSourceType().name(), dataSourceMapProperties, job.isMultiTable()); dataSourceProperties.setAlter(true); - dataSourceProperties.setTimezone(analyzedJobProperties.getOrDefault( - CreateRoutineLoadInfo.TIMEZONE, job.getTimezone())); + dataSourceProperties.setTimezone(job.getTimezone()); dataSourceProperties.analyze(); } - private void validateAlterJobProperties(RoutineLoadJob job, - TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode) throws UserException { - if (effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { + private void checkPartialUpdate() throws UserException { + if (!isPartialUpdate) { return; } + RoutineLoadJob job = Env.getCurrentEnv().getRoutineLoadManager() + .getJob(getDbName(), getDbName()); if (job.isMultiTable()) { - throw new AnalysisException("Partial update is not supported in multi-table load."); + throw new AnalysisException("load by PARTIAL_COLUMNS is not supported in multi-table load."); } Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException(job.getDbFullName()); Table table = db.getTableOrAnalysisException(job.getTableName()); - job.validateAlterJobProperties((OlapTable) table, analyzedJobProperties, effectiveUniqueKeyUpdateMode); + if (isPartialUpdate && !((OlapTable) table).getEnableUniqueKeyMergeOnWrite()) { + throw new AnalysisException("load by PARTIAL_COLUMNS is only supported in unique table MoW"); + } } - private void validateTargetTable(ConnectContext ctx, RoutineLoadJob job, - TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode) throws UserException { - if (job.getDataSourceType() != LoadDataSourceType.KAFKA) { - throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports Kafka jobs"); - } + private void validateTargetTable(ConnectContext ctx, RoutineLoadJob job) throws UserException { if (job.isMultiTable()) { throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports single-table job"); } @@ -413,7 +396,7 @@ private void validateTargetTable(ConnectContext ctx, RoutineLoadJob job, throw new AnalysisException( "if load_to_single_tablet set to true, the olap table must be with random distribution"); } - job.validateTargetTable(db, olapTable, analyzedJobProperties, effectiveUniqueKeyUpdateMode); + job.validateTargetTable(db, olapTable); this.targetTableId = olapTable.getId(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java index f69129bd74097f..ae7c8ad214de7c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java @@ -17,14 +17,8 @@ package org.apache.doris.persist; -import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.ImportColumnDesc; -import org.apache.doris.analysis.Separator; -import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.load.RoutineLoadDesc; -import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.AbstractDataSourceProperties; import org.apache.doris.persist.gson.GsonUtils; @@ -33,7 +27,6 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.List; import java.util.Map; public class AlterRoutineLoadJobOperationLog implements Writable { @@ -46,8 +39,6 @@ public class AlterRoutineLoadJobOperationLog implements Writable { private AbstractDataSourceProperties dataSourceProperties; @SerializedName(value = "targetTableId") private long targetTableId; - @SerializedName(value = "routineLoadDesc") - private RoutineLoadDescSnapshot routineLoadDesc; public AlterRoutineLoadJobOperationLog(long jobId, Map jobProperties, AbstractDataSourceProperties dataSourceProperties) { @@ -56,17 +47,10 @@ public AlterRoutineLoadJobOperationLog(long jobId, Map jobProper public AlterRoutineLoadJobOperationLog(long jobId, Map jobProperties, AbstractDataSourceProperties dataSourceProperties, long targetTableId) { - this(jobId, jobProperties, dataSourceProperties, targetTableId, null); - } - - public AlterRoutineLoadJobOperationLog(long jobId, Map jobProperties, - AbstractDataSourceProperties dataSourceProperties, long targetTableId, - RoutineLoadDesc routineLoadDesc) { this.jobId = jobId; this.jobProperties = jobProperties; this.dataSourceProperties = dataSourceProperties; this.targetTableId = targetTableId; - this.routineLoadDesc = routineLoadDesc == null ? null : new RoutineLoadDescSnapshot(routineLoadDesc); } public long getJobId() { @@ -85,69 +69,6 @@ public long getTargetTableId() { return targetTableId; } - public RoutineLoadDesc getRoutineLoadDesc() { - return routineLoadDesc == null ? null : routineLoadDesc.toRoutineLoadDesc(); - } - - private static class RoutineLoadDescSnapshot { - @SerializedName("columnSeparator") - private SeparatorSnapshot columnSeparator; - @SerializedName("lineDelimiter") - private SeparatorSnapshot lineDelimiter; - @SerializedName("columnsInfo") - private List columnsInfo; - @SerializedName("precedingFilter") - private Expr precedingFilter; - @SerializedName("filter") - private Expr filter; - @SerializedName("partitionNamesInfo") - private PartitionNamesInfo partitionNamesInfo; - @SerializedName("deleteCondition") - private Expr deleteCondition; - @SerializedName("mergeType") - private LoadTask.MergeType mergeType; - @SerializedName("sequenceColName") - private String sequenceColName; - - private RoutineLoadDescSnapshot(RoutineLoadDesc routineLoadDesc) { - this.columnSeparator = SeparatorSnapshot.fromSeparator(routineLoadDesc.getColumnSeparator()); - this.lineDelimiter = SeparatorSnapshot.fromSeparator(routineLoadDesc.getLineDelimiter()); - this.columnsInfo = routineLoadDesc.getColumnsInfo(); - this.precedingFilter = routineLoadDesc.getPrecedingFilter(); - this.filter = routineLoadDesc.getFilter(); - this.partitionNamesInfo = routineLoadDesc.getPartitionNamesInfo(); - this.deleteCondition = routineLoadDesc.getDeleteCondition(); - this.mergeType = routineLoadDesc.getMergeType(); - this.sequenceColName = routineLoadDesc.getSequenceColName(); - } - - private RoutineLoadDesc toRoutineLoadDesc() { - return new RoutineLoadDesc(SeparatorSnapshot.toSeparator(columnSeparator), - SeparatorSnapshot.toSeparator(lineDelimiter), columnsInfo, precedingFilter, filter, - partitionNamesInfo, deleteCondition, mergeType, sequenceColName); - } - } - - private static class SeparatorSnapshot { - @SerializedName("separator") - private String separator; - @SerializedName("oriSeparator") - private String oriSeparator; - - private SeparatorSnapshot(Separator separator) { - this.separator = separator.getSeparator(); - this.oriSeparator = separator.getOriSeparator(); - } - - private static SeparatorSnapshot fromSeparator(Separator separator) { - return separator == null ? null : new SeparatorSnapshot(separator); - } - - private static Separator toSeparator(SeparatorSnapshot snapshot) { - return snapshot == null ? null : new Separator(snapshot.separator, snapshot.oriSeparator); - } - } - public static AlterRoutineLoadJobOperationLog read(DataInput in) throws IOException { String json = Text.readString(in); return GsonUtils.GSON.fromJson(json, AlterRoutineLoadJobOperationLog.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index 52f251401d7119..193fc9ba22731b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -25,15 +25,11 @@ import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.info.PartitionNamesInfo; -import org.apache.doris.cloud.proto.Cloud; -import org.apache.doris.cloud.rpc.MetaServiceProxy; import org.apache.doris.common.Config; -import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.common.util.SmallFileMgr; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.kafka.KafkaUtil; import org.apache.doris.load.RoutineLoadDesc; @@ -51,13 +47,10 @@ import org.apache.doris.nereids.trees.plans.commands.load.LoadSeparator; import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.persist.EditLog; -import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TResourceInfo; -import org.apache.doris.thrift.TUniqueKeyUpdateMode; import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.kafka.common.PartitionInfo; @@ -67,6 +60,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -76,8 +70,6 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.ReentrantReadWriteLock; public class KafkaRoutineLoadJobTest { private static final Logger LOG = LogManager.getLogger(KafkaRoutineLoadJobTest.class); @@ -210,472 +202,138 @@ public void testUpdateLagRefreshesLatestOffsetCache() throws UserException { } @Test - public void testModifyPropertiesHonorsExplicitUniqueKeyUpdateModePrecedence() throws Exception { + public void testUpdateLagRebuildsConvertedPropertiesAfterReplay() throws UserException { KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "uniqueKeyUpdateMode", TUniqueKeyUpdateMode.UPSERT); - Deencapsulation.setField(routineLoadJob, "isPartialUpdate", false); - - Map jobProperties = Maps.newHashMap(); - jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, "UPSERT"); - jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, "true"); + Deencapsulation.setField(routineLoadJob, "customKafkaPartitions", Lists.newArrayList(1)); - Deencapsulation.invoke(routineLoadJob, "modifyPropertiesInternal", jobProperties, - new KafkaDataSourceProperties(Maps.newHashMap())); + Map customProperties = Maps.newHashMap(); + customProperties.put("security.protocol", "SASL_PLAINTEXT"); + customProperties.put("sasl.mechanism", "PLAIN"); + Deencapsulation.setField(routineLoadJob, "customProperties", customProperties); + Deencapsulation.setField(routineLoadJob, "convertedCustomProperties", Maps.newHashMap()); - Assert.assertEquals(TUniqueKeyUpdateMode.UPSERT, routineLoadJob.getUniqueKeyUpdateMode()); - Assert.assertFalse(routineLoadJob.isFixedPartialUpdate()); - } + Map partitionIdToOffset = Maps.newHashMap(); + partitionIdToOffset.put(1, 10L); + Deencapsulation.setField(routineLoadJob, "progress", new KafkaProgress(partitionIdToOffset)); - @Test - public void testModifyPropertiesRevalidatesWhileHoldingWriteLock() throws Exception { - KafkaRoutineLoadJob routineLoadJob = Mockito.spy(new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, - 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN)); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); - Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + Env env = Mockito.mock(Env.class); + try (MockedStatic envStatic = Mockito.mockStatic(Env.class); + MockedStatic kafkaUtilStatic = Mockito.mockStatic(KafkaUtil.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + kafkaUtilStatic.when(() -> KafkaUtil.getLatestOffsets(Mockito.eq(1L), Mockito.any(UUID.class), + Mockito.eq("127.0.0.1:9020"), Mockito.eq("topic1"), + Mockito.>argThat(properties -> + "SASL_PLAINTEXT".equals(properties.get("security.protocol")) + && "PLAIN".equals(properties.get("sasl.mechanism"))), + Mockito.argThat(partitions -> partitions.size() == 1 && partitions.contains(1)))) + .thenReturn(Lists.newArrayList(Pair.of(1, 15L))); - Mockito.doAnswer(invocation -> { - ReentrantReadWriteLock lock = Deencapsulation.getField(routineLoadJob, "lock"); - Assert.assertTrue(lock.isWriteLockedByCurrentThread()); - throw new DdlException("stop after validation"); - }).when(routineLoadJob).validateAlterJobPropertiesForMutation(command); + routineLoadJob.updateLag(); - try { - routineLoadJob.modifyProperties(command); - Assert.fail("Expected mutation-time validation to fail"); - } catch (DdlException e) { - Assert.assertTrue(e.getMessage().contains("stop after validation")); + Assert.assertEquals(5L, routineLoadJob.totalLag().longValue()); + kafkaUtilStatic.verify(() -> KafkaUtil.getLatestOffsets(Mockito.eq(1L), Mockito.any(UUID.class), + Mockito.eq("127.0.0.1:9020"), Mockito.eq("topic1"), + Mockito.>argThat(properties -> + "SASL_PLAINTEXT".equals(properties.get("security.protocol")) + && "PLAIN".equals(properties.get("sasl.mechanism"))), + Mockito.argThat(partitions -> partitions.size() == 1 && partitions.contains(1)))); } } @Test - public void testModifyPropertiesRejectsLoadDescValidatedAgainstStaleTable() throws Exception { + public void testModifyTargetTableWithJobAndDataSourceProperties() throws Exception { KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, - 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); - Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); - Mockito.when(command.getRoutineLoadDesc()).thenReturn(Mockito.mock(RoutineLoadDesc.class)); - Mockito.when(command.getLoadPropertyTableId()).thenReturn(2L); - - DdlException exception = Assert.assertThrows(DdlException.class, - () -> routineLoadJob.modifyProperties(command)); - - Assert.assertTrue(exception.getMessage().contains("please retry")); - Assert.assertEquals(1L, routineLoadJob.getTableId()); - } - - @Test - public void testModifyTargetTableUsesOnlyJobLockAndDefersCloudIoUntilResume() throws Exception { - KafkaRoutineLoadJob routineLoadJob = Mockito.spy(new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, - 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN)); + 101L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - Deencapsulation.setField(routineLoadJob, "uniqueKeyUpdateMode", TUniqueKeyUpdateMode.UPSERT); - KafkaProgress originalProgress = new KafkaProgress(Maps.newHashMap(ImmutableMap.of(1, 10L))); - Deencapsulation.setField(routineLoadJob, "progress", originalProgress); + KafkaProgress progress = new KafkaProgress(Maps.newHashMap()); + Deencapsulation.setField(routineLoadJob, "progress", progress); - Map alteredSourceProperties = Maps.newHashMap(); - alteredSourceProperties.put(KafkaConfiguration.KAFKA_TOPIC.getName(), "topic2"); - alteredSourceProperties.put(KafkaConfiguration.KAFKA_PARTITIONS.getName(), "1"); - alteredSourceProperties.put(KafkaConfiguration.KAFKA_OFFSETS.getName(), KafkaProgress.OFFSET_BEGINNING); - KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties(alteredSourceProperties); + Map sourceProperties = Maps.newHashMap(); + sourceProperties.put("property.client.id", "target-switch"); + KafkaDataSourceProperties dataSourceProperties = new KafkaDataSourceProperties(sourceProperties); + dataSourceProperties.setAlter(true); + dataSourceProperties.setTimezone("UTC"); + dataSourceProperties.analyze(); + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CreateRoutineLoadInfo.MAX_ERROR_NUMBER_PROPERTY, "10"); AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); Mockito.when(command.hasTargetTable()).thenReturn(true); - Mockito.when(command.getTargetTableId()).thenReturn(2L); - Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + Mockito.when(command.getTargetTableId()).thenReturn(202L); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); Mockito.when(command.getDataSourceProperties()).thenReturn(dataSourceProperties); Env env = Mockito.mock(Env.class); EditLog editLog = Mockito.mock(EditLog.class); Mockito.when(env.getEditLog()).thenReturn(editLog); - - AtomicBoolean kafkaPrepared = new AtomicBoolean(false); - AtomicBoolean cloudProgressReset = new AtomicBoolean(false); - Mockito.doAnswer(invocation -> { - ReentrantReadWriteLock lock = Deencapsulation.getField(routineLoadJob, "lock"); - Assert.assertTrue(lock.isWriteLockedByCurrentThread()); - Assert.assertTrue(kafkaPrepared.get()); - Assert.assertFalse(cloudProgressReset.get()); - Assert.assertEquals(2L, routineLoadJob.getTableId()); - Assert.assertEquals("topic2", routineLoadJob.getTopic()); - return null; - }).when(editLog).logAlterRoutineLoadJob(Mockito.any(AlterRoutineLoadJobOperationLog.class)); - - MetaServiceProxy metaServiceProxy = Mockito.mock(MetaServiceProxy.class); - Cloud.ResetRLProgressResponse response = Cloud.ResetRLProgressResponse.newBuilder() - .setStatus(Cloud.MetaServiceResponseStatus.newBuilder().setCode(Cloud.MetaServiceCode.OK)) - .build(); - String originalCloudUniqueId = Config.cloud_unique_id; - Config.cloud_unique_id = "test-cloud"; - try (MockedStatic envStatic = Mockito.mockStatic(Env.class); - MockedStatic kafkaUtilStatic = Mockito.mockStatic(KafkaUtil.class); - MockedStatic metaServiceProxyStatic = Mockito.mockStatic(MetaServiceProxy.class)) { - envStatic.when(Env::getCurrentEnv).thenReturn(env); - kafkaUtilStatic.when(() -> KafkaUtil.getRealOffsets(Mockito.eq("127.0.0.1:9020"), - Mockito.eq("topic2"), Mockito.anyMap(), Mockito.anyList())).thenAnswer(invocation -> { - ReentrantReadWriteLock lock = Deencapsulation.getField(routineLoadJob, "lock"); - Assert.assertTrue(lock.isWriteLockedByCurrentThread()); - kafkaPrepared.set(true); - return Lists.newArrayList(Pair.of(1, 100L)); - }); - metaServiceProxyStatic.when(MetaServiceProxy::getInstance).thenReturn(metaServiceProxy); - Mockito.when(metaServiceProxy.resetRLProgress(Mockito.any())).thenAnswer(invocation -> { - ReentrantReadWriteLock lock = Deencapsulation.getField(routineLoadJob, "lock"); - Assert.assertTrue(lock.isWriteLockedByCurrentThread()); - Assert.assertTrue(kafkaPrepared.get()); - cloudProgressReset.set(true); - return response; - }); - - routineLoadJob.modifyProperties(command); - String persistedJob = GsonUtils.GSON.toJson(routineLoadJob); - Assert.assertTrue(persistedJob.contains("\"pendingCloudProgressReset\"")); - Assert.assertTrue(persistedJob.contains("\"clearProgress\":true")); - Assert.assertTrue(persistedJob.contains("\"1\":99")); - Mockito.verifyNoInteractions(metaServiceProxy); - routineLoadJob.updateState(RoutineLoadJob.JobState.NEED_SCHEDULE, null, false); - } finally { - Config.cloud_unique_id = originalCloudUniqueId; - } - - Assert.assertTrue(kafkaPrepared.get()); - Assert.assertTrue(cloudProgressReset.get()); - Assert.assertEquals(2L, routineLoadJob.getTableId()); - Assert.assertEquals("topic2", routineLoadJob.getTopic()); - Mockito.verify(routineLoadJob, Mockito.never()).unprotectedValidateTargetTable( - Mockito.any(), Mockito.any(), Mockito.anyMap(), Mockito.any()); - Mockito.verify(editLog).logAlterRoutineLoadJob(Mockito.any(AlterRoutineLoadJobOperationLog.class)); - Mockito.verify(metaServiceProxy, Mockito.times(2)).resetRLProgress(Mockito.any()); - } - - @Test - public void testModifyPropertiesKeepsKafkaPropertiesWhenFileConversionFails() throws Exception { - KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, - 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - Map originalCustomProperties = Maps.newHashMap(); - originalCustomProperties.put("client.id", "original-client"); - Map originalConvertedProperties = Maps.newHashMap(originalCustomProperties); - Deencapsulation.setField(routineLoadJob, "customProperties", originalCustomProperties); - Deencapsulation.setField(routineLoadJob, "convertedCustomProperties", originalConvertedProperties); - - Map alteredSourceProperties = Maps.newHashMap(); - alteredSourceProperties.put("property.ssl.ca.location", "FILE:missing.pem"); - KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties(alteredSourceProperties); - AlterRoutineLoadCommand command = mockAlterCommand(dataSourceProperties); - Env env = Mockito.mock(Env.class); - SmallFileMgr smallFileMgr = Mockito.mock(SmallFileMgr.class); - Mockito.when(env.getSmallFileMgr()).thenReturn(smallFileMgr); - Mockito.when(smallFileMgr.getSmallFile(1L, KafkaRoutineLoadJob.KAFKA_FILE_CATALOG, - "missing.pem", true)).thenThrow(new DdlException("missing file")); - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); - try { - routineLoadJob.modifyProperties(command); - Assert.fail("Expected missing small file to reject ALTER"); - } catch (DdlException e) { - Assert.assertTrue(e.getMessage().contains("missing file")); - } - } - - Assert.assertEquals(Maps.newHashMap(ImmutableMap.of("client.id", "original-client")), - Deencapsulation.getField(routineLoadJob, "customProperties")); - Assert.assertEquals(Maps.newHashMap(ImmutableMap.of("client.id", "original-client")), - routineLoadJob.getConvertedCustomProperties()); - } - - @Test - public void testModifyPropertiesKeepsKafkaPropertiesWhenPartitionValidationFails() throws Exception { - KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, - 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - Deencapsulation.setField(routineLoadJob, "progress", - new KafkaProgress(Maps.newHashMap(ImmutableMap.of(1, 10L)))); - Deencapsulation.setField(routineLoadJob, "customProperties", - Maps.newHashMap(ImmutableMap.of("client.id", "original-client"))); - Deencapsulation.setField(routineLoadJob, "convertedCustomProperties", - Maps.newHashMap(ImmutableMap.of("client.id", "original-client"))); - - Map alteredSourceProperties = Maps.newHashMap(); - alteredSourceProperties.put("property.client.id", "new-client"); - alteredSourceProperties.put(KafkaConfiguration.KAFKA_PARTITIONS.getName(), "2"); - alteredSourceProperties.put(KafkaConfiguration.KAFKA_OFFSETS.getName(), "100"); - KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties(alteredSourceProperties); - - try { - routineLoadJob.modifyProperties(mockAlterCommand(dataSourceProperties)); - Assert.fail("Expected unknown partition to reject ALTER"); - } catch (DdlException e) { - Assert.assertTrue(e.getMessage().contains("2")); + routineLoadJob.modifyProperties(command); } - Assert.assertEquals("original-client", - Deencapsulation.>getField(routineLoadJob, "customProperties").get("client.id")); - Assert.assertEquals("original-client", routineLoadJob.getConvertedCustomProperties().get("client.id")); + Assert.assertEquals(202L, routineLoadJob.getTableId()); + Assert.assertSame(progress, routineLoadJob.getProgress()); + Assert.assertEquals(10L, ((Long) Deencapsulation.getField(routineLoadJob, "maxErrorNum")).longValue()); + Assert.assertEquals("target-switch", + routineLoadJob.getCustomProperties().get("property.client.id")); + ArgumentCaptor logCaptor = + ArgumentCaptor.forClass(AlterRoutineLoadJobOperationLog.class); + Mockito.verify(editLog).logAlterRoutineLoadJob(logCaptor.capture()); + Assert.assertEquals(202L, logCaptor.getValue().getTargetTableId()); } @Test - public void testModifyTopicRejectsPartitionOutsidePinnedSet() throws Exception { + public void testModifyTargetTableOnly() throws Exception { KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, - 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + 101L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - KafkaProgress progress = new KafkaProgress(Maps.newHashMap(ImmutableMap.of(1, 10L))); + KafkaProgress progress = new KafkaProgress(Maps.newHashMap()); Deencapsulation.setField(routineLoadJob, "progress", progress); - Deencapsulation.setField(routineLoadJob, "customKafkaPartitions", Lists.newArrayList(1)); - Map alteredSourceProperties = Maps.newHashMap(); - alteredSourceProperties.put(KafkaConfiguration.KAFKA_TOPIC.getName(), "topic2"); - alteredSourceProperties.put(KafkaConfiguration.KAFKA_PARTITIONS.getName(), "2"); - alteredSourceProperties.put(KafkaConfiguration.KAFKA_OFFSETS.getName(), "100"); - KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties(alteredSourceProperties); - - DdlException exception = Assert.assertThrows(DdlException.class, - () -> routineLoadJob.modifyProperties(mockAlterCommand(dataSourceProperties))); - Assert.assertTrue(exception.getMessage().contains("2")); - Assert.assertEquals("topic1", routineLoadJob.getTopic()); - Assert.assertSame(progress, routineLoadJob.getProgress()); - Assert.assertEquals(Lists.newArrayList(1), - Deencapsulation.getField(routineLoadJob, "customKafkaPartitions")); - } + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.hasTargetTable()).thenReturn(true); + Mockito.when(command.getTargetTableId()).thenReturn(202L); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + Mockito.when(command.getDataSourceProperties()).thenReturn(null); - @Test - public void testModifyPropertiesResolvesOffsetsWithEffectiveKafkaConfiguration() throws Exception { - KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, - 1L, "old-broker:9092", "old-topic", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - Map alteredSourceProperties = Maps.newHashMap(); - alteredSourceProperties.put(KafkaConfiguration.KAFKA_BROKER_LIST.getName(), "new-broker:9092"); - alteredSourceProperties.put(KafkaConfiguration.KAFKA_TOPIC.getName(), "new-topic"); - alteredSourceProperties.put(KafkaConfiguration.KAFKA_PARTITIONS.getName(), "7"); - alteredSourceProperties.put("property." + KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName(), - "2026-01-01 00:00:00"); - alteredSourceProperties.put("property.client.id", "new-client"); - KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties(alteredSourceProperties); - AlterRoutineLoadCommand command = mockAlterCommand(dataSourceProperties); Env env = Mockito.mock(Env.class); - Mockito.when(env.getEditLog()).thenReturn(Mockito.mock(EditLog.class)); - - String originalCloudUniqueId = Config.cloud_unique_id; - String originalDeployMode = Config.deploy_mode; - Config.cloud_unique_id = ""; - Config.deploy_mode = ""; - try (MockedStatic envStatic = Mockito.mockStatic(Env.class); - MockedStatic kafkaUtilStatic = Mockito.mockStatic(KafkaUtil.class)) { + EditLog editLog = Mockito.mock(EditLog.class); + Mockito.when(env.getEditLog()).thenReturn(editLog); + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); - kafkaUtilStatic.when(() -> KafkaUtil.getOffsetsForTimes(Mockito.eq("new-broker:9092"), - Mockito.eq("new-topic"), Mockito.anyMap(), Mockito.anyList())).thenAnswer(invocation -> { - Map effectiveCustomProperties = invocation.getArgument(2); - Assert.assertEquals("new-client", effectiveCustomProperties.get("client.id")); - Assert.assertFalse(effectiveCustomProperties - .containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName())); - Assert.assertFalse(effectiveCustomProperties - .containsKey(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())); - return Lists.newArrayList(Pair.of(7, 700L)); - }); - routineLoadJob.modifyProperties(command); - } finally { - Config.cloud_unique_id = originalCloudUniqueId; - Config.deploy_mode = originalDeployMode; } - Assert.assertEquals("new-broker:9092", routineLoadJob.getBrokerList()); - Assert.assertEquals("new-topic", routineLoadJob.getTopic()); - Assert.assertEquals(Long.valueOf(700L), ((KafkaProgress) routineLoadJob.getProgress()).getOffsetByPartition(7)); - Assert.assertEquals("new-client", routineLoadJob.getCustomProperties().get("property.client.id")); + Assert.assertEquals(202L, routineLoadJob.getTableId()); + Assert.assertSame(progress, routineLoadJob.getProgress()); + ArgumentCaptor logCaptor = + ArgumentCaptor.forClass(AlterRoutineLoadJobOperationLog.class); + Mockito.verify(editLog).logAlterRoutineLoadJob(logCaptor.capture()); + Assert.assertEquals(202L, logCaptor.getValue().getTargetTableId()); + Assert.assertTrue(logCaptor.getValue().getJobProperties().isEmpty()); + Assert.assertNull(logCaptor.getValue().getDataSourceProperties()); } @Test - public void testModifyCustomKafkaPropertiesDoesNotResetCloudProgress() throws Exception { + public void testReplayModifyPropertiesSwitchesTargetTableWithoutResettingProgress() { KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, - 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - KafkaProgress progress = new KafkaProgress(Maps.newHashMap(ImmutableMap.of(1, 123L))); + 101L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Map partitionToOffset = Maps.newHashMap(); + partitionToOffset.put(1, 123L); + KafkaProgress progress = new KafkaProgress(partitionToOffset); Deencapsulation.setField(routineLoadJob, "progress", progress); - Deencapsulation.setField(routineLoadJob, "customProperties", Maps.newHashMap( - ImmutableMap.of(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName(), KafkaProgress.OFFSET_BEGINNING))); - Deencapsulation.setField(routineLoadJob, "kafkaDefaultOffSet", KafkaProgress.OFFSET_BEGINNING); - - KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties( - Maps.newHashMap(ImmutableMap.of("property.client.id", "new-client"))); - Assert.assertFalse(dataSourceProperties.getCustomKafkaProperties() - .containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName())); - Env env = Mockito.mock(Env.class); - Mockito.when(env.getEditLog()).thenReturn(Mockito.mock(EditLog.class)); - MetaServiceProxy metaServiceProxy = Mockito.mock(MetaServiceProxy.class); - String originalCloudUniqueId = Config.cloud_unique_id; - Config.cloud_unique_id = "test-cloud"; - try (MockedStatic envStatic = Mockito.mockStatic(Env.class); - MockedStatic metaServiceProxyStatic = Mockito.mockStatic(MetaServiceProxy.class)) { - envStatic.when(Env::getCurrentEnv).thenReturn(env); - metaServiceProxyStatic.when(MetaServiceProxy::getInstance).thenReturn(metaServiceProxy); - - routineLoadJob.modifyProperties(mockAlterCommand(dataSourceProperties)); - } finally { - Config.cloud_unique_id = originalCloudUniqueId; - } + routineLoadJob.replayModifyProperties(new AlterRoutineLoadJobOperationLog( + 1L, Maps.newHashMap(), null)); + Assert.assertEquals(101L, routineLoadJob.getTableId()); - Mockito.verifyNoInteractions(metaServiceProxy); + routineLoadJob.replayModifyProperties(new AlterRoutineLoadJobOperationLog( + 1L, Maps.newHashMap(), null, 202L)); + Assert.assertEquals(202L, routineLoadJob.getTableId()); Assert.assertSame(progress, routineLoadJob.getProgress()); - Assert.assertEquals(Long.valueOf(123L), ((KafkaProgress) routineLoadJob.getProgress()).getOffsetByPartition(1)); - Assert.assertEquals(KafkaProgress.OFFSET_BEGINNING, - routineLoadJob.getCustomProperties().get("property.kafka_default_offsets")); - } - - @Test - public void testSymbolicDefaultOffsetOverridesPreviousDatetimeOrigin() throws Exception { - KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, - 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - Map originalCustomProperties = Maps.newHashMap(); - originalCustomProperties.put(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName(), "1767225600000"); - originalCustomProperties.put(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName(), - "2026-01-01 00:00:00"); - Deencapsulation.setField(routineLoadJob, "customProperties", originalCustomProperties); - Deencapsulation.setField(routineLoadJob, "kafkaDefaultOffSet", "2026-01-01 00:00:00"); - - KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties( - Maps.newHashMap(ImmutableMap.of("property." - + KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName(), KafkaProgress.OFFSET_BEGINNING))); - Env env = Mockito.mock(Env.class); - Mockito.when(env.getEditLog()).thenReturn(Mockito.mock(EditLog.class)); - - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { - envStatic.when(Env::getCurrentEnv).thenReturn(env); - routineLoadJob.modifyProperties(mockAlterCommand(dataSourceProperties)); - } - - Map customProperties = Deencapsulation.getField(routineLoadJob, "customProperties"); - Assert.assertEquals(KafkaProgress.OFFSET_BEGINNING, - Deencapsulation.getField(routineLoadJob, "kafkaDefaultOffSet")); - Assert.assertEquals(KafkaProgress.OFFSET_BEGINNING, - customProperties.get(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName())); - Assert.assertFalse(customProperties.containsKey(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())); - } - - @Test - public void testAlterRejectsEmptyKafkaDefaultOffset() { - Map alteredSourceProperties = Maps.newHashMap(); - alteredSourceProperties.put("property.kafka_default_offsets", ""); - - Assert.assertThrows(UserException.class, - () -> analyzedAlterDataSourceProperties(alteredSourceProperties)); - } - - @Test - public void testModifyKafkaTopicResetsCloudProgressWithoutOffsets() throws Exception { - KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, - 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - Deencapsulation.setField(routineLoadJob, "progress", - new KafkaProgress(Maps.newHashMap(ImmutableMap.of(1, 123L)))); - - KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties( - Maps.newHashMap(ImmutableMap.of(KafkaConfiguration.KAFKA_TOPIC.getName(), "topic2"))); - Env env = Mockito.mock(Env.class); - Mockito.when(env.getEditLog()).thenReturn(Mockito.mock(EditLog.class)); - MetaServiceProxy metaServiceProxy = Mockito.mock(MetaServiceProxy.class); - Cloud.ResetRLProgressResponse response = Cloud.ResetRLProgressResponse.newBuilder() - .setStatus(Cloud.MetaServiceResponseStatus.newBuilder().setCode(Cloud.MetaServiceCode.OK)) - .build(); - Mockito.when(metaServiceProxy.resetRLProgress(Mockito.any())).thenReturn(response); - - String originalCloudUniqueId = Config.cloud_unique_id; - Config.cloud_unique_id = "test-cloud"; - try (MockedStatic envStatic = Mockito.mockStatic(Env.class); - MockedStatic metaServiceProxyStatic = Mockito.mockStatic(MetaServiceProxy.class)) { - envStatic.when(Env::getCurrentEnv).thenReturn(env); - metaServiceProxyStatic.when(MetaServiceProxy::getInstance).thenReturn(metaServiceProxy); - - routineLoadJob.modifyProperties(mockAlterCommand(dataSourceProperties)); - Mockito.verifyNoInteractions(metaServiceProxy); - routineLoadJob.updateState(RoutineLoadJob.JobState.NEED_SCHEDULE, null, false); - } finally { - Config.cloud_unique_id = originalCloudUniqueId; - } - - Mockito.verify(metaServiceProxy).resetRLProgress(Mockito.argThat( - request -> request.getPartitionToOffsetMap().isEmpty())); - Assert.assertEquals("topic2", routineLoadJob.getTopic()); - Assert.assertTrue(((KafkaProgress) routineLoadJob.getProgress()).getPartitionOffsetPairs(false).isEmpty()); - } - - @Test - public void testCloudProgressResetFailureKeepsJobPausedAndPending() throws Exception { - KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, - 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties( - Maps.newHashMap(ImmutableMap.of(KafkaConfiguration.KAFKA_TOPIC.getName(), "topic2"))); - Env env = Mockito.mock(Env.class); - Mockito.when(env.getEditLog()).thenReturn(Mockito.mock(EditLog.class)); - MetaServiceProxy metaServiceProxy = Mockito.mock(MetaServiceProxy.class); - Cloud.ResetRLProgressResponse response = Cloud.ResetRLProgressResponse.newBuilder() - .setStatus(Cloud.MetaServiceResponseStatus.newBuilder() - .setCode(Cloud.MetaServiceCode.INVALID_ARGUMENT).setMsg("reset failed")) - .build(); - Mockito.when(metaServiceProxy.resetRLProgress(Mockito.any())).thenReturn(response); - - String originalCloudUniqueId = Config.cloud_unique_id; - Config.cloud_unique_id = "test-cloud"; - try (MockedStatic envStatic = Mockito.mockStatic(Env.class); - MockedStatic metaServiceProxyStatic = Mockito.mockStatic(MetaServiceProxy.class)) { - envStatic.when(Env::getCurrentEnv).thenReturn(env); - metaServiceProxyStatic.when(MetaServiceProxy::getInstance).thenReturn(metaServiceProxy); - - routineLoadJob.modifyProperties(mockAlterCommand(dataSourceProperties)); - Assert.assertThrows(DdlException.class, - () -> routineLoadJob.updateState(RoutineLoadJob.JobState.NEED_SCHEDULE, null, false)); - } finally { - Config.cloud_unique_id = originalCloudUniqueId; - } - - Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, routineLoadJob.getState()); - Assert.assertNotNull(Deencapsulation.getField(routineLoadJob, "pendingCloudProgressReset")); - } - - @Test - public void testUpdateLagRebuildsConvertedPropertiesAfterReplay() throws UserException { - KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, - 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "customKafkaPartitions", Lists.newArrayList(1)); - - Map customProperties = Maps.newHashMap(); - customProperties.put("security.protocol", "SASL_PLAINTEXT"); - customProperties.put("sasl.mechanism", "PLAIN"); - Deencapsulation.setField(routineLoadJob, "customProperties", customProperties); - Deencapsulation.setField(routineLoadJob, "convertedCustomProperties", Maps.newHashMap()); - - Map partitionIdToOffset = Maps.newHashMap(); - partitionIdToOffset.put(1, 10L); - Deencapsulation.setField(routineLoadJob, "progress", new KafkaProgress(partitionIdToOffset)); - - Env env = Mockito.mock(Env.class); - try (MockedStatic envStatic = Mockito.mockStatic(Env.class); - MockedStatic kafkaUtilStatic = Mockito.mockStatic(KafkaUtil.class)) { - envStatic.when(Env::getCurrentEnv).thenReturn(env); - kafkaUtilStatic.when(() -> KafkaUtil.getLatestOffsets(Mockito.eq(1L), Mockito.any(UUID.class), - Mockito.eq("127.0.0.1:9020"), Mockito.eq("topic1"), - Mockito.>argThat(properties -> - "SASL_PLAINTEXT".equals(properties.get("security.protocol")) - && "PLAIN".equals(properties.get("sasl.mechanism"))), - Mockito.argThat(partitions -> partitions.size() == 1 && partitions.contains(1)))) - .thenReturn(Lists.newArrayList(Pair.of(1, 15L))); - - routineLoadJob.updateLag(); - - Assert.assertEquals(5L, routineLoadJob.totalLag().longValue()); - kafkaUtilStatic.verify(() -> KafkaUtil.getLatestOffsets(Mockito.eq(1L), Mockito.any(UUID.class), - Mockito.eq("127.0.0.1:9020"), Mockito.eq("topic1"), - Mockito.>argThat(properties -> - "SASL_PLAINTEXT".equals(properties.get("security.protocol")) - && "PLAIN".equals(properties.get("sasl.mechanism"))), - Mockito.argThat(partitions -> partitions.size() == 1 && partitions.contains(1)))); - } + Assert.assertEquals(Long.valueOf(123L), + ((KafkaProgress) routineLoadJob.getProgress()).getOffsetByPartition(1)); } @Test @@ -937,64 +595,6 @@ public void testFromCreateStmt() throws UserException { } } - @Test - public void testReplayModifyPropertiesWithDataSourceSwitchesTargetTableInCloudMode() throws Exception { - KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, - 101L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); - Map partitionToOffset = Maps.newHashMap(); - partitionToOffset.put(1, 123L); - KafkaProgress progress = new KafkaProgress(partitionToOffset); - Deencapsulation.setField(routineLoadJob, "progress", progress); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - - Map replayedProperties = Maps.newHashMap(); - replayedProperties.put("property.client.id", "replayed-client"); - replayedProperties.put(KafkaConfiguration.KAFKA_PARTITIONS.getName(), "2"); - replayedProperties.put(KafkaConfiguration.KAFKA_OFFSETS.getName(), "200"); - KafkaDataSourceProperties dataSourceProperties = analyzedAlterDataSourceProperties(replayedProperties); - RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc(new Separator("|"), null, null, - null, null, null, null, LoadTask.MergeType.APPEND, null); - AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(1L, - Maps.newHashMap(), dataSourceProperties, 202L, routineLoadDesc); - MetaServiceProxy metaServiceProxy = Mockito.mock(MetaServiceProxy.class); - - String originalCloudUniqueId = Config.cloud_unique_id; - Config.cloud_unique_id = "replay-cloud"; - try (MockedStatic metaServiceProxyStatic = Mockito.mockStatic(MetaServiceProxy.class)) { - metaServiceProxyStatic.when(MetaServiceProxy::getInstance).thenReturn(metaServiceProxy); - routineLoadJob.replayModifyProperties(log); - Assert.assertNotNull(Deencapsulation.getField(routineLoadJob, "pendingCloudProgressReset")); - routineLoadJob.updateState(RoutineLoadJob.JobState.NEED_SCHEDULE, null, true); - } finally { - Config.cloud_unique_id = originalCloudUniqueId; - } - - Assert.assertEquals(202L, routineLoadJob.getTableId()); - Assert.assertSame(progress, routineLoadJob.getProgress()); - Assert.assertEquals(Long.valueOf(123L), ((KafkaProgress) routineLoadJob.getProgress()).getOffsetByPartition(1)); - Assert.assertEquals(Long.valueOf(200L), ((KafkaProgress) routineLoadJob.getProgress()).getOffsetByPartition(2)); - Assert.assertEquals("replayed-client", routineLoadJob.getCustomProperties().get("property.client.id")); - Assert.assertEquals("|", routineLoadJob.getColumnSeparator().getOriSeparator()); - Assert.assertNull(Deencapsulation.getField(routineLoadJob, "pendingCloudProgressReset")); - Mockito.verifyNoInteractions(metaServiceProxy); - } - - private KafkaDataSourceProperties analyzedAlterDataSourceProperties(Map properties) - throws UserException { - KafkaDataSourceProperties dataSourceProperties = new KafkaDataSourceProperties(properties); - dataSourceProperties.setAlter(true); - dataSourceProperties.setTimezone("UTC"); - dataSourceProperties.analyze(); - return dataSourceProperties; - } - - private AlterRoutineLoadCommand mockAlterCommand(KafkaDataSourceProperties dataSourceProperties) { - AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); - Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); - Mockito.when(command.getDataSourceProperties()).thenReturn(dataSourceProperties); - return command; - } - private CreateRoutineLoadInfo initCreateRoutineLoadInfo() { Map properties = Maps.newHashMap(); properties.put(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY, "2"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java index 65aebd0084e729..6761345627002d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java @@ -18,6 +18,7 @@ package org.apache.doris.load.routineload; import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.load.routineload.kinesis.KinesisConfiguration; @@ -25,12 +26,19 @@ import org.apache.doris.load.routineload.kinesis.KinesisProgress; import org.apache.doris.load.routineload.kinesis.KinesisRoutineLoadJob; import org.apache.doris.load.routineload.kinesis.KinesisTaskInfo; +import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; +import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; +import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; +import org.apache.doris.persist.EditLog; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.Gson; import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import java.util.HashMap; import java.util.HashSet; @@ -41,6 +49,64 @@ public class KinesisRoutineLoadJobTest { + @Test + public void testModifyTargetTableWithPropertiesAndReplay() throws Exception { + KinesisRoutineLoadJob routineLoadJob = + new KinesisRoutineLoadJob(1L, "kinesis_routine_load_job", 1L, + 101L, "ap-southeast-1", "stream-1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + KinesisProgress progress = new KinesisProgress(Maps.newHashMap()); + Deencapsulation.setField(routineLoadJob, "progress", progress); + + Map sourceProperties = Maps.newHashMap(); + sourceProperties.put("property.client.timeout", "1000"); + KinesisDataSourceProperties dataSourceProperties = new KinesisDataSourceProperties(sourceProperties); + dataSourceProperties.setAlter(true); + dataSourceProperties.setTimezone("UTC"); + dataSourceProperties.analyze(); + + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CreateRoutineLoadInfo.MAX_ERROR_NUMBER_PROPERTY, "10"); + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.hasTargetTable()).thenReturn(true); + Mockito.when(command.getTargetTableId()).thenReturn(202L); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); + Mockito.when(command.getDataSourceProperties()).thenReturn(dataSourceProperties); + + Env env = Mockito.mock(Env.class); + EditLog editLog = Mockito.mock(EditLog.class); + Mockito.when(env.getEditLog()).thenReturn(editLog); + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + routineLoadJob.modifyProperties(command); + } + + Assert.assertEquals(202L, routineLoadJob.getTableId()); + Assert.assertSame(progress, routineLoadJob.getProgress()); + Assert.assertEquals(10L, ((Long) Deencapsulation.getField(routineLoadJob, "maxErrorNum")).longValue()); + Assert.assertEquals("1000", routineLoadJob.getCustomProperties().get("property.client.timeout")); + ArgumentCaptor logCaptor = + ArgumentCaptor.forClass(AlterRoutineLoadJobOperationLog.class); + Mockito.verify(editLog).logAlterRoutineLoadJob(logCaptor.capture()); + Assert.assertEquals(202L, logCaptor.getValue().getTargetTableId()); + + KinesisRoutineLoadJob replayJob = + new KinesisRoutineLoadJob(1L, "kinesis_routine_load_job", 1L, + 303L, "ap-southeast-1", "stream-1", UserIdentity.ADMIN); + Map shardPositions = Maps.newHashMap(); + shardPositions.put("shard-0", "123"); + KinesisProgress replayProgress = new KinesisProgress(shardPositions); + Deencapsulation.setField(replayJob, "progress", replayProgress); + replayJob.replayModifyProperties(new AlterRoutineLoadJobOperationLog( + 1L, Maps.newHashMap(), null)); + Assert.assertEquals(303L, replayJob.getTableId()); + replayJob.replayModifyProperties(logCaptor.getValue()); + Assert.assertEquals(202L, replayJob.getTableId()); + Assert.assertSame(replayProgress, replayJob.getProgress()); + Assert.assertEquals("123", ((KinesisProgress) replayJob.getProgress()) + .getSequenceNumberByShard("shard-0")); + } + @Test public void testRoutineLoadTaskConcurrentNum() { int oldMaxConcurrent = Config.max_routine_load_task_concurrent_num; diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java index 6d89b17a5d52eb..263515975437fa 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java @@ -20,7 +20,6 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.cloud.transaction.TxnUtil; import org.apache.doris.common.Config; @@ -30,8 +29,6 @@ import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.kafka.KafkaUtil; -import org.apache.doris.datasource.property.fileformat.FileFormatProperties; -import org.apache.doris.datasource.property.fileformat.JsonFileFormatProperties; import org.apache.doris.load.routineload.kafka.KafkaProgress; import org.apache.doris.load.routineload.kafka.KafkaRoutineLoadJob; import org.apache.doris.load.routineload.kafka.KafkaTaskInfo; @@ -59,8 +56,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.concurrent.locks.ReentrantReadWriteLock; public class RoutineLoadJobTest { @Test @@ -380,33 +375,6 @@ public void testGetShowCreateInfo() throws UserException { Assert.assertEquals(expect, showCreateInfo); } - @Test - public void testShowTableLookupUsesJobReadLock() { - KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(111L, "test_load", 1L, - 11L, "localhost:9092", "test_topic", UserIdentity.ADMIN); - ReentrantReadWriteLock lock = Deencapsulation.getField(routineLoadJob, "lock"); - - InternalCatalog catalog = Mockito.mock(InternalCatalog.class); - Database database = Mockito.mock(Database.class); - OlapTable table = Mockito.mock(OlapTable.class); - Mockito.when(catalog.getDb(1L)).thenReturn(Optional.of(database)); - Mockito.when(database.getFullName()).thenReturn("test_db"); - Mockito.when(database.getTable(11L)).thenAnswer(invocation -> { - Assert.assertTrue(lock.getReadHoldCount() > 0); - return Optional.of((Table) table); - }); - Mockito.when(table.getName()).thenReturn("test_table"); - - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { - envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); - - Assert.assertEquals("test_table", routineLoadJob.getShowInfo().get(6)); - Assert.assertTrue(routineLoadJob.getShowCreateInfo().contains(" ON test_table\n")); - } - - Mockito.verify(database, Mockito.times(2)).getTable(11L); - } - @Test public void testParseUniqueKeyUpdateMode() { // Test valid mode strings @@ -530,44 +498,4 @@ public void testUniqueKeyUpdateModeTakesPrecedenceOverPartialColumns() throws Ex Assert.assertFalse(isPartialUpdate); } - @Test - public void testValidateFixedPartialUpdateRequiresMowTable() { - KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(); - OlapTable targetTable = Mockito.mock(OlapTable.class); - Mockito.when(targetTable.getEnableUniqueKeyMergeOnWrite()).thenReturn(false); - - UserException exception = Assert.assertThrows(UserException.class, - () -> job.validateAlterJobProperties(targetTable, Maps.newHashMap(), - TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS)); - Assert.assertTrue(exception.getMessage().contains("PARTIAL_COLUMNS")); - } - - @Test - public void testValidateFlexiblePartialUpdateUsesAlteredProperties() throws Exception { - KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(); - OlapTable targetTable = Mockito.mock(OlapTable.class); - Map currentJobProperties = Maps.newHashMap(); - currentJobProperties.put(FileFormatProperties.PROP_FORMAT, "json"); - currentJobProperties.put(JsonFileFormatProperties.PROP_FUZZY_PARSE, "false"); - currentJobProperties.put(JsonFileFormatProperties.PROP_JSON_PATHS, "$.value"); - Deencapsulation.setField(job, "jobProperties", currentJobProperties); - - Map validAlterProperties = Maps.newHashMap(); - validAlterProperties.put(JsonFileFormatProperties.PROP_JSON_PATHS, ""); - job.validateAlterJobProperties(targetTable, validAlterProperties, - TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS); - - Map invalidAlterProperties = Maps.newHashMap(); - invalidAlterProperties.put(JsonFileFormatProperties.PROP_JSON_PATHS, ""); - invalidAlterProperties.put(JsonFileFormatProperties.PROP_FUZZY_PARSE, "true"); - try { - job.validateAlterJobProperties(targetTable, invalidAlterProperties, - TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS); - Assert.fail("Expected flexible partial update validation to reject fuzzy_parse"); - } catch (UserException e) { - Assert.assertTrue(e.getMessage().contains("fuzzy_parse")); - } - Mockito.verify(targetTable, Mockito.times(2)).validateForFlexiblePartialUpdate(); - } - } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java index 17923dc02d96e7..04ad3e73a59828 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java @@ -29,6 +29,7 @@ import org.apache.doris.load.routineload.RoutineLoadJob; import org.apache.doris.load.routineload.RoutineLoadManager; import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; +import org.apache.doris.load.routineload.kinesis.KinesisDataSourceProperties; import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.exceptions.ParseException; @@ -38,14 +39,12 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.QueryState; import org.apache.doris.qe.SessionVariable; -import org.apache.doris.thrift.TUniqueKeyUpdateMode; import com.google.common.collect.Maps; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.InOrder; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -95,15 +94,11 @@ public void setUp() throws Exception { .thenReturn(routineLoadJob); Mockito.when(routineLoadJob.getDbFullName()).thenReturn("testDb"); Mockito.when(routineLoadJob.getTableName()).thenReturn("testTable"); - Mockito.when(routineLoadJob.getDbId()).thenReturn(1000L); - Mockito.when(routineLoadJob.getTableId()).thenReturn(2000L); Mockito.when(routineLoadJob.isMultiTable()).thenReturn(false); Mockito.when(routineLoadJob.getMergeType()).thenReturn(LoadTask.MergeType.APPEND); Mockito.when(routineLoadJob.getDataSourceType()).thenReturn(LoadDataSourceType.KAFKA); Mockito.when(routineLoadJob.getTimezone()).thenReturn("UTC"); Mockito.when(routineLoadJob.isLoadToSingleTablet()).thenReturn(false); - Mockito.when(routineLoadJob.getUniqueKeyUpdateMode()).thenReturn(TUniqueKeyUpdateMode.UPSERT); - Mockito.when(currentTable.getType()).thenReturn(Table.TableType.OLAP); Mockito.when(currentTable.isTemporary()).thenReturn(false); } @@ -121,12 +116,8 @@ private void runBefore() { Mockito.when(connectContext.isSkipAuth()).thenReturn(true); } - private void mockTargetTable(Table table) { - try { - Mockito.doReturn(table).when(db).getTableOrAnalysisException("testTable2"); - } catch (AnalysisException e) { - throw new RuntimeException(e); - } + private void mockTargetTable(Table table) throws AnalysisException { + Mockito.doReturn(table).when(db).getTableOrAnalysisException("testTable2"); } @Test @@ -159,18 +150,6 @@ public void testValidate() { Assertions.assertTrue(command.getAnalyzedJobProperties().containsKey(CreateRoutineLoadInfo.TIMEZONE)); } - @Test - public void testLegacyConstructorInfersDataSourceType() { - runBefore(); - Map dataSourceProperties = Maps.newHashMap(); - dataSourceProperties.put("property.client.id", "alter-client"); - AlterRoutineLoadCommand command = new AlterRoutineLoadCommand( - new LabelNameInfo("testDb", "label1"), Maps.newHashMap(), dataSourceProperties); - - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - Assertions.assertInstanceOf(KafkaDataSourceProperties.class, command.getDataSourceProperties()); - } - @Test public void testParseAlterRoutineLoadSetTargetTable() { AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( @@ -179,14 +158,15 @@ public void testParseAlterRoutineLoadSetTargetTable() { Assertions.assertEquals("label1", command.getJobName()); Assertions.assertTrue(command.hasTargetTable()); Assertions.assertEquals("testTable2", command.getTargetTableName()); - } - @Test - public void testParseAlterRoutineLoadDecodesTargetTableString() { - AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"test\"\"Table2\""); + AlterRoutineLoadCommand escapedNameCommand = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR label1 SET TARGET TABLE = \"test\"\"Table2\""); + Assertions.assertEquals("test\"Table2", escapedNameCommand.getTargetTableName()); - Assertions.assertEquals("test\"Table2", command.getTargetTableName()); + AlterRoutineLoadCommand emptyNameCommand = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR label1 SET TARGET TABLE = \"\""); + Assertions.assertTrue(emptyNameCommand.hasTargetTable()); + Assertions.assertEquals("", emptyNameCommand.getTargetTableName()); } @Test @@ -204,11 +184,10 @@ public void testValidateTargetTableWithJobAndDataSourceProperties() throws Excep runBefore(); Mockito.when(currentTable.getId()).thenReturn(3000L); mockTargetTable(currentTable); - AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" " - + "PROPERTIES(\"max_error_number\"=\"1\", \"timezone\"=\"Asia/Shanghai\") " - + "FROM `KAFKA`(\"property.client.id\"=\"target-switch\")"); + + "PROPERTIES(\"max_error_number\"=\"1\") " + + "FROM KAFKA(\"property.client.id\"=\"target-switch\")"); Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); Assertions.assertEquals("1", command.getAnalyzedJobProperties() @@ -216,233 +195,110 @@ public void testValidateTargetTableWithJobAndDataSourceProperties() throws Excep Assertions.assertInstanceOf(KafkaDataSourceProperties.class, command.getDataSourceProperties()); Assertions.assertEquals("target-switch", command.getDataSourceProperties() .getOriginalDataSourceProperties().get("property.client.id")); - Assertions.assertEquals("Asia/Shanghai", command.getDataSourceProperties().getTimezone()); Assertions.assertEquals(3000L, command.getTargetTableId()); + Mockito.verify(routineLoadJob).validateTargetTable(db, currentTable); } @Test - public void testValidateTargetTableRejectsKinesisJob() throws Exception { + public void testValidateTargetTableOnlyDoesNotRequireOtherProperties() throws Exception { runBefore(); - Mockito.when(routineLoadJob.getDataSourceType()).thenReturn(LoadDataSourceType.KINESIS); + Mockito.when(currentTable.getId()).thenReturn(3000L); + mockTargetTable(currentTable); AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); - Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, - () -> command.validate(connectContext)).getMessage().contains("only supports Kafka jobs")); + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + Assertions.assertTrue(command.getAnalyzedJobProperties().isEmpty()); + Assertions.assertNull(command.getDataSourceProperties()); + Assertions.assertEquals(3000L, command.getTargetTableId()); } @Test - public void testValidateTargetTableRejectsDataSourceTypeChange() throws Exception { + public void testValidateTargetTableWithKinesisDataSourceProperties() throws Exception { runBefore(); + Mockito.when(routineLoadJob.getDataSourceType()).thenReturn(LoadDataSourceType.KINESIS); + Mockito.when(currentTable.getId()).thenReturn(3000L); mockTargetTable(currentTable); AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" " - + "FROM KINESIS(\"kinesis_region\"=\"us-east-1\")"); - - Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, - () -> command.validate(connectContext)).getMessage().contains("KINESIS")); - } + + "PROPERTIES(\"max_error_number\"=\"1\") " + + "FROM KINESIS(\"property.client.timeout\"=\"1000\")"); - @Test - public void testValidateTargetTableRejectsInvalidProperties() throws Exception { - runBefore(); - mockTargetTable(currentTable); - AlterRoutineLoadCommand invalidJobProperty = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" " - + "PROPERTIES(\"format\"=\"json\")"); - AlterRoutineLoadCommand invalidDataSourceProperty = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" " - + "FROM KAFKA(\"invalid_key\"=\"value\")"); - AlterRoutineLoadCommand createOnlyDataSourceProperty = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" " - + "FROM KAFKA(\"kafka_table_name_location\"=\"key\")"); - - Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, - () -> invalidJobProperty.validate(connectContext)).getMessage().contains("invalid property")); - Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, - () -> invalidDataSourceProperty.validate(connectContext)).getMessage().contains("invalid kafka")); - Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, - () -> createOnlyDataSourceProperty.validate(connectContext)).getMessage().contains("only be set")); - } - - @Test - public void testValidateTargetTableOnlyDoesNotRequireOtherProperties() throws Exception { - runBefore(); - mockTargetTable(currentTable); - - AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - Assertions.assertEquals("testTable2", command.getTargetTableName()); + Assertions.assertInstanceOf(KinesisDataSourceProperties.class, command.getDataSourceProperties()); + Assertions.assertEquals("1000", command.getDataSourceProperties() + .getOriginalDataSourceProperties().get("property.client.timeout")); + Assertions.assertEquals(3000L, command.getTargetTableId()); + Mockito.verify(routineLoadJob).validateTargetTable(db, currentTable); } @Test public void testValidateTargetTableRejectsMultiTableJob() throws Exception { runBefore(); - Mockito.when(routineLoadJob.isMultiTable()).thenReturn(true); AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); - Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) - .getMessage().contains("single-table")); + Mockito.when(routineLoadJob.isMultiTable()).thenReturn(true); + Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, + () -> command.validate(connectContext)).getMessage().contains("single-table")); } @Test public void testValidateUnauthorizedTargetDoesNotResolveMetadata() throws Exception { runBefore(); Mockito.when(accessManager.checkTblPriv(Mockito.any(ConnectContext.class), Mockito.anyString(), - Mockito.eq("testDb"), Mockito.eq("testTable2"), Mockito.eq(PrivPredicate.LOAD))).thenReturn(false); + Mockito.eq("testDb"), Mockito.eq("testTable2"), Mockito.eq(PrivPredicate.LOAD))) + .thenReturn(false); AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); - Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) - .getMessage().contains("LOAD")); + Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, + () -> command.validate(connectContext)).getMessage().contains("LOAD")); Mockito.verify(routineLoadManager).checkPrivAndGetJob("testDb", "label1"); - Mockito.verify(accessManager).checkTblPriv(connectContext, InternalCatalog.INTERNAL_CATALOG_NAME, - "testDb", "testTable2", PrivPredicate.LOAD); Mockito.verify(catalog, Mockito.never()).getDbOrAnalysisException(Mockito.anyString()); - Mockito.verify(db, Mockito.never()).getTableOrAnalysisException(Mockito.anyString()); - Mockito.verify(routineLoadJob, Mockito.never()).validateTargetTable( - Mockito.any(Database.class), Mockito.any(OlapTable.class), Mockito.anyMap(), - Mockito.any(TUniqueKeyUpdateMode.class)); } @Test - public void testValidateTargetTableRejectsUnauthorizedJobBeforeTargetLookup() throws Exception { + public void testValidateUnauthorizedJobDoesNotResolveTarget() throws Exception { runBefore(); Mockito.when(routineLoadManager.checkPrivAndGetJob("testDb", "label1")) .thenThrow(new AnalysisException("access denied")); AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); - Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) - .getMessage().contains("access denied")); + Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, + () -> command.validate(connectContext)).getMessage().contains("access denied")); Mockito.verify(accessManager, Mockito.never()).checkTblPriv( Mockito.any(ConnectContext.class), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.any()); Mockito.verify(catalog, Mockito.never()).getDbOrAnalysisException(Mockito.anyString()); - Mockito.verify(db, Mockito.never()).getTableOrAnalysisException(Mockito.anyString()); } @Test - public void testValidateTargetTableAuthorizesBeforeMetadataLookup() throws Exception { + public void testValidateTargetTableConstraints() throws Exception { runBefore(); - mockTargetTable(currentTable); AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - - InOrder inOrder = Mockito.inOrder(routineLoadManager, accessManager, catalog, db); - inOrder.verify(routineLoadManager).checkPrivAndGetJob("testDb", "label1"); - inOrder.verify(accessManager).checkTblPriv(connectContext, InternalCatalog.INTERNAL_CATALOG_NAME, - "testDb", "testTable2", PrivPredicate.LOAD); - inOrder.verify(catalog).getDbOrAnalysisException("testDb"); - inOrder.verify(db).getTableOrAnalysisException("testTable2"); - } - - @Test - public void testValidateTargetTableRejectsTemporaryTable() throws Exception { - runBefore(); - OlapTable tempTable = Mockito.mock(OlapTable.class); - Mockito.when(tempTable.getType()).thenReturn(Table.TableType.OLAP); - Mockito.when(tempTable.isTemporary()).thenReturn(true); - mockTargetTable(tempTable); - AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); - - Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) - .getMessage().contains("temporary table")); - } - - @Test - public void testValidateTargetTableRejectsLoadToSingleTabletWithoutRandomDistribution() throws Exception { - runBefore(); - OlapTable newTable = Mockito.mock(OlapTable.class); - Mockito.when(newTable.getType()).thenReturn(Table.TableType.OLAP); - Mockito.when(newTable.isTemporary()).thenReturn(false); - Mockito.when(newTable.getDefaultDistributionInfo()).thenReturn(null); - mockTargetTable(newTable); - Mockito.when(routineLoadJob.isLoadToSingleTablet()).thenReturn(true); - AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); + mockTargetTable(Mockito.mock(Table.class)); + Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, + () -> command.validate(connectContext)).getMessage().contains("only supports OLAP table")); - Assertions.assertTrue(Assertions.assertThrows(Exception.class, () -> command.validate(connectContext)) - .getMessage().contains("load_to_single_tablet")); - } + OlapTable temporaryTable = Mockito.mock(OlapTable.class); + Mockito.when(temporaryTable.isTemporary()).thenReturn(true); + mockTargetTable(temporaryTable); + Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, + () -> command.validate(connectContext)).getMessage().contains("temporary table")); - @Test - public void testValidateTargetTableAllowsLoadToSingleTabletWithRandomDistribution() throws Exception { - runBefore(); - OlapTable newTable = Mockito.mock(OlapTable.class); - RandomDistributionInfo distributionInfo = Mockito.mock(RandomDistributionInfo.class); - Mockito.when(newTable.getType()).thenReturn(Table.TableType.OLAP); - Mockito.when(newTable.isTemporary()).thenReturn(false); - Mockito.when(newTable.getDefaultDistributionInfo()).thenReturn(distributionInfo); - mockTargetTable(newTable); + OlapTable hashDistributedTable = Mockito.mock(OlapTable.class); + Mockito.when(hashDistributedTable.isTemporary()).thenReturn(false); + mockTargetTable(hashDistributedTable); Mockito.when(routineLoadJob.isLoadToSingleTablet()).thenReturn(true); - AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); - - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - } - - @Test - public void testValidateTargetTablePassesTargetTableToJobValidation() throws Exception { - runBefore(); - OlapTable targetTable = Mockito.mock(OlapTable.class); - Mockito.when(targetTable.getType()).thenReturn(Table.TableType.OLAP); - Mockito.when(targetTable.isTemporary()).thenReturn(false); - Mockito.doReturn(targetTable).when(db).getTableOrAnalysisException("testTable2"); - Mockito.when(routineLoadJob.isLoadToSingleTablet()).thenReturn(false); - Mockito.when(routineLoadJob.getUniqueKeyUpdateMode()) - .thenReturn(TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS); - Mockito.doAnswer(invocation -> { - Assertions.assertSame(db, invocation.getArgument(0)); - Assertions.assertSame(targetTable, invocation.getArgument(1)); - Map alteredJobProperties = invocation.getArgument(2); - Assertions.assertEquals("UPSERT", - alteredJobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)); - Assertions.assertEquals(TUniqueKeyUpdateMode.UPSERT, invocation.getArgument(3)); - return null; - }).when(routineLoadJob).validateTargetTable(Mockito.any(Database.class), Mockito.any(OlapTable.class), - Mockito.anyMap(), Mockito.any(TUniqueKeyUpdateMode.class)); - - AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( - "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" " - + "PROPERTIES(\"unique_key_update_mode\"=\"UPSERT\")"); - - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - } - - @Test - public void testValidateAllowsExplicitUpsertToOverridePartialColumnsOnNonMowTable() { - runBefore(); - Mockito.when(routineLoadJob.getUniqueKeyUpdateMode()).thenReturn(TUniqueKeyUpdateMode.UPSERT); - Mockito.when(currentTable.getEnableUniqueKeyMergeOnWrite()).thenReturn(false); - Map jobProperties = Maps.newHashMap(); - jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, "UPSERT"); - jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, "true"); - - AlterRoutineLoadCommand command = new AlterRoutineLoadCommand( - new LabelNameInfo("testDb", "label1"), jobProperties, Maps.newHashMap()); - - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - } - - @Test - public void testValidateAllowsFlexibleAlterToReachFlexibleValidation() throws Exception { - runBefore(); - Mockito.when(routineLoadJob.getUniqueKeyUpdateMode()).thenReturn(TUniqueKeyUpdateMode.UPSERT); - Mockito.when(currentTable.getEnableUniqueKeyMergeOnWrite()).thenReturn(false); - Map jobProperties = Maps.newHashMap(); - jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, "UPDATE_FLEXIBLE_COLUMNS"); - - AlterRoutineLoadCommand command = new AlterRoutineLoadCommand( - new LabelNameInfo("testDb", "label1"), jobProperties, Maps.newHashMap()); + Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, + () -> command.validate(connectContext)).getMessage().contains("load_to_single_tablet")); + RandomDistributionInfo randomDistributionInfo = Mockito.mock(RandomDistributionInfo.class); + Mockito.when(hashDistributedTable.getDefaultDistributionInfo()).thenReturn(randomDistributionInfo); Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - Mockito.verify(routineLoadJob).validateAlterJobProperties(Mockito.eq(currentTable), Mockito.anyMap(), - Mockito.eq(TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java index 54973e0ef70abc..3e55210faba84f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java @@ -17,16 +17,12 @@ package org.apache.doris.persist; -import org.apache.doris.analysis.Separator; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.util.TimeUtils; -import org.apache.doris.load.RoutineLoadDesc; -import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.kafka.KafkaConfiguration; import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; -import org.apache.doris.persist.gson.GsonUtils; import com.google.common.collect.Maps; import org.junit.Assert; @@ -54,7 +50,6 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); long jobId = 1000; - long targetTableId = 2001; Map jobProperties = Maps.newHashMap(); jobProperties.put(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY, "5"); @@ -68,12 +63,8 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User routineLoadDataSourceProperties.setTimezone(TimeUtils.DEFAULT_TIME_ZONE); routineLoadDataSourceProperties.analyze(); - Assert.assertEquals(0L, new AlterRoutineLoadJobOperationLog( - jobId, jobProperties, routineLoadDataSourceProperties).getTargetTableId()); - RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc(new Separator("|"), null, null, - null, null, null, null, LoadTask.MergeType.APPEND, null); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(jobId, - jobProperties, routineLoadDataSourceProperties, targetTableId, routineLoadDesc); + jobProperties, routineLoadDataSourceProperties, 2000L); log.write(out); out.flush(); out.close(); @@ -93,40 +84,25 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User kafkaDataSourceProperties.getKafkaPartitionOffsets().get(0)); Assert.assertEquals(routineLoadDataSourceProperties.getKafkaPartitionOffsets().get(1), kafkaDataSourceProperties.getKafkaPartitionOffsets().get(1)); - Assert.assertEquals(targetTableId, log2.getTargetTableId()); - Assert.assertEquals("|", log2.getRoutineLoadDesc().getColumnSeparator().getOriSeparator()); - Assert.assertEquals(LoadTask.MergeType.APPEND, log2.getRoutineLoadDesc().getMergeType()); + Assert.assertEquals(2000L, log2.getTargetTableId()); in.close(); } @Test - public void testDeserializeAlterRoutineLoadOperationLogWithoutTargetTableId() throws Exception { - long jobId = 1000; - Map jobProperties = Maps.newHashMap(); - jobProperties.put(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY, "5"); - Map dataSourceProperties = Maps.newHashMap(); - dataSourceProperties.put("property.group.id", "mygroup"); - KafkaDataSourceProperties routineLoadDataSourceProperties = new KafkaDataSourceProperties( - dataSourceProperties); - routineLoadDataSourceProperties.setAlter(true); - routineLoadDataSourceProperties.setTimezone(TimeUtils.DEFAULT_TIME_ZONE); - routineLoadDataSourceProperties.analyze(); - AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(jobId, - jobProperties, routineLoadDataSourceProperties, 0L); - String legacyJson = GsonUtils.GSON.toJson(log).replace(",\"targetTableId\":0", ""); - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - try (DataOutputStream out = new DataOutputStream(byteArrayOutputStream)) { - Text.writeString(out, legacyJson); + public void testDeserializeLogWithoutTargetTableId() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(bytes)) { + Text.writeString(out, "{\"jobId\":1000,\"jobProperties\":{}," + + "\"dataSourceProperties\":null}"); } - AlterRoutineLoadJobOperationLog readLog; - try (DataInputStream in = new DataInputStream( - new ByteArrayInputStream(byteArrayOutputStream.toByteArray()))) { - readLog = AlterRoutineLoadJobOperationLog.read(in); + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + AlterRoutineLoadJobOperationLog log = AlterRoutineLoadJobOperationLog.read(in); + Assert.assertEquals(1000L, log.getJobId()); + Assert.assertEquals(0L, log.getTargetTableId()); } - - Assert.assertEquals(0L, readLog.getTargetTableId()); - Assert.assertEquals("5", readLog.getJobProperties().get(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)); } + + } diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy index 18c23258ce7a4a..b4145ddb8ee9bd 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy @@ -23,9 +23,8 @@ import org.apache.kafka.clients.producer.ProducerRecord import org.apache.kafka.clients.producer.ProducerConfig suite("test_routine_load_alter","p0") { - def kafkaCsvDataFile = "test_routine_load_alter" def kafkaCsvTpoics = [ - "test_routine_load_alter_${System.currentTimeMillis()}".toString(), + "test_routine_load_alter", ] String enabled = context.config.otherConfigs.get("enableKafkaTest") String kafka_port = context.config.otherConfigs.get("kafka_port") @@ -41,17 +40,6 @@ suite("test_routine_load_alter","p0") { props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") - def adminProps = new Properties() - adminProps.put("bootstrap.servers", "${kafka_broker}".toString()) - def mainTopicAdmin = AdminClient.create(adminProps) - try { - kafkaCsvTpoics.each { topic -> - mainTopicAdmin.createTopics([new NewTopic(topic.toString(), 1, (short) 1)]).all().get() - } - } finally { - mainTopicAdmin.close() - } - // check conenction def verifyKafkaConnection = { prod -> try { @@ -77,7 +65,7 @@ suite("test_routine_load_alter","p0") { logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { - def txt = new File("""${context.file.parent}/data/${kafkaCsvDataFile}.csv""").text + def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text def lines = txt.readLines() lines.each { line -> logger.info("=====${line}========") @@ -126,7 +114,7 @@ suite("test_routine_load_alter","p0") { """ sql "sync" - def visibleCount = 0 + def count = 0 while (true) { def res = sql "select count(*) from ${tableName}" def state = sql "show routine load for ${jobName}" @@ -136,13 +124,13 @@ suite("test_routine_load_alter","p0") { if (res[0][0] > 0) { break } - if (visibleCount >= 120) { + if (count >= 120) { log.error("routine load can not visible for long time") assertEquals(20, res[0][0]) break } sleep(5000) - visibleCount++ + count++ } qt_sql_before "select * from ${tableName} order by k1" @@ -159,7 +147,7 @@ suite("test_routine_load_alter","p0") { def producer = new KafkaProducer<>(props) for (String kafkaCsvTopic in kafkaCsvTpoics) { - def txt = new File("""${context.file.parent}/data/${kafkaCsvDataFile}.csv""").text + def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text def lines = txt.readLines() lines.each { line -> logger.info("=====${line}========") @@ -168,7 +156,7 @@ suite("test_routine_load_alter","p0") { } } - def offsetVisibleCount = 0 + count = 0 while (true) { def res = sql "select count(*) from ${tableName}" log.info("count: ${res[0][0]}".toString()) @@ -179,13 +167,13 @@ suite("test_routine_load_alter","p0") { if (res[0][0] >= 6) { break } - if (offsetVisibleCount >= 120) { + if (count >= 120) { log.error("routine load can not visible for long time") assertEquals(20, res[0][0]) break } sleep(1000) - offsetVisibleCount++ + count++ } qt_sql_alter_after "select * from ${tableName} order by k1" } finally { @@ -212,7 +200,7 @@ suite("test_routine_load_alter","p0") { sql "ALTER ROUTINE LOAD FOR ${jobName} COLUMNS(k1, k2, v1, v2, v3, v4);" sql "ALTER ROUTINE LOAD FOR ${jobName} COLUMNS TERMINATED BY ',';" sql "resume routine load for ${jobName}" - def columnVisibleCount = 0 + def count = 0 while (true) { def res = sql "select count(*) from ${tableName}" log.info("count: ${res[0][0]}".toString()) @@ -224,13 +212,13 @@ suite("test_routine_load_alter","p0") { if (res[0][0] > 0) { break } - if (columnVisibleCount >= 120) { + if (count >= 120) { log.error("routine load can not visible for long time") assertEquals(20, res[0][0]) break } sleep(1000) - columnVisibleCount++ + count++ } def res = sql "select * from ${tableName} order by k1" log.info("res: ${res.size()}".toString()) @@ -308,65 +296,51 @@ suite("test_routine_load_alter","p0") { sql "truncate table ${tableName}" } - def mainTopicCleanupProps = new Properties() - mainTopicCleanupProps.put("bootstrap.servers", "${kafka_broker}".toString()) - def mainTopicCleanupAdmin = AdminClient.create(mainTopicCleanupProps) - try { - mainTopicCleanupAdmin.deleteTopics(kafkaCsvTpoics.collect { it.toString() }, - new DeleteTopicsOptions().timeoutMs(10000)).all().get() - } catch (Exception e) { - logger.warn("failed to delete kafka topics ${kafkaCsvTpoics}: ${e.message}".toString()) - } finally { - mainTopicCleanupAdmin.close() - } - - // test alter target table + // Test switching the target together with existing job and Kafka properties. def srcTableName = "test_routine_load_alter_src" def dstTableName = "test_routine_load_alter_dst" def alterTargetTopic = "test_routine_load_alter_target_table_${System.currentTimeMillis()}" def alterTargetJob = "test_alter_target_table_${System.currentTimeMillis()}" def alterTopicProducer = null def alterTopicAdmin = null + def waitForRows = { String targetTable, long expectedRows -> + long actualRows = 0 + for (int attempt = 0; attempt < 120; attempt++) { + def countResult = sql "select count(*) from ${targetTable}" + actualRows = (countResult[0][0] as Number).longValue() + if (actualRows >= expectedRows) { + return actualRows + } + sleep(1000) + } + assertTrue(false, "timeout waiting for ${expectedRows} rows in ${targetTable}; actual=${actualRows}") + } try { - sql """ DROP TABLE IF EXISTS ${srcTableName} """ - sql """ DROP TABLE IF EXISTS ${dstTableName} """ - sql """ - CREATE TABLE IF NOT EXISTS ${srcTableName} ( - `k1` int(20) NULL, - `k2` string NULL, - `v1` date NULL, - `v2` string NULL, - `v3` datetime NULL, - `v4` string NULL - ) ENGINE=OLAP - DUPLICATE KEY(`k1`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`k1`) BUCKETS 3 - PROPERTIES ("replication_allocation" = "tag.location.default: 1"); - """ - sql """ - CREATE TABLE IF NOT EXISTS ${dstTableName} ( - `k1` int(20) NULL, - `k2` string NULL, - `v1` date NULL, - `v2` string NULL, - `v3` datetime NULL, - `v4` string NULL - ) ENGINE=OLAP - DUPLICATE KEY(`k1`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`k1`) BUCKETS 3 - PROPERTIES ("replication_allocation" = "tag.location.default: 1"); - """ - sql "sync" + sql "DROP TABLE IF EXISTS ${srcTableName}" + sql "DROP TABLE IF EXISTS ${dstTableName}" + for (String targetTable in [srcTableName, dstTableName]) { + sql """ + CREATE TABLE ${targetTable} ( + `k1` int NULL, + `k2` string NULL, + `v1` date NULL, + `v2` string NULL, + `v3` datetime NULL, + `v4` string NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ("replication_allocation" = "tag.location.default: 1") + """ + } def topicProps = new Properties() - topicProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) + topicProps.put("bootstrap.servers", kafka_broker.toString()) alterTopicAdmin = AdminClient.create(topicProps) alterTopicAdmin.createTopics([new NewTopic(alterTargetTopic, 1, (short) 1)]).all().get() def producerProps = new Properties() - producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) + producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka_broker.toString()) producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, @@ -375,8 +349,7 @@ suite("test_routine_load_alter","p0") { producerProps.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") alterTopicProducer = new KafkaProducer<>(producerProps) - def firstBatch = new File("""${context.file.parent}/data/${kafkaCsvDataFile}.csv""").readLines() - firstBatch.each { line -> + new File("${context.file.parent}/data/test_routine_load_alter.csv").readLines().each { line -> alterTopicProducer.send(new ProducerRecord<>(alterTargetTopic, null, line)).get() } alterTopicProducer.flush() @@ -396,31 +369,12 @@ suite("test_routine_load_alter","p0") { "kafka_topic" = "${alterTargetTopic}", "kafka_partitions" = "0", "kafka_offsets" = "OFFSET_BEGINNING" - ); + ) """ - sql "sync" - - def targetVisibleCount = 0 - while (true) { - def res = sql "select count(*) from ${srcTableName}" - def state = sql "show routine load for ${alterTargetJob}" - log.info("routine load state: ${state[0][8].toString()}".toString()) - log.info("routine load statistic: ${state[0][14].toString()}".toString()) - log.info("reason of state changed: ${state[0][17].toString()}".toString()) - if (res[0][0] >= 3) { - break - } - if (targetVisibleCount >= 120) { - log.error("routine load can not visible for long time") - assertEquals(3, res[0][0]) - break - } - sleep(1000) - targetVisibleCount++ - } + assertEquals(3L, waitForRows(srcTableName, 3L)) - sql "pause routine load for ${alterTargetJob}" - def showBeforeAlter = sql "show routine load for ${alterTargetJob}" + sql "PAUSE ROUTINE LOAD FOR ${alterTargetJob}" + def showBeforeAlter = sql "SHOW ROUTINE LOAD FOR ${alterTargetJob}" assertEquals(srcTableName, showBeforeAlter[0][6].toString()) def progressBeforeAlter = showBeforeAlter[0][15].toString() @@ -430,68 +384,31 @@ suite("test_routine_load_alter","p0") { PROPERTIES("max_error_number" = "10") FROM KAFKA("property.client.id" = "target-switch") """ - - def showAfterAlter = sql "show routine load for ${alterTargetJob}" + def showAfterAlter = sql "SHOW ROUTINE LOAD FOR ${alterTargetJob}" assertEquals(dstTableName, showAfterAlter[0][6].toString()) assertEquals(progressBeforeAlter, showAfterAlter[0][15].toString()) - def alteredJobProperties = new groovy.json.JsonSlurper().parseText(showAfterAlter[0][11].toString()) - def alteredCustomProperties = new groovy.json.JsonSlurper().parseText(showAfterAlter[0][13].toString()) + def alteredJobProperties = parseJson(showAfterAlter[0][11]) + def alteredCustomProperties = parseJson(showAfterAlter[0][13]) assertEquals("10", alteredJobProperties.max_error_number.toString()) assertEquals("target-switch", alteredCustomProperties["client.id"].toString()) - def secondBatch = [ + [ "4,eab,2023-07-16,def,2023-07-21:05:48:31,ghi", "5,eab,2023-07-17,def,2023-07-22:05:48:31,ghi", "6,eab,2023-07-18,def,2023-07-23:05:48:31,ghi" - ] - secondBatch.each { line -> + ].each { line -> alterTopicProducer.send(new ProducerRecord<>(alterTargetTopic, null, line)).get() } alterTopicProducer.flush() - - sql "resume routine load for ${alterTargetJob}" - - def targetAlterVisibleCount = 0 - def stableCount = 0 - while (true) { - def srcCount = sql "select count(*) from ${srcTableName}" - def dstCount = sql "select count(*) from ${dstTableName}" - long srcCountValue = (srcCount[0][0] as Number).longValue() - long dstCountValue = (dstCount[0][0] as Number).longValue() - log.info("src count: ${srcCountValue}".toString()) - log.info("dst count: ${dstCountValue}".toString()) - if (srcCountValue == 3 && dstCountValue == 3) { - stableCount++ - if (stableCount >= 5) { - sleep(2000) - def finalSrcCount = sql "select count(*) from ${srcTableName}" - def finalDstCount = sql "select count(*) from ${dstTableName}" - assertEquals(3L, (finalSrcCount[0][0] as Number).longValue()) - assertEquals(3L, (finalDstCount[0][0] as Number).longValue()) - break - } - } else { - stableCount = 0 - if (srcCountValue > 3 || dstCountValue > 3) { - assertEquals(3L, srcCountValue) - assertEquals(3L, dstCountValue) - } - } - if (targetAlterVisibleCount >= 120) { - log.error("routine load target table alter can not visible for long time") - assertEquals(3L, srcCountValue) - assertEquals(3L, dstCountValue) - break - } - sleep(1000) - targetAlterVisibleCount++ - } + sql "RESUME ROUTINE LOAD FOR ${alterTargetJob}" + waitForRows(dstTableName, 3L) + sleep(5000) qt_sql_alter_target_src "select k1, k2 from ${srcTableName} order by k1" qt_sql_alter_target_dst "select k1, k2 from ${dstTableName} order by k1" } finally { try { - sql "stop routine load for ${alterTargetJob}" + sql "STOP ROUTINE LOAD FOR ${alterTargetJob}" } catch (Exception e) { logger.warn("failed to stop alter target routine load: ${e.message}".toString()) } From 0fca4e77f1dbddb1a549d0ee791fd5c47dff4d98 Mon Sep 17 00:00:00 2001 From: Refrain Date: Thu, 23 Jul 2026 11:18:01 +0800 Subject: [PATCH 18/19] [test](routineload) Improve target-table alter coverage ### What problem does this PR solve? Issue Number: N/A Related PR: #64878 Problem Summary: FE incremental coverage did not execute the real RoutineLoadJob target-table validation path and missed compatibility branches when ALTER contains no target table. The P0 scenario also combined target switching with unrelated property changes. Add focused unit tests for planner success and failure, lock release, the single-table invariant, legacy ALTER behavior, and Kafka/Kinesis no-target compatibility. Narrow the regression scenario to a target-only switch and an incompatible-target rejection. ### Release note None ### Check List (For Author) - Test: Unit Test and Regression test - ./run-fe-ut.sh --coverage --run relevant Routine Load test classes: 57 passed - ./build.sh --fe -j48: passed - Regression case added but not run locally because this worktree has no isolated BE/Kafka environment - Behavior changed: No - Does this need documentation: No --- .../load/routineload/RoutineLoadJobTest.java | 109 ++++++++++++++++++ .../commands/AlterRoutineLoadCommandTest.java | 23 ++++ .../test_routine_load_alter.groovy | 38 ++++-- 3 files changed, 162 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java index 263515975437fa..ae787ffe679d62 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java @@ -17,9 +17,11 @@ package org.apache.doris.load.routineload; +import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.cloud.transaction.TxnUtil; import org.apache.doris.common.Config; @@ -29,10 +31,17 @@ import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.kafka.KafkaUtil; +import org.apache.doris.load.RoutineLoadDesc; +import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.kafka.KafkaProgress; import org.apache.doris.load.routineload.kafka.KafkaRoutineLoadJob; import org.apache.doris.load.routineload.kafka.KafkaTaskInfo; +import org.apache.doris.load.routineload.kinesis.KinesisRoutineLoadJob; +import org.apache.doris.nereids.load.NereidsRoutineLoadTaskInfo; +import org.apache.doris.nereids.load.NereidsStreamLoadPlanner; +import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; +import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.persist.EditLog; import org.apache.doris.thrift.TKafkaRLTaskProgress; import org.apache.doris.thrift.TLoadSourceType; @@ -50,6 +59,9 @@ import org.apache.kafka.common.PartitionInfo; import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; +import org.mockito.MockedConstruction; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -58,6 +70,103 @@ import java.util.Map; public class RoutineLoadJobTest { + @Test + public void testValidateTargetTableRejectsMultiTableJob() { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(); + Deencapsulation.setField(routineLoadJob, "isMultiTable", true); + + UserException exception = Assert.assertThrows(UserException.class, + () -> routineLoadJob.validateTargetTable( + Mockito.mock(Database.class), Mockito.mock(OlapTable.class))); + Assert.assertTrue(exception.getMessage().contains("single-table job")); + } + + @Test + public void testValidateTargetTablePlansSuccessfully() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(); + Database database = Mockito.mock(Database.class); + OlapTable targetTable = Mockito.mock(OlapTable.class); + + try (MockedConstruction plannerConstruction = + Mockito.mockConstruction(NereidsStreamLoadPlanner.class)) { + routineLoadJob.validateTargetTable(database, targetTable); + + NereidsStreamLoadPlanner planner = plannerConstruction.constructed().get(0); + InOrder inOrder = Mockito.inOrder(targetTable, planner); + inOrder.verify(targetTable).readLock(); + inOrder.verify(planner).plan(Mockito.any(TUniqueId.class)); + inOrder.verify(targetTable).readUnlock(); + } + } + + @Test + public void testValidateTargetTableUnlocksWhenPlanningFails() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(); + routineLoadJob.setRoutineLoadDesc(new RoutineLoadDesc(null, null, + Lists.newArrayList(new ImportColumnDesc("k1")), null, null, null, null, + LoadTask.MergeType.APPEND, null)); + Database database = Mockito.mock(Database.class); + OlapTable targetTable = Mockito.mock(OlapTable.class); + + try (MockedConstruction plannerConstruction = Mockito.mockConstruction( + NereidsStreamLoadPlanner.class, (planner, context) -> { + NereidsRoutineLoadTaskInfo taskInfo = + (NereidsRoutineLoadTaskInfo) context.arguments().get(2); + Assert.assertEquals("k1", taskInfo.getColumnExprDescs().descs.get(0).getColumnName()); + Mockito.doThrow(new UserException("planning failed")) + .when(planner).plan(Mockito.any(TUniqueId.class)); + })) { + UserException exception = Assert.assertThrows(UserException.class, + () -> routineLoadJob.validateTargetTable(database, targetTable)); + Assert.assertTrue(exception.getMessage().contains("planning failed")); + + NereidsStreamLoadPlanner planner = plannerConstruction.constructed().get(0); + InOrder inOrder = Mockito.inOrder(targetTable, planner); + inOrder.verify(targetTable).readLock(); + inOrder.verify(planner).plan(Mockito.any(TUniqueId.class)); + inOrder.verify(targetTable).readUnlock(); + } + } + + @Test + public void testModifyPropertiesWithoutTargetTableKeepsExistingTable() throws Exception { + List routineLoadJobs = Lists.newArrayList( + new KafkaRoutineLoadJob(1L, "kafka_job", 1L, 101L, + "127.0.0.1:9020", "topic1", UserIdentity.ADMIN), + new KinesisRoutineLoadJob(2L, "kinesis_job", 1L, 101L, + "ap-southeast-1", "stream1", UserIdentity.ADMIN)); + for (RoutineLoadJob routineLoadJob : routineLoadJobs) { + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + } + + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CreateRoutineLoadInfo.MAX_ERROR_NUMBER_PROPERTY, "10"); + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.hasTargetTable()).thenReturn(false); + Mockito.when(command.getTargetTableId()).thenReturn(0L); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); + Mockito.when(command.getDataSourceProperties()).thenReturn(null); + + Env env = Mockito.mock(Env.class); + EditLog editLog = Mockito.mock(EditLog.class); + Mockito.when(env.getEditLog()).thenReturn(editLog); + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + for (RoutineLoadJob routineLoadJob : routineLoadJobs) { + routineLoadJob.modifyProperties(command); + Assert.assertEquals(routineLoadJob.getDataSourceType().name(), + 101L, routineLoadJob.getTableId()); + } + } + + ArgumentCaptor logCaptor = + ArgumentCaptor.forClass(AlterRoutineLoadJobOperationLog.class); + Mockito.verify(editLog, Mockito.times(2)).logAlterRoutineLoadJob(logCaptor.capture()); + for (AlterRoutineLoadJobOperationLog log : logCaptor.getAllValues()) { + Assert.assertEquals(0L, log.getTargetTableId()); + } + } + @Test public void testFirstErrorMsgInTxnCommitAttachment() { String overlongFirstErrorMsg = Strings.repeat("x", Config.first_error_msg_max_length + 1); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java index 04ad3e73a59828..1dbc0ce9cb60d9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java @@ -179,6 +179,28 @@ public void testParseAlterRoutineLoadSetTargetTableRejectsUnsupportedSyntax() { "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" COLUMNS(k1)")); } + @Test + public void testValidateRejectsEmptyAlterRoutineLoad() { + runBefore(); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1"); + + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> command.validate(connectContext)); + Assertions.assertTrue(exception.getMessage().contains("No properties are specified")); + } + + @Test + public void testValidateLegacyLoadPropertyAlter() { + runBefore(); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 COLUMNS(k1)"); + + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + Assertions.assertEquals("k1", + command.getRoutineLoadDesc().getColumnsInfo().get(0).getColumnName()); + } + @Test public void testValidateTargetTableWithJobAndDataSourceProperties() throws Exception { runBefore(); @@ -211,6 +233,7 @@ public void testValidateTargetTableOnlyDoesNotRequireOtherProperties() throws Ex Assertions.assertTrue(command.getAnalyzedJobProperties().isEmpty()); Assertions.assertNull(command.getDataSourceProperties()); Assertions.assertEquals(3000L, command.getTargetTableId()); + Mockito.verify(routineLoadJob).validateTargetTable(db, currentTable); } @Test diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy index b4145ddb8ee9bd..8c15185817e343 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy @@ -296,9 +296,10 @@ suite("test_routine_load_alter","p0") { sql "truncate table ${tableName}" } - // Test switching the target together with existing job and Kafka properties. + // Test target-only switching and reject a target incompatible with the existing load description. def srcTableName = "test_routine_load_alter_src" def dstTableName = "test_routine_load_alter_dst" + def invalidDstTableName = "test_routine_load_alter_invalid_dst" def alterTargetTopic = "test_routine_load_alter_target_table_${System.currentTimeMillis()}" def alterTargetJob = "test_alter_target_table_${System.currentTimeMillis()}" def alterTopicProducer = null @@ -318,6 +319,7 @@ suite("test_routine_load_alter","p0") { try { sql "DROP TABLE IF EXISTS ${srcTableName}" sql "DROP TABLE IF EXISTS ${dstTableName}" + sql "DROP TABLE IF EXISTS ${invalidDstTableName}" for (String targetTable in [srcTableName, dstTableName]) { sql """ CREATE TABLE ${targetTable} ( @@ -333,6 +335,20 @@ suite("test_routine_load_alter","p0") { PROPERTIES ("replication_allocation" = "tag.location.default: 1") """ } + sql """ + CREATE TABLE ${invalidDstTableName} ( + `k1` int NULL, + `k2` string NULL, + `v1` date NULL, + `v2` string NULL, + `v3` datetime NULL, + `v4` string NULL, + `required_col` int NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ("replication_allocation" = "tag.location.default: 1") + """ def topicProps = new Properties() topicProps.put("bootstrap.servers", kafka_broker.toString()) @@ -356,7 +372,8 @@ suite("test_routine_load_alter","p0") { sql """ CREATE ROUTINE LOAD ${alterTargetJob} ON ${srcTableName} - COLUMNS TERMINATED BY "," + COLUMNS TERMINATED BY ",", + COLUMNS(k1, k2, v1, v2, v3, v4) PROPERTIES ( "max_batch_interval" = "5", @@ -378,19 +395,24 @@ suite("test_routine_load_alter","p0") { assertEquals(srcTableName, showBeforeAlter[0][6].toString()) def progressBeforeAlter = showBeforeAlter[0][15].toString() + test { + sql """ + ALTER ROUTINE LOAD FOR ${alterTargetJob} + SET TARGET TABLE = "${invalidDstTableName}" + """ + exception "Column has no default value, column=required_col" + } + def showAfterFailedAlter = sql "SHOW ROUTINE LOAD FOR ${alterTargetJob}" + assertEquals(srcTableName, showAfterFailedAlter[0][6].toString()) + assertEquals(progressBeforeAlter, showAfterFailedAlter[0][15].toString()) + sql """ ALTER ROUTINE LOAD FOR ${alterTargetJob} SET TARGET TABLE = "${dstTableName}" - PROPERTIES("max_error_number" = "10") - FROM KAFKA("property.client.id" = "target-switch") """ def showAfterAlter = sql "SHOW ROUTINE LOAD FOR ${alterTargetJob}" assertEquals(dstTableName, showAfterAlter[0][6].toString()) assertEquals(progressBeforeAlter, showAfterAlter[0][15].toString()) - def alteredJobProperties = parseJson(showAfterAlter[0][11]) - def alteredCustomProperties = parseJson(showAfterAlter[0][13]) - assertEquals("10", alteredJobProperties.max_error_number.toString()) - assertEquals("target-switch", alteredCustomProperties["client.id"].toString()) [ "4,eab,2023-07-16,def,2023-07-21:05:48:31,ghi", From 963ed3e911beec89ae9b8bd198a56d154568a8a4 Mon Sep 17 00:00:00 2001 From: Refrain Date: Thu, 23 Jul 2026 16:23:25 +0800 Subject: [PATCH 19/19] [fix](routineload) Restrict target-table alter to Kafka ### What problem does this PR solve? Issue Number: N/A Related PR: #64878 Problem Summary: The target-table ALTER path still accepted Kinesis jobs and persisted target-table IDs through Kinesis mutation and replay, which expanded the feature beyond its intended Kafka-only scope and exposed unrelated Kinesis source-transition issues to this PR. Reject SET TARGET TABLE for non-Kafka jobs, remove the Kinesis target-table mutation and replay changes, and replace the Kinesis positive test with a Kafka-only rejection test. Existing Kinesis property-only ALTER behavior remains unchanged. ### Release note ALTER ROUTINE LOAD SET TARGET TABLE supports Kafka Routine Load jobs only. ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommandTest,org.apache.doris.load.routineload.KafkaRoutineLoadJobTest,org.apache.doris.load.routineload.KinesisRoutineLoadJobTest - Behavior changed: Yes. Kinesis jobs now reject SET TARGET TABLE while existing property-only ALTER remains supported. - Does this need documentation: Yes. apache/doris-website#3988 --- .../kinesis/KinesisRoutineLoadJob.java | 8 +-- .../commands/AlterRoutineLoadCommand.java | 4 ++ .../KinesisRoutineLoadJobTest.java | 66 ------------------- .../commands/AlterRoutineLoadCommandTest.java | 15 ++--- 4 files changed, 10 insertions(+), 83 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index 32208975eea95f..7cebc3f5165b49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -687,12 +687,9 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti } modifyPropertiesInternal(jobProperties, dataSourceProperties); - if (command.hasTargetTable()) { - this.tableId = command.getTargetTableId(); - } AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, - jobProperties, dataSourceProperties, command.getTargetTableId()); + jobProperties, dataSourceProperties); Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); } finally { writeUnlock(); @@ -786,9 +783,6 @@ public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { try { modifyPropertiesInternal(log.getJobProperties(), (KinesisDataSourceProperties) log.getDataSourceProperties()); - if (log.getTargetTableId() != 0) { - this.tableId = log.getTargetTableId(); - } } catch (UserException e) { LOG.error("failed to replay modify kinesis routine load job: {}", id, e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java index 8ad6392e329f28..11c9b161f5891f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java @@ -35,6 +35,7 @@ import org.apache.doris.datasource.property.fileformat.JsonFileFormatProperties; import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.routineload.AbstractDataSourceProperties; +import org.apache.doris.load.routineload.LoadDataSourceType; import org.apache.doris.load.routineload.RoutineLoadDataSourcePropertyFactory; import org.apache.doris.load.routineload.RoutineLoadJob; import org.apache.doris.mysql.privilege.PrivPredicate; @@ -373,6 +374,9 @@ private void checkPartialUpdate() throws UserException { } private void validateTargetTable(ConnectContext ctx, RoutineLoadJob job) throws UserException { + if (job.getDataSourceType() != LoadDataSourceType.KAFKA) { + throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports Kafka jobs"); + } if (job.isMultiTable()) { throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports single-table job"); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java index 6761345627002d..65aebd0084e729 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java @@ -18,7 +18,6 @@ package org.apache.doris.load.routineload; import org.apache.doris.analysis.UserIdentity; -import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.load.routineload.kinesis.KinesisConfiguration; @@ -26,19 +25,12 @@ import org.apache.doris.load.routineload.kinesis.KinesisProgress; import org.apache.doris.load.routineload.kinesis.KinesisRoutineLoadJob; import org.apache.doris.load.routineload.kinesis.KinesisTaskInfo; -import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; -import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; -import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; -import org.apache.doris.persist.EditLog; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.Gson; import org.junit.Assert; import org.junit.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.MockedStatic; -import org.mockito.Mockito; import java.util.HashMap; import java.util.HashSet; @@ -49,64 +41,6 @@ public class KinesisRoutineLoadJobTest { - @Test - public void testModifyTargetTableWithPropertiesAndReplay() throws Exception { - KinesisRoutineLoadJob routineLoadJob = - new KinesisRoutineLoadJob(1L, "kinesis_routine_load_job", 1L, - 101L, "ap-southeast-1", "stream-1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - KinesisProgress progress = new KinesisProgress(Maps.newHashMap()); - Deencapsulation.setField(routineLoadJob, "progress", progress); - - Map sourceProperties = Maps.newHashMap(); - sourceProperties.put("property.client.timeout", "1000"); - KinesisDataSourceProperties dataSourceProperties = new KinesisDataSourceProperties(sourceProperties); - dataSourceProperties.setAlter(true); - dataSourceProperties.setTimezone("UTC"); - dataSourceProperties.analyze(); - - Map jobProperties = Maps.newHashMap(); - jobProperties.put(CreateRoutineLoadInfo.MAX_ERROR_NUMBER_PROPERTY, "10"); - AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); - Mockito.when(command.hasTargetTable()).thenReturn(true); - Mockito.when(command.getTargetTableId()).thenReturn(202L); - Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); - Mockito.when(command.getDataSourceProperties()).thenReturn(dataSourceProperties); - - Env env = Mockito.mock(Env.class); - EditLog editLog = Mockito.mock(EditLog.class); - Mockito.when(env.getEditLog()).thenReturn(editLog); - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { - envStatic.when(Env::getCurrentEnv).thenReturn(env); - routineLoadJob.modifyProperties(command); - } - - Assert.assertEquals(202L, routineLoadJob.getTableId()); - Assert.assertSame(progress, routineLoadJob.getProgress()); - Assert.assertEquals(10L, ((Long) Deencapsulation.getField(routineLoadJob, "maxErrorNum")).longValue()); - Assert.assertEquals("1000", routineLoadJob.getCustomProperties().get("property.client.timeout")); - ArgumentCaptor logCaptor = - ArgumentCaptor.forClass(AlterRoutineLoadJobOperationLog.class); - Mockito.verify(editLog).logAlterRoutineLoadJob(logCaptor.capture()); - Assert.assertEquals(202L, logCaptor.getValue().getTargetTableId()); - - KinesisRoutineLoadJob replayJob = - new KinesisRoutineLoadJob(1L, "kinesis_routine_load_job", 1L, - 303L, "ap-southeast-1", "stream-1", UserIdentity.ADMIN); - Map shardPositions = Maps.newHashMap(); - shardPositions.put("shard-0", "123"); - KinesisProgress replayProgress = new KinesisProgress(shardPositions); - Deencapsulation.setField(replayJob, "progress", replayProgress); - replayJob.replayModifyProperties(new AlterRoutineLoadJobOperationLog( - 1L, Maps.newHashMap(), null)); - Assert.assertEquals(303L, replayJob.getTableId()); - replayJob.replayModifyProperties(logCaptor.getValue()); - Assert.assertEquals(202L, replayJob.getTableId()); - Assert.assertSame(replayProgress, replayJob.getProgress()); - Assert.assertEquals("123", ((KinesisProgress) replayJob.getProgress()) - .getSequenceNumberByShard("shard-0")); - } - @Test public void testRoutineLoadTaskConcurrentNum() { int oldMaxConcurrent = Config.max_routine_load_task_concurrent_num; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java index 1dbc0ce9cb60d9..907f10f5ec5037 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java @@ -29,7 +29,6 @@ import org.apache.doris.load.routineload.RoutineLoadJob; import org.apache.doris.load.routineload.RoutineLoadManager; import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; -import org.apache.doris.load.routineload.kinesis.KinesisDataSourceProperties; import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.exceptions.ParseException; @@ -237,22 +236,18 @@ public void testValidateTargetTableOnlyDoesNotRequireOtherProperties() throws Ex } @Test - public void testValidateTargetTableWithKinesisDataSourceProperties() throws Exception { + public void testValidateTargetTableRejectsKinesisJob() throws Exception { runBefore(); Mockito.when(routineLoadJob.getDataSourceType()).thenReturn(LoadDataSourceType.KINESIS); - Mockito.when(currentTable.getId()).thenReturn(3000L); - mockTargetTable(currentTable); AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" " + "PROPERTIES(\"max_error_number\"=\"1\") " + "FROM KINESIS(\"property.client.timeout\"=\"1000\")"); - Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); - Assertions.assertInstanceOf(KinesisDataSourceProperties.class, command.getDataSourceProperties()); - Assertions.assertEquals("1000", command.getDataSourceProperties() - .getOriginalDataSourceProperties().get("property.client.timeout")); - Assertions.assertEquals(3000L, command.getTargetTableId()); - Mockito.verify(routineLoadJob).validateTargetTable(db, currentTable); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> command.validate(connectContext)); + Assertions.assertTrue(exception.getMessage().contains("only supports Kafka jobs")); + Mockito.verify(catalog, Mockito.never()).getDbOrAnalysisException(Mockito.anyString()); } @Test