diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 9873368f405114..7053e696488bda 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ExprToSqlVisitor; +import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.Separator; import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.analysis.UserIdentity; @@ -470,6 +471,27 @@ protected void setRoutineLoadDesc(RoutineLoadDesc routineLoadDesc) { } } + public void validateTargetTable(Database db, OlapTable targetTable) throws UserException { + if (isMultiTable) { + throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports single-table job"); + } + List columnsInfo = null; + if (columnDescs != null && !columnDescs.descs.isEmpty()) { + columnsInfo = new ArrayList<>(columnDescs.descs); + } + checkMeta(targetTable, new RoutineLoadDesc(columnSeparator, lineDelimiter, columnsInfo, + precedingFilter, whereExpr, partitionNamesInfo, deleteCondition, mergeType, sequenceCol)); + + targetTable.readLock(); + try { + NereidsStreamLoadPlanner planner = new NereidsStreamLoadPlanner(db, targetTable, + toNereidsRoutineLoadTaskInfo()); + planner.plan(new TUniqueId(0, 0)); + } finally { + targetTable.readUnlock(); + } + } + @Override public long getId() { return id; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java index 8b8247479792fc..b7f1ff0fc51af9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java @@ -782,9 +782,12 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti } modifyPropertiesInternal(jobProperties, dataSourceProperties); + if (command.hasTargetTable()) { + tableId = command.getTargetTableId(); + } AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, - jobProperties, dataSourceProperties); + jobProperties, dataSourceProperties, command.getTargetTableId()); Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); } finally { writeUnlock(); @@ -908,6 +911,9 @@ private void resetCloudProgress(Cloud.ResetRLProgressRequest.Builder builder) th public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { try { modifyPropertiesInternal(log.getJobProperties(), (KafkaDataSourceProperties) log.getDataSourceProperties()); + if (log.getTargetTableId() != 0) { + tableId = log.getTargetTableId(); + } } catch (UserException e) { // should not happen LOG.error("failed to replay modify kafka routine load job: {}", id, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 42695f85a8d34f..a3e120bfdfe64d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -9361,6 +9361,8 @@ public LogicalPlan visitAlterRoutineLoad(DorisParser.AlterRoutineLoadContext ctx throw new ParseException("only support [.]", ctx.name); } LabelNameInfo labelNameInfo = new LabelNameInfo(dbName, jobName); + String targetTableName = ctx.targetTable == null + ? null : SqlLiteralUtils.parseStringLiteral(ctx.targetTable.getText()); Map properties = new HashMap<>(); if (ctx.properties != null) { @@ -9387,7 +9389,8 @@ public LogicalPlan visitAlterRoutineLoad(DorisParser.AlterRoutineLoadContext ctx } } - return new AlterRoutineLoadCommand(labelNameInfo, loadPropertyMap, properties, dataSourceMapProperties); + return new AlterRoutineLoadCommand(labelNameInfo, targetTableName, + loadPropertyMap, properties, dataSourceMapProperties); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java index 367480c5d934d9..11c9b161f5891f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java @@ -21,18 +21,24 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.RandomDistributionInfo; import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.UserException; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; +import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; import org.apache.doris.datasource.property.fileformat.JsonFileFormatProperties; import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.routineload.AbstractDataSourceProperties; +import org.apache.doris.load.routineload.LoadDataSourceType; import org.apache.doris.load.routineload.RoutineLoadDataSourcePropertyFactory; import org.apache.doris.load.routineload.RoutineLoadJob; +import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo; @@ -52,6 +58,7 @@ /** * ALTER ROUTINE LOAD db.label + * SET TARGET TABLE = "table" * PROPERTIES( * ... * ) @@ -85,10 +92,12 @@ public class AlterRoutineLoadCommand extends AlterCommand { .build(); private final LabelNameInfo labelNameInfo; + private final String targetTableName; private final Map loadPropertyMap; private RoutineLoadDesc routineLoadDesc; private final Map jobProperties; private final Map dataSourceMapProperties; + private long targetTableId; private boolean isPartialUpdate; // save analyzed job properties. @@ -100,6 +109,7 @@ public class AlterRoutineLoadCommand extends AlterCommand { * AlterRoutineLoadCommand */ public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, + String targetTableName, Map loadPropertyMap, Map jobProperties, Map dataSourceMapProperties) { @@ -108,6 +118,7 @@ public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, Objects.requireNonNull(jobProperties, "jobProperties is null"); Objects.requireNonNull(dataSourceMapProperties, "dataSourceMapProperties is null"); this.labelNameInfo = labelNameInfo; + this.targetTableName = targetTableName; this.loadPropertyMap = loadPropertyMap == null ? Maps.newHashMap() : loadPropertyMap; this.jobProperties = jobProperties; this.dataSourceMapProperties = dataSourceMapProperties; @@ -118,7 +129,7 @@ public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, public AlterRoutineLoadCommand(LabelNameInfo labelNameInfo, Map jobProperties, Map dataSourceMapProperties) { - this(labelNameInfo, Maps.newHashMap(), jobProperties, dataSourceMapProperties); + this(labelNameInfo, null, Maps.newHashMap(), jobProperties, dataSourceMapProperties); } public String getDbName() { @@ -133,6 +144,18 @@ public Map getAnalyzedJobProperties() { return analyzedJobProperties; } + public boolean hasTargetTable() { + return targetTableName != null; + } + + public String getTargetTableName() { + return targetTableName; + } + + public long getTargetTableId() { + return targetTableId; + } + public boolean hasDataSourceProperty() { return MapUtils.isNotEmpty(dataSourceMapProperties); } @@ -164,15 +187,21 @@ public void validate(ConnectContext ctx) throws UserException { // check routine load job properties include desired concurrent number etc. checkJobProperties(); // check load properties - RoutineLoadJob job = Env.getCurrentEnv().getRoutineLoadManager() - .getJob(getDbName(), getJobName()); - this.routineLoadDesc = CreateRoutineLoadInfo.checkLoadProperties(ctx, loadPropertyMap, - job.getDbFullName(), job.getTableName(), job.isMultiTable(), job.getMergeType()); + RoutineLoadJob job = hasTargetTable() + ? Env.getCurrentEnv().getRoutineLoadManager().checkPrivAndGetJob(getDbName(), getJobName()) + : Env.getCurrentEnv().getRoutineLoadManager().getJob(getDbName(), getJobName()); + if (MapUtils.isNotEmpty(loadPropertyMap)) { + this.routineLoadDesc = CreateRoutineLoadInfo.checkLoadProperties(ctx, loadPropertyMap, + job.getDbFullName(), job.getTableName(), job.isMultiTable(), job.getMergeType()); + } // check data source properties checkDataSourceProperties(); checkPartialUpdate(); + if (hasTargetTable()) { + validateTargetTable(ctx, job); + } if (analyzedJobProperties.isEmpty() && MapUtils.isEmpty(dataSourceMapProperties) - && routineLoadDesc == null) { + && routineLoadDesc == null && !hasTargetTable()) { throw new AnalysisException("No properties are specified"); } } @@ -344,6 +373,37 @@ private void checkPartialUpdate() throws UserException { } } + private void validateTargetTable(ConnectContext ctx, RoutineLoadJob job) throws UserException { + if (job.getDataSourceType() != LoadDataSourceType.KAFKA) { + throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports Kafka jobs"); + } + if (job.isMultiTable()) { + throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports single-table job"); + } + String dbFullName = job.getDbFullName(); + if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ctx, InternalCatalog.INTERNAL_CATALOG_NAME, + dbFullName, targetTableName, PrivPredicate.LOAD)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "LOAD", + ctx.getQualifiedUser(), ctx.getRemoteIP(), dbFullName + ": " + targetTableName); + } + Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException(dbFullName); + Table table = db.getTableOrAnalysisException(targetTableName); + if (!(table instanceof OlapTable)) { + throw new AnalysisException("ALTER ROUTINE LOAD target table only supports OLAP table"); + } + OlapTable olapTable = (OlapTable) table; + if (olapTable.isTemporary()) { + throw new AnalysisException("Do not support load for temporary table " + olapTable.getDisplayName()); + } + if (job.isLoadToSingleTablet() + && !(olapTable.getDefaultDistributionInfo() instanceof RandomDistributionInfo)) { + throw new AnalysisException( + "if load_to_single_tablet set to true, the olap table must be with random distribution"); + } + job.validateTargetTable(db, olapTable); + this.targetTableId = olapTable.getId(); + } + @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitAlterRoutineLoadCommand(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java index 4729882f7927fb..ae7c8ad214de7c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java @@ -37,12 +37,20 @@ public class AlterRoutineLoadJobOperationLog implements Writable { private Map jobProperties; @SerializedName(value = "dataSourceProperties") private AbstractDataSourceProperties dataSourceProperties; + @SerializedName(value = "targetTableId") + private long targetTableId; public AlterRoutineLoadJobOperationLog(long jobId, Map jobProperties, AbstractDataSourceProperties dataSourceProperties) { + this(jobId, jobProperties, dataSourceProperties, 0L); + } + + public AlterRoutineLoadJobOperationLog(long jobId, Map jobProperties, + AbstractDataSourceProperties dataSourceProperties, long targetTableId) { this.jobId = jobId; this.jobProperties = jobProperties; this.dataSourceProperties = dataSourceProperties; + this.targetTableId = targetTableId; } public long getJobId() { @@ -57,6 +65,10 @@ public AbstractDataSourceProperties getDataSourceProperties() { return dataSourceProperties; } + public long getTargetTableId() { + return targetTableId; + } + public static AlterRoutineLoadJobOperationLog read(DataInput in) throws IOException { String json = Text.readString(in); return GsonUtils.GSON.fromJson(json, AlterRoutineLoadJobOperationLog.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index ebdad39a2744f5..193fc9ba22731b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -40,10 +40,13 @@ import org.apache.doris.load.routineload.kafka.KafkaRoutineLoadJob; import org.apache.doris.load.routineload.kafka.KafkaTaskInfo; import org.apache.doris.mysql.privilege.MockedAuth; +import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo; import org.apache.doris.nereids.trees.plans.commands.load.LoadProperty; import org.apache.doris.nereids.trees.plans.commands.load.LoadSeparator; +import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; +import org.apache.doris.persist.EditLog; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TResourceInfo; @@ -57,6 +60,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -237,6 +241,101 @@ public void testUpdateLagRebuildsConvertedPropertiesAfterReplay() throws UserExc } } + @Test + public void testModifyTargetTableWithJobAndDataSourceProperties() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, + 101L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + KafkaProgress progress = new KafkaProgress(Maps.newHashMap()); + Deencapsulation.setField(routineLoadJob, "progress", progress); + + Map sourceProperties = Maps.newHashMap(); + sourceProperties.put("property.client.id", "target-switch"); + KafkaDataSourceProperties dataSourceProperties = new KafkaDataSourceProperties(sourceProperties); + dataSourceProperties.setAlter(true); + dataSourceProperties.setTimezone("UTC"); + dataSourceProperties.analyze(); + + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CreateRoutineLoadInfo.MAX_ERROR_NUMBER_PROPERTY, "10"); + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.hasTargetTable()).thenReturn(true); + Mockito.when(command.getTargetTableId()).thenReturn(202L); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); + Mockito.when(command.getDataSourceProperties()).thenReturn(dataSourceProperties); + + Env env = Mockito.mock(Env.class); + EditLog editLog = Mockito.mock(EditLog.class); + Mockito.when(env.getEditLog()).thenReturn(editLog); + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + routineLoadJob.modifyProperties(command); + } + + Assert.assertEquals(202L, routineLoadJob.getTableId()); + Assert.assertSame(progress, routineLoadJob.getProgress()); + Assert.assertEquals(10L, ((Long) Deencapsulation.getField(routineLoadJob, "maxErrorNum")).longValue()); + Assert.assertEquals("target-switch", + routineLoadJob.getCustomProperties().get("property.client.id")); + ArgumentCaptor logCaptor = + ArgumentCaptor.forClass(AlterRoutineLoadJobOperationLog.class); + Mockito.verify(editLog).logAlterRoutineLoadJob(logCaptor.capture()); + Assert.assertEquals(202L, logCaptor.getValue().getTargetTableId()); + } + + @Test + public void testModifyTargetTableOnly() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, + 101L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + KafkaProgress progress = new KafkaProgress(Maps.newHashMap()); + Deencapsulation.setField(routineLoadJob, "progress", progress); + + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.hasTargetTable()).thenReturn(true); + Mockito.when(command.getTargetTableId()).thenReturn(202L); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + Mockito.when(command.getDataSourceProperties()).thenReturn(null); + + Env env = Mockito.mock(Env.class); + EditLog editLog = Mockito.mock(EditLog.class); + Mockito.when(env.getEditLog()).thenReturn(editLog); + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + routineLoadJob.modifyProperties(command); + } + + Assert.assertEquals(202L, routineLoadJob.getTableId()); + Assert.assertSame(progress, routineLoadJob.getProgress()); + ArgumentCaptor logCaptor = + ArgumentCaptor.forClass(AlterRoutineLoadJobOperationLog.class); + Mockito.verify(editLog).logAlterRoutineLoadJob(logCaptor.capture()); + Assert.assertEquals(202L, logCaptor.getValue().getTargetTableId()); + Assert.assertTrue(logCaptor.getValue().getJobProperties().isEmpty()); + Assert.assertNull(logCaptor.getValue().getDataSourceProperties()); + } + + @Test + public void testReplayModifyPropertiesSwitchesTargetTableWithoutResettingProgress() { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, + 101L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Map partitionToOffset = Maps.newHashMap(); + partitionToOffset.put(1, 123L); + KafkaProgress progress = new KafkaProgress(partitionToOffset); + Deencapsulation.setField(routineLoadJob, "progress", progress); + + routineLoadJob.replayModifyProperties(new AlterRoutineLoadJobOperationLog( + 1L, Maps.newHashMap(), null)); + Assert.assertEquals(101L, routineLoadJob.getTableId()); + + routineLoadJob.replayModifyProperties(new AlterRoutineLoadJobOperationLog( + 1L, Maps.newHashMap(), null, 202L)); + Assert.assertEquals(202L, routineLoadJob.getTableId()); + Assert.assertSame(progress, routineLoadJob.getProgress()); + Assert.assertEquals(Long.valueOf(123L), + ((KafkaProgress) routineLoadJob.getProgress()).getOffsetByPartition(1)); + } + @Test public void testUpdateProgressWarnsWhenReadCommittedTaskHasZeroRowsAndLag() throws UserException { KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java index 263515975437fa..ae787ffe679d62 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java @@ -17,9 +17,11 @@ package org.apache.doris.load.routineload; +import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.cloud.transaction.TxnUtil; import org.apache.doris.common.Config; @@ -29,10 +31,17 @@ import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.kafka.KafkaUtil; +import org.apache.doris.load.RoutineLoadDesc; +import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.kafka.KafkaProgress; import org.apache.doris.load.routineload.kafka.KafkaRoutineLoadJob; import org.apache.doris.load.routineload.kafka.KafkaTaskInfo; +import org.apache.doris.load.routineload.kinesis.KinesisRoutineLoadJob; +import org.apache.doris.nereids.load.NereidsRoutineLoadTaskInfo; +import org.apache.doris.nereids.load.NereidsStreamLoadPlanner; +import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; +import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.persist.EditLog; import org.apache.doris.thrift.TKafkaRLTaskProgress; import org.apache.doris.thrift.TLoadSourceType; @@ -50,6 +59,9 @@ import org.apache.kafka.common.PartitionInfo; import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; +import org.mockito.MockedConstruction; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -58,6 +70,103 @@ import java.util.Map; public class RoutineLoadJobTest { + @Test + public void testValidateTargetTableRejectsMultiTableJob() { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(); + Deencapsulation.setField(routineLoadJob, "isMultiTable", true); + + UserException exception = Assert.assertThrows(UserException.class, + () -> routineLoadJob.validateTargetTable( + Mockito.mock(Database.class), Mockito.mock(OlapTable.class))); + Assert.assertTrue(exception.getMessage().contains("single-table job")); + } + + @Test + public void testValidateTargetTablePlansSuccessfully() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(); + Database database = Mockito.mock(Database.class); + OlapTable targetTable = Mockito.mock(OlapTable.class); + + try (MockedConstruction plannerConstruction = + Mockito.mockConstruction(NereidsStreamLoadPlanner.class)) { + routineLoadJob.validateTargetTable(database, targetTable); + + NereidsStreamLoadPlanner planner = plannerConstruction.constructed().get(0); + InOrder inOrder = Mockito.inOrder(targetTable, planner); + inOrder.verify(targetTable).readLock(); + inOrder.verify(planner).plan(Mockito.any(TUniqueId.class)); + inOrder.verify(targetTable).readUnlock(); + } + } + + @Test + public void testValidateTargetTableUnlocksWhenPlanningFails() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(); + routineLoadJob.setRoutineLoadDesc(new RoutineLoadDesc(null, null, + Lists.newArrayList(new ImportColumnDesc("k1")), null, null, null, null, + LoadTask.MergeType.APPEND, null)); + Database database = Mockito.mock(Database.class); + OlapTable targetTable = Mockito.mock(OlapTable.class); + + try (MockedConstruction plannerConstruction = Mockito.mockConstruction( + NereidsStreamLoadPlanner.class, (planner, context) -> { + NereidsRoutineLoadTaskInfo taskInfo = + (NereidsRoutineLoadTaskInfo) context.arguments().get(2); + Assert.assertEquals("k1", taskInfo.getColumnExprDescs().descs.get(0).getColumnName()); + Mockito.doThrow(new UserException("planning failed")) + .when(planner).plan(Mockito.any(TUniqueId.class)); + })) { + UserException exception = Assert.assertThrows(UserException.class, + () -> routineLoadJob.validateTargetTable(database, targetTable)); + Assert.assertTrue(exception.getMessage().contains("planning failed")); + + NereidsStreamLoadPlanner planner = plannerConstruction.constructed().get(0); + InOrder inOrder = Mockito.inOrder(targetTable, planner); + inOrder.verify(targetTable).readLock(); + inOrder.verify(planner).plan(Mockito.any(TUniqueId.class)); + inOrder.verify(targetTable).readUnlock(); + } + } + + @Test + public void testModifyPropertiesWithoutTargetTableKeepsExistingTable() throws Exception { + List routineLoadJobs = Lists.newArrayList( + new KafkaRoutineLoadJob(1L, "kafka_job", 1L, 101L, + "127.0.0.1:9020", "topic1", UserIdentity.ADMIN), + new KinesisRoutineLoadJob(2L, "kinesis_job", 1L, 101L, + "ap-southeast-1", "stream1", UserIdentity.ADMIN)); + for (RoutineLoadJob routineLoadJob : routineLoadJobs) { + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + } + + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CreateRoutineLoadInfo.MAX_ERROR_NUMBER_PROPERTY, "10"); + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.hasTargetTable()).thenReturn(false); + Mockito.when(command.getTargetTableId()).thenReturn(0L); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); + Mockito.when(command.getDataSourceProperties()).thenReturn(null); + + Env env = Mockito.mock(Env.class); + EditLog editLog = Mockito.mock(EditLog.class); + Mockito.when(env.getEditLog()).thenReturn(editLog); + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + for (RoutineLoadJob routineLoadJob : routineLoadJobs) { + routineLoadJob.modifyProperties(command); + Assert.assertEquals(routineLoadJob.getDataSourceType().name(), + 101L, routineLoadJob.getTableId()); + } + } + + ArgumentCaptor logCaptor = + ArgumentCaptor.forClass(AlterRoutineLoadJobOperationLog.class); + Mockito.verify(editLog, Mockito.times(2)).logAlterRoutineLoadJob(logCaptor.capture()); + for (AlterRoutineLoadJobOperationLog log : logCaptor.getAllValues()) { + Assert.assertEquals(0L, log.getTargetTableId()); + } + } + @Test public void testFirstErrorMsgInTxnCommitAttachment() { String overlongFirstErrorMsg = Strings.repeat("x", Config.first_error_msg_max_length + 1); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java index 0989bc127b046b..907f10f5ec5037 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommandTest.java @@ -19,11 +19,20 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.RandomDistributionInfo; import org.apache.doris.catalog.Table; +import org.apache.doris.common.AnalysisException; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.load.loadv2.LoadTask; +import org.apache.doris.load.routineload.LoadDataSourceType; import org.apache.doris.load.routineload.RoutineLoadJob; import org.apache.doris.load.routineload.RoutineLoadManager; +import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.exceptions.ParseException; +import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo; import org.apache.doris.qe.ConnectContext; @@ -41,8 +50,16 @@ import java.util.Map; public class AlterRoutineLoadCommandTest { + private static final NereidsParser PARSER = new NereidsParser(); + private Env env; private ConnectContext connectContext; + private AccessControllerManager accessManager; + private InternalCatalog catalog; + private Database db; + private OlapTable currentTable; + private RoutineLoadManager routineLoadManager; + private RoutineLoadJob routineLoadJob; private MockedStatic envMockedStatic; private MockedStatic ctxMockedStatic; @@ -50,26 +67,38 @@ public class AlterRoutineLoadCommandTest { public void setUp() throws Exception { env = Mockito.mock(Env.class); connectContext = Mockito.mock(ConnectContext.class); + accessManager = Mockito.mock(AccessControllerManager.class); + catalog = Mockito.mock(InternalCatalog.class); + db = Mockito.mock(Database.class); + currentTable = Mockito.mock(OlapTable.class); + routineLoadManager = Mockito.mock(RoutineLoadManager.class); + routineLoadJob = Mockito.mock(RoutineLoadJob.class); envMockedStatic = Mockito.mockStatic(Env.class); ctxMockedStatic = Mockito.mockStatic(ConnectContext.class); envMockedStatic.when(Env::getCurrentEnv).thenReturn(env); ctxMockedStatic.when(ConnectContext::get).thenReturn(connectContext); Mockito.when(connectContext.getSessionVariable()).thenReturn(new SessionVariable()); Mockito.when(connectContext.getState()).thenReturn(new QueryState()); - InternalCatalog catalog = Mockito.mock(InternalCatalog.class); - Database db = Mockito.mock(Database.class); - Table tbl = Mockito.mock(Table.class); + Mockito.when(connectContext.getQualifiedUser()).thenReturn("testUser"); + Mockito.when(connectContext.getRemoteIP()).thenReturn("127.0.0.1"); envMockedStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); Mockito.doReturn(db).when(catalog).getDbOrAnalysisException(Mockito.anyString()); - Mockito.doReturn(tbl).when(db).getTableOrAnalysisException(Mockito.anyString()); - Mockito.when(env.getRoutineLoadManager()).thenReturn(Mockito.mock(RoutineLoadManager.class)); - RoutineLoadManager rlm = env.getRoutineLoadManager(); - RoutineLoadJob rlJob = Mockito.mock(RoutineLoadJob.class); - Mockito.when(rlm.getJob(Mockito.anyString(), Mockito.anyString())).thenReturn(rlJob); - Mockito.when(rlJob.getDbFullName()).thenReturn("testDb"); - Mockito.when(rlJob.getTableName()).thenReturn("testTable"); - Mockito.when(rlJob.isMultiTable()).thenReturn(false); - Mockito.when(rlJob.getMergeType()).thenReturn(LoadTask.MergeType.APPEND); + Mockito.doReturn(currentTable).when(db).getTableOrAnalysisException(Mockito.anyString()); + Mockito.when(env.getRoutineLoadManager()).thenReturn(routineLoadManager); + Mockito.when(env.getAccessManager()).thenReturn(accessManager); + Mockito.when(accessManager.checkTblPriv(Mockito.any(ConnectContext.class), Mockito.anyString(), + Mockito.anyString(), Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(routineLoadManager.getJob(Mockito.anyString(), Mockito.anyString())).thenReturn(routineLoadJob); + Mockito.when(routineLoadManager.checkPrivAndGetJob(Mockito.anyString(), Mockito.anyString())) + .thenReturn(routineLoadJob); + Mockito.when(routineLoadJob.getDbFullName()).thenReturn("testDb"); + Mockito.when(routineLoadJob.getTableName()).thenReturn("testTable"); + Mockito.when(routineLoadJob.isMultiTable()).thenReturn(false); + Mockito.when(routineLoadJob.getMergeType()).thenReturn(LoadTask.MergeType.APPEND); + Mockito.when(routineLoadJob.getDataSourceType()).thenReturn(LoadDataSourceType.KAFKA); + Mockito.when(routineLoadJob.getTimezone()).thenReturn("UTC"); + Mockito.when(routineLoadJob.isLoadToSingleTablet()).thenReturn(false); + Mockito.when(currentTable.isTemporary()).thenReturn(false); } @AfterEach @@ -86,6 +115,10 @@ private void runBefore() { Mockito.when(connectContext.isSkipAuth()).thenReturn(true); } + private void mockTargetTable(Table table) throws AnalysisException { + Mockito.doReturn(table).when(db).getTableOrAnalysisException("testTable2"); + } + @Test public void testValidate() { runBefore(); @@ -115,4 +148,175 @@ public void testValidate() { Assertions.assertTrue(command.getAnalyzedJobProperties().containsKey(CreateRoutineLoadInfo.STRICT_MODE)); Assertions.assertTrue(command.getAnalyzedJobProperties().containsKey(CreateRoutineLoadInfo.TIMEZONE)); } + + @Test + public void testParseAlterRoutineLoadSetTargetTable() { + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); + Assertions.assertEquals("testDb", command.getDbName()); + Assertions.assertEquals("label1", command.getJobName()); + Assertions.assertTrue(command.hasTargetTable()); + Assertions.assertEquals("testTable2", command.getTargetTableName()); + + AlterRoutineLoadCommand escapedNameCommand = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR label1 SET TARGET TABLE = \"test\"\"Table2\""); + Assertions.assertEquals("test\"Table2", escapedNameCommand.getTargetTableName()); + + AlterRoutineLoadCommand emptyNameCommand = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR label1 SET TARGET TABLE = \"\""); + Assertions.assertTrue(emptyNameCommand.hasTargetTable()); + Assertions.assertEquals("", emptyNameCommand.getTargetTableName()); + } + + @Test + public void testParseAlterRoutineLoadSetTargetTableRejectsUnsupportedSyntax() { + Assertions.assertThrows(ParseException.class, () -> PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 ON testTable2")); + Assertions.assertThrows(ParseException.class, () -> PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = testTable2")); + Assertions.assertThrows(ParseException.class, () -> PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" COLUMNS(k1)")); + } + + @Test + public void testValidateRejectsEmptyAlterRoutineLoad() { + runBefore(); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1"); + + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> command.validate(connectContext)); + Assertions.assertTrue(exception.getMessage().contains("No properties are specified")); + } + + @Test + public void testValidateLegacyLoadPropertyAlter() { + runBefore(); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 COLUMNS(k1)"); + + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + Assertions.assertEquals("k1", + command.getRoutineLoadDesc().getColumnsInfo().get(0).getColumnName()); + } + + @Test + public void testValidateTargetTableWithJobAndDataSourceProperties() throws Exception { + runBefore(); + Mockito.when(currentTable.getId()).thenReturn(3000L); + mockTargetTable(currentTable); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" " + + "PROPERTIES(\"max_error_number\"=\"1\") " + + "FROM KAFKA(\"property.client.id\"=\"target-switch\")"); + + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + Assertions.assertEquals("1", command.getAnalyzedJobProperties() + .get(CreateRoutineLoadInfo.MAX_ERROR_NUMBER_PROPERTY)); + Assertions.assertInstanceOf(KafkaDataSourceProperties.class, command.getDataSourceProperties()); + Assertions.assertEquals("target-switch", command.getDataSourceProperties() + .getOriginalDataSourceProperties().get("property.client.id")); + Assertions.assertEquals(3000L, command.getTargetTableId()); + Mockito.verify(routineLoadJob).validateTargetTable(db, currentTable); + } + + @Test + public void testValidateTargetTableOnlyDoesNotRequireOtherProperties() throws Exception { + runBefore(); + Mockito.when(currentTable.getId()).thenReturn(3000L); + mockTargetTable(currentTable); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); + + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + Assertions.assertTrue(command.getAnalyzedJobProperties().isEmpty()); + Assertions.assertNull(command.getDataSourceProperties()); + Assertions.assertEquals(3000L, command.getTargetTableId()); + Mockito.verify(routineLoadJob).validateTargetTable(db, currentTable); + } + + @Test + public void testValidateTargetTableRejectsKinesisJob() throws Exception { + runBefore(); + Mockito.when(routineLoadJob.getDataSourceType()).thenReturn(LoadDataSourceType.KINESIS); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\" " + + "PROPERTIES(\"max_error_number\"=\"1\") " + + "FROM KINESIS(\"property.client.timeout\"=\"1000\")"); + + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> command.validate(connectContext)); + Assertions.assertTrue(exception.getMessage().contains("only supports Kafka jobs")); + Mockito.verify(catalog, Mockito.never()).getDbOrAnalysisException(Mockito.anyString()); + } + + @Test + public void testValidateTargetTableRejectsMultiTableJob() throws Exception { + runBefore(); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); + + Mockito.when(routineLoadJob.isMultiTable()).thenReturn(true); + Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, + () -> command.validate(connectContext)).getMessage().contains("single-table")); + } + + @Test + public void testValidateUnauthorizedTargetDoesNotResolveMetadata() throws Exception { + runBefore(); + Mockito.when(accessManager.checkTblPriv(Mockito.any(ConnectContext.class), Mockito.anyString(), + Mockito.eq("testDb"), Mockito.eq("testTable2"), Mockito.eq(PrivPredicate.LOAD))) + .thenReturn(false); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); + + Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, + () -> command.validate(connectContext)).getMessage().contains("LOAD")); + Mockito.verify(routineLoadManager).checkPrivAndGetJob("testDb", "label1"); + Mockito.verify(catalog, Mockito.never()).getDbOrAnalysisException(Mockito.anyString()); + } + + @Test + public void testValidateUnauthorizedJobDoesNotResolveTarget() throws Exception { + runBefore(); + Mockito.when(routineLoadManager.checkPrivAndGetJob("testDb", "label1")) + .thenThrow(new AnalysisException("access denied")); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); + + Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, + () -> command.validate(connectContext)).getMessage().contains("access denied")); + Mockito.verify(accessManager, Mockito.never()).checkTblPriv( + Mockito.any(ConnectContext.class), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), + Mockito.any()); + Mockito.verify(catalog, Mockito.never()).getDbOrAnalysisException(Mockito.anyString()); + } + + @Test + public void testValidateTargetTableConstraints() throws Exception { + runBefore(); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) PARSER.parseSingle( + "ALTER ROUTINE LOAD FOR testDb.label1 SET TARGET TABLE = \"testTable2\""); + + mockTargetTable(Mockito.mock(Table.class)); + Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, + () -> command.validate(connectContext)).getMessage().contains("only supports OLAP table")); + + OlapTable temporaryTable = Mockito.mock(OlapTable.class); + Mockito.when(temporaryTable.isTemporary()).thenReturn(true); + mockTargetTable(temporaryTable); + Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, + () -> command.validate(connectContext)).getMessage().contains("temporary table")); + + OlapTable hashDistributedTable = Mockito.mock(OlapTable.class); + Mockito.when(hashDistributedTable.isTemporary()).thenReturn(false); + mockTargetTable(hashDistributedTable); + Mockito.when(routineLoadJob.isLoadToSingleTablet()).thenReturn(true); + Assertions.assertTrue(Assertions.assertThrows(AnalysisException.class, + () -> command.validate(connectContext)).getMessage().contains("load_to_single_tablet")); + + RandomDistributionInfo randomDistributionInfo = Mockito.mock(RandomDistributionInfo.class); + Mockito.when(hashDistributedTable.getDefaultDistributionInfo()).thenReturn(randomDistributionInfo); + Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java index 8a1550d48f5d13..3e55210faba84f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java @@ -18,6 +18,7 @@ package org.apache.doris.persist; import org.apache.doris.common.UserException; +import org.apache.doris.common.io.Text; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.load.routineload.kafka.KafkaConfiguration; import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; @@ -27,6 +28,8 @@ import org.junit.Assert; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.File; @@ -61,7 +64,7 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User routineLoadDataSourceProperties.analyze(); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(jobId, - jobProperties, routineLoadDataSourceProperties); + jobProperties, routineLoadDataSourceProperties, 2000L); log.write(out); out.flush(); out.close(); @@ -81,9 +84,25 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User kafkaDataSourceProperties.getKafkaPartitionOffsets().get(0)); Assert.assertEquals(routineLoadDataSourceProperties.getKafkaPartitionOffsets().get(1), kafkaDataSourceProperties.getKafkaPartitionOffsets().get(1)); + Assert.assertEquals(2000L, log2.getTargetTableId()); in.close(); } + @Test + public void testDeserializeLogWithoutTargetTableId() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(bytes)) { + Text.writeString(out, "{\"jobId\":1000,\"jobProperties\":{}," + + "\"dataSourceProperties\":null}"); + } + + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + AlterRoutineLoadJobOperationLog log = AlterRoutineLoadJobOperationLog.read(in); + Assert.assertEquals(1000L, log.getJobId()); + Assert.assertEquals(0L, log.getTargetTableId()); + } + } + } diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 446685892295e4..407454f7b091d0 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -562,6 +562,7 @@ TABLESAMPLE: 'TABLESAMPLE'; TABLET: 'TABLET'; TABLETS: 'TABLETS'; TAG: 'TAG'; +TARGET: 'TARGET'; TASK: 'TASK'; TASKS: 'TASKS'; TDE: 'TDE'; diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index b300b6185c8730..98142d8e4f7a78 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -334,8 +334,11 @@ supportedAlterStatement | ALTER SYSTEM RENAME COMPUTE GROUP name=identifier newName=identifier #alterSystemRenameComputeGroup | ALTER RESOURCE name=identifierOrText properties=propertyClause? #alterResource | ALTER REPOSITORY name=identifier properties=propertyClause? #alterRepository - | ALTER ROUTINE LOAD FOR name=multipartIdentifier - (loadProperty (COMMA loadProperty)*)? + | ALTER ROUTINE LOAD FOR name=multipartIdentifier + ( + SET TARGET TABLE EQ targetTable=STRING_LITERAL + | loadProperty (COMMA loadProperty)* + )? properties=propertyClause? (FROM type=identifier LEFT_PAREN propertyItemList RIGHT_PAREN)? #alterRoutineLoad | ALTER COLOCATE GROUP name=multipartIdentifier @@ -2356,6 +2359,7 @@ nonReserved | SUM | TABLES | TAG + | TARGET | TASK | TASKS | TDE diff --git a/regression-test/data/load_p0/routine_load/test_routine_load_alter.out b/regression-test/data/load_p0/routine_load/test_routine_load_alter.out index 427cdb2439463d..0764212aa7ae7d 100644 --- a/regression-test/data/load_p0/routine_load/test_routine_load_alter.out +++ b/regression-test/data/load_p0/routine_load/test_routine_load_alter.out @@ -10,3 +10,12 @@ 3 eab 2023-07-15 def 2023-07-20T05:48:31 "ghi" 3 eab 2023-07-15 def 2023-07-20T05:48:31 "ghi" +-- !sql_alter_target_src -- +1 eab +2 eab +3 eab + +-- !sql_alter_target_dst -- +4 eab +5 eab +6 eab diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy index 32571b5e29abd8..8c15185817e343 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy @@ -16,6 +16,8 @@ // under the License. import org.apache.kafka.clients.admin.AdminClient +import org.apache.kafka.clients.admin.DeleteTopicsOptions +import org.apache.kafka.clients.admin.NewTopic import org.apache.kafka.clients.producer.KafkaProducer import org.apache.kafka.clients.producer.ProducerRecord import org.apache.kafka.clients.producer.ProducerConfig @@ -293,5 +295,157 @@ suite("test_routine_load_alter","p0") { sql "stop routine load for ${jobName}" sql "truncate table ${tableName}" } + + // Test target-only switching and reject a target incompatible with the existing load description. + def srcTableName = "test_routine_load_alter_src" + def dstTableName = "test_routine_load_alter_dst" + def invalidDstTableName = "test_routine_load_alter_invalid_dst" + def alterTargetTopic = "test_routine_load_alter_target_table_${System.currentTimeMillis()}" + def alterTargetJob = "test_alter_target_table_${System.currentTimeMillis()}" + def alterTopicProducer = null + def alterTopicAdmin = null + def waitForRows = { String targetTable, long expectedRows -> + long actualRows = 0 + for (int attempt = 0; attempt < 120; attempt++) { + def countResult = sql "select count(*) from ${targetTable}" + actualRows = (countResult[0][0] as Number).longValue() + if (actualRows >= expectedRows) { + return actualRows + } + sleep(1000) + } + assertTrue(false, "timeout waiting for ${expectedRows} rows in ${targetTable}; actual=${actualRows}") + } + try { + sql "DROP TABLE IF EXISTS ${srcTableName}" + sql "DROP TABLE IF EXISTS ${dstTableName}" + sql "DROP TABLE IF EXISTS ${invalidDstTableName}" + for (String targetTable in [srcTableName, dstTableName]) { + sql """ + CREATE TABLE ${targetTable} ( + `k1` int NULL, + `k2` string NULL, + `v1` date NULL, + `v2` string NULL, + `v3` datetime NULL, + `v4` string NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ("replication_allocation" = "tag.location.default: 1") + """ + } + sql """ + CREATE TABLE ${invalidDstTableName} ( + `k1` int NULL, + `k2` string NULL, + `v1` date NULL, + `v2` string NULL, + `v3` datetime NULL, + `v4` string NULL, + `required_col` int NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ("replication_allocation" = "tag.location.default: 1") + """ + + def topicProps = new Properties() + topicProps.put("bootstrap.servers", kafka_broker.toString()) + alterTopicAdmin = AdminClient.create(topicProps) + alterTopicAdmin.createTopics([new NewTopic(alterTargetTopic, 1, (short) 1)]).all().get() + + def producerProps = new Properties() + producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka_broker.toString()) + producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.StringSerializer") + producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.StringSerializer") + producerProps.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + producerProps.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + alterTopicProducer = new KafkaProducer<>(producerProps) + + new File("${context.file.parent}/data/test_routine_load_alter.csv").readLines().each { line -> + alterTopicProducer.send(new ProducerRecord<>(alterTargetTopic, null, line)).get() + } + alterTopicProducer.flush() + + sql """ + CREATE ROUTINE LOAD ${alterTargetJob} ON ${srcTableName} + COLUMNS TERMINATED BY ",", + COLUMNS(k1, k2, v1, v2, v3, v4) + PROPERTIES + ( + "max_batch_interval" = "5", + "max_batch_rows" = "300000", + "max_batch_size" = "209715200" + ) + FROM KAFKA + ( + "kafka_broker_list" = "${kafka_broker}", + "kafka_topic" = "${alterTargetTopic}", + "kafka_partitions" = "0", + "kafka_offsets" = "OFFSET_BEGINNING" + ) + """ + assertEquals(3L, waitForRows(srcTableName, 3L)) + + sql "PAUSE ROUTINE LOAD FOR ${alterTargetJob}" + def showBeforeAlter = sql "SHOW ROUTINE LOAD FOR ${alterTargetJob}" + assertEquals(srcTableName, showBeforeAlter[0][6].toString()) + def progressBeforeAlter = showBeforeAlter[0][15].toString() + + test { + sql """ + ALTER ROUTINE LOAD FOR ${alterTargetJob} + SET TARGET TABLE = "${invalidDstTableName}" + """ + exception "Column has no default value, column=required_col" + } + def showAfterFailedAlter = sql "SHOW ROUTINE LOAD FOR ${alterTargetJob}" + assertEquals(srcTableName, showAfterFailedAlter[0][6].toString()) + assertEquals(progressBeforeAlter, showAfterFailedAlter[0][15].toString()) + + sql """ + ALTER ROUTINE LOAD FOR ${alterTargetJob} + SET TARGET TABLE = "${dstTableName}" + """ + def showAfterAlter = sql "SHOW ROUTINE LOAD FOR ${alterTargetJob}" + assertEquals(dstTableName, showAfterAlter[0][6].toString()) + assertEquals(progressBeforeAlter, showAfterAlter[0][15].toString()) + + [ + "4,eab,2023-07-16,def,2023-07-21:05:48:31,ghi", + "5,eab,2023-07-17,def,2023-07-22:05:48:31,ghi", + "6,eab,2023-07-18,def,2023-07-23:05:48:31,ghi" + ].each { line -> + alterTopicProducer.send(new ProducerRecord<>(alterTargetTopic, null, line)).get() + } + alterTopicProducer.flush() + sql "RESUME ROUTINE LOAD FOR ${alterTargetJob}" + waitForRows(dstTableName, 3L) + sleep(5000) + + qt_sql_alter_target_src "select k1, k2 from ${srcTableName} order by k1" + qt_sql_alter_target_dst "select k1, k2 from ${dstTableName} order by k1" + } finally { + try { + sql "STOP ROUTINE LOAD FOR ${alterTargetJob}" + } catch (Exception e) { + logger.warn("failed to stop alter target routine load: ${e.message}".toString()) + } + if (alterTopicProducer != null) { + alterTopicProducer.close() + } + if (alterTopicAdmin != null) { + try { + alterTopicAdmin.deleteTopics([alterTargetTopic.toString()], + new DeleteTopicsOptions().timeoutMs(10000)).all().get() + } catch (Exception e) { + logger.warn("failed to delete kafka topic ${alterTargetTopic}: ${e.message}".toString()) + } + alterTopicAdmin.close() + } + } } }