From a32fdec6afaf50df45bfeb922f025d7057cce853 Mon Sep 17 00:00:00 2001 From: Refrain Date: Tue, 11 Aug 2026 10:20:50 +0800 Subject: [PATCH 1/4] [fix](routineload) Persist the current load definition ### What problem does this PR solve? Issue Number: N/A Related PR: #64878 Problem Summary: Routine Load image recovery reparsed the immutable original CREATE statement, so CREATE semantics changed by ALTER were not represented in the image. Persist the current load definition, retain the original statement as the legacy-image fallback, journal altered load clauses, and validate failure-prone Kafka and Kinesis changes before mutating runtime state. ### Release note Routine Load jobs now recover the current effective load definition after ALTER. ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.load.routineload.KafkaRoutineLoadJobTest,org.apache.doris.load.routineload.KinesisRoutineLoadJobTest,org.apache.doris.persist.AlterRoutineLoadOperationLogTest - Behavior changed: Yes. Image recovery uses the current effective Routine Load definition while legacy images continue to use origStmt. - Does this need documentation: No --- .../org/apache/doris/analysis/Separator.java | 3 + .../apache/doris/load/RoutineLoadDesc.java | 10 + .../routineload/RoutineLoadDefinition.java | 128 ++++++++++++ .../load/routineload/RoutineLoadJob.java | 137 ++++++++++--- .../load/routineload/RoutineLoadManager.java | 1 - .../kafka/KafkaRoutineLoadJob.java | 186 +++++++++++------ .../kinesis/KinesisRoutineLoadJob.java | 187 +++++++++++------ .../AlterRoutineLoadJobOperationLog.java | 13 ++ .../routineload/KafkaRoutineLoadJobTest.java | 188 ++++++++++++++++++ .../KinesisRoutineLoadJobTest.java | 48 +++++ .../AlterRoutineLoadOperationLogTest.java | 24 ++- 11 files changed, 773 insertions(+), 152 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadDefinition.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java index 67515eaca5c79f..7da2e092a212ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java @@ -20,13 +20,16 @@ import org.apache.doris.common.AnalysisException; import com.google.common.base.Strings; +import com.google.gson.annotations.SerializedName; import java.io.StringWriter; public class Separator { private static final String HEX_STRING = "0123456789ABCDEF"; + @SerializedName("os") private final String oriSeparator; + @SerializedName("s") private String separator; public Separator(String separator) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java b/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java index 2c1ede0d13a352..28a429960d2b71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java @@ -26,19 +26,29 @@ import org.apache.doris.load.loadv2.LoadTask; import com.google.common.base.Strings; +import com.google.gson.annotations.SerializedName; import java.util.List; public class RoutineLoadDesc { + @SerializedName("cs") private final Separator columnSeparator; + @SerializedName("ld") private final Separator lineDelimiter; + @SerializedName("cols") private final List columnsInfo; + @SerializedName("pf") private final Expr precedingFilter; + @SerializedName("f") private final Expr filter; + @SerializedName("dc") private final Expr deleteCondition; + @SerializedName("mt") private LoadTask.MergeType mergeType; // nullable + @SerializedName("pn") private final PartitionNamesInfo partitionNamesInfo; + @SerializedName("sc") private final String sequenceColName; public RoutineLoadDesc(Separator columnSeparator, Separator lineDelimiter, List columnsInfo, diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadDefinition.java new file mode 100644 index 00000000000000..bcfb59c48d90f4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadDefinition.java @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.load.routineload; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.ExprToSqlVisitor; +import org.apache.doris.analysis.ImportColumnDesc; +import org.apache.doris.analysis.ToSqlParams; +import org.apache.doris.common.UserException; +import org.apache.doris.load.RoutineLoadDesc; +import org.apache.doris.load.loadv2.LoadTask; +import org.apache.doris.nereids.load.NereidsLoadUtils; +import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; +import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo; +import org.apache.doris.nereids.trees.plans.commands.load.LoadColumnClause; +import org.apache.doris.nereids.trees.plans.commands.load.LoadColumnDesc; +import org.apache.doris.nereids.trees.plans.commands.load.LoadDeleteOnClause; +import org.apache.doris.nereids.trees.plans.commands.load.LoadPartitionNames; +import org.apache.doris.nereids.trees.plans.commands.load.LoadPrecedingFilterClause; +import org.apache.doris.nereids.trees.plans.commands.load.LoadProperty; +import org.apache.doris.nereids.trees.plans.commands.load.LoadSeparator; +import org.apache.doris.nereids.trees.plans.commands.load.LoadSequenceClause; +import org.apache.doris.nereids.trees.plans.commands.load.LoadWhereClause; +import org.apache.doris.nereids.trees.expressions.Expression; + +import com.google.common.collect.Maps; +import com.google.gson.annotations.SerializedName; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Snapshot of the current CREATE ROUTINE LOAD semantics. + */ +public class RoutineLoadDefinition { + @SerializedName("desc") + private RoutineLoadDesc routineLoadDesc; + @SerializedName("jp") + private Map jobProperties = Maps.newHashMap(); + @SerializedName("dsp") + private Map dataSourceProperties = Maps.newHashMap(); + + public RoutineLoadDefinition(RoutineLoadDesc routineLoadDesc, + Map jobProperties, Map dataSourceProperties) { + this.routineLoadDesc = routineLoadDesc; + this.jobProperties.putAll(jobProperties); + this.dataSourceProperties.putAll(dataSourceProperties); + } + + public RoutineLoadDesc getRoutineLoadDesc() { + return routineLoadDesc; + } + + public Map getDataSourceProperties() { + return dataSourceProperties; + } + + public CreateRoutineLoadInfo toCreateInfo(String dbName, String jobName, String tableName, + LoadDataSourceType dataSourceType, String comment) throws UserException { + LoadTask.MergeType mergeType = routineLoadDesc == null + ? LoadTask.MergeType.APPEND : routineLoadDesc.getMergeType(); + return new CreateRoutineLoadInfo(new LabelNameInfo(dbName, jobName), tableName, + toLoadPropertyMap(routineLoadDesc), Maps.newHashMap(jobProperties), dataSourceType.name(), + Maps.newHashMap(dataSourceProperties), mergeType, comment); + } + + private static Map toLoadPropertyMap(RoutineLoadDesc routineLoadDesc) throws UserException { + Map loadProperties = Maps.newHashMap(); + if (routineLoadDesc == null) { + return loadProperties; + } + if (routineLoadDesc.getColumnSeparator() != null) { + put(loadProperties, new LoadSeparator(routineLoadDesc.getColumnSeparator().getOriSeparator())); + } + if (routineLoadDesc.getColumnsInfo() != null) { + List columns = new ArrayList<>(); + for (ImportColumnDesc column : routineLoadDesc.getColumnsInfo()) { + Expression expression = column.getExpr() == null ? null : parseExpression(column.getExpr()); + columns.add(new LoadColumnDesc(column.getColumnName(), expression)); + } + put(loadProperties, new LoadColumnClause(columns)); + } + if (routineLoadDesc.getPrecedingFilter() != null) { + put(loadProperties, new LoadPrecedingFilterClause( + parseExpression(routineLoadDesc.getPrecedingFilter()))); + } + if (routineLoadDesc.getFilter() != null) { + put(loadProperties, new LoadWhereClause(parseExpression(routineLoadDesc.getFilter()))); + } + if (routineLoadDesc.getPartitionNamesInfo() != null) { + put(loadProperties, new LoadPartitionNames( + routineLoadDesc.getPartitionNamesInfo().isTemp(), + routineLoadDesc.getPartitionNamesInfo().getPartitionNames())); + } + if (routineLoadDesc.getDeleteCondition() != null) { + put(loadProperties, new LoadDeleteOnClause(parseExpression(routineLoadDesc.getDeleteCondition()))); + } + if (routineLoadDesc.hasSequenceCol()) { + put(loadProperties, new LoadSequenceClause(routineLoadDesc.getSequenceColName())); + } + return loadProperties; + } + + private static Expression parseExpression(Expr expression) throws UserException { + String sql = expression.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); + return NereidsLoadUtils.parseExpressionSeq(sql).get(0); + } + + private static void put(Map loadProperties, LoadProperty loadProperty) { + loadProperties.put(loadProperty.getClass().getName(), loadProperty); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 9873368f405114..4229aca376353e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ExprToSqlVisitor; +import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.Separator; import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.analysis.UserIdentity; @@ -258,8 +259,11 @@ public boolean isFinalState() { // The tasks belong to this job protected List routineLoadTaskInfoList = Lists.newArrayList(); - // this is the origin stmt of CreateRoutineLoadStmt, we use it to persist the RoutineLoadJob, - // because we can not serialize the Expressions contained in job. + // Canonical current CREATE semantics. CREATE and ALTER must keep this snapshot current. + @SerializedName("ld") + protected RoutineLoadDefinition loadDefinition; + + // Legacy recovery input for images written before loadDefinition was persisted. @SerializedName("ostmt") protected OriginStatement origStmt; // User who submit this job. Maybe null for the old version job(before v1.1) @@ -470,6 +474,62 @@ protected void setRoutineLoadDesc(RoutineLoadDesc routineLoadDesc) { } } + protected RoutineLoadDesc getLoadDefinitionRoutineLoadDesc() { + List columnsInfo = columnDescs == null ? null : columnDescs.descs; + return new RoutineLoadDesc(columnSeparator, lineDelimiter, columnsInfo, precedingFilter, whereExpr, + partitionNamesInfo, deleteCondition, mergeType, sequenceCol); + } + + protected void initializeLoadDefinition(CreateRoutineLoadInfo info) { + Map originalDataSourceProperties = + info.getDataSourceProperties().getOriginalDataSourceProperties(); + Map dataSourceProperties = originalDataSourceProperties == null + ? Maps.newHashMap() : Maps.newHashMap(originalDataSourceProperties); + updateLoadDefinitionDataSourceProperties(dataSourceProperties); + loadDefinition = new RoutineLoadDefinition( + getLoadDefinitionRoutineLoadDesc(), snapshotLoadDefinitionJobProperties(), dataSourceProperties); + } + + protected void updateLoadDefinition(AbstractDataSourceProperties changedDataSourceProperties) { + Map dataSourceProperties = loadDefinition == null + ? Maps.newHashMap() : Maps.newHashMap(loadDefinition.getDataSourceProperties()); + if (changedDataSourceProperties != null + && changedDataSourceProperties.getOriginalDataSourceProperties() != null) { + dataSourceProperties.putAll(changedDataSourceProperties.getOriginalDataSourceProperties()); + } + updateLoadDefinitionDataSourceProperties(dataSourceProperties); + loadDefinition = new RoutineLoadDefinition( + getLoadDefinitionRoutineLoadDesc(), snapshotLoadDefinitionJobProperties(), dataSourceProperties); + } + + protected Map snapshotLoadDefinitionJobProperties() { + Map currentJobProperties = Maps.newHashMap(jobProperties); + currentJobProperties.put(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY, + String.valueOf(desireTaskConcurrentNum)); + currentJobProperties.put(CreateRoutineLoadInfo.MAX_ERROR_NUMBER_PROPERTY, String.valueOf(maxErrorNum)); + currentJobProperties.put(CreateRoutineLoadInfo.MAX_FILTER_RATIO_PROPERTY, String.valueOf(maxFilterRatio)); + currentJobProperties.put(CreateRoutineLoadInfo.MAX_BATCH_INTERVAL_SEC_PROPERTY, + String.valueOf(maxBatchIntervalS)); + currentJobProperties.put(CreateRoutineLoadInfo.MAX_BATCH_ROWS_PROPERTY, String.valueOf(maxBatchRows)); + currentJobProperties.put(CreateRoutineLoadInfo.MAX_BATCH_SIZE_PROPERTY, String.valueOf(maxBatchSizeBytes)); + currentJobProperties.put(CreateRoutineLoadInfo.EXEC_MEM_LIMIT_PROPERTY, String.valueOf(execMemLimit)); + currentJobProperties.put(CreateRoutineLoadInfo.SEND_BATCH_PARALLELISM, + String.valueOf(sendBatchParallelism)); + currentJobProperties.put(CreateRoutineLoadInfo.LOAD_TO_SINGLE_TABLET, + String.valueOf(loadToSingleTablet)); + currentJobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); + currentJobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); + if (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { + currentJobProperties.remove(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); + } else { + currentJobProperties.put(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY, + partialUpdateNewKeyPolicy.name()); + } + return currentJobProperties; + } + + protected abstract void updateLoadDefinitionDataSourceProperties(Map dataSourceProperties); + @Override public long getId() { return id; @@ -2009,45 +2069,67 @@ public void gsonPostProcess() throws IOException { ctx.getState().reset(); try { ctx.setThreadLocalInfo(); - NereidsParser nereidsParser = new NereidsParser(); - CreateRoutineLoadCommand command = (CreateRoutineLoadCommand) nereidsParser.parseSingle( - origStmt.originStmt); - CreateRoutineLoadInfo createRoutineLoadInfo = command.getCreateRoutineLoadInfo(); - // If tableId is set, resolve the current table name by ID so that - // table rename / SWAP TABLE won't cause replay to fail with stale name in origStmt. - if (!isMultiTable && tableId != 0) { - try { - Database db = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).orElse(null); - if (db != null) { - db.getTable(tableId).ifPresent( - table -> createRoutineLoadInfo.setTableName(table.getName())); - } - } catch (Exception ignored) { - // fall through; let validate() surface the real error - } + if (loadDefinition == null) { + restoreLegacyDefinition(ctx); + } else { + restoreLoadDefinition(ctx); } - createRoutineLoadInfo.validate(ctx); - setRoutineLoadDesc(createRoutineLoadInfo.getRoutineLoadDesc()); } finally { ctx.cleanup(); } } catch (Exception e) { this.state = JobState.CANCELLED; - LOG.warn("error happens when parsing create routine load stmt: " + origStmt.originStmt, e); + LOG.warn("error happens when restoring routine load definition", e); } if (userIdentity != null) { userIdentity.setIsAnalyzed(); } } + private void restoreLegacyDefinition(ConnectContext ctx) throws UserException { + NereidsParser nereidsParser = new NereidsParser(); + CreateRoutineLoadCommand command = (CreateRoutineLoadCommand) nereidsParser.parseSingle( + origStmt.originStmt); + CreateRoutineLoadInfo createRoutineLoadInfo = command.getCreateRoutineLoadInfo(); + if (!isMultiTable) { + Database db = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get(); + createRoutineLoadInfo.setTableName(db.getTable(tableId).get().getName()); + } + createRoutineLoadInfo.validate(ctx); + setRoutineLoadDesc(createRoutineLoadInfo.getRoutineLoadDesc()); + } + + private void restoreLoadDefinition(ConnectContext ctx) throws UserException { + Database db = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get(); + String tableName = isMultiTable ? null : db.getTable(tableId).get().getName(); + CreateRoutineLoadInfo createRoutineLoadInfo = loadDefinition.toCreateInfo( + db.getFullName(), name, tableName, dataSourceType, comment); + createRoutineLoadInfo.validate(ctx); + setRoutineLoadDesc(loadDefinition.getRoutineLoadDesc()); + } + public abstract void modifyProperties(AlterRoutineLoadCommand command) throws UserException; public abstract void replayModifyProperties(AlterRoutineLoadJobOperationLog log); public abstract NereidsRoutineLoadTaskInfo toNereidsRoutineLoadTaskInfo() throws UserException; - // for ALTER ROUTINE LOAD - protected void modifyCommonJobProperties(Map jobProperties) throws UserException { + protected TUniqueKeyUpdateMode validateCommonJobProperties(Map jobProperties) + throws UserException { + if (!jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { + return null; + } + TUniqueKeyUpdateMode newMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode( + jobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)); + if (newMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS) { + validateFlexiblePartialUpdateForAlter(); + } + return newMode; + } + + // for ALTER ROUTINE LOAD. All failure-prone validation must be completed before calling this method. + protected void modifyCommonJobProperties(Map jobProperties, + TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode) { if (jobProperties.containsKey(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)) { this.desireTaskConcurrentNum = Integer.parseInt( jobProperties.remove(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)); @@ -2080,13 +2162,8 @@ protected void modifyCommonJobProperties(Map jobProperties) thro } if (jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { - String modeStr = jobProperties.remove(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); - TUniqueKeyUpdateMode newMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode(modeStr); - // Validate flexible partial update constraints when changing to UPDATE_FLEXIBLE_COLUMNS - if (newMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS) { - validateFlexiblePartialUpdateForAlter(); - } - this.uniqueKeyUpdateMode = newMode; + jobProperties.remove(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); + this.uniqueKeyUpdateMode = validatedUniqueKeyUpdateMode; this.isPartialUpdate = (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS); this.jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java index df5615016bfa54..9bc9b93abd8c98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java @@ -943,7 +943,6 @@ public void alterRoutineLoadJob(AlterRoutineLoadCommand command) throws UserExce + command.getDataSourceProperties().getDataSourceType()); } job.modifyProperties(command); - job.setRoutineLoadDesc(command.getRoutineLoadDesc()); } public void replayAlterRoutineLoadJob(AlterRoutineLoadJobOperationLog log) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java index 885021440351d7..a5d1dd5f0e7ef7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java @@ -61,6 +61,7 @@ import org.apache.doris.service.FrontendOptions; import org.apache.doris.thrift.TFileCompressType; import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; +import org.apache.doris.thrift.TUniqueKeyUpdateMode; import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; @@ -216,19 +217,25 @@ private void convertCustomProperties(boolean rebuild) throws DdlException { return; } - if (rebuild) { - convertedCustomProperties.clear(); - } + Pair, String> convertedProperties = buildConvertedCustomProperties( + customProperties, kafkaDefaultOffSet); + convertedCustomProperties.clear(); + convertedCustomProperties.putAll(convertedProperties.first); + kafkaDefaultOffSet = convertedProperties.second; + } - SmallFileMgr smallFileMgr = Env.getCurrentEnv().getSmallFileMgr(); - for (Map.Entry entry : customProperties.entrySet()) { + private Pair, String> buildConvertedCustomProperties( + Map sourceProperties, String currentDefaultOffset) throws DdlException { + Map convertedProperties = Maps.newHashMap(); + for (Map.Entry entry : sourceProperties.entrySet()) { if (entry.getValue().startsWith("FILE:")) { // convert FILE:file_name -> FILE:file_id:md5 String file = entry.getValue().substring(entry.getValue().indexOf(":") + 1); + SmallFileMgr smallFileMgr = Env.getCurrentEnv().getSmallFileMgr(); SmallFile smallFile = smallFileMgr.getSmallFile(dbId, KAFKA_FILE_CATALOG, file, true); - convertedCustomProperties.put(entry.getKey(), "FILE:" + smallFile.id + ":" + smallFile.md5); + convertedProperties.put(entry.getKey(), "FILE:" + smallFile.id + ":" + smallFile.md5); } else { - convertedCustomProperties.put(entry.getKey(), entry.getValue()); + convertedProperties.put(entry.getKey(), entry.getValue()); } } @@ -237,14 +244,14 @@ private void convertCustomProperties(boolean rebuild) throws DdlException { // KAFKA_DEFAULT_OFFSETS, and this attribute will be converted into a timestamp during the analyzing phase, // thus losing some information. So we use KAFKA_ORIGIN_DEFAULT_OFFSETS to store the original datetime // formatted KAFKA_DEFAULT_OFFSETS value - if (convertedCustomProperties.containsKey(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())) { - kafkaDefaultOffSet = convertedCustomProperties + String convertedDefaultOffset = currentDefaultOffset; + if (convertedProperties.containsKey(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())) { + convertedDefaultOffset = convertedProperties .remove(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName()); - return; - } - if (convertedCustomProperties.containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName())) { - kafkaDefaultOffSet = convertedCustomProperties.remove(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); + } else if (convertedProperties.containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName())) { + convertedDefaultOffset = convertedProperties.remove(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); } + return Pair.of(convertedProperties, convertedDefaultOffset); } @Override @@ -590,6 +597,7 @@ public static KafkaRoutineLoadJob fromCreateInfo(CreateRoutineLoadInfo info, Con kafkaRoutineLoadJob.setOptional(info); kafkaRoutineLoadJob.checkCustomProperties(); kafkaRoutineLoadJob.checkCustomPartition(); + kafkaRoutineLoadJob.initializeLoadDefinition(info); return kafkaRoutineLoadJob; } @@ -775,6 +783,21 @@ public Map getCustomProperties() { return getMaskedCustomProperties("property."); } + @Override + protected void updateLoadDefinitionDataSourceProperties(Map dataSourceProperties) { + dataSourceProperties.put(KafkaConfiguration.KAFKA_BROKER_LIST.getName(), brokerList); + dataSourceProperties.put(KafkaConfiguration.KAFKA_TOPIC.getName(), topic); + dataSourceProperties.remove(KafkaConfiguration.KAFKA_OFFSETS.getName()); + if (customKafkaPartitions.isEmpty()) { + dataSourceProperties.remove(KafkaConfiguration.KAFKA_PARTITIONS.getName()); + } else { + dataSourceProperties.put(KafkaConfiguration.KAFKA_PARTITIONS.getName(), + Joiner.on(",").join(customKafkaPartitions)); + } + customProperties.forEach((key, value) -> dataSourceProperties.put( + key.startsWith("aws.") ? key : "property." + key, value)); + } + @Override public void modifyProperties(AlterRoutineLoadCommand command) throws UserException { Map jobProperties = command.getAnalyzedJobProperties(); @@ -791,9 +814,11 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti } modifyPropertiesInternal(jobProperties, dataSourceProperties); + setRoutineLoadDesc(command.getRoutineLoadDesc()); + updateLoadDefinition(dataSourceProperties); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, - jobProperties, dataSourceProperties); + jobProperties, dataSourceProperties, command.getRoutineLoadDesc()); Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); } finally { writeUnlock(); @@ -820,66 +845,89 @@ private void convertOffset(KafkaDataSourceProperties dataSourceProperties) throw private void modifyPropertiesInternal(Map jobProperties, KafkaDataSourceProperties dataSourceProperties) throws UserException { - if (null != dataSourceProperties) { - List> kafkaPartitionOffsets = Lists.newArrayList(); - Map customKafkaProperties = Maps.newHashMap(); + PreparedKafkaAlter preparedAlter = prepareAlter(jobProperties, dataSourceProperties); + applyAlter(jobProperties, dataSourceProperties, preparedAlter); + if (LOG.isDebugEnabled()) { + LOG.debug("modify the properties of kafka routine load job: {}, jobProperties: {}, " + + "datasource properties: {}", + this.id, jobProperties, dataSourceProperties); + } + } - if (MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { - kafkaPartitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); - customKafkaProperties = dataSourceProperties.getCustomKafkaProperties(); - } + private PreparedKafkaAlter prepareAlter(Map jobProperties, + KafkaDataSourceProperties dataSourceProperties) throws UserException { + TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode = validateCommonJobProperties(jobProperties); + List> kafkaPartitionOffsets = Lists.newArrayList(); + Map alteredCustomProperties = Maps.newHashMap(); + if (dataSourceProperties != null + && MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { + kafkaPartitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); + alteredCustomProperties = dataSourceProperties.getCustomKafkaProperties(); + } - // convertCustomProperties and check partitions before reset progress to make modify operation atomic - if (!customKafkaProperties.isEmpty()) { - this.customProperties.putAll(customKafkaProperties); - convertCustomProperties(true); - } + Map stagedCustomProperties = null; + Map stagedConvertedCustomProperties = null; + String stagedKafkaDefaultOffset = kafkaDefaultOffSet; + if (!alteredCustomProperties.isEmpty()) { + stagedCustomProperties = Maps.newHashMap(customProperties); + stagedCustomProperties.putAll(alteredCustomProperties); + Pair, String> convertedProperties = buildConvertedCustomProperties( + stagedCustomProperties, stagedKafkaDefaultOffset); + stagedConvertedCustomProperties = convertedProperties.first; + stagedKafkaDefaultOffset = convertedProperties.second; + } + if (!kafkaPartitionOffsets.isEmpty()) { + ((KafkaProgress) progress).checkPartitions(kafkaPartitionOffsets); + } + if (dataSourceProperties != null && Config.isCloudMode()) { + Cloud.ResetRLProgressRequest.Builder builder = Cloud.ResetRLProgressRequest.newBuilder() + .setRequestIp(FrontendOptions.getLocalHostAddressCached()); + builder.setCloudUniqueId(Config.cloud_unique_id); + builder.setDbId(dbId); + builder.setJobId(id); if (!kafkaPartitionOffsets.isEmpty()) { - ((KafkaProgress) progress).checkPartitions(kafkaPartitionOffsets); - } - - if (Config.isCloudMode()) { - Cloud.ResetRLProgressRequest.Builder builder = Cloud.ResetRLProgressRequest.newBuilder() - .setRequestIp(FrontendOptions.getLocalHostAddressCached()); - builder.setCloudUniqueId(Config.cloud_unique_id); - builder.setDbId(dbId); - builder.setJobId(id); - if (!kafkaPartitionOffsets.isEmpty()) { - Map partitionOffsetMap = new HashMap<>(); - for (Pair pair : kafkaPartitionOffsets) { - // The reason why the value recorded in MS in cloud mode needs to be subtracted by one is - // this value will be incremented - // when pulling MS persistent progress data and updating memory - // in routineLoadJob.updateCloudProgress(). - partitionOffsetMap.put(pair.first, pair.second - 1); - } - builder.putAllPartitionToOffset(partitionOffsetMap); + Map partitionOffsetMap = new HashMap<>(); + for (Pair pair : kafkaPartitionOffsets) { + // The reason why the value recorded in MS in cloud mode needs to be subtracted by one is + // this value will be incremented when pulling MS persistent progress data and updating memory + // in routineLoadJob.updateCloudProgress(). + partitionOffsetMap.put(pair.first, pair.second - 1); } - resetCloudProgress(builder); + builder.putAllPartitionToOffset(partitionOffsetMap); } + resetCloudProgress(builder); + } + return new PreparedKafkaAlter(validatedUniqueKeyUpdateMode, kafkaPartitionOffsets, + stagedCustomProperties, stagedConvertedCustomProperties, stagedKafkaDefaultOffset); + } + private void applyAlter(Map jobProperties, KafkaDataSourceProperties dataSourceProperties, + PreparedKafkaAlter preparedAlter) { + if (dataSourceProperties != null) { + if (preparedAlter.stagedCustomProperties != null) { + customProperties.clear(); + customProperties.putAll(preparedAlter.stagedCustomProperties); + convertedCustomProperties.clear(); + convertedCustomProperties.putAll(preparedAlter.stagedConvertedCustomProperties); + kafkaDefaultOffSet = preparedAlter.stagedKafkaDefaultOffset; + } // It is necessary to reset the Kafka progress cache if topic change, // and should reset cache before modifying partition offset. if (!Strings.isNullOrEmpty(dataSourceProperties.getTopic())) { - this.topic = dataSourceProperties.getTopic(); - this.progress = new KafkaProgress(); + topic = dataSourceProperties.getTopic(); + progress = new KafkaProgress(); } - - // modify partition offset - if (!kafkaPartitionOffsets.isEmpty()) { - // we can only modify the partition that is being consumed - ((KafkaProgress) progress).modifyOffset(kafkaPartitionOffsets); + if (!preparedAlter.kafkaPartitionOffsets.isEmpty()) { + ((KafkaProgress) progress).modifyOffset(preparedAlter.kafkaPartitionOffsets); } - - // modify broker list if (!Strings.isNullOrEmpty(dataSourceProperties.getBrokerList())) { - this.brokerList = dataSourceProperties.getBrokerList(); + brokerList = dataSourceProperties.getBrokerList(); } } if (!jobProperties.isEmpty()) { Map copiedJobProperties = Maps.newHashMap(jobProperties); - modifyCommonJobProperties(copiedJobProperties); + modifyCommonJobProperties(copiedJobProperties, preparedAlter.validatedUniqueKeyUpdateMode); this.jobProperties.putAll(copiedJobProperties); if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { this.isPartialUpdate = BooleanUtils.toBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); @@ -893,8 +941,26 @@ private void modifyPropertiesInternal(Map jobProperties, } } } - LOG.info("modify the properties of kafka routine load job: {}, jobProperties: {}, datasource properties: {}", - this.id, jobProperties, dataSourceProperties); + } + + private static class PreparedKafkaAlter { + private final TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode; + private final List> kafkaPartitionOffsets; + private final Map stagedCustomProperties; + private final Map stagedConvertedCustomProperties; + private final String stagedKafkaDefaultOffset; + + private PreparedKafkaAlter(TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode, + List> kafkaPartitionOffsets, + Map stagedCustomProperties, + Map stagedConvertedCustomProperties, + String stagedKafkaDefaultOffset) { + this.validatedUniqueKeyUpdateMode = validatedUniqueKeyUpdateMode; + this.kafkaPartitionOffsets = kafkaPartitionOffsets; + this.stagedCustomProperties = stagedCustomProperties; + this.stagedConvertedCustomProperties = stagedConvertedCustomProperties; + this.stagedKafkaDefaultOffset = stagedKafkaDefaultOffset; + } } private void resetCloudProgress(Cloud.ResetRLProgressRequest.Builder builder) throws DdlException { @@ -920,6 +986,8 @@ private void resetCloudProgress(Cloud.ResetRLProgressRequest.Builder builder) th public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { try { modifyPropertiesInternal(log.getJobProperties(), (KafkaDataSourceProperties) log.getDataSourceProperties()); + setRoutineLoadDesc(log.getRoutineLoadDesc()); + updateLoadDefinition(log.getDataSourceProperties()); } catch (UserException e) { // should not happen LOG.error("failed to replay modify kafka routine load job: {}", id, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index 7cebc3f5165b49..5ca8355d851c50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -52,6 +52,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TFileCompressType; import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; +import org.apache.doris.thrift.TUniqueKeyUpdateMode; import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; @@ -186,19 +187,20 @@ private void convertCustomProperties(boolean rebuild) throws DdlException { return; } - if (rebuild) { - convertedCustomProperties.clear(); - } - - for (Map.Entry entry : customProperties.entrySet()) { - convertedCustomProperties.put(entry.getKey(), entry.getValue()); - } + Pair, String> convertedProperties = buildConvertedCustomProperties( + customProperties, kinesisDefaultPosition); + convertedCustomProperties.clear(); + convertedCustomProperties.putAll(convertedProperties.first); + kinesisDefaultPosition = convertedProperties.second; + } - // Handle default position - if (convertedCustomProperties.containsKey("kinesis_default_pos")) { - kinesisDefaultPosition = convertedCustomProperties.get("kinesis_default_pos"); - // Keep it in convertedCustomProperties so BE can use it - } + private Pair, String> buildConvertedCustomProperties( + Map sourceProperties, String currentDefaultPosition) { + Map convertedProperties = Maps.newHashMap(sourceProperties); + String convertedDefaultPosition = convertedProperties.getOrDefault( + "kinesis_default_pos", currentDefaultPosition); + // Keep kinesis_default_pos in convertedProperties so BE can use it. + return Pair.of(convertedProperties, convertedDefaultPosition); } private String convertedDefaultPosition() { @@ -532,6 +534,7 @@ public static KinesisRoutineLoadJob fromCreateInfo(CreateRoutineLoadInfo info, C kinesisRoutineLoadJob.setOptional(info); kinesisRoutineLoadJob.checkCustomProperties(); + kinesisRoutineLoadJob.initializeLoadDefinition(info); return kinesisRoutineLoadJob; } @@ -660,6 +663,27 @@ public Map getCustomProperties() { return getMaskedCustomProperties("property."); } + @Override + protected void updateLoadDefinitionDataSourceProperties(Map dataSourceProperties) { + dataSourceProperties.put(KinesisConfiguration.KINESIS_REGION.getName(), region); + dataSourceProperties.put(KinesisConfiguration.KINESIS_STREAM.getName(), stream); + if (endpoint == null) { + dataSourceProperties.remove(KinesisConfiguration.KINESIS_ENDPOINT.getName()); + dataSourceProperties.remove("kinesis_endpoint"); + } else { + dataSourceProperties.put(KinesisConfiguration.KINESIS_ENDPOINT.getName(), endpoint); + } + dataSourceProperties.remove(KinesisConfiguration.KINESIS_POSITIONS.getName()); + if (customKinesisShards.isEmpty()) { + dataSourceProperties.remove(KinesisConfiguration.KINESIS_SHARDS.getName()); + } else { + dataSourceProperties.put(KinesisConfiguration.KINESIS_SHARDS.getName(), + Joiner.on(",").join(customKinesisShards)); + } + customProperties.forEach((key, value) -> dataSourceProperties.put( + key.startsWith("aws.") ? key : "property." + key, value)); + } + private Map getMaskedCustomProperties(String keyPrefix) { Map maskedProperties = new HashMap<>(); customProperties.forEach((key, value) -> { @@ -687,9 +711,11 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti } modifyPropertiesInternal(jobProperties, dataSourceProperties); + setRoutineLoadDesc(command.getRoutineLoadDesc()); + updateLoadDefinition(dataSourceProperties); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, - jobProperties, dataSourceProperties); + jobProperties, dataSourceProperties, command.getRoutineLoadDesc()); Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); } finally { writeUnlock(); @@ -699,68 +725,85 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti private void modifyPropertiesInternal(Map jobProperties, KinesisDataSourceProperties dataSourceProperties) throws UserException { - if (dataSourceProperties != null) { - List> shardPositions = Lists.newArrayList(); - Map customKinesisProperties = Maps.newHashMap(); - boolean resetProgress = false; - boolean hasExplicitShardPositions = false; - - if (MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { - shardPositions = dataSourceProperties.getKinesisShardPositions(); - customKinesisProperties = dataSourceProperties.getCustomKinesisProperties(); - hasExplicitShardPositions = !shardPositions.isEmpty(); - } + PreparedKinesisAlter preparedAlter = prepareAlter(jobProperties, dataSourceProperties); + applyAlter(jobProperties, dataSourceProperties, preparedAlter); + LOG.info("modify the properties of kinesis routine load job: {}, jobProperties: {}, datasource properties: {}", + this.id, jobProperties, dataSourceProperties); + } - // Update custom properties - if (!customKinesisProperties.isEmpty()) { - this.customProperties.putAll(customKinesisProperties); - convertCustomProperties(true); - } + private PreparedKinesisAlter prepareAlter(Map jobProperties, + KinesisDataSourceProperties dataSourceProperties) throws UserException { + TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode = validateCommonJobProperties(jobProperties); + List> shardPositions = Lists.newArrayList(); + Map alteredCustomProperties = Maps.newHashMap(); + if (dataSourceProperties != null + && MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { + shardPositions = dataSourceProperties.getKinesisShardPositions(); + alteredCustomProperties = dataSourceProperties.getCustomKinesisProperties(); + } - // Modify stream if provided + Map stagedCustomProperties = null; + Map stagedConvertedCustomProperties = null; + String stagedDefaultPosition = kinesisDefaultPosition; + if (!alteredCustomProperties.isEmpty()) { + stagedCustomProperties = Maps.newHashMap(customProperties); + stagedCustomProperties.putAll(alteredCustomProperties); + Pair, String> convertedProperties = buildConvertedCustomProperties( + stagedCustomProperties, stagedDefaultPosition); + stagedConvertedCustomProperties = convertedProperties.first; + stagedDefaultPosition = convertedProperties.second; + } + + boolean resetProgress = dataSourceProperties != null + && !Strings.isNullOrEmpty(dataSourceProperties.getStream()); + if (!shardPositions.isEmpty() && !resetProgress) { + ((KinesisProgress) progress).checkShards(shardPositions); + } + return new PreparedKinesisAlter(validatedUniqueKeyUpdateMode, shardPositions, + stagedCustomProperties, stagedConvertedCustomProperties, stagedDefaultPosition, resetProgress); + } + + private void applyAlter(Map jobProperties, KinesisDataSourceProperties dataSourceProperties, + PreparedKinesisAlter preparedAlter) { + if (dataSourceProperties != null) { + if (preparedAlter.stagedCustomProperties != null) { + customProperties.clear(); + customProperties.putAll(preparedAlter.stagedCustomProperties); + convertedCustomProperties.clear(); + convertedCustomProperties.putAll(preparedAlter.stagedConvertedCustomProperties); + kinesisDefaultPosition = preparedAlter.stagedDefaultPosition; + } if (!Strings.isNullOrEmpty(dataSourceProperties.getStream())) { - this.stream = dataSourceProperties.getStream(); - resetProgress = true; + stream = dataSourceProperties.getStream(); } - - // Modify region if provided if (!Strings.isNullOrEmpty(dataSourceProperties.getRegion())) { - this.region = dataSourceProperties.getRegion(); + region = dataSourceProperties.getRegion(); } - - // Modify endpoint if provided if (!Strings.isNullOrEmpty(dataSourceProperties.getEndpoint())) { - this.endpoint = dataSourceProperties.getEndpoint(); + endpoint = dataSourceProperties.getEndpoint(); } - - if (resetProgress) { - this.progress = new KinesisProgress(); - this.openKinesisShards.clear(); - this.closedKinesisShards.clear(); - this.cachedShardWithMillsBehindLatest.clear(); + if (preparedAlter.resetProgress) { + progress = new KinesisProgress(); + openKinesisShards.clear(); + closedKinesisShards.clear(); + cachedShardWithMillsBehindLatest.clear(); } - - if (hasExplicitShardPositions) { - this.customKinesisShards.clear(); - for (Pair shardPosition : shardPositions) { - this.customKinesisShards.add(shardPosition.first); + if (!preparedAlter.shardPositions.isEmpty()) { + customKinesisShards.clear(); + for (Pair shardPosition : preparedAlter.shardPositions) { + customKinesisShards.add(shardPosition.first); } - } else if (resetProgress) { + } else if (preparedAlter.resetProgress) { // Stream change without explicit shards should fall back to dynamic shard discovery. - this.customKinesisShards.clear(); + customKinesisShards.clear(); } - - if (!shardPositions.isEmpty()) { - if (!resetProgress) { - ((KinesisProgress) progress).checkShards(shardPositions); - } - ((KinesisProgress) progress).modifyPosition(shardPositions); + if (!preparedAlter.shardPositions.isEmpty()) { + ((KinesisProgress) progress).modifyPosition(preparedAlter.shardPositions); } } - if (!jobProperties.isEmpty()) { Map copiedJobProperties = Maps.newHashMap(jobProperties); - modifyCommonJobProperties(copiedJobProperties); + modifyCommonJobProperties(copiedJobProperties, preparedAlter.validatedUniqueKeyUpdateMode); this.jobProperties.putAll(copiedJobProperties); if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { this.isPartialUpdate = BooleanUtils.toBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); @@ -774,8 +817,28 @@ private void modifyPropertiesInternal(Map jobProperties, } } } - LOG.info("modify the properties of kinesis routine load job: {}, jobProperties: {}, datasource properties: {}", - this.id, jobProperties, dataSourceProperties); + } + + private static class PreparedKinesisAlter { + private final TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode; + private final List> shardPositions; + private final Map stagedCustomProperties; + private final Map stagedConvertedCustomProperties; + private final String stagedDefaultPosition; + private final boolean resetProgress; + + private PreparedKinesisAlter(TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode, + List> shardPositions, + Map stagedCustomProperties, + Map stagedConvertedCustomProperties, + String stagedDefaultPosition, boolean resetProgress) { + this.validatedUniqueKeyUpdateMode = validatedUniqueKeyUpdateMode; + this.shardPositions = shardPositions; + this.stagedCustomProperties = stagedCustomProperties; + this.stagedConvertedCustomProperties = stagedConvertedCustomProperties; + this.stagedDefaultPosition = stagedDefaultPosition; + this.resetProgress = resetProgress; + } } @Override @@ -783,6 +846,8 @@ public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { try { modifyPropertiesInternal(log.getJobProperties(), (KinesisDataSourceProperties) log.getDataSourceProperties()); + setRoutineLoadDesc(log.getRoutineLoadDesc()); + updateLoadDefinition(log.getDataSourceProperties()); } catch (UserException e) { LOG.error("failed to replay modify kinesis routine load job: {}", id, e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java index 4729882f7927fb..9d8064943c566a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java @@ -19,6 +19,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.routineload.AbstractDataSourceProperties; import org.apache.doris.persist.gson.GsonUtils; @@ -37,12 +38,20 @@ public class AlterRoutineLoadJobOperationLog implements Writable { private Map jobProperties; @SerializedName(value = "dataSourceProperties") private AbstractDataSourceProperties dataSourceProperties; + @SerializedName(value = "routineLoadDesc") + private RoutineLoadDesc routineLoadDesc; public AlterRoutineLoadJobOperationLog(long jobId, Map jobProperties, AbstractDataSourceProperties dataSourceProperties) { + this(jobId, jobProperties, dataSourceProperties, null); + } + + public AlterRoutineLoadJobOperationLog(long jobId, Map jobProperties, + AbstractDataSourceProperties dataSourceProperties, RoutineLoadDesc routineLoadDesc) { this.jobId = jobId; this.jobProperties = jobProperties; this.dataSourceProperties = dataSourceProperties; + this.routineLoadDesc = routineLoadDesc; } public long getJobId() { @@ -57,6 +66,10 @@ public AbstractDataSourceProperties getDataSourceProperties() { return dataSourceProperties; } + public RoutineLoadDesc getRoutineLoadDesc() { + return routineLoadDesc; + } + public static AlterRoutineLoadJobOperationLog read(DataInput in) throws IOException { String json = Text.readString(in); return GsonUtils.GSON.fromJson(json, AlterRoutineLoadJobOperationLog.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index 7f0c8588372403..c3cfad1dc1ff18 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -30,6 +30,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.kafka.KafkaUtil; import org.apache.doris.load.RoutineLoadDesc; @@ -40,11 +41,15 @@ import org.apache.doris.load.routineload.kafka.KafkaRoutineLoadJob; import org.apache.doris.load.routineload.kafka.KafkaTaskInfo; import org.apache.doris.mysql.privilege.MockedAuth; +import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo; import org.apache.doris.nereids.trees.plans.commands.load.LoadProperty; import org.apache.doris.nereids.trees.plans.commands.load.LoadSeparator; +import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; +import org.apache.doris.persist.EditLog; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; import org.apache.doris.thrift.TResourceInfo; import org.apache.doris.thrift.TRoutineLoadTask; @@ -58,14 +63,20 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; public class KafkaRoutineLoadJobTest { @@ -272,6 +283,183 @@ public void testUpdateProgressWarnsWhenReadCommittedTaskHasZeroRowsAndLag() thro Assert.assertTrue(otherMsg.contains("some records may be in uncommitted transactions")); } + @Test + public void testFailedAlterDoesNotChangeRuntimeOrLoadDefinition() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "job_atomic", 1L, + 1L, "127.0.0.1:9020", "topic-1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + Map originalCustomProperties = Maps.newHashMap(); + originalCustomProperties.put("client.id", "old-client"); + Deencapsulation.setField(routineLoadJob, "customProperties", originalCustomProperties); + Map originalConvertedProperties = Maps.newHashMap(originalCustomProperties); + Deencapsulation.setField(routineLoadJob, "convertedCustomProperties", originalConvertedProperties); + Map originalProgress = Maps.newHashMap(); + originalProgress.put(0, 10L); + Deencapsulation.setField(routineLoadJob, "progress", new KafkaProgress(originalProgress)); + routineLoadJob.updateLoadDefinition(null); + Object originalLoadDefinition = Deencapsulation.getField(routineLoadJob, "loadDefinition"); + + Map originalDataSourceProperties = Maps.newHashMap(); + originalDataSourceProperties.put("property.client.id", "new-client"); + KafkaDataSourceProperties dataSourceProperties = + new KafkaDataSourceProperties(originalDataSourceProperties); + Map alteredCustomProperties = Maps.newHashMap(); + alteredCustomProperties.put("client.id", "new-client"); + Deencapsulation.setField(dataSourceProperties, "customKafkaProperties", alteredCustomProperties); + dataSourceProperties.setKafkaPartitionOffsets(Lists.newArrayList(Pair.of(1, 20L))); + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + Mockito.when(command.getDataSourceProperties()).thenReturn(dataSourceProperties); + + try (MockedStatic kafkaUtilStatic = Mockito.mockStatic(KafkaUtil.class)) { + kafkaUtilStatic.when(() -> KafkaUtil.getRealOffsets( + Mockito.eq("127.0.0.1:9020"), Mockito.eq("topic-1"), Mockito.anyMap(), Mockito.anyList(), + Mockito.nullable(String.class))) + .thenReturn(Lists.newArrayList(Pair.of(1, 20L))); + Assert.assertThrows(UserException.class, () -> routineLoadJob.modifyProperties(command)); + } + + Assert.assertEquals("topic-1", routineLoadJob.getTopic()); + Map currentCustomProperties = Deencapsulation.getField(routineLoadJob, "customProperties"); + Map currentConvertedProperties = + Deencapsulation.getField(routineLoadJob, "convertedCustomProperties"); + Assert.assertEquals("old-client", currentCustomProperties.get("client.id")); + Assert.assertEquals("old-client", currentConvertedProperties.get("client.id")); + Assert.assertSame(originalLoadDefinition, Deencapsulation.getField(routineLoadJob, "loadDefinition")); + } + + @Test + public void testSuccessfulAlterUpdatesLoadDefinitionAndJournal() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "job1", 1L, + 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + routineLoadJob.updateLoadDefinition(null); + Object originalLoadDefinition = Deencapsulation.getField(routineLoadJob, "loadDefinition"); + + RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc(null, null, null, null, null, null, null, + LoadTask.MergeType.APPEND, "sequence_col"); + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + Mockito.when(command.getDataSourceProperties()).thenReturn(null); + Mockito.when(command.getRoutineLoadDesc()).thenReturn(routineLoadDesc); + Env env = Mockito.mock(Env.class); + EditLog editLog = Mockito.mock(EditLog.class); + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + Mockito.when(env.getEditLog()).thenReturn(editLog); + + routineLoadJob.modifyProperties(command); + + ArgumentCaptor logCaptor = + ArgumentCaptor.forClass(AlterRoutineLoadJobOperationLog.class); + Mockito.verify(editLog).logAlterRoutineLoadJob(logCaptor.capture()); + Assert.assertSame(routineLoadDesc, logCaptor.getValue().getRoutineLoadDesc()); + } + + Assert.assertEquals("sequence_col", routineLoadJob.getSequenceCol()); + Assert.assertNotSame(originalLoadDefinition, Deencapsulation.getField(routineLoadJob, "loadDefinition")); + } + + @Test + public void testImageRoundTripUsesPersistedLoadDefinitionAfterAlterReplay() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "job1", 1L, + 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + Deencapsulation.setField(routineLoadJob, "desireTaskConcurrentNum", 1); + routineLoadJob.updateLoadDefinition(null); + + RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc( + new Separator(",", ","), null, null, null, null, null, null, + LoadTask.MergeType.APPEND, "sequence_col"); + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY, "2"); + routineLoadJob.replayModifyProperties(new AlterRoutineLoadJobOperationLog( + routineLoadJob.getId(), jobProperties, null, routineLoadDesc)); + + Env env = Mockito.mock(Env.class); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Database database = Mockito.mock(Database.class); + OlapTable table = Mockito.mock(OlapTable.class); + connectContextStatic.close(); + connectContextStatic = null; + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + Mockito.when(env.getInternalCatalog()).thenReturn(catalog); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(catalogMgr.getCatalog(Mockito.anyString())).thenReturn(catalog); + Mockito.when(catalog.getDb(1L)).thenReturn(Optional.of(database)); + Mockito.when(catalog.getDb("db1")).thenReturn(Optional.of(database)); + Mockito.when(catalog.getDbOrAnalysisException("db1")).thenReturn(database); + Mockito.when(database.getName()).thenReturn("db1"); + Mockito.when(database.getFullName()).thenReturn("db1"); + Mockito.when(database.getTable(1L)).thenReturn(Optional.of((Table) table)); + Mockito.when(database.getTableOrAnalysisException("table1")).thenReturn(table); + Mockito.when(table.getName()).thenReturn("table1"); + Mockito.when(table.getType()).thenReturn(Table.TableType.OLAP); + + RoutineLoadJob restored = imageRoundTrip(routineLoadJob); + + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); + Assert.assertEquals("sequence_col", restored.getSequenceCol()); + Assert.assertEquals(",", restored.getColumnSeparator().getSeparator()); + Assert.assertEquals(2, (int) Deencapsulation.getField(restored, "desireTaskConcurrentNum")); + Assert.assertNotNull(Deencapsulation.getField(restored, "loadDefinition")); + } + } + + @Test + public void testImageRoundTripRestoresLegacyOrigStmt() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "job1", 1L, + 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + String createSql = "CREATE ROUTINE LOAD db1.job1 ON stale_table " + + "COLUMNS TERMINATED BY ',' " + + "FROM KAFKA ('kafka_broker_list' = '127.0.0.1:9020', 'kafka_topic' = 'topic1')"; + Deencapsulation.setField(routineLoadJob, "origStmt", new OriginStatement(createSql, 0)); + Deencapsulation.setField(routineLoadJob, "loadDefinition", null); + + Env env = Mockito.mock(Env.class); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Database database = Mockito.mock(Database.class); + OlapTable table = Mockito.mock(OlapTable.class); + connectContextStatic.close(); + connectContextStatic = null; + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + Mockito.when(env.getInternalCatalog()).thenReturn(catalog); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(catalogMgr.getCatalog(Mockito.anyString())).thenReturn(catalog); + Mockito.when(catalog.getDb(1L)).thenReturn(Optional.of(database)); + Mockito.when(catalog.getDb("db1")).thenReturn(Optional.of(database)); + Mockito.when(catalog.getDbOrAnalysisException("db1")).thenReturn(database); + Mockito.when(database.getName()).thenReturn("db1"); + Mockito.when(database.getTable(1L)).thenReturn(Optional.of((Table) table)); + Mockito.when(database.getTableOrAnalysisException("table1")).thenReturn(table); + Mockito.when(table.getName()).thenReturn("table1"); + Mockito.when(table.getType()).thenReturn(Table.TableType.OLAP); + + RoutineLoadJob restored = imageRoundTrip(routineLoadJob); + + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); + Assert.assertEquals(",", restored.getColumnSeparator().getSeparator()); + Assert.assertNull(Deencapsulation.getField(restored, "loadDefinition")); + } + } + + private static RoutineLoadJob imageRoundTrip(RoutineLoadJob routineLoadJob) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(bytes)) { + routineLoadJob.write(out); + } + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + return RoutineLoadJob.read(in); + } + } + @Test public void testDisplayCustomPropertiesMasksKafkaSecrets() { KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java index 65aebd0084e729..a9face05891ab1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java @@ -19,18 +19,22 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.common.Config; +import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.load.routineload.kinesis.KinesisConfiguration; import org.apache.doris.load.routineload.kinesis.KinesisDataSourceProperties; import org.apache.doris.load.routineload.kinesis.KinesisProgress; import org.apache.doris.load.routineload.kinesis.KinesisRoutineLoadJob; import org.apache.doris.load.routineload.kinesis.KinesisTaskInfo; +import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.Gson; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.util.HashMap; import java.util.HashSet; @@ -229,6 +233,50 @@ public void testModifyPropertiesShouldReplaceCustomShardsWhenExplicitShardsProvi Assert.assertEquals("202", progress.getSequenceNumberByShard("shard-2")); } + @Test + public void testFailedAlterDoesNotChangeRuntimeOrLoadDefinition() throws Exception { + KinesisRoutineLoadJob routineLoadJob = new KinesisRoutineLoadJob(1L, "job_atomic", 1L, + 1L, "ap-southeast-1", "stream-1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + Map originalCustomProperties = Maps.newHashMap(); + originalCustomProperties.put("client.id", "old-client"); + Deencapsulation.setField(routineLoadJob, "customProperties", originalCustomProperties); + Map originalConvertedProperties = Maps.newHashMap(originalCustomProperties); + Deencapsulation.setField(routineLoadJob, "convertedCustomProperties", originalConvertedProperties); + Deencapsulation.setField(routineLoadJob, "customKinesisShards", Lists.newArrayList("shard-1")); + Map originalProgress = Maps.newHashMap(); + originalProgress.put("shard-1", "10"); + Deencapsulation.setField(routineLoadJob, "progress", new KinesisProgress(originalProgress)); + routineLoadJob.updateLoadDefinition(null); + Object originalLoadDefinition = Deencapsulation.getField(routineLoadJob, "loadDefinition"); + + Map originalDataSourceProperties = Maps.newHashMap(); + originalDataSourceProperties.put("property.client.id", "new-client"); + originalDataSourceProperties.put(KinesisConfiguration.KINESIS_REGION.getName(), "us-east-1"); + KinesisDataSourceProperties dataSourceProperties = + new KinesisDataSourceProperties(originalDataSourceProperties); + Map alteredCustomProperties = Maps.newHashMap(); + alteredCustomProperties.put("client.id", "new-client"); + Deencapsulation.setField(dataSourceProperties, "customKinesisProperties", alteredCustomProperties); + Deencapsulation.setField(dataSourceProperties, "region", "us-east-1"); + dataSourceProperties.setKinesisShardPositions(Lists.newArrayList(Pair.of("shard-2", "20"))); + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + Mockito.when(command.getDataSourceProperties()).thenReturn(dataSourceProperties); + + Assert.assertThrows(UserException.class, () -> routineLoadJob.modifyProperties(command)); + + Assert.assertEquals("ap-southeast-1", routineLoadJob.getRegion()); + Assert.assertEquals(Lists.newArrayList("shard-1"), + Deencapsulation.getField(routineLoadJob, "customKinesisShards")); + Map currentCustomProperties = Deencapsulation.getField(routineLoadJob, "customProperties"); + Map currentConvertedProperties = + Deencapsulation.getField(routineLoadJob, "convertedCustomProperties"); + Assert.assertEquals("old-client", currentCustomProperties.get("client.id")); + Assert.assertEquals("old-client", currentConvertedProperties.get("client.id")); + Assert.assertSame(originalLoadDefinition, Deencapsulation.getField(routineLoadJob, "loadDefinition")); + } + @Test public void testShardRefreshShouldMoveRetiredParentToClosedUntilConsumed() throws Exception { KinesisRoutineLoadJob routineLoadJob = diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java index 8a1550d48f5d13..522c8bee4d9948 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java @@ -18,7 +18,10 @@ package org.apache.doris.persist; import org.apache.doris.common.UserException; +import org.apache.doris.common.io.Text; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.load.RoutineLoadDesc; +import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.kafka.KafkaConfiguration; import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; @@ -27,6 +30,8 @@ import org.junit.Assert; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.File; @@ -60,8 +65,10 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User routineLoadDataSourceProperties.setTimezone(TimeUtils.DEFAULT_TIME_ZONE); routineLoadDataSourceProperties.analyze(); + RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc(null, null, null, null, null, null, null, + LoadTask.MergeType.APPEND, "sequence_col"); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(jobId, - jobProperties, routineLoadDataSourceProperties); + jobProperties, routineLoadDataSourceProperties, routineLoadDesc); log.write(out); out.flush(); out.close(); @@ -81,9 +88,24 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User kafkaDataSourceProperties.getKafkaPartitionOffsets().get(0)); Assert.assertEquals(routineLoadDataSourceProperties.getKafkaPartitionOffsets().get(1), kafkaDataSourceProperties.getKafkaPartitionOffsets().get(1)); + Assert.assertEquals("sequence_col", log2.getRoutineLoadDesc().getSequenceColName()); in.close(); } + @Test + public void testDeserializeLegacyLogWithoutRoutineLoadDesc() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(bytes)) { + Text.writeString(out, "{\"jobId\":1000,\"jobProperties\":{}," + + "\"dataSourceProperties\":null}"); + } + + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + AlterRoutineLoadJobOperationLog log = AlterRoutineLoadJobOperationLog.read(in); + Assert.assertEquals(1000L, log.getJobId()); + Assert.assertNull(log.getRoutineLoadDesc()); + } + } } From d0505b77ac1e64fdec04cc2d3b12c9cc54a3bf17 Mon Sep 17 00:00:00 2001 From: Refrain Date: Tue, 11 Aug 2026 11:00:45 +0800 Subject: [PATCH 2/4] [test](routineload) Define rollback compatibility boundary ### What problem does this PR solve? Issue Number: N/A Related PR: #64878 Problem Summary: Keep the original CREATE statement in new images so an older FE can ignore the new load definition field and use its existing recovery path. ALTERed load clauses are outside the downgrade compatibility guarantee and may not survive rollback. ### Release note Document that Routine Load ALTER semantics are not guaranteed after rolling back to an older FE. ### Check List (For Author) - Test: Not run (per request; compatibility coverage was added) - Behavior changed: No. This records and tests the intended rollback compatibility boundary. - Does this need documentation: Yes. The rollback limitation must be called out in the feature documentation. --- .../load/routineload/RoutineLoadJob.java | 3 +- .../routineload/KafkaRoutineLoadJobTest.java | 38 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 4229aca376353e..aefbb5e5d66013 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -263,7 +263,8 @@ public boolean isFinalState() { @SerializedName("ld") protected RoutineLoadDefinition loadDefinition; - // Legacy recovery input for images written before loadDefinition was persisted. + // Keep the original CREATE statement for downgrade compatibility. Older FEs ignore loadDefinition + // and restore from this field, so ALTER semantics are not guaranteed after rollback. @SerializedName("ostmt") protected OriginStatement origStmt; // User who submit this job. Maybe null for the old version job(before v1.1) diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index c3cfad1dc1ff18..462803af886af1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -29,6 +29,7 @@ import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; +import org.apache.doris.common.io.Text; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InternalCatalog; @@ -56,6 +57,8 @@ import com.google.common.base.Joiner; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import org.apache.kafka.common.PartitionInfo; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -366,6 +369,10 @@ public void testImageRoundTripUsesPersistedLoadDefinitionAfterAlterReplay() thro 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); Deencapsulation.setField(routineLoadJob, "desireTaskConcurrentNum", 1); + String originalCreateSql = "CREATE ROUTINE LOAD db1.job1 ON table1 " + + "COLUMNS TERMINATED BY '|' " + + "FROM KAFKA ('kafka_broker_list' = '127.0.0.1:9020', 'kafka_topic' = 'topic1')"; + Deencapsulation.setField(routineLoadJob, "origStmt", new OriginStatement(originalCreateSql, 0)); routineLoadJob.updateLoadDefinition(null); RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc( @@ -406,6 +413,14 @@ public void testImageRoundTripUsesPersistedLoadDefinitionAfterAlterReplay() thro Assert.assertEquals(",", restored.getColumnSeparator().getSeparator()); Assert.assertEquals(2, (int) Deencapsulation.getField(restored, "desireTaskConcurrentNum")); Assert.assertNotNull(Deencapsulation.getField(restored, "loadDefinition")); + + // Simulate an older FE ignoring the unknown loadDefinition field. It can read the image + // through origStmt, but the ALTERed load clauses are intentionally not preserved on rollback. + RoutineLoadJob rollbackRestored = imageRoundTripWithoutLoadDefinition(routineLoadJob); + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, rollbackRestored.getState()); + Assert.assertEquals("|", rollbackRestored.getColumnSeparator().getSeparator()); + Assert.assertNull(rollbackRestored.getSequenceCol()); + Assert.assertNull(Deencapsulation.getField(rollbackRestored, "loadDefinition")); } } @@ -460,6 +475,29 @@ private static RoutineLoadJob imageRoundTrip(RoutineLoadJob routineLoadJob) thro } } + private static RoutineLoadJob imageRoundTripWithoutLoadDefinition(RoutineLoadJob routineLoadJob) + throws Exception { + ByteArrayOutputStream image = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(image)) { + routineLoadJob.write(out); + } + + String json; + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(image.toByteArray()))) { + json = Text.readString(in); + } + JsonObject jobJson = JsonParser.parseString(json).getAsJsonObject(); + Assert.assertNotNull(jobJson.remove("ld")); + + ByteArrayOutputStream legacyImage = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(legacyImage)) { + Text.writeString(out, jobJson.toString()); + } + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(legacyImage.toByteArray()))) { + return RoutineLoadJob.read(in); + } + } + @Test public void testDisplayCustomPropertiesMasksKafkaSecrets() { KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, From aac38e8db61826ac15afc6f0912055edd11c09cf Mon Sep 17 00:00:00 2001 From: Refrain Date: Fri, 14 Aug 2026 14:17:49 +0800 Subject: [PATCH 3/4] temp --- .../routineload/RoutineLoadDefinition.java | 128 ------ .../load/routineload/RoutineLoadJob.java | 313 +++++++------ .../kafka/KafkaRoutineLoadJob.java | 196 +++----- .../kinesis/KinesisRoutineLoadJob.java | 197 +++----- .../routineload/KafkaRoutineLoadJobTest.java | 229 ++-------- .../KinesisRoutineLoadJobTest.java | 131 ++++-- .../RoutineLoadJobPersistenceTest.java | 426 ++++++++++++++++++ .../AlterRoutineLoadOperationLogTest.java | 100 ++-- 8 files changed, 922 insertions(+), 798 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadDefinition.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadDefinition.java deleted file mode 100644 index bcfb59c48d90f4..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadDefinition.java +++ /dev/null @@ -1,128 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.load.routineload; - -import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.ExprToSqlVisitor; -import org.apache.doris.analysis.ImportColumnDesc; -import org.apache.doris.analysis.ToSqlParams; -import org.apache.doris.common.UserException; -import org.apache.doris.load.RoutineLoadDesc; -import org.apache.doris.load.loadv2.LoadTask; -import org.apache.doris.nereids.load.NereidsLoadUtils; -import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; -import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo; -import org.apache.doris.nereids.trees.plans.commands.load.LoadColumnClause; -import org.apache.doris.nereids.trees.plans.commands.load.LoadColumnDesc; -import org.apache.doris.nereids.trees.plans.commands.load.LoadDeleteOnClause; -import org.apache.doris.nereids.trees.plans.commands.load.LoadPartitionNames; -import org.apache.doris.nereids.trees.plans.commands.load.LoadPrecedingFilterClause; -import org.apache.doris.nereids.trees.plans.commands.load.LoadProperty; -import org.apache.doris.nereids.trees.plans.commands.load.LoadSeparator; -import org.apache.doris.nereids.trees.plans.commands.load.LoadSequenceClause; -import org.apache.doris.nereids.trees.plans.commands.load.LoadWhereClause; -import org.apache.doris.nereids.trees.expressions.Expression; - -import com.google.common.collect.Maps; -import com.google.gson.annotations.SerializedName; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Snapshot of the current CREATE ROUTINE LOAD semantics. - */ -public class RoutineLoadDefinition { - @SerializedName("desc") - private RoutineLoadDesc routineLoadDesc; - @SerializedName("jp") - private Map jobProperties = Maps.newHashMap(); - @SerializedName("dsp") - private Map dataSourceProperties = Maps.newHashMap(); - - public RoutineLoadDefinition(RoutineLoadDesc routineLoadDesc, - Map jobProperties, Map dataSourceProperties) { - this.routineLoadDesc = routineLoadDesc; - this.jobProperties.putAll(jobProperties); - this.dataSourceProperties.putAll(dataSourceProperties); - } - - public RoutineLoadDesc getRoutineLoadDesc() { - return routineLoadDesc; - } - - public Map getDataSourceProperties() { - return dataSourceProperties; - } - - public CreateRoutineLoadInfo toCreateInfo(String dbName, String jobName, String tableName, - LoadDataSourceType dataSourceType, String comment) throws UserException { - LoadTask.MergeType mergeType = routineLoadDesc == null - ? LoadTask.MergeType.APPEND : routineLoadDesc.getMergeType(); - return new CreateRoutineLoadInfo(new LabelNameInfo(dbName, jobName), tableName, - toLoadPropertyMap(routineLoadDesc), Maps.newHashMap(jobProperties), dataSourceType.name(), - Maps.newHashMap(dataSourceProperties), mergeType, comment); - } - - private static Map toLoadPropertyMap(RoutineLoadDesc routineLoadDesc) throws UserException { - Map loadProperties = Maps.newHashMap(); - if (routineLoadDesc == null) { - return loadProperties; - } - if (routineLoadDesc.getColumnSeparator() != null) { - put(loadProperties, new LoadSeparator(routineLoadDesc.getColumnSeparator().getOriSeparator())); - } - if (routineLoadDesc.getColumnsInfo() != null) { - List columns = new ArrayList<>(); - for (ImportColumnDesc column : routineLoadDesc.getColumnsInfo()) { - Expression expression = column.getExpr() == null ? null : parseExpression(column.getExpr()); - columns.add(new LoadColumnDesc(column.getColumnName(), expression)); - } - put(loadProperties, new LoadColumnClause(columns)); - } - if (routineLoadDesc.getPrecedingFilter() != null) { - put(loadProperties, new LoadPrecedingFilterClause( - parseExpression(routineLoadDesc.getPrecedingFilter()))); - } - if (routineLoadDesc.getFilter() != null) { - put(loadProperties, new LoadWhereClause(parseExpression(routineLoadDesc.getFilter()))); - } - if (routineLoadDesc.getPartitionNamesInfo() != null) { - put(loadProperties, new LoadPartitionNames( - routineLoadDesc.getPartitionNamesInfo().isTemp(), - routineLoadDesc.getPartitionNamesInfo().getPartitionNames())); - } - if (routineLoadDesc.getDeleteCondition() != null) { - put(loadProperties, new LoadDeleteOnClause(parseExpression(routineLoadDesc.getDeleteCondition()))); - } - if (routineLoadDesc.hasSequenceCol()) { - put(loadProperties, new LoadSequenceClause(routineLoadDesc.getSequenceColName())); - } - return loadProperties; - } - - private static Expression parseExpression(Expr expression) throws UserException { - String sql = expression.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); - return NereidsLoadUtils.parseExpressionSeq(sql).get(0); - } - - private static void put(Map loadProperties, LoadProperty loadProperty) { - loadProperties.put(loadProperty.getClass().getName(), loadProperty); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index aefbb5e5d66013..57497876059a81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -19,7 +19,6 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ExprToSqlVisitor; -import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.Separator; import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.analysis.UserIdentity; @@ -117,6 +116,7 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback implements Writable, LoadTaskInfo, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(RoutineLoadJob.class); + private static final int CURRENT_ROUTINE_LOAD_PERSISTENCE_VERSION = 1; public static final long DEFAULT_MAX_ERROR_NUM = 0; public static final double DEFAULT_MAX_FILTER_RATIO = 1.0; @@ -180,14 +180,23 @@ public boolean isFinalState() { protected long dbId; @SerializedName("tbid") protected long tableId; + // An absent version identifies a legacy record whose CREATE statement still needs to be migrated. + @SerializedName("rlpv") + private int routineLoadPersistenceVersion; // this code is used to verify be task request protected long authCode; // protected RoutineLoadDesc routineLoadDesc; // optional + @SerializedName("pni") protected PartitionNamesInfo partitionNamesInfo; // optional + @SerializedName("cds") protected ImportColumnDescs columnDescs; // optional + @SerializedName("pf") protected Expr precedingFilter; // optional + @SerializedName("we") protected Expr whereExpr; // optional + @SerializedName("cs") protected Separator columnSeparator; // optional + @SerializedName("lidel") protected Separator lineDelimiter; @SerializedName("dtcn") protected int desireTaskConcurrentNum; // optional @@ -202,6 +211,7 @@ public boolean isFinalState() { @SerializedName("men") protected long maxErrorNum = DEFAULT_MAX_ERROR_NUM; // optional protected double maxFilterRatio = DEFAULT_MAX_FILTER_RATIO; + @SerializedName("eml") protected long execMemLimit = DEFAULT_EXEC_MEM_LIMIT; protected int sendBatchParallelism = DEFAULT_SEND_BATCH_PARALLELISM; protected boolean loadToSingleTablet = DEFAULT_LOAD_TO_SINGLE_TABLET; @@ -231,8 +241,10 @@ public boolean isFinalState() { protected TPartialUpdateNewRowPolicy partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.APPEND; protected TUniqueKeyUpdateMode uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPSERT; + @SerializedName("sc") protected String sequenceCol; + @SerializedName("mosn") protected boolean memtableOnSinkNode = false; protected int currentTaskConcurrentNum; @@ -259,12 +271,7 @@ public boolean isFinalState() { // The tasks belong to this job protected List routineLoadTaskInfoList = Lists.newArrayList(); - // Canonical current CREATE semantics. CREATE and ALTER must keep this snapshot current. - @SerializedName("ld") - protected RoutineLoadDefinition loadDefinition; - - // Keep the original CREATE statement for downgrade compatibility. Older FEs ignore loadDefinition - // and restore from this field, so ALTER semantics are not guaranteed after rollback. + // Keep the original CREATE statement for downgrade compatibility and legacy image migration. @SerializedName("ostmt") protected OriginStatement origStmt; // User who submit this job. Maybe null for the old version job(before v1.1) @@ -275,7 +282,9 @@ public boolean isFinalState() { protected String comment = ""; protected ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); + @SerializedName("mt") protected LoadTask.MergeType mergeType = LoadTask.MergeType.APPEND; // default is all data is load no delete + @SerializedName("dc") protected Expr deleteCondition; // TODO(ml): error sample @@ -321,6 +330,7 @@ public RoutineLoadJob(Long id, String name, this.tableId = tableId; this.authCode = 0; this.userIdentity = userIdentity; + this.routineLoadPersistenceVersion = CURRENT_ROUTINE_LOAD_PERSISTENCE_VERSION; if (ConnectContext.get() != null) { SessionVariable var = ConnectContext.get().getSessionVariable(); @@ -350,6 +360,7 @@ public RoutineLoadJob(Long id, String name, this.authCode = 0; this.userIdentity = userIdentity; this.isMultiTable = true; + this.routineLoadPersistenceVersion = CURRENT_ROUTINE_LOAD_PERSISTENCE_VERSION; if (ConnectContext.get() != null) { SessionVariable var = ConnectContext.get().getSessionVariable(); @@ -475,62 +486,6 @@ protected void setRoutineLoadDesc(RoutineLoadDesc routineLoadDesc) { } } - protected RoutineLoadDesc getLoadDefinitionRoutineLoadDesc() { - List columnsInfo = columnDescs == null ? null : columnDescs.descs; - return new RoutineLoadDesc(columnSeparator, lineDelimiter, columnsInfo, precedingFilter, whereExpr, - partitionNamesInfo, deleteCondition, mergeType, sequenceCol); - } - - protected void initializeLoadDefinition(CreateRoutineLoadInfo info) { - Map originalDataSourceProperties = - info.getDataSourceProperties().getOriginalDataSourceProperties(); - Map dataSourceProperties = originalDataSourceProperties == null - ? Maps.newHashMap() : Maps.newHashMap(originalDataSourceProperties); - updateLoadDefinitionDataSourceProperties(dataSourceProperties); - loadDefinition = new RoutineLoadDefinition( - getLoadDefinitionRoutineLoadDesc(), snapshotLoadDefinitionJobProperties(), dataSourceProperties); - } - - protected void updateLoadDefinition(AbstractDataSourceProperties changedDataSourceProperties) { - Map dataSourceProperties = loadDefinition == null - ? Maps.newHashMap() : Maps.newHashMap(loadDefinition.getDataSourceProperties()); - if (changedDataSourceProperties != null - && changedDataSourceProperties.getOriginalDataSourceProperties() != null) { - dataSourceProperties.putAll(changedDataSourceProperties.getOriginalDataSourceProperties()); - } - updateLoadDefinitionDataSourceProperties(dataSourceProperties); - loadDefinition = new RoutineLoadDefinition( - getLoadDefinitionRoutineLoadDesc(), snapshotLoadDefinitionJobProperties(), dataSourceProperties); - } - - protected Map snapshotLoadDefinitionJobProperties() { - Map currentJobProperties = Maps.newHashMap(jobProperties); - currentJobProperties.put(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY, - String.valueOf(desireTaskConcurrentNum)); - currentJobProperties.put(CreateRoutineLoadInfo.MAX_ERROR_NUMBER_PROPERTY, String.valueOf(maxErrorNum)); - currentJobProperties.put(CreateRoutineLoadInfo.MAX_FILTER_RATIO_PROPERTY, String.valueOf(maxFilterRatio)); - currentJobProperties.put(CreateRoutineLoadInfo.MAX_BATCH_INTERVAL_SEC_PROPERTY, - String.valueOf(maxBatchIntervalS)); - currentJobProperties.put(CreateRoutineLoadInfo.MAX_BATCH_ROWS_PROPERTY, String.valueOf(maxBatchRows)); - currentJobProperties.put(CreateRoutineLoadInfo.MAX_BATCH_SIZE_PROPERTY, String.valueOf(maxBatchSizeBytes)); - currentJobProperties.put(CreateRoutineLoadInfo.EXEC_MEM_LIMIT_PROPERTY, String.valueOf(execMemLimit)); - currentJobProperties.put(CreateRoutineLoadInfo.SEND_BATCH_PARALLELISM, - String.valueOf(sendBatchParallelism)); - currentJobProperties.put(CreateRoutineLoadInfo.LOAD_TO_SINGLE_TABLET, - String.valueOf(loadToSingleTablet)); - currentJobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); - currentJobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); - if (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { - currentJobProperties.remove(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); - } else { - currentJobProperties.put(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY, - partialUpdateNewKeyPolicy.name()); - } - return currentJobProperties; - } - - protected abstract void updateLoadDefinitionDataSourceProperties(Map dataSourceProperties); - @Override public long getId() { return id; @@ -2028,85 +1983,105 @@ public void gsonPostProcess() throws IOException { if (tableId == 0) { isMultiTable = true; } - // Process UNIQUE_KEY_UPDATE_MODE first to ensure correct backward compatibility - // with PARTIAL_COLUMNS (HashMap iteration order is not guaranteed) - if (jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { - String modeValue = jobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); - TUniqueKeyUpdateMode mode = CreateRoutineLoadInfo.parseUniqueKeyUpdateMode(modeValue); - if (mode != null) { - uniqueKeyUpdateMode = mode; - isPartialUpdate = (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS); - } else { - uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPSERT; - } + if (routineLoadPersistenceVersion == 0) { + // Legacy images did not persist this create-time session option. Preserve their historical + // post-restart behavior instead of inheriting the image-loading thread's ConnectContext. + memtableOnSinkNode = false; } - // Process remaining properties - jobProperties.forEach((k, v) -> { - if (k.equals(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { - // Backward compatibility: only use partial_columns if unique_key_update_mode is not set - // unique_key_update_mode takes precedence - if (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { - isPartialUpdate = Boolean.parseBoolean(v); - if (isPartialUpdate) { - uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS; - } - } - } else if (k.equals(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { - if ("ERROR".equalsIgnoreCase(v)) { - partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.ERROR; - } else { - partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.APPEND; - } - } - }); try { - ConnectContext ctx = new ConnectContext(); - ctx.setDatabase(Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get().getName()); - StatementContext statementContext = new StatementContext(); - statementContext.setConnectContext(ctx); - ctx.setStatementContext(statementContext); - ctx.setEnv(Env.getCurrentEnv()); - ctx.setCurrentUserIdentity(UserIdentity.ADMIN); - ctx.getState().reset(); - try { - ctx.setThreadLocalInfo(); - if (loadDefinition == null) { - restoreLegacyDefinition(ctx); - } else { - restoreLoadDefinition(ctx); - } - } finally { - ctx.cleanup(); + hydrateJobProperties(); + if (routineLoadPersistenceVersion == 0) { + restoreLegacyDefinition(); + routineLoadPersistenceVersion = CURRENT_ROUTINE_LOAD_PERSISTENCE_VERSION; } } catch (Exception e) { this.state = JobState.CANCELLED; - LOG.warn("error happens when restoring routine load definition", e); + LOG.warn("error happens when restoring routine load job", e); } if (userIdentity != null) { userIdentity.setIsAnalyzed(); } } - private void restoreLegacyDefinition(ConnectContext ctx) throws UserException { - NereidsParser nereidsParser = new NereidsParser(); - CreateRoutineLoadCommand command = (CreateRoutineLoadCommand) nereidsParser.parseSingle( - origStmt.originStmt); - CreateRoutineLoadInfo createRoutineLoadInfo = command.getCreateRoutineLoadInfo(); - if (!isMultiTable) { - Database db = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get(); - createRoutineLoadInfo.setTableName(db.getTable(tableId).get().getName()); + private void hydrateJobProperties() throws UserException { + if (jobProperties.containsKey(CreateRoutineLoadInfo.MAX_FILTER_RATIO_PROPERTY)) { + maxFilterRatio = Double.parseDouble( + jobProperties.get(CreateRoutineLoadInfo.MAX_FILTER_RATIO_PROPERTY)); + } + if (jobProperties.containsKey(CreateRoutineLoadInfo.SEND_BATCH_PARALLELISM)) { + sendBatchParallelism = Integer.parseInt( + jobProperties.get(CreateRoutineLoadInfo.SEND_BATCH_PARALLELISM)); + } + if (jobProperties.containsKey(CreateRoutineLoadInfo.LOAD_TO_SINGLE_TABLET)) { + loadToSingleTablet = Boolean.parseBoolean( + jobProperties.get(CreateRoutineLoadInfo.LOAD_TO_SINGLE_TABLET)); + } + + boolean hasUniqueKeyUpdateMode = jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); + if (hasUniqueKeyUpdateMode) { + TUniqueKeyUpdateMode mode = CreateRoutineLoadInfo.parseUniqueKeyUpdateMode( + jobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)); + uniqueKeyUpdateMode = mode == null ? TUniqueKeyUpdateMode.UPSERT : mode; + isPartialUpdate = uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS; + } + if (!hasUniqueKeyUpdateMode && jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { + isPartialUpdate = Boolean.parseBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); + if (isPartialUpdate) { + uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS; + } + } + if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { + partialUpdateNewKeyPolicy = "ERROR".equalsIgnoreCase( + jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) + ? TPartialUpdateNewRowPolicy.ERROR : TPartialUpdateNewRowPolicy.APPEND; + } + + if (jobProperties.containsKey(CsvFileFormatProperties.PROP_ENCLOSE)) { + enclose = parseEnclose(jobProperties.get(CsvFileFormatProperties.PROP_ENCLOSE)); + } + if (jobProperties.containsKey(CsvFileFormatProperties.PROP_ESCAPE)) { + escape = parseEscape(jobProperties.get(CsvFileFormatProperties.PROP_ESCAPE)); + } + if (jobProperties.containsKey(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)) { + emptyFieldAsNull = Boolean.parseBoolean( + jobProperties.get(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)); } - createRoutineLoadInfo.validate(ctx); - setRoutineLoadDesc(createRoutineLoadInfo.getRoutineLoadDesc()); } - private void restoreLoadDefinition(ConnectContext ctx) throws UserException { - Database db = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get(); - String tableName = isMultiTable ? null : db.getTable(tableId).get().getName(); - CreateRoutineLoadInfo createRoutineLoadInfo = loadDefinition.toCreateInfo( - db.getFullName(), name, tableName, dataSourceType, comment); - createRoutineLoadInfo.validate(ctx); - setRoutineLoadDesc(loadDefinition.getRoutineLoadDesc()); + private void restoreLegacyDefinition() throws UserException { + ConnectContext ctx = new ConnectContext(); + ctx.setDatabase(Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get().getName()); + StatementContext statementContext = new StatementContext(); + statementContext.setConnectContext(ctx); + ctx.setStatementContext(statementContext); + ctx.setEnv(Env.getCurrentEnv()); + ctx.setCurrentUserIdentity(UserIdentity.ADMIN); + ctx.getState().reset(); + try { + ctx.setThreadLocalInfo(); + NereidsParser nereidsParser = new NereidsParser(); + CreateRoutineLoadCommand command = (CreateRoutineLoadCommand) nereidsParser.parseSingle( + origStmt.originStmt); + CreateRoutineLoadInfo createRoutineLoadInfo = command.getCreateRoutineLoadInfo(); + // Resolve the current table name by ID so table rename or SWAP TABLE does not leave the + // legacy CREATE statement pointing at a stale table name. + if (!isMultiTable && tableId != 0) { + try { + Database db = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).orElse(null); + if (db != null) { + db.getTable(tableId).ifPresent( + table -> createRoutineLoadInfo.setTableName(table.getName())); + } + } catch (Exception ignored) { + // Let validate() below surface the original catalog error. + } + } + createRoutineLoadInfo.validate(ctx); + setRoutineLoadDesc(createRoutineLoadInfo.getRoutineLoadDesc()); + execMemLimit = createRoutineLoadInfo.getExecMemLimit(); + } finally { + ctx.cleanup(); + } } public abstract void modifyProperties(AlterRoutineLoadCommand command) throws UserException; @@ -2115,22 +2090,27 @@ private void restoreLoadDefinition(ConnectContext ctx) throws UserException { public abstract NereidsRoutineLoadTaskInfo toNereidsRoutineLoadTaskInfo() throws UserException; - protected TUniqueKeyUpdateMode validateCommonJobProperties(Map jobProperties) - throws UserException { - if (!jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { - return null; + protected void validateCommonJobProperties(Map jobProperties) throws UserException { + validateCsvFormatProperties(jobProperties); + if (jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { + TUniqueKeyUpdateMode newMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode( + jobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)); + if (newMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS) { + validateFlexiblePartialUpdateForAlter(); + } } - TUniqueKeyUpdateMode newMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode( - jobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)); - if (newMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS) { - validateFlexiblePartialUpdateForAlter(); + if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { + String policy = jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); + if (!"APPEND".equalsIgnoreCase(policy) && !"ERROR".equalsIgnoreCase(policy)) { + throw new AnalysisException(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY + + " should be one of {'APPEND', 'ERROR'}, but found " + policy); + } } - return newMode; } - // for ALTER ROUTINE LOAD. All failure-prone validation must be completed before calling this method. - protected void modifyCommonJobProperties(Map jobProperties, - TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode) { + // for ALTER ROUTINE LOAD. Validate all common properties before changing any common runtime state. + protected void modifyCommonJobProperties(Map jobProperties) throws UserException { + validateCommonJobProperties(jobProperties); if (jobProperties.containsKey(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)) { this.desireTaskConcurrentNum = Integer.parseInt( jobProperties.remove(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)); @@ -2163,8 +2143,8 @@ protected void modifyCommonJobProperties(Map jobProperties, } if (jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { - jobProperties.remove(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); - this.uniqueKeyUpdateMode = validatedUniqueKeyUpdateMode; + String modeStr = jobProperties.remove(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); + this.uniqueKeyUpdateMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode(modeStr); this.isPartialUpdate = (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS); this.jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); @@ -2181,6 +2161,55 @@ protected void modifyCommonJobProperties(Map jobProperties, this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); this.jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); } + + if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { + String policy = jobProperties.remove(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); + partialUpdateNewKeyPolicy = "ERROR".equalsIgnoreCase(policy) + ? TPartialUpdateNewRowPolicy.ERROR : TPartialUpdateNewRowPolicy.APPEND; + this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY, + partialUpdateNewKeyPolicy.name()); + } + + if (jobProperties.containsKey(CsvFileFormatProperties.PROP_ENCLOSE)) { + String value = jobProperties.remove(CsvFileFormatProperties.PROP_ENCLOSE); + enclose = parseEnclose(value); + this.jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, value); + } + if (jobProperties.containsKey(CsvFileFormatProperties.PROP_ESCAPE)) { + String value = jobProperties.remove(CsvFileFormatProperties.PROP_ESCAPE); + escape = parseEscape(value); + this.jobProperties.put(CsvFileFormatProperties.PROP_ESCAPE, value); + } + if (jobProperties.containsKey(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)) { + String value = jobProperties.remove(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL); + emptyFieldAsNull = Boolean.parseBoolean(value); + this.jobProperties.put(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL, value); + } + } + + private static void validateCsvFormatProperties(Map jobProperties) { + if (!jobProperties.containsKey(CsvFileFormatProperties.PROP_ENCLOSE) + && !jobProperties.containsKey(CsvFileFormatProperties.PROP_ESCAPE) + && !jobProperties.containsKey(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)) { + return; + } + Map csvProperties = Maps.newHashMap(); + for (String property : new String[] {CsvFileFormatProperties.PROP_ENCLOSE, + CsvFileFormatProperties.PROP_ESCAPE, CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL}) { + if (jobProperties.containsKey(property)) { + csvProperties.put(property, jobProperties.get(property)); + } + } + CsvFileFormatProperties properties = new CsvFileFormatProperties(FileFormatProperties.FORMAT_CSV); + properties.analyzeFileFormatProperties(csvProperties, false); + } + + private static byte parseEnclose(String value) { + return Strings.isNullOrEmpty(value) ? 0 : (byte) value.charAt(0); + } + + private static byte parseEscape(String value) { + return Strings.isNullOrEmpty(value) ? 0 : value.getBytes()[0]; } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java index a5d1dd5f0e7ef7..be124f7c72e6c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java @@ -60,8 +60,6 @@ import org.apache.doris.rpc.RpcException; import org.apache.doris.service.FrontendOptions; import org.apache.doris.thrift.TFileCompressType; -import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; -import org.apache.doris.thrift.TUniqueKeyUpdateMode; import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; @@ -75,7 +73,6 @@ import com.google.gson.annotations.SerializedName; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.BooleanUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -217,25 +214,19 @@ private void convertCustomProperties(boolean rebuild) throws DdlException { return; } - Pair, String> convertedProperties = buildConvertedCustomProperties( - customProperties, kafkaDefaultOffSet); - convertedCustomProperties.clear(); - convertedCustomProperties.putAll(convertedProperties.first); - kafkaDefaultOffSet = convertedProperties.second; - } + if (rebuild) { + convertedCustomProperties.clear(); + } - private Pair, String> buildConvertedCustomProperties( - Map sourceProperties, String currentDefaultOffset) throws DdlException { - Map convertedProperties = Maps.newHashMap(); - for (Map.Entry entry : sourceProperties.entrySet()) { + SmallFileMgr smallFileMgr = Env.getCurrentEnv().getSmallFileMgr(); + for (Map.Entry entry : customProperties.entrySet()) { if (entry.getValue().startsWith("FILE:")) { // convert FILE:file_name -> FILE:file_id:md5 String file = entry.getValue().substring(entry.getValue().indexOf(":") + 1); - SmallFileMgr smallFileMgr = Env.getCurrentEnv().getSmallFileMgr(); SmallFile smallFile = smallFileMgr.getSmallFile(dbId, KAFKA_FILE_CATALOG, file, true); - convertedProperties.put(entry.getKey(), "FILE:" + smallFile.id + ":" + smallFile.md5); + convertedCustomProperties.put(entry.getKey(), "FILE:" + smallFile.id + ":" + smallFile.md5); } else { - convertedProperties.put(entry.getKey(), entry.getValue()); + convertedCustomProperties.put(entry.getKey(), entry.getValue()); } } @@ -244,14 +235,14 @@ private Pair, String> buildConvertedCustomProperties( // KAFKA_DEFAULT_OFFSETS, and this attribute will be converted into a timestamp during the analyzing phase, // thus losing some information. So we use KAFKA_ORIGIN_DEFAULT_OFFSETS to store the original datetime // formatted KAFKA_DEFAULT_OFFSETS value - String convertedDefaultOffset = currentDefaultOffset; - if (convertedProperties.containsKey(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())) { - convertedDefaultOffset = convertedProperties + if (convertedCustomProperties.containsKey(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())) { + kafkaDefaultOffSet = convertedCustomProperties .remove(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName()); - } else if (convertedProperties.containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName())) { - convertedDefaultOffset = convertedProperties.remove(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); + return; + } + if (convertedCustomProperties.containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName())) { + kafkaDefaultOffSet = convertedCustomProperties.remove(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); } - return Pair.of(convertedProperties, convertedDefaultOffset); } @Override @@ -597,7 +588,6 @@ public static KafkaRoutineLoadJob fromCreateInfo(CreateRoutineLoadInfo info, Con kafkaRoutineLoadJob.setOptional(info); kafkaRoutineLoadJob.checkCustomProperties(); kafkaRoutineLoadJob.checkCustomPartition(); - kafkaRoutineLoadJob.initializeLoadDefinition(info); return kafkaRoutineLoadJob; } @@ -783,24 +773,10 @@ public Map getCustomProperties() { return getMaskedCustomProperties("property."); } - @Override - protected void updateLoadDefinitionDataSourceProperties(Map dataSourceProperties) { - dataSourceProperties.put(KafkaConfiguration.KAFKA_BROKER_LIST.getName(), brokerList); - dataSourceProperties.put(KafkaConfiguration.KAFKA_TOPIC.getName(), topic); - dataSourceProperties.remove(KafkaConfiguration.KAFKA_OFFSETS.getName()); - if (customKafkaPartitions.isEmpty()) { - dataSourceProperties.remove(KafkaConfiguration.KAFKA_PARTITIONS.getName()); - } else { - dataSourceProperties.put(KafkaConfiguration.KAFKA_PARTITIONS.getName(), - Joiner.on(",").join(customKafkaPartitions)); - } - customProperties.forEach((key, value) -> dataSourceProperties.put( - key.startsWith("aws.") ? key : "property." + key, value)); - } - @Override public void modifyProperties(AlterRoutineLoadCommand command) throws UserException { Map jobProperties = command.getAnalyzedJobProperties(); + validateCommonJobProperties(jobProperties); KafkaDataSourceProperties dataSourceProperties = (KafkaDataSourceProperties) command.getDataSourceProperties(); if (null != dataSourceProperties) { // if the partition offset is set by timestamp, convert it to real offset @@ -815,7 +791,6 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti modifyPropertiesInternal(jobProperties, dataSourceProperties); setRoutineLoadDesc(command.getRoutineLoadDesc()); - updateLoadDefinition(dataSourceProperties); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, jobProperties, dataSourceProperties, command.getRoutineLoadDesc()); @@ -845,122 +820,70 @@ private void convertOffset(KafkaDataSourceProperties dataSourceProperties) throw private void modifyPropertiesInternal(Map jobProperties, KafkaDataSourceProperties dataSourceProperties) throws UserException { - PreparedKafkaAlter preparedAlter = prepareAlter(jobProperties, dataSourceProperties); - applyAlter(jobProperties, dataSourceProperties, preparedAlter); - if (LOG.isDebugEnabled()) { - LOG.debug("modify the properties of kafka routine load job: {}, jobProperties: {}, " - + "datasource properties: {}", - this.id, jobProperties, dataSourceProperties); - } - } + if (null != dataSourceProperties) { + List> kafkaPartitionOffsets = Lists.newArrayList(); + Map customKafkaProperties = Maps.newHashMap(); - private PreparedKafkaAlter prepareAlter(Map jobProperties, - KafkaDataSourceProperties dataSourceProperties) throws UserException { - TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode = validateCommonJobProperties(jobProperties); - List> kafkaPartitionOffsets = Lists.newArrayList(); - Map alteredCustomProperties = Maps.newHashMap(); - if (dataSourceProperties != null - && MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { - kafkaPartitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); - alteredCustomProperties = dataSourceProperties.getCustomKafkaProperties(); - } + if (MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { + kafkaPartitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); + customKafkaProperties = dataSourceProperties.getCustomKafkaProperties(); + } - Map stagedCustomProperties = null; - Map stagedConvertedCustomProperties = null; - String stagedKafkaDefaultOffset = kafkaDefaultOffSet; - if (!alteredCustomProperties.isEmpty()) { - stagedCustomProperties = Maps.newHashMap(customProperties); - stagedCustomProperties.putAll(alteredCustomProperties); - Pair, String> convertedProperties = buildConvertedCustomProperties( - stagedCustomProperties, stagedKafkaDefaultOffset); - stagedConvertedCustomProperties = convertedProperties.first; - stagedKafkaDefaultOffset = convertedProperties.second; - } + // convertCustomProperties and check partitions before reset progress to make modify operation atomic + if (!customKafkaProperties.isEmpty()) { + this.customProperties.putAll(customKafkaProperties); + convertCustomProperties(true); + } - if (!kafkaPartitionOffsets.isEmpty()) { - ((KafkaProgress) progress).checkPartitions(kafkaPartitionOffsets); - } - if (dataSourceProperties != null && Config.isCloudMode()) { - Cloud.ResetRLProgressRequest.Builder builder = Cloud.ResetRLProgressRequest.newBuilder() - .setRequestIp(FrontendOptions.getLocalHostAddressCached()); - builder.setCloudUniqueId(Config.cloud_unique_id); - builder.setDbId(dbId); - builder.setJobId(id); if (!kafkaPartitionOffsets.isEmpty()) { - Map partitionOffsetMap = new HashMap<>(); - for (Pair pair : kafkaPartitionOffsets) { - // The reason why the value recorded in MS in cloud mode needs to be subtracted by one is - // this value will be incremented when pulling MS persistent progress data and updating memory - // in routineLoadJob.updateCloudProgress(). - partitionOffsetMap.put(pair.first, pair.second - 1); - } - builder.putAllPartitionToOffset(partitionOffsetMap); + ((KafkaProgress) progress).checkPartitions(kafkaPartitionOffsets); } - resetCloudProgress(builder); - } - return new PreparedKafkaAlter(validatedUniqueKeyUpdateMode, kafkaPartitionOffsets, - stagedCustomProperties, stagedConvertedCustomProperties, stagedKafkaDefaultOffset); - } - private void applyAlter(Map jobProperties, KafkaDataSourceProperties dataSourceProperties, - PreparedKafkaAlter preparedAlter) { - if (dataSourceProperties != null) { - if (preparedAlter.stagedCustomProperties != null) { - customProperties.clear(); - customProperties.putAll(preparedAlter.stagedCustomProperties); - convertedCustomProperties.clear(); - convertedCustomProperties.putAll(preparedAlter.stagedConvertedCustomProperties); - kafkaDefaultOffSet = preparedAlter.stagedKafkaDefaultOffset; + if (Config.isCloudMode()) { + Cloud.ResetRLProgressRequest.Builder builder = Cloud.ResetRLProgressRequest.newBuilder() + .setRequestIp(FrontendOptions.getLocalHostAddressCached()); + builder.setCloudUniqueId(Config.cloud_unique_id); + builder.setDbId(dbId); + builder.setJobId(id); + if (!kafkaPartitionOffsets.isEmpty()) { + Map partitionOffsetMap = new HashMap<>(); + for (Pair pair : kafkaPartitionOffsets) { + // The reason why the value recorded in MS in cloud mode needs to be subtracted by one is + // this value will be incremented + // when pulling MS persistent progress data and updating memory + // in routineLoadJob.updateCloudProgress(). + partitionOffsetMap.put(pair.first, pair.second - 1); + } + builder.putAllPartitionToOffset(partitionOffsetMap); + } + resetCloudProgress(builder); } + // It is necessary to reset the Kafka progress cache if topic change, // and should reset cache before modifying partition offset. if (!Strings.isNullOrEmpty(dataSourceProperties.getTopic())) { - topic = dataSourceProperties.getTopic(); - progress = new KafkaProgress(); + this.topic = dataSourceProperties.getTopic(); + this.progress = new KafkaProgress(); } - if (!preparedAlter.kafkaPartitionOffsets.isEmpty()) { - ((KafkaProgress) progress).modifyOffset(preparedAlter.kafkaPartitionOffsets); + + // modify partition offset + if (!kafkaPartitionOffsets.isEmpty()) { + // we can only modify the partition that is being consumed + ((KafkaProgress) progress).modifyOffset(kafkaPartitionOffsets); } + + // modify broker list if (!Strings.isNullOrEmpty(dataSourceProperties.getBrokerList())) { - brokerList = dataSourceProperties.getBrokerList(); + this.brokerList = dataSourceProperties.getBrokerList(); } } if (!jobProperties.isEmpty()) { Map copiedJobProperties = Maps.newHashMap(jobProperties); - modifyCommonJobProperties(copiedJobProperties, preparedAlter.validatedUniqueKeyUpdateMode); + modifyCommonJobProperties(copiedJobProperties); this.jobProperties.putAll(copiedJobProperties); - if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { - this.isPartialUpdate = BooleanUtils.toBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); - } - if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { - String policy = jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); - if ("ERROR".equalsIgnoreCase(policy)) { - this.partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.ERROR; - } else { - this.partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.APPEND; - } - } - } - } - - private static class PreparedKafkaAlter { - private final TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode; - private final List> kafkaPartitionOffsets; - private final Map stagedCustomProperties; - private final Map stagedConvertedCustomProperties; - private final String stagedKafkaDefaultOffset; - - private PreparedKafkaAlter(TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode, - List> kafkaPartitionOffsets, - Map stagedCustomProperties, - Map stagedConvertedCustomProperties, - String stagedKafkaDefaultOffset) { - this.validatedUniqueKeyUpdateMode = validatedUniqueKeyUpdateMode; - this.kafkaPartitionOffsets = kafkaPartitionOffsets; - this.stagedCustomProperties = stagedCustomProperties; - this.stagedConvertedCustomProperties = stagedConvertedCustomProperties; - this.stagedKafkaDefaultOffset = stagedKafkaDefaultOffset; } + LOG.info("modify the properties of kafka routine load job: {}, jobProperties: {}, datasource properties: {}", + this.id, jobProperties, dataSourceProperties); } private void resetCloudProgress(Cloud.ResetRLProgressRequest.Builder builder) throws DdlException { @@ -987,7 +910,6 @@ public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { try { modifyPropertiesInternal(log.getJobProperties(), (KafkaDataSourceProperties) log.getDataSourceProperties()); setRoutineLoadDesc(log.getRoutineLoadDesc()); - updateLoadDefinition(log.getDataSourceProperties()); } catch (UserException e) { // should not happen LOG.error("failed to replay modify kafka routine load job: {}", id, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index 5ca8355d851c50..9c5bac1c7895a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -51,8 +51,6 @@ import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TFileCompressType; -import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; -import org.apache.doris.thrift.TUniqueKeyUpdateMode; import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; @@ -66,7 +64,6 @@ import com.google.gson.annotations.SerializedName; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.BooleanUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -187,20 +184,19 @@ private void convertCustomProperties(boolean rebuild) throws DdlException { return; } - Pair, String> convertedProperties = buildConvertedCustomProperties( - customProperties, kinesisDefaultPosition); - convertedCustomProperties.clear(); - convertedCustomProperties.putAll(convertedProperties.first); - kinesisDefaultPosition = convertedProperties.second; - } + if (rebuild) { + convertedCustomProperties.clear(); + } + + for (Map.Entry entry : customProperties.entrySet()) { + convertedCustomProperties.put(entry.getKey(), entry.getValue()); + } - private Pair, String> buildConvertedCustomProperties( - Map sourceProperties, String currentDefaultPosition) { - Map convertedProperties = Maps.newHashMap(sourceProperties); - String convertedDefaultPosition = convertedProperties.getOrDefault( - "kinesis_default_pos", currentDefaultPosition); - // Keep kinesis_default_pos in convertedProperties so BE can use it. - return Pair.of(convertedProperties, convertedDefaultPosition); + // Handle default position + if (convertedCustomProperties.containsKey("kinesis_default_pos")) { + kinesisDefaultPosition = convertedCustomProperties.get("kinesis_default_pos"); + // Keep it in convertedCustomProperties so BE can use it + } } private String convertedDefaultPosition() { @@ -534,7 +530,6 @@ public static KinesisRoutineLoadJob fromCreateInfo(CreateRoutineLoadInfo info, C kinesisRoutineLoadJob.setOptional(info); kinesisRoutineLoadJob.checkCustomProperties(); - kinesisRoutineLoadJob.initializeLoadDefinition(info); return kinesisRoutineLoadJob; } @@ -663,27 +658,6 @@ public Map getCustomProperties() { return getMaskedCustomProperties("property."); } - @Override - protected void updateLoadDefinitionDataSourceProperties(Map dataSourceProperties) { - dataSourceProperties.put(KinesisConfiguration.KINESIS_REGION.getName(), region); - dataSourceProperties.put(KinesisConfiguration.KINESIS_STREAM.getName(), stream); - if (endpoint == null) { - dataSourceProperties.remove(KinesisConfiguration.KINESIS_ENDPOINT.getName()); - dataSourceProperties.remove("kinesis_endpoint"); - } else { - dataSourceProperties.put(KinesisConfiguration.KINESIS_ENDPOINT.getName(), endpoint); - } - dataSourceProperties.remove(KinesisConfiguration.KINESIS_POSITIONS.getName()); - if (customKinesisShards.isEmpty()) { - dataSourceProperties.remove(KinesisConfiguration.KINESIS_SHARDS.getName()); - } else { - dataSourceProperties.put(KinesisConfiguration.KINESIS_SHARDS.getName(), - Joiner.on(",").join(customKinesisShards)); - } - customProperties.forEach((key, value) -> dataSourceProperties.put( - key.startsWith("aws.") ? key : "property." + key, value)); - } - private Map getMaskedCustomProperties(String keyPrefix) { Map maskedProperties = new HashMap<>(); customProperties.forEach((key, value) -> { @@ -712,7 +686,6 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti modifyPropertiesInternal(jobProperties, dataSourceProperties); setRoutineLoadDesc(command.getRoutineLoadDesc()); - updateLoadDefinition(dataSourceProperties); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, jobProperties, dataSourceProperties, command.getRoutineLoadDesc()); @@ -725,120 +698,73 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti private void modifyPropertiesInternal(Map jobProperties, KinesisDataSourceProperties dataSourceProperties) throws UserException { - PreparedKinesisAlter preparedAlter = prepareAlter(jobProperties, dataSourceProperties); - applyAlter(jobProperties, dataSourceProperties, preparedAlter); - LOG.info("modify the properties of kinesis routine load job: {}, jobProperties: {}, datasource properties: {}", - this.id, jobProperties, dataSourceProperties); - } - - private PreparedKinesisAlter prepareAlter(Map jobProperties, - KinesisDataSourceProperties dataSourceProperties) throws UserException { - TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode = validateCommonJobProperties(jobProperties); - List> shardPositions = Lists.newArrayList(); - Map alteredCustomProperties = Maps.newHashMap(); - if (dataSourceProperties != null - && MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { - shardPositions = dataSourceProperties.getKinesisShardPositions(); - alteredCustomProperties = dataSourceProperties.getCustomKinesisProperties(); - } - - Map stagedCustomProperties = null; - Map stagedConvertedCustomProperties = null; - String stagedDefaultPosition = kinesisDefaultPosition; - if (!alteredCustomProperties.isEmpty()) { - stagedCustomProperties = Maps.newHashMap(customProperties); - stagedCustomProperties.putAll(alteredCustomProperties); - Pair, String> convertedProperties = buildConvertedCustomProperties( - stagedCustomProperties, stagedDefaultPosition); - stagedConvertedCustomProperties = convertedProperties.first; - stagedDefaultPosition = convertedProperties.second; - } - - boolean resetProgress = dataSourceProperties != null - && !Strings.isNullOrEmpty(dataSourceProperties.getStream()); - if (!shardPositions.isEmpty() && !resetProgress) { - ((KinesisProgress) progress).checkShards(shardPositions); - } - return new PreparedKinesisAlter(validatedUniqueKeyUpdateMode, shardPositions, - stagedCustomProperties, stagedConvertedCustomProperties, stagedDefaultPosition, resetProgress); - } - - private void applyAlter(Map jobProperties, KinesisDataSourceProperties dataSourceProperties, - PreparedKinesisAlter preparedAlter) { + validateCommonJobProperties(jobProperties); if (dataSourceProperties != null) { - if (preparedAlter.stagedCustomProperties != null) { - customProperties.clear(); - customProperties.putAll(preparedAlter.stagedCustomProperties); - convertedCustomProperties.clear(); - convertedCustomProperties.putAll(preparedAlter.stagedConvertedCustomProperties); - kinesisDefaultPosition = preparedAlter.stagedDefaultPosition; + List> shardPositions = Lists.newArrayList(); + Map customKinesisProperties = Maps.newHashMap(); + boolean resetProgress = false; + boolean hasExplicitShardPositions = false; + + if (MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { + shardPositions = dataSourceProperties.getKinesisShardPositions(); + customKinesisProperties = dataSourceProperties.getCustomKinesisProperties(); + hasExplicitShardPositions = !shardPositions.isEmpty(); + } + + // Update custom properties + if (!customKinesisProperties.isEmpty()) { + this.customProperties.putAll(customKinesisProperties); + convertCustomProperties(true); } + + // Modify stream if provided if (!Strings.isNullOrEmpty(dataSourceProperties.getStream())) { - stream = dataSourceProperties.getStream(); + this.stream = dataSourceProperties.getStream(); + resetProgress = true; } + + // Modify region if provided if (!Strings.isNullOrEmpty(dataSourceProperties.getRegion())) { - region = dataSourceProperties.getRegion(); + this.region = dataSourceProperties.getRegion(); } + + // Modify endpoint if provided if (!Strings.isNullOrEmpty(dataSourceProperties.getEndpoint())) { - endpoint = dataSourceProperties.getEndpoint(); + this.endpoint = dataSourceProperties.getEndpoint(); } - if (preparedAlter.resetProgress) { - progress = new KinesisProgress(); - openKinesisShards.clear(); - closedKinesisShards.clear(); - cachedShardWithMillsBehindLatest.clear(); + + if (resetProgress) { + this.progress = new KinesisProgress(); + this.openKinesisShards.clear(); + this.closedKinesisShards.clear(); + this.cachedShardWithMillsBehindLatest.clear(); } - if (!preparedAlter.shardPositions.isEmpty()) { - customKinesisShards.clear(); - for (Pair shardPosition : preparedAlter.shardPositions) { - customKinesisShards.add(shardPosition.first); + + if (hasExplicitShardPositions) { + this.customKinesisShards.clear(); + for (Pair shardPosition : shardPositions) { + this.customKinesisShards.add(shardPosition.first); } - } else if (preparedAlter.resetProgress) { + } else if (resetProgress) { // Stream change without explicit shards should fall back to dynamic shard discovery. - customKinesisShards.clear(); + this.customKinesisShards.clear(); } - if (!preparedAlter.shardPositions.isEmpty()) { - ((KinesisProgress) progress).modifyPosition(preparedAlter.shardPositions); + + if (!shardPositions.isEmpty()) { + if (!resetProgress) { + ((KinesisProgress) progress).checkShards(shardPositions); + } + ((KinesisProgress) progress).modifyPosition(shardPositions); } } + if (!jobProperties.isEmpty()) { Map copiedJobProperties = Maps.newHashMap(jobProperties); - modifyCommonJobProperties(copiedJobProperties, preparedAlter.validatedUniqueKeyUpdateMode); + modifyCommonJobProperties(copiedJobProperties); this.jobProperties.putAll(copiedJobProperties); - if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { - this.isPartialUpdate = BooleanUtils.toBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); - } - if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { - String policy = jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); - if ("ERROR".equalsIgnoreCase(policy)) { - this.partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.ERROR; - } else { - this.partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.APPEND; - } - } - } - } - - private static class PreparedKinesisAlter { - private final TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode; - private final List> shardPositions; - private final Map stagedCustomProperties; - private final Map stagedConvertedCustomProperties; - private final String stagedDefaultPosition; - private final boolean resetProgress; - - private PreparedKinesisAlter(TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode, - List> shardPositions, - Map stagedCustomProperties, - Map stagedConvertedCustomProperties, - String stagedDefaultPosition, boolean resetProgress) { - this.validatedUniqueKeyUpdateMode = validatedUniqueKeyUpdateMode; - this.shardPositions = shardPositions; - this.stagedCustomProperties = stagedCustomProperties; - this.stagedConvertedCustomProperties = stagedConvertedCustomProperties; - this.stagedDefaultPosition = stagedDefaultPosition; - this.resetProgress = resetProgress; } + LOG.info("modify the properties of kinesis routine load job: {}, jobProperties: {}, datasource properties: {}", + this.id, jobProperties, dataSourceProperties); } @Override @@ -847,7 +773,6 @@ public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { modifyPropertiesInternal(log.getJobProperties(), (KinesisDataSourceProperties) log.getDataSourceProperties()); setRoutineLoadDesc(log.getRoutineLoadDesc()); - updateLoadDefinition(log.getDataSourceProperties()); } catch (UserException e) { LOG.error("failed to replay modify kinesis routine load job: {}", id, e); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index 462803af886af1..ca0748775e0f28 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -29,11 +29,10 @@ import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; -import org.apache.doris.common.io.Text; import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.kafka.KafkaUtil; +import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.kafka.KafkaConfiguration; @@ -50,15 +49,12 @@ import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.persist.EditLog; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.OriginStatement; import org.apache.doris.thrift.TResourceInfo; import org.apache.doris.thrift.TRoutineLoadTask; import com.google.common.base.Joiner; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; import org.apache.kafka.common.PartitionInfo; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -79,7 +75,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.UUID; public class KafkaRoutineLoadJobTest { @@ -287,182 +282,71 @@ public void testUpdateProgressWarnsWhenReadCommittedTaskHasZeroRowsAndLag() thro } @Test - public void testFailedAlterDoesNotChangeRuntimeOrLoadDefinition() throws Exception { - KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "job_atomic", 1L, - 1L, "127.0.0.1:9020", "topic-1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - Map originalCustomProperties = Maps.newHashMap(); - originalCustomProperties.put("client.id", "old-client"); - Deencapsulation.setField(routineLoadJob, "customProperties", originalCustomProperties); - Map originalConvertedProperties = Maps.newHashMap(originalCustomProperties); - Deencapsulation.setField(routineLoadJob, "convertedCustomProperties", originalConvertedProperties); - Map originalProgress = Maps.newHashMap(); - originalProgress.put(0, 10L); - Deencapsulation.setField(routineLoadJob, "progress", new KafkaProgress(originalProgress)); - routineLoadJob.updateLoadDefinition(null); - Object originalLoadDefinition = Deencapsulation.getField(routineLoadJob, "loadDefinition"); - - Map originalDataSourceProperties = Maps.newHashMap(); - originalDataSourceProperties.put("property.client.id", "new-client"); - KafkaDataSourceProperties dataSourceProperties = - new KafkaDataSourceProperties(originalDataSourceProperties); - Map alteredCustomProperties = Maps.newHashMap(); - alteredCustomProperties.put("client.id", "new-client"); - Deencapsulation.setField(dataSourceProperties, "customKafkaProperties", alteredCustomProperties); - dataSourceProperties.setKafkaPartitionOffsets(Lists.newArrayList(Pair.of(1, 20L))); - AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); - Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); - Mockito.when(command.getDataSourceProperties()).thenReturn(dataSourceProperties); - - try (MockedStatic kafkaUtilStatic = Mockito.mockStatic(KafkaUtil.class)) { - kafkaUtilStatic.when(() -> KafkaUtil.getRealOffsets( - Mockito.eq("127.0.0.1:9020"), Mockito.eq("topic-1"), Mockito.anyMap(), Mockito.anyList(), - Mockito.nullable(String.class))) - .thenReturn(Lists.newArrayList(Pair.of(1, 20L))); - Assert.assertThrows(UserException.class, () -> routineLoadJob.modifyProperties(command)); - } - - Assert.assertEquals("topic-1", routineLoadJob.getTopic()); - Map currentCustomProperties = Deencapsulation.getField(routineLoadJob, "customProperties"); - Map currentConvertedProperties = - Deencapsulation.getField(routineLoadJob, "convertedCustomProperties"); - Assert.assertEquals("old-client", currentCustomProperties.get("client.id")); - Assert.assertEquals("old-client", currentConvertedProperties.get("client.id")); - Assert.assertSame(originalLoadDefinition, Deencapsulation.getField(routineLoadJob, "loadDefinition")); - } - - @Test - public void testSuccessfulAlterUpdatesLoadDefinitionAndJournal() throws Exception { - KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "job1", 1L, - 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - routineLoadJob.updateLoadDefinition(null); - Object originalLoadDefinition = Deencapsulation.getField(routineLoadJob, "loadDefinition"); + public void testAlterPersistsLoadDescAndCsvPropertiesForReplay() throws Exception { + KafkaRoutineLoadJob leader = createPausedJob(); + KafkaRoutineLoadJob follower = createPausedJob(); + RoutineLoadDesc originalDesc = new RoutineLoadDesc(new Separator("|", "|"), null, null, + null, null, null, null, LoadTask.MergeType.APPEND, "original_sequence"); + leader.setRoutineLoadDesc(originalDesc); + follower.setRoutineLoadDesc(originalDesc); - RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc(null, null, null, null, null, null, null, - LoadTask.MergeType.APPEND, "sequence_col"); + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "\""); + jobProperties.put(CsvFileFormatProperties.PROP_ESCAPE, "\\"); + jobProperties.put(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL, "true"); + RoutineLoadDesc delta = new RoutineLoadDesc(null, new Separator("\n", "\\n"), null, + null, null, null, null, LoadTask.MergeType.APPEND, null); AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); - Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); Mockito.when(command.getDataSourceProperties()).thenReturn(null); - Mockito.when(command.getRoutineLoadDesc()).thenReturn(routineLoadDesc); + Mockito.when(command.getRoutineLoadDesc()).thenReturn(delta); + Env env = Mockito.mock(Env.class); EditLog editLog = Mockito.mock(EditLog.class); + AlterRoutineLoadJobOperationLog alterLog; try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); Mockito.when(env.getEditLog()).thenReturn(editLog); - routineLoadJob.modifyProperties(command); + leader.modifyProperties(command); ArgumentCaptor logCaptor = ArgumentCaptor.forClass(AlterRoutineLoadJobOperationLog.class); Mockito.verify(editLog).logAlterRoutineLoadJob(logCaptor.capture()); - Assert.assertSame(routineLoadDesc, logCaptor.getValue().getRoutineLoadDesc()); + alterLog = logCaptor.getValue(); } - Assert.assertEquals("sequence_col", routineLoadJob.getSequenceCol()); - Assert.assertNotSame(originalLoadDefinition, Deencapsulation.getField(routineLoadJob, "loadDefinition")); - } - - @Test - public void testImageRoundTripUsesPersistedLoadDefinitionAfterAlterReplay() throws Exception { - KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "job1", 1L, - 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - Deencapsulation.setField(routineLoadJob, "desireTaskConcurrentNum", 1); - String originalCreateSql = "CREATE ROUTINE LOAD db1.job1 ON table1 " - + "COLUMNS TERMINATED BY '|' " - + "FROM KAFKA ('kafka_broker_list' = '127.0.0.1:9020', 'kafka_topic' = 'topic1')"; - Deencapsulation.setField(routineLoadJob, "origStmt", new OriginStatement(originalCreateSql, 0)); - routineLoadJob.updateLoadDefinition(null); - - RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc( - new Separator(",", ","), null, null, null, null, null, null, - LoadTask.MergeType.APPEND, "sequence_col"); - Map jobProperties = Maps.newHashMap(); - jobProperties.put(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY, "2"); - routineLoadJob.replayModifyProperties(new AlterRoutineLoadJobOperationLog( - routineLoadJob.getId(), jobProperties, null, routineLoadDesc)); + Assert.assertSame(delta, alterLog.getRoutineLoadDesc()); + Assert.assertEquals(jobProperties, alterLog.getJobProperties()); + assertAlterState(leader); - Env env = Mockito.mock(Env.class); - InternalCatalog catalog = Mockito.mock(InternalCatalog.class); - CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); - Database database = Mockito.mock(Database.class); - OlapTable table = Mockito.mock(OlapTable.class); - connectContextStatic.close(); - connectContextStatic = null; - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { - envStatic.when(Env::getCurrentEnv).thenReturn(env); - envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); - Mockito.when(env.getInternalCatalog()).thenReturn(catalog); - Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); - Mockito.when(catalogMgr.getCatalog(Mockito.anyString())).thenReturn(catalog); - Mockito.when(catalog.getDb(1L)).thenReturn(Optional.of(database)); - Mockito.when(catalog.getDb("db1")).thenReturn(Optional.of(database)); - Mockito.when(catalog.getDbOrAnalysisException("db1")).thenReturn(database); - Mockito.when(database.getName()).thenReturn("db1"); - Mockito.when(database.getFullName()).thenReturn("db1"); - Mockito.when(database.getTable(1L)).thenReturn(Optional.of((Table) table)); - Mockito.when(database.getTableOrAnalysisException("table1")).thenReturn(table); - Mockito.when(table.getName()).thenReturn("table1"); - Mockito.when(table.getType()).thenReturn(Table.TableType.OLAP); + follower.replayModifyProperties(alterLog); + assertAlterState(follower); - RoutineLoadJob restored = imageRoundTrip(routineLoadJob); - - Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); - Assert.assertEquals("sequence_col", restored.getSequenceCol()); - Assert.assertEquals(",", restored.getColumnSeparator().getSeparator()); - Assert.assertEquals(2, (int) Deencapsulation.getField(restored, "desireTaskConcurrentNum")); - Assert.assertNotNull(Deencapsulation.getField(restored, "loadDefinition")); - - // Simulate an older FE ignoring the unknown loadDefinition field. It can read the image - // through origStmt, but the ALTERed load clauses are intentionally not preserved on rollback. - RoutineLoadJob rollbackRestored = imageRoundTripWithoutLoadDefinition(routineLoadJob); - Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, rollbackRestored.getState()); - Assert.assertEquals("|", rollbackRestored.getColumnSeparator().getSeparator()); - Assert.assertNull(rollbackRestored.getSequenceCol()); - Assert.assertNull(Deencapsulation.getField(rollbackRestored, "loadDefinition")); - } + assertAlterState(imageRoundTrip(leader)); + assertAlterState(imageRoundTrip(follower)); } - @Test - public void testImageRoundTripRestoresLegacyOrigStmt() throws Exception { - KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "job1", 1L, + private static KafkaRoutineLoadJob createPausedJob() { + KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(1L, "job1", 1L, 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - String createSql = "CREATE ROUTINE LOAD db1.job1 ON stale_table " - + "COLUMNS TERMINATED BY ',' " - + "FROM KAFKA ('kafka_broker_list' = '127.0.0.1:9020', 'kafka_topic' = 'topic1')"; - Deencapsulation.setField(routineLoadJob, "origStmt", new OriginStatement(createSql, 0)); - Deencapsulation.setField(routineLoadJob, "loadDefinition", null); - - Env env = Mockito.mock(Env.class); - InternalCatalog catalog = Mockito.mock(InternalCatalog.class); - CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); - Database database = Mockito.mock(Database.class); - OlapTable table = Mockito.mock(OlapTable.class); - connectContextStatic.close(); - connectContextStatic = null; - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { - envStatic.when(Env::getCurrentEnv).thenReturn(env); - envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); - Mockito.when(env.getInternalCatalog()).thenReturn(catalog); - Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); - Mockito.when(catalogMgr.getCatalog(Mockito.anyString())).thenReturn(catalog); - Mockito.when(catalog.getDb(1L)).thenReturn(Optional.of(database)); - Mockito.when(catalog.getDb("db1")).thenReturn(Optional.of(database)); - Mockito.when(catalog.getDbOrAnalysisException("db1")).thenReturn(database); - Mockito.when(database.getName()).thenReturn("db1"); - Mockito.when(database.getTable(1L)).thenReturn(Optional.of((Table) table)); - Mockito.when(database.getTableOrAnalysisException("table1")).thenReturn(table); - Mockito.when(table.getName()).thenReturn("table1"); - Mockito.when(table.getType()).thenReturn(Table.TableType.OLAP); - - RoutineLoadJob restored = imageRoundTrip(routineLoadJob); + Deencapsulation.setField(job, "state", RoutineLoadJob.JobState.PAUSED); + return job; + } - Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); - Assert.assertEquals(",", restored.getColumnSeparator().getSeparator()); - Assert.assertNull(Deencapsulation.getField(restored, "loadDefinition")); - } + private static void assertAlterState(RoutineLoadJob job) { + Assert.assertEquals("|", job.getColumnSeparator().getSeparator()); + Assert.assertEquals("\n", job.getLineDelimiter().getSeparator()); + Assert.assertEquals("original_sequence", job.getSequenceCol()); + Assert.assertEquals((byte) '"', job.getEnclose()); + Assert.assertEquals((byte) '\\', job.getEscape()); + Assert.assertTrue(job.getEmptyFieldAsNull()); + Assert.assertEquals(Boolean.TRUE, Deencapsulation.getField(job, "emptyFieldAsNull")); + + Map persistedJobProperties = Deencapsulation.getField(job, "jobProperties"); + Assert.assertEquals("\"", persistedJobProperties.get(CsvFileFormatProperties.PROP_ENCLOSE)); + Assert.assertEquals("\\", persistedJobProperties.get(CsvFileFormatProperties.PROP_ESCAPE)); + Assert.assertEquals("true", persistedJobProperties.get(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)); } private static RoutineLoadJob imageRoundTrip(RoutineLoadJob routineLoadJob) throws Exception { @@ -475,29 +359,6 @@ private static RoutineLoadJob imageRoundTrip(RoutineLoadJob routineLoadJob) thro } } - private static RoutineLoadJob imageRoundTripWithoutLoadDefinition(RoutineLoadJob routineLoadJob) - throws Exception { - ByteArrayOutputStream image = new ByteArrayOutputStream(); - try (DataOutputStream out = new DataOutputStream(image)) { - routineLoadJob.write(out); - } - - String json; - try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(image.toByteArray()))) { - json = Text.readString(in); - } - JsonObject jobJson = JsonParser.parseString(json).getAsJsonObject(); - Assert.assertNotNull(jobJson.remove("ld")); - - ByteArrayOutputStream legacyImage = new ByteArrayOutputStream(); - try (DataOutputStream out = new DataOutputStream(legacyImage)) { - Text.writeString(out, jobJson.toString()); - } - try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(legacyImage.toByteArray()))) { - return RoutineLoadJob.read(in); - } - } - @Test public void testDisplayCustomPropertiesMasksKafkaSecrets() { KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java index a9face05891ab1..ea64e7e2871b0c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java @@ -17,25 +17,39 @@ package org.apache.doris.load.routineload; +import org.apache.doris.analysis.Separator; import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; -import org.apache.doris.common.Pair; -import org.apache.doris.common.UserException; +import org.apache.doris.common.io.Text; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; +import org.apache.doris.load.RoutineLoadDesc; +import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.kinesis.KinesisConfiguration; import org.apache.doris.load.routineload.kinesis.KinesisDataSourceProperties; import org.apache.doris.load.routineload.kinesis.KinesisProgress; import org.apache.doris.load.routineload.kinesis.KinesisRoutineLoadJob; import org.apache.doris.load.routineload.kinesis.KinesisTaskInfo; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; +import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; +import org.apache.doris.persist.EditLog; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.Gson; +import com.google.gson.JsonParser; import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -234,47 +248,55 @@ public void testModifyPropertiesShouldReplaceCustomShardsWhenExplicitShardsProvi } @Test - public void testFailedAlterDoesNotChangeRuntimeOrLoadDefinition() throws Exception { - KinesisRoutineLoadJob routineLoadJob = new KinesisRoutineLoadJob(1L, "job_atomic", 1L, - 1L, "ap-southeast-1", "stream-1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - Map originalCustomProperties = Maps.newHashMap(); - originalCustomProperties.put("client.id", "old-client"); - Deencapsulation.setField(routineLoadJob, "customProperties", originalCustomProperties); - Map originalConvertedProperties = Maps.newHashMap(originalCustomProperties); - Deencapsulation.setField(routineLoadJob, "convertedCustomProperties", originalConvertedProperties); - Deencapsulation.setField(routineLoadJob, "customKinesisShards", Lists.newArrayList("shard-1")); - Map originalProgress = Maps.newHashMap(); - originalProgress.put("shard-1", "10"); - Deencapsulation.setField(routineLoadJob, "progress", new KinesisProgress(originalProgress)); - routineLoadJob.updateLoadDefinition(null); - Object originalLoadDefinition = Deencapsulation.getField(routineLoadJob, "loadDefinition"); - - Map originalDataSourceProperties = Maps.newHashMap(); - originalDataSourceProperties.put("property.client.id", "new-client"); - originalDataSourceProperties.put(KinesisConfiguration.KINESIS_REGION.getName(), "us-east-1"); - KinesisDataSourceProperties dataSourceProperties = - new KinesisDataSourceProperties(originalDataSourceProperties); - Map alteredCustomProperties = Maps.newHashMap(); - alteredCustomProperties.put("client.id", "new-client"); - Deencapsulation.setField(dataSourceProperties, "customKinesisProperties", alteredCustomProperties); - Deencapsulation.setField(dataSourceProperties, "region", "us-east-1"); - dataSourceProperties.setKinesisShardPositions(Lists.newArrayList(Pair.of("shard-2", "20"))); + public void testAlterReplayKeepsDeltaAndCsvCachesInCheckpointParity() throws Exception { + KinesisRoutineLoadJob leader = createPausedJobWithInitialLoadDesc(); + KinesisRoutineLoadJob replay = createPausedJobWithInitialLoadDesc(); + + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "\""); + jobProperties.put(CsvFileFormatProperties.PROP_ESCAPE, "\\"); + jobProperties.put(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL, "true"); + RoutineLoadDesc delta = new RoutineLoadDesc(null, new Separator("\n", "\\n"), + null, null, null, null, null, LoadTask.MergeType.APPEND, "sequence_col"); AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); - Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); + Mockito.when(command.getDataSourceProperties()).thenReturn(null); + Mockito.when(command.getRoutineLoadDesc()).thenReturn(delta); + + Env env = Mockito.mock(Env.class); + EditLog editLog = Mockito.mock(EditLog.class); + ArgumentCaptor logCaptor = + ArgumentCaptor.forClass(AlterRoutineLoadJobOperationLog.class); + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + Mockito.when(env.getEditLog()).thenReturn(editLog); + leader.modifyProperties(command); + Mockito.verify(editLog).logAlterRoutineLoadJob(logCaptor.capture()); + } + + AlterRoutineLoadJobOperationLog log = logCaptor.getValue(); + replay.replayModifyProperties(log); + + Assert.assertSame(delta, log.getRoutineLoadDesc()); + assertAlterResult(leader); + assertAlterResult(replay); + Assert.assertEquals(JsonParser.parseString(checkpointJson(leader)), + JsonParser.parseString(checkpointJson(replay))); + } + + @Test + public void testAlterValidatesCsvBeforeDataSourceMutation() { + KinesisRoutineLoadJob job = createPausedJobWithInitialLoadDesc(); + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "invalid"); + KinesisDataSourceProperties dataSourceProperties = Mockito.mock(KinesisDataSourceProperties.class); + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); Mockito.when(command.getDataSourceProperties()).thenReturn(dataSourceProperties); - Assert.assertThrows(UserException.class, () -> routineLoadJob.modifyProperties(command)); - - Assert.assertEquals("ap-southeast-1", routineLoadJob.getRegion()); - Assert.assertEquals(Lists.newArrayList("shard-1"), - Deencapsulation.getField(routineLoadJob, "customKinesisShards")); - Map currentCustomProperties = Deencapsulation.getField(routineLoadJob, "customProperties"); - Map currentConvertedProperties = - Deencapsulation.getField(routineLoadJob, "convertedCustomProperties"); - Assert.assertEquals("old-client", currentCustomProperties.get("client.id")); - Assert.assertEquals("old-client", currentConvertedProperties.get("client.id")); - Assert.assertSame(originalLoadDefinition, Deencapsulation.getField(routineLoadJob, "loadDefinition")); + Assert.assertThrows(AnalysisException.class, () -> job.modifyProperties(command)); + Assert.assertEquals("stream-1", job.getStream()); + Mockito.verifyNoInteractions(dataSourceProperties); } @Test @@ -387,6 +409,35 @@ public void testDisplayCustomPropertiesMasksKinesisSecrets() { Assert.assertEquals("role_arn_value", showCreateCustomProperties.get("property.aws.role_arn")); } + private KinesisRoutineLoadJob createPausedJobWithInitialLoadDesc() { + KinesisRoutineLoadJob job = new KinesisRoutineLoadJob(1L, "kinesis_routine_load_job", 1L, + 1L, "ap-southeast-1", "stream-1", UserIdentity.ADMIN); + Deencapsulation.setField(job, "state", RoutineLoadJob.JobState.PAUSED); + Deencapsulation.setField(job, "createTimestamp", 123L); + job.setRoutineLoadDesc(new RoutineLoadDesc(new Separator("|", "|"), null, + null, null, null, null, null, LoadTask.MergeType.APPEND, null)); + return job; + } + + private void assertAlterResult(KinesisRoutineLoadJob job) { + Assert.assertEquals("|", job.getColumnSeparator().getSeparator()); + Assert.assertEquals("\n", job.getLineDelimiter().getSeparator()); + Assert.assertEquals("sequence_col", job.getSequenceCol()); + Assert.assertEquals((byte) '"', job.getEnclose()); + Assert.assertEquals((byte) '\\', job.getEscape()); + Assert.assertTrue(job.getEmptyFieldAsNull()); + } + + private String checkpointJson(RoutineLoadJob job) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(bytes)) { + job.write(out); + } + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + return Text.readString(in); + } + } + private Set collectAssignedShards(KinesisRoutineLoadJob routineLoadJob) { List routineLoadTaskInfoList = Deencapsulation.getField(routineLoadJob, "routineLoadTaskInfoList"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java new file mode 100644 index 00000000000000..7da9e1d3308589 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java @@ -0,0 +1,426 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.load.routineload; + +import org.apache.doris.analysis.BinaryPredicate; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.ImportColumnDesc; +import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.analysis.Separator; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.info.PartitionNamesInfo; +import org.apache.doris.common.io.Text; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; +import org.apache.doris.load.RoutineLoadDesc; +import org.apache.doris.load.loadv2.LoadTask; +import org.apache.doris.load.routineload.kafka.KafkaConfiguration; +import org.apache.doris.load.routineload.kafka.KafkaRoutineLoadJob; +import org.apache.doris.load.routineload.kinesis.KinesisRoutineLoadJob; +import org.apache.doris.nereids.load.NereidsRoutineLoadTaskInfo; +import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; +import org.apache.doris.qe.OriginStatement; +import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; +import org.apache.doris.thrift.TUniqueKeyUpdateMode; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public class RoutineLoadJobPersistenceTest { + private static final String LEGACY_IMAGE = + "/upgrade/routine-load/a8928245/routine-load-kafka-image.b64"; + + @Test + public void testDirectStateImageRoundTripDoesNotParseOrigStmt() throws Exception { + KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(1001L, "direct_job", 1002L, + 1003L, "127.0.0.1:9092", "direct_topic", UserIdentity.ADMIN); + job.state = RoutineLoadJob.JobState.PAUSED; + job.origStmt = new OriginStatement("this is deliberately not valid SQL", 0); + + Separator columnSeparator = analyzedSeparator("\\x01"); + Separator lineDelimiter = analyzedSeparator("\\n"); + List columns = Lists.newArrayList( + new ImportColumnDesc("source_col"), + new ImportColumnDesc("mapped_col", new IntLiteral(7L))); + Expr precedingFilter = predicate(BinaryPredicate.Operator.GT, "source_col", 1L); + Expr whereExpr = predicate(BinaryPredicate.Operator.LE, "mapped_col", 10L); + Expr deleteCondition = predicate(BinaryPredicate.Operator.EQ, "delete_flag", 1L); + PartitionNamesInfo partitions = new PartitionNamesInfo(false, Lists.newArrayList("p1", "p2")); + job.setRoutineLoadDesc(new RoutineLoadDesc(columnSeparator, lineDelimiter, columns, + precedingFilter, whereExpr, partitions, deleteCondition, LoadTask.MergeType.MERGE, "seq_col")); + + job.desireTaskConcurrentNum = 5; + job.maxErrorNum = 17L; + job.maxBatchIntervalS = 23L; + job.maxBatchRows = 300001L; + job.maxBatchSizeBytes = 104857601L; + job.execMemLimit = 345678901L; + job.maxFilterRatio = 0.99; + job.sendBatchParallelism = 99; + job.loadToSingleTablet = false; + job.memtableOnSinkNode = true; + + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CreateRoutineLoadInfo.MAX_FILTER_RATIO_PROPERTY, "0.25"); + jobProperties.put(CreateRoutineLoadInfo.SEND_BATCH_PARALLELISM, "4"); + jobProperties.put(CreateRoutineLoadInfo.LOAD_TO_SINGLE_TABLET, "true"); + jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, "UPDATE_FIXED_COLUMNS"); + jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, "true"); + jobProperties.put(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY, "ERROR"); + jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "\""); + jobProperties.put(CsvFileFormatProperties.PROP_ESCAPE, "\\"); + jobProperties.put(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL, "true"); + job.jobProperties = jobProperties; + + JsonObject json = imageJson(job); + Assert.assertEquals(1, json.get("rlpv").getAsInt()); + for (String key : Lists.newArrayList( + "pni", "cds", "pf", "we", "cs", "lidel", "sc", "mt", "dc", "eml", "mosn")) { + Assert.assertTrue("missing direct-state key " + key, json.has(key)); + } + Assert.assertFalse(json.has("ld")); + Assert.assertEquals("\\x01", json.getAsJsonObject("cs").get("os").getAsString()); + Assert.assertEquals("\u0001", json.getAsJsonObject("cs").get("s").getAsString()); + Assert.assertEquals("\\n", json.getAsJsonObject("lidel").get("os").getAsString()); + Assert.assertEquals("\n", json.getAsJsonObject("lidel").get("s").getAsString()); + Assert.assertEquals(2, json.getAsJsonObject("cds").getAsJsonArray("des").size()); + + RoutineLoadJob restored; + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + restored = imageRoundTrip(job); + envStatic.verifyNoInteractions(); + } + + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); + Assert.assertEquals(Lists.newArrayList("p1", "p2"), restored.getPartitionNamesInfo().getPartitionNames()); + Assert.assertEquals(2, restored.columnDescs.descs.size()); + Assert.assertEquals("source_col", restored.columnDescs.descs.get(0).getColumnName()); + Assert.assertEquals("mapped_col", restored.columnDescs.descs.get(1).getColumnName()); + Assert.assertNotNull(restored.columnDescs.descs.get(1).getExpr()); + Assert.assertNotNull(restored.getPrecedingFilter()); + Assert.assertNotNull(restored.getWhereExpr()); + Assert.assertEquals("\\x01", restored.getColumnSeparator().getOriSeparator()); + Assert.assertEquals("\u0001", restored.getColumnSeparator().getSeparator()); + Assert.assertEquals("\\n", restored.getLineDelimiter().getOriSeparator()); + Assert.assertEquals("\n", restored.getLineDelimiter().getSeparator()); + Assert.assertEquals("seq_col", restored.getSequenceCol()); + Assert.assertEquals(LoadTask.MergeType.MERGE, restored.getMergeType()); + Assert.assertNotNull(restored.getDeleteCondition()); + Assert.assertEquals(345678901L, restored.getMemLimit()); + Assert.assertTrue(restored.isMemtableOnSinkNode()); + Assert.assertEquals(5, restored.desireTaskConcurrentNum); + Assert.assertEquals(17L, restored.maxErrorNum); + Assert.assertEquals(23L, restored.getMaxBatchIntervalS()); + Assert.assertEquals(300001L, restored.getMaxBatchRows()); + Assert.assertEquals(104857601L, restored.getMaxBatchSizeBytes()); + + NereidsRoutineLoadTaskInfo taskInfo = restored.toNereidsRoutineLoadTaskInfo(); + Assert.assertEquals(345678901L, taskInfo.getMemLimit()); + Assert.assertEquals(0.25, taskInfo.getMaxFilterRatio(), 0.0); + Assert.assertEquals(4, taskInfo.getSendBatchParallelism()); + Assert.assertTrue(taskInfo.isLoadToSingleTablet()); + Assert.assertEquals(TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS, taskInfo.getUniqueKeyUpdateMode()); + Assert.assertTrue(taskInfo.isFixedPartialUpdate()); + Assert.assertEquals(TPartialUpdateNewRowPolicy.ERROR, taskInfo.getPartialUpdateNewRowPolicy()); + Assert.assertEquals((byte) '"', taskInfo.getEnclose()); + Assert.assertEquals((byte) '\\', taskInfo.getEscape()); + Assert.assertTrue(taskInfo.getEmptyFieldAsNull()); + Assert.assertTrue(taskInfo.isMemtableOnSinkNode()); + Assert.assertEquals(LoadTask.MergeType.MERGE, taskInfo.getMergeType()); + Assert.assertNotNull(taskInfo.getDeleteCondition()); + Assert.assertEquals("seq_col", taskInfo.getSequenceCol()); + Assert.assertEquals(Lists.newArrayList("p1", "p2"), + taskInfo.getPartitionNamesInfo().getPartitionNames()); + Assert.assertEquals(2, taskInfo.getColumnExprDescs().descs.size()); + Assert.assertNotNull(taskInfo.getPrecedingFilter()); + Assert.assertNotNull(taskInfo.getWhereExpr()); + Assert.assertEquals("\u0001", taskInfo.getColumnSeparator().getSeparator()); + Assert.assertEquals("\n", taskInfo.getLineDelimiter().getSeparator()); + } + + @Test + public void testDirectStateImageWithNoLoadClausesDoesNotFallback() throws Exception { + KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(2001L, "empty_job", 2002L, + 2003L, "127.0.0.1:9092", "empty_topic", UserIdentity.ADMIN); + job.state = RoutineLoadJob.JobState.PAUSED; + job.origStmt = new OriginStatement("also not valid SQL", 0); + + JsonObject json = imageJson(job); + Assert.assertEquals(1, json.get("rlpv").getAsInt()); + for (String key : Lists.newArrayList("pni", "cds", "pf", "we", "cs", "lidel", "sc", "dc")) { + Assert.assertFalse("unexpected nullable direct-state key " + key, json.has(key)); + } + + RoutineLoadJob restored; + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + restored = imageRoundTrip(job); + envStatic.verifyNoInteractions(); + } + + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); + Assert.assertNull(restored.getPartitionNamesInfo()); + Assert.assertNull(restored.columnDescs); + Assert.assertNull(restored.getPrecedingFilter()); + Assert.assertNull(restored.getWhereExpr()); + Assert.assertNull(restored.getColumnSeparator()); + Assert.assertNull(restored.getLineDelimiter()); + Assert.assertNull(restored.getSequenceCol()); + Assert.assertNull(restored.getDeleteCondition()); + Assert.assertEquals(LoadTask.MergeType.APPEND, restored.getMergeType()); + } + + @Test + public void testLegacyImageMigratesOnce() throws Exception { + Env env = Mockito.mock(Env.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Database database = Mockito.mock(Database.class); + OlapTable table = Mockito.mock(OlapTable.class); + Mockito.when(env.getInternalCatalog()).thenReturn(catalog); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(catalogMgr.getCatalog(Mockito.anyString())).thenReturn(catalog); + Mockito.when(catalog.getDb(8001L)).thenReturn(Optional.of(database)); + Mockito.when(catalog.getDb("legacy_db")).thenReturn(Optional.of(database)); + Mockito.when(catalog.getDbOrAnalysisException("legacy_db")).thenReturn(database); + Mockito.when(database.getName()).thenReturn("legacy_db"); + Mockito.when(database.getTable(9001L)).thenReturn(Optional.of((Table) table)); + Mockito.when(database.getTableOrAnalysisException("current_table")).thenReturn(table); + Mockito.when(table.getName()).thenReturn("current_table"); + Mockito.when(table.getType()).thenReturn(Table.TableType.OLAP); + Mockito.when(table.getEnableUniqueKeyMergeOnWrite()).thenReturn(true); + + RoutineLoadJob migrated; + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + migrated = readImage(loadBase64Fixture(LEGACY_IMAGE)); + } + + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, migrated.getState()); + Assert.assertEquals("|", migrated.getColumnSeparator().getOriSeparator()); + Assert.assertEquals("|", migrated.getColumnSeparator().getSeparator()); + Assert.assertNull(migrated.getSequenceCol()); + Assert.assertEquals(33554432L, migrated.getMemLimit()); + Assert.assertEquals(0.25, migrated.getMaxFilterRatio(), 0.0); + Assert.assertEquals(3, migrated.getSendBatchParallelism()); + Assert.assertTrue(migrated.isLoadToSingleTablet()); + Assert.assertEquals(TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS, migrated.getUniqueKeyUpdateMode()); + Assert.assertTrue(migrated.isFixedPartialUpdate()); + Assert.assertEquals(TPartialUpdateNewRowPolicy.ERROR, migrated.partialUpdateNewKeyPolicy); + Assert.assertEquals((byte) '"', migrated.getEnclose()); + Assert.assertEquals((byte) '\\', migrated.getEscape()); + Assert.assertTrue(migrated.getEmptyFieldAsNull()); + Assert.assertFalse(migrated.isMemtableOnSinkNode()); + + JsonObject migratedJson = imageJson(migrated); + Assert.assertEquals(1, migratedJson.get("rlpv").getAsInt()); + Assert.assertTrue(migratedJson.has("cs")); + Assert.assertTrue(migratedJson.has("eml")); + Assert.assertTrue(migratedJson.has("mosn")); + migrated.origStmt = new OriginStatement("invalid after successful migration", 0); + + RoutineLoadJob restoredAgain; + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + restoredAgain = imageRoundTrip(migrated); + envStatic.verifyNoInteractions(); + } + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restoredAgain.getState()); + Assert.assertEquals("|", restoredAgain.getColumnSeparator().getSeparator()); + Assert.assertEquals(33554432L, restoredAgain.getMemLimit()); + Assert.assertFalse(restoredAgain.isMemtableOnSinkNode()); + } + + @Test + public void testKafkaDerivedStateIsRebuiltFromDurableProperties() throws Exception { + KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(3001L, "kafka_derived", 3002L, + 3003L, "127.0.0.1:9092", "derived_topic", UserIdentity.ADMIN); + job.origStmt = new OriginStatement("invalid SQL must stay unused", 0); + Map customProperties = Maps.newHashMap(); + customProperties.put("client.id", "durable-client"); + customProperties.put(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName(), "OFFSET_BEGINNING"); + Deencapsulation.setField(job, "customProperties", customProperties); + Deencapsulation.setField(job, "customKafkaPartitions", Lists.newArrayList(9)); + Deencapsulation.setField(job, "currentKafkaPartitions", Lists.newArrayList(1, 2)); + Deencapsulation.setField(job, "convertedCustomProperties", + Maps.newHashMap(ImmutableMap.of("stale", "value"))); + Deencapsulation.setField(job, "cachedPartitionWithLatestOffsets", + Maps.newHashMap(ImmutableMap.of(1, 100L))); + Deencapsulation.setField(job, "newCurrentKafkaPartition", Lists.newArrayList(3)); + Deencapsulation.setField(job, "kafkaDefaultOffSet", "OFFSET_END"); + + JsonObject json = imageJson(job); + Assert.assertEquals("127.0.0.1:9092", json.get("bl").getAsString()); + Assert.assertEquals("derived_topic", json.get("tp").getAsString()); + Assert.assertEquals("durable-client", json.getAsJsonObject("prop").get("client.id").getAsString()); + Assert.assertEquals(1, json.getAsJsonArray("cskp").size()); + assertNoJavaFieldNames(json, "currentKafkaPartitions", "convertedCustomProperties", + "cachedPartitionWithLatestOffsets", "newCurrentKafkaPartition", "kafkaDefaultOffSet"); + + KafkaRoutineLoadJob restored = (KafkaRoutineLoadJob) imageRoundTrip(job); + Assert.assertEquals("127.0.0.1:9092", restored.getBrokerList()); + Assert.assertEquals("derived_topic", restored.getTopic()); + Assert.assertEquals(Lists.newArrayList(9), Deencapsulation.getField(restored, "customKafkaPartitions")); + Assert.assertTrue(((List) Deencapsulation.getField(restored, "currentKafkaPartitions")).isEmpty()); + Assert.assertTrue(restored.getConvertedCustomProperties().isEmpty()); + Assert.assertTrue(((Map) Deencapsulation.getField( + restored, "cachedPartitionWithLatestOffsets")).isEmpty()); + Assert.assertEquals("", Deencapsulation.getField(restored, "kafkaDefaultOffSet")); + + Env env = Mockito.mock(Env.class); + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + restored.prepare(); + } + Assert.assertEquals("durable-client", restored.getConvertedCustomProperties().get("client.id")); + Assert.assertFalse(restored.getConvertedCustomProperties().containsKey( + KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())); + Assert.assertEquals("OFFSET_BEGINNING", Deencapsulation.getField(restored, "kafkaDefaultOffSet")); + } + + @Test + public void testKinesisDerivedStateIsRebuiltFromDurableProperties() throws Exception { + KinesisRoutineLoadJob job = new KinesisRoutineLoadJob(4001L, "kinesis_derived", 4002L, + 4003L, "us-east-1", "derived_stream", UserIdentity.ADMIN); + job.origStmt = new OriginStatement("invalid SQL must stay unused", 0); + Deencapsulation.setField(job, "endpoint", "https://kinesis.example.test"); + Map customProperties = Maps.newHashMap(); + customProperties.put("client.setting", "durable-value"); + customProperties.put("kinesis_default_pos", "TRIM_HORIZON"); + Deencapsulation.setField(job, "customProperties", customProperties); + Deencapsulation.setField(job, "customKinesisShards", Lists.newArrayList("custom-shard")); + Deencapsulation.setField(job, "openKinesisShards", Lists.newArrayList("open-shard")); + Deencapsulation.setField(job, "closedKinesisShards", Lists.newArrayList("closed-shard")); + Deencapsulation.setField(job, "convertedCustomProperties", + Maps.newHashMap(ImmutableMap.of("stale", "value"))); + Deencapsulation.setField(job, "cachedShardWithMillsBehindLatest", + Maps.newHashMap(ImmutableMap.of("open-shard", 99L))); + Deencapsulation.setField(job, "newCurrentKinesisShards", Lists.newArrayList("new-shard")); + Deencapsulation.setField(job, "kinesisDefaultPosition", "LATEST"); + + JsonObject json = imageJson(job); + Assert.assertEquals("us-east-1", json.get("rg").getAsString()); + Assert.assertEquals("derived_stream", json.get("stm").getAsString()); + Assert.assertEquals("https://kinesis.example.test", json.get("ep").getAsString()); + Assert.assertEquals("durable-value", + json.getAsJsonObject("prop").get("client.setting").getAsString()); + Assert.assertEquals("custom-shard", json.getAsJsonArray("csks").get(0).getAsString()); + Assert.assertEquals("open-shard", json.getAsJsonArray("opks").get(0).getAsString()); + Assert.assertEquals("closed-shard", json.getAsJsonArray("clks").get(0).getAsString()); + assertNoJavaFieldNames(json, "convertedCustomProperties", "cachedShardWithMillsBehindLatest", + "newCurrentKinesisShards", "kinesisDefaultPosition"); + + KinesisRoutineLoadJob restored = (KinesisRoutineLoadJob) imageRoundTrip(job); + Assert.assertEquals("us-east-1", restored.getRegion()); + Assert.assertEquals("derived_stream", restored.getStream()); + Assert.assertEquals("https://kinesis.example.test", restored.getEndpoint()); + Assert.assertEquals(Lists.newArrayList("custom-shard"), + Deencapsulation.getField(restored, "customKinesisShards")); + Assert.assertEquals(Lists.newArrayList("open-shard"), + Deencapsulation.getField(restored, "openKinesisShards")); + Assert.assertEquals(Lists.newArrayList("closed-shard"), + Deencapsulation.getField(restored, "closedKinesisShards")); + Assert.assertTrue(restored.getConvertedCustomProperties().isEmpty()); + Assert.assertTrue(((Map) Deencapsulation.getField( + restored, "cachedShardWithMillsBehindLatest")).isEmpty()); + Assert.assertTrue(((List) Deencapsulation.getField(restored, "newCurrentKinesisShards")).isEmpty()); + Assert.assertEquals("", Deencapsulation.getField(restored, "kinesisDefaultPosition")); + + restored.prepare(); + Assert.assertEquals("durable-value", restored.getConvertedCustomProperties().get("client.setting")); + Assert.assertEquals("TRIM_HORIZON", + restored.getConvertedCustomProperties().get("kinesis_default_pos")); + Assert.assertEquals("TRIM_HORIZON", Deencapsulation.getField(restored, "kinesisDefaultPosition")); + } + + private static Separator analyzedSeparator(String value) throws Exception { + Separator separator = new Separator(value); + separator.analyze(); + return separator; + } + + private static Expr predicate(BinaryPredicate.Operator operator, String column, long value) { + return new BinaryPredicate(operator, new SlotRef(null, column), new IntLiteral(value)); + } + + private static JsonObject imageJson(RoutineLoadJob job) throws IOException { + byte[] image = writeImage(job); + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(image))) { + return JsonParser.parseString(Text.readString(in)).getAsJsonObject(); + } + } + + private static RoutineLoadJob imageRoundTrip(RoutineLoadJob job) throws IOException { + return readImage(writeImage(job)); + } + + private static byte[] writeImage(RoutineLoadJob job) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(bytes)) { + job.write(out); + } + return bytes.toByteArray(); + } + + private static RoutineLoadJob readImage(byte[] image) throws IOException { + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(image))) { + return RoutineLoadJob.read(in); + } + } + + private static byte[] loadBase64Fixture(String resource) throws IOException { + try (InputStream in = RoutineLoadJobPersistenceTest.class.getResourceAsStream(resource)) { + if (in == null) { + throw new IOException("missing fixture " + resource); + } + String base64 = new String(in.readAllBytes(), StandardCharsets.UTF_8).trim(); + return Base64.getDecoder().decode(base64); + } + } + + private static void assertNoJavaFieldNames(JsonObject json, String... fieldNames) { + for (String fieldName : fieldNames) { + Assert.assertFalse("derived field leaked into image: " + fieldName, json.has(fieldName)); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java index 522c8bee4d9948..058cf42f949d54 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java @@ -17,15 +17,22 @@ package org.apache.doris.persist; +import org.apache.doris.analysis.BinaryPredicate; +import org.apache.doris.analysis.ImportColumnDesc; +import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.analysis.Separator; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.common.UserException; -import org.apache.doris.common.io.Text; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.kafka.KafkaConfiguration; import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; +import org.apache.doris.persist.gson.GsonUtils; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Test; @@ -34,23 +41,19 @@ import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; import java.util.Map; public class AlterRoutineLoadOperationLogTest { - private static String fileName = "./AlterRoutineLoadOperationLogTest"; + private static final String A8928245_LEGACY_LOG = + "/upgrade/routine-load/a8928245/alter-routine-load-log.b64"; @Test public void testSerializeAlterRoutineLoadOperationLog() throws IOException, UserException { - // 1. Write objects to file - File file = new File(fileName); - file.createNewFile(); - file.deleteOnExit(); - DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); - long jobId = 1000; Map jobProperties = Maps.newHashMap(); jobProperties.put(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY, "5"); @@ -65,18 +68,31 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User routineLoadDataSourceProperties.setTimezone(TimeUtils.DEFAULT_TIME_ZONE); routineLoadDataSourceProperties.analyze(); - RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc(null, null, null, null, null, null, null, - LoadTask.MergeType.APPEND, "sequence_col"); + Separator columnSeparator = new Separator(",", "\\x2c"); + Separator lineDelimiter = new Separator("\n", "\\n"); + List columns = Lists.newArrayList( + new ImportColumnDesc("source_col"), + new ImportColumnDesc("mapped_col", new StringLiteral("mapped_value"))); + BinaryPredicate precedingFilter = new BinaryPredicate(BinaryPredicate.Operator.GT, + new IntLiteral(3L), new IntLiteral(2L)); + BinaryPredicate where = new BinaryPredicate(BinaryPredicate.Operator.EQ, + new StringLiteral("selected"), new StringLiteral("selected")); + PartitionNamesInfo partitions = new PartitionNamesInfo(true, Lists.newArrayList("p1", "p2")); + BinaryPredicate deleteCondition = new BinaryPredicate(BinaryPredicate.Operator.EQ, + new IntLiteral(1L), new IntLiteral(1L)); + RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc(columnSeparator, lineDelimiter, columns, + precedingFilter, where, partitions, deleteCondition, LoadTask.MergeType.MERGE, "sequence_col"); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(jobId, jobProperties, routineLoadDataSourceProperties, routineLoadDesc); - log.write(out); - out.flush(); - out.close(); - - // 2. Read objects from file - DataInputStream in = new DataInputStream(new FileInputStream(file)); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(bytes)) { + log.write(out); + } - AlterRoutineLoadJobOperationLog log2 = AlterRoutineLoadJobOperationLog.read(in); + AlterRoutineLoadJobOperationLog log2; + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + log2 = AlterRoutineLoadJobOperationLog.read(in); + } Assert.assertEquals(1, log2.getJobProperties().size()); Assert.assertEquals("5", log2.getJobProperties().get(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)); KafkaDataSourceProperties kafkaDataSourceProperties = (KafkaDataSourceProperties) log2.getDataSourceProperties(); @@ -88,24 +104,46 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User kafkaDataSourceProperties.getKafkaPartitionOffsets().get(0)); Assert.assertEquals(routineLoadDataSourceProperties.getKafkaPartitionOffsets().get(1), kafkaDataSourceProperties.getKafkaPartitionOffsets().get(1)); - Assert.assertEquals("sequence_col", log2.getRoutineLoadDesc().getSequenceColName()); - - in.close(); + RoutineLoadDesc restoredDesc = log2.getRoutineLoadDesc(); + Assert.assertEquals(",", restoredDesc.getColumnSeparator().getSeparator()); + Assert.assertEquals("\\x2c", restoredDesc.getColumnSeparator().getOriSeparator()); + Assert.assertEquals("\n", restoredDesc.getLineDelimiter().getSeparator()); + Assert.assertEquals("\\n", restoredDesc.getLineDelimiter().getOriSeparator()); + Assert.assertEquals(2, restoredDesc.getColumnsInfo().size()); + Assert.assertEquals("source_col", restoredDesc.getColumnsInfo().get(0).getColumnName()); + Assert.assertEquals("mapped_col", restoredDesc.getColumnsInfo().get(1).getColumnName()); + Assert.assertNotNull(restoredDesc.getColumnsInfo().get(1).getExpr()); + Assert.assertNotNull(restoredDesc.getPrecedingFilter()); + Assert.assertNotNull(restoredDesc.getFilter()); + Assert.assertTrue(restoredDesc.getPartitionNamesInfo().isTemp()); + Assert.assertEquals(Lists.newArrayList("p1", "p2"), + restoredDesc.getPartitionNamesInfo().getPartitionNames()); + Assert.assertNotNull(restoredDesc.getDeleteCondition()); + Assert.assertEquals(LoadTask.MergeType.MERGE, restoredDesc.getMergeType()); + Assert.assertEquals("sequence_col", restoredDesc.getSequenceColName()); + Assert.assertEquals(GsonUtils.GSON.toJson(routineLoadDesc), GsonUtils.GSON.toJson(restoredDesc)); } @Test public void testDeserializeLegacyLogWithoutRoutineLoadDesc() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - try (DataOutputStream out = new DataOutputStream(bytes)) { - Text.writeString(out, "{\"jobId\":1000,\"jobProperties\":{}," - + "\"dataSourceProperties\":null}"); - } - - try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + byte[] bytes = loadBase64Fixture(A8928245_LEGACY_LOG); + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes))) { AlterRoutineLoadJobOperationLog log = AlterRoutineLoadJobOperationLog.read(in); - Assert.assertEquals(1000L, log.getJobId()); + Assert.assertEquals(7001L, log.getJobId()); + Assert.assertTrue(log.getJobProperties().isEmpty()); + Assert.assertNull(log.getDataSourceProperties()); Assert.assertNull(log.getRoutineLoadDesc()); } } + private static byte[] loadBase64Fixture(String resource) throws IOException { + try (InputStream in = AlterRoutineLoadOperationLogTest.class.getResourceAsStream(resource)) { + if (in == null) { + throw new IOException("missing fixture " + resource); + } + String base64 = new String(in.readAllBytes(), StandardCharsets.UTF_8).trim(); + return Base64.getDecoder().decode(base64); + } + } + } From 419ac458fcbce0e374f3cc6e8999ab11b07d688a Mon Sep 17 00:00:00 2001 From: Refrain Date: Mon, 17 Aug 2026 00:34:56 +0800 Subject: [PATCH 4/4] temp --- .../load/routineload/RoutineLoadJob.java | 20 ++++++++---------- .../RoutineLoadJobPersistenceTest.java | 21 ++++++++++++++----- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 57497876059a81..c01edd1ddcd28b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -116,8 +116,6 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback implements Writable, LoadTaskInfo, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(RoutineLoadJob.class); - private static final int CURRENT_ROUTINE_LOAD_PERSISTENCE_VERSION = 1; - public static final long DEFAULT_MAX_ERROR_NUM = 0; public static final double DEFAULT_MAX_FILTER_RATIO = 1.0; @@ -180,9 +178,6 @@ public boolean isFinalState() { protected long dbId; @SerializedName("tbid") protected long tableId; - // An absent version identifies a legacy record whose CREATE statement still needs to be migrated. - @SerializedName("rlpv") - private int routineLoadPersistenceVersion; // this code is used to verify be task request protected long authCode; // protected RoutineLoadDesc routineLoadDesc; // optional @@ -283,7 +278,7 @@ public boolean isFinalState() { protected ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); @SerializedName("mt") - protected LoadTask.MergeType mergeType = LoadTask.MergeType.APPEND; // default is all data is load no delete + protected LoadTask.MergeType mergeType; @SerializedName("dc") protected Expr deleteCondition; // TODO(ml): error sample @@ -330,7 +325,7 @@ public RoutineLoadJob(Long id, String name, this.tableId = tableId; this.authCode = 0; this.userIdentity = userIdentity; - this.routineLoadPersistenceVersion = CURRENT_ROUTINE_LOAD_PERSISTENCE_VERSION; + this.mergeType = LoadTask.MergeType.APPEND; if (ConnectContext.get() != null) { SessionVariable var = ConnectContext.get().getSessionVariable(); @@ -360,7 +355,7 @@ public RoutineLoadJob(Long id, String name, this.authCode = 0; this.userIdentity = userIdentity; this.isMultiTable = true; - this.routineLoadPersistenceVersion = CURRENT_ROUTINE_LOAD_PERSISTENCE_VERSION; + this.mergeType = LoadTask.MergeType.APPEND; if (ConnectContext.get() != null) { SessionVariable var = ConnectContext.get().getSessionVariable(); @@ -1983,16 +1978,19 @@ public void gsonPostProcess() throws IOException { if (tableId == 0) { isMultiTable = true; } - if (routineLoadPersistenceVersion == 0) { + // Legacy images did not persist mergeType. New images always contain it, including jobs + // without any load clause, so its absence is sufficient to identify the one-time fallback. + boolean isOldImage = mergeType == null; + if (isOldImage) { + mergeType = LoadTask.MergeType.APPEND; // Legacy images did not persist this create-time session option. Preserve their historical // post-restart behavior instead of inheriting the image-loading thread's ConnectContext. memtableOnSinkNode = false; } try { hydrateJobProperties(); - if (routineLoadPersistenceVersion == 0) { + if (isOldImage) { restoreLegacyDefinition(); - routineLoadPersistenceVersion = CURRENT_ROUTINE_LOAD_PERSISTENCE_VERSION; } } catch (Exception e) { this.state = JobState.CANCELLED; diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java index 7da9e1d3308589..5c221334e22254 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java @@ -114,7 +114,8 @@ public void testDirectStateImageRoundTripDoesNotParseOrigStmt() throws Exception job.jobProperties = jobProperties; JsonObject json = imageJson(job); - Assert.assertEquals(1, json.get("rlpv").getAsInt()); + Assert.assertTrue(json.has("ostmt")); + Assert.assertEquals(LoadTask.MergeType.MERGE.name(), json.get("mt").getAsString()); for (String key : Lists.newArrayList( "pni", "cds", "pf", "we", "cs", "lidel", "sc", "mt", "dc", "eml", "mosn")) { Assert.assertTrue("missing direct-state key " + key, json.has(key)); @@ -187,7 +188,8 @@ public void testDirectStateImageWithNoLoadClausesDoesNotFallback() throws Except job.origStmt = new OriginStatement("also not valid SQL", 0); JsonObject json = imageJson(job); - Assert.assertEquals(1, json.get("rlpv").getAsInt()); + Assert.assertTrue(json.has("ostmt")); + Assert.assertEquals(LoadTask.MergeType.APPEND.name(), json.get("mt").getAsString()); for (String key : Lists.newArrayList("pni", "cds", "pf", "we", "cs", "lidel", "sc", "dc")) { Assert.assertFalse("unexpected nullable direct-state key " + key, json.has(key)); } @@ -230,11 +232,16 @@ public void testLegacyImageMigratesOnce() throws Exception { Mockito.when(table.getType()).thenReturn(Table.TableType.OLAP); Mockito.when(table.getEnableUniqueKeyMergeOnWrite()).thenReturn(true); + byte[] legacyImage = loadBase64Fixture(LEGACY_IMAGE); + JsonObject legacyJson = imageJson(legacyImage); + Assert.assertFalse(legacyJson.has("mt")); + Assert.assertTrue(legacyJson.has("ostmt")); + RoutineLoadJob migrated; try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); - migrated = readImage(loadBase64Fixture(LEGACY_IMAGE)); + migrated = readImage(legacyImage); } Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, migrated.getState()); @@ -254,7 +261,8 @@ public void testLegacyImageMigratesOnce() throws Exception { Assert.assertFalse(migrated.isMemtableOnSinkNode()); JsonObject migratedJson = imageJson(migrated); - Assert.assertEquals(1, migratedJson.get("rlpv").getAsInt()); + Assert.assertTrue(migratedJson.has("ostmt")); + Assert.assertEquals(LoadTask.MergeType.APPEND.name(), migratedJson.get("mt").getAsString()); Assert.assertTrue(migratedJson.has("cs")); Assert.assertTrue(migratedJson.has("eml")); Assert.assertTrue(migratedJson.has("mosn")); @@ -384,7 +392,10 @@ private static Expr predicate(BinaryPredicate.Operator operator, String column, } private static JsonObject imageJson(RoutineLoadJob job) throws IOException { - byte[] image = writeImage(job); + return imageJson(writeImage(job)); + } + + private static JsonObject imageJson(byte[] image) throws IOException { try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(image))) { return JsonParser.parseString(Text.readString(in)).getAsJsonObject(); }