Skip to content

Commit 4a34f9f

Browse files
committed
[feature](routine-load) Support composable target table alteration
### What problem does this PR solve? Issue Number: None Related PR: #64878 Problem Summary: ALTER ROUTINE LOAD previously used the ON target-table syntax and could not combine a target switch with supported job or data-source properties. Add the explicit SET TARGET TABLE syntax, preserve Doris property constraints, validate and apply target/job/source changes atomically, and keep unspecified Kafka and Kinesis default positions unchanged. ### Release note ALTER ROUTINE LOAD now uses SET TARGET TABLE = "table" and can combine a target table switch with supported PROPERTIES and FROM data-source properties. ### Check List (For Author) - Test: Regression test / Unit Test / Build - FE routine-load and parser unit tests - load_p0/routine_load/test_routine_load_alter regression test - ./build.sh --fe -j48 - Behavior changed: Yes. ALTER ROUTINE LOAD target-table syntax and composability changed as described above. - Does this need documentation: Yes (tracked by Related PR #64878)
1 parent 0f1be9e commit 4a34f9f

15 files changed

Lines changed: 817 additions & 145 deletions

File tree

fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java

Lines changed: 74 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -470,6 +470,12 @@ protected void setRoutineLoadDesc(RoutineLoadDesc routineLoadDesc) {
470470
}
471471

472472
public void validateTargetTable(Database db, OlapTable targetTable) throws UserException {
473+
validateTargetTable(db, targetTable, Maps.newHashMap(), uniqueKeyUpdateMode);
474+
}
475+
476+
public void validateTargetTable(Database db, OlapTable targetTable,
477+
Map<String, String> alteredJobProperties, TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode)
478+
throws UserException {
473479
if (isMultiTable) {
474480
throw new AnalysisException("ALTER ROUTINE LOAD target table change only supports single-table job");
475481
}
@@ -479,17 +485,75 @@ public void validateTargetTable(Database db, OlapTable targetTable) throws UserE
479485
}
480486
checkMeta(targetTable, new RoutineLoadDesc(columnSeparator, lineDelimiter, columnsInfo, precedingFilter,
481487
whereExpr, partitionNamesInfo, deleteCondition, mergeType, sequenceCol));
488+
validateAlterJobProperties(targetTable, alteredJobProperties, effectiveUniqueKeyUpdateMode);
482489

483490
targetTable.readLock();
484491
try {
485-
NereidsStreamLoadPlanner planner = new NereidsStreamLoadPlanner(db, targetTable,
486-
toNereidsRoutineLoadTaskInfo());
492+
NereidsRoutineLoadTaskInfo taskInfo = toNereidsRoutineLoadTaskInfo();
493+
taskInfo.applyAlterPropertiesForValidation(alteredJobProperties, effectiveUniqueKeyUpdateMode);
494+
NereidsStreamLoadPlanner planner = new NereidsStreamLoadPlanner(db, targetTable, taskInfo);
487495
planner.plan(new TUniqueId(0, 0));
488496
} finally {
489497
targetTable.readUnlock();
490498
}
491499
}
492500

501+
public void validateAlterJobProperties(OlapTable targetTable, Map<String, String> alteredJobProperties,
502+
TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode) throws UserException {
503+
if (effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) {
504+
return;
505+
}
506+
if (isMultiTable) {
507+
throw new AnalysisException("Partial update is not supported in multi-table load.");
508+
}
509+
if (effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS) {
510+
if (!targetTable.getEnableUniqueKeyMergeOnWrite()) {
511+
throw new AnalysisException("load by PARTIAL_COLUMNS is only supported in unique table MoW");
512+
}
513+
return;
514+
}
515+
Preconditions.checkState(effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS,
516+
effectiveUniqueKeyUpdateMode);
517+
validateFlexiblePartialUpdateForAlter(targetTable, alteredJobProperties);
518+
}
519+
520+
public static TUniqueKeyUpdateMode getEffectiveUniqueKeyUpdateMode(TUniqueKeyUpdateMode currentUniqueKeyUpdateMode,
521+
Map<String, String> alteredJobProperties) {
522+
if (alteredJobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) {
523+
return TUniqueKeyUpdateMode.valueOf(
524+
alteredJobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE));
525+
}
526+
if (alteredJobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)
527+
&& Boolean.parseBoolean(alteredJobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS))
528+
&& currentUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) {
529+
return TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS;
530+
}
531+
return currentUniqueKeyUpdateMode;
532+
}
533+
534+
protected void validateAlterJobPropertiesForMutation(AlterRoutineLoadCommand command) throws UserException {
535+
Map<String, String> alteredJobProperties = command.getAnalyzedJobProperties();
536+
TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode = getEffectiveUniqueKeyUpdateMode(
537+
uniqueKeyUpdateMode, alteredJobProperties);
538+
if (effectiveUniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) {
539+
return;
540+
}
541+
542+
Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId);
543+
if (db == null) {
544+
throw new DdlException("Database not found: " + dbId);
545+
}
546+
long effectiveTableId = command.hasTargetTable() ? command.getTargetTableId() : tableId;
547+
Table table = db.getTableNullable(effectiveTableId);
548+
if (table == null) {
549+
throw new DdlException("Table not found: " + effectiveTableId);
550+
}
551+
if (!(table instanceof OlapTable)) {
552+
throw new DdlException("Partial update is only supported for OLAP tables");
553+
}
554+
validateAlterJobProperties((OlapTable) table, alteredJobProperties, effectiveUniqueKeyUpdateMode);
555+
}
556+
493557
@Override
494558
public long getId() {
495559
return id;
@@ -2093,10 +2157,6 @@ protected void modifyCommonJobProperties(Map<String, String> jobProperties) thro
20932157
if (hasExplicitUniqueKeyUpdateMode) {
20942158
String modeStr = jobProperties.remove(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE);
20952159
TUniqueKeyUpdateMode newMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode(modeStr);
2096-
// Validate flexible partial update constraints when changing to UPDATE_FLEXIBLE_COLUMNS
2097-
if (newMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS) {
2098-
validateFlexiblePartialUpdateForAlter();
2099-
}
21002160
this.uniqueKeyUpdateMode = newMode;
21012161
this.isPartialUpdate = (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS);
21022162
this.jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name());
@@ -2120,43 +2180,27 @@ protected void modifyCommonJobProperties(Map<String, String> jobProperties) thro
21202180
/**
21212181
* Validate flexible partial update constraints when altering routine load job.
21222182
*/
2123-
private void validateFlexiblePartialUpdateForAlter() throws UserException {
2124-
// Multi-table load does not support flexible partial update
2125-
if (isMultiTable) {
2126-
throw new DdlException("Flexible partial update is not supported in multi-table load");
2127-
}
2128-
2129-
// Get the table to check table-level constraints
2130-
Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId);
2131-
if (db == null) {
2132-
throw new DdlException("Database not found: " + dbId);
2133-
}
2134-
Table table = db.getTableNullable(tableId);
2135-
if (table == null) {
2136-
throw new DdlException("Table not found: " + tableId);
2137-
}
2138-
if (!(table instanceof OlapTable)) {
2139-
throw new DdlException("Flexible partial update is only supported for OLAP tables");
2140-
}
2141-
OlapTable olapTable = (OlapTable) table;
2142-
2183+
private void validateFlexiblePartialUpdateForAlter(OlapTable targetTable,
2184+
Map<String, String> alteredJobProperties) throws UserException {
21432185
// Validate table-level constraints (MoW, skip_bitmap, light_schema_change, variant columns)
2144-
olapTable.validateForFlexiblePartialUpdate();
2186+
targetTable.validateForFlexiblePartialUpdate();
2187+
Map<String, String> effectiveJobProperties = Maps.newHashMap(jobProperties);
2188+
effectiveJobProperties.putAll(alteredJobProperties);
21452189

21462190
// Routine load specific validations
21472191
// Must use JSON format
2148-
String format = this.jobProperties.getOrDefault(FileFormatProperties.PROP_FORMAT, "csv");
2192+
String format = effectiveJobProperties.getOrDefault(FileFormatProperties.PROP_FORMAT, "csv");
21492193
if (!"json".equalsIgnoreCase(format)) {
21502194
throw new DdlException("Flexible partial update only supports JSON format, but current job uses: "
21512195
+ format);
21522196
}
21532197
// Cannot use fuzzy_parse
2154-
if (Boolean.parseBoolean(this.jobProperties.getOrDefault(
2198+
if (Boolean.parseBoolean(effectiveJobProperties.getOrDefault(
21552199
JsonFileFormatProperties.PROP_FUZZY_PARSE, "false"))) {
21562200
throw new DdlException("Flexible partial update does not support fuzzy_parse");
21572201
}
21582202
// Cannot use jsonpaths
2159-
String jsonPaths = getJsonPaths();
2203+
String jsonPaths = effectiveJobProperties.get(JsonFileFormatProperties.PROP_JSON_PATHS);
21602204
if (jsonPaths != null && !jsonPaths.isEmpty()) {
21612205
throw new DdlException("Flexible partial update does not support jsonpaths");
21622206
}

fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaDataSourceProperties.java

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,14 @@ public class KafkaDataSourceProperties extends AbstractDataSourceProperties {
105105
.add(KafkaConfiguration.KAFKA_TEXT_TABLE_NAME_FIELD_INDEX.getName())
106106
.build();
107107

108+
private static final ImmutableSet<String> CREATE_ONLY_MULTI_TABLE_PROPERTIES_SET =
109+
new ImmutableSet.Builder<String>()
110+
.add(KafkaConfiguration.KAFKA_TABLE_NAME_LOCATION.getName())
111+
.add(KafkaConfiguration.KAFKA_TABLE_NAME_FORMAT.getName())
112+
.add(KafkaConfiguration.KAFKA_TEXT_TABLE_NAME_FIELD_DELIMITER.getName())
113+
.add(KafkaConfiguration.KAFKA_TEXT_TABLE_NAME_FIELD_INDEX.getName())
114+
.build();
115+
108116
public KafkaDataSourceProperties(Map<String, String> dataSourceProperties, boolean multiLoad) {
109117
super(dataSourceProperties, multiLoad);
110118
}
@@ -132,6 +140,14 @@ public void convertAndCheckDataSourceProperties() throws UserException {
132140
if (optional.isPresent()) {
133141
throw new AnalysisException(optional.get() + " is invalid kafka property or can not be set");
134142
}
143+
if (isAlter()) {
144+
Optional<String> createOnlyProperty = originalDataSourceProperties.keySet().stream()
145+
.filter(CREATE_ONLY_MULTI_TABLE_PROPERTIES_SET::contains).findFirst();
146+
if (createOnlyProperty.isPresent()) {
147+
throw new AnalysisException(createOnlyProperty.get() + " can only be set when creating "
148+
+ "a multi-table routine load job");
149+
}
150+
}
135151

136152
this.brokerList = KafkaConfiguration.KAFKA_BROKER_LIST.getParameterValue(originalDataSourceProperties
137153
.get(KafkaConfiguration.KAFKA_BROKER_LIST.getName()));
@@ -166,24 +182,41 @@ public void convertAndCheckDataSourceProperties() throws UserException {
166182
//check offset
167183
List<String> offsets = KafkaConfiguration.KAFKA_OFFSETS.getParameterValue(originalDataSourceProperties
168184
.get(KafkaConfiguration.KAFKA_OFFSETS.getName()));
169-
String defaultOffsetString = originalDataSourceProperties
185+
boolean hasCustomDefaultOffset = customKafkaProperties
186+
.containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName());
187+
String defaultOffsetString = customKafkaProperties.get(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName());
188+
boolean hasDirectDefaultOffset = originalDataSourceProperties
189+
.containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName());
190+
String directDefaultOffsetString = originalDataSourceProperties
170191
.get(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName());
171-
if (CollectionUtils.isNotEmpty(offsets) && StringUtils.isNotBlank(defaultOffsetString)) {
192+
if (hasDirectDefaultOffset) {
193+
if (hasCustomDefaultOffset) {
194+
throw new AnalysisException("Only one of " + KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()
195+
+ " and property." + KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName() + " can be set.");
196+
}
197+
defaultOffsetString = directDefaultOffsetString;
198+
customKafkaProperties.put(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName(), defaultOffsetString);
199+
}
200+
boolean hasDefaultOffset = hasCustomDefaultOffset || hasDirectDefaultOffset;
201+
if (CollectionUtils.isNotEmpty(offsets) && hasDefaultOffset) {
172202
throw new AnalysisException("Only one of " + KafkaConfiguration.KAFKA_OFFSETS.getName() + " and "
173203
+ KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName() + " can be set.");
174204
}
175205
if (multiTable) {
176206
checkAndSetMultiLoadProperties();
177207
}
178208
if (isAlter() && CollectionUtils.isNotEmpty(partitions) && CollectionUtils.isEmpty(offsets)
179-
&& StringUtils.isBlank(defaultOffsetString)) {
209+
&& !hasDefaultOffset) {
180210
// if this is an alter operation, the partition and (default)offset must be set together.
181211
throw new AnalysisException("Must set offset or default offset with partition property");
182212
}
183213
if (CollectionUtils.isNotEmpty(offsets)) {
184214
this.isOffsetsForTimes = analyzeKafkaOffsetProperty(offsets);
185215
return;
186216
}
217+
if (isAlter() && CollectionUtils.isEmpty(partitions) && !hasDefaultOffset) {
218+
return;
219+
}
187220
this.isOffsetsForTimes = analyzeKafkaDefaultOffsetProperty();
188221
if (CollectionUtils.isNotEmpty(kafkaPartitionOffsets)) {
189222
defaultOffsetString = customKafkaProperties.get(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName());

0 commit comments

Comments
 (0)