From 62970fb84885638d60db27a179d433f4b69aa358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 26 May 2026 14:00:12 +0800 Subject: [PATCH 01/16] [core][flink] Detect row-id reassign conflicts for index commits --- .../java/org/apache/paimon/CoreOptions.java | 12 ++ .../main/java/org/apache/paimon/Snapshot.java | 1 + .../btree/BTreeGlobalIndexBuilder.java | 9 + .../paimon/operation/FileStoreCommitImpl.java | 41 ++++- .../operation/commit/ConflictDetection.java | 80 +++++++++ .../paimon/operation/FileStoreCommitTest.java | 125 +++++++++++++- .../commit/ConflictDetectionTest.java | 162 ++++++++++++++++++ .../flink/btree/BTreeIndexTopoBuilder.java | 34 +++- .../GenericGlobalIndexBuilder.java | 21 ++- .../globalindex/GenericIndexTopoBuilder.java | 33 +++- 10 files changed, 502 insertions(+), 16 deletions(-) diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index da734fa9386f..1f211c9e8abc 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -2207,6 +2207,14 @@ public InlineElement getDescription() { + "APPEND snapshot which committed files to fixed bucket, commit will be aborted." + "If the value of this option is -1, committer will not check for its first commit."); + public static final ConfigOption COMMIT_ROW_ID_REASSIGN_LAST_SAFE_SNAPSHOT = + ConfigOptions.key("commit.row-id-reassign.last-safe-snapshot") + .longType() + .noDefaultValue() + .withDescription( + "If set, committer will check if there are row-id reassignment snapshots starting from the " + + "snapshot after this one. If found, commit will be aborted."); + public static final ConfigOption CLUSTERING_COLUMNS = key("clustering.columns") .stringType() @@ -3873,6 +3881,10 @@ public Optional commitStrictModeLastSafeSnapshot() { return options.getOptional(COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT); } + public Optional commitRowIdReassignLastSafeSnapshot() { + return options.getOptional(COMMIT_ROW_ID_REASSIGN_LAST_SAFE_SNAPSHOT); + } + public List clusteringColumns() { return clusteringColumns(options.get(CLUSTERING_COLUMNS)); } diff --git a/paimon-api/src/main/java/org/apache/paimon/Snapshot.java b/paimon-api/src/main/java/org/apache/paimon/Snapshot.java index 93f525a7e3fb..d7f0ca2131cf 100644 --- a/paimon-api/src/main/java/org/apache/paimon/Snapshot.java +++ b/paimon-api/src/main/java/org/apache/paimon/Snapshot.java @@ -45,6 +45,7 @@ public class Snapshot implements Serializable { private static final long serialVersionUID = 1L; public static final long FIRST_SNAPSHOT_ID = 1; + public static final String ROW_ID_REASSIGN_PROPERTY = "row-id-reassign"; protected static final int CURRENT_VERSION = 3; diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexBuilder.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexBuilder.java index ad68d83eb340..febf86d055db 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexBuilder.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexBuilder.java @@ -97,6 +97,7 @@ public class BTreeGlobalIndexBuilder implements Serializable { // readRowType is composed by partition fields, indexed field and _ROW_ID field private RowType readRowType; @Nullable private Snapshot snapshot; + @Nullable private Long scanSnapshotId; @Nullable private PartitionPredicate partitionPredicate; @@ -133,6 +134,10 @@ public BTreeGlobalIndexBuilder withSnapshot(Snapshot snapshot) { return this; } + public Optional scanSnapshotId() { + return Optional.ofNullable(scanSnapshotId); + } + public Optional>> scan() { SnapshotReader snapshotReader = table.newSnapshotReader(); if (partitionPredicate != null) { @@ -143,8 +148,10 @@ public Optional>> scan() { ? this.snapshot : snapshotReader.snapshotManager().latestSnapshot(); if (snapshot == null) { + scanSnapshotId = null; return Optional.empty(); } + scanSnapshotId = snapshot.id(); snapshotReader = withManifestEntryFilter(snapshotReader.withSnapshot(snapshot)); Range dataRange = new Range(0, snapshot.nextRowId() - 1); @@ -164,8 +171,10 @@ public Optional>> incrementalScan() { ? this.snapshot : snapshotReader.snapshotManager().latestSnapshot(); if (snapshot == null) { + scanSnapshotId = null; return Optional.empty(); } + scanSnapshotId = snapshot.id(); snapshotReader = withManifestEntryFilter(snapshotReader.withSnapshot(snapshot)); Preconditions.checkArgument(indexField != null, "indexField must be set before scan."); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java index 10c9b20a0467..ad5bc4a04455 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java @@ -220,6 +220,8 @@ public FileStoreCommitImpl( id)) .orElse(null); this.conflictDetection = conflictDetectFactory.create(scanner); + options.commitRowIdReassignLastSafeSnapshot() + .ifPresent(this.conflictDetection::setRowIdReassignCheckFromSnapshot); this.commitCleaner = new CommitCleaner(manifestList, manifestFile, indexManifestFile); } @@ -327,6 +329,9 @@ public int commit(ManifestCommittable committable, boolean checkAppendFiles) { checkAppendFiles = true; allowRollback = true; } + if (conflictDetection.hasRowIdReassignCheckFromSnapshot()) { + checkAppendFiles = true; + } attempts += tryCommit( @@ -1116,7 +1121,8 @@ public boolean replaceManifestList( baseManifestList, deltaManifestList, latest.indexManifest(), - latest.nextRowId()); + latest.nextRowId(), + withoutRowIdReassignProperties(latest.properties())); } public boolean replaceManifestList( @@ -1126,6 +1132,24 @@ public boolean replaceManifestList( Pair deltaManifestList, @Nullable String indexManifest, @Nullable Long nextRowId) { + return replaceManifestList( + latest, + totalRecordCount, + baseManifestList, + deltaManifestList, + indexManifest, + nextRowId, + withoutRowIdReassignProperties(latest.properties())); + } + + public boolean replaceManifestList( + Snapshot latest, + long totalRecordCount, + Pair baseManifestList, + Pair deltaManifestList, + @Nullable String indexManifest, + @Nullable Long nextRowId, + @Nullable Map properties) { Snapshot newSnapshot = new Snapshot( latest.id() + 1, @@ -1147,7 +1171,7 @@ public boolean replaceManifestList( latest.watermark(), latest.statistics(), // if empty properties, just set to null - latest.properties(), + properties, nextRowId); return commitSnapshotImpl(newSnapshot, emptyList()); @@ -1226,12 +1250,23 @@ private boolean compactManifestOnce() { null, latestSnapshot.watermark(), latestSnapshot.statistics(), - latestSnapshot.properties(), + withoutRowIdReassignProperties(latestSnapshot.properties()), latestSnapshot.nextRowId()); return commitSnapshotImpl(newSnapshot, emptyList()); } + private static @Nullable Map withoutRowIdReassignProperties( + @Nullable Map properties) { + if (properties == null || !properties.containsKey(Snapshot.ROW_ID_REASSIGN_PROPERTY)) { + return properties; + } + + Map copied = new HashMap<>(properties); + copied.remove(Snapshot.ROW_ID_REASSIGN_PROPERTY); + return copied.isEmpty() ? null : copied; + } + private boolean commitSnapshotImpl(Snapshot newSnapshot, List deltaStatistics) { try { List statistics = new ArrayList<>(deltaStatistics.size()); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java index 493d13d88dee..fc4e1afe77ff 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java @@ -95,6 +95,7 @@ protected boolean removeEldestEntry(Map.Entry eldest) { private @Nullable PartitionExpire partitionExpire; private @Nullable Long rowIdCheckFromSnapshot = null; + private @Nullable Long rowIdReassignCheckFromSnapshot = null; public ConflictDetection( String tableName, @@ -131,6 +132,14 @@ public boolean hasRowIdCheckFromSnapshot() { return rowIdCheckFromSnapshot != null; } + public void setRowIdReassignCheckFromSnapshot(@Nullable Long rowIdReassignCheckFromSnapshot) { + this.rowIdReassignCheckFromSnapshot = rowIdReassignCheckFromSnapshot; + } + + public boolean hasRowIdReassignCheckFromSnapshot() { + return rowIdReassignCheckFromSnapshot != null; + } + @Nullable public Comparator keyComparator() { return keyComparator; @@ -237,6 +246,11 @@ public Optional checkConflicts( return exception; } + exception = checkRowIdReassignConflicts(latestSnapshot, deltaEntries, deltaIndexEntries); + if (exception.isPresent()) { + return exception; + } + return checkForRowIdFromSnapshot( latestSnapshot, deltaEntries, deltaIndexEntries, rowIdColumnConflictChecker); } @@ -544,6 +558,72 @@ private Optional checkForRowIdFromSnapshot( return Optional.empty(); } + private Optional checkRowIdReassignConflicts( + Snapshot latestSnapshot, + List deltaEntries, + List deltaIndexEntries) { + if (!dataEvolutionEnabled) { + return Optional.empty(); + } + if (rowIdReassignCheckFromSnapshot == null) { + return Optional.empty(); + } + if (latestSnapshot.id() <= rowIdReassignCheckFromSnapshot) { + return Optional.empty(); + } + + List changedPartitions = + changedPartitionsIncludingAllIndexFiles(deltaEntries, deltaIndexEntries); + for (long id = rowIdReassignCheckFromSnapshot + 1; id <= latestSnapshot.id(); id++) { + Snapshot snapshot = snapshotManager.snapshot(id); + if (snapshot.commitKind() != CommitKind.OVERWRITE) { + continue; + } + if (hasRowIdReassignProperty(snapshot.properties())) { + return Optional.of( + new RuntimeException( + String.format( + "Row-id reassignment snapshot %s was committed after the " + + "task planned from snapshot %s. The task must " + + "be retried with the latest row ids.", + id, rowIdReassignCheckFromSnapshot))); + } + if (overwriteChangedTargetPartitions(snapshot, changedPartitions)) { + return Optional.of( + new RuntimeException( + String.format( + "Overwrite snapshot %s changed partitions after the " + + "task planned from snapshot %s. The task must " + + "be retried with the latest row ids.", + id, rowIdReassignCheckFromSnapshot))); + } + } + return Optional.empty(); + } + + private boolean hasRowIdReassignProperty(@Nullable Map properties) { + return properties != null + && Boolean.parseBoolean(properties.get(Snapshot.ROW_ID_REASSIGN_PROPERTY)); + } + + private boolean overwriteChangedTargetPartitions( + Snapshot snapshot, List changedPartitions) { + return !changedPartitions.isEmpty() + && !commitScanner.readIncrementalEntries(snapshot, changedPartitions).isEmpty(); + } + + private List changedPartitionsIncludingAllIndexFiles( + List dataFileChanges, List indexFileChanges) { + Set changedPartitions = new HashSet<>(); + for (SimpleFileEntry file : dataFileChanges) { + changedPartitions.add(file.partition()); + } + for (IndexManifestEntry file : indexFileChanges) { + changedPartitions.add(file.partition()); + } + return new ArrayList<>(changedPartitions); + } + Optional checkRowIdExistence( List baseEntries, List deltaEntries, diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java index e13b11d474b6..122bd609a98e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java @@ -57,6 +57,7 @@ import org.apache.paimon.types.RowKind; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.FailingFileIO; +import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.SnapshotManager; import org.apache.paimon.utils.TraceableFileIO; @@ -1008,6 +1009,100 @@ public void testCommitManifestWithProperties() throws Exception { } } + @Test + public void testReplaceManifestListWithRowIdReassignProperty() throws Exception { + TestFileStore store = createStore(false); + + List keyValues = generateDataList(1); + BinaryRow partition = gen.getPartition(keyValues.get(0)); + Snapshot latest = store.commitData(keyValues, s -> partition, kv -> 0).get(0); + + Map reassignProperties = new HashMap<>(); + reassignProperties.put("keep", "v1"); + reassignProperties.put(Snapshot.ROW_ID_REASSIGN_PROPERTY, "true"); + try (FileStoreCommitImpl commit = store.newCommit()) { + assertThat( + commit.replaceManifestList( + latest, + latest.totalRecordCount(), + baseManifestList(latest), + deltaManifestList(latest), + latest.indexManifest(), + latest.nextRowId(), + reassignProperties)) + .isTrue(); + } + + Snapshot reassignSnapshot = checkNotNull(store.snapshotManager().latestSnapshot()); + assertThat(reassignSnapshot.properties()).isEqualTo(reassignProperties); + + try (FileStoreCommitImpl commit = store.newCommit()) { + assertThat( + commit.replaceManifestList( + reassignSnapshot, + reassignSnapshot.totalRecordCount(), + baseManifestList(reassignSnapshot), + deltaManifestList(reassignSnapshot), + reassignSnapshot.indexManifest(), + reassignSnapshot.nextRowId())) + .isTrue(); + } + + Snapshot normalSnapshot = checkNotNull(store.snapshotManager().latestSnapshot()); + assertThat(normalSnapshot.properties()) + .containsEntry("keep", "v1") + .doesNotContainKey(Snapshot.ROW_ID_REASSIGN_PROPERTY); + } + + @Test + public void testRowIdReassignConflictFromOptions() throws Exception { + TestFileStore store = createStore(false); + + List keyValues = generateDataList(1); + BinaryRow partition = gen.getPartition(keyValues.get(0)); + Snapshot latest = store.commitData(keyValues, s -> partition, kv -> 0).get(0); + + Map reassignProperties = new HashMap<>(); + reassignProperties.put(Snapshot.ROW_ID_REASSIGN_PROPERTY, "true"); + try (FileStoreCommitImpl commit = store.newCommit()) { + assertThat( + commit.replaceManifestList( + latest, + latest.totalRecordCount(), + baseManifestList(latest), + deltaManifestList(latest), + latest.indexManifest(), + latest.nextRowId(), + reassignProperties)) + .isTrue(); + } + + AtomicReference committableRef = new AtomicReference<>(); + store.commitDataImpl( + generateDataList(1), + gen::getPartition, + kv -> 0, + false, + null, + null, + Collections.emptyList(), + (commit, committable) -> committableRef.set(committable)); + + Map dynamicOptions = new HashMap<>(store.options().toMap()); + dynamicOptions.put(CoreOptions.COMMIT_ROW_ID_REASSIGN_LAST_SAFE_SNAPSHOT.key(), "1"); + try (FileStoreCommitImpl commit = + newCommitWithSnapshotCommit( + store, + "row-id-reassign-check", + new RenamingSnapshotCommit(store.snapshotManager(), Lock.empty()), + new CoreOptions(dynamicOptions), + true)) { + assertThatThrownBy(() -> commit.commit(checkNotNull(committableRef.get()), false)) + .hasMessageContaining("Row-id reassignment snapshot 2") + .hasMessageContaining("task planned from snapshot 1"); + } + } + @Test public void testCommitTwiceWithDifferentKind() throws Exception { TestFileStore store = createStore(false); @@ -1082,6 +1177,20 @@ public void testCommitRetryAfterFalseSuccessDoesNotCleanManifest() throws Except private FileStoreCommitImpl newCommitWithSnapshotCommit( TestFileStore store, String commitUser, SnapshotCommit snapshotCommit) { + return newCommitWithSnapshotCommit( + store, + commitUser, + snapshotCommit, + store.options(), + store.options().dataEvolutionEnabled()); + } + + private FileStoreCommitImpl newCommitWithSnapshotCommit( + TestFileStore store, + String commitUser, + SnapshotCommit snapshotCommit, + CoreOptions options, + boolean dataEvolutionEnabled) { String tableName = store.options().path().getName(); return new FileStoreCommitImpl( snapshotCommit, @@ -1090,7 +1199,7 @@ private FileStoreCommitImpl newCommitWithSnapshotCommit( tableName, commitUser, store.partitionType(), - store.options(), + options, store.pathFactory(), store.snapshotManager(), store.manifestFileFactory(), @@ -1109,9 +1218,9 @@ private FileStoreCommitImpl newCommitWithSnapshotCommit( store.pathFactory(), store.newKeyComparator(), store.bucketMode(), - store.options().deletionVectorsEnabled(), - store.options().dataEvolutionEnabled(), - store.options().pkClusteringOverride(), + options.deletionVectorsEnabled(), + dataEvolutionEnabled, + options.pkClusteringOverride(), store.newIndexFileHandler(), store.snapshotManager(), scanner), @@ -1153,6 +1262,14 @@ private TestFileStore createStore(boolean failing, Map options) return createStore(failing, 1, CoreOptions.ChangelogProducer.NONE, options); } + private Pair baseManifestList(Snapshot snapshot) { + return Pair.of(snapshot.baseManifestList(), snapshot.baseManifestListSize()); + } + + private Pair deltaManifestList(Snapshot snapshot) { + return Pair.of(snapshot.deltaManifestList(), snapshot.deltaManifestListSize()); + } + private TestFileStore createStore(boolean failing) throws Exception { return createStore(failing, 1); } diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java index 1c36b9e09ee7..e243fe3b8ecd 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java @@ -18,15 +18,20 @@ package org.apache.paimon.operation.commit; +import org.apache.paimon.Snapshot; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.index.DeletionVectorMeta; +import org.apache.paimon.index.GlobalIndexMeta; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.manifest.FileEntry; import org.apache.paimon.manifest.FileKind; import org.apache.paimon.manifest.IndexManifestEntry; +import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.SimpleFileEntry; import org.apache.paimon.manifest.SimpleFileEntryWithDV; import org.apache.paimon.table.BucketMode; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.SnapshotManager; import org.junit.jupiter.api.Test; @@ -36,8 +41,10 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import static org.apache.paimon.data.BinaryRow.EMPTY_ROW; @@ -47,6 +54,8 @@ import static org.apache.paimon.operation.commit.ConflictDetection.buildBaseEntriesWithDV; import static org.apache.paimon.operation.commit.ConflictDetection.buildDeltaEntriesWithDV; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; class ConflictDetectionTest { @@ -330,6 +339,15 @@ private IndexManifestEntry createDvIndexEntry( DELETION_VECTORS_INDEX, fileName, 11, dvRanges.size(), dvRanges, null)); } + private IndexManifestEntry createIndexEntry( + String fileName, FileKind kind, BinaryRow partition) { + return new IndexManifestEntry( + kind, + partition, + 0, + new IndexFileMeta("btree", fileName, 11, 1, (GlobalIndexMeta) null, null)); + } + private void assertConflict( List baseEntries, List deltaEntries) { ArrayList simpleFileEntryWithDVS = new ArrayList<>(baseEntries); @@ -489,7 +507,122 @@ private SimpleFileEntry createFileEntryWithRowId( firstRowId); } + @Test + void testDetectsRowIdReassignSnapshotConflict() { + SnapshotManager snapshotManager = mock(SnapshotManager.class); + Map reassignProperties = new HashMap<>(); + reassignProperties.put(Snapshot.ROW_ID_REASSIGN_PROPERTY, "true"); + when(snapshotManager.snapshot(2L)) + .thenReturn(snapshot(2, Snapshot.CommitKind.OVERWRITE, reassignProperties)); + + ConflictDetection detection = createConflictDetection(snapshotManager); + detection.setRowIdReassignCheckFromSnapshot(1L); + + Optional exception = + detection.checkConflicts( + snapshot(3, null), + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), + null, + Snapshot.CommitKind.COMPACT); + + assertThat(exception).isPresent(); + assertThat(exception.get()) + .hasMessageContaining("Row-id reassignment snapshot 2") + .hasMessageContaining("task planned from snapshot 1"); + } + + @Test + void testIgnoresRowIdReassignPropertyOnNonOverwriteSnapshot() { + SnapshotManager snapshotManager = mock(SnapshotManager.class); + Map reassignProperties = new HashMap<>(); + reassignProperties.put(Snapshot.ROW_ID_REASSIGN_PROPERTY, "true"); + when(snapshotManager.snapshot(2L)) + .thenReturn(snapshot(2, Snapshot.CommitKind.APPEND, reassignProperties)); + + ConflictDetection detection = createConflictDetection(snapshotManager); + detection.setRowIdReassignCheckFromSnapshot(1L); + + Optional exception = + detection.checkConflicts( + snapshot(2, null), + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), + null, + Snapshot.CommitKind.COMPACT); + + assertThat(exception).isNotPresent(); + } + + @Test + void testDetectsOverlappedOverwriteSnapshotForIndexCommit() { + SnapshotManager snapshotManager = mock(SnapshotManager.class); + CommitScanner commitScanner = mock(CommitScanner.class); + Snapshot overwriteSnapshot = snapshot(2, Snapshot.CommitKind.OVERWRITE, null); + when(snapshotManager.snapshot(2L)).thenReturn(overwriteSnapshot); + + BinaryRow partition = BinaryRow.singleColumn(1); + when(commitScanner.readIncrementalEntries( + overwriteSnapshot, Collections.singletonList(partition))) + .thenReturn(Collections.singletonList(mock(ManifestEntry.class))); + + ConflictDetection detection = createConflictDetection(snapshotManager, commitScanner); + detection.setRowIdReassignCheckFromSnapshot(1L); + + Optional exception = + detection.checkConflicts( + snapshot(2, null), + Collections.emptyList(), + Collections.emptyList(), + Collections.singletonList(createIndexEntry("idx", ADD, partition)), + null, + Snapshot.CommitKind.COMPACT); + + assertThat(exception).isPresent(); + assertThat(exception.get()) + .hasMessageContaining("Overwrite snapshot 2") + .hasMessageContaining("task planned from snapshot 1"); + } + + @Test + void testIgnoresNonOverlappedOverwriteSnapshotForIndexCommit() { + SnapshotManager snapshotManager = mock(SnapshotManager.class); + CommitScanner commitScanner = mock(CommitScanner.class); + Snapshot overwriteSnapshot = snapshot(2, Snapshot.CommitKind.OVERWRITE, null); + when(snapshotManager.snapshot(2L)).thenReturn(overwriteSnapshot); + + BinaryRow partition = BinaryRow.singleColumn(1); + when(commitScanner.readIncrementalEntries( + overwriteSnapshot, Collections.singletonList(partition))) + .thenReturn(Collections.emptyList()); + + ConflictDetection detection = createConflictDetection(snapshotManager, commitScanner); + detection.setRowIdReassignCheckFromSnapshot(1L); + + Optional exception = + detection.checkConflicts( + snapshot(2, null), + Collections.emptyList(), + Collections.emptyList(), + Collections.singletonList(createIndexEntry("idx", ADD, partition)), + null, + Snapshot.CommitKind.COMPACT); + + assertThat(exception).isNotPresent(); + } + private ConflictDetection createConflictDetection() { + return createConflictDetection(null); + } + + private ConflictDetection createConflictDetection(@Nullable SnapshotManager snapshotManager) { + return createConflictDetection(snapshotManager, null); + } + + private ConflictDetection createConflictDetection( + @Nullable SnapshotManager snapshotManager, @Nullable CommitScanner commitScanner) { return new ConflictDetection( "test-table", "test-user", @@ -501,7 +634,36 @@ private ConflictDetection createConflictDetection() { true, false, null, + snapshotManager, + commitScanner); + } + + private Snapshot snapshot(long id, @Nullable Map properties) { + return snapshot(id, Snapshot.CommitKind.APPEND, properties); + } + + private Snapshot snapshot( + long id, Snapshot.CommitKind commitKind, @Nullable Map properties) { + return new Snapshot( + id, + 0, + null, + null, + null, + null, + null, + null, + null, + "commit-user", + id, + commitKind, + id, + 0, + 0, + null, + null, null, + properties, null); } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java index 9542eea4a435..896e87a10ff7 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java @@ -95,6 +95,7 @@ public static boolean buildIndex( Options userOptions) throws Exception { List> allStreams = new ArrayList<>(); + Long rowIdReassignCheckFromSnapshot = null; for (String indexColumn : indexColumns) { BTreeGlobalIndexBuilder indexBuilder = indexBuilderSupplier.get().withIndexField(indexColumn); @@ -107,6 +108,12 @@ public static boolean buildIndex( if (!indexRangeAndSplits.isPresent()) { continue; } + if (indexBuilder.scanSnapshotId().isPresent()) { + rowIdReassignCheckFromSnapshot = + minSnapshot( + rowIdReassignCheckFromSnapshot, + indexBuilder.scanSnapshotId().get()); + } Pair> scanResult = indexRangeAndSplits.get(); List splits = splitByContiguousRowRange(scanResult.getRight()); @@ -194,13 +201,17 @@ public static boolean buildIndex( @SuppressWarnings("unchecked") DataStream[] rest = allStreams.subList(1, allStreams.size()).toArray(new DataStream[0]); - commit(table, allStreams.get(0).union(rest)); + commit(table, allStreams.get(0).union(rest), rowIdReassignCheckFromSnapshot); return true; } return false; } + private static Long minSnapshot(Long left, long right) { + return left == null ? right : Math.min(left, right); + } + public static void buildIndexAndExecute( StreamExecutionEnvironment env, FileStoreTable table, @@ -312,7 +323,11 @@ private static RowType withBuildTaskId(RowType readType, String buildTaskIdField return new RowType(readType.isNullable(), fields); } - private static void commit(FileStoreTable table, DataStream written) { + private static void commit( + FileStoreTable table, + DataStream written, + Long rowIdReassignCheckFromSnapshot) { + FileStoreTable commitTable = withRowIdReassignCheck(table, rowIdReassignCheckFromSnapshot); OneInputStreamOperatorFactory committerOperator = new CommitterOperatorFactory<>( false, @@ -320,7 +335,9 @@ private static void commit(FileStoreTable table, DataStream written "BTreeIndexCommitter-" + UUID.randomUUID(), context -> new StoreCommitter( - table, table.newCommit(context.commitUser()), context), + commitTable, + commitTable.newCommit(context.commitUser()), + context), new NoopCommittableStateManager()); written.transform("COMMIT OPERATOR", new CommittableTypeInfo(), committerOperator) @@ -328,6 +345,17 @@ private static void commit(FileStoreTable table, DataStream written .setMaxParallelism(1); } + private static FileStoreTable withRowIdReassignCheck( + FileStoreTable table, Long rowIdReassignCheckFromSnapshot) { + if (rowIdReassignCheckFromSnapshot == null) { + return table; + } + return table.copy( + Collections.singletonMap( + CoreOptions.COMMIT_ROW_ID_REASSIGN_LAST_SAFE_SNAPSHOT.key(), + String.valueOf(rowIdReassignCheckFromSnapshot))); + } + /** Operator to read data from splits. */ private static class ReadDataOperator extends org.apache.flink.table.runtime.operators.TableStreamOperator diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericGlobalIndexBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericGlobalIndexBuilder.java index 48ea935f5b62..da4644524c1c 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericGlobalIndexBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericGlobalIndexBuilder.java @@ -18,6 +18,7 @@ package org.apache.paimon.flink.globalindex; +import org.apache.paimon.Snapshot; import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.partition.PartitionPredicate; @@ -28,6 +29,7 @@ import java.io.Serializable; import java.util.Collections; import java.util.List; +import java.util.Optional; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -39,6 +41,7 @@ public class GenericGlobalIndexBuilder implements Serializable { protected final FileStoreTable table; @Nullable protected PartitionPredicate partitionPredicate; + @Nullable protected Long scanSnapshotId; public GenericGlobalIndexBuilder(FileStoreTable table) { this.table = table; @@ -53,6 +56,10 @@ public FileStoreTable table() { return table; } + public Optional scanSnapshotId() { + return Optional.ofNullable(scanSnapshotId); + } + /** * Scans manifest entries to determine which files need to be indexed. * @@ -72,7 +79,19 @@ public List scan() { + "deleted rows to be indexed.", table.name()); - return table.store().newScan().withPartitionFilter(partitionPredicate).plan().files(); + Snapshot snapshot = table.snapshotManager().latestSnapshot(); + if (snapshot == null) { + scanSnapshotId = null; + return Collections.emptyList(); + } + scanSnapshotId = snapshot.id(); + + return table.store() + .newScan() + .withSnapshot(snapshot) + .withPartitionFilter(partitionPredicate) + .plan() + .files(); } /** Returns old index file entries that should be deleted after new indexes are built. */ diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java index 5896503ce09d..0e4e0dec19f7 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java @@ -179,6 +179,10 @@ public static boolean buildIndex( List entries = indexBuilder.scan(); List deletedIndexEntries = indexBuilder.deletedIndexEntries(); + Long rowIdReassignCheckFromSnapshot = + indexBuilder.scanSnapshotId().isPresent() + ? indexBuilder.scanSnapshotId().get() + : null; return buildTopology( env, @@ -188,7 +192,8 @@ public static boolean buildIndex( userOptions, entries, deletedIndexEntries, - maxIndexedRowId); + maxIndexedRowId, + rowIdReassignCheckFromSnapshot); } /** @@ -208,7 +213,8 @@ private static boolean buildTopology( Options userOptions, List entries, List deletedIndexEntries, - long maxIndexedRowId) + long maxIndexedRowId, + Long rowIdReassignCheckFromSnapshot) throws Exception { long totalRowCount = entries.stream().mapToLong(e -> e.file().rowCount()).sum(); LOG.info( @@ -294,7 +300,7 @@ private static boolean buildTopology( built = built.union(deletes); } - commit(table, indexType, built); + commit(table, indexType, built, rowIdReassignCheckFromSnapshot); return true; } @@ -506,7 +512,11 @@ private static List createDeleteCommittables( } private static void commit( - FileStoreTable table, String indexType, DataStream written) { + FileStoreTable table, + String indexType, + DataStream written, + Long rowIdReassignCheckFromSnapshot) { + FileStoreTable commitTable = withRowIdReassignCheck(table, rowIdReassignCheckFromSnapshot); OneInputStreamOperatorFactory committerOperator = new CommitterOperatorFactory<>( false, @@ -514,7 +524,9 @@ private static void commit( "GenericIndexCommitter-" + indexType + "-" + UUID.randomUUID(), context -> new StoreCommitter( - table, table.newCommit(context.commitUser()), context), + commitTable, + commitTable.newCommit(context.commitUser()), + context), new NoopCommittableStateManager()); written.transform("COMMIT OPERATOR", new CommittableTypeInfo(), committerOperator) @@ -522,6 +534,17 @@ private static void commit( .setMaxParallelism(1); } + private static FileStoreTable withRowIdReassignCheck( + FileStoreTable table, Long rowIdReassignCheckFromSnapshot) { + if (rowIdReassignCheckFromSnapshot == null) { + return table; + } + return table.copy( + Collections.singletonMap( + CoreOptions.COMMIT_ROW_ID_REASSIGN_LAST_SAFE_SNAPSHOT.key(), + String.valueOf(rowIdReassignCheckFromSnapshot))); + } + /** Serializable descriptor for one shard's work. Each shard has its own DataSplit and Range. */ static class ShardTask implements Serializable { private static final long serialVersionUID = 1L; From d1014a14038f3efba8039097000b9ea400420b61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 26 May 2026 16:36:04 +0800 Subject: [PATCH 02/16] [docs] Document row-id reassign commit option --- docs/generated/core_configuration.html | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 1a66fb0ab0b8..42023c88b02d 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -320,6 +320,12 @@ Long If set, committer will check if there are other commit user's snapshot starting from the snapshot after this one. If found a COMPACT / OVERWRITE snapshot, or found a APPEND snapshot which committed files to fixed bucket, commit will be aborted.If the value of this option is -1, committer will not check for its first commit. + +
commit.row-id-reassign.last-safe-snapshot
+ (none) + Long + If set, committer will check if there are row-id reassignment snapshots starting from the snapshot after this one. If found, commit will be aborted. +
commit.timeout
(none) From 308565ca15295ad921d9624267cf6496307ac435 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 26 May 2026 17:36:18 +0800 Subject: [PATCH 03/16] [core][flink] Address row-id reassign review comments --- .../paimon/operation/FileStoreCommitTest.java | 49 +++++++++++++++++++ .../flink/btree/BTreeIndexTopoBuilder.java | 16 ++---- .../globalindex/GenericIndexTopoBuilder.java | 20 ++------ .../globalindex/GlobalIndexCommitUtils.java | 43 ++++++++++++++++ 4 files changed, 100 insertions(+), 28 deletions(-) create mode 100644 paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GlobalIndexCommitUtils.java diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java index 122bd609a98e..d5701f87e76d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java @@ -1054,6 +1054,55 @@ public void testReplaceManifestListWithRowIdReassignProperty() throws Exception .doesNotContainKey(Snapshot.ROW_ID_REASSIGN_PROPERTY); } + @Test + public void testCompactManifestWithRowIdReassignProperty() throws Exception { + TestFileStore store = createStore(false); + + List keyValues = generateDataList(1); + BinaryRow partition = gen.getPartition(keyValues.get(0)); + store.commitData(keyValues, s -> partition, kv -> 0); + store.overwriteData(keyValues, s -> partition, kv -> 0, Collections.emptyMap()); + Snapshot latest = + store.overwriteData(keyValues, s -> partition, kv -> 0, Collections.emptyMap()) + .get(0); + + long deleteNum = + store.manifestListFactory().create().readDataManifests(latest).stream() + .mapToLong(ManifestFileMeta::numDeletedFiles) + .sum(); + assertThat(deleteNum).isGreaterThan(0); + + Map reassignProperties = new HashMap<>(); + reassignProperties.put("keep", "v1"); + reassignProperties.put(Snapshot.ROW_ID_REASSIGN_PROPERTY, "true"); + try (FileStoreCommitImpl commit = store.newCommit()) { + assertThat( + commit.replaceManifestList( + latest, + latest.totalRecordCount(), + baseManifestList(latest), + deltaManifestList(latest), + latest.indexManifest(), + latest.nextRowId(), + reassignProperties)) + .isTrue(); + } + + Snapshot reassignSnapshot = checkNotNull(store.snapshotManager().latestSnapshot()); + assertThat(reassignSnapshot.properties()).isEqualTo(reassignProperties); + + try (FileStoreCommit commit = store.newCommit()) { + commit.compactManifest(); + } + + Snapshot normalSnapshot = checkNotNull(store.snapshotManager().latestSnapshot()); + assertThat(normalSnapshot.id()).isGreaterThan(reassignSnapshot.id()); + assertThat(normalSnapshot.commitKind()).isEqualTo(Snapshot.CommitKind.COMPACT); + assertThat(normalSnapshot.properties()) + .containsEntry("keep", "v1") + .doesNotContainKey(Snapshot.ROW_ID_REASSIGN_PROPERTY); + } + @Test public void testRowIdReassignConflictFromOptions() throws Exception { TestFileStore store = createStore(false); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java index 896e87a10ff7..4a0655ac6590 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java @@ -27,6 +27,7 @@ import org.apache.paimon.flink.FlinkRowData; import org.apache.paimon.flink.FlinkRowWrapper; import org.apache.paimon.flink.LogicalTypeConversion; +import org.apache.paimon.flink.globalindex.GlobalIndexCommitUtils; import org.apache.paimon.flink.sink.Committable; import org.apache.paimon.flink.sink.CommittableTypeInfo; import org.apache.paimon.flink.sink.CommitterOperatorFactory; @@ -327,7 +328,9 @@ private static void commit( FileStoreTable table, DataStream written, Long rowIdReassignCheckFromSnapshot) { - FileStoreTable commitTable = withRowIdReassignCheck(table, rowIdReassignCheckFromSnapshot); + FileStoreTable commitTable = + GlobalIndexCommitUtils.withRowIdReassignCheck( + table, rowIdReassignCheckFromSnapshot); OneInputStreamOperatorFactory committerOperator = new CommitterOperatorFactory<>( false, @@ -345,17 +348,6 @@ private static void commit( .setMaxParallelism(1); } - private static FileStoreTable withRowIdReassignCheck( - FileStoreTable table, Long rowIdReassignCheckFromSnapshot) { - if (rowIdReassignCheckFromSnapshot == null) { - return table; - } - return table.copy( - Collections.singletonMap( - CoreOptions.COMMIT_ROW_ID_REASSIGN_LAST_SAFE_SNAPSHOT.key(), - String.valueOf(rowIdReassignCheckFromSnapshot))); - } - /** Operator to read data from splits. */ private static class ReadDataOperator extends org.apache.flink.table.runtime.operators.TableStreamOperator diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java index 0e4e0dec19f7..93a8ad9e15af 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java @@ -179,10 +179,7 @@ public static boolean buildIndex( List entries = indexBuilder.scan(); List deletedIndexEntries = indexBuilder.deletedIndexEntries(); - Long rowIdReassignCheckFromSnapshot = - indexBuilder.scanSnapshotId().isPresent() - ? indexBuilder.scanSnapshotId().get() - : null; + Long rowIdReassignCheckFromSnapshot = indexBuilder.scanSnapshotId().orElse(null); return buildTopology( env, @@ -516,7 +513,9 @@ private static void commit( String indexType, DataStream written, Long rowIdReassignCheckFromSnapshot) { - FileStoreTable commitTable = withRowIdReassignCheck(table, rowIdReassignCheckFromSnapshot); + FileStoreTable commitTable = + GlobalIndexCommitUtils.withRowIdReassignCheck( + table, rowIdReassignCheckFromSnapshot); OneInputStreamOperatorFactory committerOperator = new CommitterOperatorFactory<>( false, @@ -534,17 +533,6 @@ private static void commit( .setMaxParallelism(1); } - private static FileStoreTable withRowIdReassignCheck( - FileStoreTable table, Long rowIdReassignCheckFromSnapshot) { - if (rowIdReassignCheckFromSnapshot == null) { - return table; - } - return table.copy( - Collections.singletonMap( - CoreOptions.COMMIT_ROW_ID_REASSIGN_LAST_SAFE_SNAPSHOT.key(), - String.valueOf(rowIdReassignCheckFromSnapshot))); - } - /** Serializable descriptor for one shard's work. Each shard has its own DataSplit and Range. */ static class ShardTask implements Serializable { private static final long serialVersionUID = 1L; diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GlobalIndexCommitUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GlobalIndexCommitUtils.java new file mode 100644 index 000000000000..4365887b9740 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GlobalIndexCommitUtils.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.globalindex; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.table.FileStoreTable; + +import javax.annotation.Nullable; + +import java.util.Collections; + +/** Utilities for global index commit topology. */ +public final class GlobalIndexCommitUtils { + + private GlobalIndexCommitUtils() {} + + public static FileStoreTable withRowIdReassignCheck( + FileStoreTable table, @Nullable Long rowIdReassignCheckFromSnapshot) { + if (rowIdReassignCheckFromSnapshot == null) { + return table; + } + return table.copy( + Collections.singletonMap( + CoreOptions.COMMIT_ROW_ID_REASSIGN_LAST_SAFE_SNAPSHOT.key(), + String.valueOf(rowIdReassignCheckFromSnapshot))); + } +} From 3e5297f4f92528cbf456c0bbbb0462cec0e0443a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 26 May 2026 17:59:02 +0800 Subject: [PATCH 04/16] [core][flink] Rename row-id overwrite conflict barrier --- docs/generated/core_configuration.html | 4 +- .../java/org/apache/paimon/CoreOptions.java | 14 +++-- .../main/java/org/apache/paimon/Snapshot.java | 2 +- .../DataEvolutionRowIdReassigner.java | 4 +- .../paimon/operation/FileStoreCommitImpl.java | 19 +++--- .../operation/commit/ConflictDetection.java | 35 ++++++----- .../DataEvolutionRowIdReassignerTest.java | 2 + .../paimon/operation/FileStoreCommitTest.java | 61 ++++++++++--------- .../commit/ConflictDetectionTest.java | 26 ++++---- .../flink/btree/BTreeIndexTopoBuilder.java | 14 ++--- .../globalindex/GenericIndexTopoBuilder.java | 14 ++--- .../globalindex/GlobalIndexCommitUtils.java | 10 +-- 12 files changed, 108 insertions(+), 97 deletions(-) diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 42023c88b02d..94a558a6b074 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -321,10 +321,10 @@ If set, committer will check if there are other commit user's snapshot starting from the snapshot after this one. If found a COMPACT / OVERWRITE snapshot, or found a APPEND snapshot which committed files to fixed bucket, commit will be aborted.If the value of this option is -1, committer will not check for its first commit. -
commit.row-id-reassign.last-safe-snapshot
+
commit.row-id-overwrite-conflict.last-safe-snapshot
(none) Long - If set, committer will check if there are row-id reassignment snapshots starting from the snapshot after this one. If found, commit will be aborted. + If set, committer will check OVERWRITE snapshots starting from the snapshot after this one. If a row-id overwrite barrier snapshot or an ordinary overwrite snapshot which changed target partitions is found, commit will be aborted.
commit.timeout
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 1f211c9e8abc..bc73bde01011 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -2207,13 +2207,15 @@ public InlineElement getDescription() { + "APPEND snapshot which committed files to fixed bucket, commit will be aborted." + "If the value of this option is -1, committer will not check for its first commit."); - public static final ConfigOption COMMIT_ROW_ID_REASSIGN_LAST_SAFE_SNAPSHOT = - ConfigOptions.key("commit.row-id-reassign.last-safe-snapshot") + public static final ConfigOption COMMIT_ROW_ID_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT = + ConfigOptions.key("commit.row-id-overwrite-conflict.last-safe-snapshot") .longType() .noDefaultValue() .withDescription( - "If set, committer will check if there are row-id reassignment snapshots starting from the " - + "snapshot after this one. If found, commit will be aborted."); + "If set, committer will check OVERWRITE snapshots starting from the " + + "snapshot after this one. If a row-id overwrite barrier snapshot " + + "or an ordinary overwrite snapshot which changed target partitions " + + "is found, commit will be aborted."); public static final ConfigOption CLUSTERING_COLUMNS = key("clustering.columns") @@ -3881,8 +3883,8 @@ public Optional commitStrictModeLastSafeSnapshot() { return options.getOptional(COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT); } - public Optional commitRowIdReassignLastSafeSnapshot() { - return options.getOptional(COMMIT_ROW_ID_REASSIGN_LAST_SAFE_SNAPSHOT); + public Optional commitRowIdOverwriteConflictLastSafeSnapshot() { + return options.getOptional(COMMIT_ROW_ID_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT); } public List clusteringColumns() { diff --git a/paimon-api/src/main/java/org/apache/paimon/Snapshot.java b/paimon-api/src/main/java/org/apache/paimon/Snapshot.java index d7f0ca2131cf..d661847662c8 100644 --- a/paimon-api/src/main/java/org/apache/paimon/Snapshot.java +++ b/paimon-api/src/main/java/org/apache/paimon/Snapshot.java @@ -45,7 +45,7 @@ public class Snapshot implements Serializable { private static final long serialVersionUID = 1L; public static final long FIRST_SNAPSHOT_ID = 1; - public static final String ROW_ID_REASSIGN_PROPERTY = "row-id-reassign"; + public static final String ROW_ID_OVERWRITE_BARRIER_PROPERTY = "row-id-overwrite-barrier"; protected static final int CURRENT_VERSION = 3; diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassigner.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassigner.java index 1faa4cf66ab2..0c5c80555855 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassigner.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassigner.java @@ -149,7 +149,9 @@ public Result reassign(String commitUser) { baseManifestList, deltaManifestList, rewrittenIndexManifest.indexManifest, - assignment.nextRowId); + assignment.nextRowId, + Collections.singletonMap( + Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY, "true")); if (!success) { throw new RuntimeException( "Failed to reassign row IDs because a newer snapshot has been committed."); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java index ad5bc4a04455..25afe3e1cd67 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java @@ -220,8 +220,8 @@ public FileStoreCommitImpl( id)) .orElse(null); this.conflictDetection = conflictDetectFactory.create(scanner); - options.commitRowIdReassignLastSafeSnapshot() - .ifPresent(this.conflictDetection::setRowIdReassignCheckFromSnapshot); + options.commitRowIdOverwriteConflictLastSafeSnapshot() + .ifPresent(this.conflictDetection::setRowIdOverwriteConflictCheckFromSnapshot); this.commitCleaner = new CommitCleaner(manifestList, manifestFile, indexManifestFile); } @@ -329,7 +329,7 @@ public int commit(ManifestCommittable committable, boolean checkAppendFiles) { checkAppendFiles = true; allowRollback = true; } - if (conflictDetection.hasRowIdReassignCheckFromSnapshot()) { + if (conflictDetection.hasRowIdOverwriteConflictCheckFromSnapshot()) { checkAppendFiles = true; } @@ -1122,7 +1122,7 @@ public boolean replaceManifestList( deltaManifestList, latest.indexManifest(), latest.nextRowId(), - withoutRowIdReassignProperties(latest.properties())); + withoutRowIdOverwriteBarrierProperties(latest.properties())); } public boolean replaceManifestList( @@ -1139,7 +1139,7 @@ public boolean replaceManifestList( deltaManifestList, indexManifest, nextRowId, - withoutRowIdReassignProperties(latest.properties())); + withoutRowIdOverwriteBarrierProperties(latest.properties())); } public boolean replaceManifestList( @@ -1250,20 +1250,21 @@ private boolean compactManifestOnce() { null, latestSnapshot.watermark(), latestSnapshot.statistics(), - withoutRowIdReassignProperties(latestSnapshot.properties()), + withoutRowIdOverwriteBarrierProperties(latestSnapshot.properties()), latestSnapshot.nextRowId()); return commitSnapshotImpl(newSnapshot, emptyList()); } - private static @Nullable Map withoutRowIdReassignProperties( + private static @Nullable Map withoutRowIdOverwriteBarrierProperties( @Nullable Map properties) { - if (properties == null || !properties.containsKey(Snapshot.ROW_ID_REASSIGN_PROPERTY)) { + if (properties == null + || !properties.containsKey(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY)) { return properties; } Map copied = new HashMap<>(properties); - copied.remove(Snapshot.ROW_ID_REASSIGN_PROPERTY); + copied.remove(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY); return copied.isEmpty() ? null : copied; } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java index fc4e1afe77ff..51ed37ac4fa9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java @@ -95,7 +95,7 @@ protected boolean removeEldestEntry(Map.Entry eldest) { private @Nullable PartitionExpire partitionExpire; private @Nullable Long rowIdCheckFromSnapshot = null; - private @Nullable Long rowIdReassignCheckFromSnapshot = null; + private @Nullable Long rowIdOverwriteConflictCheckFromSnapshot = null; public ConflictDetection( String tableName, @@ -132,12 +132,13 @@ public boolean hasRowIdCheckFromSnapshot() { return rowIdCheckFromSnapshot != null; } - public void setRowIdReassignCheckFromSnapshot(@Nullable Long rowIdReassignCheckFromSnapshot) { - this.rowIdReassignCheckFromSnapshot = rowIdReassignCheckFromSnapshot; + public void setRowIdOverwriteConflictCheckFromSnapshot( + @Nullable Long rowIdOverwriteConflictCheckFromSnapshot) { + this.rowIdOverwriteConflictCheckFromSnapshot = rowIdOverwriteConflictCheckFromSnapshot; } - public boolean hasRowIdReassignCheckFromSnapshot() { - return rowIdReassignCheckFromSnapshot != null; + public boolean hasRowIdOverwriteConflictCheckFromSnapshot() { + return rowIdOverwriteConflictCheckFromSnapshot != null; } @Nullable @@ -246,7 +247,7 @@ public Optional checkConflicts( return exception; } - exception = checkRowIdReassignConflicts(latestSnapshot, deltaEntries, deltaIndexEntries); + exception = checkRowIdOverwriteConflicts(latestSnapshot, deltaEntries, deltaIndexEntries); if (exception.isPresent()) { return exception; } @@ -558,35 +559,37 @@ private Optional checkForRowIdFromSnapshot( return Optional.empty(); } - private Optional checkRowIdReassignConflicts( + private Optional checkRowIdOverwriteConflicts( Snapshot latestSnapshot, List deltaEntries, List deltaIndexEntries) { if (!dataEvolutionEnabled) { return Optional.empty(); } - if (rowIdReassignCheckFromSnapshot == null) { + if (rowIdOverwriteConflictCheckFromSnapshot == null) { return Optional.empty(); } - if (latestSnapshot.id() <= rowIdReassignCheckFromSnapshot) { + if (latestSnapshot.id() <= rowIdOverwriteConflictCheckFromSnapshot) { return Optional.empty(); } List changedPartitions = changedPartitionsIncludingAllIndexFiles(deltaEntries, deltaIndexEntries); - for (long id = rowIdReassignCheckFromSnapshot + 1; id <= latestSnapshot.id(); id++) { + for (long id = rowIdOverwriteConflictCheckFromSnapshot + 1; + id <= latestSnapshot.id(); + id++) { Snapshot snapshot = snapshotManager.snapshot(id); if (snapshot.commitKind() != CommitKind.OVERWRITE) { continue; } - if (hasRowIdReassignProperty(snapshot.properties())) { + if (hasRowIdOverwriteBarrierProperty(snapshot.properties())) { return Optional.of( new RuntimeException( String.format( - "Row-id reassignment snapshot %s was committed after the " + "Row-id overwrite barrier snapshot %s was committed after the " + "task planned from snapshot %s. The task must " + "be retried with the latest row ids.", - id, rowIdReassignCheckFromSnapshot))); + id, rowIdOverwriteConflictCheckFromSnapshot))); } if (overwriteChangedTargetPartitions(snapshot, changedPartitions)) { return Optional.of( @@ -595,15 +598,15 @@ private Optional checkRowIdReassignConflicts( "Overwrite snapshot %s changed partitions after the " + "task planned from snapshot %s. The task must " + "be retried with the latest row ids.", - id, rowIdReassignCheckFromSnapshot))); + id, rowIdOverwriteConflictCheckFromSnapshot))); } } return Optional.empty(); } - private boolean hasRowIdReassignProperty(@Nullable Map properties) { + private boolean hasRowIdOverwriteBarrierProperty(@Nullable Map properties) { return properties != null - && Boolean.parseBoolean(properties.get(Snapshot.ROW_ID_REASSIGN_PROPERTY)); + && Boolean.parseBoolean(properties.get(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY)); } private boolean overwriteChangedTargetPartitions( diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java index 2cacf4722e1f..c75af66c77f3 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java @@ -125,6 +125,8 @@ public void testReassignRowIdsByPartition() throws Exception { assertThat(result.fileCount).isEqualTo(5L); assertThat(result.rowCount).isEqualTo(5L); assertThat(result.indexFileCount).isEqualTo(0L); + assertThat(table.snapshotManager().latestSnapshot().properties()) + .containsEntry(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY, "true"); Map> rowIdsByPartition = rowIdsByPartition(table); assertThat(rowIdsByPartition).hasSize(2); diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java index d5701f87e76d..ac83f4b32535 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java @@ -1010,16 +1010,16 @@ public void testCommitManifestWithProperties() throws Exception { } @Test - public void testReplaceManifestListWithRowIdReassignProperty() throws Exception { + public void testReplaceManifestListWithRowIdOverwriteBarrierProperty() throws Exception { TestFileStore store = createStore(false); List keyValues = generateDataList(1); BinaryRow partition = gen.getPartition(keyValues.get(0)); Snapshot latest = store.commitData(keyValues, s -> partition, kv -> 0).get(0); - Map reassignProperties = new HashMap<>(); - reassignProperties.put("keep", "v1"); - reassignProperties.put(Snapshot.ROW_ID_REASSIGN_PROPERTY, "true"); + Map barrierProperties = new HashMap<>(); + barrierProperties.put("keep", "v1"); + barrierProperties.put(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY, "true"); try (FileStoreCommitImpl commit = store.newCommit()) { assertThat( commit.replaceManifestList( @@ -1029,33 +1029,33 @@ public void testReplaceManifestListWithRowIdReassignProperty() throws Exception deltaManifestList(latest), latest.indexManifest(), latest.nextRowId(), - reassignProperties)) + barrierProperties)) .isTrue(); } - Snapshot reassignSnapshot = checkNotNull(store.snapshotManager().latestSnapshot()); - assertThat(reassignSnapshot.properties()).isEqualTo(reassignProperties); + Snapshot barrierSnapshot = checkNotNull(store.snapshotManager().latestSnapshot()); + assertThat(barrierSnapshot.properties()).isEqualTo(barrierProperties); try (FileStoreCommitImpl commit = store.newCommit()) { assertThat( commit.replaceManifestList( - reassignSnapshot, - reassignSnapshot.totalRecordCount(), - baseManifestList(reassignSnapshot), - deltaManifestList(reassignSnapshot), - reassignSnapshot.indexManifest(), - reassignSnapshot.nextRowId())) + barrierSnapshot, + barrierSnapshot.totalRecordCount(), + baseManifestList(barrierSnapshot), + deltaManifestList(barrierSnapshot), + barrierSnapshot.indexManifest(), + barrierSnapshot.nextRowId())) .isTrue(); } Snapshot normalSnapshot = checkNotNull(store.snapshotManager().latestSnapshot()); assertThat(normalSnapshot.properties()) .containsEntry("keep", "v1") - .doesNotContainKey(Snapshot.ROW_ID_REASSIGN_PROPERTY); + .doesNotContainKey(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY); } @Test - public void testCompactManifestWithRowIdReassignProperty() throws Exception { + public void testCompactManifestWithRowIdOverwriteBarrierProperty() throws Exception { TestFileStore store = createStore(false); List keyValues = generateDataList(1); @@ -1072,9 +1072,9 @@ public void testCompactManifestWithRowIdReassignProperty() throws Exception { .sum(); assertThat(deleteNum).isGreaterThan(0); - Map reassignProperties = new HashMap<>(); - reassignProperties.put("keep", "v1"); - reassignProperties.put(Snapshot.ROW_ID_REASSIGN_PROPERTY, "true"); + Map barrierProperties = new HashMap<>(); + barrierProperties.put("keep", "v1"); + barrierProperties.put(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY, "true"); try (FileStoreCommitImpl commit = store.newCommit()) { assertThat( commit.replaceManifestList( @@ -1084,35 +1084,35 @@ public void testCompactManifestWithRowIdReassignProperty() throws Exception { deltaManifestList(latest), latest.indexManifest(), latest.nextRowId(), - reassignProperties)) + barrierProperties)) .isTrue(); } - Snapshot reassignSnapshot = checkNotNull(store.snapshotManager().latestSnapshot()); - assertThat(reassignSnapshot.properties()).isEqualTo(reassignProperties); + Snapshot barrierSnapshot = checkNotNull(store.snapshotManager().latestSnapshot()); + assertThat(barrierSnapshot.properties()).isEqualTo(barrierProperties); try (FileStoreCommit commit = store.newCommit()) { commit.compactManifest(); } Snapshot normalSnapshot = checkNotNull(store.snapshotManager().latestSnapshot()); - assertThat(normalSnapshot.id()).isGreaterThan(reassignSnapshot.id()); + assertThat(normalSnapshot.id()).isGreaterThan(barrierSnapshot.id()); assertThat(normalSnapshot.commitKind()).isEqualTo(Snapshot.CommitKind.COMPACT); assertThat(normalSnapshot.properties()) .containsEntry("keep", "v1") - .doesNotContainKey(Snapshot.ROW_ID_REASSIGN_PROPERTY); + .doesNotContainKey(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY); } @Test - public void testRowIdReassignConflictFromOptions() throws Exception { + public void testRowIdOverwriteConflictFromOptions() throws Exception { TestFileStore store = createStore(false); List keyValues = generateDataList(1); BinaryRow partition = gen.getPartition(keyValues.get(0)); Snapshot latest = store.commitData(keyValues, s -> partition, kv -> 0).get(0); - Map reassignProperties = new HashMap<>(); - reassignProperties.put(Snapshot.ROW_ID_REASSIGN_PROPERTY, "true"); + Map barrierProperties = new HashMap<>(); + barrierProperties.put(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY, "true"); try (FileStoreCommitImpl commit = store.newCommit()) { assertThat( commit.replaceManifestList( @@ -1122,7 +1122,7 @@ public void testRowIdReassignConflictFromOptions() throws Exception { deltaManifestList(latest), latest.indexManifest(), latest.nextRowId(), - reassignProperties)) + barrierProperties)) .isTrue(); } @@ -1138,16 +1138,17 @@ public void testRowIdReassignConflictFromOptions() throws Exception { (commit, committable) -> committableRef.set(committable)); Map dynamicOptions = new HashMap<>(store.options().toMap()); - dynamicOptions.put(CoreOptions.COMMIT_ROW_ID_REASSIGN_LAST_SAFE_SNAPSHOT.key(), "1"); + dynamicOptions.put( + CoreOptions.COMMIT_ROW_ID_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT.key(), "1"); try (FileStoreCommitImpl commit = newCommitWithSnapshotCommit( store, - "row-id-reassign-check", + "row-id-overwrite-barrier-check", new RenamingSnapshotCommit(store.snapshotManager(), Lock.empty()), new CoreOptions(dynamicOptions), true)) { assertThatThrownBy(() -> commit.commit(checkNotNull(committableRef.get()), false)) - .hasMessageContaining("Row-id reassignment snapshot 2") + .hasMessageContaining("Row-id overwrite barrier snapshot 2") .hasMessageContaining("task planned from snapshot 1"); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java index e243fe3b8ecd..55c25d7753d8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java @@ -508,15 +508,15 @@ private SimpleFileEntry createFileEntryWithRowId( } @Test - void testDetectsRowIdReassignSnapshotConflict() { + void testDetectsRowIdOverwriteBarrierSnapshotConflict() { SnapshotManager snapshotManager = mock(SnapshotManager.class); - Map reassignProperties = new HashMap<>(); - reassignProperties.put(Snapshot.ROW_ID_REASSIGN_PROPERTY, "true"); + Map barrierProperties = new HashMap<>(); + barrierProperties.put(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY, "true"); when(snapshotManager.snapshot(2L)) - .thenReturn(snapshot(2, Snapshot.CommitKind.OVERWRITE, reassignProperties)); + .thenReturn(snapshot(2, Snapshot.CommitKind.OVERWRITE, barrierProperties)); ConflictDetection detection = createConflictDetection(snapshotManager); - detection.setRowIdReassignCheckFromSnapshot(1L); + detection.setRowIdOverwriteConflictCheckFromSnapshot(1L); Optional exception = detection.checkConflicts( @@ -529,20 +529,20 @@ void testDetectsRowIdReassignSnapshotConflict() { assertThat(exception).isPresent(); assertThat(exception.get()) - .hasMessageContaining("Row-id reassignment snapshot 2") + .hasMessageContaining("Row-id overwrite barrier snapshot 2") .hasMessageContaining("task planned from snapshot 1"); } @Test - void testIgnoresRowIdReassignPropertyOnNonOverwriteSnapshot() { + void testIgnoresRowIdOverwriteBarrierPropertyOnNonOverwriteSnapshot() { SnapshotManager snapshotManager = mock(SnapshotManager.class); - Map reassignProperties = new HashMap<>(); - reassignProperties.put(Snapshot.ROW_ID_REASSIGN_PROPERTY, "true"); + Map barrierProperties = new HashMap<>(); + barrierProperties.put(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY, "true"); when(snapshotManager.snapshot(2L)) - .thenReturn(snapshot(2, Snapshot.CommitKind.APPEND, reassignProperties)); + .thenReturn(snapshot(2, Snapshot.CommitKind.APPEND, barrierProperties)); ConflictDetection detection = createConflictDetection(snapshotManager); - detection.setRowIdReassignCheckFromSnapshot(1L); + detection.setRowIdOverwriteConflictCheckFromSnapshot(1L); Optional exception = detection.checkConflicts( @@ -569,7 +569,7 @@ void testDetectsOverlappedOverwriteSnapshotForIndexCommit() { .thenReturn(Collections.singletonList(mock(ManifestEntry.class))); ConflictDetection detection = createConflictDetection(snapshotManager, commitScanner); - detection.setRowIdReassignCheckFromSnapshot(1L); + detection.setRowIdOverwriteConflictCheckFromSnapshot(1L); Optional exception = detection.checkConflicts( @@ -599,7 +599,7 @@ void testIgnoresNonOverlappedOverwriteSnapshotForIndexCommit() { .thenReturn(Collections.emptyList()); ConflictDetection detection = createConflictDetection(snapshotManager, commitScanner); - detection.setRowIdReassignCheckFromSnapshot(1L); + detection.setRowIdOverwriteConflictCheckFromSnapshot(1L); Optional exception = detection.checkConflicts( diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java index 4a0655ac6590..13fc3a891a53 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java @@ -96,7 +96,7 @@ public static boolean buildIndex( Options userOptions) throws Exception { List> allStreams = new ArrayList<>(); - Long rowIdReassignCheckFromSnapshot = null; + Long rowIdOverwriteConflictCheckFromSnapshot = null; for (String indexColumn : indexColumns) { BTreeGlobalIndexBuilder indexBuilder = indexBuilderSupplier.get().withIndexField(indexColumn); @@ -110,9 +110,9 @@ public static boolean buildIndex( continue; } if (indexBuilder.scanSnapshotId().isPresent()) { - rowIdReassignCheckFromSnapshot = + rowIdOverwriteConflictCheckFromSnapshot = minSnapshot( - rowIdReassignCheckFromSnapshot, + rowIdOverwriteConflictCheckFromSnapshot, indexBuilder.scanSnapshotId().get()); } @@ -202,7 +202,7 @@ public static boolean buildIndex( @SuppressWarnings("unchecked") DataStream[] rest = allStreams.subList(1, allStreams.size()).toArray(new DataStream[0]); - commit(table, allStreams.get(0).union(rest), rowIdReassignCheckFromSnapshot); + commit(table, allStreams.get(0).union(rest), rowIdOverwriteConflictCheckFromSnapshot); return true; } @@ -327,10 +327,10 @@ private static RowType withBuildTaskId(RowType readType, String buildTaskIdField private static void commit( FileStoreTable table, DataStream written, - Long rowIdReassignCheckFromSnapshot) { + Long rowIdOverwriteConflictCheckFromSnapshot) { FileStoreTable commitTable = - GlobalIndexCommitUtils.withRowIdReassignCheck( - table, rowIdReassignCheckFromSnapshot); + GlobalIndexCommitUtils.withRowIdOverwriteConflictCheck( + table, rowIdOverwriteConflictCheckFromSnapshot); OneInputStreamOperatorFactory committerOperator = new CommitterOperatorFactory<>( false, diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java index 93a8ad9e15af..9fa820b761dd 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java @@ -179,7 +179,7 @@ public static boolean buildIndex( List entries = indexBuilder.scan(); List deletedIndexEntries = indexBuilder.deletedIndexEntries(); - Long rowIdReassignCheckFromSnapshot = indexBuilder.scanSnapshotId().orElse(null); + Long rowIdOverwriteConflictCheckFromSnapshot = indexBuilder.scanSnapshotId().orElse(null); return buildTopology( env, @@ -190,7 +190,7 @@ public static boolean buildIndex( entries, deletedIndexEntries, maxIndexedRowId, - rowIdReassignCheckFromSnapshot); + rowIdOverwriteConflictCheckFromSnapshot); } /** @@ -211,7 +211,7 @@ private static boolean buildTopology( List entries, List deletedIndexEntries, long maxIndexedRowId, - Long rowIdReassignCheckFromSnapshot) + Long rowIdOverwriteConflictCheckFromSnapshot) throws Exception { long totalRowCount = entries.stream().mapToLong(e -> e.file().rowCount()).sum(); LOG.info( @@ -297,7 +297,7 @@ private static boolean buildTopology( built = built.union(deletes); } - commit(table, indexType, built, rowIdReassignCheckFromSnapshot); + commit(table, indexType, built, rowIdOverwriteConflictCheckFromSnapshot); return true; } @@ -512,10 +512,10 @@ private static void commit( FileStoreTable table, String indexType, DataStream written, - Long rowIdReassignCheckFromSnapshot) { + Long rowIdOverwriteConflictCheckFromSnapshot) { FileStoreTable commitTable = - GlobalIndexCommitUtils.withRowIdReassignCheck( - table, rowIdReassignCheckFromSnapshot); + GlobalIndexCommitUtils.withRowIdOverwriteConflictCheck( + table, rowIdOverwriteConflictCheckFromSnapshot); OneInputStreamOperatorFactory committerOperator = new CommitterOperatorFactory<>( false, diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GlobalIndexCommitUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GlobalIndexCommitUtils.java index 4365887b9740..be383fbb4859 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GlobalIndexCommitUtils.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GlobalIndexCommitUtils.java @@ -30,14 +30,14 @@ public final class GlobalIndexCommitUtils { private GlobalIndexCommitUtils() {} - public static FileStoreTable withRowIdReassignCheck( - FileStoreTable table, @Nullable Long rowIdReassignCheckFromSnapshot) { - if (rowIdReassignCheckFromSnapshot == null) { + public static FileStoreTable withRowIdOverwriteConflictCheck( + FileStoreTable table, @Nullable Long rowIdOverwriteConflictCheckFromSnapshot) { + if (rowIdOverwriteConflictCheckFromSnapshot == null) { return table; } return table.copy( Collections.singletonMap( - CoreOptions.COMMIT_ROW_ID_REASSIGN_LAST_SAFE_SNAPSHOT.key(), - String.valueOf(rowIdReassignCheckFromSnapshot))); + CoreOptions.COMMIT_ROW_ID_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT.key(), + String.valueOf(rowIdOverwriteConflictCheckFromSnapshot))); } } From ac184234dd36c8e811ddd553882cd8758ae11693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 26 May 2026 18:18:14 +0800 Subject: [PATCH 05/16] [core][flink] Rename overwrite conflict option --- docs/generated/core_configuration.html | 2 +- .../java/org/apache/paimon/CoreOptions.java | 8 +++--- .../paimon/operation/FileStoreCommitImpl.java | 6 ++-- .../operation/commit/ConflictDetection.java | 28 +++++++++---------- .../paimon/operation/FileStoreCommitTest.java | 5 ++-- .../commit/ConflictDetectionTest.java | 8 +++--- .../flink/btree/BTreeIndexTopoBuilder.java | 14 +++++----- .../globalindex/GenericIndexTopoBuilder.java | 14 +++++----- .../globalindex/GlobalIndexCommitUtils.java | 10 +++---- 9 files changed, 46 insertions(+), 49 deletions(-) diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 94a558a6b074..001249c946a1 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -321,7 +321,7 @@ If set, committer will check if there are other commit user's snapshot starting from the snapshot after this one. If found a COMPACT / OVERWRITE snapshot, or found a APPEND snapshot which committed files to fixed bucket, commit will be aborted.If the value of this option is -1, committer will not check for its first commit. -
commit.row-id-overwrite-conflict.last-safe-snapshot
+
commit.overwrite-conflict.last-safe-snapshot
(none) Long If set, committer will check OVERWRITE snapshots starting from the snapshot after this one. If a row-id overwrite barrier snapshot or an ordinary overwrite snapshot which changed target partitions is found, commit will be aborted. diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index bc73bde01011..07247f5959b1 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -2207,8 +2207,8 @@ public InlineElement getDescription() { + "APPEND snapshot which committed files to fixed bucket, commit will be aborted." + "If the value of this option is -1, committer will not check for its first commit."); - public static final ConfigOption COMMIT_ROW_ID_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT = - ConfigOptions.key("commit.row-id-overwrite-conflict.last-safe-snapshot") + public static final ConfigOption COMMIT_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT = + ConfigOptions.key("commit.overwrite-conflict.last-safe-snapshot") .longType() .noDefaultValue() .withDescription( @@ -3883,8 +3883,8 @@ public Optional commitStrictModeLastSafeSnapshot() { return options.getOptional(COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT); } - public Optional commitRowIdOverwriteConflictLastSafeSnapshot() { - return options.getOptional(COMMIT_ROW_ID_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT); + public Optional commitOverwriteConflictLastSafeSnapshot() { + return options.getOptional(COMMIT_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT); } public List clusteringColumns() { diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java index 25afe3e1cd67..2290ea946234 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java @@ -220,8 +220,8 @@ public FileStoreCommitImpl( id)) .orElse(null); this.conflictDetection = conflictDetectFactory.create(scanner); - options.commitRowIdOverwriteConflictLastSafeSnapshot() - .ifPresent(this.conflictDetection::setRowIdOverwriteConflictCheckFromSnapshot); + options.commitOverwriteConflictLastSafeSnapshot() + .ifPresent(this.conflictDetection::setOverwriteConflictCheckFromSnapshot); this.commitCleaner = new CommitCleaner(manifestList, manifestFile, indexManifestFile); } @@ -329,7 +329,7 @@ public int commit(ManifestCommittable committable, boolean checkAppendFiles) { checkAppendFiles = true; allowRollback = true; } - if (conflictDetection.hasRowIdOverwriteConflictCheckFromSnapshot()) { + if (conflictDetection.hasOverwriteConflictCheckFromSnapshot()) { checkAppendFiles = true; } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java index 51ed37ac4fa9..f126312e30dc 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java @@ -95,7 +95,7 @@ protected boolean removeEldestEntry(Map.Entry eldest) { private @Nullable PartitionExpire partitionExpire; private @Nullable Long rowIdCheckFromSnapshot = null; - private @Nullable Long rowIdOverwriteConflictCheckFromSnapshot = null; + private @Nullable Long overwriteConflictCheckFromSnapshot = null; public ConflictDetection( String tableName, @@ -132,13 +132,13 @@ public boolean hasRowIdCheckFromSnapshot() { return rowIdCheckFromSnapshot != null; } - public void setRowIdOverwriteConflictCheckFromSnapshot( - @Nullable Long rowIdOverwriteConflictCheckFromSnapshot) { - this.rowIdOverwriteConflictCheckFromSnapshot = rowIdOverwriteConflictCheckFromSnapshot; + public void setOverwriteConflictCheckFromSnapshot( + @Nullable Long overwriteConflictCheckFromSnapshot) { + this.overwriteConflictCheckFromSnapshot = overwriteConflictCheckFromSnapshot; } - public boolean hasRowIdOverwriteConflictCheckFromSnapshot() { - return rowIdOverwriteConflictCheckFromSnapshot != null; + public boolean hasOverwriteConflictCheckFromSnapshot() { + return overwriteConflictCheckFromSnapshot != null; } @Nullable @@ -247,7 +247,7 @@ public Optional checkConflicts( return exception; } - exception = checkRowIdOverwriteConflicts(latestSnapshot, deltaEntries, deltaIndexEntries); + exception = checkOverwriteConflicts(latestSnapshot, deltaEntries, deltaIndexEntries); if (exception.isPresent()) { return exception; } @@ -559,25 +559,23 @@ private Optional checkForRowIdFromSnapshot( return Optional.empty(); } - private Optional checkRowIdOverwriteConflicts( + private Optional checkOverwriteConflicts( Snapshot latestSnapshot, List deltaEntries, List deltaIndexEntries) { if (!dataEvolutionEnabled) { return Optional.empty(); } - if (rowIdOverwriteConflictCheckFromSnapshot == null) { + if (overwriteConflictCheckFromSnapshot == null) { return Optional.empty(); } - if (latestSnapshot.id() <= rowIdOverwriteConflictCheckFromSnapshot) { + if (latestSnapshot.id() <= overwriteConflictCheckFromSnapshot) { return Optional.empty(); } List changedPartitions = changedPartitionsIncludingAllIndexFiles(deltaEntries, deltaIndexEntries); - for (long id = rowIdOverwriteConflictCheckFromSnapshot + 1; - id <= latestSnapshot.id(); - id++) { + for (long id = overwriteConflictCheckFromSnapshot + 1; id <= latestSnapshot.id(); id++) { Snapshot snapshot = snapshotManager.snapshot(id); if (snapshot.commitKind() != CommitKind.OVERWRITE) { continue; @@ -589,7 +587,7 @@ private Optional checkRowIdOverwriteConflicts( "Row-id overwrite barrier snapshot %s was committed after the " + "task planned from snapshot %s. The task must " + "be retried with the latest row ids.", - id, rowIdOverwriteConflictCheckFromSnapshot))); + id, overwriteConflictCheckFromSnapshot))); } if (overwriteChangedTargetPartitions(snapshot, changedPartitions)) { return Optional.of( @@ -598,7 +596,7 @@ private Optional checkRowIdOverwriteConflicts( "Overwrite snapshot %s changed partitions after the " + "task planned from snapshot %s. The task must " + "be retried with the latest row ids.", - id, rowIdOverwriteConflictCheckFromSnapshot))); + id, overwriteConflictCheckFromSnapshot))); } } return Optional.empty(); diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java index ac83f4b32535..a09cf2e2ae55 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java @@ -1104,7 +1104,7 @@ public void testCompactManifestWithRowIdOverwriteBarrierProperty() throws Except } @Test - public void testRowIdOverwriteConflictFromOptions() throws Exception { + public void testOverwriteConflictFromOptions() throws Exception { TestFileStore store = createStore(false); List keyValues = generateDataList(1); @@ -1138,8 +1138,7 @@ public void testRowIdOverwriteConflictFromOptions() throws Exception { (commit, committable) -> committableRef.set(committable)); Map dynamicOptions = new HashMap<>(store.options().toMap()); - dynamicOptions.put( - CoreOptions.COMMIT_ROW_ID_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT.key(), "1"); + dynamicOptions.put(CoreOptions.COMMIT_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT.key(), "1"); try (FileStoreCommitImpl commit = newCommitWithSnapshotCommit( store, diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java index 55c25d7753d8..75452fdb08ed 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java @@ -516,7 +516,7 @@ void testDetectsRowIdOverwriteBarrierSnapshotConflict() { .thenReturn(snapshot(2, Snapshot.CommitKind.OVERWRITE, barrierProperties)); ConflictDetection detection = createConflictDetection(snapshotManager); - detection.setRowIdOverwriteConflictCheckFromSnapshot(1L); + detection.setOverwriteConflictCheckFromSnapshot(1L); Optional exception = detection.checkConflicts( @@ -542,7 +542,7 @@ void testIgnoresRowIdOverwriteBarrierPropertyOnNonOverwriteSnapshot() { .thenReturn(snapshot(2, Snapshot.CommitKind.APPEND, barrierProperties)); ConflictDetection detection = createConflictDetection(snapshotManager); - detection.setRowIdOverwriteConflictCheckFromSnapshot(1L); + detection.setOverwriteConflictCheckFromSnapshot(1L); Optional exception = detection.checkConflicts( @@ -569,7 +569,7 @@ void testDetectsOverlappedOverwriteSnapshotForIndexCommit() { .thenReturn(Collections.singletonList(mock(ManifestEntry.class))); ConflictDetection detection = createConflictDetection(snapshotManager, commitScanner); - detection.setRowIdOverwriteConflictCheckFromSnapshot(1L); + detection.setOverwriteConflictCheckFromSnapshot(1L); Optional exception = detection.checkConflicts( @@ -599,7 +599,7 @@ void testIgnoresNonOverlappedOverwriteSnapshotForIndexCommit() { .thenReturn(Collections.emptyList()); ConflictDetection detection = createConflictDetection(snapshotManager, commitScanner); - detection.setRowIdOverwriteConflictCheckFromSnapshot(1L); + detection.setOverwriteConflictCheckFromSnapshot(1L); Optional exception = detection.checkConflicts( diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java index 13fc3a891a53..f70b8bf68911 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java @@ -96,7 +96,7 @@ public static boolean buildIndex( Options userOptions) throws Exception { List> allStreams = new ArrayList<>(); - Long rowIdOverwriteConflictCheckFromSnapshot = null; + Long overwriteConflictCheckFromSnapshot = null; for (String indexColumn : indexColumns) { BTreeGlobalIndexBuilder indexBuilder = indexBuilderSupplier.get().withIndexField(indexColumn); @@ -110,9 +110,9 @@ public static boolean buildIndex( continue; } if (indexBuilder.scanSnapshotId().isPresent()) { - rowIdOverwriteConflictCheckFromSnapshot = + overwriteConflictCheckFromSnapshot = minSnapshot( - rowIdOverwriteConflictCheckFromSnapshot, + overwriteConflictCheckFromSnapshot, indexBuilder.scanSnapshotId().get()); } @@ -202,7 +202,7 @@ public static boolean buildIndex( @SuppressWarnings("unchecked") DataStream[] rest = allStreams.subList(1, allStreams.size()).toArray(new DataStream[0]); - commit(table, allStreams.get(0).union(rest), rowIdOverwriteConflictCheckFromSnapshot); + commit(table, allStreams.get(0).union(rest), overwriteConflictCheckFromSnapshot); return true; } @@ -327,10 +327,10 @@ private static RowType withBuildTaskId(RowType readType, String buildTaskIdField private static void commit( FileStoreTable table, DataStream written, - Long rowIdOverwriteConflictCheckFromSnapshot) { + Long overwriteConflictCheckFromSnapshot) { FileStoreTable commitTable = - GlobalIndexCommitUtils.withRowIdOverwriteConflictCheck( - table, rowIdOverwriteConflictCheckFromSnapshot); + GlobalIndexCommitUtils.withOverwriteConflictCheck( + table, overwriteConflictCheckFromSnapshot); OneInputStreamOperatorFactory committerOperator = new CommitterOperatorFactory<>( false, diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java index 9fa820b761dd..7c8863d06c8d 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java @@ -179,7 +179,7 @@ public static boolean buildIndex( List entries = indexBuilder.scan(); List deletedIndexEntries = indexBuilder.deletedIndexEntries(); - Long rowIdOverwriteConflictCheckFromSnapshot = indexBuilder.scanSnapshotId().orElse(null); + Long overwriteConflictCheckFromSnapshot = indexBuilder.scanSnapshotId().orElse(null); return buildTopology( env, @@ -190,7 +190,7 @@ public static boolean buildIndex( entries, deletedIndexEntries, maxIndexedRowId, - rowIdOverwriteConflictCheckFromSnapshot); + overwriteConflictCheckFromSnapshot); } /** @@ -211,7 +211,7 @@ private static boolean buildTopology( List entries, List deletedIndexEntries, long maxIndexedRowId, - Long rowIdOverwriteConflictCheckFromSnapshot) + Long overwriteConflictCheckFromSnapshot) throws Exception { long totalRowCount = entries.stream().mapToLong(e -> e.file().rowCount()).sum(); LOG.info( @@ -297,7 +297,7 @@ private static boolean buildTopology( built = built.union(deletes); } - commit(table, indexType, built, rowIdOverwriteConflictCheckFromSnapshot); + commit(table, indexType, built, overwriteConflictCheckFromSnapshot); return true; } @@ -512,10 +512,10 @@ private static void commit( FileStoreTable table, String indexType, DataStream written, - Long rowIdOverwriteConflictCheckFromSnapshot) { + Long overwriteConflictCheckFromSnapshot) { FileStoreTable commitTable = - GlobalIndexCommitUtils.withRowIdOverwriteConflictCheck( - table, rowIdOverwriteConflictCheckFromSnapshot); + GlobalIndexCommitUtils.withOverwriteConflictCheck( + table, overwriteConflictCheckFromSnapshot); OneInputStreamOperatorFactory committerOperator = new CommitterOperatorFactory<>( false, diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GlobalIndexCommitUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GlobalIndexCommitUtils.java index be383fbb4859..44ff40120d1e 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GlobalIndexCommitUtils.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GlobalIndexCommitUtils.java @@ -30,14 +30,14 @@ public final class GlobalIndexCommitUtils { private GlobalIndexCommitUtils() {} - public static FileStoreTable withRowIdOverwriteConflictCheck( - FileStoreTable table, @Nullable Long rowIdOverwriteConflictCheckFromSnapshot) { - if (rowIdOverwriteConflictCheckFromSnapshot == null) { + public static FileStoreTable withOverwriteConflictCheck( + FileStoreTable table, @Nullable Long overwriteConflictCheckFromSnapshot) { + if (overwriteConflictCheckFromSnapshot == null) { return table; } return table.copy( Collections.singletonMap( - CoreOptions.COMMIT_ROW_ID_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT.key(), - String.valueOf(rowIdOverwriteConflictCheckFromSnapshot))); + CoreOptions.COMMIT_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT.key(), + String.valueOf(overwriteConflictCheckFromSnapshot))); } } From db143c9587538b8fdc86a801870241039269f9e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 26 May 2026 18:22:36 +0800 Subject: [PATCH 06/16] [core] Rename overwrite barrier snapshot property --- docs/generated/core_configuration.html | 2 +- .../java/org/apache/paimon/CoreOptions.java | 2 +- .../main/java/org/apache/paimon/Snapshot.java | 2 +- .../DataEvolutionRowIdReassigner.java | 3 +-- .../paimon/operation/FileStoreCommitImpl.java | 13 ++++++------- .../operation/commit/ConflictDetection.java | 8 ++++---- .../DataEvolutionRowIdReassignerTest.java | 2 +- .../paimon/operation/FileStoreCommitTest.java | 18 +++++++++--------- .../commit/ConflictDetectionTest.java | 10 +++++----- 9 files changed, 29 insertions(+), 31 deletions(-) diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 001249c946a1..ec99b5edd2f7 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -324,7 +324,7 @@
commit.overwrite-conflict.last-safe-snapshot
(none) Long - If set, committer will check OVERWRITE snapshots starting from the snapshot after this one. If a row-id overwrite barrier snapshot or an ordinary overwrite snapshot which changed target partitions is found, commit will be aborted. + If set, committer will check OVERWRITE snapshots starting from the snapshot after this one. If an overwrite barrier snapshot or an ordinary overwrite snapshot which changed target partitions is found, commit will be aborted.
commit.timeout
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 07247f5959b1..4513630c73f5 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -2213,7 +2213,7 @@ public InlineElement getDescription() { .noDefaultValue() .withDescription( "If set, committer will check OVERWRITE snapshots starting from the " - + "snapshot after this one. If a row-id overwrite barrier snapshot " + + "snapshot after this one. If an overwrite barrier snapshot " + "or an ordinary overwrite snapshot which changed target partitions " + "is found, commit will be aborted."); diff --git a/paimon-api/src/main/java/org/apache/paimon/Snapshot.java b/paimon-api/src/main/java/org/apache/paimon/Snapshot.java index d661847662c8..480e29f687ec 100644 --- a/paimon-api/src/main/java/org/apache/paimon/Snapshot.java +++ b/paimon-api/src/main/java/org/apache/paimon/Snapshot.java @@ -45,7 +45,7 @@ public class Snapshot implements Serializable { private static final long serialVersionUID = 1L; public static final long FIRST_SNAPSHOT_ID = 1; - public static final String ROW_ID_OVERWRITE_BARRIER_PROPERTY = "row-id-overwrite-barrier"; + public static final String OVERWRITE_BARRIER_PROPERTY = "overwrite-barrier"; protected static final int CURRENT_VERSION = 3; diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassigner.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassigner.java index 0c5c80555855..b7ee0b19b836 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassigner.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassigner.java @@ -150,8 +150,7 @@ public Result reassign(String commitUser) { deltaManifestList, rewrittenIndexManifest.indexManifest, assignment.nextRowId, - Collections.singletonMap( - Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY, "true")); + Collections.singletonMap(Snapshot.OVERWRITE_BARRIER_PROPERTY, "true")); if (!success) { throw new RuntimeException( "Failed to reassign row IDs because a newer snapshot has been committed."); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java index 2290ea946234..ceaebc018614 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java @@ -1122,7 +1122,7 @@ public boolean replaceManifestList( deltaManifestList, latest.indexManifest(), latest.nextRowId(), - withoutRowIdOverwriteBarrierProperties(latest.properties())); + withoutOverwriteBarrierProperties(latest.properties())); } public boolean replaceManifestList( @@ -1139,7 +1139,7 @@ public boolean replaceManifestList( deltaManifestList, indexManifest, nextRowId, - withoutRowIdOverwriteBarrierProperties(latest.properties())); + withoutOverwriteBarrierProperties(latest.properties())); } public boolean replaceManifestList( @@ -1250,21 +1250,20 @@ private boolean compactManifestOnce() { null, latestSnapshot.watermark(), latestSnapshot.statistics(), - withoutRowIdOverwriteBarrierProperties(latestSnapshot.properties()), + withoutOverwriteBarrierProperties(latestSnapshot.properties()), latestSnapshot.nextRowId()); return commitSnapshotImpl(newSnapshot, emptyList()); } - private static @Nullable Map withoutRowIdOverwriteBarrierProperties( + private static @Nullable Map withoutOverwriteBarrierProperties( @Nullable Map properties) { - if (properties == null - || !properties.containsKey(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY)) { + if (properties == null || !properties.containsKey(Snapshot.OVERWRITE_BARRIER_PROPERTY)) { return properties; } Map copied = new HashMap<>(properties); - copied.remove(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY); + copied.remove(Snapshot.OVERWRITE_BARRIER_PROPERTY); return copied.isEmpty() ? null : copied; } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java index f126312e30dc..ca26dce08901 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java @@ -580,11 +580,11 @@ private Optional checkOverwriteConflicts( if (snapshot.commitKind() != CommitKind.OVERWRITE) { continue; } - if (hasRowIdOverwriteBarrierProperty(snapshot.properties())) { + if (hasOverwriteBarrierProperty(snapshot.properties())) { return Optional.of( new RuntimeException( String.format( - "Row-id overwrite barrier snapshot %s was committed after the " + "Overwrite barrier snapshot %s was committed after the " + "task planned from snapshot %s. The task must " + "be retried with the latest row ids.", id, overwriteConflictCheckFromSnapshot))); @@ -602,9 +602,9 @@ private Optional checkOverwriteConflicts( return Optional.empty(); } - private boolean hasRowIdOverwriteBarrierProperty(@Nullable Map properties) { + private boolean hasOverwriteBarrierProperty(@Nullable Map properties) { return properties != null - && Boolean.parseBoolean(properties.get(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY)); + && Boolean.parseBoolean(properties.get(Snapshot.OVERWRITE_BARRIER_PROPERTY)); } private boolean overwriteChangedTargetPartitions( diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java index c75af66c77f3..ae6f5d3f2504 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java @@ -126,7 +126,7 @@ public void testReassignRowIdsByPartition() throws Exception { assertThat(result.rowCount).isEqualTo(5L); assertThat(result.indexFileCount).isEqualTo(0L); assertThat(table.snapshotManager().latestSnapshot().properties()) - .containsEntry(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY, "true"); + .containsEntry(Snapshot.OVERWRITE_BARRIER_PROPERTY, "true"); Map> rowIdsByPartition = rowIdsByPartition(table); assertThat(rowIdsByPartition).hasSize(2); diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java index a09cf2e2ae55..15a1b868a395 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java @@ -1010,7 +1010,7 @@ public void testCommitManifestWithProperties() throws Exception { } @Test - public void testReplaceManifestListWithRowIdOverwriteBarrierProperty() throws Exception { + public void testReplaceManifestListWithOverwriteBarrierProperty() throws Exception { TestFileStore store = createStore(false); List keyValues = generateDataList(1); @@ -1019,7 +1019,7 @@ public void testReplaceManifestListWithRowIdOverwriteBarrierProperty() throws Ex Map barrierProperties = new HashMap<>(); barrierProperties.put("keep", "v1"); - barrierProperties.put(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY, "true"); + barrierProperties.put(Snapshot.OVERWRITE_BARRIER_PROPERTY, "true"); try (FileStoreCommitImpl commit = store.newCommit()) { assertThat( commit.replaceManifestList( @@ -1051,11 +1051,11 @@ public void testReplaceManifestListWithRowIdOverwriteBarrierProperty() throws Ex Snapshot normalSnapshot = checkNotNull(store.snapshotManager().latestSnapshot()); assertThat(normalSnapshot.properties()) .containsEntry("keep", "v1") - .doesNotContainKey(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY); + .doesNotContainKey(Snapshot.OVERWRITE_BARRIER_PROPERTY); } @Test - public void testCompactManifestWithRowIdOverwriteBarrierProperty() throws Exception { + public void testCompactManifestWithOverwriteBarrierProperty() throws Exception { TestFileStore store = createStore(false); List keyValues = generateDataList(1); @@ -1074,7 +1074,7 @@ public void testCompactManifestWithRowIdOverwriteBarrierProperty() throws Except Map barrierProperties = new HashMap<>(); barrierProperties.put("keep", "v1"); - barrierProperties.put(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY, "true"); + barrierProperties.put(Snapshot.OVERWRITE_BARRIER_PROPERTY, "true"); try (FileStoreCommitImpl commit = store.newCommit()) { assertThat( commit.replaceManifestList( @@ -1100,7 +1100,7 @@ public void testCompactManifestWithRowIdOverwriteBarrierProperty() throws Except assertThat(normalSnapshot.commitKind()).isEqualTo(Snapshot.CommitKind.COMPACT); assertThat(normalSnapshot.properties()) .containsEntry("keep", "v1") - .doesNotContainKey(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY); + .doesNotContainKey(Snapshot.OVERWRITE_BARRIER_PROPERTY); } @Test @@ -1112,7 +1112,7 @@ public void testOverwriteConflictFromOptions() throws Exception { Snapshot latest = store.commitData(keyValues, s -> partition, kv -> 0).get(0); Map barrierProperties = new HashMap<>(); - barrierProperties.put(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY, "true"); + barrierProperties.put(Snapshot.OVERWRITE_BARRIER_PROPERTY, "true"); try (FileStoreCommitImpl commit = store.newCommit()) { assertThat( commit.replaceManifestList( @@ -1142,12 +1142,12 @@ public void testOverwriteConflictFromOptions() throws Exception { try (FileStoreCommitImpl commit = newCommitWithSnapshotCommit( store, - "row-id-overwrite-barrier-check", + "overwrite-barrier-check", new RenamingSnapshotCommit(store.snapshotManager(), Lock.empty()), new CoreOptions(dynamicOptions), true)) { assertThatThrownBy(() -> commit.commit(checkNotNull(committableRef.get()), false)) - .hasMessageContaining("Row-id overwrite barrier snapshot 2") + .hasMessageContaining("Overwrite barrier snapshot 2") .hasMessageContaining("task planned from snapshot 1"); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java index 75452fdb08ed..de5c72a576f5 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java @@ -508,10 +508,10 @@ private SimpleFileEntry createFileEntryWithRowId( } @Test - void testDetectsRowIdOverwriteBarrierSnapshotConflict() { + void testDetectsOverwriteBarrierSnapshotConflict() { SnapshotManager snapshotManager = mock(SnapshotManager.class); Map barrierProperties = new HashMap<>(); - barrierProperties.put(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY, "true"); + barrierProperties.put(Snapshot.OVERWRITE_BARRIER_PROPERTY, "true"); when(snapshotManager.snapshot(2L)) .thenReturn(snapshot(2, Snapshot.CommitKind.OVERWRITE, barrierProperties)); @@ -529,15 +529,15 @@ void testDetectsRowIdOverwriteBarrierSnapshotConflict() { assertThat(exception).isPresent(); assertThat(exception.get()) - .hasMessageContaining("Row-id overwrite barrier snapshot 2") + .hasMessageContaining("Overwrite barrier snapshot 2") .hasMessageContaining("task planned from snapshot 1"); } @Test - void testIgnoresRowIdOverwriteBarrierPropertyOnNonOverwriteSnapshot() { + void testIgnoresOverwriteBarrierPropertyOnNonOverwriteSnapshot() { SnapshotManager snapshotManager = mock(SnapshotManager.class); Map barrierProperties = new HashMap<>(); - barrierProperties.put(Snapshot.ROW_ID_OVERWRITE_BARRIER_PROPERTY, "true"); + barrierProperties.put(Snapshot.OVERWRITE_BARRIER_PROPERTY, "true"); when(snapshotManager.snapshot(2L)) .thenReturn(snapshot(2, Snapshot.CommitKind.APPEND, barrierProperties)); From 2157bdd29dc7ee5e9fd4bcae66b185f858f790dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Thu, 28 May 2026 14:27:46 +0800 Subject: [PATCH 07/16] [core] Clarify overwrite barrier property cleanup --- .../apache/paimon/operation/FileStoreCommitImpl.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java index ceaebc018614..93eddc0aff52 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java @@ -1122,7 +1122,7 @@ public boolean replaceManifestList( deltaManifestList, latest.indexManifest(), latest.nextRowId(), - withoutOverwriteBarrierProperties(latest.properties())); + withoutOverwriteBarrierProperty(latest.properties())); } public boolean replaceManifestList( @@ -1139,7 +1139,7 @@ public boolean replaceManifestList( deltaManifestList, indexManifest, nextRowId, - withoutOverwriteBarrierProperties(latest.properties())); + withoutOverwriteBarrierProperty(latest.properties())); } public boolean replaceManifestList( @@ -1250,13 +1250,15 @@ private boolean compactManifestOnce() { null, latestSnapshot.watermark(), latestSnapshot.statistics(), - withoutOverwriteBarrierProperties(latestSnapshot.properties()), + withoutOverwriteBarrierProperty(latestSnapshot.properties()), latestSnapshot.nextRowId()); return commitSnapshotImpl(newSnapshot, emptyList()); } - private static @Nullable Map withoutOverwriteBarrierProperties( + // The overwrite barrier is a one-shot marker for the snapshot that rewrites row IDs. Derived + // snapshots, such as manifest compaction snapshots, must not inherit it and become barriers. + private static @Nullable Map withoutOverwriteBarrierProperty( @Nullable Map properties) { if (properties == null || !properties.containsKey(Snapshot.OVERWRITE_BARRIER_PROPERTY)) { return properties; From 105329fb878a4dffc90b5d049d10f289686678f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Thu, 28 May 2026 14:39:30 +0800 Subject: [PATCH 08/16] [flink] Inline global index commit table options --- .../flink/btree/BTreeIndexTopoBuilder.java | 23 +++++----- .../globalindex/GenericIndexTopoBuilder.java | 30 ++++++------- .../globalindex/GlobalIndexCommitUtils.java | 43 ------------------- 3 files changed, 26 insertions(+), 70 deletions(-) delete mode 100644 paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GlobalIndexCommitUtils.java diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java index f70b8bf68911..9a0dd644d320 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java @@ -27,7 +27,6 @@ import org.apache.paimon.flink.FlinkRowData; import org.apache.paimon.flink.FlinkRowWrapper; import org.apache.paimon.flink.LogicalTypeConversion; -import org.apache.paimon.flink.globalindex.GlobalIndexCommitUtils; import org.apache.paimon.flink.sink.Committable; import org.apache.paimon.flink.sink.CommittableTypeInfo; import org.apache.paimon.flink.sink.CommitterOperatorFactory; @@ -202,7 +201,15 @@ public static boolean buildIndex( @SuppressWarnings("unchecked") DataStream[] rest = allStreams.subList(1, allStreams.size()).toArray(new DataStream[0]); - commit(table, allStreams.get(0).union(rest), overwriteConflictCheckFromSnapshot); + FileStoreTable commitTable = + overwriteConflictCheckFromSnapshot == null + ? table + : table.copy( + Collections.singletonMap( + CoreOptions.COMMIT_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT + .key(), + String.valueOf(overwriteConflictCheckFromSnapshot))); + commit(commitTable, allStreams.get(0).union(rest)); return true; } @@ -324,13 +331,7 @@ private static RowType withBuildTaskId(RowType readType, String buildTaskIdField return new RowType(readType.isNullable(), fields); } - private static void commit( - FileStoreTable table, - DataStream written, - Long overwriteConflictCheckFromSnapshot) { - FileStoreTable commitTable = - GlobalIndexCommitUtils.withOverwriteConflictCheck( - table, overwriteConflictCheckFromSnapshot); + private static void commit(FileStoreTable table, DataStream written) { OneInputStreamOperatorFactory committerOperator = new CommitterOperatorFactory<>( false, @@ -338,9 +339,7 @@ private static void commit( "BTreeIndexCommitter-" + UUID.randomUUID(), context -> new StoreCommitter( - commitTable, - commitTable.newCommit(context.commitUser()), - context), + table, table.newCommit(context.commitUser()), context), new NoopCommittableStateManager()); written.transform("COMMIT OPERATOR", new CommittableTypeInfo(), committerOperator) diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java index 7c8863d06c8d..bf2df8069968 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java @@ -180,17 +180,25 @@ public static boolean buildIndex( List entries = indexBuilder.scan(); List deletedIndexEntries = indexBuilder.deletedIndexEntries(); Long overwriteConflictCheckFromSnapshot = indexBuilder.scanSnapshotId().orElse(null); + FileStoreTable commitTable = + overwriteConflictCheckFromSnapshot == null + ? table + : table.copy( + Collections.singletonMap( + CoreOptions.COMMIT_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT + .key(), + String.valueOf(overwriteConflictCheckFromSnapshot))); return buildTopology( env, table, + commitTable, indexColumn, indexType, userOptions, entries, deletedIndexEntries, - maxIndexedRowId, - overwriteConflictCheckFromSnapshot); + maxIndexedRowId); } /** @@ -205,13 +213,13 @@ public static boolean buildIndex( private static boolean buildTopology( StreamExecutionEnvironment env, FileStoreTable table, + FileStoreTable commitTable, String indexColumn, String indexType, Options userOptions, List entries, List deletedIndexEntries, - long maxIndexedRowId, - Long overwriteConflictCheckFromSnapshot) + long maxIndexedRowId) throws Exception { long totalRowCount = entries.stream().mapToLong(e -> e.file().rowCount()).sum(); LOG.info( @@ -297,7 +305,7 @@ private static boolean buildTopology( built = built.union(deletes); } - commit(table, indexType, built, overwriteConflictCheckFromSnapshot); + commit(commitTable, indexType, built); return true; } @@ -509,13 +517,7 @@ private static List createDeleteCommittables( } private static void commit( - FileStoreTable table, - String indexType, - DataStream written, - Long overwriteConflictCheckFromSnapshot) { - FileStoreTable commitTable = - GlobalIndexCommitUtils.withOverwriteConflictCheck( - table, overwriteConflictCheckFromSnapshot); + FileStoreTable table, String indexType, DataStream written) { OneInputStreamOperatorFactory committerOperator = new CommitterOperatorFactory<>( false, @@ -523,9 +525,7 @@ private static void commit( "GenericIndexCommitter-" + indexType + "-" + UUID.randomUUID(), context -> new StoreCommitter( - commitTable, - commitTable.newCommit(context.commitUser()), - context), + table, table.newCommit(context.commitUser()), context), new NoopCommittableStateManager()); written.transform("COMMIT OPERATOR", new CommittableTypeInfo(), committerOperator) diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GlobalIndexCommitUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GlobalIndexCommitUtils.java deleted file mode 100644 index 44ff40120d1e..000000000000 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GlobalIndexCommitUtils.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.flink.globalindex; - -import org.apache.paimon.CoreOptions; -import org.apache.paimon.table.FileStoreTable; - -import javax.annotation.Nullable; - -import java.util.Collections; - -/** Utilities for global index commit topology. */ -public final class GlobalIndexCommitUtils { - - private GlobalIndexCommitUtils() {} - - public static FileStoreTable withOverwriteConflictCheck( - FileStoreTable table, @Nullable Long overwriteConflictCheckFromSnapshot) { - if (overwriteConflictCheckFromSnapshot == null) { - return table; - } - return table.copy( - Collections.singletonMap( - CoreOptions.COMMIT_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT.key(), - String.valueOf(overwriteConflictCheckFromSnapshot))); - } -} From 61245eb86022c4eb09c212aad1adbd60cb659a34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Thu, 28 May 2026 15:42:24 +0800 Subject: [PATCH 09/16] [core] Scope overwrite conflict check to index commits --- .../java/org/apache/paimon/CoreOptions.java | 13 ++-- .../paimon/operation/FileStoreCommitImpl.java | 7 +- .../operation/commit/ConflictDetection.java | 44 +++++------ .../paimon/operation/FileStoreCommitTest.java | 76 +++++++++++++++++-- .../commit/ConflictDetectionTest.java | 14 ++-- .../flink/btree/BTreeIndexTopoBuilder.java | 3 +- .../globalindex/GenericIndexTopoBuilder.java | 3 +- 7 files changed, 115 insertions(+), 45 deletions(-) diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 4513630c73f5..26dd546b8a89 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -2207,14 +2207,15 @@ public InlineElement getDescription() { + "APPEND snapshot which committed files to fixed bucket, commit will be aborted." + "If the value of this option is -1, committer will not check for its first commit."); - public static final ConfigOption COMMIT_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT = - ConfigOptions.key("commit.overwrite-conflict.last-safe-snapshot") + public static final ConfigOption COMMIT_OVERWRITE_CONFLICT_WITH_INDEX_LAST_SAFE_SNAPSHOT = + ConfigOptions.key("commit.overwrite-conflict-with-index.last-safe-snapshot") .longType() .noDefaultValue() .withDescription( - "If set, committer will check OVERWRITE snapshots starting from the " + "If set, committer will check OVERWRITE snapshots against index file " + + "changes starting from the " + "snapshot after this one. If an overwrite barrier snapshot " - + "or an ordinary overwrite snapshot which changed target partitions " + + "or an ordinary overwrite snapshot which changed target index partitions " + "is found, commit will be aborted."); public static final ConfigOption CLUSTERING_COLUMNS = @@ -3883,8 +3884,8 @@ public Optional commitStrictModeLastSafeSnapshot() { return options.getOptional(COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT); } - public Optional commitOverwriteConflictLastSafeSnapshot() { - return options.getOptional(COMMIT_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT); + public Optional commitOverwriteConflictWithIndexLastSafeSnapshot() { + return options.getOptional(COMMIT_OVERWRITE_CONFLICT_WITH_INDEX_LAST_SAFE_SNAPSHOT); } public List clusteringColumns() { diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java index 93eddc0aff52..ed96316feb64 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java @@ -220,8 +220,8 @@ public FileStoreCommitImpl( id)) .orElse(null); this.conflictDetection = conflictDetectFactory.create(scanner); - options.commitOverwriteConflictLastSafeSnapshot() - .ifPresent(this.conflictDetection::setOverwriteConflictCheckFromSnapshot); + options.commitOverwriteConflictWithIndexLastSafeSnapshot() + .ifPresent(this.conflictDetection::setOverwriteConflictWithIndexCheckFromSnapshot); this.commitCleaner = new CommitCleaner(manifestList, manifestFile, indexManifestFile); } @@ -329,7 +329,8 @@ public int commit(ManifestCommittable committable, boolean checkAppendFiles) { checkAppendFiles = true; allowRollback = true; } - if (conflictDetection.hasOverwriteConflictCheckFromSnapshot()) { + if (conflictDetection.hasOverwriteConflictWithIndexCheckFromSnapshot() + && !changes.appendIndexFiles.isEmpty()) { checkAppendFiles = true; } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java index ca26dce08901..672bc881d744 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java @@ -95,7 +95,7 @@ protected boolean removeEldestEntry(Map.Entry eldest) { private @Nullable PartitionExpire partitionExpire; private @Nullable Long rowIdCheckFromSnapshot = null; - private @Nullable Long overwriteConflictCheckFromSnapshot = null; + private @Nullable Long overwriteConflictWithIndexCheckFromSnapshot = null; public ConflictDetection( String tableName, @@ -132,13 +132,14 @@ public boolean hasRowIdCheckFromSnapshot() { return rowIdCheckFromSnapshot != null; } - public void setOverwriteConflictCheckFromSnapshot( - @Nullable Long overwriteConflictCheckFromSnapshot) { - this.overwriteConflictCheckFromSnapshot = overwriteConflictCheckFromSnapshot; + public void setOverwriteConflictWithIndexCheckFromSnapshot( + @Nullable Long overwriteConflictWithIndexCheckFromSnapshot) { + this.overwriteConflictWithIndexCheckFromSnapshot = + overwriteConflictWithIndexCheckFromSnapshot; } - public boolean hasOverwriteConflictCheckFromSnapshot() { - return overwriteConflictCheckFromSnapshot != null; + public boolean hasOverwriteConflictWithIndexCheckFromSnapshot() { + return overwriteConflictWithIndexCheckFromSnapshot != null; } @Nullable @@ -247,7 +248,7 @@ public Optional checkConflicts( return exception; } - exception = checkOverwriteConflicts(latestSnapshot, deltaEntries, deltaIndexEntries); + exception = checkOverwriteConflicts(latestSnapshot, deltaIndexEntries); if (exception.isPresent()) { return exception; } @@ -560,22 +561,25 @@ private Optional checkForRowIdFromSnapshot( } private Optional checkOverwriteConflicts( - Snapshot latestSnapshot, - List deltaEntries, - List deltaIndexEntries) { + Snapshot latestSnapshot, List deltaIndexEntries) { if (!dataEvolutionEnabled) { return Optional.empty(); } - if (overwriteConflictCheckFromSnapshot == null) { + if (overwriteConflictWithIndexCheckFromSnapshot == null) { + return Optional.empty(); + } + if (latestSnapshot.id() <= overwriteConflictWithIndexCheckFromSnapshot) { return Optional.empty(); } - if (latestSnapshot.id() <= overwriteConflictCheckFromSnapshot) { + + List changedPartitions = changedIndexPartitions(deltaIndexEntries); + if (changedPartitions.isEmpty()) { return Optional.empty(); } - List changedPartitions = - changedPartitionsIncludingAllIndexFiles(deltaEntries, deltaIndexEntries); - for (long id = overwriteConflictCheckFromSnapshot + 1; id <= latestSnapshot.id(); id++) { + for (long id = overwriteConflictWithIndexCheckFromSnapshot + 1; + id <= latestSnapshot.id(); + id++) { Snapshot snapshot = snapshotManager.snapshot(id); if (snapshot.commitKind() != CommitKind.OVERWRITE) { continue; @@ -587,7 +591,7 @@ private Optional checkOverwriteConflicts( "Overwrite barrier snapshot %s was committed after the " + "task planned from snapshot %s. The task must " + "be retried with the latest row ids.", - id, overwriteConflictCheckFromSnapshot))); + id, overwriteConflictWithIndexCheckFromSnapshot))); } if (overwriteChangedTargetPartitions(snapshot, changedPartitions)) { return Optional.of( @@ -596,7 +600,7 @@ private Optional checkOverwriteConflicts( "Overwrite snapshot %s changed partitions after the " + "task planned from snapshot %s. The task must " + "be retried with the latest row ids.", - id, overwriteConflictCheckFromSnapshot))); + id, overwriteConflictWithIndexCheckFromSnapshot))); } } return Optional.empty(); @@ -613,12 +617,8 @@ private boolean overwriteChangedTargetPartitions( && !commitScanner.readIncrementalEntries(snapshot, changedPartitions).isEmpty(); } - private List changedPartitionsIncludingAllIndexFiles( - List dataFileChanges, List indexFileChanges) { + private List changedIndexPartitions(List indexFileChanges) { Set changedPartitions = new HashSet<>(); - for (SimpleFileEntry file : dataFileChanges) { - changedPartitions.add(file.partition()); - } for (IndexManifestEntry file : indexFileChanges) { changedPartitions.add(file.partition()); } diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java index 15a1b868a395..7bb75cc94788 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java @@ -31,8 +31,11 @@ import org.apache.paimon.deletionvectors.DeletionVector; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.index.GlobalIndexMeta; import org.apache.paimon.index.IndexFileHandler; import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.io.CompactIncrement; +import org.apache.paimon.io.DataIncrement; import org.apache.paimon.manifest.FileKind; import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.manifest.ManifestCommittable; @@ -1104,7 +1107,49 @@ public void testCompactManifestWithOverwriteBarrierProperty() throws Exception { } @Test - public void testOverwriteConflictFromOptions() throws Exception { + public void testOverwriteConflictWithIndexFromOptions() throws Exception { + TestFileStore store = createStore(false); + + List keyValues = generateDataList(1); + BinaryRow partition = gen.getPartition(keyValues.get(0)); + Snapshot latest = store.commitData(keyValues, s -> partition, kv -> 0).get(0); + + Map barrierProperties = new HashMap<>(); + barrierProperties.put(Snapshot.OVERWRITE_BARRIER_PROPERTY, "true"); + try (FileStoreCommitImpl commit = store.newCommit()) { + assertThat( + commit.replaceManifestList( + latest, + latest.totalRecordCount(), + baseManifestList(latest), + deltaManifestList(latest), + latest.indexManifest(), + latest.nextRowId(), + barrierProperties)) + .isTrue(); + } + + Map dynamicOptions = new HashMap<>(store.options().toMap()); + dynamicOptions.put( + CoreOptions.COMMIT_OVERWRITE_CONFLICT_WITH_INDEX_LAST_SAFE_SNAPSHOT.key(), "1"); + try (FileStoreCommitImpl commit = + newCommitWithSnapshotCommit( + store, + "overwrite-barrier-check", + new RenamingSnapshotCommit(store.snapshotManager(), Lock.empty()), + new CoreOptions(dynamicOptions), + true)) { + assertThatThrownBy( + () -> + commit.commit( + indexCommittable(partition, "barrier-index"), false)) + .hasMessageContaining("Overwrite barrier snapshot 2") + .hasMessageContaining("task planned from snapshot 1"); + } + } + + @Test + public void testOverwriteConflictWithIndexIgnoresDataOnlyDelta() throws Exception { TestFileStore store = createStore(false); List keyValues = generateDataList(1); @@ -1138,17 +1183,16 @@ public void testOverwriteConflictFromOptions() throws Exception { (commit, committable) -> committableRef.set(committable)); Map dynamicOptions = new HashMap<>(store.options().toMap()); - dynamicOptions.put(CoreOptions.COMMIT_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT.key(), "1"); + dynamicOptions.put( + CoreOptions.COMMIT_OVERWRITE_CONFLICT_WITH_INDEX_LAST_SAFE_SNAPSHOT.key(), "1"); try (FileStoreCommitImpl commit = newCommitWithSnapshotCommit( store, - "overwrite-barrier-check", + "overwrite-barrier-data-only-check", new RenamingSnapshotCommit(store.snapshotManager(), Lock.empty()), new CoreOptions(dynamicOptions), true)) { - assertThatThrownBy(() -> commit.commit(checkNotNull(committableRef.get()), false)) - .hasMessageContaining("Overwrite barrier snapshot 2") - .hasMessageContaining("task planned from snapshot 1"); + commit.commit(checkNotNull(committableRef.get()), false); } } @@ -1276,6 +1320,26 @@ private FileStoreCommitImpl newCommitWithSnapshotCommit( null); } + private ManifestCommittable indexCommittable(BinaryRow partition, String fileName) { + ManifestCommittable committable = new ManifestCommittable(0); + committable.addFileCommittable( + new CommitMessageImpl( + partition, + 0, + null, + DataIncrement.indexIncrement( + Collections.singletonList( + new IndexFileMeta( + "btree", + fileName, + 1, + 1, + (GlobalIndexMeta) null, + null))), + CompactIncrement.emptyIncrement())); + return committable; + } + private static class FalseSuccessSnapshotCommit implements SnapshotCommit { private final SnapshotCommit delegate; diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java index de5c72a576f5..de51152e0bce 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java @@ -516,14 +516,15 @@ void testDetectsOverwriteBarrierSnapshotConflict() { .thenReturn(snapshot(2, Snapshot.CommitKind.OVERWRITE, barrierProperties)); ConflictDetection detection = createConflictDetection(snapshotManager); - detection.setOverwriteConflictCheckFromSnapshot(1L); + detection.setOverwriteConflictWithIndexCheckFromSnapshot(1L); Optional exception = detection.checkConflicts( snapshot(3, null), Collections.emptyList(), Collections.emptyList(), - Collections.emptyList(), + Collections.singletonList( + createIndexEntry("idx", ADD, BinaryRow.EMPTY_ROW)), null, Snapshot.CommitKind.COMPACT); @@ -542,14 +543,15 @@ void testIgnoresOverwriteBarrierPropertyOnNonOverwriteSnapshot() { .thenReturn(snapshot(2, Snapshot.CommitKind.APPEND, barrierProperties)); ConflictDetection detection = createConflictDetection(snapshotManager); - detection.setOverwriteConflictCheckFromSnapshot(1L); + detection.setOverwriteConflictWithIndexCheckFromSnapshot(1L); Optional exception = detection.checkConflicts( snapshot(2, null), Collections.emptyList(), Collections.emptyList(), - Collections.emptyList(), + Collections.singletonList( + createIndexEntry("idx", ADD, BinaryRow.EMPTY_ROW)), null, Snapshot.CommitKind.COMPACT); @@ -569,7 +571,7 @@ void testDetectsOverlappedOverwriteSnapshotForIndexCommit() { .thenReturn(Collections.singletonList(mock(ManifestEntry.class))); ConflictDetection detection = createConflictDetection(snapshotManager, commitScanner); - detection.setOverwriteConflictCheckFromSnapshot(1L); + detection.setOverwriteConflictWithIndexCheckFromSnapshot(1L); Optional exception = detection.checkConflicts( @@ -599,7 +601,7 @@ void testIgnoresNonOverlappedOverwriteSnapshotForIndexCommit() { .thenReturn(Collections.emptyList()); ConflictDetection detection = createConflictDetection(snapshotManager, commitScanner); - detection.setOverwriteConflictCheckFromSnapshot(1L); + detection.setOverwriteConflictWithIndexCheckFromSnapshot(1L); Optional exception = detection.checkConflicts( diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java index 9a0dd644d320..bcec33331437 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java @@ -206,7 +206,8 @@ public static boolean buildIndex( ? table : table.copy( Collections.singletonMap( - CoreOptions.COMMIT_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT + CoreOptions + .COMMIT_OVERWRITE_CONFLICT_WITH_INDEX_LAST_SAFE_SNAPSHOT .key(), String.valueOf(overwriteConflictCheckFromSnapshot))); commit(commitTable, allStreams.get(0).union(rest)); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java index bf2df8069968..6b2ae265a2cb 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java @@ -185,7 +185,8 @@ public static boolean buildIndex( ? table : table.copy( Collections.singletonMap( - CoreOptions.COMMIT_OVERWRITE_CONFLICT_LAST_SAFE_SNAPSHOT + CoreOptions + .COMMIT_OVERWRITE_CONFLICT_WITH_INDEX_LAST_SAFE_SNAPSHOT .key(), String.valueOf(overwriteConflictCheckFromSnapshot))); From 0e79146cfa26d29131f41342d8aa88da462cb0cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Fri, 29 May 2026 10:50:45 +0800 Subject: [PATCH 10/16] [docs] Update overwrite conflict option docs --- docs/generated/core_configuration.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index ec99b5edd2f7..f52f398f2d18 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -321,10 +321,10 @@ If set, committer will check if there are other commit user's snapshot starting from the snapshot after this one. If found a COMPACT / OVERWRITE snapshot, or found a APPEND snapshot which committed files to fixed bucket, commit will be aborted.If the value of this option is -1, committer will not check for its first commit. -
commit.overwrite-conflict.last-safe-snapshot
+
commit.overwrite-conflict-with-index.last-safe-snapshot
(none) Long - If set, committer will check OVERWRITE snapshots starting from the snapshot after this one. If an overwrite barrier snapshot or an ordinary overwrite snapshot which changed target partitions is found, commit will be aborted. + If set, committer will check OVERWRITE snapshots against index file changes starting from the snapshot after this one. If an overwrite barrier snapshot or an ordinary overwrite snapshot which changed target index partitions is found, commit will be aborted.
commit.timeout
From 23ff1f042802fb92ff8ffacbc327118a22479a32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Wed, 3 Jun 2026 11:12:51 +0800 Subject: [PATCH 11/16] Fix comment --- docs/generated/core_configuration.html | 6 - .../java/org/apache/paimon/CoreOptions.java | 15 -- .../main/java/org/apache/paimon/Snapshot.java | 1 - .../DataEvolutionRowIdReassigner.java | 3 +- .../btree/BTreeGlobalIndexBuilder.java | 5 - .../paimon/operation/FileStoreCommitImpl.java | 43 +--- .../operation/commit/ConflictDetection.java | 153 ++++++++----- .../commit/ManifestEntryChanges.java | 3 +- .../DataEvolutionRowIdReassignerTest.java | 2 - .../paimon/operation/FileStoreCommitTest.java | 204 +++--------------- .../commit/ConflictDetectionTest.java | 154 +++++-------- .../flink/btree/BTreeIndexTopoBuilder.java | 22 +- .../GenericGlobalIndexBuilder.java | 8 - .../globalindex/GenericIndexTopoBuilder.java | 14 +- 14 files changed, 191 insertions(+), 442 deletions(-) diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index f52f398f2d18..1a66fb0ab0b8 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -320,12 +320,6 @@ Long If set, committer will check if there are other commit user's snapshot starting from the snapshot after this one. If found a COMPACT / OVERWRITE snapshot, or found a APPEND snapshot which committed files to fixed bucket, commit will be aborted.If the value of this option is -1, committer will not check for its first commit. - -
commit.overwrite-conflict-with-index.last-safe-snapshot
- (none) - Long - If set, committer will check OVERWRITE snapshots against index file changes starting from the snapshot after this one. If an overwrite barrier snapshot or an ordinary overwrite snapshot which changed target index partitions is found, commit will be aborted. -
commit.timeout
(none) diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 26dd546b8a89..da734fa9386f 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -2207,17 +2207,6 @@ public InlineElement getDescription() { + "APPEND snapshot which committed files to fixed bucket, commit will be aborted." + "If the value of this option is -1, committer will not check for its first commit."); - public static final ConfigOption COMMIT_OVERWRITE_CONFLICT_WITH_INDEX_LAST_SAFE_SNAPSHOT = - ConfigOptions.key("commit.overwrite-conflict-with-index.last-safe-snapshot") - .longType() - .noDefaultValue() - .withDescription( - "If set, committer will check OVERWRITE snapshots against index file " - + "changes starting from the " - + "snapshot after this one. If an overwrite barrier snapshot " - + "or an ordinary overwrite snapshot which changed target index partitions " - + "is found, commit will be aborted."); - public static final ConfigOption CLUSTERING_COLUMNS = key("clustering.columns") .stringType() @@ -3884,10 +3873,6 @@ public Optional commitStrictModeLastSafeSnapshot() { return options.getOptional(COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT); } - public Optional commitOverwriteConflictWithIndexLastSafeSnapshot() { - return options.getOptional(COMMIT_OVERWRITE_CONFLICT_WITH_INDEX_LAST_SAFE_SNAPSHOT); - } - public List clusteringColumns() { return clusteringColumns(options.get(CLUSTERING_COLUMNS)); } diff --git a/paimon-api/src/main/java/org/apache/paimon/Snapshot.java b/paimon-api/src/main/java/org/apache/paimon/Snapshot.java index 480e29f687ec..93f525a7e3fb 100644 --- a/paimon-api/src/main/java/org/apache/paimon/Snapshot.java +++ b/paimon-api/src/main/java/org/apache/paimon/Snapshot.java @@ -45,7 +45,6 @@ public class Snapshot implements Serializable { private static final long serialVersionUID = 1L; public static final long FIRST_SNAPSHOT_ID = 1; - public static final String OVERWRITE_BARRIER_PROPERTY = "overwrite-barrier"; protected static final int CURRENT_VERSION = 3; diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassigner.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassigner.java index b7ee0b19b836..1faa4cf66ab2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassigner.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassigner.java @@ -149,8 +149,7 @@ public Result reassign(String commitUser) { baseManifestList, deltaManifestList, rewrittenIndexManifest.indexManifest, - assignment.nextRowId, - Collections.singletonMap(Snapshot.OVERWRITE_BARRIER_PROPERTY, "true")); + assignment.nextRowId); if (!success) { throw new RuntimeException( "Failed to reassign row IDs because a newer snapshot has been committed."); diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexBuilder.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexBuilder.java index febf86d055db..dcac4b599e97 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexBuilder.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexBuilder.java @@ -97,7 +97,6 @@ public class BTreeGlobalIndexBuilder implements Serializable { // readRowType is composed by partition fields, indexed field and _ROW_ID field private RowType readRowType; @Nullable private Snapshot snapshot; - @Nullable private Long scanSnapshotId; @Nullable private PartitionPredicate partitionPredicate; @@ -134,10 +133,6 @@ public BTreeGlobalIndexBuilder withSnapshot(Snapshot snapshot) { return this; } - public Optional scanSnapshotId() { - return Optional.ofNullable(scanSnapshotId); - } - public Optional>> scan() { SnapshotReader snapshotReader = table.newSnapshotReader(); if (partitionPredicate != null) { diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java index ed96316feb64..f62bd5b3b199 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java @@ -220,8 +220,6 @@ public FileStoreCommitImpl( id)) .orElse(null); this.conflictDetection = conflictDetectFactory.create(scanner); - options.commitOverwriteConflictWithIndexLastSafeSnapshot() - .ifPresent(this.conflictDetection::setOverwriteConflictWithIndexCheckFromSnapshot); this.commitCleaner = new CommitCleaner(manifestList, manifestFile, indexManifestFile); } @@ -329,8 +327,7 @@ public int commit(ManifestCommittable committable, boolean checkAppendFiles) { checkAppendFiles = true; allowRollback = true; } - if (conflictDetection.hasOverwriteConflictWithIndexCheckFromSnapshot() - && !changes.appendIndexFiles.isEmpty()) { + if (conflictDetection.hasGlobalIndexFileAddition(changes.appendIndexFiles)) { checkAppendFiles = true; } @@ -1122,8 +1119,7 @@ public boolean replaceManifestList( baseManifestList, deltaManifestList, latest.indexManifest(), - latest.nextRowId(), - withoutOverwriteBarrierProperty(latest.properties())); + latest.nextRowId()); } public boolean replaceManifestList( @@ -1133,24 +1129,6 @@ public boolean replaceManifestList( Pair deltaManifestList, @Nullable String indexManifest, @Nullable Long nextRowId) { - return replaceManifestList( - latest, - totalRecordCount, - baseManifestList, - deltaManifestList, - indexManifest, - nextRowId, - withoutOverwriteBarrierProperty(latest.properties())); - } - - public boolean replaceManifestList( - Snapshot latest, - long totalRecordCount, - Pair baseManifestList, - Pair deltaManifestList, - @Nullable String indexManifest, - @Nullable Long nextRowId, - @Nullable Map properties) { Snapshot newSnapshot = new Snapshot( latest.id() + 1, @@ -1172,7 +1150,7 @@ public boolean replaceManifestList( latest.watermark(), latest.statistics(), // if empty properties, just set to null - properties, + latest.properties(), nextRowId); return commitSnapshotImpl(newSnapshot, emptyList()); @@ -1251,25 +1229,12 @@ private boolean compactManifestOnce() { null, latestSnapshot.watermark(), latestSnapshot.statistics(), - withoutOverwriteBarrierProperty(latestSnapshot.properties()), + latestSnapshot.properties(), latestSnapshot.nextRowId()); return commitSnapshotImpl(newSnapshot, emptyList()); } - // The overwrite barrier is a one-shot marker for the snapshot that rewrites row IDs. Derived - // snapshots, such as manifest compaction snapshots, must not inherit it and become barriers. - private static @Nullable Map withoutOverwriteBarrierProperty( - @Nullable Map properties) { - if (properties == null || !properties.containsKey(Snapshot.OVERWRITE_BARRIER_PROPERTY)) { - return properties; - } - - Map copied = new HashMap<>(properties); - copied.remove(Snapshot.OVERWRITE_BARRIER_PROPERTY); - return copied.isEmpty() ? null : copied; - } - private boolean commitSnapshotImpl(Snapshot newSnapshot, List deltaStatistics) { try { List statistics = new ArrayList<>(deltaStatistics.size()); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java index 672bc881d744..66a43712dd7e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java @@ -23,6 +23,7 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.index.DeletionVectorMeta; +import org.apache.paimon.index.GlobalIndexMeta; import org.apache.paimon.index.IndexFileHandler; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.io.DataFileMeta; @@ -37,6 +38,7 @@ import org.apache.paimon.types.RowType; import org.apache.paimon.utils.FileStorePathFactory; import org.apache.paimon.utils.Pair; +import org.apache.paimon.utils.Range; import org.apache.paimon.utils.RangeHelper; import org.apache.paimon.utils.SnapshotManager; @@ -95,7 +97,6 @@ protected boolean removeEldestEntry(Map.Entry eldest) { private @Nullable PartitionExpire partitionExpire; private @Nullable Long rowIdCheckFromSnapshot = null; - private @Nullable Long overwriteConflictWithIndexCheckFromSnapshot = null; public ConflictDetection( String tableName, @@ -132,16 +133,6 @@ public boolean hasRowIdCheckFromSnapshot() { return rowIdCheckFromSnapshot != null; } - public void setOverwriteConflictWithIndexCheckFromSnapshot( - @Nullable Long overwriteConflictWithIndexCheckFromSnapshot) { - this.overwriteConflictWithIndexCheckFromSnapshot = - overwriteConflictWithIndexCheckFromSnapshot; - } - - public boolean hasOverwriteConflictWithIndexCheckFromSnapshot() { - return overwriteConflictWithIndexCheckFromSnapshot != null; - } - @Nullable public Comparator keyComparator() { return keyComparator; @@ -166,6 +157,15 @@ public boolean shouldBeOverwriteCommit( return false; } + public boolean hasGlobalIndexFileAddition(List indexFileChanges) { + for (IndexManifestEntry entry : indexFileChanges) { + if (entry.kind() == FileKind.ADD && entry.indexFile().globalIndexMeta() != null) { + return true; + } + } + return false; + } + public Optional checkConflicts( Snapshot latestSnapshot, List baseEntries, @@ -248,7 +248,7 @@ public Optional checkConflicts( return exception; } - exception = checkOverwriteConflicts(latestSnapshot, deltaIndexEntries); + exception = checkGlobalIndexRowIdExistence(baseEntries, deltaIndexEntries); if (exception.isPresent()) { return exception; } @@ -560,69 +560,85 @@ private Optional checkForRowIdFromSnapshot( return Optional.empty(); } - private Optional checkOverwriteConflicts( - Snapshot latestSnapshot, List deltaIndexEntries) { + private Optional checkGlobalIndexRowIdExistence( + List baseEntries, List deltaIndexEntries) { if (!dataEvolutionEnabled) { return Optional.empty(); } - if (overwriteConflictWithIndexCheckFromSnapshot == null) { - return Optional.empty(); - } - if (latestSnapshot.id() <= overwriteConflictWithIndexCheckFromSnapshot) { - return Optional.empty(); - } - List changedPartitions = changedIndexPartitions(deltaIndexEntries); - if (changedPartitions.isEmpty()) { + List indexesToCheck = globalIndexFileAdditions(deltaIndexEntries); + if (indexesToCheck.isEmpty()) { return Optional.empty(); } - for (long id = overwriteConflictWithIndexCheckFromSnapshot + 1; - id <= latestSnapshot.id(); - id++) { - Snapshot snapshot = snapshotManager.snapshot(id); - if (snapshot.commitKind() != CommitKind.OVERWRITE) { - continue; + Map> dataRanges = new HashMap<>(); + for (SimpleFileEntry entry : baseEntries) { + if (entry.kind() == FileKind.ADD && entry.firstRowId() != null) { + dataRanges + .computeIfAbsent( + new PartitionBucketKey(entry.partition(), entry.bucket()), + ignored -> new ArrayList<>()) + .add(entry.nonNullRowIdRange()); } - if (hasOverwriteBarrierProperty(snapshot.properties())) { - return Optional.of( - new RuntimeException( - String.format( - "Overwrite barrier snapshot %s was committed after the " - + "task planned from snapshot %s. The task must " - + "be retried with the latest row ids.", - id, overwriteConflictWithIndexCheckFromSnapshot))); - } - if (overwriteChangedTargetPartitions(snapshot, changedPartitions)) { + } + for (Map.Entry> entry : dataRanges.entrySet()) { + entry.setValue(Range.sortAndMergeOverlap(entry.getValue(), true)); + } + + for (IndexManifestEntry indexEntry : indexesToCheck) { + GlobalIndexMeta globalIndex = indexEntry.indexFile().globalIndexMeta(); + checkState(globalIndex != null, "Global index meta must not be null."); + Range indexRange = globalIndex.rowRange(); + List currentRanges = + dataRanges.get( + new PartitionBucketKey(indexEntry.partition(), indexEntry.bucket())); + if (!containsRange(currentRanges, indexRange)) { return Optional.of( new RuntimeException( String.format( - "Overwrite snapshot %s changed partitions after the " - + "task planned from snapshot %s. The task must " - + "be retried with the latest row ids.", - id, overwriteConflictWithIndexCheckFromSnapshot))); + "Global index row ID existence conflict: index file '%s' " + + "references row range %s in bucket %d, but this " + + "range is not fully covered by current data " + + "files. The referenced row IDs may have been " + + "reassigned or removed by a concurrent commit.", + indexEntry.indexFile().fileName(), + indexRange, + indexEntry.bucket()))); } } return Optional.empty(); } - private boolean hasOverwriteBarrierProperty(@Nullable Map properties) { - return properties != null - && Boolean.parseBoolean(properties.get(Snapshot.OVERWRITE_BARRIER_PROPERTY)); + private List globalIndexFileAdditions( + List indexFileChanges) { + List result = new ArrayList<>(); + for (IndexManifestEntry entry : indexFileChanges) { + if (entry.kind() == FileKind.ADD && entry.indexFile().globalIndexMeta() != null) { + result.add(entry); + } + } + return result; } - private boolean overwriteChangedTargetPartitions( - Snapshot snapshot, List changedPartitions) { - return !changedPartitions.isEmpty() - && !commitScanner.readIncrementalEntries(snapshot, changedPartitions).isEmpty(); - } + private boolean containsRange(@Nullable List ranges, Range target) { + if (ranges == null || ranges.isEmpty()) { + return false; + } - private List changedIndexPartitions(List indexFileChanges) { - Set changedPartitions = new HashSet<>(); - for (IndexManifestEntry file : indexFileChanges) { - changedPartitions.add(file.partition()); + long next = target.from; + for (Range range : ranges) { + if (range.to < next) { + continue; + } + if (range.from > next) { + return false; + } + if (range.to >= target.to) { + return true; + } + next = range.to + 1; } - return new ArrayList<>(changedPartitions); + return false; } Optional checkRowIdExistence( @@ -718,6 +734,33 @@ public int hashCode() { } } + private static class PartitionBucketKey { + private final BinaryRow partition; + private final int bucket; + + PartitionBucketKey(BinaryRow partition, int bucket) { + this.partition = partition; + this.bucket = bucket; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PartitionBucketKey that = (PartitionBucketKey) o; + return bucket == that.bucket && Objects.equals(partition, that.partition); + } + + @Override + public int hashCode() { + return Objects.hash(partition, bucket); + } + } + private static boolean dedicatedStorageFile(String fileName) { return isBlobFile(fileName) || isVectorStoreFile(fileName); } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ManifestEntryChanges.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ManifestEntryChanges.java index f32f6c9ef5ff..faa65bba4750 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ManifestEntryChanges.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ManifestEntryChanges.java @@ -170,7 +170,8 @@ public static List changedPartitions( changedPartitions.add(file.partition()); } for (IndexManifestEntry file : indexFileChanges) { - if (file.indexFile().indexType().equals(DELETION_VECTORS_INDEX)) { + if (file.indexFile().indexType().equals(DELETION_VECTORS_INDEX) + || file.indexFile().globalIndexMeta() != null) { changedPartitions.add(file.partition()); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java index ae6f5d3f2504..2cacf4722e1f 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java @@ -125,8 +125,6 @@ public void testReassignRowIdsByPartition() throws Exception { assertThat(result.fileCount).isEqualTo(5L); assertThat(result.rowCount).isEqualTo(5L); assertThat(result.indexFileCount).isEqualTo(0L); - assertThat(table.snapshotManager().latestSnapshot().properties()) - .containsEntry(Snapshot.OVERWRITE_BARRIER_PROPERTY, "true"); Map> rowIdsByPartition = rowIdsByPartition(table); assertThat(rowIdsByPartition).hasSize(2); diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java index 7bb75cc94788..d04832902d21 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java @@ -60,7 +60,6 @@ import org.apache.paimon.types.RowKind; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.FailingFileIO; -import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.SnapshotManager; import org.apache.paimon.utils.TraceableFileIO; @@ -1013,186 +1012,44 @@ public void testCommitManifestWithProperties() throws Exception { } @Test - public void testReplaceManifestListWithOverwriteBarrierProperty() throws Exception { - TestFileStore store = createStore(false); + public void testGlobalIndexCommitChecksExistingRowIds() throws Exception { + TestFileStore store = createRowTrackingDataEvolutionStore(); List keyValues = generateDataList(1); BinaryRow partition = gen.getPartition(keyValues.get(0)); - Snapshot latest = store.commitData(keyValues, s -> partition, kv -> 0).get(0); + Snapshot dataSnapshot = store.commitData(keyValues, s -> partition, kv -> 0).get(0); + assertThat(dataSnapshot.nextRowId()).isEqualTo(1L); - Map barrierProperties = new HashMap<>(); - barrierProperties.put("keep", "v1"); - barrierProperties.put(Snapshot.OVERWRITE_BARRIER_PROPERTY, "true"); try (FileStoreCommitImpl commit = store.newCommit()) { - assertThat( - commit.replaceManifestList( - latest, - latest.totalRecordCount(), - baseManifestList(latest), - deltaManifestList(latest), - latest.indexManifest(), - latest.nextRowId(), - barrierProperties)) - .isTrue(); + commit.commit(indexCommittable(partition, "existing-index", 0, 0), false); } - Snapshot barrierSnapshot = checkNotNull(store.snapshotManager().latestSnapshot()); - assertThat(barrierSnapshot.properties()).isEqualTo(barrierProperties); - - try (FileStoreCommitImpl commit = store.newCommit()) { - assertThat( - commit.replaceManifestList( - barrierSnapshot, - barrierSnapshot.totalRecordCount(), - baseManifestList(barrierSnapshot), - deltaManifestList(barrierSnapshot), - barrierSnapshot.indexManifest(), - barrierSnapshot.nextRowId())) - .isTrue(); - } - - Snapshot normalSnapshot = checkNotNull(store.snapshotManager().latestSnapshot()); - assertThat(normalSnapshot.properties()) - .containsEntry("keep", "v1") - .doesNotContainKey(Snapshot.OVERWRITE_BARRIER_PROPERTY); + Snapshot latest = checkNotNull(store.snapshotManager().latestSnapshot()); + assertThat(latest.indexManifest()).isNotNull(); } @Test - public void testCompactManifestWithOverwriteBarrierProperty() throws Exception { - TestFileStore store = createStore(false); + public void testGlobalIndexCommitFailsForMissingRowIds() throws Exception { + TestFileStore store = createRowTrackingDataEvolutionStore(); List keyValues = generateDataList(1); BinaryRow partition = gen.getPartition(keyValues.get(0)); - store.commitData(keyValues, s -> partition, kv -> 0); - store.overwriteData(keyValues, s -> partition, kv -> 0, Collections.emptyMap()); - Snapshot latest = - store.overwriteData(keyValues, s -> partition, kv -> 0, Collections.emptyMap()) - .get(0); - - long deleteNum = - store.manifestListFactory().create().readDataManifests(latest).stream() - .mapToLong(ManifestFileMeta::numDeletedFiles) - .sum(); - assertThat(deleteNum).isGreaterThan(0); + Snapshot dataSnapshot = store.commitData(keyValues, s -> partition, kv -> 0).get(0); + long missingRowId = checkNotNull(dataSnapshot.nextRowId()); - Map barrierProperties = new HashMap<>(); - barrierProperties.put("keep", "v1"); - barrierProperties.put(Snapshot.OVERWRITE_BARRIER_PROPERTY, "true"); try (FileStoreCommitImpl commit = store.newCommit()) { - assertThat( - commit.replaceManifestList( - latest, - latest.totalRecordCount(), - baseManifestList(latest), - deltaManifestList(latest), - latest.indexManifest(), - latest.nextRowId(), - barrierProperties)) - .isTrue(); - } - - Snapshot barrierSnapshot = checkNotNull(store.snapshotManager().latestSnapshot()); - assertThat(barrierSnapshot.properties()).isEqualTo(barrierProperties); - - try (FileStoreCommit commit = store.newCommit()) { - commit.compactManifest(); - } - - Snapshot normalSnapshot = checkNotNull(store.snapshotManager().latestSnapshot()); - assertThat(normalSnapshot.id()).isGreaterThan(barrierSnapshot.id()); - assertThat(normalSnapshot.commitKind()).isEqualTo(Snapshot.CommitKind.COMPACT); - assertThat(normalSnapshot.properties()) - .containsEntry("keep", "v1") - .doesNotContainKey(Snapshot.OVERWRITE_BARRIER_PROPERTY); - } - - @Test - public void testOverwriteConflictWithIndexFromOptions() throws Exception { - TestFileStore store = createStore(false); - - List keyValues = generateDataList(1); - BinaryRow partition = gen.getPartition(keyValues.get(0)); - Snapshot latest = store.commitData(keyValues, s -> partition, kv -> 0).get(0); - - Map barrierProperties = new HashMap<>(); - barrierProperties.put(Snapshot.OVERWRITE_BARRIER_PROPERTY, "true"); - try (FileStoreCommitImpl commit = store.newCommit()) { - assertThat( - commit.replaceManifestList( - latest, - latest.totalRecordCount(), - baseManifestList(latest), - deltaManifestList(latest), - latest.indexManifest(), - latest.nextRowId(), - barrierProperties)) - .isTrue(); - } - - Map dynamicOptions = new HashMap<>(store.options().toMap()); - dynamicOptions.put( - CoreOptions.COMMIT_OVERWRITE_CONFLICT_WITH_INDEX_LAST_SAFE_SNAPSHOT.key(), "1"); - try (FileStoreCommitImpl commit = - newCommitWithSnapshotCommit( - store, - "overwrite-barrier-check", - new RenamingSnapshotCommit(store.snapshotManager(), Lock.empty()), - new CoreOptions(dynamicOptions), - true)) { assertThatThrownBy( () -> commit.commit( - indexCommittable(partition, "barrier-index"), false)) - .hasMessageContaining("Overwrite barrier snapshot 2") - .hasMessageContaining("task planned from snapshot 1"); - } - } - - @Test - public void testOverwriteConflictWithIndexIgnoresDataOnlyDelta() throws Exception { - TestFileStore store = createStore(false); - - List keyValues = generateDataList(1); - BinaryRow partition = gen.getPartition(keyValues.get(0)); - Snapshot latest = store.commitData(keyValues, s -> partition, kv -> 0).get(0); - - Map barrierProperties = new HashMap<>(); - barrierProperties.put(Snapshot.OVERWRITE_BARRIER_PROPERTY, "true"); - try (FileStoreCommitImpl commit = store.newCommit()) { - assertThat( - commit.replaceManifestList( - latest, - latest.totalRecordCount(), - baseManifestList(latest), - deltaManifestList(latest), - latest.indexManifest(), - latest.nextRowId(), - barrierProperties)) - .isTrue(); - } - - AtomicReference committableRef = new AtomicReference<>(); - store.commitDataImpl( - generateDataList(1), - gen::getPartition, - kv -> 0, - false, - null, - null, - Collections.emptyList(), - (commit, committable) -> committableRef.set(committable)); - - Map dynamicOptions = new HashMap<>(store.options().toMap()); - dynamicOptions.put( - CoreOptions.COMMIT_OVERWRITE_CONFLICT_WITH_INDEX_LAST_SAFE_SNAPSHOT.key(), "1"); - try (FileStoreCommitImpl commit = - newCommitWithSnapshotCommit( - store, - "overwrite-barrier-data-only-check", - new RenamingSnapshotCommit(store.snapshotManager(), Lock.empty()), - new CoreOptions(dynamicOptions), - true)) { - commit.commit(checkNotNull(committableRef.get()), false); + indexCommittable( + partition, + "missing-index", + missingRowId, + missingRowId), + false)) + .hasMessageContaining("Global index row ID existence conflict") + .hasMessageContaining("missing-index") + .hasMessageContaining("[" + missingRowId + ", " + missingRowId + "]"); } } @@ -1320,7 +1177,8 @@ private FileStoreCommitImpl newCommitWithSnapshotCommit( null); } - private ManifestCommittable indexCommittable(BinaryRow partition, String fileName) { + private ManifestCommittable indexCommittable( + BinaryRow partition, String fileName, long rowRangeStart, long rowRangeEnd) { ManifestCommittable committable = new ManifestCommittable(0); committable.addFileCommittable( new CommitMessageImpl( @@ -1334,7 +1192,12 @@ private ManifestCommittable indexCommittable(BinaryRow partition, String fileNam fileName, 1, 1, - (GlobalIndexMeta) null, + new GlobalIndexMeta( + rowRangeStart, + rowRangeEnd, + 0, + null, + null), null))), CompactIncrement.emptyIncrement())); return committable; @@ -1375,12 +1238,11 @@ private TestFileStore createStore(boolean failing, Map options) return createStore(failing, 1, CoreOptions.ChangelogProducer.NONE, options); } - private Pair baseManifestList(Snapshot snapshot) { - return Pair.of(snapshot.baseManifestList(), snapshot.baseManifestListSize()); - } - - private Pair deltaManifestList(Snapshot snapshot) { - return Pair.of(snapshot.deltaManifestList(), snapshot.deltaManifestListSize()); + private TestFileStore createRowTrackingDataEvolutionStore() throws Exception { + Map options = new HashMap<>(); + options.put(CoreOptions.ROW_TRACKING_ENABLED.key(), "true"); + options.put(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true"); + return createStore(false, options); } private TestFileStore createStore(boolean failing) throws Exception { diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java index de51152e0bce..b73bdbc54d4b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java @@ -26,12 +26,10 @@ import org.apache.paimon.manifest.FileEntry; import org.apache.paimon.manifest.FileKind; import org.apache.paimon.manifest.IndexManifestEntry; -import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.SimpleFileEntry; import org.apache.paimon.manifest.SimpleFileEntryWithDV; import org.apache.paimon.table.BucketMode; import org.apache.paimon.types.RowType; -import org.apache.paimon.utils.SnapshotManager; import org.junit.jupiter.api.Test; @@ -41,10 +39,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import static org.apache.paimon.data.BinaryRow.EMPTY_ROW; @@ -54,8 +50,6 @@ import static org.apache.paimon.operation.commit.ConflictDetection.buildBaseEntriesWithDV; import static org.apache.paimon.operation.commit.ConflictDetection.buildDeltaEntriesWithDV; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; class ConflictDetectionTest { @@ -339,13 +333,19 @@ private IndexManifestEntry createDvIndexEntry( DELETION_VECTORS_INDEX, fileName, 11, dvRanges.size(), dvRanges, null)); } - private IndexManifestEntry createIndexEntry( - String fileName, FileKind kind, BinaryRow partition) { + private IndexManifestEntry createGlobalIndexEntry( + String fileName, FileKind kind, BinaryRow partition, long from, long to) { return new IndexManifestEntry( kind, partition, 0, - new IndexFileMeta("btree", fileName, 11, 1, (GlobalIndexMeta) null, null)); + new IndexFileMeta( + "btree", + fileName, + 11, + 1, + new GlobalIndexMeta(from, to, 0, null, null), + null)); } private void assertConflict( @@ -393,6 +393,18 @@ void testShouldBeOverwriteCommit() { .isFalse(); } + @Test + void testChangedPartitionsIncludesGlobalIndexFiles() { + BinaryRow partition = BinaryRow.singleColumn(1); + + assertThat( + ManifestEntryChanges.changedPartitions( + Collections.emptyList(), + Collections.singletonList( + createGlobalIndexEntry("idx", ADD, partition, 0, 99)))) + .containsExactly(partition); + } + @Test void testCheckRowIdExistenceNoConflict() { ConflictDetection detection = createConflictDetection(); @@ -508,123 +520,64 @@ private SimpleFileEntry createFileEntryWithRowId( } @Test - void testDetectsOverwriteBarrierSnapshotConflict() { - SnapshotManager snapshotManager = mock(SnapshotManager.class); - Map barrierProperties = new HashMap<>(); - barrierProperties.put(Snapshot.OVERWRITE_BARRIER_PROPERTY, "true"); - when(snapshotManager.snapshot(2L)) - .thenReturn(snapshot(2, Snapshot.CommitKind.OVERWRITE, barrierProperties)); - - ConflictDetection detection = createConflictDetection(snapshotManager); - detection.setOverwriteConflictWithIndexCheckFromSnapshot(1L); - - Optional exception = - detection.checkConflicts( - snapshot(3, null), - Collections.emptyList(), - Collections.emptyList(), - Collections.singletonList( - createIndexEntry("idx", ADD, BinaryRow.EMPTY_ROW)), - null, - Snapshot.CommitKind.COMPACT); - - assertThat(exception).isPresent(); - assertThat(exception.get()) - .hasMessageContaining("Overwrite barrier snapshot 2") - .hasMessageContaining("task planned from snapshot 1"); - } - - @Test - void testIgnoresOverwriteBarrierPropertyOnNonOverwriteSnapshot() { - SnapshotManager snapshotManager = mock(SnapshotManager.class); - Map barrierProperties = new HashMap<>(); - barrierProperties.put(Snapshot.OVERWRITE_BARRIER_PROPERTY, "true"); - when(snapshotManager.snapshot(2L)) - .thenReturn(snapshot(2, Snapshot.CommitKind.APPEND, barrierProperties)); - - ConflictDetection detection = createConflictDetection(snapshotManager); - detection.setOverwriteConflictWithIndexCheckFromSnapshot(1L); + void testCheckGlobalIndexRowIdExistenceNoConflict() { + ConflictDetection detection = createConflictDetection(); Optional exception = detection.checkConflicts( - snapshot(2, null), - Collections.emptyList(), + snapshot(1), + Arrays.asList( + createFileEntryWithRowId("f1", ADD, 0L, 100L), + createFileEntryWithRowId("f2", ADD, 100L, 50L)), Collections.emptyList(), Collections.singletonList( - createIndexEntry("idx", ADD, BinaryRow.EMPTY_ROW)), + createGlobalIndexEntry("idx", ADD, BinaryRow.EMPTY_ROW, 0, 149)), null, - Snapshot.CommitKind.COMPACT); + Snapshot.CommitKind.APPEND); assertThat(exception).isNotPresent(); } @Test - void testDetectsOverlappedOverwriteSnapshotForIndexCommit() { - SnapshotManager snapshotManager = mock(SnapshotManager.class); - CommitScanner commitScanner = mock(CommitScanner.class); - Snapshot overwriteSnapshot = snapshot(2, Snapshot.CommitKind.OVERWRITE, null); - when(snapshotManager.snapshot(2L)).thenReturn(overwriteSnapshot); - - BinaryRow partition = BinaryRow.singleColumn(1); - when(commitScanner.readIncrementalEntries( - overwriteSnapshot, Collections.singletonList(partition))) - .thenReturn(Collections.singletonList(mock(ManifestEntry.class))); - - ConflictDetection detection = createConflictDetection(snapshotManager, commitScanner); - detection.setOverwriteConflictWithIndexCheckFromSnapshot(1L); + void testCheckGlobalIndexRowIdExistenceBaseFileRemoved() { + ConflictDetection detection = createConflictDetection(); Optional exception = detection.checkConflicts( - snapshot(2, null), + snapshot(1), + Collections.singletonList(createFileEntryWithRowId("f1", ADD, 0L, 100L)), Collections.emptyList(), - Collections.emptyList(), - Collections.singletonList(createIndexEntry("idx", ADD, partition)), + Collections.singletonList( + createGlobalIndexEntry("idx", ADD, BinaryRow.EMPTY_ROW, 0, 149)), null, - Snapshot.CommitKind.COMPACT); + Snapshot.CommitKind.APPEND); assertThat(exception).isPresent(); assertThat(exception.get()) - .hasMessageContaining("Overwrite snapshot 2") - .hasMessageContaining("task planned from snapshot 1"); + .hasMessageContaining("Global index row ID existence conflict") + .hasMessageContaining("idx") + .hasMessageContaining("[0, 149]"); } @Test - void testIgnoresNonOverlappedOverwriteSnapshotForIndexCommit() { - SnapshotManager snapshotManager = mock(SnapshotManager.class); - CommitScanner commitScanner = mock(CommitScanner.class); - Snapshot overwriteSnapshot = snapshot(2, Snapshot.CommitKind.OVERWRITE, null); - when(snapshotManager.snapshot(2L)).thenReturn(overwriteSnapshot); - - BinaryRow partition = BinaryRow.singleColumn(1); - when(commitScanner.readIncrementalEntries( - overwriteSnapshot, Collections.singletonList(partition))) - .thenReturn(Collections.emptyList()); - - ConflictDetection detection = createConflictDetection(snapshotManager, commitScanner); - detection.setOverwriteConflictWithIndexCheckFromSnapshot(1L); + void testCheckGlobalIndexRowIdExistenceSkipsDeleteIndexEntry() { + ConflictDetection detection = createConflictDetection(); Optional exception = detection.checkConflicts( - snapshot(2, null), + snapshot(1), Collections.emptyList(), Collections.emptyList(), - Collections.singletonList(createIndexEntry("idx", ADD, partition)), + Collections.singletonList( + createGlobalIndexEntry( + "idx", DELETE, BinaryRow.EMPTY_ROW, 0, 149)), null, - Snapshot.CommitKind.COMPACT); + Snapshot.CommitKind.APPEND); assertThat(exception).isNotPresent(); } private ConflictDetection createConflictDetection() { - return createConflictDetection(null); - } - - private ConflictDetection createConflictDetection(@Nullable SnapshotManager snapshotManager) { - return createConflictDetection(snapshotManager, null); - } - - private ConflictDetection createConflictDetection( - @Nullable SnapshotManager snapshotManager, @Nullable CommitScanner commitScanner) { return new ConflictDetection( "test-table", "test-user", @@ -636,16 +589,11 @@ private ConflictDetection createConflictDetection( true, false, null, - snapshotManager, - commitScanner); - } - - private Snapshot snapshot(long id, @Nullable Map properties) { - return snapshot(id, Snapshot.CommitKind.APPEND, properties); + null, + null); } - private Snapshot snapshot( - long id, Snapshot.CommitKind commitKind, @Nullable Map properties) { + private Snapshot snapshot(long id) { return new Snapshot( id, 0, @@ -658,14 +606,14 @@ private Snapshot snapshot( null, "commit-user", id, - commitKind, + Snapshot.CommitKind.APPEND, id, 0, 0, null, null, null, - properties, + null, null); } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java index bcec33331437..9542eea4a435 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java @@ -95,7 +95,6 @@ public static boolean buildIndex( Options userOptions) throws Exception { List> allStreams = new ArrayList<>(); - Long overwriteConflictCheckFromSnapshot = null; for (String indexColumn : indexColumns) { BTreeGlobalIndexBuilder indexBuilder = indexBuilderSupplier.get().withIndexField(indexColumn); @@ -108,12 +107,6 @@ public static boolean buildIndex( if (!indexRangeAndSplits.isPresent()) { continue; } - if (indexBuilder.scanSnapshotId().isPresent()) { - overwriteConflictCheckFromSnapshot = - minSnapshot( - overwriteConflictCheckFromSnapshot, - indexBuilder.scanSnapshotId().get()); - } Pair> scanResult = indexRangeAndSplits.get(); List splits = splitByContiguousRowRange(scanResult.getRight()); @@ -201,26 +194,13 @@ public static boolean buildIndex( @SuppressWarnings("unchecked") DataStream[] rest = allStreams.subList(1, allStreams.size()).toArray(new DataStream[0]); - FileStoreTable commitTable = - overwriteConflictCheckFromSnapshot == null - ? table - : table.copy( - Collections.singletonMap( - CoreOptions - .COMMIT_OVERWRITE_CONFLICT_WITH_INDEX_LAST_SAFE_SNAPSHOT - .key(), - String.valueOf(overwriteConflictCheckFromSnapshot))); - commit(commitTable, allStreams.get(0).union(rest)); + commit(table, allStreams.get(0).union(rest)); return true; } return false; } - private static Long minSnapshot(Long left, long right) { - return left == null ? right : Math.min(left, right); - } - public static void buildIndexAndExecute( StreamExecutionEnvironment env, FileStoreTable table, diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericGlobalIndexBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericGlobalIndexBuilder.java index da4644524c1c..415c1590a59c 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericGlobalIndexBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericGlobalIndexBuilder.java @@ -29,7 +29,6 @@ import java.io.Serializable; import java.util.Collections; import java.util.List; -import java.util.Optional; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -41,7 +40,6 @@ public class GenericGlobalIndexBuilder implements Serializable { protected final FileStoreTable table; @Nullable protected PartitionPredicate partitionPredicate; - @Nullable protected Long scanSnapshotId; public GenericGlobalIndexBuilder(FileStoreTable table) { this.table = table; @@ -56,10 +54,6 @@ public FileStoreTable table() { return table; } - public Optional scanSnapshotId() { - return Optional.ofNullable(scanSnapshotId); - } - /** * Scans manifest entries to determine which files need to be indexed. * @@ -81,10 +75,8 @@ public List scan() { Snapshot snapshot = table.snapshotManager().latestSnapshot(); if (snapshot == null) { - scanSnapshotId = null; return Collections.emptyList(); } - scanSnapshotId = snapshot.id(); return table.store() .newScan() diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java index 6b2ae265a2cb..5896503ce09d 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java @@ -179,21 +179,10 @@ public static boolean buildIndex( List entries = indexBuilder.scan(); List deletedIndexEntries = indexBuilder.deletedIndexEntries(); - Long overwriteConflictCheckFromSnapshot = indexBuilder.scanSnapshotId().orElse(null); - FileStoreTable commitTable = - overwriteConflictCheckFromSnapshot == null - ? table - : table.copy( - Collections.singletonMap( - CoreOptions - .COMMIT_OVERWRITE_CONFLICT_WITH_INDEX_LAST_SAFE_SNAPSHOT - .key(), - String.valueOf(overwriteConflictCheckFromSnapshot))); return buildTopology( env, table, - commitTable, indexColumn, indexType, userOptions, @@ -214,7 +203,6 @@ public static boolean buildIndex( private static boolean buildTopology( StreamExecutionEnvironment env, FileStoreTable table, - FileStoreTable commitTable, String indexColumn, String indexType, Options userOptions, @@ -306,7 +294,7 @@ private static boolean buildTopology( built = built.union(deletes); } - commit(commitTable, indexType, built); + commit(table, indexType, built); return true; } From 2ba7fc87cf9cc9c619cb5215cea7e96f5c2d3cfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Wed, 3 Jun 2026 12:16:27 +0800 Subject: [PATCH 12/16] [core] Inline global index conflict check condition --- .../org/apache/paimon/operation/FileStoreCommitImpl.java | 6 +++++- .../paimon/operation/commit/ConflictDetection.java | 9 --------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java index f62bd5b3b199..70ef1283c10e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java @@ -327,7 +327,11 @@ public int commit(ManifestCommittable committable, boolean checkAppendFiles) { checkAppendFiles = true; allowRollback = true; } - if (conflictDetection.hasGlobalIndexFileAddition(changes.appendIndexFiles)) { + if (changes.appendIndexFiles.stream() + .anyMatch( + entry -> + entry.kind() == FileKind.ADD + && entry.indexFile().globalIndexMeta() != null)) { checkAppendFiles = true; } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java index 66a43712dd7e..e0c56e11c853 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java @@ -157,15 +157,6 @@ public boolean shouldBeOverwriteCommit( return false; } - public boolean hasGlobalIndexFileAddition(List indexFileChanges) { - for (IndexManifestEntry entry : indexFileChanges) { - if (entry.kind() == FileKind.ADD && entry.indexFile().globalIndexMeta() != null) { - return true; - } - } - return false; - } - public Optional checkConflicts( Snapshot latestSnapshot, List baseEntries, From d5833f0bdafb446a96b57c49bb934016ad867538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Wed, 3 Jun 2026 14:03:19 +0800 Subject: [PATCH 13/16] [common][core] Use RowRangeIndex for row id existence check --- .../apache/paimon/utils/RowRangeIndex.java | 7 ++++ .../paimon/utils/RowRangeIndexTest.java | 41 +++++++++++++++++++ .../operation/commit/ConflictDetection.java | 31 +++----------- .../paimon/operation/FileStoreCommitTest.java | 6 +-- .../commit/ConflictDetectionTest.java | 3 +- 5 files changed, 56 insertions(+), 32 deletions(-) create mode 100644 paimon-common/src/test/java/org/apache/paimon/utils/RowRangeIndexTest.java diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/RowRangeIndex.java b/paimon-common/src/main/java/org/apache/paimon/utils/RowRangeIndex.java index a068d04f728d..ffd83eddd227 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/RowRangeIndex.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/RowRangeIndex.java @@ -56,6 +56,13 @@ public boolean intersects(long start, long end) { return candidate < starts.length && starts[candidate] <= end; } + public boolean contains(Range range) { + int candidate = lowerBound(ends, range.from); + return candidate < starts.length + && starts[candidate] <= range.from + && ends[candidate] >= range.to; + } + public List intersectedRanges(long start, long end) { int left = lowerBound(ends, start); if (left >= ranges.size()) { diff --git a/paimon-common/src/test/java/org/apache/paimon/utils/RowRangeIndexTest.java b/paimon-common/src/test/java/org/apache/paimon/utils/RowRangeIndexTest.java new file mode 100644 index 000000000000..7c4a9b155013 --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/utils/RowRangeIndexTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.utils; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link RowRangeIndex}. */ +class RowRangeIndexTest { + + @Test + void testContains() { + RowRangeIndex index = + RowRangeIndex.create( + Arrays.asList(new Range(0, 99), new Range(100, 149), new Range(200, 299))); + + assertThat(index.contains(new Range(0, 149))).isTrue(); + assertThat(index.contains(new Range(50, 120))).isTrue(); + assertThat(index.contains(new Range(150, 199))).isFalse(); + assertThat(index.contains(new Range(100, 200))).isFalse(); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java index e0c56e11c853..5ab4c8151673 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java @@ -40,6 +40,7 @@ import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.Range; import org.apache.paimon.utils.RangeHelper; +import org.apache.paimon.utils.RowRangeIndex; import org.apache.paimon.utils.SnapshotManager; import org.slf4j.Logger; @@ -572,18 +573,19 @@ private Optional checkGlobalIndexRowIdExistence( .add(entry.nonNullRowIdRange()); } } + Map rowRangeIndexes = new HashMap<>(); for (Map.Entry> entry : dataRanges.entrySet()) { - entry.setValue(Range.sortAndMergeOverlap(entry.getValue(), true)); + rowRangeIndexes.put(entry.getKey(), RowRangeIndex.create(entry.getValue())); } for (IndexManifestEntry indexEntry : indexesToCheck) { GlobalIndexMeta globalIndex = indexEntry.indexFile().globalIndexMeta(); checkState(globalIndex != null, "Global index meta must not be null."); Range indexRange = globalIndex.rowRange(); - List currentRanges = - dataRanges.get( + RowRangeIndex currentRanges = + rowRangeIndexes.get( new PartitionBucketKey(indexEntry.partition(), indexEntry.bucket())); - if (!containsRange(currentRanges, indexRange)) { + if (currentRanges == null || !currentRanges.contains(indexRange)) { return Optional.of( new RuntimeException( String.format( @@ -611,27 +613,6 @@ private List globalIndexFileAdditions( return result; } - private boolean containsRange(@Nullable List ranges, Range target) { - if (ranges == null || ranges.isEmpty()) { - return false; - } - - long next = target.from; - for (Range range : ranges) { - if (range.to < next) { - continue; - } - if (range.from > next) { - return false; - } - if (range.to >= target.to) { - return true; - } - next = range.to + 1; - } - return false; - } - Optional checkRowIdExistence( List baseEntries, List deltaEntries, diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java index d04832902d21..8acf0ad0ec70 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java @@ -1193,11 +1193,7 @@ private ManifestCommittable indexCommittable( 1, 1, new GlobalIndexMeta( - rowRangeStart, - rowRangeEnd, - 0, - null, - null), + rowRangeStart, rowRangeEnd, 0, null, null), null))), CompactIncrement.emptyIncrement())); return committable; diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java index b73bdbc54d4b..d7282fe66173 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java @@ -569,8 +569,7 @@ void testCheckGlobalIndexRowIdExistenceSkipsDeleteIndexEntry() { Collections.emptyList(), Collections.emptyList(), Collections.singletonList( - createGlobalIndexEntry( - "idx", DELETE, BinaryRow.EMPTY_ROW, 0, 149)), + createGlobalIndexEntry("idx", DELETE, BinaryRow.EMPTY_ROW, 0, 149)), null, Snapshot.CommitKind.APPEND); From 674aab3252b5b39610788efbe835f0aaaf9393e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Wed, 3 Jun 2026 14:21:52 +0800 Subject: [PATCH 14/16] [core] Simplify global index row id check --- .../operation/commit/ConflictDetection.java | 53 +++---------------- .../paimon/operation/FileStoreCommitTest.java | 10 ++-- 2 files changed, 14 insertions(+), 49 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java index 5ab4c8151673..1f73b705f6f2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java @@ -563,40 +563,28 @@ private Optional checkGlobalIndexRowIdExistence( return Optional.empty(); } - Map> dataRanges = new HashMap<>(); + List dataRanges = new ArrayList<>(); for (SimpleFileEntry entry : baseEntries) { if (entry.kind() == FileKind.ADD && entry.firstRowId() != null) { - dataRanges - .computeIfAbsent( - new PartitionBucketKey(entry.partition(), entry.bucket()), - ignored -> new ArrayList<>()) - .add(entry.nonNullRowIdRange()); + dataRanges.add(entry.nonNullRowIdRange()); } } - Map rowRangeIndexes = new HashMap<>(); - for (Map.Entry> entry : dataRanges.entrySet()) { - rowRangeIndexes.put(entry.getKey(), RowRangeIndex.create(entry.getValue())); - } + RowRangeIndex rowRangeIndex = RowRangeIndex.create(dataRanges); for (IndexManifestEntry indexEntry : indexesToCheck) { GlobalIndexMeta globalIndex = indexEntry.indexFile().globalIndexMeta(); checkState(globalIndex != null, "Global index meta must not be null."); Range indexRange = globalIndex.rowRange(); - RowRangeIndex currentRanges = - rowRangeIndexes.get( - new PartitionBucketKey(indexEntry.partition(), indexEntry.bucket())); - if (currentRanges == null || !currentRanges.contains(indexRange)) { + if (!rowRangeIndex.contains(indexRange)) { return Optional.of( new RuntimeException( String.format( "Global index row ID existence conflict: index file '%s' " - + "references row range %s in bucket %d, but this " - + "range is not fully covered by current data " + + "references row range %s, but this range " + + "is not fully covered by current data " + "files. The referenced row IDs may have been " + "reassigned or removed by a concurrent commit.", - indexEntry.indexFile().fileName(), - indexRange, - indexEntry.bucket()))); + indexEntry.indexFile().fileName(), indexRange))); } } return Optional.empty(); @@ -706,33 +694,6 @@ public int hashCode() { } } - private static class PartitionBucketKey { - private final BinaryRow partition; - private final int bucket; - - PartitionBucketKey(BinaryRow partition, int bucket) { - this.partition = partition; - this.bucket = bucket; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - PartitionBucketKey that = (PartitionBucketKey) o; - return bucket == that.bucket && Objects.equals(partition, that.partition); - } - - @Override - public int hashCode() { - return Objects.hash(partition, bucket); - } - } - private static boolean dedicatedStorageFile(String fileName) { return isBlobFile(fileName) || isVectorStoreFile(fileName); } diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java index 8acf0ad0ec70..ef40a0fa2c4d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java @@ -1238,7 +1238,7 @@ private TestFileStore createRowTrackingDataEvolutionStore() throws Exception { Map options = new HashMap<>(); options.put(CoreOptions.ROW_TRACKING_ENABLED.key(), "true"); options.put(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true"); - return createStore(false, options); + return createStore(false, -1, CoreOptions.ChangelogProducer.NONE, options); } private TestFileStore createStore(boolean failing) throws Exception { @@ -1267,14 +1267,18 @@ private TestFileStore createStore( ? FailingFileIO.getFailingPath(failingName, tempDir.toString()) : TraceableFileIO.SCHEME + "://" + tempDir.toString(); Path path = new Path(tempDir.toUri()); + List primaryKeys = + Boolean.parseBoolean(options.get(CoreOptions.ROW_TRACKING_ENABLED.key())) + ? Collections.emptyList() + : TestKeyValueGenerator.getPrimaryKeys( + TestKeyValueGenerator.GeneratorMode.MULTI_PARTITIONED); TableSchema tableSchema = SchemaUtils.forceCommit( new SchemaManager(new LocalFileIO(), path), new Schema( TestKeyValueGenerator.DEFAULT_ROW_TYPE.getFields(), TestKeyValueGenerator.DEFAULT_PART_TYPE.getFieldNames(), - TestKeyValueGenerator.getPrimaryKeys( - TestKeyValueGenerator.GeneratorMode.MULTI_PARTITIONED), + primaryKeys, options, null)); return new TestFileStore.Builder( From 030e6ceacedfcbcbfc4e9ba6f92b1ca0ec1fcc94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Wed, 3 Jun 2026 14:23:10 +0800 Subject: [PATCH 15/16] [build] Ignore Python build output --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f533fd540d20..e62797c76052 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ paimon-python/dist/ paimon-python/*.egg-info/ paimon-python/dev/log paimon-spark/paimon-spark-ut/PaimonLambdaFunctionfunction_test.java +paimon-python/build/ ### Misc ### *.swp From 826a88ff8545d35fa81ad95eec74be53b65a50d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Thu, 4 Jun 2026 14:54:26 +0800 Subject: [PATCH 16/16] [core] Restore btree index scan snapshot state --- .../paimon/globalindex/btree/BTreeGlobalIndexBuilder.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexBuilder.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexBuilder.java index dcac4b599e97..febf86d055db 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexBuilder.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexBuilder.java @@ -97,6 +97,7 @@ public class BTreeGlobalIndexBuilder implements Serializable { // readRowType is composed by partition fields, indexed field and _ROW_ID field private RowType readRowType; @Nullable private Snapshot snapshot; + @Nullable private Long scanSnapshotId; @Nullable private PartitionPredicate partitionPredicate; @@ -133,6 +134,10 @@ public BTreeGlobalIndexBuilder withSnapshot(Snapshot snapshot) { return this; } + public Optional scanSnapshotId() { + return Optional.ofNullable(scanSnapshotId); + } + public Optional>> scan() { SnapshotReader snapshotReader = table.newSnapshotReader(); if (partitionPredicate != null) {