From 8d9f5df35cc223f4b793736a84ecdbde124639ed Mon Sep 17 00:00:00 2001 From: fhan Date: Thu, 13 Aug 2026 20:35:45 +0800 Subject: [PATCH 01/11] [lake/paimon] Support clean and legacy Paimon lake table schemas (#3902) --- .../fluss/lake/paimon/PaimonLakeCatalog.java | 33 ++-- .../paimon/source/PaimonRecordReader.java | 67 +++++-- .../tiering/FlussRecordAsPaimonRow.java | 22 ++- .../lake/paimon/tiering/PaimonLakeWriter.java | 14 +- .../lake/paimon/tiering/RecordWriter.java | 6 +- .../append/AppendOnlyArrowBatchHelper.java | 16 +- .../tiering/append/AppendOnlyWriter.java | 12 +- .../tiering/mergetree/MergeTreeWriter.java | 25 ++- .../lake/paimon/utils/PaimonConversions.java | 41 +++-- .../paimon/utils/PaimonRowAsFlussRow.java | 20 ++- .../paimon/utils/PaimonSystemColumns.java | 164 ++++++++++++++++++ .../paimon/utils/PaimonTableValidation.java | 43 +++++ .../tiering/FlussRecordAsPaimonRowTest.java | 43 ++--- 13 files changed, 420 insertions(+), 86 deletions(-) create mode 100644 fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSystemColumns.java diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeCatalog.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeCatalog.java index bd846cec28d..e23bae37023 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeCatalog.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeCatalog.java @@ -23,6 +23,7 @@ import org.apache.fluss.exception.TableAlreadyExistException; import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.lake.lakestorage.LakeCatalog; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns; import org.apache.fluss.metadata.TableChange; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TablePath; @@ -39,7 +40,6 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; import org.apache.paimon.types.DataType; -import org.apache.paimon.types.DataTypes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,27 +53,21 @@ import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimonSchemaChanges; import static org.apache.fluss.lake.paimon.utils.PaimonTableValidation.checkTableIsEmpty; import static org.apache.fluss.lake.paimon.utils.PaimonTableValidation.isPaimonSchemaCompatible; -import static org.apache.fluss.metadata.TableDescriptor.BUCKET_COLUMN_NAME; -import static org.apache.fluss.metadata.TableDescriptor.OFFSET_COLUMN_NAME; -import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; /** A Paimon implementation of {@link LakeCatalog}. */ public class PaimonLakeCatalog implements LakeCatalog { private static final Logger LOG = LoggerFactory.getLogger(PaimonLakeCatalog.class); private static final String PAIMON_PATH_KEY = "paimon.path"; - public static final LinkedHashMap SYSTEM_COLUMNS = new LinkedHashMap<>(); - - static { - // We need __bucket system column to filter out the given bucket - // for paimon bucket-unaware append only table. - // It's not required for paimon bucket-aware table like primary key table - // and bucket-aware append only table, but we always add the system column - // for consistent behavior - SYSTEM_COLUMNS.put(BUCKET_COLUMN_NAME, DataTypes.INT()); - SYSTEM_COLUMNS.put(OFFSET_COLUMN_NAME, DataTypes.BIGINT()); - SYSTEM_COLUMNS.put(TIMESTAMP_COLUMN_NAME, DataTypes.TIMESTAMP_LTZ_MILLIS()); - } + + /** + * The three Fluss system columns and their Paimon types, kept for readers, writers and the + * lookuper to recognise legacy tables by column name. Retained as an alias of {@link + * PaimonSystemColumns#SYSTEM_COLUMNS}; under FIP-27 these columns are no longer added to newly + * created (clean) tables. + */ + public static final LinkedHashMap SYSTEM_COLUMNS = + PaimonSystemColumns.SYSTEM_COLUMNS; private final Catalog paimonCatalog; @@ -127,13 +121,15 @@ public void alterTable(TablePath tablePath, List tableChanges, Cont } Schema currentPaimonSchema = fileStoreTable.schema().toSchema(); + PaimonSystemColumns.LakeLayout lakeLayout = + PaimonSystemColumns.detectLayout(fileStoreTable.schema().logicalRowType()); List paimonSchemaChanges; if (isPaimonSchemaCompatible( currentPaimonSchema, toPaimonSchema(context.getCurrentTable()))) { // if the paimon schema is same as current fluss schema, directly apply all the // changes. - paimonSchemaChanges = toPaimonSchemaChanges(changesToApply); + paimonSchemaChanges = toPaimonSchemaChanges(changesToApply, lakeLayout); } else if (isPaimonSchemaCompatible( currentPaimonSchema, toPaimonSchema(context.getExpectedTable()))) { // if the schema is same as applied fluss schema , skip adding columns. @@ -144,7 +140,8 @@ currentPaimonSchema, toPaimonSchema(context.getExpectedTable()))) { tableChange -> !(tableChange instanceof TableChange.AddColumn)) - .collect(Collectors.toList())); + .collect(Collectors.toList()), + lakeLayout); } else { throw new InvalidAlterTableException( String.format( diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/PaimonRecordReader.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/PaimonRecordReader.java index 3af7467cfab..577b17658db 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/PaimonRecordReader.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/PaimonRecordReader.java @@ -19,6 +19,8 @@ package org.apache.fluss.lake.paimon.source; import org.apache.fluss.lake.paimon.utils.PaimonRowAsFlussRow; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.lake.source.RecordReader; import org.apache.fluss.record.ChangeType; import org.apache.fluss.record.GenericRecord; @@ -46,6 +48,14 @@ /** Record reader for paimon table. */ public class PaimonRecordReader implements RecordReader { + /** + * Sentinel log offset / timestamp emitted for rows read from a clean lake table, which does not + * store the {@code __offset} / {@code __timestamp} system columns. A negative offset is + * interpreted downstream as "no valid offset" (snapshot phase), see {@code + * LakeRecordRecordEmitter}. + */ + private static final long NO_SYSTEM_COLUMN_VALUE = -1L; + protected PaimonRowAsFlussRecordIterator iterator; protected @Nullable int[][] project; protected RowType paimonRowType; @@ -56,10 +66,12 @@ public PaimonRecordReader( @Nullable int[][] project, @Nullable Predicate predicate) throws IOException { + LakeLayout lakeLayout = + PaimonSystemColumns.detectLayout(fileStoreTable.schema().logicalRowType()); ReadBuilder readBuilder = fileStoreTable.newReadBuilder(); RowType paimonFullRowType = fileStoreTable.rowType(); if (project != null) { - readBuilder = applyProject(readBuilder, project, paimonFullRowType); + readBuilder = applyProject(readBuilder, project, paimonFullRowType, lakeLayout); } if (predicate != null) { @@ -71,13 +83,15 @@ public PaimonRecordReader( if (split == null) { iterator = new PaimonRecordReader.PaimonRowAsFlussRecordIterator( - org.apache.paimon.utils.CloseableIterator.empty(), paimonRowType); + org.apache.paimon.utils.CloseableIterator.empty(), + paimonRowType, + lakeLayout); } else { org.apache.paimon.reader.RecordReader recordReader = tableRead.createReader(split.dataSplit()); iterator = new PaimonRecordReader.PaimonRowAsFlussRecordIterator( - recordReader.toCloseableIterator(), paimonRowType); + recordReader.toCloseableIterator(), paimonRowType, lakeLayout); } } @@ -87,9 +101,19 @@ public CloseableIterator read() throws IOException { } private ReadBuilder applyProject( - ReadBuilder readBuilder, int[][] projects, RowType paimonFullRowType) { + ReadBuilder readBuilder, + int[][] projects, + RowType paimonFullRowType, + LakeLayout lakeLayout) { int[] projectIds = Arrays.stream(projects).mapToInt(project -> project[0]).toArray(); + if (lakeLayout == LakeLayout.CLEAN) { + // Clean tables have no system columns to read, so project the business columns only. + return readBuilder.withProjection(projectIds); + } + + // Legacy tables carry __offset/__timestamp, which the iterator needs to recover the log + // offset and timestamp of each record; append them to the projection. int offsetFieldPos = paimonFullRowType.getFieldIndex(OFFSET_COLUMN_NAME); int timestampFieldPos = paimonFullRowType.getFieldIndex(TIMESTAMP_COLUMN_NAME); @@ -115,13 +139,28 @@ public static class PaimonRowAsFlussRecordIterator implements CloseableIterator< public PaimonRowAsFlussRecordIterator( org.apache.paimon.utils.CloseableIterator paimonRowIterator, - RowType paimonRowType) { + RowType paimonRowType, + LakeLayout lakeLayout) { this.paimonRowIterator = paimonRowIterator; - this.logOffsetColIndex = paimonRowType.getFieldIndex(OFFSET_COLUMN_NAME); - this.timestampColIndex = paimonRowType.getFieldIndex(TIMESTAMP_COLUMN_NAME); - int[] project = IntStream.range(0, paimonRowType.getFieldCount() - 2).toArray(); - projectedRow = ProjectedRow.from(project); + int fieldCount = paimonRowType.getFieldCount(); + if (lakeLayout == LakeLayout.CLEAN) { + // No system columns are read; all projected fields are business fields, and the + // log offset / timestamp are not available from the lake table. + this.logOffsetColIndex = -1; + this.timestampColIndex = -1; + projectedRow = ProjectedRow.from(IntStream.range(0, fieldCount).toArray()); + } else { + // Legacy layout: applyProject appended exactly __offset and __timestamp (not + // __bucket) as the last two projected fields, so the business fields are all fields + // except those trailing two. + this.logOffsetColIndex = paimonRowType.getFieldIndex(OFFSET_COLUMN_NAME); + this.timestampColIndex = paimonRowType.getFieldIndex(TIMESTAMP_COLUMN_NAME); + int[] project = IntStream.range(0, fieldCount - 2).toArray(); + projectedRow = ProjectedRow.from(project); + } + // The wrapped row is only ever accessed by index through projectedRow, which already + // drops the system columns, so no trailing-system-column trimming is needed here. paimonRowAsFlussRow = new PaimonRowAsFlussRow(); } @@ -143,8 +182,14 @@ public boolean hasNext() { public LogRecord next() { InternalRow paimonRow = paimonRowIterator.next(); ChangeType changeType = toChangeType(paimonRow.getRowKind()); - long offset = paimonRow.getLong(logOffsetColIndex); - long timestamp = paimonRow.getTimestamp(timestampColIndex, 6).getMillisecond(); + long offset = + logOffsetColIndex < 0 + ? NO_SYSTEM_COLUMN_VALUE + : paimonRow.getLong(logOffsetColIndex); + long timestamp = + timestampColIndex < 0 + ? NO_SYSTEM_COLUMN_VALUE + : paimonRow.getTimestamp(timestampColIndex, 6).getMillisecond(); return new GenericRecord( offset, diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRow.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRow.java index bc030301037..0705ac4ed99 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRow.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRow.java @@ -18,6 +18,7 @@ package org.apache.fluss.lake.paimon.tiering; import org.apache.fluss.lake.paimon.source.FlussRowAsPaimonRow; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.record.LogRecord; import org.apache.paimon.data.InternalRow; @@ -34,6 +35,7 @@ public class FlussRecordAsPaimonRow extends FlussRowAsPaimonRow { private final int bucket; + private final LakeLayout lakeLayout; private LogRecord logRecord; private int originRowFieldCount; private final int businessFieldCount; @@ -41,13 +43,23 @@ public class FlussRecordAsPaimonRow extends FlussRowAsPaimonRow { private final int offsetFieldIndex; private final int timestampFieldIndex; - public FlussRecordAsPaimonRow(int bucket, RowType tableTowType) { + public FlussRecordAsPaimonRow(int bucket, RowType tableTowType, LakeLayout lakeLayout) { super(tableTowType); this.bucket = bucket; - this.businessFieldCount = tableRowType.getFieldCount() - SYSTEM_COLUMNS.size(); - this.bucketFieldIndex = businessFieldCount; - this.offsetFieldIndex = businessFieldCount + 1; - this.timestampFieldIndex = businessFieldCount + 2; + this.lakeLayout = lakeLayout; + if (lakeLayout == LakeLayout.LEGACY) { + // Legacy tables append the three system columns after the business columns. + this.businessFieldCount = tableRowType.getFieldCount() - SYSTEM_COLUMNS.size(); + this.bucketFieldIndex = businessFieldCount; + this.offsetFieldIndex = businessFieldCount + 1; + this.timestampFieldIndex = businessFieldCount + 2; + } else { + // Clean tables contain only business columns; there are no system fields to emit. + this.businessFieldCount = tableRowType.getFieldCount(); + this.bucketFieldIndex = -1; + this.offsetFieldIndex = -1; + this.timestampFieldIndex = -1; + } } public void setFlussRecord(LogRecord logRecord) { diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java index 2a38e388a06..cb82976c9bb 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java @@ -21,6 +21,8 @@ import org.apache.fluss.lake.batch.RecordBatch; import org.apache.fluss.lake.paimon.tiering.append.AppendOnlyWriter; import org.apache.fluss.lake.paimon.tiering.mergetree.MergeTreeWriter; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.lake.writer.LakeWriter; import org.apache.fluss.lake.writer.SupportsRecordBatchWrite; import org.apache.fluss.lake.writer.WriterInitContext; @@ -58,6 +60,12 @@ public PaimonLakeWriter( List partitionKeys = fileStoreTable.partitionKeys(); RowType flussRowType = writerInitContext.tableInfo().getRowType(); + // FIP-27: detect whether the target Paimon table is a clean table (only user columns) or a + // legacy table (carrying the three Fluss system columns). Writers emit system columns only + // for legacy tables. + LakeLayout lakeLayout = + PaimonSystemColumns.detectLayout(fileStoreTable.schema().logicalRowType()); + this.recordWriter = fileStoreTable.primaryKeys().isEmpty() ? new AppendOnlyWriter( @@ -65,14 +73,16 @@ public PaimonLakeWriter( writerInitContext.tableBucket(), writerInitContext.partition(), partitionKeys, - flussRowType) + flussRowType, + lakeLayout) : new MergeTreeWriter( fileStoreTable, writerInitContext.tableBucket(), writerInitContext.partition(), partitionKeys, flussRowType, - writerInitContext.ioTmpDirs()); + writerInitContext.ioTmpDirs(), + lakeLayout); } @Override diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java index 173407bb9c7..dad3f7e8f04 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java @@ -17,6 +17,7 @@ package org.apache.fluss.lake.paimon.tiering; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.metadata.ResolvedPartitionSpec; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.record.LogRecord; @@ -49,7 +50,8 @@ public RecordWriter( TableBucket tableBucket, @Nullable String partition, List partitionKeys, - org.apache.fluss.types.RowType flussRowType) { + org.apache.fluss.types.RowType flussRowType, + LakeLayout lakeLayout) { this.tableWrite = tableWrite; this.tableRowType = tableRowType; this.bucket = tableBucket.getBucket(); @@ -62,7 +64,7 @@ public RecordWriter( this.partition = resolvePartition(partition, partitionKeys, flussRowType); } this.flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket.getBucket(), tableRowType); + new FlussRecordAsPaimonRow(tableBucket.getBucket(), tableRowType, lakeLayout); } public abstract void write(LogRecord record) throws Exception; diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java index 4fdb8bce79b..4036ef96396 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java @@ -17,6 +17,7 @@ package org.apache.fluss.lake.paimon.tiering.append; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.record.ArrowBatchData; @@ -57,6 +58,7 @@ class AppendOnlyArrowBatchHelper implements AutoCloseable { private final TableWriteImpl tableWrite; private final RowType tableRowType; private final int bucket; + private final LakeLayout lakeLayout; private static final Field BUCKET_FIELD = new Field( @@ -88,11 +90,13 @@ class AppendOnlyArrowBatchHelper implements AutoCloseable { FileStoreTable fileStoreTable, TableWriteImpl tableWrite, RowType tableRowType, - int bucket) { + int bucket, + LakeLayout lakeLayout) { this.fileStoreTable = fileStoreTable; this.tableWrite = tableWrite; this.tableRowType = tableRowType; this.bucket = bucket; + this.lakeLayout = lakeLayout; } /** @@ -107,6 +111,16 @@ void writeArrowBatch(ArrowBatchData arrowBatchData, BinaryRow partition) throws } VectorSchemaRoot originalRoot = arrowBatchData.getVectorSchemaRoot(); + + if (lakeLayout == LakeLayout.CLEAN) { + // Clean tables contain only user columns, so the incoming Arrow batch already matches + // the Paimon table schema. Write it directly without enriching system columns. + ArrowBundleRecords cleanRecords = + new ArrowBundleRecords(originalRoot, tableRowType, false); + tableWrite.writeBundle(partition, writtenBucket, cleanRecords); + return; + } + long baseOffset = arrowBatchData.getBaseLogOffset(); long timestamp = arrowBatchData.getTimestamp(); int rowCount = originalRoot.getRowCount(); diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java index 0caaed97c4b..601afc531ea 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java @@ -18,6 +18,7 @@ package org.apache.fluss.lake.paimon.tiering.append; import org.apache.fluss.lake.paimon.tiering.RecordWriter; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.record.ArrowBatchData; import org.apache.fluss.record.LogRecord; @@ -46,12 +47,15 @@ public class AppendOnlyWriter extends RecordWriter { */ @Nullable private AutoCloseable arrowBatchHelper; + private final LakeLayout lakeLayout; + public AppendOnlyWriter( FileStoreTable fileStoreTable, TableBucket tableBucket, @Nullable String partition, List partitionKeys, - RowType flussRowType) { + RowType flussRowType, + LakeLayout lakeLayout) { //noinspection unchecked super( (TableWriteImpl) @@ -61,8 +65,10 @@ public AppendOnlyWriter( tableBucket, partition, partitionKeys, - flussRowType); + flussRowType, + lakeLayout); this.fileStoreTable = fileStoreTable; + this.lakeLayout = lakeLayout; } @Override @@ -90,7 +96,7 @@ public void writeArrowBatch(ArrowBatchData arrowBatchData) throws Exception { if (arrowBatchHelper == null) { helper = new AppendOnlyArrowBatchHelper( - fileStoreTable, tableWrite, tableRowType, bucket); + fileStoreTable, tableWrite, tableRowType, bucket, lakeLayout); arrowBatchHelper = helper; } else { helper = (AppendOnlyArrowBatchHelper) arrowBatchHelper; diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java index b3e1ecdeed8..9b0b2c0f744 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java @@ -18,6 +18,7 @@ package org.apache.fluss.lake.paimon.tiering.mergetree; import org.apache.fluss.lake.paimon.tiering.RecordWriter; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.record.LogRecord; import org.apache.fluss.types.RowType; @@ -49,8 +50,16 @@ public MergeTreeWriter( TableBucket tableBucket, @Nullable String partition, List partitionKeys, - RowType flussRowType) { - this(fileStoreTable, tableBucket, partition, partitionKeys, flussRowType, (String[]) null); + RowType flussRowType, + LakeLayout lakeLayout) { + this( + fileStoreTable, + tableBucket, + partition, + partitionKeys, + flussRowType, + (String[]) null, + lakeLayout); } public MergeTreeWriter( @@ -59,14 +68,16 @@ public MergeTreeWriter( @Nullable String partition, List partitionKeys, RowType flussRowType, - @Nullable String[] ioTmpDirs) { + @Nullable String[] ioTmpDirs, + LakeLayout lakeLayout) { this( fileStoreTable, createIOManager(ioTmpDirs), tableBucket, partition, partitionKeys, - flussRowType); + flussRowType, + lakeLayout); } MergeTreeWriter( @@ -75,14 +86,16 @@ public MergeTreeWriter( TableBucket tableBucket, @Nullable String partition, List partitionKeys, - RowType flussRowType) { + RowType flussRowType, + LakeLayout lakeLayout) { super( createTableWrite(fileStoreTable, ioManager), fileStoreTable.rowType(), tableBucket, partition, partitionKeys, - flussRowType); + flussRowType, + lakeLayout); this.rowKeyExtractor = fileStoreTable.createRowKeyExtractor(); this.ioManager = ioManager; } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonConversions.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonConversions.java index becf8a2f056..c90f0e91438 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonConversions.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonConversions.java @@ -51,6 +51,7 @@ import java.util.function.Function; import static org.apache.fluss.lake.paimon.PaimonLakeCatalog.SYSTEM_COLUMNS; +import static org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import static org.apache.fluss.utils.Preconditions.checkState; /** Utils for conversion between Paimon and Fluss. */ @@ -172,7 +173,8 @@ public static BinaryRow toPaimonPartition( return partitionExtractor.apply(new FlussRowAsPaimonRow(partitionRow, paimonRowType)); } - public static List toPaimonSchemaChanges(List tableChanges) { + public static List toPaimonSchemaChanges( + List tableChanges, LakeLayout lakeLayout) { List schemaChanges = new ArrayList<>(tableChanges.size()); for (TableChange tableChange : tableChanges) { @@ -203,14 +205,27 @@ public static List toPaimonSchemaChanges(List tableCh org.apache.paimon.types.DataType paimonDataType = flussDataType.accept(FlussDataTypeToPaimonDataType.INSTANCE); - String firstSystemColumnName = SYSTEM_COLUMNS.keySet().iterator().next(); - schemaChanges.add( - SchemaChange.addColumn( - addColumn.getName(), - paimonDataType, - addColumn.getComment(), - SchemaChange.Move.before( - addColumn.getName(), firstSystemColumnName))); + if (lakeLayout == LakeLayout.LEGACY) { + // Legacy tables keep the three system columns as the last physical columns, so + // a new business column must be inserted right before the first system column. + String firstSystemColumnName = SYSTEM_COLUMNS.keySet().iterator().next(); + schemaChanges.add( + SchemaChange.addColumn( + addColumn.getName(), + paimonDataType, + addColumn.getComment(), + SchemaChange.Move.before( + addColumn.getName(), firstSystemColumnName))); + } else { + // Clean tables have no trailing system columns, so a new business column is + // simply appended at the end. + schemaChanges.add( + SchemaChange.addColumn( + addColumn.getName(), + paimonDataType, + addColumn.getComment(), + null)); + } } else { throw new UnsupportedOperationException( "Unsupported table change: " + tableChange.getClass()); @@ -264,10 +279,10 @@ public static Schema toPaimonSchema(TableDescriptor tableDescriptor) { column.getComment().orElse(null)); } - // add system metadata columns to schema - for (Map.Entry systemColumn : SYSTEM_COLUMNS.entrySet()) { - schemaBuilder.column(systemColumn.getKey(), systemColumn.getValue()); - } + // FIP-27: newly created lake tables use a clean physical schema containing only + // user-defined columns. The three Fluss system columns (__bucket, __offset, __timestamp) + // are no longer added. Existing legacy tables that still carry these columns remain + // readable and writable, see PaimonSystemColumns#detectLayout. // set pk if (tableDescriptor.hasPrimaryKey()) { diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonRowAsFlussRow.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonRowAsFlussRow.java index fe956561a69..9e8be46c65b 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonRowAsFlussRow.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonRowAsFlussRow.java @@ -17,6 +17,7 @@ package org.apache.fluss.lake.paimon.utils; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.row.BinaryString; import org.apache.fluss.row.Decimal; import org.apache.fluss.row.InternalArray; @@ -27,17 +28,28 @@ import org.apache.paimon.data.Timestamp; -import static org.apache.fluss.lake.paimon.PaimonLakeCatalog.SYSTEM_COLUMNS; - /** Adapter for paimon row as fluss row. */ public class PaimonRowAsFlussRow implements InternalRow { private org.apache.paimon.data.InternalRow paimonRow; - public PaimonRowAsFlussRow() {} + // Number of trailing Fluss system columns carried by the wrapped Paimon row that must be + // excluded from the exposed field count. This is only non-zero for a legacy table's top-level + // physical row; clean tables and nested/projected rows carry no system columns. + private final int trailingSystemColumns; + + public PaimonRowAsFlussRow() { + this.trailingSystemColumns = 0; + } + + public PaimonRowAsFlussRow(LakeLayout lakeLayout) { + this.trailingSystemColumns = + lakeLayout == LakeLayout.LEGACY ? PaimonSystemColumns.systemColumnCount() : 0; + } public PaimonRowAsFlussRow(org.apache.paimon.data.InternalRow paimonRow) { this.paimonRow = paimonRow; + this.trailingSystemColumns = 0; } public PaimonRowAsFlussRow replaceRow(org.apache.paimon.data.InternalRow paimonRow) { @@ -47,7 +59,7 @@ public PaimonRowAsFlussRow replaceRow(org.apache.paimon.data.InternalRow paimonR @Override public int getFieldCount() { - return paimonRow.getFieldCount() - SYSTEM_COLUMNS.size(); + return paimonRow.getFieldCount() - trailingSystemColumns; } @Override diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSystemColumns.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSystemColumns.java new file mode 100644 index 00000000000..ef9d38ff268 --- /dev/null +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSystemColumns.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.lake.paimon.utils; + +import org.apache.fluss.exception.InvalidTableException; + +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.fluss.metadata.TableDescriptor.BUCKET_COLUMN_NAME; +import static org.apache.fluss.metadata.TableDescriptor.OFFSET_COLUMN_NAME; +import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; + +/** + * Utilities describing the two physical layouts a Paimon lake table can have under FIP-27, and the + * single place that detects which layout a given Paimon table uses. + * + *
    + *
  • {@link LakeLayout#CLEAN} - the table only contains user-defined columns. This is the + * layout of every newly created lake table. + *
  • {@link LakeLayout#LEGACY} - the table was created before FIP-27 and carries the three + * mandatory Fluss system columns {@code __bucket}, {@code __offset}, {@code __timestamp} as + * its last three physical columns. + *
+ * + *

Detection is based purely on the physical Paimon schema, so no extra metadata or table + * property is needed and existing tables are never migrated. A table that carries only some of the + * system columns, or carries them with an unexpected type, is neither a clean nor a valid legacy + * table and is rejected with a clear error. + */ +public class PaimonSystemColumns { + + /** + * The three mandatory Fluss system columns and their expected Paimon types, in physical order. + * The {@code __timestamp} type is compared with relaxed precision (see {@link + * #isSystemTimestampType}) to stay compatible with legacy tables written by older clusters. + */ + public static final LinkedHashMap SYSTEM_COLUMNS = new LinkedHashMap<>(); + + static { + // We need __bucket system column to filter out the given bucket + // for paimon bucket-unaware append only table. + // It's not required for paimon bucket-aware table like primary key table + // and bucket-aware append only table, but legacy tables always carry the system column + // for consistent behavior. + SYSTEM_COLUMNS.put(BUCKET_COLUMN_NAME, DataTypes.INT()); + SYSTEM_COLUMNS.put(OFFSET_COLUMN_NAME, DataTypes.BIGINT()); + SYSTEM_COLUMNS.put(TIMESTAMP_COLUMN_NAME, DataTypes.TIMESTAMP_LTZ_MILLIS()); + } + + /** The physical layout of a Paimon lake table with respect to Fluss system columns. */ + public enum LakeLayout { + /** Only user-defined columns are present (FIP-27 default for new tables). */ + CLEAN, + /** The three Fluss system columns are appended as the last physical columns. */ + LEGACY + } + + private PaimonSystemColumns() {} + + /** Returns the number of system columns carried by a {@link LakeLayout#LEGACY} table. */ + public static int systemColumnCount() { + return SYSTEM_COLUMNS.size(); + } + + public static boolean isSystemColumn(String columnName) { + return SYSTEM_COLUMNS.containsKey(columnName); + } + + /** + * Detects whether a Paimon table with the given physical row type uses the clean or the legacy + * layout. + * + *

The detection tolerates the {@code __timestamp} precision difference between old (precision + * 6) and new (precision 3) clusters, mirroring {@link + * PaimonTableValidation#equalIgnoreSystemColumnTimestampPrecision}. + * + * @throws InvalidTableException if the table carries only some of the system columns, carries + * them out of order, with an incompatible type, or embeds a system column name among the + * business columns. Such a table is neither clean nor a valid legacy table. + */ + public static LakeLayout detectLayout(RowType paimonRowType) { + List fields = paimonRowType.getFields(); + + int firstSystemColumnPos = -1; + for (int i = 0; i < fields.size(); i++) { + if (SYSTEM_COLUMNS.containsKey(fields.get(i).name())) { + firstSystemColumnPos = i; + break; + } + } + + // No system column anywhere -> clean layout. + if (firstSystemColumnPos < 0) { + return LakeLayout.CLEAN; + } + + // A system column exists. For a valid legacy table, all three must appear, in the canonical + // order, as the very last physical columns, each with a compatible type. + int businessFieldCount = fields.size() - SYSTEM_COLUMNS.size(); + if (firstSystemColumnPos != businessFieldCount) { + throw partialLayoutException(paimonRowType); + } + + int pos = businessFieldCount; + for (Map.Entry systemColumn : SYSTEM_COLUMNS.entrySet()) { + DataField field = fields.get(pos); + if (!field.name().equals(systemColumn.getKey()) + || !isSystemColumnTypeCompatible(field.name(), field.type())) { + throw partialLayoutException(paimonRowType); + } + pos++; + } + + return LakeLayout.LEGACY; + } + + private static boolean isSystemColumnTypeCompatible(String name, DataType actualType) { + if (TIMESTAMP_COLUMN_NAME.equals(name)) { + // Old clusters wrote precision 6, new clusters write precision 3; both are accepted. + return isSystemTimestampType(actualType); + } + // Compare the type family and precision, ignoring nullability: legacy system columns were + // written as non-null, but we only care that the physical type matches. + DataType expected = SYSTEM_COLUMNS.get(name); + return actualType.copy(true).equalsIgnoreFieldId(expected.copy(true)); + } + + private static boolean isSystemTimestampType(DataType actualType) { + return actualType.equalsIgnoreFieldId(DataTypes.TIMESTAMP_LTZ_MILLIS()) + || actualType.equalsIgnoreFieldId(DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE()); + } + + private static InvalidTableException partialLayoutException(RowType paimonRowType) { + return new InvalidTableException( + String.format( + "The Paimon table has an incompatible system-column layout. A table must " + + "either contain none of the Fluss system columns (clean layout) or " + + "contain all of %s as its last columns, in this order, with " + + "compatible types (legacy layout). Actual schema: %s.", + SYSTEM_COLUMNS.keySet(), paimonRowType)); + } +} diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java index 0b8f56052f9..2a6466fc032 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java @@ -34,12 +34,23 @@ import static org.apache.fluss.lake.paimon.utils.PaimonConversions.PAIMON_UNSETTABLE_OPTIONS; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.PARTITION_GENERATE_LEGACY_NAME_OPTION_KEY; +import static org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; /** Utils to verify whether the existing Paimon table is compatible with the table to be created. */ public class PaimonTableValidation { public static boolean isPaimonSchemaCompatible(Schema existingSchema, Schema newSchema) { + // FIP-27: newly generated schemas are always clean (no system columns). When the existing + // table is a legacy table that still carries the three system columns, re-enabling lake + // tiering must keep that physical layout. Enrich the clean new schema with the trailing + // system columns before comparison, so an existing legacy table is recognised as + // compatible and its layout is preserved. Detection also rejects a partial/type-mismatched + // legacy layout with a clear error. + if (PaimonSystemColumns.detectLayout(existingSchema.rowType()) == LakeLayout.LEGACY) { + newSchema = appendSystemColumns(newSchema); + } + if (!equalPhysicalSchema(existingSchema, newSchema)) { // Allow different precisions for __timestamp column for backward compatibility, // old cluster will use precision 6, but new cluster will use precision 3, @@ -50,6 +61,33 @@ public static boolean isPaimonSchemaCompatible(Schema existingSchema, Schema new return true; } + /** + * Returns a copy of {@code cleanSchema} with the three Fluss system columns appended as the + * last physical columns, so a clean schema generated by the current cluster can be compared + * against an existing legacy table. Partition keys, primary keys, options and comment are + * preserved. System-column field ids continue the existing id sequence to avoid collisions; + * they are irrelevant to {@link #equalPhysicalSchema}, which ignores field ids. + */ + private static Schema appendSystemColumns(Schema cleanSchema) { + List fields = new ArrayList<>(cleanSchema.fields()); + int nextFieldId = 0; + for (DataField field : fields) { + nextFieldId = Math.max(nextFieldId, field.id() + 1); + } + for (Map.Entry systemColumn : + PaimonSystemColumns.SYSTEM_COLUMNS.entrySet()) { + fields.add( + new DataField( + nextFieldId++, systemColumn.getKey(), systemColumn.getValue())); + } + return new Schema( + fields, + cleanSchema.partitionKeys(), + cleanSchema.primaryKeys(), + cleanSchema.options(), + cleanSchema.comment()); + } + /** * Check if the {@code existingSchema} is compatible with {@code newSchema} by ignoring the * precision difference of the system column {@code __timestamp}. @@ -67,6 +105,11 @@ public static boolean isPaimonSchemaCompatible(Schema existingSchema, Schema new public static boolean equalIgnoreSystemColumnTimestampPrecision( Schema existingSchema, Schema newSchema) { List existingFields = new ArrayList<>(existingSchema.fields()); + // Only legacy tables carry a trailing __timestamp system column. Clean tables have no + // system columns, so there is no precision to relax and we compare them directly. + if (existingFields.isEmpty()) { + return equalPhysicalSchema(existingSchema, newSchema); + } DataField systemTimestampField = existingFields.get(existingFields.size() - 1); if (systemTimestampField.name().equals(TIMESTAMP_COLUMN_NAME) && systemTimestampField diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRowTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRowTest.java index e085b694ba0..b57a4315f6e 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRowTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRowTest.java @@ -17,6 +17,7 @@ package org.apache.fluss.lake.paimon.tiering; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.record.GenericRecord; import org.apache.fluss.record.LogRecord; import org.apache.fluss.row.BinaryString; @@ -75,7 +76,7 @@ void testLogTableRecordAllTypes() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(14); @@ -143,7 +144,7 @@ void testPrimaryKeyTableRecord() { new org.apache.paimon.types.BigIntType(), new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -185,7 +186,7 @@ void testArrayTypeWithIntElements() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 10; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(2); @@ -224,7 +225,7 @@ void testArrayTypeWithStringElements() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 5; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -259,7 +260,7 @@ void testNestedArrayType() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -313,7 +314,7 @@ void testArrayWithAllPrimitiveTypes() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(7); @@ -386,7 +387,7 @@ void testArrayWithDecimalElements() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -420,7 +421,7 @@ void testArrayWithTimestampElements() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -454,7 +455,7 @@ void testArrayWithBinaryElements() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -483,7 +484,7 @@ void testNullArray() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -508,7 +509,7 @@ void testArrayWithNullableElements() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -537,7 +538,7 @@ void testEmptyArray() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -564,7 +565,7 @@ void testPaimonSchemaWiderThanFlussRecord() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 7L; long timeStamp = System.currentTimeMillis(); @@ -595,7 +596,7 @@ void testFlussRecordWiderThanPaimonSchema() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 7L; long timeStamp = System.currentTimeMillis(); @@ -690,7 +691,7 @@ void testNestedRowType() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(8); @@ -918,7 +919,7 @@ void testMapWithAllTypes() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, nestedMapRowType); + new FlussRecordAsPaimonRow(tableBucket, nestedMapRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow nestedMapGenericRow = new GenericRow(1); @@ -971,7 +972,7 @@ private void testMapType( new org.apache.paimon.types.BigIntType(), new org.apache.paimon.types.LocalZonedTimestampType(3)); - FlussRecordAsPaimonRow flussRow = new FlussRecordAsPaimonRow(tableBucket, rowType); + FlussRecordAsPaimonRow flussRow = new FlussRecordAsPaimonRow(tableBucket, rowType, LakeLayout.LEGACY); GenericRow genericRow = new GenericRow(1); genericRow.setField(0, new GenericMap(mapData)); LogRecord logRecord = new GenericRecord(logOffset, timeStamp, APPEND_ONLY, genericRow); @@ -997,7 +998,7 @@ void testNullMap() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -1023,7 +1024,7 @@ void testMapWithNullableValues() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -1058,7 +1059,7 @@ void testEmptyMap() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -1082,7 +1083,7 @@ void testAccessRowBeforeSetThrowsIllegalState() { new org.apache.paimon.types.IntType(), new org.apache.paimon.types.BigIntType(), new org.apache.paimon.types.LocalZonedTimestampType(3)); - FlussRecordAsPaimonRow row = new FlussRecordAsPaimonRow(0, rowType); + FlussRecordAsPaimonRow row = new FlussRecordAsPaimonRow(0, rowType, LakeLayout.LEGACY); assertThatThrownBy(row::getRowKind) .isInstanceOf(IllegalStateException.class) .hasMessageContaining(expectedMsg); From dd1f6b0c09a690f50efb38f53521bbbe867cd1c7 Mon Sep 17 00:00:00 2001 From: fhan Date: Fri, 14 Aug 2026 10:40:03 +0800 Subject: [PATCH 02/11] [lake/paimon] fix format violations --- .../fluss/lake/paimon/utils/PaimonSystemColumns.java | 8 ++++---- .../fluss/lake/paimon/utils/PaimonTableValidation.java | 3 +-- .../lake/paimon/tiering/FlussRecordAsPaimonRowTest.java | 3 ++- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSystemColumns.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSystemColumns.java index ef9d38ff268..52b1c66dd80 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSystemColumns.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSystemColumns.java @@ -37,8 +37,8 @@ * single place that detects which layout a given Paimon table uses. * *

    - *
  • {@link LakeLayout#CLEAN} - the table only contains user-defined columns. This is the - * layout of every newly created lake table. + *
  • {@link LakeLayout#CLEAN} - the table only contains user-defined columns. This is the layout + * of every newly created lake table. *
  • {@link LakeLayout#LEGACY} - the table was created before FIP-27 and carries the three * mandatory Fluss system columns {@code __bucket}, {@code __offset}, {@code __timestamp} as * its last three physical columns. @@ -92,8 +92,8 @@ public static boolean isSystemColumn(String columnName) { * Detects whether a Paimon table with the given physical row type uses the clean or the legacy * layout. * - *

    The detection tolerates the {@code __timestamp} precision difference between old (precision - * 6) and new (precision 3) clusters, mirroring {@link + *

    The detection tolerates the {@code __timestamp} precision difference between old + * (precision 6) and new (precision 3) clusters, mirroring {@link * PaimonTableValidation#equalIgnoreSystemColumnTimestampPrecision}. * * @throws InvalidTableException if the table carries only some of the system columns, carries diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java index 2a6466fc032..b10e485dc85 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java @@ -77,8 +77,7 @@ private static Schema appendSystemColumns(Schema cleanSchema) { for (Map.Entry systemColumn : PaimonSystemColumns.SYSTEM_COLUMNS.entrySet()) { fields.add( - new DataField( - nextFieldId++, systemColumn.getKey(), systemColumn.getValue())); + new DataField(nextFieldId++, systemColumn.getKey(), systemColumn.getValue())); } return new Schema( fields, diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRowTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRowTest.java index b57a4315f6e..dacb0479c80 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRowTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRowTest.java @@ -972,7 +972,8 @@ private void testMapType( new org.apache.paimon.types.BigIntType(), new org.apache.paimon.types.LocalZonedTimestampType(3)); - FlussRecordAsPaimonRow flussRow = new FlussRecordAsPaimonRow(tableBucket, rowType, LakeLayout.LEGACY); + FlussRecordAsPaimonRow flussRow = + new FlussRecordAsPaimonRow(tableBucket, rowType, LakeLayout.LEGACY); GenericRow genericRow = new GenericRow(1); genericRow.setField(0, new GenericMap(mapData)); LogRecord logRecord = new GenericRecord(logOffset, timeStamp, APPEND_ONLY, genericRow); From 3a08bd15a6e83d8b5e4f8d2c55284f4cb8b88972 Mon Sep 17 00:00:00 2001 From: fhan Date: Fri, 14 Aug 2026 12:55:05 +0800 Subject: [PATCH 03/11] [lake/paimon] fix test failures --- .../paimon/utils/PaimonSystemColumns.java | 14 +++++++++++-- .../lake/paimon/PaimonLakeCatalogTest.java | 20 ++----------------- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSystemColumns.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSystemColumns.java index 52b1c66dd80..35e4a265f52 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSystemColumns.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSystemColumns.java @@ -148,8 +148,18 @@ private static boolean isSystemColumnTypeCompatible(String name, DataType actual } private static boolean isSystemTimestampType(DataType actualType) { - return actualType.equalsIgnoreFieldId(DataTypes.TIMESTAMP_LTZ_MILLIS()) - || actualType.equalsIgnoreFieldId(DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE()); + // Legacy tables carry __timestamp with varying timestamp types depending on the cluster + // that created them: with or without local time zone, and precision 3 (new clusters) or 6 + // (old clusters). Accept the whole timestamp family and let the reader handle the + // precision, mirroring the relaxed check in + // PaimonTableValidation#equalIgnoreSystemColumnTimestampPrecision. + switch (actualType.getTypeRoot()) { + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return true; + default: + return false; + } } private static InvalidTableException partialLayoutException(RowType paimonRowType) { diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/PaimonLakeCatalogTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/PaimonLakeCatalogTest.java index 1683533e506..1e67a92a281 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/PaimonLakeCatalogTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/PaimonLakeCatalogTest.java @@ -160,15 +160,7 @@ void testAlterTableAddColumnLastNullable() throws Exception { Table table = flussPaimonCatalog.getPaimonCatalog().getTable(identifier); assertThat(table.rowType().getFieldNames()) - .containsSequence( - "id", - "name", - "amount", - "address", - "new_col", - "__bucket", - "__offset", - "__timestamp"); + .containsSequence("id", "name", "amount", "address", "new_col"); } @Test @@ -205,15 +197,7 @@ void testAlterTableAddColumnIgnoresPaimonCommentsAndOptions() throws Exception { assertThat(((FileStoreTable) table).schema().toSchema().comment()).isEqualTo(""); assertThat(table.options().get("fluss.key")).isEqualTo("value"); assertThat(table.rowType().getFieldNames()) - .containsSequence( - "id", - "name", - "amount", - "address", - "is_direct_play", - "__bucket", - "__offset", - "__timestamp"); + .containsSequence("id", "name", "amount", "address", "is_direct_play"); } @Test From c238900aaa6429efa7885491dcd209f816d9a659 Mon Sep 17 00:00:00 2001 From: fhan Date: Fri, 14 Aug 2026 16:20:27 +0800 Subject: [PATCH 04/11] [lake/paimon] fix IT test failures --- .../paimon/LakeEnabledTableCreateITCase.java | 196 ++++-------------- .../FlinkUnionReadPrimaryKeyTableITCase.java | 8 +- .../paimon/tiering/PaimonTieringITCase.java | 16 +- 3 files changed, 55 insertions(+), 165 deletions(-) diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/LakeEnabledTableCreateITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/LakeEnabledTableCreateITCase.java index f1d75d27810..547e31cce28 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/LakeEnabledTableCreateITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/LakeEnabledTableCreateITCase.java @@ -44,7 +44,6 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; -import org.apache.paimon.data.Timestamp; import org.apache.paimon.options.Options; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.table.FileStoreTable; @@ -172,19 +171,9 @@ void testCreateLakeEnabledTable() throws Exception { RowType.of( new DataType[] { org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.STRING(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING() }, - new String[] { - "log_c1", - "log_c2", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"log_c1", "log_c2"}), "log_c1,log_c2", BUCKET_NUM); @@ -210,19 +199,9 @@ void testCreateLakeEnabledTable() throws Exception { RowType.of( new DataType[] { org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.STRING(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING() }, - new String[] { - "log_c1", - "log_c2", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"log_c1", "log_c2"}), null, BUCKET_NUM); @@ -249,19 +228,9 @@ void testCreateLakeEnabledTable() throws Exception { RowType.of( new DataType[] { org.apache.paimon.types.DataTypes.INT().notNull(), - org.apache.paimon.types.DataTypes.STRING(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING() }, - new String[] { - "pk_c1", - "pk_c2", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"pk_c1", "pk_c2"}), "pk_c1", BUCKET_NUM); @@ -292,20 +261,9 @@ void testCreateLakeEnabledTable() throws Exception { new DataType[] { org.apache.paimon.types.DataTypes.INT().notNull(), org.apache.paimon.types.DataTypes.STRING(), - org.apache.paimon.types.DataTypes.STRING().notNull(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING().notNull() }, - new String[] { - "c1", - "c2", - "c3", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"c1", "c2", "c3"}), "c1", BUCKET_NUM); } @@ -359,11 +317,7 @@ void testCreateLakeEnabledTableWithAllTypes() throws Exception { org.apache.paimon.types.DataTypes.DATE(), org.apache.paimon.types.DataTypes.TIME(), org.apache.paimon.types.DataTypes.TIMESTAMP(), - org.apache.paimon.types.DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE() }, new String[] { "log_c1", @@ -381,10 +335,7 @@ void testCreateLakeEnabledTableWithAllTypes() throws Exception { "log_c13", "log_c14", "log_c15", - "log_c16", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME + "log_c16" }), null, BUCKET_NUM); @@ -599,10 +550,10 @@ void testCreateLakeEnableTableWithExistLakeTable() throws Exception { .hasMessageContaining( "The table `fluss`.`log_table_with_exist_lake_table` already exists in Paimon catalog, but the table schema is not compatible.") .hasMessageContaining( - "Existing schema: UpdateSchema{fields=[`c1` STRING, `c2` INT, `__bucket` INT, `__offset` BIGINT, `__timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE]") + "Existing schema: UpdateSchema{fields=[`c1` STRING, `c2` INT]") .hasMessageContaining("options={bucket=-1") .hasMessageContaining( - "new schema: UpdateSchema{fields=[`c1` STRING, `c2` INT, `__bucket` INT, `__offset` BIGINT, `__timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE]") + "new schema: UpdateSchema{fields=[`c1` STRING, `c2` INT]") .hasMessageContaining("options={bucket=3") .hasMessageContaining("bucket-key=c1,c2") .hasMessageEndingWith( @@ -624,9 +575,9 @@ void testCreateLakeEnableTableWithExistLakeTable() throws Exception { .hasMessageContaining( "The table `fluss`.`log_table_with_exist_lake_table` already exists in Paimon catalog, but the table schema is not compatible.") .hasMessageContaining( - "Existing schema: UpdateSchema{fields=[`c1` STRING, `c2` INT, `__bucket` INT, `__offset` BIGINT, `__timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE]") + "Existing schema: UpdateSchema{fields=[`c1` STRING, `c2` INT]") .hasMessageContaining( - "new schema: UpdateSchema{fields=[`c1` STRING, `c2` INT, `c3` STRING, `__bucket` INT, `__offset` BIGINT, `__timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE]") + "new schema: UpdateSchema{fields=[`c1` STRING, `c2` INT, `c3` STRING]") .hasMessageEndingWith( "Please first drop the table in Paimon catalog or use a new table name."); @@ -781,19 +732,9 @@ void testAlterLakeEnabledLogTable() throws Exception { RowType.of( new DataType[] { org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.STRING(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING() }, - new String[] { - "log_c1", - "log_c2", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"log_c1", "log_c2"}), "log_c1,log_c2", BUCKET_NUM); @@ -888,19 +829,9 @@ void testAlterLakeEnabledTableProperties() throws Exception { RowType.of( new DataType[] { org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.STRING(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING() }, - new String[] { - "c1", - "c2", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"c1", "c2"}), "c1,c2", BUCKET_NUM); @@ -919,19 +850,9 @@ void testAlterLakeEnabledTableProperties() throws Exception { RowType.of( new DataType[] { org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.STRING(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING() }, - new String[] { - "c1", - "c2", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"c1", "c2"}), "c1,c2", BUCKET_NUM); @@ -966,19 +887,9 @@ void testAlterLakeEnabledTableProperties() throws Exception { RowType.of( new DataType[] { org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.STRING(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING() }, - new String[] { - "c1", - "c2", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"c1", "c2"}), "c1,c2", BUCKET_NUM); @@ -1047,19 +958,9 @@ void testEnableLakeTableAfterAlterTableProperties() throws Exception { RowType.of( new DataType[] { org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.STRING(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING() }, - new String[] { - "c1", - "c2", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"c1", "c2"}), "c1,c2", BUCKET_NUM); @@ -1100,19 +1001,9 @@ void testAlterLakeEnabledTableSchema() throws Exception { RowType.of( new DataType[] { org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.STRING(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING() }, - new String[] { - "c1", - "c2", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"c1", "c2"}), "c1,c2", BUCKET_NUM); @@ -1131,15 +1022,8 @@ void testAlterLakeEnabledTableSchema() throws Exception { paimonCatalog.getTable(Identifier.create(DATABASE, tablePath.getTableName())); // Verify the new column c3 with comment was added to Paimon table RowType alteredRowType = alteredPaimonTable.rowType(); - assertThat(alteredRowType.getFieldCount()).isEqualTo(6); - assertThat(alteredRowType.getFieldNames()) - .containsExactly( - "c1", - "c2", - "c3", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME); + assertThat(alteredRowType.getFieldCount()).isEqualTo(3); + assertThat(alteredRowType.getFieldNames()).containsExactly("c1", "c2", "c3"); // Verify c3 column has the correct type and comment assertThat(alteredRowType.getField("c3").type()) .isEqualTo(org.apache.paimon.types.DataTypes.INT()); @@ -1159,12 +1043,20 @@ void testEnableLakeTableWithLegacySystemTimestampColumn() throws Exception { Identifier paimonIdentifier = Identifier.create(DATABASE, tablePath.getTableName()); - // alter to TIMESTAMP_WITH_LOCAL_TIME_ZONE to mock the legacy behavior + // FIP-27: a newly created table is clean (no system columns). To exercise the legacy + // compatibility path, first turn it into a legacy table by appending the three trailing + // system columns, using TIMESTAMP_WITH_LOCAL_TIME_ZONE for __timestamp to mock the + // precision-6 layout written by an old cluster. paimonCatalog.alterTable( paimonIdentifier, - SchemaChange.updateColumnType( - TIMESTAMP_COLUMN_NAME, - org.apache.paimon.types.DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE()), + Arrays.asList( + SchemaChange.addColumn( + BUCKET_COLUMN_NAME, org.apache.paimon.types.DataTypes.INT()), + SchemaChange.addColumn( + OFFSET_COLUMN_NAME, org.apache.paimon.types.DataTypes.BIGINT()), + SchemaChange.addColumn( + TIMESTAMP_COLUMN_NAME, + org.apache.paimon.types.DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE())), false); // disable data lake @@ -1310,13 +1202,7 @@ private void writeData(Table table) throws Exception { BatchTableCommit commit = writeBuilder.newCommit()) { for (int i = 0; i < 10; i++) { - GenericRow row = - GenericRow.of( - i, - BinaryString.fromString("row-" + i), - 0, - (long) i, - Timestamp.fromEpochMillis(System.currentTimeMillis())); + GenericRow row = GenericRow.of(i, BinaryString.fromString("row-" + i)); write.write(row); } diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/flink/FlinkUnionReadPrimaryKeyTableITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/flink/FlinkUnionReadPrimaryKeyTableITCase.java index 81fa9b05018..f91e56c27ea 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/flink/FlinkUnionReadPrimaryKeyTableITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/flink/FlinkUnionReadPrimaryKeyTableITCase.java @@ -123,7 +123,9 @@ void testUnionReadFullType(Boolean isPartitioned) throws Exception { CollectionUtil.iteratorToList(tableResult.collect()).stream() .map( row -> { - int userColumnCount = row.getArity() - 3; + // FIP-27: a clean lake table exposes only user columns via + // $lake, so there are no trailing system columns to strip. + int userColumnCount = row.getArity(); Object[] fields = new Object[userColumnCount]; for (int i = 0; i < userColumnCount; i++) { fields[i] = row.getField(i); @@ -277,7 +279,9 @@ void testUnionReadFullType(Boolean isPartitioned) throws Exception { .stream() .map( row -> { - int columnCount = row.getArity() - 3; + // FIP-27: a clean lake table exposes only user columns via + // $lake, so there are no trailing system columns to strip. + int columnCount = row.getArity(); Object[] fields = new Object[columnCount]; for (int i = 0; i < columnCount; i++) { fields[i] = row.getField(i); diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java index 3d4da4fe50c..7101e0069df 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java @@ -504,9 +504,9 @@ private void checkDataInPaimonAppendOnlyTable( InternalRow flussRow = flussRowIterator.next(); assertThat(row.getInt(0)).isEqualTo(flussRow.getInt(0)); assertThat(row.getString(1).toString()).isEqualTo(flussRow.getString(1).toString()); - // system columns are always the last three: __bucket, __offset, __timestamp - int offsetIndex = row.getFieldCount() - 2; - assertThat(row.getLong(offsetIndex)).isEqualTo(startingOffset++); + // FIP-27: a clean lake table only stores user columns, so there are no trailing + // __bucket/__offset/__timestamp columns to verify here. + assertThat(row.getFieldCount()).isEqualTo(2); } assertThat(flussRowIterator.hasNext()).isFalse(); } @@ -526,8 +526,9 @@ private void checkDataInPaimonAppendOnlyPartitionedTable( assertThat(row.getInt(0)).isEqualTo(flussRow.getInt(0)); assertThat(row.getString(1).toString()).isEqualTo(flussRow.getString(1).toString()); assertThat(row.getString(2).toString()).isEqualTo(flussRow.getString(2).toString()); - // the idx 3 is __bucket, so use 4 - assertThat(row.getLong(4)).isEqualTo(startingOffset++); + // FIP-27: a clean lake table only stores user columns, so there are no trailing + // __bucket/__offset/__timestamp columns to verify here. + assertThat(row.getFieldCount()).isEqualTo(3); } assertThat(flussRowIterator.hasNext()).isFalse(); } @@ -594,9 +595,8 @@ void testTieringWithAddColumn() throws Exception { FileStoreTable paimonTable = (FileStoreTable) paimonCatalog.getTable(tableIdentifier); List fieldNames = paimonTable.rowType().getFieldNames(); - // Should have exact fields in order: a, b, c3, __bucket, __offset, __timestamp - assertThat(fieldNames) - .containsExactly("a", "b", "c3", "__bucket", "__offset", "__timestamp"); + // FIP-27: a clean lake table only stores user columns, in order: a, b, c3. + assertThat(fieldNames).containsExactly("a", "b", "c3"); // 9. Verify both schema evolution and data correctness // For initial rows (before ADD COLUMN), c3 should be NULL From 0230acf3d93c4b90352bf3d116816f7b01f50c33 Mon Sep 17 00:00:00 2001 From: fhan Date: Fri, 14 Aug 2026 18:31:00 +0800 Subject: [PATCH 05/11] [lake/paimon] fix format violations --- .../paimon/LakeEnabledTableCreateITCase.java | 25 +++++-------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/LakeEnabledTableCreateITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/LakeEnabledTableCreateITCase.java index 547e31cce28..fafc95c5871 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/LakeEnabledTableCreateITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/LakeEnabledTableCreateITCase.java @@ -320,22 +320,9 @@ void testCreateLakeEnabledTableWithAllTypes() throws Exception { org.apache.paimon.types.DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE() }, new String[] { - "log_c1", - "log_c2", - "log_c3", - "log_c4", - "log_c5", - "log_c6", - "log_c7", - "log_c8", - "log_c9", - "log_c10", - "log_c11", - "log_c12", - "log_c13", - "log_c14", - "log_c15", - "log_c16" + "log_c1", "log_c2", "log_c3", "log_c4", "log_c5", "log_c6", "log_c7", + "log_c8", "log_c9", "log_c10", "log_c11", "log_c12", "log_c13", + "log_c14", "log_c15", "log_c16" }), null, BUCKET_NUM); @@ -552,8 +539,7 @@ void testCreateLakeEnableTableWithExistLakeTable() throws Exception { .hasMessageContaining( "Existing schema: UpdateSchema{fields=[`c1` STRING, `c2` INT]") .hasMessageContaining("options={bucket=-1") - .hasMessageContaining( - "new schema: UpdateSchema{fields=[`c1` STRING, `c2` INT]") + .hasMessageContaining("new schema: UpdateSchema{fields=[`c1` STRING, `c2` INT]") .hasMessageContaining("options={bucket=3") .hasMessageContaining("bucket-key=c1,c2") .hasMessageEndingWith( @@ -1056,7 +1042,8 @@ void testEnableLakeTableWithLegacySystemTimestampColumn() throws Exception { OFFSET_COLUMN_NAME, org.apache.paimon.types.DataTypes.BIGINT()), SchemaChange.addColumn( TIMESTAMP_COLUMN_NAME, - org.apache.paimon.types.DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE())), + org.apache.paimon.types.DataTypes + .TIMESTAMP_WITH_LOCAL_TIME_ZONE())), false); // disable data lake From 2afdccddac40f8021044d26906b334059781333f Mon Sep 17 00:00:00 2001 From: fhan Date: Sun, 16 Aug 2026 12:40:24 +0800 Subject: [PATCH 06/11] [lake/paimon] code refine according to #2493 --- .../fluss/lake/paimon/PaimonLakeCatalog.java | 33 ++-- .../paimon/source/PaimonRecordReader.java | 33 ++-- .../tiering/FlussRecordAsPaimonRow.java | 52 +++--- .../lake/paimon/tiering/PaimonLakeWriter.java | 16 +- .../lake/paimon/tiering/RecordWriter.java | 6 +- .../append/AppendOnlyArrowBatchHelper.java | 9 +- .../tiering/append/AppendOnlyWriter.java | 15 +- .../tiering/mergetree/MergeTreeWriter.java | 13 +- .../lake/paimon/utils/PaimonConversions.java | 16 +- .../paimon/utils/PaimonRowAsFlussRow.java | 18 +- .../paimon/utils/PaimonSystemColumns.java | 174 ------------------ .../paimon/utils/PaimonTableValidation.java | 26 +-- .../paimon/LakeEnabledTableCreateITCase.java | 44 +++++ .../FlinkUnionReadPrimaryKeyTableITCase.java | 43 +++++ .../paimon/testutils/PaimonTestUtils.java | 53 ++++++ .../tiering/FlussRecordAsPaimonRowTest.java | 76 +++++--- .../paimon/tiering/PaimonTieringITCase.java | 66 ++++++- 17 files changed, 372 insertions(+), 321 deletions(-) delete mode 100644 fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSystemColumns.java create mode 100644 fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/testutils/PaimonTestUtils.java diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeCatalog.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeCatalog.java index e23bae37023..e7c1b446a31 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeCatalog.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeCatalog.java @@ -23,7 +23,6 @@ import org.apache.fluss.exception.TableAlreadyExistException; import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.lake.lakestorage.LakeCatalog; -import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns; import org.apache.fluss.metadata.TableChange; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TablePath; @@ -40,6 +39,7 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,21 +53,28 @@ import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimonSchemaChanges; import static org.apache.fluss.lake.paimon.utils.PaimonTableValidation.checkTableIsEmpty; import static org.apache.fluss.lake.paimon.utils.PaimonTableValidation.isPaimonSchemaCompatible; +import static org.apache.fluss.metadata.TableDescriptor.BUCKET_COLUMN_NAME; +import static org.apache.fluss.metadata.TableDescriptor.OFFSET_COLUMN_NAME; +import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; /** A Paimon implementation of {@link LakeCatalog}. */ public class PaimonLakeCatalog implements LakeCatalog { private static final Logger LOG = LoggerFactory.getLogger(PaimonLakeCatalog.class); private static final String PAIMON_PATH_KEY = "paimon.path"; + public static final LinkedHashMap SYSTEM_COLUMNS = new LinkedHashMap<>(); - /** - * The three Fluss system columns and their Paimon types, kept for readers, writers and the - * lookuper to recognise legacy tables by column name. Retained as an alias of {@link - * PaimonSystemColumns#SYSTEM_COLUMNS}; under FIP-27 these columns are no longer added to newly - * created (clean) tables. - */ - public static final LinkedHashMap SYSTEM_COLUMNS = - PaimonSystemColumns.SYSTEM_COLUMNS; + static { + // We need __bucket system column to filter out the given bucket + // for paimon bucket-unaware append only table. + // It's not required for paimon bucket-aware table like primary key table + // and bucket-aware append only table, but legacy tables always carry the system column + // for consistent behavior. Under FIP-27 these columns are no longer added to newly created + // (clean) tables; they only remain on legacy tables created before FIP-27. + SYSTEM_COLUMNS.put(BUCKET_COLUMN_NAME, DataTypes.INT()); + SYSTEM_COLUMNS.put(OFFSET_COLUMN_NAME, DataTypes.BIGINT()); + SYSTEM_COLUMNS.put(TIMESTAMP_COLUMN_NAME, DataTypes.TIMESTAMP_LTZ_MILLIS()); + } private final Catalog paimonCatalog; @@ -121,27 +128,25 @@ public void alterTable(TablePath tablePath, List tableChanges, Cont } Schema currentPaimonSchema = fileStoreTable.schema().toSchema(); - PaimonSystemColumns.LakeLayout lakeLayout = - PaimonSystemColumns.detectLayout(fileStoreTable.schema().logicalRowType()); List paimonSchemaChanges; if (isPaimonSchemaCompatible( currentPaimonSchema, toPaimonSchema(context.getCurrentTable()))) { // if the paimon schema is same as current fluss schema, directly apply all the // changes. - paimonSchemaChanges = toPaimonSchemaChanges(changesToApply, lakeLayout); + paimonSchemaChanges = toPaimonSchemaChanges(table, changesToApply); } else if (isPaimonSchemaCompatible( currentPaimonSchema, toPaimonSchema(context.getExpectedTable()))) { // if the schema is same as applied fluss schema , skip adding columns. paimonSchemaChanges = toPaimonSchemaChanges( + table, changesToApply.stream() .filter( tableChange -> !(tableChange instanceof TableChange.AddColumn)) - .collect(Collectors.toList()), - lakeLayout); + .collect(Collectors.toList())); } else { throw new InvalidAlterTableException( String.format( diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/PaimonRecordReader.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/PaimonRecordReader.java index 577b17658db..036afbcd177 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/PaimonRecordReader.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/PaimonRecordReader.java @@ -19,8 +19,6 @@ package org.apache.fluss.lake.paimon.source; import org.apache.fluss.lake.paimon.utils.PaimonRowAsFlussRow; -import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns; -import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.lake.source.RecordReader; import org.apache.fluss.record.ChangeType; import org.apache.fluss.record.GenericRecord; @@ -66,12 +64,10 @@ public PaimonRecordReader( @Nullable int[][] project, @Nullable Predicate predicate) throws IOException { - LakeLayout lakeLayout = - PaimonSystemColumns.detectLayout(fileStoreTable.schema().logicalRowType()); ReadBuilder readBuilder = fileStoreTable.newReadBuilder(); RowType paimonFullRowType = fileStoreTable.rowType(); if (project != null) { - readBuilder = applyProject(readBuilder, project, paimonFullRowType, lakeLayout); + readBuilder = applyProject(readBuilder, project, paimonFullRowType); } if (predicate != null) { @@ -83,15 +79,13 @@ public PaimonRecordReader( if (split == null) { iterator = new PaimonRecordReader.PaimonRowAsFlussRecordIterator( - org.apache.paimon.utils.CloseableIterator.empty(), - paimonRowType, - lakeLayout); + org.apache.paimon.utils.CloseableIterator.empty(), paimonRowType); } else { org.apache.paimon.reader.RecordReader recordReader = tableRead.createReader(split.dataSplit()); iterator = new PaimonRecordReader.PaimonRowAsFlussRecordIterator( - recordReader.toCloseableIterator(), paimonRowType, lakeLayout); + recordReader.toCloseableIterator(), paimonRowType); } } @@ -101,13 +95,10 @@ public CloseableIterator read() throws IOException { } private ReadBuilder applyProject( - ReadBuilder readBuilder, - int[][] projects, - RowType paimonFullRowType, - LakeLayout lakeLayout) { + ReadBuilder readBuilder, int[][] projects, RowType paimonFullRowType) { int[] projectIds = Arrays.stream(projects).mapToInt(project -> project[0]).toArray(); - if (lakeLayout == LakeLayout.CLEAN) { + if (!hasSystemColumn(paimonFullRowType)) { // Clean tables have no system columns to read, so project the business columns only. return readBuilder.withProjection(projectIds); } @@ -126,6 +117,15 @@ private ReadBuilder applyProject( return readBuilder.withProjection(paimonProject); } + /** A legacy table carries the three system columns, ending with {@code __timestamp}. */ + private static boolean hasSystemColumn(RowType paimonRowType) { + return paimonRowType + .getFields() + .get(paimonRowType.getFieldCount() - 1) + .name() + .equals(TIMESTAMP_COLUMN_NAME); + } + /** Iterator for paimon row as fluss record. */ public static class PaimonRowAsFlussRecordIterator implements CloseableIterator { @@ -139,12 +139,11 @@ public static class PaimonRowAsFlussRecordIterator implements CloseableIterator< public PaimonRowAsFlussRecordIterator( org.apache.paimon.utils.CloseableIterator paimonRowIterator, - RowType paimonRowType, - LakeLayout lakeLayout) { + RowType paimonRowType) { this.paimonRowIterator = paimonRowIterator; int fieldCount = paimonRowType.getFieldCount(); - if (lakeLayout == LakeLayout.CLEAN) { + if (!hasSystemColumn(paimonRowType)) { // No system columns are read; all projected fields are business fields, and the // log offset / timestamp are not available from the lake table. this.logOffsetColIndex = -1; diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRow.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRow.java index 0705ac4ed99..5633b95ab16 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRow.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRow.java @@ -18,7 +18,6 @@ package org.apache.fluss.lake.paimon.tiering; import org.apache.fluss.lake.paimon.source.FlussRowAsPaimonRow; -import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.record.LogRecord; import org.apache.paimon.data.InternalRow; @@ -34,8 +33,8 @@ /** To wrap Fluss {@link LogRecord} as paimon {@link InternalRow}. */ public class FlussRecordAsPaimonRow extends FlussRowAsPaimonRow { + private final boolean paimonIncludingSystemColumns; private final int bucket; - private final LakeLayout lakeLayout; private LogRecord logRecord; private int originRowFieldCount; private final int businessFieldCount; @@ -43,23 +42,22 @@ public class FlussRecordAsPaimonRow extends FlussRowAsPaimonRow { private final int offsetFieldIndex; private final int timestampFieldIndex; - public FlussRecordAsPaimonRow(int bucket, RowType tableTowType, LakeLayout lakeLayout) { - super(tableTowType); + public FlussRecordAsPaimonRow(int bucket, RowType tableRowType) { + this(bucket, tableRowType, false); + } + + public FlussRecordAsPaimonRow( + int bucket, RowType tableRowType, boolean paimonIncludingSystemColumns) { + super(tableRowType); this.bucket = bucket; - this.lakeLayout = lakeLayout; - if (lakeLayout == LakeLayout.LEGACY) { - // Legacy tables append the three system columns after the business columns. - this.businessFieldCount = tableRowType.getFieldCount() - SYSTEM_COLUMNS.size(); - this.bucketFieldIndex = businessFieldCount; - this.offsetFieldIndex = businessFieldCount + 1; - this.timestampFieldIndex = businessFieldCount + 2; - } else { - // Clean tables contain only business columns; there are no system fields to emit. - this.businessFieldCount = tableRowType.getFieldCount(); - this.bucketFieldIndex = -1; - this.offsetFieldIndex = -1; - this.timestampFieldIndex = -1; - } + this.paimonIncludingSystemColumns = paimonIncludingSystemColumns; + this.businessFieldCount = + tableRowType.getFieldCount() + - (paimonIncludingSystemColumns ? SYSTEM_COLUMNS.size() : 0); + // only valid when paimon includes the system columns + this.bucketFieldIndex = businessFieldCount; + this.offsetFieldIndex = businessFieldCount + 1; + this.timestampFieldIndex = businessFieldCount + 2; } public void setFlussRecord(LogRecord logRecord) { @@ -109,7 +107,7 @@ public boolean isNullAt(int pos) { @Override public int getInt(int pos) { - if (pos == bucketFieldIndex) { + if (paimonIncludingSystemColumns && pos == bucketFieldIndex) { // bucket system column return bucket; } @@ -126,12 +124,14 @@ public int getInt(int pos) { @Override public long getLong(int pos) { checkState(logRecord != null, "setFlussRecord() must be called before accessing the row."); - if (pos == offsetFieldIndex) { - // offset system column - return logRecord.logOffset(); - } else if (pos == timestampFieldIndex) { - // timestamp system column - return logRecord.timestamp(); + if (paimonIncludingSystemColumns) { + if (pos == offsetFieldIndex) { + // offset system column + return logRecord.logOffset(); + } else if (pos == timestampFieldIndex) { + // timestamp system column + return logRecord.timestamp(); + } } if (pos >= originRowFieldCount) { throw new IllegalStateException( @@ -147,7 +147,7 @@ public long getLong(int pos) { public Timestamp getTimestamp(int pos, int precision) { checkState(logRecord != null, "setFlussRecord() must be called before accessing the row."); // it's timestamp system column - if (pos == timestampFieldIndex) { + if (paimonIncludingSystemColumns && pos == timestampFieldIndex) { return Timestamp.fromEpochMillis(logRecord.timestamp()); } if (pos >= originRowFieldCount) { diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java index cb82976c9bb..a1cb63b31d0 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java @@ -21,8 +21,6 @@ import org.apache.fluss.lake.batch.RecordBatch; import org.apache.fluss.lake.paimon.tiering.append.AppendOnlyWriter; import org.apache.fluss.lake.paimon.tiering.mergetree.MergeTreeWriter; -import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns; -import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.lake.writer.LakeWriter; import org.apache.fluss.lake.writer.SupportsRecordBatchWrite; import org.apache.fluss.lake.writer.WriterInitContext; @@ -41,6 +39,7 @@ import java.util.Map; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimon; +import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; /** Implementation of {@link LakeWriter} for Paimon. */ public class PaimonLakeWriter implements LakeWriter, SupportsRecordBatchWrite { @@ -61,10 +60,11 @@ public PaimonLakeWriter( RowType flussRowType = writerInitContext.tableInfo().getRowType(); // FIP-27: detect whether the target Paimon table is a clean table (only user columns) or a - // legacy table (carrying the three Fluss system columns). Writers emit system columns only - // for legacy tables. - LakeLayout lakeLayout = - PaimonSystemColumns.detectLayout(fileStoreTable.schema().logicalRowType()); + // legacy table (carrying the three Fluss system columns). A legacy table is recognisable by + // the presence of the __timestamp system column. Writers emit system columns only for + // legacy tables. + boolean paimonIncludingSystemColumns = + fileStoreTable.rowType().getFieldIndex(TIMESTAMP_COLUMN_NAME) >= 0; this.recordWriter = fileStoreTable.primaryKeys().isEmpty() @@ -74,7 +74,7 @@ public PaimonLakeWriter( writerInitContext.partition(), partitionKeys, flussRowType, - lakeLayout) + paimonIncludingSystemColumns) : new MergeTreeWriter( fileStoreTable, writerInitContext.tableBucket(), @@ -82,7 +82,7 @@ public PaimonLakeWriter( partitionKeys, flussRowType, writerInitContext.ioTmpDirs(), - lakeLayout); + paimonIncludingSystemColumns); } @Override diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java index dad3f7e8f04..2260d553bc9 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java @@ -17,7 +17,6 @@ package org.apache.fluss.lake.paimon.tiering; -import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.metadata.ResolvedPartitionSpec; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.record.LogRecord; @@ -51,7 +50,7 @@ public RecordWriter( @Nullable String partition, List partitionKeys, org.apache.fluss.types.RowType flussRowType, - LakeLayout lakeLayout) { + boolean paimonIncludingSystemColumns) { this.tableWrite = tableWrite; this.tableRowType = tableRowType; this.bucket = tableBucket.getBucket(); @@ -64,7 +63,8 @@ public RecordWriter( this.partition = resolvePartition(partition, partitionKeys, flussRowType); } this.flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket.getBucket(), tableRowType, lakeLayout); + new FlussRecordAsPaimonRow( + tableBucket.getBucket(), tableRowType, paimonIncludingSystemColumns); } public abstract void write(LogRecord record) throws Exception; diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java index 4036ef96396..a19c2628b14 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java @@ -17,7 +17,6 @@ package org.apache.fluss.lake.paimon.tiering.append; -import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.record.ArrowBatchData; @@ -58,7 +57,7 @@ class AppendOnlyArrowBatchHelper implements AutoCloseable { private final TableWriteImpl tableWrite; private final RowType tableRowType; private final int bucket; - private final LakeLayout lakeLayout; + private final boolean paimonIncludingSystemColumns; private static final Field BUCKET_FIELD = new Field( @@ -91,12 +90,12 @@ class AppendOnlyArrowBatchHelper implements AutoCloseable { TableWriteImpl tableWrite, RowType tableRowType, int bucket, - LakeLayout lakeLayout) { + boolean paimonIncludingSystemColumns) { this.fileStoreTable = fileStoreTable; this.tableWrite = tableWrite; this.tableRowType = tableRowType; this.bucket = bucket; - this.lakeLayout = lakeLayout; + this.paimonIncludingSystemColumns = paimonIncludingSystemColumns; } /** @@ -112,7 +111,7 @@ void writeArrowBatch(ArrowBatchData arrowBatchData, BinaryRow partition) throws VectorSchemaRoot originalRoot = arrowBatchData.getVectorSchemaRoot(); - if (lakeLayout == LakeLayout.CLEAN) { + if (!paimonIncludingSystemColumns) { // Clean tables contain only user columns, so the incoming Arrow batch already matches // the Paimon table schema. Write it directly without enriching system columns. ArrowBundleRecords cleanRecords = diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java index 601afc531ea..23f61a33171 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java @@ -18,7 +18,6 @@ package org.apache.fluss.lake.paimon.tiering.append; import org.apache.fluss.lake.paimon.tiering.RecordWriter; -import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.record.ArrowBatchData; import org.apache.fluss.record.LogRecord; @@ -47,7 +46,7 @@ public class AppendOnlyWriter extends RecordWriter { */ @Nullable private AutoCloseable arrowBatchHelper; - private final LakeLayout lakeLayout; + private final boolean paimonIncludingSystemColumns; public AppendOnlyWriter( FileStoreTable fileStoreTable, @@ -55,7 +54,7 @@ public AppendOnlyWriter( @Nullable String partition, List partitionKeys, RowType flussRowType, - LakeLayout lakeLayout) { + boolean paimonIncludingSystemColumns) { //noinspection unchecked super( (TableWriteImpl) @@ -66,9 +65,9 @@ public AppendOnlyWriter( partition, partitionKeys, flussRowType, - lakeLayout); + paimonIncludingSystemColumns); this.fileStoreTable = fileStoreTable; - this.lakeLayout = lakeLayout; + this.paimonIncludingSystemColumns = paimonIncludingSystemColumns; } @Override @@ -96,7 +95,11 @@ public void writeArrowBatch(ArrowBatchData arrowBatchData) throws Exception { if (arrowBatchHelper == null) { helper = new AppendOnlyArrowBatchHelper( - fileStoreTable, tableWrite, tableRowType, bucket, lakeLayout); + fileStoreTable, + tableWrite, + tableRowType, + bucket, + paimonIncludingSystemColumns); arrowBatchHelper = helper; } else { helper = (AppendOnlyArrowBatchHelper) arrowBatchHelper; diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java index 9b0b2c0f744..37aeef7afe6 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java @@ -18,7 +18,6 @@ package org.apache.fluss.lake.paimon.tiering.mergetree; import org.apache.fluss.lake.paimon.tiering.RecordWriter; -import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.record.LogRecord; import org.apache.fluss.types.RowType; @@ -51,7 +50,7 @@ public MergeTreeWriter( @Nullable String partition, List partitionKeys, RowType flussRowType, - LakeLayout lakeLayout) { + boolean paimonIncludingSystemColumns) { this( fileStoreTable, tableBucket, @@ -59,7 +58,7 @@ public MergeTreeWriter( partitionKeys, flussRowType, (String[]) null, - lakeLayout); + paimonIncludingSystemColumns); } public MergeTreeWriter( @@ -69,7 +68,7 @@ public MergeTreeWriter( List partitionKeys, RowType flussRowType, @Nullable String[] ioTmpDirs, - LakeLayout lakeLayout) { + boolean paimonIncludingSystemColumns) { this( fileStoreTable, createIOManager(ioTmpDirs), @@ -77,7 +76,7 @@ public MergeTreeWriter( partition, partitionKeys, flussRowType, - lakeLayout); + paimonIncludingSystemColumns); } MergeTreeWriter( @@ -87,7 +86,7 @@ public MergeTreeWriter( @Nullable String partition, List partitionKeys, RowType flussRowType, - LakeLayout lakeLayout) { + boolean paimonIncludingSystemColumns) { super( createTableWrite(fileStoreTable, ioManager), fileStoreTable.rowType(), @@ -95,7 +94,7 @@ public MergeTreeWriter( partition, partitionKeys, flussRowType, - lakeLayout); + paimonIncludingSystemColumns); this.rowKeyExtractor = fileStoreTable.createRowKeyExtractor(); this.ioManager = ioManager; } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonConversions.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonConversions.java index c90f0e91438..198291b569b 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonConversions.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonConversions.java @@ -37,6 +37,7 @@ import org.apache.paimon.options.Options; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.table.Table; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.RowKind; @@ -51,7 +52,7 @@ import java.util.function.Function; import static org.apache.fluss.lake.paimon.PaimonLakeCatalog.SYSTEM_COLUMNS; -import static org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; +import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; import static org.apache.fluss.utils.Preconditions.checkState; /** Utils for conversion between Paimon and Fluss. */ @@ -174,7 +175,11 @@ public static BinaryRow toPaimonPartition( } public static List toPaimonSchemaChanges( - List tableChanges, LakeLayout lakeLayout) { + Table paimonTable, List tableChanges) { + // A legacy table (created before FIP-27) still carries the three trailing system columns, + // recognisable by the presence of the __timestamp column. A clean table has none of them. + boolean paimonIncludingSystemColumns = + paimonTable.rowType().getFieldIndex(TIMESTAMP_COLUMN_NAME) >= 0; List schemaChanges = new ArrayList<>(tableChanges.size()); for (TableChange tableChange : tableChanges) { @@ -205,7 +210,7 @@ public static List toPaimonSchemaChanges( org.apache.paimon.types.DataType paimonDataType = flussDataType.accept(FlussDataTypeToPaimonDataType.INSTANCE); - if (lakeLayout == LakeLayout.LEGACY) { + if (paimonIncludingSystemColumns) { // Legacy tables keep the three system columns as the last physical columns, so // a new business column must be inserted right before the first system column. String firstSystemColumnName = SYSTEM_COLUMNS.keySet().iterator().next(); @@ -224,7 +229,7 @@ public static List toPaimonSchemaChanges( addColumn.getName(), paimonDataType, addColumn.getComment(), - null)); + SchemaChange.Move.last(addColumn.getName()))); } } else { throw new UnsupportedOperationException( @@ -282,7 +287,8 @@ public static Schema toPaimonSchema(TableDescriptor tableDescriptor) { // FIP-27: newly created lake tables use a clean physical schema containing only // user-defined columns. The three Fluss system columns (__bucket, __offset, __timestamp) // are no longer added. Existing legacy tables that still carry these columns remain - // readable and writable, see PaimonSystemColumns#detectLayout. + // readable and writable; such tables are recognised by the presence of the __timestamp + // column. // set pk if (tableDescriptor.hasPrimaryKey()) { diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonRowAsFlussRow.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonRowAsFlussRow.java index 9e8be46c65b..5fa5a2feed6 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonRowAsFlussRow.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonRowAsFlussRow.java @@ -17,7 +17,6 @@ package org.apache.fluss.lake.paimon.utils; -import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.row.BinaryString; import org.apache.fluss.row.Decimal; import org.apache.fluss.row.InternalArray; @@ -33,23 +32,10 @@ public class PaimonRowAsFlussRow implements InternalRow { private org.apache.paimon.data.InternalRow paimonRow; - // Number of trailing Fluss system columns carried by the wrapped Paimon row that must be - // excluded from the exposed field count. This is only non-zero for a legacy table's top-level - // physical row; clean tables and nested/projected rows carry no system columns. - private final int trailingSystemColumns; - - public PaimonRowAsFlussRow() { - this.trailingSystemColumns = 0; - } - - public PaimonRowAsFlussRow(LakeLayout lakeLayout) { - this.trailingSystemColumns = - lakeLayout == LakeLayout.LEGACY ? PaimonSystemColumns.systemColumnCount() : 0; - } + public PaimonRowAsFlussRow() {} public PaimonRowAsFlussRow(org.apache.paimon.data.InternalRow paimonRow) { this.paimonRow = paimonRow; - this.trailingSystemColumns = 0; } public PaimonRowAsFlussRow replaceRow(org.apache.paimon.data.InternalRow paimonRow) { @@ -59,7 +45,7 @@ public PaimonRowAsFlussRow replaceRow(org.apache.paimon.data.InternalRow paimonR @Override public int getFieldCount() { - return paimonRow.getFieldCount() - trailingSystemColumns; + return paimonRow.getFieldCount(); } @Override diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSystemColumns.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSystemColumns.java deleted file mode 100644 index 35e4a265f52..00000000000 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSystemColumns.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.lake.paimon.utils; - -import org.apache.fluss.exception.InvalidTableException; - -import org.apache.paimon.types.DataField; -import org.apache.paimon.types.DataType; -import org.apache.paimon.types.DataTypes; -import org.apache.paimon.types.RowType; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import static org.apache.fluss.metadata.TableDescriptor.BUCKET_COLUMN_NAME; -import static org.apache.fluss.metadata.TableDescriptor.OFFSET_COLUMN_NAME; -import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; - -/** - * Utilities describing the two physical layouts a Paimon lake table can have under FIP-27, and the - * single place that detects which layout a given Paimon table uses. - * - *

      - *
    • {@link LakeLayout#CLEAN} - the table only contains user-defined columns. This is the layout - * of every newly created lake table. - *
    • {@link LakeLayout#LEGACY} - the table was created before FIP-27 and carries the three - * mandatory Fluss system columns {@code __bucket}, {@code __offset}, {@code __timestamp} as - * its last three physical columns. - *
    - * - *

    Detection is based purely on the physical Paimon schema, so no extra metadata or table - * property is needed and existing tables are never migrated. A table that carries only some of the - * system columns, or carries them with an unexpected type, is neither a clean nor a valid legacy - * table and is rejected with a clear error. - */ -public class PaimonSystemColumns { - - /** - * The three mandatory Fluss system columns and their expected Paimon types, in physical order. - * The {@code __timestamp} type is compared with relaxed precision (see {@link - * #isSystemTimestampType}) to stay compatible with legacy tables written by older clusters. - */ - public static final LinkedHashMap SYSTEM_COLUMNS = new LinkedHashMap<>(); - - static { - // We need __bucket system column to filter out the given bucket - // for paimon bucket-unaware append only table. - // It's not required for paimon bucket-aware table like primary key table - // and bucket-aware append only table, but legacy tables always carry the system column - // for consistent behavior. - SYSTEM_COLUMNS.put(BUCKET_COLUMN_NAME, DataTypes.INT()); - SYSTEM_COLUMNS.put(OFFSET_COLUMN_NAME, DataTypes.BIGINT()); - SYSTEM_COLUMNS.put(TIMESTAMP_COLUMN_NAME, DataTypes.TIMESTAMP_LTZ_MILLIS()); - } - - /** The physical layout of a Paimon lake table with respect to Fluss system columns. */ - public enum LakeLayout { - /** Only user-defined columns are present (FIP-27 default for new tables). */ - CLEAN, - /** The three Fluss system columns are appended as the last physical columns. */ - LEGACY - } - - private PaimonSystemColumns() {} - - /** Returns the number of system columns carried by a {@link LakeLayout#LEGACY} table. */ - public static int systemColumnCount() { - return SYSTEM_COLUMNS.size(); - } - - public static boolean isSystemColumn(String columnName) { - return SYSTEM_COLUMNS.containsKey(columnName); - } - - /** - * Detects whether a Paimon table with the given physical row type uses the clean or the legacy - * layout. - * - *

    The detection tolerates the {@code __timestamp} precision difference between old - * (precision 6) and new (precision 3) clusters, mirroring {@link - * PaimonTableValidation#equalIgnoreSystemColumnTimestampPrecision}. - * - * @throws InvalidTableException if the table carries only some of the system columns, carries - * them out of order, with an incompatible type, or embeds a system column name among the - * business columns. Such a table is neither clean nor a valid legacy table. - */ - public static LakeLayout detectLayout(RowType paimonRowType) { - List fields = paimonRowType.getFields(); - - int firstSystemColumnPos = -1; - for (int i = 0; i < fields.size(); i++) { - if (SYSTEM_COLUMNS.containsKey(fields.get(i).name())) { - firstSystemColumnPos = i; - break; - } - } - - // No system column anywhere -> clean layout. - if (firstSystemColumnPos < 0) { - return LakeLayout.CLEAN; - } - - // A system column exists. For a valid legacy table, all three must appear, in the canonical - // order, as the very last physical columns, each with a compatible type. - int businessFieldCount = fields.size() - SYSTEM_COLUMNS.size(); - if (firstSystemColumnPos != businessFieldCount) { - throw partialLayoutException(paimonRowType); - } - - int pos = businessFieldCount; - for (Map.Entry systemColumn : SYSTEM_COLUMNS.entrySet()) { - DataField field = fields.get(pos); - if (!field.name().equals(systemColumn.getKey()) - || !isSystemColumnTypeCompatible(field.name(), field.type())) { - throw partialLayoutException(paimonRowType); - } - pos++; - } - - return LakeLayout.LEGACY; - } - - private static boolean isSystemColumnTypeCompatible(String name, DataType actualType) { - if (TIMESTAMP_COLUMN_NAME.equals(name)) { - // Old clusters wrote precision 6, new clusters write precision 3; both are accepted. - return isSystemTimestampType(actualType); - } - // Compare the type family and precision, ignoring nullability: legacy system columns were - // written as non-null, but we only care that the physical type matches. - DataType expected = SYSTEM_COLUMNS.get(name); - return actualType.copy(true).equalsIgnoreFieldId(expected.copy(true)); - } - - private static boolean isSystemTimestampType(DataType actualType) { - // Legacy tables carry __timestamp with varying timestamp types depending on the cluster - // that created them: with or without local time zone, and precision 3 (new clusters) or 6 - // (old clusters). Accept the whole timestamp family and let the reader handle the - // precision, mirroring the relaxed check in - // PaimonTableValidation#equalIgnoreSystemColumnTimestampPrecision. - switch (actualType.getTypeRoot()) { - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - return true; - default: - return false; - } - } - - private static InvalidTableException partialLayoutException(RowType paimonRowType) { - return new InvalidTableException( - String.format( - "The Paimon table has an incompatible system-column layout. A table must " - + "either contain none of the Fluss system columns (clean layout) or " - + "contain all of %s as its last columns, in this order, with " - + "compatible types (legacy layout). Actual schema: %s.", - SYSTEM_COLUMNS.keySet(), paimonRowType)); - } -} diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java index b10e485dc85..4f2eeedfe62 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java @@ -32,9 +32,9 @@ import java.util.Map; import java.util.Objects; +import static org.apache.fluss.lake.paimon.PaimonLakeCatalog.SYSTEM_COLUMNS; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.PAIMON_UNSETTABLE_OPTIONS; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.PARTITION_GENERATE_LEGACY_NAME_OPTION_KEY; -import static org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; /** Utils to verify whether the existing Paimon table is compatible with the table to be created. */ @@ -42,12 +42,11 @@ public class PaimonTableValidation { public static boolean isPaimonSchemaCompatible(Schema existingSchema, Schema newSchema) { // FIP-27: newly generated schemas are always clean (no system columns). When the existing - // table is a legacy table that still carries the three system columns, re-enabling lake - // tiering must keep that physical layout. Enrich the clean new schema with the trailing - // system columns before comparison, so an existing legacy table is recognised as - // compatible and its layout is preserved. Detection also rejects a partial/type-mismatched - // legacy layout with a clear error. - if (PaimonSystemColumns.detectLayout(existingSchema.rowType()) == LakeLayout.LEGACY) { + // table is a legacy table that still carries the three system columns (recognisable by the + // presence of the __timestamp column), re-enabling lake tiering must keep that physical + // layout. Enrich the clean new schema with the trailing system columns before comparison, + // so an existing legacy table is recognised as compatible and its layout is preserved. + if (existingSchema.rowType().getFieldIndex(TIMESTAMP_COLUMN_NAME) >= 0) { newSchema = appendSystemColumns(newSchema); } @@ -75,7 +74,7 @@ private static Schema appendSystemColumns(Schema cleanSchema) { nextFieldId = Math.max(nextFieldId, field.id() + 1); } for (Map.Entry systemColumn : - PaimonSystemColumns.SYSTEM_COLUMNS.entrySet()) { + SYSTEM_COLUMNS.entrySet()) { fields.add( new DataField(nextFieldId++, systemColumn.getKey(), systemColumn.getValue())); } @@ -104,9 +103,14 @@ private static Schema appendSystemColumns(Schema cleanSchema) { public static boolean equalIgnoreSystemColumnTimestampPrecision( Schema existingSchema, Schema newSchema) { List existingFields = new ArrayList<>(existingSchema.fields()); - // Only legacy tables carry a trailing __timestamp system column. Clean tables have no - // system columns, so there is no precision to relax and we compare them directly. - if (existingFields.isEmpty()) { + // The precision relaxation only applies to a legacy table's trailing __timestamp system + // column. A clean table (or any table whose last column is not __timestamp) has no such + // column, so there is nothing to relax and we compare the schemas directly. + if (existingFields.isEmpty() + || !existingFields + .get(existingFields.size() - 1) + .name() + .equals(TIMESTAMP_COLUMN_NAME)) { return equalPhysicalSchema(existingSchema, newSchema); } DataField systemTimestampField = existingFields.get(existingFields.size() - 1); diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/LakeEnabledTableCreateITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/LakeEnabledTableCreateITCase.java index fafc95c5871..0a0c07d9735 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/LakeEnabledTableCreateITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/LakeEnabledTableCreateITCase.java @@ -73,6 +73,7 @@ import java.util.Set; import java.util.stream.Stream; +import static org.apache.fluss.lake.paimon.testutils.PaimonTestUtils.adjustToLegacyV1Table; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.PAIMON_UNSETTABLE_OPTIONS; import static org.apache.fluss.metadata.TableDescriptor.BUCKET_COLUMN_NAME; import static org.apache.fluss.metadata.TableDescriptor.OFFSET_COLUMN_NAME; @@ -1069,6 +1070,49 @@ void testEnableLakeTableWithLegacySystemTimestampColumn() throws Exception { .isTrue(); } + @Test + void testAddColumnForLegacyTableWithSystemColumns() throws Exception { + TablePath tablePath = TablePath.of(DATABASE, "legacy_add_column_table"); + TableDescriptor tableDescriptor = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("c1", DataTypes.INT()) + .column("c2", DataTypes.STRING()) + .build()) + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .build(); + admin.createTable(tablePath, tableDescriptor, false).get(); + + // turn the freshly created clean table into a legacy table carrying the three system + // columns + adjustToLegacyV1Table(tablePath, paimonCatalog); + + // adding a business column to a legacy table must still work, and the new column must be + // inserted before the trailing system columns so the legacy physical layout is preserved. + admin.alterTable( + tablePath, + Collections.singletonList( + TableChange.addColumn( + "c3", + DataTypes.INT(), + "c3 comment", + TableChange.ColumnPosition.last())), + false) + .get(); + + Identifier identifier = Identifier.create(DATABASE, tablePath.getTableName()); + RowType rowType = paimonCatalog.getTable(identifier).rowType(); + assertThat(rowType.getFieldNames()) + .containsExactly( + "c1", + "c2", + "c3", + BUCKET_COLUMN_NAME, + OFFSET_COLUMN_NAME, + TIMESTAMP_COLUMN_NAME); + } + @Test void testCreatePaimonDvTableWithNonStringPartitionColumn() throws Exception { TablePath tablePath = TablePath.of(DATABASE, "invalid_dv_table"); diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/flink/FlinkUnionReadPrimaryKeyTableITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/flink/FlinkUnionReadPrimaryKeyTableITCase.java index f91e56c27ea..9efa832e23d 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/flink/FlinkUnionReadPrimaryKeyTableITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/flink/FlinkUnionReadPrimaryKeyTableITCase.java @@ -70,6 +70,7 @@ import static org.apache.fluss.flink.source.testutils.FlinkRowAssertionsUtils.assertRowResultsIgnoreOrder; import static org.apache.fluss.flink.source.testutils.FlinkRowAssertionsUtils.collectBatchRows; import static org.apache.fluss.flink.source.testutils.FlinkRowAssertionsUtils.collectRowsWithTimeout; +import static org.apache.fluss.lake.paimon.testutils.PaimonTestUtils.adjustToLegacyV1Table; import static org.apache.fluss.testutils.DataTestUtils.row; import static org.apache.fluss.testutils.common.CommonTestUtils.retry; import static org.assertj.core.api.Assertions.assertThat; @@ -550,6 +551,48 @@ void testUnionReadWhenSomeBucketNotTiered() throws Exception { .isEqualTo("[+I[0, v01, v02], +I[1, v11, v12], +I[2, v21, v22], +I[3, v31, v32]]"); } + @Test + void testUnionReadLegacyTable() throws Exception { + // FIP-27: union read (lake snapshot merged with fresh Fluss log) must still work for a + // legacy table that carries the three trailing system columns. + String tableName = "pk_table_union_read_legacy"; + TablePath t1 = TablePath.of(DEFAULT_DB, tableName); + int bucketNum = 3; + long tableId = createSimplePkTable(t1, bucketNum, false, true); + + // turn the freshly created clean table into a legacy table before any tiering happens + adjustToLegacyV1Table(t1, paimonCatalog); + + // tier an initial version of every row into the lake + JobClient jobClient = buildTieringJob(execEnv); + List rows = new ArrayList<>(); + Map bucketLogEndOffset = new HashMap<>(); + for (int i = 0; i < bucketNum; i++) { + rows.add( + GenericRow.of( + i, BinaryString.fromString("v1_1"), BinaryString.fromString("v1_2"))); + bucketLogEndOffset.put(new TableBucket(tableId, i), 1L); + } + writeRows(t1, rows, false); + waitUntilBucketsSynced(bucketLogEndOffset.keySet()); + assertReplicaStatus(bucketLogEndOffset); + + // stop tiering, then overwrite every row so the fresh values only exist in the Fluss log + jobClient.cancel().get(); + rows.clear(); + for (int i = 0; i < bucketNum; i++) { + rows.add( + GenericRow.of( + i, BinaryString.fromString("v2_1"), BinaryString.fromString("v2_2"))); + } + writeRows(t1, rows, false); + + // union read must merge the lake snapshot with the fresh log and return the latest values + List result = toSortedRows(batchTEnv.executeSql("select * from " + tableName)); + assertThat(result.toString()) + .isEqualTo("[+I[0, v2_1, v2_2], +I[1, v2_1, v2_2], +I[2, v2_1, v2_2]]"); + } + @ParameterizedTest @ValueSource(booleans = {false, true}) void testUnionReadInStreamMode(Boolean isPartitioned) throws Exception { diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/testutils/PaimonTestUtils.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/testutils/PaimonTestUtils.java new file mode 100644 index 00000000000..590ff4886cd --- /dev/null +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/testutils/PaimonTestUtils.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.lake.paimon.testutils; + +import org.apache.fluss.metadata.TablePath; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.types.DataTypes; + +import java.util.Arrays; + +import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimon; +import static org.apache.fluss.metadata.TableDescriptor.BUCKET_COLUMN_NAME; +import static org.apache.fluss.metadata.TableDescriptor.OFFSET_COLUMN_NAME; +import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; + +/** The utils for paimon testing. */ +public class PaimonTestUtils { + + /** + * Adjusts the Paimon table to a legacy table by appending the three trailing Fluss system + * columns ({@code __bucket}, {@code __offset}, {@code __timestamp}). This simulates a table + * created before FIP-27, when the lake table always carried these system columns, so tests can + * verify that legacy tables remain readable and writable. + */ + public static void adjustToLegacyV1Table(TablePath tablePath, Catalog paimonCatalog) + throws Exception { + paimonCatalog.alterTable( + toPaimon(tablePath), + Arrays.asList( + SchemaChange.addColumn(BUCKET_COLUMN_NAME, DataTypes.INT()), + SchemaChange.addColumn(OFFSET_COLUMN_NAME, DataTypes.BIGINT()), + SchemaChange.addColumn( + TIMESTAMP_COLUMN_NAME, DataTypes.TIMESTAMP_LTZ_MILLIS())), + false); + } +} diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRowTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRowTest.java index dacb0479c80..891ce12c55a 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRowTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRowTest.java @@ -17,7 +17,6 @@ package org.apache.fluss.lake.paimon.tiering; -import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.record.GenericRecord; import org.apache.fluss.record.LogRecord; import org.apache.fluss.row.BinaryString; @@ -50,6 +49,38 @@ /** Test for {@link FlussRecordAsPaimonRow}. */ class FlussRecordAsPaimonRowTest { + @Test + void testCleanTableWritesNoSystemColumns() { + // A clean table (FIP-27 default) contains only user columns; the writer must not emit any + // of the three system columns. + int tableBucket = 3; + RowType tableRowType = + RowType.of( + new org.apache.paimon.types.IntType(), + new org.apache.paimon.types.VarCharType()); + + FlussRecordAsPaimonRow flussRecordAsPaimonRow = + new FlussRecordAsPaimonRow(tableBucket, tableRowType, false); + + long logOffset = 7L; + long timeStamp = System.currentTimeMillis(); + GenericRow genericRow = new GenericRow(2); + genericRow.setField(0, 1); + genericRow.setField(1, BinaryString.fromString("v1")); + LogRecord logRecord = new GenericRecord(logOffset, timeStamp, APPEND_ONLY, genericRow); + flussRecordAsPaimonRow.setFlussRecord(logRecord); + + // exposes exactly the two business columns, no trailing system columns + assertThat(flussRecordAsPaimonRow.getFieldCount()).isEqualTo(2); + assertThat(flussRecordAsPaimonRow.getInt(0)).isEqualTo(1); + assertThat(flussRecordAsPaimonRow.getString(1).toString()).isEqualTo("v1"); + + // the no-arg-layout constructor defaults to the clean layout + FlussRecordAsPaimonRow defaultRow = new FlussRecordAsPaimonRow(tableBucket, tableRowType); + defaultRow.setFlussRecord(logRecord); + assertThat(defaultRow.getFieldCount()).isEqualTo(2); + } + @Test void testLogTableRecordAllTypes() { // Construct a FlussRecordAsPaimonRow instance @@ -76,7 +107,7 @@ void testLogTableRecordAllTypes() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, true); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(14); @@ -144,7 +175,7 @@ void testPrimaryKeyTableRecord() { new org.apache.paimon.types.BigIntType(), new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, true); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -186,7 +217,7 @@ void testArrayTypeWithIntElements() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, true); long logOffset = 10; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(2); @@ -225,7 +256,7 @@ void testArrayTypeWithStringElements() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, true); long logOffset = 5; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -260,7 +291,7 @@ void testNestedArrayType() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, true); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -314,7 +345,7 @@ void testArrayWithAllPrimitiveTypes() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, true); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(7); @@ -387,7 +418,7 @@ void testArrayWithDecimalElements() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, true); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -421,7 +452,7 @@ void testArrayWithTimestampElements() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, true); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -455,7 +486,7 @@ void testArrayWithBinaryElements() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, true); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -484,7 +515,7 @@ void testNullArray() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, true); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -509,7 +540,7 @@ void testArrayWithNullableElements() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, true); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -538,7 +569,7 @@ void testEmptyArray() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, true); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -565,7 +596,7 @@ void testPaimonSchemaWiderThanFlussRecord() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, true); long logOffset = 7L; long timeStamp = System.currentTimeMillis(); @@ -596,7 +627,7 @@ void testFlussRecordWiderThanPaimonSchema() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, true); long logOffset = 7L; long timeStamp = System.currentTimeMillis(); @@ -691,7 +722,7 @@ void testNestedRowType() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, true); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(8); @@ -919,7 +950,7 @@ void testMapWithAllTypes() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, nestedMapRowType, LakeLayout.LEGACY); + new FlussRecordAsPaimonRow(tableBucket, nestedMapRowType, true); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow nestedMapGenericRow = new GenericRow(1); @@ -972,8 +1003,7 @@ private void testMapType( new org.apache.paimon.types.BigIntType(), new org.apache.paimon.types.LocalZonedTimestampType(3)); - FlussRecordAsPaimonRow flussRow = - new FlussRecordAsPaimonRow(tableBucket, rowType, LakeLayout.LEGACY); + FlussRecordAsPaimonRow flussRow = new FlussRecordAsPaimonRow(tableBucket, rowType, true); GenericRow genericRow = new GenericRow(1); genericRow.setField(0, new GenericMap(mapData)); LogRecord logRecord = new GenericRecord(logOffset, timeStamp, APPEND_ONLY, genericRow); @@ -999,7 +1029,7 @@ void testNullMap() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, true); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -1025,7 +1055,7 @@ void testMapWithNullableValues() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, true); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -1060,7 +1090,7 @@ void testEmptyMap() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, true); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -1084,7 +1114,7 @@ void testAccessRowBeforeSetThrowsIllegalState() { new org.apache.paimon.types.IntType(), new org.apache.paimon.types.BigIntType(), new org.apache.paimon.types.LocalZonedTimestampType(3)); - FlussRecordAsPaimonRow row = new FlussRecordAsPaimonRow(0, rowType, LakeLayout.LEGACY); + FlussRecordAsPaimonRow row = new FlussRecordAsPaimonRow(0, rowType, true); assertThatThrownBy(row::getRowKind) .isInstanceOf(IllegalStateException.class) .hasMessageContaining(expectedMsg); diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java index 7101e0069df..6d035aec098 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java @@ -67,6 +67,7 @@ import java.util.Map; import java.util.stream.Stream; +import static org.apache.fluss.lake.paimon.testutils.PaimonTestUtils.adjustToLegacyV1Table; import static org.apache.fluss.testutils.DataTestUtils.row; import static org.assertj.core.api.Assertions.assertThat; @@ -188,6 +189,37 @@ void testTiering() throws Exception { } } + @Test + void testTieringForLegacyTable() throws Exception { + // FIP-27: verify that tiering still works for a legacy table that carries the three + // trailing + // system columns (__bucket, __offset, __timestamp). + TablePath t = TablePath.of(DEFAULT_DB, "legacyLogTable"); + long tId = createLogTable(t); + + // turn the freshly created clean table into a legacy table before writing/tiering + adjustToLegacyV1Table(t, paimonCatalog); + + TableBucket tBucket = new TableBucket(tId, 0); + List flussRows = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + List rows = Arrays.asList(row(1, "v1"), row(2, "v2"), row(3, "v3")); + flussRows.addAll(rows); + writeRows(t, rows, true); + } + + JobClient jobClient = buildTieringJob(execEnv); + try { + assertReplicaStatus(tBucket, 30); + assertThat(getLeaderReplica(tBucket).getLogTablet().getLakeMaxTimestamp()) + .isGreaterThan(-1); + // the legacy table keeps the system columns, and the tiered __offset must be correct + checkDataInPaimonLegacyAppendOnlyTable(t, flussRows, 0); + } finally { + jobClient.cancel().get(); + } + } + private static Stream tieringAllTypesWriteArgs() { return Stream.of(Arguments.of(true), Arguments.of(false)); } @@ -504,9 +536,30 @@ private void checkDataInPaimonAppendOnlyTable( InternalRow flussRow = flussRowIterator.next(); assertThat(row.getInt(0)).isEqualTo(flussRow.getInt(0)); assertThat(row.getString(1).toString()).isEqualTo(flussRow.getString(1).toString()); - // FIP-27: a clean lake table only stores user columns, so there are no trailing - // __bucket/__offset/__timestamp columns to verify here. - assertThat(row.getFieldCount()).isEqualTo(2); + // FIP-27: a clean lake table only stores user columns, so the Paimon row has exactly + // the same field count as the expected Fluss row (no trailing + // __bucket/__offset/__timestamp columns). + assertThat(row.getFieldCount()).isEqualTo(flussRow.getFieldCount()); + } + assertThat(flussRowIterator.hasNext()).isFalse(); + } + + private void checkDataInPaimonLegacyAppendOnlyTable( + TablePath tablePath, List expectedRows, long startingOffset) + throws Exception { + Iterator paimonRowIterator = + getPaimonRowCloseableIterator(tablePath); + Iterator flussRowIterator = expectedRows.iterator(); + while (paimonRowIterator.hasNext()) { + org.apache.paimon.data.InternalRow row = paimonRowIterator.next(); + InternalRow flussRow = flussRowIterator.next(); + assertThat(row.getInt(0)).isEqualTo(flussRow.getInt(0)); + assertThat(row.getString(1).toString()).isEqualTo(flussRow.getString(1).toString()); + // A legacy table keeps the two business columns plus the three trailing system columns + // __bucket/__offset/__timestamp, so the tiered offset is stored at fieldCount - 2. + assertThat(row.getFieldCount()).isEqualTo(flussRow.getFieldCount() + 3); + int offsetIndex = row.getFieldCount() - 2; + assertThat(row.getLong(offsetIndex)).isEqualTo(startingOffset++); } assertThat(flussRowIterator.hasNext()).isFalse(); } @@ -526,9 +579,10 @@ private void checkDataInPaimonAppendOnlyPartitionedTable( assertThat(row.getInt(0)).isEqualTo(flussRow.getInt(0)); assertThat(row.getString(1).toString()).isEqualTo(flussRow.getString(1).toString()); assertThat(row.getString(2).toString()).isEqualTo(flussRow.getString(2).toString()); - // FIP-27: a clean lake table only stores user columns, so there are no trailing - // __bucket/__offset/__timestamp columns to verify here. - assertThat(row.getFieldCount()).isEqualTo(3); + // FIP-27: a clean lake table only stores user columns, so the Paimon row has exactly + // the same field count as the expected Fluss row (no trailing + // __bucket/__offset/__timestamp columns). + assertThat(row.getFieldCount()).isEqualTo(flussRow.getFieldCount()); } assertThat(flussRowIterator.hasNext()).isFalse(); } From 4029e83ba3987541d1d769992a8cc4637aaf612d Mon Sep 17 00:00:00 2001 From: fhan Date: Sun, 16 Aug 2026 18:42:28 +0800 Subject: [PATCH 07/11] [lake/paimon] fix CI errors in spark3-lake module --- .../paimon/SparkLakePaimonCatalogTest.scala | 52 +++++-------------- 1 file changed, 12 insertions(+), 40 deletions(-) diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/paimon/SparkLakePaimonCatalogTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/paimon/SparkLakePaimonCatalogTest.scala index fe5d3eb5d6e..e9ff7fc2d08 100644 --- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/paimon/SparkLakePaimonCatalogTest.scala +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/paimon/SparkLakePaimonCatalogTest.scala @@ -21,7 +21,6 @@ import org.apache.fluss.config.{ConfigOptions, Configuration, FlussConfigUtils} import org.apache.fluss.exception.FlussRuntimeException import org.apache.fluss.lake.paimon.utils.PaimonConversions import org.apache.fluss.metadata._ -import org.apache.fluss.metadata.TableDescriptor.{BUCKET_COLUMN_NAME, OFFSET_COLUMN_NAME, TIMESTAMP_COLUMN_NAME} import org.apache.fluss.server.utils.LakeStorageUtils import org.apache.fluss.spark.SparkCatalogTest import org.apache.fluss.spark.SparkConnectorOptions.{BUCKET_KEY, BUCKET_NUMBER, PRIMARY_KEY} @@ -154,12 +153,9 @@ class SparkLakePaimonCatalogTest extends SparkCatalogTest { .of( Array.apply( org.apache.paimon.types.DataTypes.INT, - org.apache.paimon.types.DataTypes.STRING, - org.apache.paimon.types.DataTypes.INT, - org.apache.paimon.types.DataTypes.BIGINT, - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS + org.apache.paimon.types.DataTypes.STRING ), - Array.apply("id", "name", BUCKET_COLUMN_NAME, OFFSET_COLUMN_NAME, TIMESTAMP_COLUMN_NAME) + Array.apply("id", "name") ), "id", 2 @@ -187,12 +183,9 @@ class SparkLakePaimonCatalogTest extends SparkCatalogTest { .of( Array.apply( org.apache.paimon.types.DataTypes.INT, - org.apache.paimon.types.DataTypes.STRING, - org.apache.paimon.types.DataTypes.INT, - org.apache.paimon.types.DataTypes.BIGINT, - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS + org.apache.paimon.types.DataTypes.STRING ), - Array.apply("id", "name", BUCKET_COLUMN_NAME, OFFSET_COLUMN_NAME, TIMESTAMP_COLUMN_NAME) + Array.apply("id", "name") ), null, 2 @@ -222,18 +215,12 @@ class SparkLakePaimonCatalogTest extends SparkCatalogTest { Array.apply( org.apache.paimon.types.DataTypes.INT.notNull(), org.apache.paimon.types.DataTypes.STRING, - org.apache.paimon.types.DataTypes.STRING.notNull(), - org.apache.paimon.types.DataTypes.INT, - org.apache.paimon.types.DataTypes.BIGINT, - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS + org.apache.paimon.types.DataTypes.STRING.notNull() ), Array.apply( "id", "name", - "pk1", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME) + "pk1") ), "id", 2 @@ -265,19 +252,13 @@ class SparkLakePaimonCatalogTest extends SparkCatalogTest { org.apache.paimon.types.DataTypes.INT.notNull(), org.apache.paimon.types.DataTypes.STRING, org.apache.paimon.types.DataTypes.STRING.notNull(), - org.apache.paimon.types.DataTypes.STRING.notNull(), - org.apache.paimon.types.DataTypes.INT, - org.apache.paimon.types.DataTypes.BIGINT, - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS + org.apache.paimon.types.DataTypes.STRING.notNull() ), Array.apply( "id", "name", "pk1", - "pt1", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME) + "pt1") ), "id", 2 @@ -328,10 +309,7 @@ class SparkLakePaimonCatalogTest extends SparkCatalogTest { ), org.apache.paimon.types.DataTypes.MAP( org.apache.paimon.types.DataTypes.STRING.notNull(), - org.apache.paimon.types.DataTypes.INT), - org.apache.paimon.types.DataTypes.INT, - org.apache.paimon.types.DataTypes.BIGINT, - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS + org.apache.paimon.types.DataTypes.INT) ), Array.apply( "c1", @@ -349,10 +327,7 @@ class SparkLakePaimonCatalogTest extends SparkCatalogTest { "c13", "c14", "c15", - "c16", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME) + "c16") ), null, 2 @@ -402,12 +377,9 @@ class SparkLakePaimonCatalogTest extends SparkCatalogTest { .of( Array.apply( org.apache.paimon.types.DataTypes.INT, - org.apache.paimon.types.DataTypes.STRING, - org.apache.paimon.types.DataTypes.INT, - org.apache.paimon.types.DataTypes.BIGINT, - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS + org.apache.paimon.types.DataTypes.STRING ), - Array.apply("id", "name", BUCKET_COLUMN_NAME, OFFSET_COLUMN_NAME, TIMESTAMP_COLUMN_NAME) + Array.apply("id", "name") ), null, 2 From 6b313fb25c706f0d9f3fdc065c1a5405be8331a1 Mon Sep 17 00:00:00 2001 From: fhan Date: Sun, 16 Aug 2026 21:09:51 +0800 Subject: [PATCH 08/11] [lake/paimon] fix format violations --- .../lake/paimon/SparkLakePaimonCatalogTest.scala | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/paimon/SparkLakePaimonCatalogTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/paimon/SparkLakePaimonCatalogTest.scala index e9ff7fc2d08..c08fe5aa780 100644 --- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/paimon/SparkLakePaimonCatalogTest.scala +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/paimon/SparkLakePaimonCatalogTest.scala @@ -217,10 +217,7 @@ class SparkLakePaimonCatalogTest extends SparkCatalogTest { org.apache.paimon.types.DataTypes.STRING, org.apache.paimon.types.DataTypes.STRING.notNull() ), - Array.apply( - "id", - "name", - "pk1") + Array.apply("id", "name", "pk1") ), "id", 2 @@ -254,11 +251,7 @@ class SparkLakePaimonCatalogTest extends SparkCatalogTest { org.apache.paimon.types.DataTypes.STRING.notNull(), org.apache.paimon.types.DataTypes.STRING.notNull() ), - Array.apply( - "id", - "name", - "pk1", - "pt1") + Array.apply("id", "name", "pk1", "pt1") ), "id", 2 From 66d2d5e8974dbfccb6ce1cdd5ba92e1d2385227a Mon Sep 17 00:00:00 2001 From: fhan Date: Sun, 16 Aug 2026 22:03:50 +0800 Subject: [PATCH 09/11] [lake/paimon] refine code impl according to review comments --- .../fluss/lake/paimon/PaimonLakeCatalog.java | 9 +- .../lookup/PaimonLakeTableLookuper.java | 4 +- .../paimon/source/PaimonRecordReader.java | 89 +++++-------------- .../tiering/FlussRecordAsPaimonRow.java | 4 +- .../lake/paimon/tiering/PaimonLakeWriter.java | 10 +-- .../lake/paimon/utils/PaimonConversions.java | 23 +++-- .../paimon/utils/PaimonTableValidation.java | 6 +- .../fluss/lake/paimon/utils/PaimonUtils.java | 40 +++++++++ .../paimon/tiering/PaimonTieringITCase.java | 52 ----------- 9 files changed, 88 insertions(+), 149 deletions(-) create mode 100644 fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonUtils.java diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeCatalog.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeCatalog.java index e7c1b446a31..3982e7474b7 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeCatalog.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeCatalog.java @@ -62,7 +62,8 @@ public class PaimonLakeCatalog implements LakeCatalog { private static final Logger LOG = LoggerFactory.getLogger(PaimonLakeCatalog.class); private static final String PAIMON_PATH_KEY = "paimon.path"; - public static final LinkedHashMap SYSTEM_COLUMNS = new LinkedHashMap<>(); + public static final LinkedHashMap LEGACY_SYSTEM_COLUMNS = + new LinkedHashMap<>(); static { // We need __bucket system column to filter out the given bucket @@ -71,9 +72,9 @@ public class PaimonLakeCatalog implements LakeCatalog { // and bucket-aware append only table, but legacy tables always carry the system column // for consistent behavior. Under FIP-27 these columns are no longer added to newly created // (clean) tables; they only remain on legacy tables created before FIP-27. - SYSTEM_COLUMNS.put(BUCKET_COLUMN_NAME, DataTypes.INT()); - SYSTEM_COLUMNS.put(OFFSET_COLUMN_NAME, DataTypes.BIGINT()); - SYSTEM_COLUMNS.put(TIMESTAMP_COLUMN_NAME, DataTypes.TIMESTAMP_LTZ_MILLIS()); + LEGACY_SYSTEM_COLUMNS.put(BUCKET_COLUMN_NAME, DataTypes.INT()); + LEGACY_SYSTEM_COLUMNS.put(OFFSET_COLUMN_NAME, DataTypes.BIGINT()); + LEGACY_SYSTEM_COLUMNS.put(TIMESTAMP_COLUMN_NAME, DataTypes.TIMESTAMP_LTZ_MILLIS()); } private final Catalog paimonCatalog; diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java index 18afe2d982e..9e04ce15ee8 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java @@ -68,7 +68,7 @@ import java.util.Set; import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_2; -import static org.apache.fluss.lake.paimon.PaimonLakeCatalog.SYSTEM_COLUMNS; +import static org.apache.fluss.lake.paimon.PaimonLakeCatalog.LEGACY_SYSTEM_COLUMNS; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimon; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimonPartition; import static org.apache.fluss.utils.Preconditions.checkArgument; @@ -280,7 +280,7 @@ private static int[] businessFieldProjection(FileStoreTable fileStoreTable) { List fields = fileStoreTable.schema().logicalRowType().getFields(); List projectedFields = new ArrayList<>(); for (int i = 0; i < fields.size(); i++) { - if (!SYSTEM_COLUMNS.containsKey(fields.get(i).name())) { + if (!LEGACY_SYSTEM_COLUMNS.containsKey(fields.get(i).name())) { projectedFields.add(i); } } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/PaimonRecordReader.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/PaimonRecordReader.java index 036afbcd177..dd2fca065ee 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/PaimonRecordReader.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/PaimonRecordReader.java @@ -19,6 +19,7 @@ package org.apache.fluss.lake.paimon.source; import org.apache.fluss.lake.paimon.utils.PaimonRowAsFlussRow; +import org.apache.fluss.lake.paimon.utils.PaimonUtils; import org.apache.fluss.lake.source.RecordReader; import org.apache.fluss.record.ChangeType; import org.apache.fluss.record.GenericRecord; @@ -39,18 +40,17 @@ import java.util.Arrays; import java.util.stream.IntStream; +import static org.apache.fluss.lake.paimon.PaimonLakeCatalog.LEGACY_SYSTEM_COLUMNS; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toChangeType; -import static org.apache.fluss.metadata.TableDescriptor.OFFSET_COLUMN_NAME; -import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; /** Record reader for paimon table. */ public class PaimonRecordReader implements RecordReader { /** - * Sentinel log offset / timestamp emitted for rows read from a clean lake table, which does not - * store the {@code __offset} / {@code __timestamp} system columns. A negative offset is - * interpreted downstream as "no valid offset" (snapshot phase), see {@code - * LakeRecordRecordEmitter}. + * Sentinel log offset / timestamp emitted for rows read from a lake table. The lake table does + * not carry a per-record log offset (a clean table has no system columns, and for a legacy + * table we no longer read them), so a negative value is emitted and interpreted downstream as + * "no valid offset" (snapshot phase), see {@code LakeRecordRecordEmitter}. */ private static final long NO_SYSTEM_COLUMN_VALUE = -1L; @@ -65,9 +65,8 @@ public PaimonRecordReader( @Nullable Predicate predicate) throws IOException { ReadBuilder readBuilder = fileStoreTable.newReadBuilder(); - RowType paimonFullRowType = fileStoreTable.rowType(); if (project != null) { - readBuilder = applyProject(readBuilder, project, paimonFullRowType); + readBuilder = applyProject(readBuilder, project); } if (predicate != null) { @@ -94,36 +93,11 @@ public CloseableIterator read() throws IOException { return iterator; } - private ReadBuilder applyProject( - ReadBuilder readBuilder, int[][] projects, RowType paimonFullRowType) { + private ReadBuilder applyProject(ReadBuilder readBuilder, int[][] projects) { + // The projected column ids reference the user (business) columns; the log offset / + // timestamp are not read from the lake table, so no system column needs to be projected. int[] projectIds = Arrays.stream(projects).mapToInt(project -> project[0]).toArray(); - - if (!hasSystemColumn(paimonFullRowType)) { - // Clean tables have no system columns to read, so project the business columns only. - return readBuilder.withProjection(projectIds); - } - - // Legacy tables carry __offset/__timestamp, which the iterator needs to recover the log - // offset and timestamp of each record; append them to the projection. - int offsetFieldPos = paimonFullRowType.getFieldIndex(OFFSET_COLUMN_NAME); - int timestampFieldPos = paimonFullRowType.getFieldIndex(TIMESTAMP_COLUMN_NAME); - - int[] paimonProject = - IntStream.concat( - IntStream.of(projectIds), - IntStream.of(offsetFieldPos, timestampFieldPos)) - .toArray(); - - return readBuilder.withProjection(paimonProject); - } - - /** A legacy table carries the three system columns, ending with {@code __timestamp}. */ - private static boolean hasSystemColumn(RowType paimonRowType) { - return paimonRowType - .getFields() - .get(paimonRowType.getFieldCount() - 1) - .name() - .equals(TIMESTAMP_COLUMN_NAME); + return readBuilder.withProjection(projectIds); } /** Iterator for paimon row as fluss record. */ @@ -134,32 +108,20 @@ public static class PaimonRowAsFlussRecordIterator implements CloseableIterator< private final ProjectedRow projectedRow; private final PaimonRowAsFlussRow paimonRowAsFlussRow; - private final int logOffsetColIndex; - private final int timestampColIndex; - public PaimonRowAsFlussRecordIterator( org.apache.paimon.utils.CloseableIterator paimonRowIterator, RowType paimonRowType) { this.paimonRowIterator = paimonRowIterator; + // A legacy table read without projection still exposes its three trailing system + // columns; trim them so only the business columns are emitted. A clean table (or any + // projected read) has no system columns to trim. int fieldCount = paimonRowType.getFieldCount(); - if (!hasSystemColumn(paimonRowType)) { - // No system columns are read; all projected fields are business fields, and the - // log offset / timestamp are not available from the lake table. - this.logOffsetColIndex = -1; - this.timestampColIndex = -1; - projectedRow = ProjectedRow.from(IntStream.range(0, fieldCount).toArray()); - } else { - // Legacy layout: applyProject appended exactly __offset and __timestamp (not - // __bucket) as the last two projected fields, so the business fields are all fields - // except those trailing two. - this.logOffsetColIndex = paimonRowType.getFieldIndex(OFFSET_COLUMN_NAME); - this.timestampColIndex = paimonRowType.getFieldIndex(TIMESTAMP_COLUMN_NAME); - int[] project = IntStream.range(0, fieldCount - 2).toArray(); - projectedRow = ProjectedRow.from(project); - } - // The wrapped row is only ever accessed by index through projectedRow, which already - // drops the system columns, so no trailing-system-column trimming is needed here. + int businessFieldCount = + PaimonUtils.isLegacyTable(paimonRowType) + ? fieldCount - LEGACY_SYSTEM_COLUMNS.size() + : fieldCount; + projectedRow = ProjectedRow.from(IntStream.range(0, businessFieldCount).toArray()); paimonRowAsFlussRow = new PaimonRowAsFlussRow(); } @@ -181,18 +143,9 @@ public boolean hasNext() { public LogRecord next() { InternalRow paimonRow = paimonRowIterator.next(); ChangeType changeType = toChangeType(paimonRow.getRowKind()); - long offset = - logOffsetColIndex < 0 - ? NO_SYSTEM_COLUMN_VALUE - : paimonRow.getLong(logOffsetColIndex); - long timestamp = - timestampColIndex < 0 - ? NO_SYSTEM_COLUMN_VALUE - : paimonRow.getTimestamp(timestampColIndex, 6).getMillisecond(); - return new GenericRecord( - offset, - timestamp, + NO_SYSTEM_COLUMN_VALUE, + NO_SYSTEM_COLUMN_VALUE, changeType, projectedRow.replaceRow(paimonRowAsFlussRow.replaceRow(paimonRow))); } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRow.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRow.java index 5633b95ab16..78ddf30450b 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRow.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRow.java @@ -25,7 +25,7 @@ import org.apache.paimon.types.RowKind; import org.apache.paimon.types.RowType; -import static org.apache.fluss.lake.paimon.PaimonLakeCatalog.SYSTEM_COLUMNS; +import static org.apache.fluss.lake.paimon.PaimonLakeCatalog.LEGACY_SYSTEM_COLUMNS; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toRowKind; import static org.apache.fluss.utils.Preconditions.checkNotNull; import static org.apache.fluss.utils.Preconditions.checkState; @@ -53,7 +53,7 @@ public FlussRecordAsPaimonRow( this.paimonIncludingSystemColumns = paimonIncludingSystemColumns; this.businessFieldCount = tableRowType.getFieldCount() - - (paimonIncludingSystemColumns ? SYSTEM_COLUMNS.size() : 0); + - (paimonIncludingSystemColumns ? LEGACY_SYSTEM_COLUMNS.size() : 0); // only valid when paimon includes the system columns this.bucketFieldIndex = businessFieldCount; this.offsetFieldIndex = businessFieldCount + 1; diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java index a1cb63b31d0..9b82403b6ce 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java @@ -21,6 +21,7 @@ import org.apache.fluss.lake.batch.RecordBatch; import org.apache.fluss.lake.paimon.tiering.append.AppendOnlyWriter; import org.apache.fluss.lake.paimon.tiering.mergetree.MergeTreeWriter; +import org.apache.fluss.lake.paimon.utils.PaimonUtils; import org.apache.fluss.lake.writer.LakeWriter; import org.apache.fluss.lake.writer.SupportsRecordBatchWrite; import org.apache.fluss.lake.writer.WriterInitContext; @@ -39,7 +40,6 @@ import java.util.Map; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimon; -import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; /** Implementation of {@link LakeWriter} for Paimon. */ public class PaimonLakeWriter implements LakeWriter, SupportsRecordBatchWrite { @@ -60,11 +60,9 @@ public PaimonLakeWriter( RowType flussRowType = writerInitContext.tableInfo().getRowType(); // FIP-27: detect whether the target Paimon table is a clean table (only user columns) or a - // legacy table (carrying the three Fluss system columns). A legacy table is recognisable by - // the presence of the __timestamp system column. Writers emit system columns only for - // legacy tables. - boolean paimonIncludingSystemColumns = - fileStoreTable.rowType().getFieldIndex(TIMESTAMP_COLUMN_NAME) >= 0; + // legacy table (carrying the three Fluss system columns). Writers emit system columns only + // for legacy tables. + boolean paimonIncludingSystemColumns = PaimonUtils.isLegacyTable(fileStoreTable.rowType()); this.recordWriter = fileStoreTable.primaryKeys().isEmpty() diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonConversions.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonConversions.java index 198291b569b..cf3fc034827 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonConversions.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonConversions.java @@ -51,8 +51,7 @@ import java.util.Set; import java.util.function.Function; -import static org.apache.fluss.lake.paimon.PaimonLakeCatalog.SYSTEM_COLUMNS; -import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; +import static org.apache.fluss.lake.paimon.PaimonLakeCatalog.LEGACY_SYSTEM_COLUMNS; import static org.apache.fluss.utils.Preconditions.checkState; /** Utils for conversion between Paimon and Fluss. */ @@ -178,8 +177,7 @@ public static List toPaimonSchemaChanges( Table paimonTable, List tableChanges) { // A legacy table (created before FIP-27) still carries the three trailing system columns, // recognisable by the presence of the __timestamp column. A clean table has none of them. - boolean paimonIncludingSystemColumns = - paimonTable.rowType().getFieldIndex(TIMESTAMP_COLUMN_NAME) >= 0; + boolean paimonIncludingSystemColumns = PaimonUtils.isLegacyTable(paimonTable.rowType()); List schemaChanges = new ArrayList<>(tableChanges.size()); for (TableChange tableChange : tableChanges) { @@ -196,6 +194,13 @@ public static List toPaimonSchemaChanges( } else if (tableChange instanceof TableChange.AddColumn) { TableChange.AddColumn addColumn = (TableChange.AddColumn) tableChange; + if (LEGACY_SYSTEM_COLUMNS.containsKey(addColumn.getName())) { + throw new InvalidTableException( + "Column " + + addColumn.getName() + + " conflicts with a system column name of paimon table, please rename the column."); + } + if (!(addColumn.getPosition() instanceof TableChange.Last)) { throw new UnsupportedOperationException( "Only support to add column at last for paimon table."); @@ -213,7 +218,7 @@ public static List toPaimonSchemaChanges( if (paimonIncludingSystemColumns) { // Legacy tables keep the three system columns as the last physical columns, so // a new business column must be inserted right before the first system column. - String firstSystemColumnName = SYSTEM_COLUMNS.keySet().iterator().next(); + String firstSystemColumnName = LEGACY_SYSTEM_COLUMNS.keySet().iterator().next(); schemaChanges.add( SchemaChange.addColumn( addColumn.getName(), @@ -272,7 +277,7 @@ public static Schema toPaimonSchema(TableDescriptor tableDescriptor) { for (org.apache.fluss.metadata.Schema.Column column : tableDescriptor.getSchema().getColumns()) { String columnName = column.getName(); - if (SYSTEM_COLUMNS.containsKey(columnName)) { + if (LEGACY_SYSTEM_COLUMNS.containsKey(columnName)) { throw new InvalidTableException( "Column " + columnName @@ -284,12 +289,6 @@ public static Schema toPaimonSchema(TableDescriptor tableDescriptor) { column.getComment().orElse(null)); } - // FIP-27: newly created lake tables use a clean physical schema containing only - // user-defined columns. The three Fluss system columns (__bucket, __offset, __timestamp) - // are no longer added. Existing legacy tables that still carry these columns remain - // readable and writable; such tables are recognised by the presence of the __timestamp - // column. - // set pk if (tableDescriptor.hasPrimaryKey()) { schemaBuilder.primaryKey( diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java index 4f2eeedfe62..c6254084c17 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java @@ -32,7 +32,7 @@ import java.util.Map; import java.util.Objects; -import static org.apache.fluss.lake.paimon.PaimonLakeCatalog.SYSTEM_COLUMNS; +import static org.apache.fluss.lake.paimon.PaimonLakeCatalog.LEGACY_SYSTEM_COLUMNS; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.PAIMON_UNSETTABLE_OPTIONS; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.PARTITION_GENERATE_LEGACY_NAME_OPTION_KEY; import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; @@ -46,7 +46,7 @@ public static boolean isPaimonSchemaCompatible(Schema existingSchema, Schema new // presence of the __timestamp column), re-enabling lake tiering must keep that physical // layout. Enrich the clean new schema with the trailing system columns before comparison, // so an existing legacy table is recognised as compatible and its layout is preserved. - if (existingSchema.rowType().getFieldIndex(TIMESTAMP_COLUMN_NAME) >= 0) { + if (PaimonUtils.isLegacyTable(existingSchema.rowType())) { newSchema = appendSystemColumns(newSchema); } @@ -74,7 +74,7 @@ private static Schema appendSystemColumns(Schema cleanSchema) { nextFieldId = Math.max(nextFieldId, field.id() + 1); } for (Map.Entry systemColumn : - SYSTEM_COLUMNS.entrySet()) { + LEGACY_SYSTEM_COLUMNS.entrySet()) { fields.add( new DataField(nextFieldId++, systemColumn.getKey(), systemColumn.getValue())); } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonUtils.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonUtils.java new file mode 100644 index 00000000000..6b58f3f2b29 --- /dev/null +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonUtils.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.lake.paimon.utils; + +import org.apache.paimon.types.RowType; + +import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; + +/** Common utilities for the Paimon lake integration. */ +public final class PaimonUtils { + + private PaimonUtils() {} + + /** + * Returns whether the physical Paimon table uses the legacy layout, i.e. it still carries the + * three trailing Fluss system columns ({@code __bucket}, {@code __offset}, {@code + * __timestamp}). + * + *

    A legacy table is identified by the presence of the {@code __timestamp} system column; + * under FIP-27 newly created tables are clean and carry none of these columns. + */ + public static boolean isLegacyTable(RowType tableRowType) { + return tableRowType.getFieldIndex(TIMESTAMP_COLUMN_NAME) >= 0; + } +} diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java index 6d035aec098..e20a2d3e608 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java @@ -67,7 +67,6 @@ import java.util.Map; import java.util.stream.Stream; -import static org.apache.fluss.lake.paimon.testutils.PaimonTestUtils.adjustToLegacyV1Table; import static org.apache.fluss.testutils.DataTestUtils.row; import static org.assertj.core.api.Assertions.assertThat; @@ -189,37 +188,6 @@ void testTiering() throws Exception { } } - @Test - void testTieringForLegacyTable() throws Exception { - // FIP-27: verify that tiering still works for a legacy table that carries the three - // trailing - // system columns (__bucket, __offset, __timestamp). - TablePath t = TablePath.of(DEFAULT_DB, "legacyLogTable"); - long tId = createLogTable(t); - - // turn the freshly created clean table into a legacy table before writing/tiering - adjustToLegacyV1Table(t, paimonCatalog); - - TableBucket tBucket = new TableBucket(tId, 0); - List flussRows = new ArrayList<>(); - for (int i = 0; i < 10; i++) { - List rows = Arrays.asList(row(1, "v1"), row(2, "v2"), row(3, "v3")); - flussRows.addAll(rows); - writeRows(t, rows, true); - } - - JobClient jobClient = buildTieringJob(execEnv); - try { - assertReplicaStatus(tBucket, 30); - assertThat(getLeaderReplica(tBucket).getLogTablet().getLakeMaxTimestamp()) - .isGreaterThan(-1); - // the legacy table keeps the system columns, and the tiered __offset must be correct - checkDataInPaimonLegacyAppendOnlyTable(t, flussRows, 0); - } finally { - jobClient.cancel().get(); - } - } - private static Stream tieringAllTypesWriteArgs() { return Stream.of(Arguments.of(true), Arguments.of(false)); } @@ -544,26 +512,6 @@ private void checkDataInPaimonAppendOnlyTable( assertThat(flussRowIterator.hasNext()).isFalse(); } - private void checkDataInPaimonLegacyAppendOnlyTable( - TablePath tablePath, List expectedRows, long startingOffset) - throws Exception { - Iterator paimonRowIterator = - getPaimonRowCloseableIterator(tablePath); - Iterator flussRowIterator = expectedRows.iterator(); - while (paimonRowIterator.hasNext()) { - org.apache.paimon.data.InternalRow row = paimonRowIterator.next(); - InternalRow flussRow = flussRowIterator.next(); - assertThat(row.getInt(0)).isEqualTo(flussRow.getInt(0)); - assertThat(row.getString(1).toString()).isEqualTo(flussRow.getString(1).toString()); - // A legacy table keeps the two business columns plus the three trailing system columns - // __bucket/__offset/__timestamp, so the tiered offset is stored at fieldCount - 2. - assertThat(row.getFieldCount()).isEqualTo(flussRow.getFieldCount() + 3); - int offsetIndex = row.getFieldCount() - 2; - assertThat(row.getLong(offsetIndex)).isEqualTo(startingOffset++); - } - assertThat(flussRowIterator.hasNext()).isFalse(); - } - private void checkDataInPaimonAppendOnlyPartitionedTable( TablePath tablePath, Map partitionSpec, From 606312846bfc9fe3e186b37b4367052b8e4dd53e Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Mon, 17 Aug 2026 10:18:08 +0800 Subject: [PATCH 10/11] [lake/paimon] Restore legacy tiering test Restore append-only legacy table coverage removed in the previous review round. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 52/52 --- .../paimon/tiering/PaimonTieringITCase.java | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java index e20a2d3e608..6d035aec098 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java @@ -67,6 +67,7 @@ import java.util.Map; import java.util.stream.Stream; +import static org.apache.fluss.lake.paimon.testutils.PaimonTestUtils.adjustToLegacyV1Table; import static org.apache.fluss.testutils.DataTestUtils.row; import static org.assertj.core.api.Assertions.assertThat; @@ -188,6 +189,37 @@ void testTiering() throws Exception { } } + @Test + void testTieringForLegacyTable() throws Exception { + // FIP-27: verify that tiering still works for a legacy table that carries the three + // trailing + // system columns (__bucket, __offset, __timestamp). + TablePath t = TablePath.of(DEFAULT_DB, "legacyLogTable"); + long tId = createLogTable(t); + + // turn the freshly created clean table into a legacy table before writing/tiering + adjustToLegacyV1Table(t, paimonCatalog); + + TableBucket tBucket = new TableBucket(tId, 0); + List flussRows = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + List rows = Arrays.asList(row(1, "v1"), row(2, "v2"), row(3, "v3")); + flussRows.addAll(rows); + writeRows(t, rows, true); + } + + JobClient jobClient = buildTieringJob(execEnv); + try { + assertReplicaStatus(tBucket, 30); + assertThat(getLeaderReplica(tBucket).getLogTablet().getLakeMaxTimestamp()) + .isGreaterThan(-1); + // the legacy table keeps the system columns, and the tiered __offset must be correct + checkDataInPaimonLegacyAppendOnlyTable(t, flussRows, 0); + } finally { + jobClient.cancel().get(); + } + } + private static Stream tieringAllTypesWriteArgs() { return Stream.of(Arguments.of(true), Arguments.of(false)); } @@ -512,6 +544,26 @@ private void checkDataInPaimonAppendOnlyTable( assertThat(flussRowIterator.hasNext()).isFalse(); } + private void checkDataInPaimonLegacyAppendOnlyTable( + TablePath tablePath, List expectedRows, long startingOffset) + throws Exception { + Iterator paimonRowIterator = + getPaimonRowCloseableIterator(tablePath); + Iterator flussRowIterator = expectedRows.iterator(); + while (paimonRowIterator.hasNext()) { + org.apache.paimon.data.InternalRow row = paimonRowIterator.next(); + InternalRow flussRow = flussRowIterator.next(); + assertThat(row.getInt(0)).isEqualTo(flussRow.getInt(0)); + assertThat(row.getString(1).toString()).isEqualTo(flussRow.getString(1).toString()); + // A legacy table keeps the two business columns plus the three trailing system columns + // __bucket/__offset/__timestamp, so the tiered offset is stored at fieldCount - 2. + assertThat(row.getFieldCount()).isEqualTo(flussRow.getFieldCount() + 3); + int offsetIndex = row.getFieldCount() - 2; + assertThat(row.getLong(offsetIndex)).isEqualTo(startingOffset++); + } + assertThat(flussRowIterator.hasNext()).isFalse(); + } + private void checkDataInPaimonAppendOnlyPartitionedTable( TablePath tablePath, Map partitionSpec, From 89077ed807f85aaeddbb8c52f4042eaa36ed7e19 Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Mon, 17 Aug 2026 10:25:33 +0800 Subject: [PATCH 11/11] [server] Validate system columns during schema alter Apply the existing reserved-system-column validation to altered schemas. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 1/1 AI-Contributed/UT: 0/0 --- .../org/apache/fluss/server/utils/TableDescriptorValidation.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java index e8ba9714dd6..e898420e1e9 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java @@ -137,6 +137,7 @@ public static void validateTableDescriptor( /** Validates the schema after altering table columns. */ @Internal public static void validateAlterTableSchema(TableInfo table, Schema newSchema) { + checkSystemColumns(newSchema.getRowType()); if (table.getTableConfig() .getMergeEngineType() .map(MergeEngineType.AGGREGATION::equals)