Skip to content

[Pipe] Use buffered input for TsFile scan parser #18254

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Caideyipi wants to merge 4 commits into
base: master
Choose a base branch
from
+38 −5
Open

[Pipe] Use buffered input for TsFile scan parser #18254

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
24a476d
[Pipe] Use buffered input for TsFile scan parser
Caideyipi Jul 20, 2026
a1f2b5f
[Pipe] Use new TsFile input reader API
Caideyipi Jul 21, 2026
0672b08
Fix aligned TVList bitmap memory accounting
Caideyipi Jul 22, 2026
63cb989
Merge branch 'master' of https://github.com/apache/iotdb into pipe-bu…
Caideyipi Jul 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 25 additions & 5 deletions .../apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
import org.apache.tsfile.read.common.BatchData;
import org.apache.tsfile.read.common.Chunk;
import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.read.reader.BufferedTsFileInput;
import org.apache.tsfile.read.reader.IChunkReader;
import org.apache.tsfile.read.reader.chunk.AlignedChunkReader;
import org.apache.tsfile.read.reader.chunk.ChunkReader;
Expand All @@ -83,6 +84,8 @@

public class TsFileInsertionEventScanParser extends TsFileInsertionEventParser {

private static final int TS_FILE_INPUT_BUFFER_SIZE_IN_BYTES = 8 * 1024;

private final long startTime;
private final long endTime;
private final Filter filter;
Expand All @@ -91,6 +94,7 @@ public class TsFileInsertionEventScanParser extends TsFileInsertionEventParser {
private BatchData data;
private final PipeMemoryBlock allocatedMemoryBlockForBatchData;
private final PipeMemoryBlock allocatedMemoryBlockForChunk;
private PipeMemoryBlock allocatedMemoryBlockForTsFileInput;

private boolean currentIsMultiPage;
private IDeviceID currentDevice;
Expand Down Expand Up @@ -156,11 +160,7 @@ public TsFileInsertionEventScanParser(
PipeDataNodeResourceManager.memory()
.forceAllocateForTabletWithRetry(currentModifications.ramBytesUsed());

tsFileSequenceReader =
new TsFileSequenceReader(
tsFile.getAbsolutePath(),
!currentModifications.isEmpty(),
!currentModifications.isEmpty());
tsFileSequenceReader = createTsFileSequenceReader(tsFile, !currentModifications.isEmpty());
tsFileSequenceReader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1);

prepareData();
Expand Down Expand Up @@ -193,6 +193,22 @@ public TsFileInsertionEventScanParser(
isWithMod);
}

private TsFileSequenceReader createTsFileSequenceReader(
final File tsFile, final boolean hasModifications) throws IOException {
if (hasModifications) {
return new TsFileSequenceReader(tsFile.getAbsolutePath(), true, true);
}

allocatedMemoryBlockForTsFileInput =
PipeDataNodeResourceManager.memory()
.forceAllocateForTabletWithRetry(TS_FILE_INPUT_BUFFER_SIZE_IN_BYTES);
return new TsFileSequenceReader(
new BufferedTsFileInput(tsFile.toPath(), TS_FILE_INPUT_BUFFER_SIZE_IN_BYTES),
false,
false,
null);
}

@Override
public Iterable<TabletInsertionEvent> toTabletInsertionEvents() {
if (tabletInsertionIterable == null) {
Expand Down Expand Up @@ -1098,6 +1114,10 @@ public void close() {
if (allocatedMemoryBlockForChunk != null) {
allocatedMemoryBlockForChunk.close();
}

if (allocatedMemoryBlockForTsFileInput != null) {
allocatedMemoryBlockForTsFileInput.close();
}
}

private Statistics findAlignedChunkStatistics(
Expand Down
13 changes: 13 additions & 0 deletions ...ore/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,22 @@
import java.util.List;

import static org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager.ARRAY_SIZE;
import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF;

public class AlignedTVListTest {

@Test
public void testValueListArrayMemCostIncludesBitmapImplementation() {
long expected =
(long) ARRAY_SIZE * Long.BYTES
+ BitMap.createBitMapDynamically(ARRAY_SIZE).ramBytesUsed()
+ NUM_BYTES_ARRAY_HEADER
+ 2L * NUM_BYTES_OBJECT_REF;

Assert.assertEquals(expected, AlignedTVList.valueListArrayMemCost(TSDataType.INT64));
}

@Test
public void testAlignedTVList1() {
List<TSDataType> dataTypes = new ArrayList<>();
Expand Down