From c76fc6fba469feb2b5a65433e9886d0574950b07 Mon Sep 17 00:00:00 2001 From: umi Date: Thu, 4 Jun 2026 15:48:16 +0800 Subject: [PATCH 1/4] refactor --- .../paimon/operation/ManifestFileSorter.java | 311 +++++++++++------- 1 file changed, 198 insertions(+), 113 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java index 39ef0bab5299..234f87d66eef 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java @@ -66,7 +66,7 @@ static class CompactionContext { final boolean fullCompaction; final RecordComparator fieldComparator; final Set deleteEntries; - final Map defaultCompactionMap; + final Map needsDefaultCompaction; final List levelRuns; final List pickedRuns; @@ -74,31 +74,44 @@ static class CompactionContext { boolean fullCompaction, RecordComparator fieldComparator, Set deleteEntries, - Map defaultCompactionMap, + Map needsDefaultCompaction, List levelRuns, List pickedRuns) { this.fullCompaction = fullCompaction; this.fieldComparator = fieldComparator; this.deleteEntries = deleteEntries; - this.defaultCompactionMap = defaultCompactionMap; + this.needsDefaultCompaction = needsDefaultCompaction; this.levelRuns = levelRuns; this.pickedRuns = pickedRuns; } + + /** Check whether the given manifest file is marked for default compaction. */ + boolean isMarkedForDefaultCompaction(ManifestFileMeta file) { + return needsDefaultCompaction.containsKey(file); + } } /** Result of classifying manifest files. */ private static class ClassifyResult { final List lsmFiles; final Set deleteEntries; - final Map defaultCompactionMap; + /** + * Manifest files that need default compaction. + * + *

Key: manifest file metadata + * + *

Value: true if the file overlaps with delete partitions and fullCompaction is true + * file + */ + final Map needsDefaultCompaction; ClassifyResult( List lsmFiles, Set deleteEntries, - Map defaultCompactionMap) { + Map needsDefaultCompaction) { this.lsmFiles = lsmFiles; this.deleteEntries = deleteEntries; - this.defaultCompactionMap = defaultCompactionMap; + this.needsDefaultCompaction = needsDefaultCompaction; } } @@ -201,7 +214,7 @@ private static Optional> tryFullCompaction( List levelRuns = ctx.levelRuns; List pickedRuns = ctx.pickedRuns; - if (pickedRuns.isEmpty() && ctx.defaultCompactionMap.isEmpty()) { + if (pickedRuns.isEmpty() && ctx.needsDefaultCompaction.isEmpty()) { LOG.debug( "Manifest sort full compact skipped: no runs picked and no defaultCompaction files."); return Optional.empty(); @@ -213,7 +226,7 @@ private static Optional> tryFullCompaction( input.size(), levelRuns.size(), pickedRuns.size(), - ctx.defaultCompactionMap.size()); + ctx.needsDefaultCompaction.size()); // Step 3: Collect reused files (not picked) and picked files Set pickedSet = new HashSet<>(pickedRuns); @@ -227,11 +240,10 @@ private static Optional> tryFullCompaction( for (ManifestAdjacentSortedRun run : pickedRuns) { pickedFiles.addAll(run.files()); } - pickedFiles.addAll(ctx.defaultCompactionMap.keySet()); + pickedFiles.addAll(ctx.needsDefaultCompaction.keySet()); // Step 4: Split into sections and merge small adjacent sections - List

sections = - splitIntoSections(pickedFiles, ctx.fieldComparator, ctx.defaultCompactionMap); + List
sections = splitIntoSections(pickedFiles, ctx); sections = mergeSmallAdjacentSections(sections, suggestedMetaSize); LOG.info( @@ -293,7 +305,7 @@ private static List tryMinorCompaction( List levelRuns = ctx.levelRuns; List pickedRuns = ctx.pickedRuns; - if (pickedRuns.isEmpty() && ctx.defaultCompactionMap.isEmpty()) { + if (pickedRuns.isEmpty() && ctx.needsDefaultCompaction.isEmpty()) { LOG.debug( "Manifest sort minor compact skipped: no runs picked and no defaultCompaction files."); return input; @@ -305,7 +317,7 @@ private static List tryMinorCompaction( input.size(), levelRuns.size(), pickedRuns.size(), - ctx.defaultCompactionMap.size()); + ctx.needsDefaultCompaction.size()); // Step 2: Build fileName -> index mapping and initialize 2D result Map fileNameToIndex = new HashMap<>(); @@ -332,7 +344,7 @@ private static List tryMinorCompaction( for (ManifestAdjacentSortedRun run : pickedRuns) { pickedFiles.addAll(run.files()); } - pickedFiles.addAll(ctx.defaultCompactionMap.keySet()); + pickedFiles.addAll(ctx.needsDefaultCompaction.keySet()); // Step 4: Compute index range int minIdx = Integer.MAX_VALUE; @@ -347,8 +359,7 @@ private static List tryMinorCompaction( Pair indexRange = Pair.of(minIdx, maxIdx); // Step 5: Split into sections and merge small adjacent sections - List
sections = - splitIntoSections(pickedFiles, ctx.fieldComparator, ctx.defaultCompactionMap); + List
sections = splitIntoSections(pickedFiles, ctx); sections = mergeSmallAdjacentSections(sections, suggestedMetaSize); LOG.info( @@ -436,7 +447,7 @@ private static CompactionContext prepareCompaction( fullCompaction, fieldComparator, classifyResult.deleteEntries, - classifyResult.defaultCompactionMap, + classifyResult.needsDefaultCompaction, levelRuns, pickedRuns); } @@ -445,12 +456,12 @@ private static CompactionContext prepareCompaction( * Classify manifest files into default-compaction group and LSM group. * *

Full compaction: small files and files overlapping delete partitions go into - * defaultCompactionMap; the rest are returned as lsmFiles. + * needsDefaultCompaction; the rest are returned as lsmFiles. * - *

Non-full compaction: small files go to defaultCompactionMap for minor-style merge; the + *

Non-full compaction: small files go to needsDefaultCompaction for minor-style merge; the * rest are returned as lsmFiles. * - * @return ClassifyResult containing lsmFiles, deleteEntries, and defaultCompactionMap + * @return ClassifyResult containing lsmFiles, deleteEntries, and needsDefaultCompaction */ private static ClassifyResult classifyManifests( List input, @@ -460,7 +471,7 @@ private static ClassifyResult classifyManifests( long suggestedMetaSize, @Nullable Integer manifestReadParallelism) { // Initialize classification containers and read delete entries - Map classifiedDefaultMap = new LinkedHashMap<>(); + Map needsDefaultCompaction = new LinkedHashMap<>(); List lsmFiles = new LinkedList<>(input); Set classifiedDeleteEntries = Collections.emptySet(); PartitionPredicate predicate = null; @@ -496,11 +507,11 @@ private static ClassifyResult classifyManifests( file.partitionStats().nullCounts()); if (small || inDeleteRange) { iterator.remove(); - classifiedDefaultMap.put(file, inDeleteRange); + needsDefaultCompaction.put(file, inDeleteRange); } } - return new ClassifyResult(lsmFiles, classifiedDeleteEntries, classifiedDefaultMap); + return new ClassifyResult(lsmFiles, classifiedDeleteEntries, needsDefaultCompaction); } /** @@ -591,9 +602,8 @@ static List buildLevelSortedRuns( * section. Each section is built with pre-computed totalSize and hasDefaultCompactMeta. */ static List

splitIntoSections( - List pickedFiles, - RecordComparator fieldComparator, - Map defaultCompactionMap) { + List pickedFiles, CompactionContext ctx) { + RecordComparator fieldComparator = ctx.fieldComparator; pickedFiles.sort( (a, b) -> { int cmp = @@ -607,13 +617,13 @@ static List
splitIntoSections( }); List
sections = new ArrayList<>(); - List currentFiles = new ArrayList<>(); - long currentTotalSize = 0; - boolean currentHasDefault = false; + List currentSectionFiles = new ArrayList<>(); + long currentSectionTotalSize = 0; ManifestFileMeta first = pickedFiles.get(0); - currentFiles.add(first); - currentTotalSize += first.fileSize(); - currentHasDefault = defaultCompactionMap.containsKey(first); + + currentSectionFiles.add(first); + currentSectionTotalSize += first.fileSize(); + boolean currentSectionHasCompactMeta = ctx.isMarkedForDefaultCompaction(first); BinaryRow sectionMaxBound = first.partitionStats().maxValues(); for (int i = 1; i < pickedFiles.size(); i++) { @@ -633,18 +643,23 @@ static List
splitIntoSections( // they may be placed in the same SortedRun during buildLevelSortedRuns (which uses >= 0 // comparison). This dual behavior is intentional and documented in class comments. if (fieldComparator.compare(file.partitionStats().minValues(), sectionMaxBound) >= 0) { - sections.add(new Section(currentFiles, currentTotalSize, currentHasDefault)); - currentFiles = new ArrayList<>(); - currentTotalSize = 0; - currentFiles.add(file); - currentTotalSize += file.fileSize(); - currentHasDefault = defaultCompactionMap.containsKey(file); + sections.add( + new Section( + currentSectionFiles, + currentSectionTotalSize, + currentSectionHasCompactMeta)); + // start a new section + currentSectionFiles = new ArrayList<>(); + currentSectionTotalSize = 0; + currentSectionFiles.add(file); + currentSectionTotalSize += file.fileSize(); + currentSectionHasCompactMeta = ctx.isMarkedForDefaultCompaction(file); sectionMaxBound = file.partitionStats().maxValues(); } else { - currentFiles.add(file); - currentTotalSize += file.fileSize(); - if (!currentHasDefault && defaultCompactionMap.containsKey(file)) { - currentHasDefault = true; + currentSectionFiles.add(file); + currentSectionTotalSize += file.fileSize(); + if (!currentSectionHasCompactMeta && ctx.isMarkedForDefaultCompaction(file)) { + currentSectionHasCompactMeta = true; } if (fieldComparator.compare(file.partitionStats().maxValues(), sectionMaxBound) > 0) { @@ -652,7 +667,11 @@ static List
splitIntoSections( } } } - sections.add(new Section(currentFiles, currentTotalSize, currentHasDefault)); + sections.add( + new Section( + currentSectionFiles, + currentSectionTotalSize, + currentSectionHasCompactMeta)); return sections; } @@ -695,14 +714,14 @@ private static List
mergeSmallAdjacentSections( *
  • First overflow: The current section is split. The rewritable part is sorted and * rewritten. The remaining part is appended back to the sections queue for later * processing. - *
  • Subsequent overflows: If the section has files in defaultCompactionMap (needs default - * compaction), rewriteSubSegments is called to process it in smaller chunks. Otherwise, - * the section is skipped. + *
  • Subsequent overflows: If the section has files in needsDefaultCompaction (needs default + * compaction), defaultCompactSection is called to process it in smaller chunks. + * Otherwise, the section is skipped. * * *

    This design ensures that the budget only limits the aggressive sort rewrite, while still * allowing necessary cleanup operations (delete entry elimination, small file merge) through - * the rewriteSubSegments fallback path. + * the defaultCompactSection fallback path. */ private static void rewriteSections( List

    sections, @@ -715,11 +734,14 @@ private static void rewriteSections( long maxRewriteSize, @Nullable Integer manifestReadParallelism) throws Exception { - long processedSize = 0; - boolean reachedLimit = false; + // Total data size that has been sort-rewritten so far, used to enforce maxRewriteSize. + long currentRewrittenSize = 0; + boolean budgetExhausted = false; // Whether currentRewrittenSize reaches maxRewriteSize. for (int i = 0; i < sections.size(); i++) { Section section = sections.get(i); + + // A single-file section is always handled directly, regardless of the budget. if (section.files.size() == 1) { sortAndRewriteSection( section.files, @@ -731,55 +753,45 @@ private static void rewriteSections( continue; } - if (processedSize + section.totalSize <= maxRewriteSize) { - processedSize += section.totalSize; - sortAndRewriteSection( - section.files, - output, - sortNewFiles, - ctx, - manifestFile, - manifestReadParallelism); - } else if (!reachedLimit) { - long rewriteTotalSize = maxRewriteSize - processedSize; - processedSize += section.totalSize; - List rewriteFiles = new ArrayList<>(); - List remainingFiles = new ArrayList<>(); - long rewriteSize = 0; - long remainingSize = 0; - boolean remainingHasDefault = false; - - for (ManifestFileMeta file : section.files) { - if (rewriteSize + file.fileSize() <= rewriteTotalSize) { - rewriteFiles.add(file); - rewriteSize += file.fileSize(); - } else { - remainingFiles.add(file); - remainingSize += file.fileSize(); - if (ctx.defaultCompactionMap.containsKey(file)) { - remainingHasDefault = true; - } + // Phase 1: budget not yet exhausted -- perform aggressive sort rewrite. + if (!budgetExhausted) { + // Phase 1a: section fits within the remaining budget -- sort and rewrite it + // wholly. + if (currentRewrittenSize + section.totalSize <= maxRewriteSize) { + currentRewrittenSize += section.totalSize; + sortAndRewriteSection( + section.files, + output, + sortNewFiles, + ctx, + manifestFile, + manifestReadParallelism); + } else { + // Phase 1b: first overflow -- split the section at the budget boundary, + // rewrite the affordable head, and append the remaining tail back for later + // (Phase 2) handling. + long remainingBudget = maxRewriteSize - currentRewrittenSize; + currentRewrittenSize += section.totalSize; + Section remaining = + splitSectionAndRewriteHead( + section, + remainingBudget, + output, + sortNewFiles, + ctx, + manifestFile, + manifestReadParallelism); + if (remaining != null) { + // global ManifestMeta section order by sort key is not a required invariant + sections.add(remaining); } + budgetExhausted = true; } - - sortAndRewriteSection( - rewriteFiles, - output, - sortNewFiles, - ctx, - manifestFile, - manifestReadParallelism); - - if (!remainingFiles.isEmpty()) { - Section remainingSection = - new Section(remainingFiles, remainingSize, remainingHasDefault); - // global manifest file metas order by sort key is not a required invariant - sections.add(remainingSection); - } - reachedLimit = true; - } else if (section.hasDefaultCompactMeta) { - rewriteSubSegments( - section.files, + } else { + // Phase 2: budget already exhausted -- only do default compact, skip aggressive + // sort rewrite. + rewriteSectionBeyondBudget( + section, output, sortNewFiles, ctx, @@ -787,10 +799,82 @@ private static void rewriteSections( suggestedMetaSize, suggestedMinMetaCount, manifestReadParallelism); + } + } + } + + /** + * Split a section at the rewrite budget boundary: sort and rewrite the head part that fits + * within the remaining budget, and return the remaining tail as a new Section (or null if the + * whole section fits and no tail is left). + */ + private static Section splitSectionAndRewriteHead( + Section section, + long remainingBudget, + RewriteOutput output, + List sortNewFiles, + CompactionContext ctx, + ManifestFile manifestFile, + @Nullable Integer manifestReadParallelism) + throws Exception { + List headFiles = new ArrayList<>(); + List tailFiles = new ArrayList<>(); + long headSize = 0; + long tailSize = 0; + // Whether tail section has files in needsDefaultCompaction, if true, the section need to be + // rewritten. + boolean tailHasDefaultCompactMeta = false; + + for (ManifestFileMeta file : section.files) { + if (headSize + file.fileSize() <= remainingBudget) { + headFiles.add(file); + headSize += file.fileSize(); } else { - output.addAllUnchanged(section.files); + tailFiles.add(file); + tailSize += file.fileSize(); + if (ctx.isMarkedForDefaultCompaction(file)) { + tailHasDefaultCompactMeta = true; + } } } + + sortAndRewriteSection( + headFiles, output, sortNewFiles, ctx, manifestFile, manifestReadParallelism); + + if (tailFiles.isEmpty()) { + return null; + } + return new Section(tailFiles, tailSize, tailHasDefaultCompactMeta); + } + + /** + * Handle a section after the sort rewrite budget is exhausted. Sections that contain + * default-compaction files (small files / delete entries) still go through + * defaultCompactSection for necessary cleanup; otherwise they are kept unchanged. + */ + private static void rewriteSectionBeyondBudget( + Section section, + RewriteOutput output, + List sortNewFiles, + CompactionContext ctx, + ManifestFile manifestFile, + long suggestedMetaSize, + int suggestedMinMetaCount, + @Nullable Integer manifestReadParallelism) + throws Exception { + if (section.hasDefaultCompactMeta) { + defaultCompactSection( + section.files, + output, + sortNewFiles, + ctx, + manifestFile, + suggestedMetaSize, + suggestedMinMetaCount, + manifestReadParallelism); + } else { + output.addAllUnchanged(section.files); + } } /** @@ -798,8 +882,8 @@ private static void rewriteSections( * *

    Semantics difference from old minor merge: In the old ManifestFileMerger path, the * trailing candidates are kept unchanged when their count is below manifest.merge-min-count. In - * this sort path, rewriteSubSegments is triggered when defaultCompactionMap is non-empty, - * regardless of the manifest count. This is because files in defaultCompactionMap either: + * this sort path, defaultCompactSection is triggered when needsDefaultCompaction is non-empty, + * regardless of the manifest count. This is because files in needsDefaultCompaction either: * *

      *
    • Are small files needing consolidation @@ -810,7 +894,7 @@ private static void rewriteSections( * acting as a conservative gate to avoid unnecessary rewrite when there are no delete entries * and the tail is too small. */ - private static void rewriteSubSegments( + private static void defaultCompactSection( List section, RewriteOutput output, List sortNewFiles, @@ -820,36 +904,36 @@ private static void rewriteSubSegments( int suggestedMinMetaCount, @Nullable Integer manifestReadParallelism) throws Exception { - List subSegment = new ArrayList<>(); - long subSegmentSize = 0; + List candidates = new ArrayList<>(); + long candidatesSize = 0; for (ManifestFileMeta m : section) { - subSegmentSize += m.fileSize(); - subSegment.add(m); + candidatesSize += m.fileSize(); + candidates.add(m); - if (subSegmentSize >= suggestedMetaSize) { + if (candidatesSize >= suggestedMetaSize) { sortAndRewriteSection( - subSegment, + candidates, output, sortNewFiles, ctx, manifestFile, manifestReadParallelism); - subSegment.clear(); - subSegmentSize = 0; + candidates.clear(); + candidatesSize = 0; } } // Flush tail only if delete entries exist or file count >= minCount. - if (!subSegment.isEmpty()) { - if (!ctx.deleteEntries.isEmpty() || subSegment.size() >= suggestedMinMetaCount) { + if (!candidates.isEmpty()) { + if (!ctx.deleteEntries.isEmpty() || candidates.size() >= suggestedMinMetaCount) { sortAndRewriteSection( - subSegment, + candidates, output, sortNewFiles, ctx, manifestFile, manifestReadParallelism); } else { - output.addAllUnchanged(subSegment); + output.addAllUnchanged(candidates); } } } @@ -869,7 +953,8 @@ private static void sortAndRewriteSection( @Nullable Integer manifestReadParallelism) throws Exception { // Skip rewrite for single file not in delete-range. - if (section.size() == 1 && !ctx.defaultCompactionMap.getOrDefault(section.get(0), false)) { + if (section.size() == 1 + && !ctx.needsDefaultCompaction.getOrDefault(section.get(0), false)) { output.addUnchanged(section.get(0)); return; } From 2d63d8bd202ee1062108aecaf56a6fffb0b51a78 Mon Sep 17 00:00:00 2001 From: umi Date: Thu, 4 Jun 2026 16:05:53 +0800 Subject: [PATCH 2/4] rename --- .../paimon/operation/ManifestFileSorter.java | 111 +++++++++--------- 1 file changed, 57 insertions(+), 54 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java index 234f87d66eef..a442784bf7b7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java @@ -66,7 +66,7 @@ static class CompactionContext { final boolean fullCompaction; final RecordComparator fieldComparator; final Set deleteEntries; - final Map needsDefaultCompaction; + final Map needsUnsortedCompaction; final List levelRuns; final List pickedRuns; @@ -74,20 +74,20 @@ static class CompactionContext { boolean fullCompaction, RecordComparator fieldComparator, Set deleteEntries, - Map needsDefaultCompaction, + Map needsUnsortedCompaction, List levelRuns, List pickedRuns) { this.fullCompaction = fullCompaction; this.fieldComparator = fieldComparator; this.deleteEntries = deleteEntries; - this.needsDefaultCompaction = needsDefaultCompaction; + this.needsUnsortedCompaction = needsUnsortedCompaction; this.levelRuns = levelRuns; this.pickedRuns = pickedRuns; } - /** Check whether the given manifest file is marked for default compaction. */ - boolean isMarkedForDefaultCompaction(ManifestFileMeta file) { - return needsDefaultCompaction.containsKey(file); + /** Check whether the given manifest file is marked for unsorted compaction. */ + boolean isMarkedForUnsortedCompaction(ManifestFileMeta file) { + return needsUnsortedCompaction.containsKey(file); } } @@ -96,22 +96,22 @@ private static class ClassifyResult { final List lsmFiles; final Set deleteEntries; /** - * Manifest files that need default compaction. + * Manifest files that need unsorted compaction. * *

      Key: manifest file metadata * *

      Value: true if the file overlaps with delete partitions and fullCompaction is true * file */ - final Map needsDefaultCompaction; + final Map needsUnsortedCompaction; ClassifyResult( List lsmFiles, Set deleteEntries, - Map needsDefaultCompaction) { + Map needsUnsortedCompaction) { this.lsmFiles = lsmFiles; this.deleteEntries = deleteEntries; - this.needsDefaultCompaction = needsDefaultCompaction; + this.needsUnsortedCompaction = needsUnsortedCompaction; } } @@ -214,19 +214,19 @@ private static Optional> tryFullCompaction( List levelRuns = ctx.levelRuns; List pickedRuns = ctx.pickedRuns; - if (pickedRuns.isEmpty() && ctx.needsDefaultCompaction.isEmpty()) { + if (pickedRuns.isEmpty() && ctx.needsUnsortedCompaction.isEmpty()) { LOG.debug( - "Manifest sort full compact skipped: no runs picked and no defaultCompaction files."); + "Manifest sort full compact skipped: no runs picked and no unsortedCompaction files."); return Optional.empty(); } LOG.info( "Manifest sort full compact: input={} files, lsm={} runs, picked={} runs, " - + "defaultCompaction={} files.", + + "unsortedCompaction={} files.", input.size(), levelRuns.size(), pickedRuns.size(), - ctx.needsDefaultCompaction.size()); + ctx.needsUnsortedCompaction.size()); // Step 3: Collect reused files (not picked) and picked files Set pickedSet = new HashSet<>(pickedRuns); @@ -240,7 +240,7 @@ private static Optional> tryFullCompaction( for (ManifestAdjacentSortedRun run : pickedRuns) { pickedFiles.addAll(run.files()); } - pickedFiles.addAll(ctx.needsDefaultCompaction.keySet()); + pickedFiles.addAll(ctx.needsUnsortedCompaction.keySet()); // Step 4: Split into sections and merge small adjacent sections List

      sections = splitIntoSections(pickedFiles, ctx); @@ -305,19 +305,19 @@ private static List tryMinorCompaction( List levelRuns = ctx.levelRuns; List pickedRuns = ctx.pickedRuns; - if (pickedRuns.isEmpty() && ctx.needsDefaultCompaction.isEmpty()) { + if (pickedRuns.isEmpty() && ctx.needsUnsortedCompaction.isEmpty()) { LOG.debug( - "Manifest sort minor compact skipped: no runs picked and no defaultCompaction files."); + "Manifest sort minor compact skipped: no runs picked and no unsortedCompaction files."); return input; } LOG.info( "Manifest sort minor compact: input={} files, lsm={} runs, picked={} runs, " - + "defaultCompaction={} files.", + + "unsortedCompaction={} files.", input.size(), levelRuns.size(), pickedRuns.size(), - ctx.needsDefaultCompaction.size()); + ctx.needsUnsortedCompaction.size()); // Step 2: Build fileName -> index mapping and initialize 2D result Map fileNameToIndex = new HashMap<>(); @@ -344,7 +344,7 @@ private static List tryMinorCompaction( for (ManifestAdjacentSortedRun run : pickedRuns) { pickedFiles.addAll(run.files()); } - pickedFiles.addAll(ctx.needsDefaultCompaction.keySet()); + pickedFiles.addAll(ctx.needsUnsortedCompaction.keySet()); // Step 4: Compute index range int minIdx = Integer.MAX_VALUE; @@ -447,7 +447,7 @@ private static CompactionContext prepareCompaction( fullCompaction, fieldComparator, classifyResult.deleteEntries, - classifyResult.needsDefaultCompaction, + classifyResult.needsUnsortedCompaction, levelRuns, pickedRuns); } @@ -456,12 +456,12 @@ private static CompactionContext prepareCompaction( * Classify manifest files into default-compaction group and LSM group. * *

      Full compaction: small files and files overlapping delete partitions go into - * needsDefaultCompaction; the rest are returned as lsmFiles. + * needsUnsortedCompaction; the rest are returned as lsmFiles. * - *

      Non-full compaction: small files go to needsDefaultCompaction for minor-style merge; the + *

      Non-full compaction: small files go to needsUnsortedCompaction for minor-style merge; the * rest are returned as lsmFiles. * - * @return ClassifyResult containing lsmFiles, deleteEntries, and needsDefaultCompaction + * @return ClassifyResult containing lsmFiles, deleteEntries, and needsUnsortedCompaction */ private static ClassifyResult classifyManifests( List input, @@ -471,7 +471,7 @@ private static ClassifyResult classifyManifests( long suggestedMetaSize, @Nullable Integer manifestReadParallelism) { // Initialize classification containers and read delete entries - Map needsDefaultCompaction = new LinkedHashMap<>(); + Map needsUnsortedCompaction = new LinkedHashMap<>(); List lsmFiles = new LinkedList<>(input); Set classifiedDeleteEntries = Collections.emptySet(); PartitionPredicate predicate = null; @@ -507,11 +507,11 @@ private static ClassifyResult classifyManifests( file.partitionStats().nullCounts()); if (small || inDeleteRange) { iterator.remove(); - needsDefaultCompaction.put(file, inDeleteRange); + needsUnsortedCompaction.put(file, inDeleteRange); } } - return new ClassifyResult(lsmFiles, classifiedDeleteEntries, needsDefaultCompaction); + return new ClassifyResult(lsmFiles, classifiedDeleteEntries, needsUnsortedCompaction); } /** @@ -599,7 +599,7 @@ static List buildLevelSortedRuns( /** * Split picked files into sections. Files with overlapping sort-key intervals go into the same - * section. Each section is built with pre-computed totalSize and hasDefaultCompactMeta. + * section. Each section is built with pre-computed totalSize and hasUnsortedCompactMeta. */ static List

      splitIntoSections( List pickedFiles, CompactionContext ctx) { @@ -623,7 +623,7 @@ static List
      splitIntoSections( currentSectionFiles.add(first); currentSectionTotalSize += first.fileSize(); - boolean currentSectionHasCompactMeta = ctx.isMarkedForDefaultCompaction(first); + boolean currentSectionHasUnsortedCompactMeta = ctx.isMarkedForUnsortedCompaction(first); BinaryRow sectionMaxBound = first.partitionStats().maxValues(); for (int i = 1; i < pickedFiles.size(); i++) { @@ -647,19 +647,20 @@ static List
      splitIntoSections( new Section( currentSectionFiles, currentSectionTotalSize, - currentSectionHasCompactMeta)); + currentSectionHasUnsortedCompactMeta)); // start a new section currentSectionFiles = new ArrayList<>(); currentSectionTotalSize = 0; currentSectionFiles.add(file); currentSectionTotalSize += file.fileSize(); - currentSectionHasCompactMeta = ctx.isMarkedForDefaultCompaction(file); + currentSectionHasUnsortedCompactMeta = ctx.isMarkedForUnsortedCompaction(file); sectionMaxBound = file.partitionStats().maxValues(); } else { currentSectionFiles.add(file); currentSectionTotalSize += file.fileSize(); - if (!currentSectionHasCompactMeta && ctx.isMarkedForDefaultCompaction(file)) { - currentSectionHasCompactMeta = true; + if (!currentSectionHasUnsortedCompactMeta + && ctx.isMarkedForUnsortedCompaction(file)) { + currentSectionHasUnsortedCompactMeta = true; } if (fieldComparator.compare(file.partitionStats().maxValues(), sectionMaxBound) > 0) { @@ -671,7 +672,7 @@ static List
      splitIntoSections( new Section( currentSectionFiles, currentSectionTotalSize, - currentSectionHasCompactMeta)); + currentSectionHasUnsortedCompactMeta)); return sections; } @@ -714,14 +715,14 @@ private static List
      mergeSmallAdjacentSections( *
    • First overflow: The current section is split. The rewritable part is sorted and * rewritten. The remaining part is appended back to the sections queue for later * processing. - *
    • Subsequent overflows: If the section has files in needsDefaultCompaction (needs default - * compaction), defaultCompactSection is called to process it in smaller chunks. + *
    • Subsequent overflows: If the section has files in needsUnsortedCompaction (needs + * unsorted compaction), unsortedCompactSection is called to process it in smaller chunks. * Otherwise, the section is skipped. *
    * *

    This design ensures that the budget only limits the aggressive sort rewrite, while still * allowing necessary cleanup operations (delete entry elimination, small file merge) through - * the defaultCompactSection fallback path. + * the unsortedCompactSection fallback path. */ private static void rewriteSections( List

    sections, @@ -788,7 +789,7 @@ private static void rewriteSections( budgetExhausted = true; } } else { - // Phase 2: budget already exhausted -- only do default compact, skip aggressive + // Phase 2: budget already exhausted -- only do unsorted compact, skip aggressive // sort rewrite. rewriteSectionBeyondBudget( section, @@ -821,9 +822,10 @@ private static Section splitSectionAndRewriteHead( List tailFiles = new ArrayList<>(); long headSize = 0; long tailSize = 0; - // Whether tail section has files in needsDefaultCompaction, if true, the section need to be + // Whether tail section has files in needsUnsortedCompaction, if true, the section need to + // be // rewritten. - boolean tailHasDefaultCompactMeta = false; + boolean tailHasUnsortedCompactMeta = false; for (ManifestFileMeta file : section.files) { if (headSize + file.fileSize() <= remainingBudget) { @@ -832,8 +834,8 @@ private static Section splitSectionAndRewriteHead( } else { tailFiles.add(file); tailSize += file.fileSize(); - if (ctx.isMarkedForDefaultCompaction(file)) { - tailHasDefaultCompactMeta = true; + if (ctx.isMarkedForUnsortedCompaction(file)) { + tailHasUnsortedCompactMeta = true; } } } @@ -844,13 +846,13 @@ private static Section splitSectionAndRewriteHead( if (tailFiles.isEmpty()) { return null; } - return new Section(tailFiles, tailSize, tailHasDefaultCompactMeta); + return new Section(tailFiles, tailSize, tailHasUnsortedCompactMeta); } /** * Handle a section after the sort rewrite budget is exhausted. Sections that contain * default-compaction files (small files / delete entries) still go through - * defaultCompactSection for necessary cleanup; otherwise they are kept unchanged. + * unsortedCompactSection for necessary cleanup; otherwise they are kept unchanged. */ private static void rewriteSectionBeyondBudget( Section section, @@ -862,8 +864,8 @@ private static void rewriteSectionBeyondBudget( int suggestedMinMetaCount, @Nullable Integer manifestReadParallelism) throws Exception { - if (section.hasDefaultCompactMeta) { - defaultCompactSection( + if (section.hasUnsortedCompactMeta) { + unsortedCompactSection( section.files, output, sortNewFiles, @@ -882,8 +884,9 @@ private static void rewriteSectionBeyondBudget( * *

    Semantics difference from old minor merge: In the old ManifestFileMerger path, the * trailing candidates are kept unchanged when their count is below manifest.merge-min-count. In - * this sort path, defaultCompactSection is triggered when needsDefaultCompaction is non-empty, - * regardless of the manifest count. This is because files in needsDefaultCompaction either: + * this sort path, unsortedCompactSection is triggered when needsUnsortedCompaction is + * non-empty, regardless of the manifest count. This is because files in needsUnsortedCompaction + * either: * *

      *
    • Are small files needing consolidation @@ -894,7 +897,7 @@ private static void rewriteSectionBeyondBudget( * acting as a conservative gate to avoid unnecessary rewrite when there are no delete entries * and the tail is too small. */ - private static void defaultCompactSection( + private static void unsortedCompactSection( List section, RewriteOutput output, List sortNewFiles, @@ -954,7 +957,7 @@ private static void sortAndRewriteSection( throws Exception { // Skip rewrite for single file not in delete-range. if (section.size() == 1 - && !ctx.needsDefaultCompaction.getOrDefault(section.get(0), false)) { + && !ctx.needsUnsortedCompaction.getOrDefault(section.get(0), false)) { output.addUnchanged(section.get(0)); return; } @@ -1213,12 +1216,12 @@ public void addDeleteFiles(List files) { static class Section { final List files; final long totalSize; - final boolean hasDefaultCompactMeta; + final boolean hasUnsortedCompactMeta; - Section(List files, long totalSize, boolean hasDefaultCompactMeta) { + Section(List files, long totalSize, boolean hasUnsortedCompactMeta) { this.files = files; this.totalSize = totalSize; - this.hasDefaultCompactMeta = hasDefaultCompactMeta; + this.hasUnsortedCompactMeta = hasUnsortedCompactMeta; } /** Create a merged section from two sections. */ @@ -1228,7 +1231,7 @@ static Section merge(Section a, Section b) { return new Section( merged, a.totalSize + b.totalSize, - a.hasDefaultCompactMeta || b.hasDefaultCompactMeta); + a.hasUnsortedCompactMeta || b.hasUnsortedCompactMeta); } } } From 1bc850294a5d85944f939f08f43db3b744684455 Mon Sep 17 00:00:00 2001 From: umi Date: Thu, 4 Jun 2026 16:12:36 +0800 Subject: [PATCH 3/4] rewriteSection --- .../paimon/operation/ManifestFileSorter.java | 31 +++++++++---------- .../paimon/manifest/ManifestFileMetaTest.java | 2 +- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java index a442784bf7b7..ddebbd8e24f8 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java @@ -172,7 +172,7 @@ static List trySortCompaction( /** * Full compaction path: totalDeltaFileSize >= sizeTrigger. * - *

      Does not build index mapping. sortAndRewriteSection writes all entries (ADD+DELETE merged) + *

      Does not build index mapping. rewriteSection writes all entries (ADD+DELETE merged) * together without separating them. */ private static Optional> tryFullCompaction( @@ -274,8 +274,8 @@ private static Optional> tryFullCompaction( /** * Minor compaction path: totalDeltaFileSize < sizeTrigger. * - *

      Builds index mapping to preserve original positions. sortAndRewriteSection separates ADD - * and DELETE entries, placing ADD at result[minIdx] and DELETE at result[maxIdx]. + *

      Builds index mapping to preserve original positions. rewriteSection separates ADD and + * DELETE entries, placing ADD at result[minIdx] and DELETE at result[maxIdx]. */ private static List tryMinorCompaction( List input, @@ -744,7 +744,7 @@ private static void rewriteSections( // A single-file section is always handled directly, regardless of the budget. if (section.files.size() == 1) { - sortAndRewriteSection( + rewriteSection( section.files, output, sortNewFiles, @@ -760,7 +760,7 @@ private static void rewriteSections( // wholly. if (currentRewrittenSize + section.totalSize <= maxRewriteSize) { currentRewrittenSize += section.totalSize; - sortAndRewriteSection( + rewriteSection( section.files, output, sortNewFiles, @@ -840,8 +840,7 @@ private static Section splitSectionAndRewriteHead( } } - sortAndRewriteSection( - headFiles, output, sortNewFiles, ctx, manifestFile, manifestReadParallelism); + rewriteSection(headFiles, output, sortNewFiles, ctx, manifestFile, manifestReadParallelism); if (tailFiles.isEmpty()) { return null; @@ -914,7 +913,7 @@ private static void unsortedCompactSection( candidates.add(m); if (candidatesSize >= suggestedMetaSize) { - sortAndRewriteSection( + rewriteSection( candidates, output, sortNewFiles, @@ -928,7 +927,7 @@ private static void unsortedCompactSection( // Flush tail only if delete entries exist or file count >= minCount. if (!candidates.isEmpty()) { if (!ctx.deleteEntries.isEmpty() || candidates.size() >= suggestedMinMetaCount) { - sortAndRewriteSection( + rewriteSection( candidates, output, sortNewFiles, @@ -942,12 +941,12 @@ private static void unsortedCompactSection( } /** - * Sort and rewrite a section. Dispatches to full or minor compact path. + * Rewrite a section. Dispatches to full or minor compact path. * *

      sortNewFiles is the same reference as newFilesForAbort, ensuring newly written files are * cleaned up on exception by the caller's catch block. */ - private static void sortAndRewriteSection( + private static void rewriteSection( List section, RewriteOutput output, List sortNewFiles, @@ -963,11 +962,9 @@ private static void sortAndRewriteSection( } if (ctx.fullCompaction) { - sortAndRewriteFull( - section, output, sortNewFiles, ctx, manifestFile, manifestReadParallelism); + rewriteFull(section, output, sortNewFiles, ctx, manifestFile, manifestReadParallelism); } else { - sortAndRewriteMinor( - section, output, sortNewFiles, ctx, manifestFile, manifestReadParallelism); + rewriteMinor(section, output, sortNewFiles, ctx, manifestFile, manifestReadParallelism); } } @@ -975,7 +972,7 @@ private static void sortAndRewriteSection( * Full compaction path: read all surviving entries (ADD merged with DELETE), sort them * together, and write to output as a single sorted stream. */ - private static void sortAndRewriteFull( + private static void rewriteFull( List section, RewriteOutput output, List sortNewFiles, @@ -1022,7 +1019,7 @@ private static void sortAndRewriteFull( * entries into ADD and DELETE within each file, returning a Pair. Results are merged in the * main thread. */ - private static void sortAndRewriteMinor( + private static void rewriteMinor( List section, RewriteOutput output, List sortNewFiles, diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java index 75a1ab0a84df..2b91824d9afd 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java @@ -964,7 +964,7 @@ public void testManifestSortWithOverlappingPartitions() { * Test that sort rewrite correctly eliminates DELETE entries and their corresponding ADD * entries. The key condition is that totalDeltaFileSize must reach manifestFullCompactionSize * to trigger the full compaction path inside trySortRewrite, which reads deleteEntries and - * passes them to sortAndRewriteSection for elimination. + * passes them to rewriteSection for elimination. * *

      Design: * From 77c464493b07f59b8203badb99ecd2384cd0ad81 Mon Sep 17 00:00:00 2001 From: umi Date: Thu, 4 Jun 2026 18:33:55 +0800 Subject: [PATCH 4/4] fix --- .../paimon/operation/ManifestFileSorter.java | 80 ++++++++++--------- 1 file changed, 43 insertions(+), 37 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java index ddebbd8e24f8..bbdf5e14bc4d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java @@ -66,7 +66,16 @@ static class CompactionContext { final boolean fullCompaction; final RecordComparator fieldComparator; final Set deleteEntries; - final Map needsUnsortedCompaction; + /** + * Manifest files that need unsorted compaction. + * + *

      Key: manifest file metadata + * + *

      Value: true if fullCompaction is true and the file overlaps with delete partitions. It + * means the file needs to eliminate delete entries file + */ + final Map compactWithoutSort; + final List levelRuns; final List pickedRuns; @@ -74,20 +83,20 @@ static class CompactionContext { boolean fullCompaction, RecordComparator fieldComparator, Set deleteEntries, - Map needsUnsortedCompaction, + Map compactWithoutSort, List levelRuns, List pickedRuns) { this.fullCompaction = fullCompaction; this.fieldComparator = fieldComparator; this.deleteEntries = deleteEntries; - this.needsUnsortedCompaction = needsUnsortedCompaction; + this.compactWithoutSort = compactWithoutSort; this.levelRuns = levelRuns; this.pickedRuns = pickedRuns; } /** Check whether the given manifest file is marked for unsorted compaction. */ boolean isMarkedForUnsortedCompaction(ManifestFileMeta file) { - return needsUnsortedCompaction.containsKey(file); + return compactWithoutSort.containsKey(file); } } @@ -100,18 +109,18 @@ private static class ClassifyResult { * *

      Key: manifest file metadata * - *

      Value: true if the file overlaps with delete partitions and fullCompaction is true - * file + *

      Value: true if fullCompaction is true and the file overlaps with delete partitions. It + * means the file needs to eliminate delete entries file */ - final Map needsUnsortedCompaction; + final Map compactWithoutSort; ClassifyResult( List lsmFiles, Set deleteEntries, - Map needsUnsortedCompaction) { + Map compactWithoutSort) { this.lsmFiles = lsmFiles; this.deleteEntries = deleteEntries; - this.needsUnsortedCompaction = needsUnsortedCompaction; + this.compactWithoutSort = compactWithoutSort; } } @@ -214,19 +223,19 @@ private static Optional> tryFullCompaction( List levelRuns = ctx.levelRuns; List pickedRuns = ctx.pickedRuns; - if (pickedRuns.isEmpty() && ctx.needsUnsortedCompaction.isEmpty()) { + if (pickedRuns.isEmpty() && ctx.compactWithoutSort.isEmpty()) { LOG.debug( - "Manifest sort full compact skipped: no runs picked and no unsortedCompaction files."); + "Manifest sort full compact skipped: no runs picked and no compactWithoutSort files."); return Optional.empty(); } LOG.info( "Manifest sort full compact: input={} files, lsm={} runs, picked={} runs, " - + "unsortedCompaction={} files.", + + "compactWithoutSort={} files.", input.size(), levelRuns.size(), pickedRuns.size(), - ctx.needsUnsortedCompaction.size()); + ctx.compactWithoutSort.size()); // Step 3: Collect reused files (not picked) and picked files Set pickedSet = new HashSet<>(pickedRuns); @@ -240,7 +249,7 @@ private static Optional> tryFullCompaction( for (ManifestAdjacentSortedRun run : pickedRuns) { pickedFiles.addAll(run.files()); } - pickedFiles.addAll(ctx.needsUnsortedCompaction.keySet()); + pickedFiles.addAll(ctx.compactWithoutSort.keySet()); // Step 4: Split into sections and merge small adjacent sections List

      sections = splitIntoSections(pickedFiles, ctx); @@ -305,19 +314,19 @@ private static List tryMinorCompaction( List levelRuns = ctx.levelRuns; List pickedRuns = ctx.pickedRuns; - if (pickedRuns.isEmpty() && ctx.needsUnsortedCompaction.isEmpty()) { + if (pickedRuns.isEmpty() && ctx.compactWithoutSort.isEmpty()) { LOG.debug( - "Manifest sort minor compact skipped: no runs picked and no unsortedCompaction files."); + "Manifest sort minor compact skipped: no runs picked and no compactWithoutSort files."); return input; } LOG.info( "Manifest sort minor compact: input={} files, lsm={} runs, picked={} runs, " - + "unsortedCompaction={} files.", + + "compactWithoutSort={} files.", input.size(), levelRuns.size(), pickedRuns.size(), - ctx.needsUnsortedCompaction.size()); + ctx.compactWithoutSort.size()); // Step 2: Build fileName -> index mapping and initialize 2D result Map fileNameToIndex = new HashMap<>(); @@ -344,7 +353,7 @@ private static List tryMinorCompaction( for (ManifestAdjacentSortedRun run : pickedRuns) { pickedFiles.addAll(run.files()); } - pickedFiles.addAll(ctx.needsUnsortedCompaction.keySet()); + pickedFiles.addAll(ctx.compactWithoutSort.keySet()); // Step 4: Compute index range int minIdx = Integer.MAX_VALUE; @@ -447,7 +456,7 @@ private static CompactionContext prepareCompaction( fullCompaction, fieldComparator, classifyResult.deleteEntries, - classifyResult.needsUnsortedCompaction, + classifyResult.compactWithoutSort, levelRuns, pickedRuns); } @@ -456,12 +465,12 @@ private static CompactionContext prepareCompaction( * Classify manifest files into default-compaction group and LSM group. * *

      Full compaction: small files and files overlapping delete partitions go into - * needsUnsortedCompaction; the rest are returned as lsmFiles. + * compactWithoutSort; the rest are returned as lsmFiles. * - *

      Non-full compaction: small files go to needsUnsortedCompaction for minor-style merge; the - * rest are returned as lsmFiles. + *

      Non-full compaction: small files go to compactWithoutSort for minor-style merge; the rest + * are returned as lsmFiles. * - * @return ClassifyResult containing lsmFiles, deleteEntries, and needsUnsortedCompaction + * @return ClassifyResult containing lsmFiles, deleteEntries, and compactWithoutSort */ private static ClassifyResult classifyManifests( List input, @@ -471,7 +480,7 @@ private static ClassifyResult classifyManifests( long suggestedMetaSize, @Nullable Integer manifestReadParallelism) { // Initialize classification containers and read delete entries - Map needsUnsortedCompaction = new LinkedHashMap<>(); + Map compactWithoutSort = new LinkedHashMap<>(); List lsmFiles = new LinkedList<>(input); Set classifiedDeleteEntries = Collections.emptySet(); PartitionPredicate predicate = null; @@ -507,11 +516,11 @@ private static ClassifyResult classifyManifests( file.partitionStats().nullCounts()); if (small || inDeleteRange) { iterator.remove(); - needsUnsortedCompaction.put(file, inDeleteRange); + compactWithoutSort.put(file, inDeleteRange); } } - return new ClassifyResult(lsmFiles, classifiedDeleteEntries, needsUnsortedCompaction); + return new ClassifyResult(lsmFiles, classifiedDeleteEntries, compactWithoutSort); } /** @@ -715,8 +724,8 @@ private static List

      mergeSmallAdjacentSections( *
    • First overflow: The current section is split. The rewritable part is sorted and * rewritten. The remaining part is appended back to the sections queue for later * processing. - *
    • Subsequent overflows: If the section has files in needsUnsortedCompaction (needs - * unsorted compaction), unsortedCompactSection is called to process it in smaller chunks. + *
    • Subsequent overflows: If the section has files in compactWithoutSort (needs unsorted + * compaction), unsortedCompactSection is called to process it in smaller chunks. * Otherwise, the section is skipped. *
    * @@ -822,9 +831,8 @@ private static Section splitSectionAndRewriteHead( List tailFiles = new ArrayList<>(); long headSize = 0; long tailSize = 0; - // Whether tail section has files in needsUnsortedCompaction, if true, the section need to - // be - // rewritten. + // Whether tail section has files in compactWithoutSort, if true, the section need to + // be rewritten. boolean tailHasUnsortedCompactMeta = false; for (ManifestFileMeta file : section.files) { @@ -883,9 +891,8 @@ private static void rewriteSectionBeyondBudget( * *

    Semantics difference from old minor merge: In the old ManifestFileMerger path, the * trailing candidates are kept unchanged when their count is below manifest.merge-min-count. In - * this sort path, unsortedCompactSection is triggered when needsUnsortedCompaction is - * non-empty, regardless of the manifest count. This is because files in needsUnsortedCompaction - * either: + * this sort path, unsortedCompactSection is triggered when compactWithoutSort is non-empty, + * regardless of the manifest count. This is because files in compactWithoutSort either: * *

      *
    • Are small files needing consolidation @@ -955,8 +962,7 @@ private static void rewriteSection( @Nullable Integer manifestReadParallelism) throws Exception { // Skip rewrite for single file not in delete-range. - if (section.size() == 1 - && !ctx.needsUnsortedCompaction.getOrDefault(section.get(0), false)) { + if (section.size() == 1 && !ctx.compactWithoutSort.getOrDefault(section.get(0), false)) { output.addUnchanged(section.get(0)); return; }