From b798ae0c7f22cb3906040baed34f442c0a070d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Fri, 29 May 2026 14:42:13 +0800 Subject: [PATCH 01/19] [flink-action][server][client] add orphan files cleanup action for remote storage --- .../java/org/apache/fluss/fs/FileStatus.java | 14 + .../fluss/fs/local/LocalFileStatus.java | 5 + .../org/apache/fluss/utils/FlussPaths.java | 2 +- .../fluss/fs/hdfs/HadoopFileStatus.java | 5 + fluss-flink/fluss-flink-action/pom.xml | 81 ++ .../action/FlussFlinkActionEntrypoint.java | 40 + .../org/apache/fluss/flink/action/Action.java | 31 + .../fluss/flink/action/ActionFactory.java | 43 + .../fluss/flink/action/ActionLoader.java | 83 ++ .../flink/action/orphan/OrphanCleanUtils.java | 151 +++ .../action/orphan/OrphanFilesCleanAction.java | 75 ++ .../orphan/OrphanFilesCleanActionFactory.java | 77 ++ .../action/orphan/RpcErrorClassifier.java | 87 ++ .../action/orphan/audit/AuditLogger.java | 203 +++ .../orphan/build/ActiveRefsFetcher.java | 357 +++++ .../orphan/build/KvActiveRefsFetchResult.java | 92 ++ .../build/LogActiveRefsFetchResult.java | 156 +++ .../orphan/build/MaxKnownIdsTracker.java | 62 + .../action/orphan/build/RpcListStatus.java | 58 + .../orphan/config/OrphanCleanConfig.java | 336 +++++ .../flink/action/orphan/fs/SafeDeleter.java | 121 ++ .../action/orphan/job/BucketCleanTask.java | 103 ++ .../action/orphan/job/BucketCleaner.java | 142 ++ .../flink/action/orphan/job/CleanStats.java | 89 ++ .../flink/action/orphan/job/CleanTask.java | 30 + .../action/orphan/job/EmptyDirSweeper.java | 166 +++ .../action/orphan/job/OrphanDirCleanTask.java | 59 + .../orphan/job/OrphanFilesCleanJob.java | 109 ++ .../orphan/job/ScanAndCleanFunction.java | 222 ++++ .../orphan/job/ScopeEnumeratorFunction.java | 572 ++++++++ .../orphan/job/StatsAggregateOperator.java | 112 ++ .../action/orphan/rule/BucketActiveRefs.java | 84 ++ .../flink/action/orphan/rule/Decision.java | 42 + .../flink/action/orphan/rule/FileMeta.java | 48 + .../flink/action/orphan/rule/FileRule.java | 38 + .../action/orphan/rule/KvSharedSstRule.java | 53 + .../orphan/rule/KvSnapshotFileRule.java | 96 ++ .../action/orphan/rule/LogManifestRule.java | 80 ++ .../action/orphan/rule/LogSegmentRule.java | 98 ++ .../action/orphan/rule/OrphanDirDetector.java | 110 ++ .../action/orphan/rule/RuleDispatcher.java | 82 ++ .../flink/action/orphan/rule/RuleId.java | 46 + ...rg.apache.fluss.flink.action.ActionFactory | 19 + .../action/orphan/OrphanFilesCleanITCase.java | 1157 +++++++++++++++++ .../action/orphan/RpcErrorClassifierTest.java | 78 ++ .../orphan/build/ActiveRefsFetcherTest.java | 428 ++++++ .../orphan/build/MaxKnownIdsTrackerTest.java | 58 + .../orphan/config/OrphanCleanConfigTest.java | 186 +++ .../action/orphan/fs/SafeDeleterTest.java | 128 ++ .../orphan/job/EmptyDirSweeperTest.java | 86 ++ .../orphan/rule/KvSharedSstRuleTest.java | 80 ++ .../orphan/rule/KvSnapshotFileRuleTest.java | 120 ++ .../orphan/rule/LogManifestRuleTest.java | 114 ++ .../orphan/rule/LogSegmentRuleTest.java | 115 ++ .../orphan/rule/OrphanDirDetectorTest.java | 85 ++ .../orphan/rule/RuleDispatcherTest.java | 66 + .../sink/testutils/TestAdminAdapter.java | 14 + fluss-flink/pom.xml | 1 + .../rpc/gateway/AdminReadOnlyGateway.java | 25 + .../rpc/TestingTabletGatewayService.java | 16 + .../fluss/server/tablet/TabletService.java | 28 + .../tablet/TestTabletServerGateway.java | 16 + 62 files changed, 7279 insertions(+), 1 deletion(-) create mode 100644 fluss-flink/fluss-flink-action/pom.xml create mode 100644 fluss-flink/fluss-flink-action/src/main/java/org/apache/fluss/flink/action/FlussFlinkActionEntrypoint.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/Action.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/ActionFactory.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/ActionLoader.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanCleanUtils.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanAction.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanActionFactory.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/RpcErrorClassifier.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/audit/AuditLogger.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcher.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/KvActiveRefsFetchResult.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/LogActiveRefsFetchResult.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/MaxKnownIdsTracker.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/RpcListStatus.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfig.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleter.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleanTask.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/CleanStats.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/CleanTask.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeper.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanDirCleanTask.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/BucketActiveRefs.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/Decision.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/FileMeta.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/FileRule.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/KvSharedSstRule.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/KvSnapshotFileRule.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/LogManifestRule.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/LogSegmentRule.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/OrphanDirDetector.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/RuleDispatcher.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/RuleId.java create mode 100644 fluss-flink/fluss-flink-common/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/RpcErrorClassifierTest.java create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcherTest.java create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/MaxKnownIdsTrackerTest.java create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfigTest.java create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleterTest.java create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeperTest.java create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/KvSharedSstRuleTest.java create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/KvSnapshotFileRuleTest.java create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/LogManifestRuleTest.java create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/LogSegmentRuleTest.java create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/OrphanDirDetectorTest.java create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/RuleDispatcherTest.java diff --git a/fluss-common/src/main/java/org/apache/fluss/fs/FileStatus.java b/fluss-common/src/main/java/org/apache/fluss/fs/FileStatus.java index ad5708e3e9..74b51571ee 100644 --- a/fluss-common/src/main/java/org/apache/fluss/fs/FileStatus.java +++ b/fluss-common/src/main/java/org/apache/fluss/fs/FileStatus.java @@ -46,4 +46,18 @@ public interface FileStatus { * @return the corresponding Path to the FileStatus */ FsPath getPath(); + + /** + * Returns the modification time of the file in milliseconds since the epoch. + * + *

The default implementation returns {@link Long#MAX_VALUE}, which is interpreted by + * time-based filters (e.g. orphan-files cleanup) as "always fresh" - effectively a fail-closed + * default that prevents deletion when modification time is unavailable. File system + * implementations that can expose modification time SHOULD override this. + * + * @return the modification time in epoch millis, or {@link Long#MAX_VALUE} when unavailable + */ + default long getModificationTime() { + return Long.MAX_VALUE; + } } diff --git a/fluss-common/src/main/java/org/apache/fluss/fs/local/LocalFileStatus.java b/fluss-common/src/main/java/org/apache/fluss/fs/local/LocalFileStatus.java index 09184a9756..b8b04aa63b 100644 --- a/fluss-common/src/main/java/org/apache/fluss/fs/local/LocalFileStatus.java +++ b/fluss-common/src/main/java/org/apache/fluss/fs/local/LocalFileStatus.java @@ -67,6 +67,11 @@ public FsPath getPath() { return this.path; } + @Override + public long getModificationTime() { + return this.file.lastModified(); + } + public File getFile() { return this.file; } diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java b/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java index 9a0659f180..1c75663ba3 100644 --- a/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java +++ b/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java @@ -74,7 +74,7 @@ public class FlussPaths { public static final String REMOTE_LOG_DIR_NAME = "log"; /** The directory name for storing metadata files (e.g., manifest) for a log tablet. */ - private static final String REMOTE_LOG_METADATA_DIR_NAME = "metadata"; + public static final String REMOTE_LOG_METADATA_DIR_NAME = "metadata"; /** Suffix of a manifest file. */ private static final String REMOTE_LOG_MANIFEST_FILE_SUFFIX = ".manifest"; diff --git a/fluss-filesystems/fluss-fs-hadoop/src/main/java/org/apache/fluss/fs/hdfs/HadoopFileStatus.java b/fluss-filesystems/fluss-fs-hadoop/src/main/java/org/apache/fluss/fs/hdfs/HadoopFileStatus.java index f54033a693..47c9febcfe 100644 --- a/fluss-filesystems/fluss-fs-hadoop/src/main/java/org/apache/fluss/fs/hdfs/HadoopFileStatus.java +++ b/fluss-filesystems/fluss-fs-hadoop/src/main/java/org/apache/fluss/fs/hdfs/HadoopFileStatus.java @@ -52,6 +52,11 @@ public boolean isDir() { return fileStatus.isDirectory(); } + @Override + public long getModificationTime() { + return fileStatus.getModificationTime(); + } + // ------------------------------------------------------------------------ /** diff --git a/fluss-flink/fluss-flink-action/pom.xml b/fluss-flink/fluss-flink-action/pom.xml new file mode 100644 index 0000000000..7512bcd0b7 --- /dev/null +++ b/fluss-flink/fluss-flink-action/pom.xml @@ -0,0 +1,81 @@ + + + + + 4.0.0 + + org.apache.fluss + fluss-flink + 1.0-SNAPSHOT + + + jar + + fluss-flink-action + Fluss : Flink : Action + + + 1.20.3 + + + + + org.apache.fluss + fluss-flink-common + ${project.version} + + + + org.apache.flink + flink-streaming-java + ${flink.minor.version} + provided + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-fluss + package + + shade + + + + + org.apache.fluss.flink.action.FlussFlinkActionEntrypoint + + + + + + + + + + + diff --git a/fluss-flink/fluss-flink-action/src/main/java/org/apache/fluss/flink/action/FlussFlinkActionEntrypoint.java b/fluss-flink/fluss-flink-action/src/main/java/org/apache/fluss/flink/action/FlussFlinkActionEntrypoint.java new file mode 100644 index 0000000000..c83ea3b304 --- /dev/null +++ b/fluss-flink/fluss-flink-action/src/main/java/org/apache/fluss/flink/action/FlussFlinkActionEntrypoint.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action; + +import java.util.Optional; + +/** Main entrypoint for the Fluss Flink action jar. Delegates to {@link ActionLoader}. */ +public class FlussFlinkActionEntrypoint { + + public static void main(String[] args) throws Exception { + Optional action; + try { + action = ActionLoader.createAction(args); + } catch (IllegalArgumentException e) { + System.err.println(e.getMessage()); + System.exit(1); + return; + } + if (!action.isPresent()) { + return; + } + action.get().build(); + action.get().run(); + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/Action.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/Action.java new file mode 100644 index 0000000000..98af1da48a --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/Action.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action; + +import org.apache.fluss.annotation.Internal; + +/** Pluggable Flink action invoked from CLI via {@link FlussFlinkActionEntrypoint}. */ +@Internal +public interface Action { + + /** Optional setup hook called once before {@link #run()}. */ + default void build() throws Exception {} + + /** Execute the action. */ + void run() throws Exception; +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/ActionFactory.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/ActionFactory.java new file mode 100644 index 0000000000..da90751709 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/ActionFactory.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action; + +import org.apache.fluss.annotation.Internal; + +import org.apache.flink.api.java.utils.MultipleParameterTool; + +import java.util.Optional; + +/** SPI for {@link Action} factories, registered via JDK {@link java.util.ServiceLoader}. */ +@Internal +public interface ActionFactory { + + /** + * Identifier matched against the first CLI argument after lowercasing and replacing {@code -} + * with {@code _}. + */ + String identifier(); + + /** Construct the action from parsed CLI parameters. Empty when {@code --help} is requested. */ + Optional create(MultipleParameterTool params); + + /** Help text printed when {@code --help} is passed. */ + default String help() { + return ""; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/ActionLoader.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/ActionLoader.java new file mode 100644 index 0000000000..0b51915ea1 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/ActionLoader.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action; + +import org.apache.fluss.annotation.Internal; + +import org.apache.flink.api.java.utils.MultipleParameterTool; + +import java.util.Arrays; +import java.util.Optional; +import java.util.ServiceLoader; + +/** + * Discovers {@link ActionFactory} implementations via {@link ServiceLoader} and dispatches CLI + * arguments to the appropriate {@link Action}. + */ +@Internal +public final class ActionLoader { + + private ActionLoader() {} + + /** + * Resolve and create an action from CLI arguments. + * + *

Returns {@link Optional#empty()} when no arguments are provided or when {@code --help} is + * requested. Throws {@link IllegalArgumentException} when the requested identifier does not + * resolve to a known factory. + */ + public static Optional createAction(String[] args) { + if (args.length < 1) { + printDefaultHelp(); + return Optional.empty(); + } + String name = args[0].toLowerCase().replace('-', '_'); + ActionFactory factory = + findFactory(name) + .orElseThrow( + () -> + new IllegalArgumentException( + "Unknown action: " + + args[0] + + ". Run with --help for available actions.")); + String[] remaining = Arrays.copyOfRange(args, 1, args.length); + MultipleParameterTool params = MultipleParameterTool.fromArgs(remaining); + if (params.has("help")) { + System.out.println(factory.help()); + return Optional.empty(); + } + return factory.create(params); + } + + private static Optional findFactory(String identifier) { + for (ActionFactory f : ServiceLoader.load(ActionFactory.class)) { + if (f.identifier().equals(identifier)) { + return Optional.of(f); + } + } + return Optional.empty(); + } + + private static void printDefaultHelp() { + System.out.println("Usage: [options]"); + System.out.println("Available actions:"); + for (ActionFactory f : ServiceLoader.load(ActionFactory.class)) { + System.out.println(" " + f.identifier()); + } + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanCleanUtils.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanCleanUtils.java new file mode 100644 index 0000000000..acf2dc7214 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanCleanUtils.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.cluster.ConfigEntry; +import org.apache.fluss.fs.FileStatus; +import org.apache.fluss.fs.FileSystem; +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.PhysicalTablePath; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Shared utility methods for the orphan files cleanup action. */ +@Internal +public final class OrphanCleanUtils { + + private OrphanCleanUtils() {} + + /** + * Constructs a {@link PhysicalTablePath} from a table path and an optional partition. Returns + * the non-partitioned form when {@code partitionInfo} is null. + */ + public static PhysicalTablePath physicalPath( + TablePath tablePath, @Nullable PartitionInfo partitionInfo) { + if (partitionInfo == null) { + return PhysicalTablePath.of(tablePath); + } + return PhysicalTablePath.of(tablePath, partitionInfo.getPartitionName()); + } + + /** + * Enumerates all {@link TableBucket} instances for a table (or a single partition of that + * table). + */ + public static List enumerateBuckets( + TableInfo tableInfo, @Nullable PartitionInfo partitionInfo) { + int n = tableInfo.getNumBuckets(); + List buckets = new ArrayList(n); + long tableId = tableInfo.getTableId(); + for (int b = 0; b < n; b++) { + if (partitionInfo == null) { + buckets.add(new TableBucket(tableId, b)); + } else { + buckets.add(new TableBucket(tableId, partitionInfo.getPartitionId(), b)); + } + } + return buckets; + } + + /** + * Resolves the effective remote data directory for a table/partition target using the + * three-level fallback: partition-level → table-level → cluster-level. + */ + @Nullable + public static String resolveRemoteDataDir( + TableInfo tableInfo, + @Nullable PartitionInfo partitionInfo, + @Nullable String clusterRemoteDataDir) { + if (partitionInfo != null && partitionInfo.getRemoteDataDir() != null) { + return partitionInfo.getRemoteDataDir(); + } + if (tableInfo.getRemoteDataDir() != null) { + return tableInfo.getRemoteDataDir(); + } + return clusterRemoteDataDir; + } + + /** + * Resolves the cluster-level {@code remote.data.dir} by querying the coordinator's runtime + * configuration. + */ + @Nullable + public static String resolveClusterRemoteDataDir(Admin admin) throws Exception { + Collection entries = admin.describeClusterConfigs().get(); + Map map = new HashMap(); + for (ConfigEntry entry : entries) { + map.put(entry.key(), entry.value()); + } + return map.get(ConfigOptions.REMOTE_DATA_DIR.key()); + } + + /** Constructs a remote sub-directory path, normalizing trailing slashes on the root. */ + public static FsPath remoteSubDir(String remoteDataDir, String subDir) { + return new FsPath(normalizeRoot(remoteDataDir) + "/" + subDir); + } + + /** Strips a trailing slash from a remote data directory string. */ + public static String normalizeRoot(String remoteDataDir) { + return remoteDataDir.endsWith("/") + ? remoteDataDir.substring(0, remoteDataDir.length() - 1) + : remoteDataDir; + } + + /** + * Lists the entries of a directory, returning {@code null} on {@link IOException} (directory + * does not exist or is inaccessible). + */ + @Nullable + public static FileStatus[] listStatuses(FileSystem fs, FsPath dir) { + try { + return fs.listStatus(dir); + } catch (IOException e) { + return null; + } + } + + /** + * Returns the {@link FileSystem} for a path if the path exists, or {@code null} otherwise. + * + * @throws IOException if resolving the filesystem itself fails + */ + @Nullable + public static FileSystem getFileSystemIfExists(FsPath dir) throws IOException { + FileSystem fs = dir.getFileSystem(); + return fs.exists(dir) ? fs : null; + } + + /** Formats a bucket-scope key for audit/logging purposes. */ + public static String bucketScopeKey(long tableId, Long partitionId, int bucketId) { + return tableId + ":" + partitionId + ":" + bucketId; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanAction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanAction.java new file mode 100644 index 0000000000..c5b49944bc --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanAction.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.flink.action.Action; +import org.apache.fluss.flink.action.orphan.config.OrphanCleanConfig; +import org.apache.fluss.flink.action.orphan.job.CleanStats; +import org.apache.fluss.flink.action.orphan.job.OrphanFilesCleanJob; + +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Orphan files cleanup action. Delegates to a distributed Flink Batch job ({@link + * OrphanFilesCleanJob}) that executes a 3-stage DAG: + * + *

    + *
  1. ScopeEnumerator (p=1): coordinator RPCs to enumerate scope and emit work items. + *
  2. ScanAndClean (p=N): parallel FS scan + rate-limited delete. + *
  3. StatsAggregate (p=1): merge stats + empty-directory sweep. + *
+ */ +@Internal +public class OrphanFilesCleanAction implements Action { + + private static final Logger LOG = LoggerFactory.getLogger(OrphanFilesCleanAction.class); + + private final OrphanCleanConfig config; + + public OrphanFilesCleanAction(OrphanCleanConfig config) { + this.config = config; + } + + @Override + public void run() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + CleanStats stats = + OrphanFilesCleanJob.execute(env, config, config.parallelism().orElse(null)); + LOG.info( + "orphan_files_clean done: scope={} scanned={} deleted={} failures={}" + + " bytesReclaimed={} dryRun={}", + scopeDescription(), + stats.scanned(), + stats.deleted(), + stats.deleteFailures(), + stats.bytesReclaimed(), + config.dryRun()); + } + + private String scopeDescription() { + String scope = + config.allDatabases() ? "all-databases" : config.database().orElse("unknown"); + if (config.table().isPresent()) { + return scope + "." + config.table().get(); + } + return scope; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanActionFactory.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanActionFactory.java new file mode 100644 index 0000000000..b0fd0b29dc --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanActionFactory.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.flink.action.Action; +import org.apache.fluss.flink.action.ActionFactory; +import org.apache.fluss.flink.action.orphan.config.OrphanCleanConfig; + +import org.apache.flink.api.java.utils.MultipleParameterTool; + +import java.util.Optional; + +/** Factory for the shell-mode orphan files cleanup action. */ +@Internal +public class OrphanFilesCleanActionFactory implements ActionFactory { + + @Override + public String identifier() { + return "orphan_files_clean"; + } + + @Override + public Optional create(MultipleParameterTool params) { + return Optional.of( + new OrphanFilesCleanAction(OrphanCleanConfig.fromParams(params))); + } + + @Override + public String help() { + return "Usage: orphan_files_clean --bootstrap-server \n" + + " (--database [--table ] | --all-databases)\n" + + " [--scan-root ]...\n" + + " [--older-than 'yyyy-MM-dd HH:mm:ss']\n" + + " [--delete-rate-limit-per-second 100] [--dry-run]\n" + + " [--allow-delete-manifest]\n" + + " [--allow-clean-orphan-tables]\n" + + " [--allow-clean-orphan-partitions]\n" + + " [--conf =]...\n" + + "\n" + + "Notes:\n" + + " --older-than is an absolute wall-clock cutoff (server local timezone). Files\n" + + " with mtime strictly less than the cutoff are deletion-eligible. Default:\n" + + " now - 3d, computed once at startup. The cutoff is frozen for the run, so a\n" + + " long scan cannot accidentally pull in files written after the action started.\n" + + " The cutoff must be at least 1d before now (closer cutoffs would race with\n" + + " mid-write files).\n" + + " Orphan directory detection (table/partition) relies solely on ID guards\n" + + " (maxKnownTableId / maxKnownPartitionId), not mtime.\n" + + " --table also disables the orphan-table scan (no sibling orphan-table scan in\n" + + " the db).\n" + + " --conf passes filesystem configuration for remote storage authentication.\n" + + " Keys use the same format as server.yaml (e.g. fs.oss.accessKeyId,\n" + + " fs.oss.accessKeySecret, fs.oss.endpoint, fs.oss.region). Repeatable.\n" + + "\n" + + "Examples:\n" + + " orphan_files_clean --bootstrap-server host:9123 --all-databases\n" + + " --conf fs.oss.accessKeyId=XXXX --conf fs.oss.accessKeySecret=YYYY\n" + + " --conf fs.oss.endpoint=oss-cn-hangzhou-internal.aliyuncs.com\n" + + " --conf fs.oss.region=cn-hangzhou"; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/RpcErrorClassifier.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/RpcErrorClassifier.java new file mode 100644 index 0000000000..8f0994213f --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/RpcErrorClassifier.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.exception.FlussRuntimeException; +import org.apache.fluss.exception.PartitionNotExistException; +import org.apache.fluss.exception.TableNotExistException; + +import java.io.IOException; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; + +/** + * Classifies RPC exceptions raised during scope enumeration and per-target active-set fetch into a + * small, audit-stable vocabulary. The category name is what surfaces as the {@code reason=} field + * of {@code skip_log_target} / {@code skip_kv_target} audit events, so operators triage by exact + * string and the enum must not be widened lightly. + * + *
    + *
  • {@link Category#NOT_FOUND} — legitimate "object does not exist"; the enumerator treats it + * as the target having disappeared concurrently and silently skips it without alarm. + *
  • {@link Category#TRANSIENT} — IO / timeout / ZK connection loss; the target is skipped this + * round and naturally retried in the next cleanup round. + *
  • {@link Category#SERVER_ERROR} — server-side failure; same skip, but audited at higher + * severity so an operator can investigate. + *
  • {@link Category#UNKNOWN} — anything not matched above; conservatively skipped + audited. + *
+ */ +@Internal +public final class RpcErrorClassifier { + + private RpcErrorClassifier() {} + + /** Categories of RPC errors. */ + public enum Category { + NOT_FOUND, + TRANSIENT, + SERVER_ERROR, + UNKNOWN + } + + /** + * Classifies a thrown exception. Unwraps {@link CompletionException}/{@link + * ExecutionException}. + */ + public static Category classify(Throwable t) { + Throwable cause = unwrap(t); + if (cause instanceof TableNotExistException + || cause instanceof PartitionNotExistException) { + return Category.NOT_FOUND; + } + if (cause instanceof IOException || cause instanceof TimeoutException) { + return Category.TRANSIENT; + } + if (cause instanceof FlussRuntimeException) { + return Category.SERVER_ERROR; + } + return Category.UNKNOWN; + } + + private static Throwable unwrap(Throwable t) { + while (t instanceof CompletionException || t instanceof ExecutionException) { + if (t.getCause() == null) { + return t; + } + t = t.getCause(); + } + return t; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/audit/AuditLogger.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/audit/AuditLogger.java new file mode 100644 index 0000000000..a47f0d7741 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/audit/AuditLogger.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.audit; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.flink.action.orphan.rule.RuleId; +import org.apache.fluss.fs.FsPath; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; + +/** + * Structured audit log writer for the orphan files cleanup action. + * + *

The dedicated logger name {@code fluss.orphan.audit} can be routed to a separate sink (e.g. + * SLS) by deployment-specific log4j configuration. + */ +@Internal +public final class AuditLogger { + + private static final Logger AUDIT = LoggerFactory.getLogger("fluss.orphan.audit"); + + /** + * Formats cutoff epoch-ms back to the {@code yyyy-MM-dd HH:mm:ss} CLI grammar in the server's + * local zone, so the audit line and the original {@code --older-than} value can be compared + * verbatim. + */ + private static final DateTimeFormatter CUTOFF_FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneId.systemDefault()); + + /** + * One-shot startup event recording the frozen file cutoff that drives this run's deletion + * decisions. Emitted before any other audit line so log readers can recover the exact threshold + * without having to re-parse the original CLI arguments. + */ + public void logCutoff(long olderThanMillis) { + AUDIT.info( + "action=cutoff older_than_iso={} older_than_ms={} ts={}", + CUTOFF_FORMATTER.format(Instant.ofEpochMilli(olderThanMillis)), + olderThanMillis, + Instant.now()); + } + + public void logDeleted(FsPath path, RuleId ruleId, boolean ok) { + AUDIT.info("action=deleted rule={} path={} ok={} ts={}", ruleId, path, ok, Instant.now()); + } + + public void logWouldDelete(FsPath path, RuleId ruleId) { + AUDIT.info("action=would_delete rule={} path={} ts={}", ruleId, path, Instant.now()); + } + + public void logDirDeleted(FsPath dir) { + AUDIT.info("action=dir_deleted path={} ts={}", dir, Instant.now()); + } + + public void logWouldDeleteDir(FsPath dir) { + AUDIT.info("action=would_delete_dir path={} ts={}", dir, Instant.now()); + } + + public void logSkipUnknown(FsPath path, RuleId ruleId) { + AUDIT.warn("action=skip_unknown rule={} path={} ts={}", ruleId, path, Instant.now()); + } + + public void logDeferred(FsPath path, RuleId ruleId) { + AUDIT.debug("action=deferred rule={} path={}", ruleId, path); + } + + public void logBucketAborted(String bucketStr, String reason) { + AUDIT.error( + "action=bucket_aborted bucket={} reason={} ts={}", + bucketStr, + reason, + Instant.now()); + } + + /** Skip an entire database during scope enumeration due to listTables failure. */ + public void logSkipDb(String dbName, String reason) { + AUDIT.warn("action=skip_db reason={} db={} ts={}", reason, dbName, Instant.now()); + } + + /** Skip a single table during scope enumeration due to getTableInfo or RPC failure. */ + public void logSkipTable(String dbName, String tableName, String reason) { + AUDIT.warn( + "action=skip_table reason={} db={} table={} ts={}", + reason, + dbName, + tableName, + Instant.now()); + } + + /** + * Skip listPartitionInfos for a table due to RPC failure (both active-partition cleanup and + * orphan-partition scan are suppressed for this table). + */ + public void logSkipPartitionList(String dbName, String tableName, String reason) { + AUDIT.warn( + "action=skip_partition_list reason={} db={} table={} ts={}", + reason, + dbName, + tableName, + Instant.now()); + } + + /** + * Skip KV cleanup for one (tableId, partitionId) target — emitted when {@code ListKvSnapshots} + * fails after retries. {@code partitionId} is null for non-partitioned tables. + */ + public void logSkipKvTarget(long tableId, Long partitionId, String reason) { + AUDIT.warn( + "action=skip_kv_target reason={} table_id={} partition_id={} ts={}", + reason, + tableId, + partitionId, + Instant.now()); + } + + /** + * Skip KV cleanup for a single bucket whose {@code ListKvSnapshots} response carried no + * active-snapshot entries. Empty per-bucket active set is treated as "cannot prove what is + * active" and the bucket is skipped to avoid mis-deletion. + */ + public void logSkipKvBucket(long tableId, Long partitionId, int bucketId, String reason) { + AUDIT.warn( + "action=skip_kv_bucket reason={} table_id={} partition_id={} bucket_id={} ts={}", + reason, + tableId, + partitionId, + bucketId, + Instant.now()); + } + + /** + * Skip log cleanup for one (tableId, partitionId) target — emitted when {@code + * ListRemoteLogManifests} fails after retries. {@code partitionId} is null for non-partitioned + * tables. + */ + public void logSkipLogTarget(long tableId, Long partitionId, String reason) { + AUDIT.warn( + "action=skip_log_target reason={} table_id={} partition_id={} ts={}", + reason, + tableId, + partitionId, + Instant.now()); + } + + /** + * Skip log cleanup for a single bucket whose remote manifest was not returned by the {@code + * ListRemoteLogManifests} RPC (the bucket has not yet committed any remote manifest). + */ + public void logSkipLogBucket(long tableId, Long partitionId, int bucketId, String reason) { + AUDIT.warn( + "action=skip_log_bucket reason={} table_id={} partition_id={} bucket_id={} ts={}", + reason, + tableId, + partitionId, + bucketId, + Instant.now()); + } + + /** Default-conservative skip of an orphan-table dir (opt-in flag not set). */ + public void logSkipOrphanTable(FsPath dir, String reason) { + AUDIT.info("action=skip_orphan_table reason={} path={} ts={}", reason, dir, Instant.now()); + } + + /** + * Skip the orphan-table scan for a database whose table-info set is incomplete (e.g. {@code + * --table} single-table mode, or {@code listTables}/{@code getTableInfo} failures left holes in + * the active table id set). Distinct from {@link #logSkipDb}, which means the whole database + * scope is dropped. + */ + public void logSkipOrphanTableScan(String dbName, String reason) { + AUDIT.warn( + "action=skip_orphan_table_scan reason={} db={} ts={}", + reason, + dbName, + Instant.now()); + } + + /** Default-conservative skip of an orphan-partition dir (opt-in flag not set). */ + public void logSkipOrphanPartition(FsPath dir, String reason) { + AUDIT.info( + "action=skip_orphan_partition reason={} path={} ts={}", reason, dir, Instant.now()); + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcher.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcher.java new file mode 100644 index 0000000000..8abf1184ce --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcher.java @@ -0,0 +1,357 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.build; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.annotation.VisibleForTesting; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.flink.action.orphan.RpcErrorClassifier; +import org.apache.fluss.flink.action.orphan.rule.BucketActiveRefs; +import org.apache.fluss.fs.FSDataInputStream; +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.rpc.messages.ListKvSnapshotsResponse; +import org.apache.fluss.rpc.messages.ListRemoteLogManifestsResponse; +import org.apache.fluss.rpc.messages.PbKvSnapshot; +import org.apache.fluss.rpc.messages.PbRemoteLogManifestEntry; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.fluss.utils.FlussPaths; +import org.apache.fluss.utils.IOUtils; +import org.apache.fluss.utils.RetryUtils; + +import javax.annotation.Nullable; + +import java.io.ByteArrayOutputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkArgument; + +/** + * Builds the active reference set for a single {@code (tableId, partitionId|null)} target, sourced + * from coordinator metadata via RPC (not from filesystem listing). + * + *

Log path: discovers each bucket's current remote log manifest path via {@code + * LIST_REMOTE_LOG_MANIFESTS}, then second-reads the manifest file from object storage. The + * per-target RPC is retried with exponential backoff via {@link RetryUtils}; per-bucket + * second-reads make a single attempt — a {@link FileNotFoundException} (manifest upserted between + * RPC and read) or any other IO failure immediately marks the bucket as {@link + * LogActiveRefsFetchResult.ManifestReadStatus#READ_FAILED} and recovery is left to the next cleanup + * round, avoiding {@code N × retries × IO} blow-up on cluster-wide turbulence. + * + *

KV path: {@code LIST_KV_SNAPSHOTS} returns snapshot ids directly (no second-read), so the + * per-target RPC retry alone is sufficient symmetry with the log path. + */ +@Internal +public final class ActiveRefsFetcher { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static final String REMOTE_LOG_SEGMENTS_FIELD = "remote_log_segments"; + private static final String SEGMENT_ID_FIELD = "segment_id"; + private static final String START_OFFSET_FIELD = "start_offset"; + private static final String END_OFFSET_FIELD = "end_offset"; + + /** + * Retry backoff base used by {@link RetryUtils} for per-target RPCs. With the default 3 retries + * and exponential backoff (200 → 400 → cap) this caps total retry delay at ~600ms — negligible + * vs the smoothing it gives over server jitter. + */ + private static final long DEFAULT_BACKOFF_MILLIS = 200L; + + private static final long MAX_BACKOFF_MILLIS = 2000L; + + private static final MetadataReader DEFAULT_METADATA_READER = + new MetadataReader() { + @Override + public byte[] read(FsPath path) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try (FSDataInputStream inputStream = path.getFileSystem().open(path)) { + IOUtils.copyBytes(inputStream, outputStream); + } + return outputStream.toByteArray(); + } + }; + + private final AdminFacade admin; + private final MetadataReader metadataReader; + private final int maxRetries; + private final long backoffMillis; + + public ActiveRefsFetcher(Admin admin, int maxRetries) { + this(wrap(admin), DEFAULT_METADATA_READER, maxRetries, DEFAULT_BACKOFF_MILLIS); + } + + public ActiveRefsFetcher(Admin admin, MetadataReader metadataReader, int maxRetries) { + this(wrap(admin), metadataReader, maxRetries, DEFAULT_BACKOFF_MILLIS); + } + + /** Test constructor: defaults backoff to 0 so unit tests don't pay retry sleep. */ + @VisibleForTesting + ActiveRefsFetcher(AdminFacade admin, MetadataReader metadataReader, int maxRetries) { + this(admin, metadataReader, maxRetries, 0L); + } + + @VisibleForTesting + ActiveRefsFetcher( + AdminFacade admin, MetadataReader metadataReader, int maxRetries, long backoffMillis) { + checkArgument(maxRetries >= 1, "maxRetries must be >= 1, got %s", maxRetries); + checkArgument(backoffMillis >= 0L, "backoffMillis must be >= 0, got %s", backoffMillis); + this.admin = admin; + this.metadataReader = metadataReader; + this.maxRetries = maxRetries; + this.backoffMillis = backoffMillis; + } + + private static AdminFacade wrap(Admin admin) { + return new AdminFacade() { + @Override + public CompletableFuture listRemoteLogManifests( + long tableId, @Nullable Long partitionId) { + return admin.listRemoteLogManifests(tableId, partitionId); + } + + @Override + public CompletableFuture listKvSnapshots( + long tableId, @Nullable Long partitionId) { + return admin.listKvSnapshots(tableId, partitionId); + } + }; + } + + /** + * Fetches per-bucket log active refs for a single {@code (tableId, partitionId|null)} target. + * Each bucket whose remote manifest is returned by the RPC is second-read in a single attempt; + * a {@link FileNotFoundException} or any other IO failure marks the bucket as {@link + * LogActiveRefsFetchResult.ManifestReadStatus#READ_FAILED} without affecting siblings. + * Per-target RPC failure (after retries) is reported via {@link + * LogActiveRefsFetchResult#listOk()}. + */ + public LogActiveRefsFetchResult fetchLogActiveRefsByBucket( + long tableId, @Nullable Long partitionId) { + ListRemoteLogManifestsResponse rpc; + try { + rpc = + RetryUtils.executeWithRetry( + () -> admin.listRemoteLogManifests(tableId, partitionId).get(), + "listRemoteLogManifests", + maxRetries, + backoffMillis, + MAX_BACKOFF_MILLIS, + e -> + RpcErrorClassifier.classify(e) + != RpcErrorClassifier.Category.NOT_FOUND); + } catch (IOException e) { + return LogActiveRefsFetchResult.listFailed( + formatRpcFailureReason(tableId, partitionId, e.getCause())); + } + + Map> entriesByBucket = new HashMap<>(); + for (PbRemoteLogManifestEntry entry : rpc.getManifestsList()) { + int bucketId = entry.getTableBucket().getBucketId(); + entriesByBucket.computeIfAbsent(bucketId, id -> new ArrayList<>()).add(entry); + } + + Map resolved = new HashMap<>(); + Map readFailures = new HashMap<>(); + for (Map.Entry> bucketEntries : + entriesByBucket.entrySet()) { + int bucketId = bucketEntries.getKey(); + try { + resolved.put(bucketId, buildBucketActiveRefs(bucketEntries.getValue())); + } catch (FileNotFoundException e) { + readFailures.put( + bucketId, + formatBucketReadFailureReason( + "Manifest not found (likely upserted concurrently)", + tableId, + partitionId, + bucketId, + e)); + } catch (ManifestParseException | JsonProcessingException e) { + // Manifest payload is unreadable as JSON or violates the expected shape — corrupt + // or schema-skewed, not a transient FS hiccup. Distinct reason so operators triage + // separately (re-running the action will not recover). + readFailures.put( + bucketId, + formatBucketReadFailureReason( + "Manifest parse failure (corrupt or unexpected schema)", + tableId, + partitionId, + bucketId, + e)); + } catch (IOException e) { + readFailures.put( + bucketId, + formatBucketReadFailureReason( + "IO error reading manifest", tableId, partitionId, bucketId, e)); + } + } + return LogActiveRefsFetchResult.ofPerBucket(resolved, readFailures); + } + + /** + * Fetches the per-bucket active snapshot directories ({@code snap-{id}} names) for one {@code + * (tableId, partitionId|null)} target. The set per bucket is the union of RETAINED and + * STILL_IN_USE entries returned by {@link Admin#listKvSnapshots(long, Long)}. Per-target RPC + * failure (after retries) is reported via {@link KvActiveRefsFetchResult#listOk()}, symmetric + * with the log path. + */ + public KvActiveRefsFetchResult fetchKvActiveSnapDirs(long tableId, @Nullable Long partitionId) { + ListKvSnapshotsResponse rpc; + try { + rpc = + RetryUtils.executeWithRetry( + () -> admin.listKvSnapshots(tableId, partitionId).get(), + "listKvSnapshots", + maxRetries, + backoffMillis, + MAX_BACKOFF_MILLIS, + e -> + RpcErrorClassifier.classify(e) + != RpcErrorClassifier.Category.NOT_FOUND); + } catch (IOException e) { + return KvActiveRefsFetchResult.listFailed( + formatRpcFailureReason(tableId, partitionId, e.getCause())); + } + Map> dirsByBucket = new HashMap<>(); + for (PbKvSnapshot snapshot : rpc.getActiveSnapshotsList()) { + int bucketId = snapshot.getBucketId(); + String dirName = FlussPaths.REMOTE_KV_SNAPSHOT_DIR_PREFIX + snapshot.getSnapshotId(); + dirsByBucket.computeIfAbsent(bucketId, b -> new HashSet<>()).add(dirName); + } + return KvActiveRefsFetchResult.ok(dirsByBucket); + } + + private static String formatRpcFailureReason( + long tableId, @Nullable Long partitionId, @Nullable Throwable cause) { + String reason = + String.format("RPC failure for tableId=%s partitionId=%s", tableId, partitionId); + if (cause != null && cause.getMessage() != null) { + reason = reason + ": " + cause.getMessage(); + } + return reason; + } + + private static String formatBucketReadFailureReason( + String prefix, + long tableId, + @Nullable Long partitionId, + int bucketId, + Throwable cause) { + String reason = + String.format( + "%s for tableId=%s partitionId=%s bucketId=%s", + prefix, tableId, partitionId, bucketId); + if (cause != null && cause.getMessage() != null) { + reason = reason + ": " + cause.getMessage(); + } + return reason; + } + + private BucketActiveRefs buildBucketActiveRefs(List entries) + throws IOException { + Set manifestPaths = new HashSet<>(); + Set segmentRelpaths = new HashSet<>(); + for (PbRemoteLogManifestEntry entry : entries) { + String path = entry.getRemoteLogManifestPath(); + manifestPaths.add(path); + byte[] manifestBytes = metadataReader.read(new FsPath(path)); + segmentRelpaths.addAll(parseLogSegmentRelativePaths(manifestBytes)); + } + return new BucketActiveRefs(segmentRelpaths, Collections.emptySet(), manifestPaths); + } + + private Set parseLogSegmentRelativePaths(byte[] manifestBytes) throws IOException { + JsonNode root = OBJECT_MAPPER.readTree(manifestBytes); + JsonNode segmentsNode = requiredNode(root, REMOTE_LOG_SEGMENTS_FIELD); + Set relativePaths = new HashSet<>(); + Iterator iterator = segmentsNode.elements(); + while (iterator.hasNext()) { + JsonNode segmentNode = iterator.next(); + String segmentId = requiredNode(segmentNode, SEGMENT_ID_FIELD).asText(); + long startOffset = requiredNode(segmentNode, START_OFFSET_FIELD).asLong(); + long endOffset = requiredNode(segmentNode, END_OFFSET_FIELD).asLong(); + String baseOffset = FlussPaths.filenamePrefixFromOffset(startOffset); + String writerOffset = FlussPaths.filenamePrefixFromOffset(endOffset); + + relativePaths.add(segmentId + "/" + baseOffset + FlussPaths.LOG_FILE_SUFFIX); + relativePaths.add(segmentId + "/" + baseOffset + FlussPaths.INDEX_FILE_SUFFIX); + relativePaths.add(segmentId + "/" + baseOffset + FlussPaths.TIME_INDEX_FILE_SUFFIX); + relativePaths.add( + segmentId + "/" + writerOffset + FlussPaths.WRITER_SNAPSHOT_FILE_SUFFIX); + } + return relativePaths; + } + + private static JsonNode requiredNode(JsonNode node, String fieldName) + throws ManifestParseException { + JsonNode field = node.get(fieldName); + if (field == null) { + throw new ManifestParseException("Missing required field: " + fieldName); + } + return field; + } + + /** + * Thrown when a remote-log manifest payload is structurally invalid (missing required field, + * wrong shape). Distinct from {@link IOException} so the bucket-read failure handler can route + * it to the {@code "Manifest parse failure"} reason instead of the generic {@code "IO error"} + * bucket — same skip-this-round outcome, different operator triage. + */ + static final class ManifestParseException extends IOException { + ManifestParseException(String message) { + super(message); + } + } + + /** + * Thin abstraction over the {@link FlussAdmin} read-only RPCs the builder depends on ({@code + * listRemoteLogManifests} for the log active manifest, {@code listKvSnapshots} for the KV + * active snapshot dirs). Exposed for test injection. + */ + @VisibleForTesting + interface AdminFacade { + CompletableFuture listRemoteLogManifests( + long tableId, @Nullable Long partitionId); + + CompletableFuture listKvSnapshots( + long tableId, @Nullable Long partitionId); + } + + /** + * Abstraction for reading manifest files from object storage. Must throw {@link + * FileNotFoundException} (and not a wrapped variant) when the path is absent, so the caller can + * distinguish "manifest pointer upserted concurrently" from genuine IO failures and surface + * each with a distinct failure reason. + */ + @VisibleForTesting + interface MetadataReader { + byte[] read(FsPath path) throws IOException; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/KvActiveRefsFetchResult.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/KvActiveRefsFetchResult.java new file mode 100644 index 0000000000..7b1c6c7873 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/KvActiveRefsFetchResult.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.build; + +import org.apache.fluss.annotation.Internal; + +import javax.annotation.Nullable; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Result of KV active-snapshot-dir fetch for one {@code (tableId, partitionId|null)} target. + * + *

Mirrors the per-target {@code listOk + listFailureReason} axis of {@link + * LogActiveRefsFetchResult}. KV has no per-bucket failure dimension because the {@code + * LIST_KV_SNAPSHOTS} RPC returns snapshot ids directly (no second-read of an external file), so the + * per-bucket payload is just {@code Map>} of {@code snap-{id}} directory + * names. Buckets absent from the map are treated by the consumer as "empty active set → skip". + */ +@Internal +public final class KvActiveRefsFetchResult { + + private final RpcListStatus list; + private final Map> activeSnapDirsByBucket; + + private KvActiveRefsFetchResult( + RpcListStatus list, Map> activeSnapDirsByBucket) { + this.list = list; + Map> copy = new HashMap<>(); + for (Map.Entry> e : activeSnapDirsByBucket.entrySet()) { + copy.put(e.getKey(), Collections.unmodifiableSet(new HashSet<>(e.getValue()))); + } + this.activeSnapDirsByBucket = Collections.unmodifiableMap(copy); + } + + /** Result for a target whose {@code LIST_KV_SNAPSHOTS} RPC failed and exhausted retries. */ + public static KvActiveRefsFetchResult listFailed(String reason) { + return new KvActiveRefsFetchResult( + RpcListStatus.listFailed(reason), Collections.emptyMap()); + } + + /** Result for a target whose {@code LIST_KV_SNAPSHOTS} RPC succeeded. */ + static KvActiveRefsFetchResult ok(Map> activeSnapDirsByBucket) { + return new KvActiveRefsFetchResult(RpcListStatus.ok(), activeSnapDirsByBucket); + } + + /** Whether the per-target {@code LIST_KV_SNAPSHOTS} RPC succeeded. */ + public boolean listOk() { + return list.isOk(); + } + + /** Reason the per-target RPC failed; {@code null} when {@link #listOk()} is true. */ + @Nullable + public String listFailureReason() { + return list.reason(); + } + + /** + * Per-bucket active snapshot directory names ({@code snap-{id}}). Empty map when {@link + * #listOk()} is false. + * + *

Bucket absent from the map means "the RPC returned no active-snapshot entries for this + * bucket", which the consumer must treat as "cannot prove what is active here → skip KV + * cleanup for this bucket and emit {@code skip_kv_bucket reason=empty_active_set}". Empty does + * not mean "no active snapshots exist": the server enumerates buckets from ZK and that path can + * transiently underreport (partial reads, znode creation lag, stale historical bucket counts), + * so treating empty as no-op-skip is the only response compatible with the action's "may leak, + * must not mis-delete" hard constraint. + */ + public Map> activeSnapDirsByBucket() { + return activeSnapDirsByBucket; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/LogActiveRefsFetchResult.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/LogActiveRefsFetchResult.java new file mode 100644 index 0000000000..af94c500b1 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/LogActiveRefsFetchResult.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.build; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.flink.action.orphan.rule.BucketActiveRefs; + +import javax.annotation.Nullable; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Result of log active-refs fetch for one {@code (tableId, partitionId|null)} target. + * + *

The result is split along two orthogonal axes so each axis can be queried independently: + * + *

    + *
  • Per-target: {@link #listOk()} reports whether the {@code LIST_REMOTE_LOG_MANIFESTS} + * RPC succeeded. When it fails the per-bucket axis is meaningless and the caller should emit + * a single per-target skip and bypass the per-bucket loop entirely. + *
  • Per-bucket: {@link #statusFor(int)} reports one of {@link + * ManifestReadStatus#RESOLVED}, {@link ManifestReadStatus#READ_FAILED}, or {@link + * ManifestReadStatus#NOT_LISTED} for every bucket enumerated from table metadata. Only + * meaningful when {@link #listOk()} is true. + *
+ */ +@Internal +public final class LogActiveRefsFetchResult { + + /** Per-bucket outcome (only meaningful when {@link #listOk()} is true). */ + public enum ManifestReadStatus { + /** The RPC returned an entry for this bucket and its manifest was read successfully. */ + RESOLVED, + /** + * Per-bucket manifest second-read failed (FileNotFound from manifest upsert race, or other + * IO failure). The failing bucket is skipped for this round; recovery is by the next + * cleanup round. + */ + READ_FAILED, + /** + * Table metadata enumerates the bucket, but the {@code LIST_REMOTE_LOG_MANIFESTS} response + * did not include an entry for it — typically because the bucket has not yet committed any + * remote manifest (e.g. log tiering has not produced one), or an occasional server-side + * underreport (e.g. partial ZK read). Cleanup has nothing to clean for this bucket. + */ + NOT_LISTED + } + + private final RpcListStatus list; + private final Map resolved; + private final Map readFailures; + + private LogActiveRefsFetchResult( + RpcListStatus list, + Map resolved, + Map readFailures) { + this.list = list; + this.resolved = Collections.unmodifiableMap(new HashMap<>(resolved)); + this.readFailures = Collections.unmodifiableMap(new HashMap<>(readFailures)); + } + + /** + * Result for a target whose {@code LIST_REMOTE_LOG_MANIFESTS} RPC failed and exhausted retries. + */ + public static LogActiveRefsFetchResult listFailed(String reason) { + return new LogActiveRefsFetchResult( + RpcListStatus.listFailed(reason), Collections.emptyMap(), Collections.emptyMap()); + } + + /** + * Result for a target whose {@code LIST_REMOTE_LOG_MANIFESTS} RPC succeeded. {@code resolved} + * carries the per-bucket active refs for RESOLVED buckets; {@code readFailures} carries the + * per-bucket failure reasons for READ_FAILED buckets. Any bucket not present in either map is + * reported as {@link ManifestReadStatus#NOT_LISTED}. + */ + static LogActiveRefsFetchResult ofPerBucket( + Map resolved, Map readFailures) { + return new LogActiveRefsFetchResult(RpcListStatus.ok(), resolved, readFailures); + } + + /** Whether the per-target {@code LIST_REMOTE_LOG_MANIFESTS} RPC succeeded. */ + public boolean listOk() { + return list.isOk(); + } + + /** Reason the per-target RPC failed; {@code null} when {@link #listOk()} is true. */ + @Nullable + public String listFailureReason() { + return list.reason(); + } + + /** + * Per-bucket manifest read status for a bucket enumerated from table metadata. Callers must + * first check {@link #listOk()} and skip the per-bucket loop entirely when it is false. + */ + public ManifestReadStatus statusFor(int bucketId) { + if (!list.isOk()) { + throw new IllegalStateException("Per-bucket status is not available when listOk=false"); + } + if (resolved.containsKey(bucketId)) { + return ManifestReadStatus.RESOLVED; + } + if (readFailures.containsKey(bucketId)) { + return ManifestReadStatus.READ_FAILED; + } + return ManifestReadStatus.NOT_LISTED; + } + + /** Active refs for a RESOLVED bucket. */ + public BucketActiveRefs activeRefsOf(int bucketId) { + BucketActiveRefs activeRefs = resolved.get(bucketId); + if (activeRefs == null) { + throw new IllegalStateException("Bucket " + bucketId + " is not RESOLVED"); + } + return activeRefs; + } + + /** Failure reason for a READ_FAILED bucket. */ + public String readFailureReason(int bucketId) { + String reason = readFailures.get(bucketId); + if (reason == null) { + throw new IllegalStateException("Bucket " + bucketId + " is not READ_FAILED"); + } + return reason; + } + + /** + * Bucket ids for which the RPC returned an entry (i.e. RESOLVED or READ_FAILED). Buckets + * enumerated from table metadata but absent from this set are {@link + * ManifestReadStatus#NOT_LISTED}. + */ + public Set respondedBucketIds() { + Set ids = new HashSet<>(resolved.keySet()); + ids.addAll(readFailures.keySet()); + return Collections.unmodifiableSet(ids); + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/MaxKnownIdsTracker.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/MaxKnownIdsTracker.java new file mode 100644 index 0000000000..c77d03323b --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/MaxKnownIdsTracker.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.build; + +import org.apache.fluss.annotation.Internal; + +/** + * Accumulates {@code maxKnownTableId} and {@code maxKnownPartitionId} during a single cleanup run. + * + *

Values are updated from the successful scope-enumeration metadata lookups that already + * materialize concrete ids for cleanup orchestration: {@code getTableInfo()} for tables and {@code + * listPartitionInfos()} for partitions. The tracker is therefore pure RPC-derived and never sourced + * from FS dir-name parsing. + * + *

The tracked maximums serve as ID guards for orphan directory detection: only + * directories whose parsed ID is {@code <=} the observed maximum can be classified as orphan + * candidates. Directories with higher IDs are conservatively skipped as potentially freshly + * allocated. Because RPC failures cause the tracker to observe fewer IDs, the maximums are always a + * lower bound of the true cluster-wide maximum — making the guard strictly more conservative (safe + * direction) under partial failures. + */ +@Internal +public final class MaxKnownIdsTracker { + + private long maxKnownTableId = -1L; + private long maxKnownPartitionId = -1L; + + public void observeTableId(long tableId) { + if (tableId > maxKnownTableId) { + maxKnownTableId = tableId; + } + } + + public void observePartitionId(long partitionId) { + if (partitionId > maxKnownPartitionId) { + maxKnownPartitionId = partitionId; + } + } + + public long maxKnownTableId() { + return maxKnownTableId; + } + + public long maxKnownPartitionId() { + return maxKnownPartitionId; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/RpcListStatus.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/RpcListStatus.java new file mode 100644 index 0000000000..4113dd500c --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/RpcListStatus.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.build; + +import javax.annotation.Nullable; + +/** + * Per-target status of a list RPC (target = one {@code (tableId, partitionId|null)} pair), shared + * by {@link LogActiveRefsFetchResult} and {@link KvActiveRefsFetchResult}. + * + *

Captures the {@code listOk + listFailureReason} pair so both result types can delegate the + * per-target axis to a single value and surface identical {@code listOk()} / {@code + * listFailureReason()} APIs to consumers. + */ +final class RpcListStatus { + + private static final RpcListStatus OK = new RpcListStatus(true, null); + + private final boolean ok; + @Nullable private final String reason; + + private RpcListStatus(boolean ok, @Nullable String reason) { + this.ok = ok; + this.reason = reason; + } + + static RpcListStatus ok() { + return OK; + } + + static RpcListStatus listFailed(String reason) { + return new RpcListStatus(false, reason); + } + + boolean isOk() { + return ok; + } + + @Nullable + String reason() { + return reason; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfig.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfig.java new file mode 100644 index 0000000000..59257fbb48 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfig.java @@ -0,0 +1,336 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.config; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.utils.StringUtils; + +import org.apache.flink.api.java.utils.MultipleParameterTool; + +import javax.annotation.Nullable; + +import java.io.Serializable; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** Parsed command-line options for the orphan files cleanup action. */ +@Internal +public final class OrphanCleanConfig implements Serializable { + + private static final long serialVersionUID = 1L; + + /** + * Minimum gap between any user-supplied cutoff and {@code now}. A cutoff closer to {@code now} + * would risk classifying files that are mid-write (committed file written, snapshot/manifest + * not yet visible to {@code ListRemoteLogManifests} / {@code ListKvSnapshots}) as orphan and + * deleting them. + */ + private static final Duration HARD_LOWER_BOUND = Duration.ofDays(1); + + /** Default file-level cutoff: files written before {@code now - 3d} are deletion-eligible. */ + private static final Duration DEFAULT_OLDER_THAN = Duration.ofDays(3); + + private static final long DEFAULT_DELETE_RATE_LIMIT_PER_SECOND = 100L; + + /** + * Wall-clock timestamp format accepted on the CLI ({@code yyyy-MM-dd HH:mm:ss}, interpreted in + * the server's local time zone). Matches Apache Paimon's {@code orphan_files_clean older_than} + * grammar to minimize operator context-switching between systems. + */ + private static final DateTimeFormatter CUTOFF_FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + private final String bootstrapServer; + private final boolean allDatabases; + private final @Nullable String database; + private final @Nullable String table; + private final long olderThanMillis; + private final boolean dryRun; + private final long deleteRateLimitPerSecond; + private final @Nullable Integer parallelism; + private final List scanRoots; + private final boolean allowDeleteManifest; + private final boolean allowCleanOrphanTables; + private final boolean allowCleanOrphanPartitions; + private final Map extraConfigs; + + private OrphanCleanConfig( + String bootstrapServer, + boolean allDatabases, + @Nullable String database, + @Nullable String table, + long olderThanMillis, + boolean dryRun, + long deleteRateLimitPerSecond, + @Nullable Integer parallelism, + List scanRoots, + boolean allowDeleteManifest, + boolean allowCleanOrphanTables, + boolean allowCleanOrphanPartitions, + Map extraConfigs) { + this.bootstrapServer = bootstrapServer; + this.allDatabases = allDatabases; + this.database = database; + this.table = table; + this.olderThanMillis = olderThanMillis; + this.dryRun = dryRun; + this.deleteRateLimitPerSecond = deleteRateLimitPerSecond; + this.parallelism = parallelism; + this.scanRoots = Collections.unmodifiableList(new ArrayList(scanRoots)); + this.allowDeleteManifest = allowDeleteManifest; + this.allowCleanOrphanTables = allowCleanOrphanTables; + this.allowCleanOrphanPartitions = allowCleanOrphanPartitions; + this.extraConfigs = Collections.unmodifiableMap(new HashMap<>(extraConfigs)); + } + + /** Parses a cleanup config from CLI parameters. */ + public static OrphanCleanConfig fromParams(MultipleParameterTool params) { + String bootstrapServer = params.get("bootstrap-server"); + if (StringUtils.isNullOrWhitespaceOnly(bootstrapServer)) { + throw new IllegalArgumentException("--bootstrap-server is required"); + } + + boolean allDatabases = params.has("all-databases"); + String database = params.get("database"); + if (allDatabases && !StringUtils.isNullOrWhitespaceOnly(database)) { + throw new IllegalArgumentException( + "--database and --all-databases are mutually exclusive"); + } + if (!allDatabases && StringUtils.isNullOrWhitespaceOnly(database)) { + throw new IllegalArgumentException( + "Either --database or --all-databases must be provided"); + } + if (allDatabases && !StringUtils.isNullOrWhitespaceOnly(params.get("table"))) { + throw new IllegalArgumentException( + "--table requires --database and cannot be used with --all-databases"); + } + + long now = System.currentTimeMillis(); + long olderThanMillis = + parseCutoff("--older-than", params.get("older-than"), now, DEFAULT_OLDER_THAN); + long deleteRateLimitPerSecond = + parseDeleteRateLimit(params.get("delete-rate-limit-per-second")); + Integer parallelism = parseParallelism(params.get("parallelism")); + boolean allowDeleteManifest = params.has("allow-delete-manifest"); + boolean allowCleanOrphanTables = params.has("allow-clean-orphan-tables"); + boolean allowCleanOrphanPartitions = params.has("allow-clean-orphan-partitions"); + + return new OrphanCleanConfig( + bootstrapServer, + allDatabases, + database, + params.get("table"), + olderThanMillis, + params.has("dry-run"), + deleteRateLimitPerSecond, + parallelism, + parseScanRoots(params.getMultiParameter("scan-root")), + allowDeleteManifest, + allowCleanOrphanTables, + allowCleanOrphanPartitions, + parseExtraConfigs(params.getMultiParameter("conf"))); + } + + /** + * Parses a CLI cutoff value into an absolute epoch-ms timestamp. Empty input falls back to + * {@code now - defaultGap}. Explicit input must parse as {@code yyyy-MM-dd HH:mm:ss} in the + * server's local time zone and must be at least {@link #HARD_LOWER_BOUND} earlier than {@code + * now} — closer-to-now cutoffs would race with active writes (see {@code HARD_LOWER_BOUND} + * javadoc). + */ + private static long parseCutoff( + String flag, @Nullable String value, long now, Duration defaultGap) { + if (StringUtils.isNullOrWhitespaceOnly(value)) { + return now - defaultGap.toMillis(); + } + LocalDateTime parsed; + try { + parsed = LocalDateTime.parse(value, CUTOFF_FORMATTER); + } catch (DateTimeParseException e) { + throw new IllegalArgumentException( + flag + + " must be a timestamp in 'yyyy-MM-dd HH:mm:ss' (server local TZ), got: " + + value, + e); + } + long parsedMillis = parsed.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + long maxAllowed = now - HARD_LOWER_BOUND.toMillis(); + if (parsedMillis > maxAllowed) { + throw new IllegalArgumentException( + flag + + " must be at least 1d before now (got " + + Instant.ofEpochMilli(parsedMillis) + + ", now is " + + Instant.ofEpochMilli(now) + + "); a closer cutoff would race with mid-write files"); + } + return parsedMillis; + } + + private static long parseDeleteRateLimit(@Nullable String value) { + if (StringUtils.isNullOrWhitespaceOnly(value)) { + return DEFAULT_DELETE_RATE_LIMIT_PER_SECOND; + } + long rate = Long.parseLong(value); + if (rate <= 0) { + throw new IllegalArgumentException("--delete-rate-limit-per-second must be positive"); + } + return rate; + } + + @Nullable + private static Integer parseParallelism(@Nullable String value) { + if (StringUtils.isNullOrWhitespaceOnly(value)) { + return null; + } + int p = Integer.parseInt(value); + if (p <= 0) { + throw new IllegalArgumentException("--parallelism must be positive"); + } + return p; + } + + private static List parseScanRoots(@Nullable Collection values) { + if (values == null || values.isEmpty()) { + return Collections.emptyList(); + } + + List scanRoots = new ArrayList(values.size()); + for (String value : values) { + if (StringUtils.isNullOrWhitespaceOnly(value)) { + throw new IllegalArgumentException("--scan-root must not be blank"); + } + scanRoots.add(value); + } + return scanRoots; + } + + private static Map parseExtraConfigs(@Nullable Collection values) { + if (values == null || values.isEmpty()) { + return Collections.emptyMap(); + } + Map configs = new HashMap(); + for (String kv : values) { + int eqIdx = kv.indexOf('='); + if (eqIdx <= 0) { + throw new IllegalArgumentException( + "--conf must be in key=value format, got: " + kv); + } + configs.put(kv.substring(0, eqIdx), kv.substring(eqIdx + 1)); + } + return configs; + } + + /** Returns the bootstrap server list used to connect to Fluss. */ + public String bootstrapServer() { + return bootstrapServer; + } + + /** Returns whether the cleanup targets all databases. */ + public boolean allDatabases() { + return allDatabases; + } + + /** Returns the single targeted database when the action is not scoped to all databases. */ + public Optional database() { + return Optional.ofNullable(database); + } + + /** Returns the optional targeted table name. */ + public Optional table() { + return Optional.ofNullable(table); + } + + /** + * Returns the file-level cutoff as an absolute epoch-millis timestamp, frozen at action + * startup. A candidate file is deletion-eligible iff its mtime is strictly less than this + * value. The cutoff does not slide during the run — long scans cannot accidentally pull in + * files written after startup. + */ + public long olderThanMillis() { + return olderThanMillis; + } + + /** Returns whether the action runs in dry-run mode. */ + public boolean dryRun() { + return dryRun; + } + + /** Returns the maximum number of actual delete calls per second. */ + public long deleteRateLimitPerSecond() { + return deleteRateLimitPerSecond; + } + + /** Returns the optional parallelism for the ScanAndClean stage. */ + public Optional parallelism() { + return Optional.ofNullable(parallelism); + } + + /** Returns additional remote.data.dir roots to scan. */ + public List scanRoots() { + return scanRoots; + } + + /** + * Opt-in to delete {@code .manifest} files. Default {@code false}: mis-deleting an active + * manifest leaves the coordinator's manifest pointer dangling and breaks the bucket's metadata + * chain — the failure mode is catastrophic and asymmetric vs the trivial space cost of keeping + * orphan manifests (KB-sized files), so deletion is gated behind an explicit operator flag. + */ + public boolean allowDeleteManifest() { + return allowDeleteManifest; + } + + /** + * Opt-in to recursively clean files inside an orphan-table directory. Default {@code false}: + * the action only audits the detected orphan dir and leaves its contents untouched, because an + * id-based misclassification of a freshly-created table as orphan would otherwise be + * unrecoverable. Operators flip this on once they have reviewed the audit log. + */ + public boolean allowCleanOrphanTables() { + return allowCleanOrphanTables; + } + + /** + * Opt-in to recursively clean files inside an orphan-partition directory. Same default-audit + * rationale as {@link #allowCleanOrphanTables()}. + */ + public boolean allowCleanOrphanPartitions() { + return allowCleanOrphanPartitions; + } + + /** + * Returns extra configuration entries passed via {@code --conf key=value}. These are propagated + * to {@link org.apache.fluss.fs.FileSystem#initialize} for remote filesystem authentication + * (e.g. {@code fs.oss.accessKeyId}, {@code fs.oss.accessKeySecret}). + */ + public Map extraConfigs() { + return extraConfigs; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleter.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleter.java new file mode 100644 index 0000000000..81875974a0 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleter.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.fs; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.flink.action.orphan.audit.AuditLogger; +import org.apache.fluss.flink.action.orphan.rule.Decision; +import org.apache.fluss.flink.action.orphan.rule.RuleId; +import org.apache.fluss.fs.FileStatus; +import org.apache.fluss.fs.FileSystem; +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.RateLimiter; + +import java.io.IOException; + +import static org.apache.fluss.utils.Preconditions.checkArgument; + +/** + * Sole entry point for filesystem deletion within the orphan cleanup package. + * + *

Only two operations are exposed: + * + *

    + *
  • {@link #deleteFile} - delete a single file (never recursive). + *
  • {@link #deleteEmptyDir} - delete a directory only if it is currently empty. + *
+ * + *

By design there is no recursive-delete API; any caller that needs deletion under {@code + * fluss-flink-common/.../action/orphan/} should go through this class. The single-entry-point + * invariant is currently enforced only by convention — there is no Checkstyle rule guarding it. + */ +@Internal +public final class SafeDeleter { + + private final FileSystem fs; + private final boolean dryRun; + private final AuditLogger audit; + private final RateLimiter rateLimiter; + + public SafeDeleter(FileSystem fs, boolean dryRun, AuditLogger audit) { + this(fs, dryRun, audit, RateLimiter.create(100.0)); + } + + public SafeDeleter(FileSystem fs, boolean dryRun, AuditLogger audit, RateLimiter rateLimiter) { + this.fs = fs; + this.dryRun = dryRun; + this.audit = audit; + this.rateLimiter = rateLimiter; + } + + /** + * Delete a single file. + * + * @return {@code true} if the file was actually deleted (or recorded as would-be-deleted under + * {@code dryRun}); {@code false} if {@link FileSystem#delete} returned {@code false} + * (deletion silently failed — e.g. permissions, transient remote-store error). Callers + * should track {@code false} returns as delete failures in their run summary. + */ + public boolean deleteFile(FsPath file, Decision decision, RuleId ruleId) throws IOException { + checkArgument( + decision == Decision.DELETE, + "deleteFile must only be called for Decision.DELETE, got %s", + decision); + if (dryRun) { + audit.logWouldDelete(file, ruleId); + return true; + } + rateLimiter.acquire(); + boolean ok = fs.delete(file, false); + audit.logDeleted(file, ruleId, ok); + return ok; + } + + /** + * Delete a directory only if it is currently empty. + * + * @return {@code true} if the directory was actually deleted (or recorded as would-be-deleted + * under {@code dryRun}); {@code false} if the directory was non-empty / unreadable, or if + * {@link FileSystem#delete} returned {@code false}. Callers should not increment a "deleted + * directory" counter when this returns {@code false}. + */ + public boolean deleteEmptyDir(FsPath dir) throws IOException { + FileStatus[] children = listChildrenSilently(dir); + if (children == null || children.length > 0) { + return false; + } + if (dryRun) { + audit.logWouldDeleteDir(dir); + return true; + } + rateLimiter.acquire(); + boolean ok = fs.delete(dir, false); + if (ok) { + audit.logDirDeleted(dir); + } + return ok; + } + + private FileStatus[] listChildrenSilently(FsPath dir) { + try { + return fs.listStatus(dir); + } catch (IOException ignored) { + return null; + } + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleanTask.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleanTask.java new file mode 100644 index 0000000000..70499fd285 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleanTask.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.job; + +import org.apache.fluss.annotation.Internal; + +import javax.annotation.Nullable; + +import java.util.HashSet; +import java.util.Set; + +/** + * Work item for a single bucket's file-level cleanup. Carries everything needed to execute cleanup + * without coordinator interaction: FS paths, manifest locations for second-read, and the + * already-resolved KV active snapshot directory names. + */ +@Internal +public final class BucketCleanTask implements CleanTask { + + private static final long serialVersionUID = 1L; + + @Nullable private final String logTabletDir; + @Nullable private final String kvTabletDir; + private final Set logSegmentRelativePaths; + private final Set logActiveManifestPaths; + private final Set kvActiveSnapDirs; + private final long cutoffMillis; + private final boolean dryRun; + private final boolean allowDeleteManifest; + + public BucketCleanTask( + @Nullable String logTabletDir, + @Nullable String kvTabletDir, + Set logSegmentRelativePaths, + Set logActiveManifestPaths, + Set kvActiveSnapDirs, + long cutoffMillis, + boolean dryRun, + boolean allowDeleteManifest) { + this.logTabletDir = logTabletDir; + this.kvTabletDir = kvTabletDir; + this.logSegmentRelativePaths = new HashSet<>(logSegmentRelativePaths); + this.logActiveManifestPaths = new HashSet<>(logActiveManifestPaths); + this.kvActiveSnapDirs = new HashSet<>(kvActiveSnapDirs); + this.cutoffMillis = cutoffMillis; + this.dryRun = dryRun; + this.allowDeleteManifest = allowDeleteManifest; + } + + @Nullable + public String logTabletDir() { + return logTabletDir; + } + + @Nullable + public String kvTabletDir() { + return kvTabletDir; + } + + /** Active log segment relative paths (already resolved from manifests in Stage 1). */ + public Set logSegmentRelativePaths() { + return logSegmentRelativePaths; + } + + /** Active manifest paths (already resolved from RPC in Stage 1). */ + public Set logActiveManifestPaths() { + return logActiveManifestPaths; + } + + /** + * KV active snapshot directory names (already resolved from RPC, no further FS read needed). + */ + public Set kvActiveSnapDirs() { + return kvActiveSnapDirs; + } + + public long cutoffMillis() { + return cutoffMillis; + } + + public boolean dryRun() { + return dryRun; + } + + public boolean allowDeleteManifest() { + return allowDeleteManifest; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java new file mode 100644 index 0000000000..ea7d9ea161 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.job; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.flink.action.orphan.audit.AuditLogger; +import org.apache.fluss.flink.action.orphan.fs.SafeDeleter; +import org.apache.fluss.flink.action.orphan.rule.BucketActiveRefs; +import org.apache.fluss.flink.action.orphan.rule.Decision; +import org.apache.fluss.flink.action.orphan.rule.FileMeta; +import org.apache.fluss.flink.action.orphan.rule.FileRule; +import org.apache.fluss.flink.action.orphan.rule.RuleDispatcher; +import org.apache.fluss.fs.FileStatus; +import org.apache.fluss.fs.FileSystem; +import org.apache.fluss.fs.FsPath; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; + +/** + * Per-bucket orphan cleanup for live buckets: walks the provided bucket directories and dispatches + * each file to the appropriate {@link FileRule} using the caller-supplied active reference set. + * + *

All deletions go through {@link SafeDeleter} (no recursive deletes). Unknown file types are + * skipped with an audit warning per the design's "unknown-types-not-deleted" principle. + */ +@Internal +public final class BucketCleaner { + + private final RuleDispatcher dispatcher; + private final SafeDeleter safeDeleter; + private final AuditLogger audit; + private final long cutoffMillis; + + public BucketCleaner( + RuleDispatcher dispatcher, + SafeDeleter safeDeleter, + AuditLogger audit, + long cutoffMillis) { + this.dispatcher = dispatcher; + this.safeDeleter = safeDeleter; + this.audit = audit; + this.cutoffMillis = cutoffMillis; + } + + /** Cleans one bucket's log/kv subtrees using the caller-supplied active reference set. */ + public BucketCleanStats clean(BucketActiveRefs activeRefs, FsPath... bucketDirs) + throws IOException { + BucketCleanStats stats = BucketCleanStats.empty(); + for (FsPath bucketDir : bucketDirs) { + if (bucketDir != null) { + walkAndCleanDir(bucketDir, activeRefs, stats); + } + } + return stats; + } + + private void walkAndCleanDir(FsPath root, BucketActiveRefs activeRefs, BucketCleanStats stats) + throws IOException { + FileSystem fs = root.getFileSystem(); + if (!fs.exists(root)) { + return; + } + Deque stack = new ArrayDeque(); + stack.push(root); + while (!stack.isEmpty()) { + FsPath dir = stack.pop(); + FileStatus[] children; + try { + children = fs.listStatus(dir); + } catch (IOException ignored) { + continue; + } + if (children == null) { + continue; + } + for (FileStatus child : children) { + FsPath childPath = child.getPath(); + if (child.isDir()) { + stack.push(childPath); + continue; + } + if (childPath.getName().startsWith(".")) { + continue; + } + FileMeta meta = + new FileMeta(childPath, child.getLen(), child.getModificationTime()); + FileRule rule = dispatcher.dispatch(meta); + Decision decision = rule.evaluate(meta, activeRefs, cutoffMillis); + stats.scanned++; + switch (decision) { + case DELETE: + if (safeDeleter.deleteFile(meta.path(), decision, rule.id())) { + stats.deleted++; + stats.bytesReclaimed += meta.size(); + } else { + stats.deleteFailures++; + } + break; + case SKIP_UNKNOWN: + audit.logSkipUnknown(meta.path(), rule.id()); + break; + case KEEP_ACTIVE: + case DEFER: + // no-op + break; + default: + // unknown decision — skip defensively + break; + } + } + } + } + + /** Per-bucket cleanup statistics. */ + public static final class BucketCleanStats { + public long scanned; + public long deleted; + public long deleteFailures; + public long bytesReclaimed; + + public static BucketCleanStats empty() { + return new BucketCleanStats(); + } + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/CleanStats.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/CleanStats.java new file mode 100644 index 0000000000..31e1e66f10 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/CleanStats.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.job; + +import org.apache.fluss.annotation.Internal; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * Aggregatable cleanup statistics emitted by each {@link ScanAndCleanFunction} subtask. The {@code + * touchedDirs} list is collected by the final aggregator for empty-directory sweeping after all + * subtasks complete. + */ +@Internal +public final class CleanStats implements Serializable { + + private static final long serialVersionUID = 1L; + + private final long scanned; + private final long deleted; + private final long deleteFailures; + private final long bytesReclaimed; + private final List touchedDirs; + + public CleanStats( + long scanned, + long deleted, + long deleteFailures, + long bytesReclaimed, + List touchedDirs) { + this.scanned = scanned; + this.deleted = deleted; + this.deleteFailures = deleteFailures; + this.bytesReclaimed = bytesReclaimed; + this.touchedDirs = new ArrayList<>(touchedDirs); + } + + public static CleanStats empty() { + return new CleanStats(0L, 0L, 0L, 0L, new ArrayList()); + } + + public long scanned() { + return scanned; + } + + public long deleted() { + return deleted; + } + + public long deleteFailures() { + return deleteFailures; + } + + public long bytesReclaimed() { + return bytesReclaimed; + } + + public List touchedDirs() { + return touchedDirs; + } + + public CleanStats merge(CleanStats other) { + List mergedDirs = new ArrayList<>(this.touchedDirs); + mergedDirs.addAll(other.touchedDirs); + return new CleanStats( + this.scanned + other.scanned, + this.deleted + other.deleted, + this.deleteFailures + other.deleteFailures, + this.bytesReclaimed + other.bytesReclaimed, + mergedDirs); + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/CleanTask.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/CleanTask.java new file mode 100644 index 0000000000..69f691ce99 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/CleanTask.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.job; + +import org.apache.fluss.annotation.Internal; + +import java.io.Serializable; + +/** + * Marker interface for work items emitted by {@link ScopeEnumeratorFunction} and consumed by {@link + * ScanAndCleanFunction}. Implementations carry enough context for a single subtask to execute + * cleanup independently (no further coordinator interaction needed). + */ +@Internal +public interface CleanTask extends Serializable {} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeper.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeper.java new file mode 100644 index 0000000000..191ba87638 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeper.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.job; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.flink.action.orphan.audit.AuditLogger; +import org.apache.fluss.flink.action.orphan.fs.SafeDeleter; +import org.apache.fluss.fs.FileStatus; +import org.apache.fluss.fs.FileSystem; +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.RateLimiter; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * End-of-run empty-directory reclaim. Walks every registered "touched" directory tree depth-first + * and asks {@link SafeDeleter} to remove any empty directories encountered, bottom up; non-empty + * directories are no-ops via {@code SafeDeleter}'s contract. + * + *

Run exactly once, after all per-table / per-db cleanup has finished. Any sub-flow that touched + * a tablet dir or descended into an orphan dir is expected to register that root via {@link + * #registerTouched(FsPath)} during its own pass, so this single end-of-run sweep can collect the + * leftover empties without re-walking the live cleanup paths. + * + *

The sweeper deliberately does not own a {@link FileSystem} — it derives one per-root from the + * given {@link FsPath} so different remote stores can coexist. + */ +@Internal +public final class EmptyDirSweeper { + + private final boolean dryRun; + private final AuditLogger audit; + private final RateLimiter rateLimiter; + private final Set touchedRoots = new HashSet(); + + public EmptyDirSweeper(boolean dryRun, AuditLogger audit) { + this(dryRun, audit, RateLimiter.create(100.0)); + } + + public EmptyDirSweeper(boolean dryRun, AuditLogger audit, RateLimiter rateLimiter) { + this.dryRun = dryRun; + this.audit = audit; + this.rateLimiter = rateLimiter; + } + + /** + * Register a directory root whose subtree should be considered by the final empty-dir sweep. + * Call sites: every cleanup sub-flow that may have removed files under {@code root} (live log / + * KV tablet dirs, orphan partition / orphan table dirs). Multiple registrations of the same + * root are deduplicated; the actual sweep is deferred until {@link #sweep()} runs at end of + * action. + */ + public void registerTouched(FsPath root) { + if (root != null) { + touchedRoots.add(root); + } + } + + /** + * Sweeps every registered subtree, removing empty leaf directories first and propagating up to + * the registered root. + * + * @return the number of empty directories deleted, or that would be deleted in dry-run mode + */ + public long sweep() throws IOException { + long removed = 0L; + for (FsPath root : touchedRoots) { + removed += sweepOne(root); + } + return removed; + } + + private long sweepOne(FsPath root) throws IOException { + FileSystem fs = root.getFileSystem(); + SafeDeleter safeDeleter = new SafeDeleter(fs, dryRun, audit, rateLimiter); + if (!fs.exists(root)) { + return 0L; + } + // First, gather all directories (root and descendants) in pre-order; then process in + // reverse order so deeper directories are visited before their parents. + List dirs = new ArrayList(); + Deque stack = new ArrayDeque(); + stack.push(root); + while (!stack.isEmpty()) { + FsPath dir = stack.pop(); + dirs.add(dir); + FileStatus[] children; + try { + children = fs.listStatus(dir); + } catch (IOException ignored) { + continue; + } + if (children == null) { + continue; + } + for (FileStatus child : children) { + if (child.isDir()) { + stack.push(child.getPath()); + } + } + } + long deleted = 0L; + if (dryRun) { + Set virtuallyDeletedDirs = new HashSet(); + for (int i = dirs.size() - 1; i >= 0; i--) { + FsPath dir = dirs.get(i); + if (!fs.exists(dir)) { + continue; + } + if (!isEffectivelyEmpty(fs, dir, virtuallyDeletedDirs)) { + continue; + } + audit.logWouldDeleteDir(dir); + virtuallyDeletedDirs.add(dir.toString()); + deleted++; + } + } else { + for (int i = dirs.size() - 1; i >= 0; i--) { + if (safeDeleter.deleteEmptyDir(dirs.get(i))) { + deleted++; + } + } + } + return deleted; + } + + private boolean isEffectivelyEmpty( + FileSystem fs, FsPath dir, Set virtuallyDeletedDirs) { + FileStatus[] remaining; + try { + remaining = fs.listStatus(dir); + } catch (IOException ignored) { + return false; + } + if (remaining == null) { + return false; + } + for (FileStatus child : remaining) { + if (!child.isDir() || !virtuallyDeletedDirs.contains(child.getPath().toString())) { + return false; + } + } + return true; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanDirCleanTask.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanDirCleanTask.java new file mode 100644 index 0000000000..cd564e5b78 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanDirCleanTask.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.job; + +import org.apache.fluss.annotation.Internal; + +/** + * Work item for cleaning an orphan table or partition directory. The directory has already been + * identified as an orphan candidate by {@link ScopeEnumeratorFunction} (ID guard satisfied). + */ +@Internal +public final class OrphanDirCleanTask implements CleanTask { + + private static final long serialVersionUID = 1L; + + private final String dirPath; + private final long cutoffMillis; + private final boolean dryRun; + private final boolean allowDeleteManifest; + + public OrphanDirCleanTask( + String dirPath, long cutoffMillis, boolean dryRun, boolean allowDeleteManifest) { + this.dirPath = dirPath; + this.cutoffMillis = cutoffMillis; + this.dryRun = dryRun; + this.allowDeleteManifest = allowDeleteManifest; + } + + public String dirPath() { + return dirPath; + } + + public long cutoffMillis() { + return cutoffMillis; + } + + public boolean dryRun() { + return dryRun; + } + + public boolean allowDeleteManifest() { + return allowDeleteManifest; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java new file mode 100644 index 0000000000..62f780e096 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.job; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.flink.action.orphan.config.OrphanCleanConfig; + +import org.apache.flink.api.common.RuntimeExecutionMode; +import org.apache.flink.api.common.typeinfo.TypeHint; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; + +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +/** + * Builds and executes the 3-stage Flink Batch DAG for orphan files cleanup. + * + * 

+ * Stage 1: ScopeEnumerator (p=1)   — coordinator RPCs, emits CleanTask
+ * Stage 2: ScanAndClean (p=N)      — FS scan + rate-limited delete, emits CleanStats
+ * Stage 3: StatsAggregate (p=1)    — merge stats + empty-dir sweep, emits final CleanStats
+ *
+ */ +@Internal +public final class OrphanFilesCleanJob { + + private OrphanFilesCleanJob() {} + + /** + * Builds the DAG, executes it in batch mode, and returns the final aggregated cleanup + * statistics. + * + * @param env the Flink execution environment (caller configures classpath, etc.) + * @param config parsed orphan cleanup configuration + * @param parallelism the parallelism for Stage 2 (ScanAndClean); null uses env default + * @return the final cleanup statistics + */ + public static CleanStats execute( + StreamExecutionEnvironment env, OrphanCleanConfig config, Integer parallelism) + throws Exception { + env.setRuntimeMode(RuntimeExecutionMode.BATCH); + + // Stage 1: ScopeEnumerator (parallelism=1) + DataStream trigger = + env.fromCollection(Collections.singletonList(1), TypeInformation.of(Integer.class)); + + SingleOutputStreamOperator tasks = + trigger.process(new ScopeEnumeratorFunction(config)) + .returns(TypeInformation.of(new TypeHint() {})) + .setParallelism(1) + .name("ScopeEnumerator"); + + // Stage 2: ScanAndClean (parallelism=N) + SingleOutputStreamOperator stats = + tasks.rebalance() + .process( + new ScanAndCleanFunction( + config.deleteRateLimitPerSecond(), config.extraConfigs())) + .returns(TypeInformation.of(new TypeHint() {})) + .name("ScanAndClean"); + if (parallelism != null) { + stats = stats.setParallelism(parallelism); + } + + // Stage 3: StatsAggregate + EmptyDirSweep (parallelism=1) + SingleOutputStreamOperator result = + stats.transform( + "StatsAggregate", + TypeInformation.of(new TypeHint() {}), + new StatsAggregateOperator(config.dryRun())) + .setParallelism(1); + + // Execute and collect the single result + List collected = collectResults(result); + if (collected.isEmpty()) { + return CleanStats.empty(); + } + return collected.get(0); + } + + @SuppressWarnings("deprecation") + private static List collectResults(DataStream result) throws Exception { + Iterator iterator = result.executeAndCollect("OrphanFilesClean"); + List results = new java.util.ArrayList(); + while (iterator.hasNext()) { + results.add(iterator.next()); + } + return results; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java new file mode 100644 index 0000000000..c6e2cae92d --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.job; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.flink.action.orphan.audit.AuditLogger; +import org.apache.fluss.flink.action.orphan.fs.SafeDeleter; +import org.apache.fluss.flink.action.orphan.rule.BucketActiveRefs; +import org.apache.fluss.flink.action.orphan.rule.Decision; +import org.apache.fluss.flink.action.orphan.rule.FileMeta; +import org.apache.fluss.flink.action.orphan.rule.FileRule; +import org.apache.fluss.flink.action.orphan.rule.RuleDispatcher; +import org.apache.fluss.fs.FileStatus; +import org.apache.fluss.fs.FileSystem; +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.RateLimiter; + +import org.apache.flink.streaming.api.functions.ProcessFunction; +import org.apache.flink.util.Collector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; +import java.util.Map; + +/** + * Stage 2 of the orphan files cleanup job. Runs at user-configured parallelism (N) and performs + * pure FS operations — no coordinator RPC interaction. + * + *

Each subtask processes assigned {@link CleanTask} items serially: + * + *

    + *
  • {@link BucketCleanTask}: second-reads manifests from object storage to build the active + * reference set, then walks log/kv directories and deletes orphan files. + *
  • {@link OrphanDirCleanTask}: recursively walks the orphan directory and deletes all files + * older than the cutoff. + *
+ * + *

Each task emits its own {@link CleanStats} immediately upon completion. Delete rate is limited + * per-subtask: {@code configuredRate / runtimeParallelism}. The serial processing within each + * subtask guarantees no concurrent throttler access. + */ +@Internal +public final class ScanAndCleanFunction extends ProcessFunction { + + private static final long serialVersionUID = 1L; + private static final Logger LOG = LoggerFactory.getLogger(ScanAndCleanFunction.class); + + private final long deleteRateLimitPerSecond; + private final Map extraConfigs; + + private transient AuditLogger audit; + + public ScanAndCleanFunction(long deleteRateLimitPerSecond, Map extraConfigs) { + this.deleteRateLimitPerSecond = deleteRateLimitPerSecond; + this.extraConfigs = extraConfigs; + } + + @Override + public void open(org.apache.flink.configuration.Configuration parameters) { + if (!extraConfigs.isEmpty()) { + FileSystem.initialize(Configuration.fromMap(extraConfigs), null); + } + audit = new AuditLogger(); + } + + @Override + public void processElement(CleanTask task, Context ctx, Collector out) + throws Exception { + if (task instanceof BucketCleanTask) { + out.collect(processBucketTask((BucketCleanTask) task)); + } else if (task instanceof OrphanDirCleanTask) { + out.collect(processOrphanDirTask((OrphanDirCleanTask) task)); + } + } + + // ------------------------------------------------------------------------- + // BucketCleanTask processing + // ------------------------------------------------------------------------- + + private CleanStats processBucketTask(BucketCleanTask task) throws IOException { + FsPath logDir = task.logTabletDir() != null ? new FsPath(task.logTabletDir()) : null; + FsPath kvDir = task.kvTabletDir() != null ? new FsPath(task.kvTabletDir()) : null; + + FsPath anyDir = logDir != null ? logDir : kvDir; + if (anyDir == null) { + return CleanStats.empty(); + } + + BucketActiveRefs activeRefs = + new BucketActiveRefs( + task.logSegmentRelativePaths(), + task.kvActiveSnapDirs(), + task.logActiveManifestPaths()); + RuleDispatcher dispatcher = new RuleDispatcher(task.allowDeleteManifest()); + SafeDeleter safeDeleter = createSafeDeleter(anyDir.getFileSystem(), task.dryRun()); + BucketCleaner cleaner = + new BucketCleaner(dispatcher, safeDeleter, audit, task.cutoffMillis()); + + BucketCleaner.BucketCleanStats bucketStats = cleaner.clean(activeRefs, logDir, kvDir); + + List touchedDirs = new ArrayList(); + if (logDir != null) { + touchedDirs.add(logDir.toString()); + } + if (kvDir != null) { + touchedDirs.add(kvDir.toString()); + } + + return new CleanStats( + bucketStats.scanned, + bucketStats.deleted, + bucketStats.deleteFailures, + bucketStats.bytesReclaimed, + touchedDirs); + } + + // ------------------------------------------------------------------------- + // OrphanDirCleanTask processing + // ------------------------------------------------------------------------- + + private CleanStats processOrphanDirTask(OrphanDirCleanTask task) throws IOException { + FsPath dirPath = new FsPath(task.dirPath()); + FileSystem fs = dirPath.getFileSystem(); + if (!fs.exists(dirPath)) { + return CleanStats.empty(); + } + + SafeDeleter safeDeleter = createSafeDeleter(fs, task.dryRun()); + RuleDispatcher dispatcher = new RuleDispatcher(task.allowDeleteManifest(), true); + + long scanned = 0L; + long deleted = 0L; + long deleteFailures = 0L; + long bytesReclaimed = 0L; + + Deque stack = new ArrayDeque(); + stack.push(dirPath); + while (!stack.isEmpty()) { + FsPath dir = stack.pop(); + FileStatus[] children; + try { + children = fs.listStatus(dir); + } catch (IOException ignored) { + continue; + } + if (children == null) { + continue; + } + for (FileStatus child : children) { + FsPath childPath = child.getPath(); + if (child.isDir()) { + stack.push(childPath); + continue; + } + if (childPath.getName().startsWith(".")) { + continue; + } + scanned++; + if (child.getModificationTime() >= task.cutoffMillis()) { + continue; + } + FileMeta meta = + new FileMeta(childPath, child.getLen(), child.getModificationTime()); + FileRule rule = dispatcher.dispatch(meta); + Decision decision = + rule.evaluate(meta, BucketActiveRefs.empty(), task.cutoffMillis()); + switch (decision) { + case DELETE: + if (safeDeleter.deleteFile(meta.path(), decision, rule.id())) { + deleted++; + bytesReclaimed += meta.size(); + } else { + deleteFailures++; + } + break; + case SKIP_UNKNOWN: + audit.logSkipUnknown(meta.path(), rule.id()); + break; + case KEEP_ACTIVE: + case DEFER: + default: + break; + } + } + } + + List touchedDirs = new ArrayList(); + touchedDirs.add(dirPath.toString()); + return new CleanStats(scanned, deleted, deleteFailures, bytesReclaimed, touchedDirs); + } + + // ------------------------------------------------------------------------- + // Helpers + // ------------------------------------------------------------------------- + + private SafeDeleter createSafeDeleter(FileSystem fs, boolean dryRun) { + int parallelism = getRuntimeContext().getTaskInfo().getNumberOfParallelSubtasks(); + double perSubtaskRate = Math.max(1.0, (double) deleteRateLimitPerSecond / parallelism); + return new SafeDeleter(fs, dryRun, audit, RateLimiter.create(perSubtaskRate)); + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java new file mode 100644 index 0000000000..03059a92a4 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java @@ -0,0 +1,572 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.job; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.flink.action.orphan.OrphanCleanUtils; +import org.apache.fluss.flink.action.orphan.RpcErrorClassifier; +import org.apache.fluss.flink.action.orphan.audit.AuditLogger; +import org.apache.fluss.flink.action.orphan.build.ActiveRefsFetcher; +import org.apache.fluss.flink.action.orphan.build.KvActiveRefsFetchResult; +import org.apache.fluss.flink.action.orphan.build.LogActiveRefsFetchResult; +import org.apache.fluss.flink.action.orphan.build.MaxKnownIdsTracker; +import org.apache.fluss.flink.action.orphan.config.OrphanCleanConfig; +import org.apache.fluss.flink.action.orphan.rule.OrphanDirDetector; +import org.apache.fluss.fs.FileStatus; +import org.apache.fluss.fs.FileSystem; +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.utils.FlussPaths; + +import org.apache.flink.streaming.api.functions.ProcessFunction; +import org.apache.flink.util.Collector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; + +import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.enumerateBuckets; +import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.getFileSystemIfExists; +import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.listStatuses; +import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.physicalPath; +import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.remoteSubDir; +import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.resolveClusterRemoteDataDir; +import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.resolveRemoteDataDir; + +/** + * Stage 1 of the orphan files cleanup job. Runs at parallelism=1 and concentrates all coordinator + * RPC interaction in a single subtask. + * + *

For each live bucket, emits a {@link BucketCleanTask} containing the FS paths and manifest + * locations needed for Stage 2 to execute cleanup without coordinator access. For each detected + * orphan directory, emits an {@link OrphanDirCleanTask}. + */ +@Internal +public final class ScopeEnumeratorFunction extends ProcessFunction { + + private static final long serialVersionUID = 1L; + private static final Logger LOG = LoggerFactory.getLogger(ScopeEnumeratorFunction.class); + private static final String[] TOP_LEVEL_DIRS = { + FlussPaths.REMOTE_LOG_DIR_NAME, FlussPaths.REMOTE_KV_DIR_NAME + }; + + private final OrphanCleanConfig config; + + public ScopeEnumeratorFunction(OrphanCleanConfig config) { + this.config = config; + } + + @Override + public void processElement(Integer trigger, Context ctx, Collector out) + throws Exception { + if (!config.extraConfigs().isEmpty()) { + FileSystem.initialize(Configuration.fromMap(config.extraConfigs()), null); + } + + Configuration flussConfig = new Configuration(); + flussConfig.setString(ConfigOptions.BOOTSTRAP_SERVERS.key(), config.bootstrapServer()); + + try (Connection connection = ConnectionFactory.createConnection(flussConfig); + Admin admin = connection.getAdmin()) { + AuditLogger audit = new AuditLogger(); + audit.logCutoff(config.olderThanMillis()); + + ActiveRefsFetcher fetcher = new ActiveRefsFetcher(admin, 3); + MaxKnownIdsTracker tracker = new MaxKnownIdsTracker(); + String clusterRemoteDataDir = resolveClusterRemoteDataDir(admin); + + Map dbStates = enumerateActiveScope(admin, audit, tracker); + + for (DbScanState dbState : dbStates.values()) { + for (LiveTableScope liveTable : dbState.liveTables) { + emitBucketTasks(liveTable, fetcher, audit, clusterRemoteDataDir, out); + emitOrphanPartitionDirTasks( + liveTable, tracker, clusterRemoteDataDir, audit, out); + } + emitOrphanTableDirTasks(dbState, tracker, clusterRemoteDataDir, audit, out); + } + } + } + + // ------------------------------------------------------------------------- + // Scope enumeration (coordinator RPCs only) + // ------------------------------------------------------------------------- + + private Map enumerateActiveScope( + Admin admin, AuditLogger audit, MaxKnownIdsTracker tracker) { + List dbs = resolveDatabasesToScan(admin, audit); + Map result = new LinkedHashMap(); + for (String dbName : dbs) { + DbScanState dbState = new DbScanState(dbName); + result.put(dbName, dbState); + if (config.table().isPresent()) { + dbState.tableInfosComplete = false; + resolveTable(admin, audit, tracker, dbState, config.table().get(), true); + continue; + } + List tableNames; + try { + tableNames = admin.listTables(dbName).get(); + } catch (Exception e) { + audit.logSkipDb(dbName, classifyName(e)); + dbState.tableInfosComplete = false; + continue; + } + for (String tableName : tableNames) { + resolveTable(admin, audit, tracker, dbState, tableName, false); + } + } + return result; + } + + private List resolveDatabasesToScan(Admin admin, AuditLogger audit) { + if (config.allDatabases()) { + try { + return admin.listDatabases().get(); + } catch (Exception e) { + audit.logSkipDb("*", classifyName(e)); + return Collections.emptyList(); + } + } + String databaseName = config.database().get(); + try { + if (admin.databaseExists(databaseName).get()) { + return Collections.singletonList(databaseName); + } + } catch (Exception e) { + audit.logSkipDb(databaseName, classifyName(e)); + return Collections.emptyList(); + } + audit.logSkipDb(databaseName, RpcErrorClassifier.Category.NOT_FOUND.name()); + return Collections.emptyList(); + } + + private void resolveTable( + Admin admin, + AuditLogger audit, + MaxKnownIdsTracker tracker, + DbScanState dbState, + String tableName, + boolean explicitTableTarget) { + TablePath tablePath = TablePath.of(dbState.dbName, tableName); + TableInfo tableInfo; + try { + tableInfo = admin.getTableInfo(tablePath).get(); + } catch (Exception e) { + RpcErrorClassifier.Category category = RpcErrorClassifier.classify(e); + if (category != RpcErrorClassifier.Category.NOT_FOUND || explicitTableTarget) { + audit.logSkipTable(dbState.dbName, tableName, category.name()); + dbState.tableInfosComplete = false; + } + return; + } + tracker.observeTableId(tableInfo.getTableId()); + dbState.activeTableIds.add(tableInfo.getTableId()); + + LiveTableScope liveTable = new LiveTableScope(dbState.dbName, tableName, tableInfo); + dbState.liveTables.add(liveTable); + if (!tableInfo.isPartitioned()) { + return; + } + try { + List partitions = admin.listPartitionInfos(tablePath).get(); + TableInfo confirm = admin.getTableInfo(tablePath).get(); + if (confirm.getTableId() != tableInfo.getTableId()) { + audit.logSkipTable(dbState.dbName, tableName, "ABA"); + liveTable.partitionInfosComplete = false; + return; + } + for (PartitionInfo partition : partitions) { + liveTable.partitions.add(partition); + liveTable.activePartitionIds.add(partition.getPartitionId()); + tracker.observePartitionId(partition.getPartitionId()); + } + } catch (Exception e) { + audit.logSkipPartitionList(dbState.dbName, tableName, classifyName(e)); + liveTable.partitionInfosComplete = false; + } + } + + // ------------------------------------------------------------------------- + // Emit BucketCleanTasks (per-target RPC + per-bucket task emission) + // ------------------------------------------------------------------------- + + private void emitBucketTasks( + LiveTableScope liveTable, + ActiveRefsFetcher fetcher, + AuditLogger audit, + @Nullable String clusterRemoteDataDir, + Collector out) { + if (liveTable.partitioned && !liveTable.partitionInfosComplete) { + return; + } + List partitionTargets = + liveTable.partitioned + ? liveTable.partitions + : Collections.singletonList(null); + for (PartitionInfo partitionInfo : partitionTargets) { + emitBucketTasksForTarget( + liveTable, partitionInfo, fetcher, audit, clusterRemoteDataDir, out); + } + } + + private void emitBucketTasksForTarget( + LiveTableScope liveTable, + @Nullable PartitionInfo partitionInfo, + ActiveRefsFetcher fetcher, + AuditLogger audit, + @Nullable String clusterRemoteDataDir, + Collector out) { + Long partitionId = partitionInfo == null ? null : partitionInfo.getPartitionId(); + + LogActiveRefsFetchResult logResult = + fetcher.fetchLogActiveRefsByBucket(liveTable.tableId, partitionId); + if (!logResult.listOk()) { + audit.logSkipLogTarget(liveTable.tableId, partitionId, logResult.listFailureReason()); + } + + Map> kvActiveByBucket = Collections.emptyMap(); + boolean kvTargetOk = false; + if (liveTable.tableInfo.hasPrimaryKey()) { + KvActiveRefsFetchResult kvResult = + fetcher.fetchKvActiveSnapDirs(liveTable.tableId, partitionId); + if (kvResult.listOk()) { + kvActiveByBucket = kvResult.activeSnapDirsByBucket(); + kvTargetOk = true; + } else { + audit.logSkipKvTarget(liveTable.tableId, partitionId, kvResult.listFailureReason()); + } + } + + String remoteDataDir = + resolveRemoteDataDir(liveTable.tableInfo, partitionInfo, clusterRemoteDataDir); + if (remoteDataDir == null) { + LOG.warn( + "Table {} partition {} has no resolvable remote.data.dir; skipping", + liveTable.tablePath, + partitionId); + return; + } + + FsPath remoteLogDir = remoteSubDir(remoteDataDir, FlussPaths.REMOTE_LOG_DIR_NAME); + FsPath remoteKvDir = remoteSubDir(remoteDataDir, FlussPaths.REMOTE_KV_DIR_NAME); + + for (TableBucket tableBucket : enumerateBuckets(liveTable.tableInfo, partitionInfo)) { + int bucketId = tableBucket.getBucket(); + + String logTabletDir = null; + + Set logSegmentRelativePaths = Collections.emptySet(); + Set logActiveManifestPaths = Collections.emptySet(); + + if (logResult.listOk()) { + switch (logResult.statusFor(bucketId)) { + case RESOLVED: + logTabletDir = + FlussPaths.remoteLogTabletDir( + remoteLogDir, + physicalPath(liveTable.tablePath, partitionInfo), + tableBucket) + .toString(); + logSegmentRelativePaths = + logResult.activeRefsOf(bucketId).logSegmentRelativePaths(); + logActiveManifestPaths = + logResult.activeRefsOf(bucketId).logActiveManifestPaths(); + break; + case READ_FAILED: + audit.logBucketAborted( + OrphanCleanUtils.bucketScopeKey( + liveTable.tableId, partitionId, bucketId), + logResult.readFailureReason(bucketId)); + break; + case NOT_LISTED: + audit.logSkipLogBucket( + liveTable.tableId, partitionId, bucketId, "no_remote_manifest"); + break; + default: + break; + } + } + + String kvTabletDir = null; + Set kvActiveSnaps = Collections.emptySet(); + if (kvTargetOk && kvActiveByBucket.containsKey(bucketId)) { + kvTabletDir = + FlussPaths.remoteKvTabletDir( + remoteKvDir, + physicalPath(liveTable.tablePath, partitionInfo), + tableBucket) + .toString(); + kvActiveSnaps = kvActiveByBucket.get(bucketId); + } else if (kvTargetOk) { + audit.logSkipKvBucket(liveTable.tableId, partitionId, bucketId, "empty_active_set"); + } + + if (logTabletDir == null && kvTabletDir == null) { + continue; + } + + out.collect( + new BucketCleanTask( + logTabletDir, + kvTabletDir, + logSegmentRelativePaths, + logActiveManifestPaths, + kvActiveSnaps, + config.olderThanMillis(), + config.dryRun(), + config.allowDeleteManifest())); + } + } + + // ------------------------------------------------------------------------- + // Emit OrphanDirCleanTasks + // ------------------------------------------------------------------------- + + private void emitOrphanTableDirTasks( + DbScanState dbState, + MaxKnownIdsTracker tracker, + @Nullable String clusterRemoteDataDir, + AuditLogger audit, + Collector out) + throws IOException { + if (!dbState.tableInfosComplete) { + audit.logSkipOrphanTableScan(dbState.dbName, "tableInfos-incomplete"); + return; + } + Set activeTableIds = dbState.activeTableIds; + long maxKnownTableId = tracker.maxKnownTableId(); + boolean emit = config.allowCleanOrphanTables(); + for (String root : rootsToScan(clusterRemoteDataDir)) { + for (String topLevel : TOP_LEVEL_DIRS) { + FsPath dbDir = remoteSubDir(root, topLevel + "/" + dbState.dbName); + if (emit) { + emitOrphanDirsUnderParent( + dbDir, + dirName -> + OrphanDirDetector.isOrphanTable( + dirName, activeTableIds, maxKnownTableId), + out); + } else { + logSkippedOrphanDirsUnderParent( + dbDir, + dirName -> + OrphanDirDetector.isOrphanTable( + dirName, activeTableIds, maxKnownTableId), + audit); + } + } + } + } + + private void emitOrphanPartitionDirTasks( + LiveTableScope liveTable, + MaxKnownIdsTracker tracker, + @Nullable String clusterRemoteDataDir, + AuditLogger audit, + Collector out) + throws IOException { + if (!liveTable.partitioned || !liveTable.partitionInfosComplete) { + return; + } + Set activePartitionIds = liveTable.activePartitionIds; + long maxKnownPartitionId = tracker.maxKnownPartitionId(); + boolean emit = config.allowCleanOrphanPartitions(); + for (String root : rootsForLiveTable(liveTable, clusterRemoteDataDir)) { + for (String topLevel : TOP_LEVEL_DIRS) { + FsPath tableDir = + FlussPaths.remoteTableDir( + remoteSubDir(root, topLevel), + liveTable.tablePath, + liveTable.tableId); + if (emit) { + emitOrphanDirsUnderParent( + tableDir, + dirName -> + OrphanDirDetector.isOrphanPartition( + dirName, activePartitionIds, maxKnownPartitionId), + out); + } else { + logSkippedOrphanPartitionDirsUnderParent( + tableDir, + dirName -> + OrphanDirDetector.isOrphanPartition( + dirName, activePartitionIds, maxKnownPartitionId), + audit); + } + } + } + } + + private void emitOrphanDirsUnderParent( + FsPath parentDir, Predicate isOrphan, Collector out) + throws IOException { + FileSystem fs = getFileSystemIfExists(parentDir); + if (fs == null) { + return; + } + FileStatus[] entries = listStatuses(fs, parentDir); + if (entries == null) { + return; + } + for (FileStatus entry : entries) { + if (!entry.isDir()) { + continue; + } + if (!isOrphan.test(entry.getPath().getName())) { + continue; + } + out.collect( + new OrphanDirCleanTask( + entry.getPath().toString(), + config.olderThanMillis(), + config.dryRun(), + config.allowDeleteManifest())); + } + } + + private void logSkippedOrphanDirsUnderParent( + FsPath parentDir, Predicate isOrphan, AuditLogger audit) throws IOException { + FileSystem fs = getFileSystemIfExists(parentDir); + if (fs == null) { + return; + } + FileStatus[] entries = listStatuses(fs, parentDir); + if (entries == null) { + return; + } + for (FileStatus entry : entries) { + if (!entry.isDir()) { + continue; + } + if (!isOrphan.test(entry.getPath().getName())) { + continue; + } + audit.logSkipOrphanTable(entry.getPath(), "default-conservative"); + } + } + + private void logSkippedOrphanPartitionDirsUnderParent( + FsPath parentDir, Predicate isOrphan, AuditLogger audit) throws IOException { + FileSystem fs = getFileSystemIfExists(parentDir); + if (fs == null) { + return; + } + FileStatus[] entries = listStatuses(fs, parentDir); + if (entries == null) { + return; + } + for (FileStatus entry : entries) { + if (!entry.isDir()) { + continue; + } + if (!isOrphan.test(entry.getPath().getName())) { + continue; + } + audit.logSkipOrphanPartition(entry.getPath(), "default-conservative"); + } + } + + // ------------------------------------------------------------------------- + // Helpers + // ------------------------------------------------------------------------- + + private List rootsToScan(@Nullable String clusterRemoteDataDir) { + LinkedHashSet roots = new LinkedHashSet(); + if (clusterRemoteDataDir != null) { + roots.add(clusterRemoteDataDir); + } + roots.addAll(config.scanRoots()); + return new ArrayList(roots); + } + + private List rootsForLiveTable( + LiveTableScope liveTable, @Nullable String clusterRemoteDataDir) { + LinkedHashSet roots = new LinkedHashSet(rootsToScan(clusterRemoteDataDir)); + String tableRoot = resolveRemoteDataDir(liveTable.tableInfo, null, clusterRemoteDataDir); + if (tableRoot != null) { + roots.add(tableRoot); + } + for (PartitionInfo partitionInfo : liveTable.partitions) { + String partitionRoot = + resolveRemoteDataDir(liveTable.tableInfo, partitionInfo, clusterRemoteDataDir); + if (partitionRoot != null) { + roots.add(partitionRoot); + } + } + return new ArrayList(roots); + } + + private static String classifyName(Throwable e) { + return RpcErrorClassifier.classify(e).name(); + } + + // ------------------------------------------------------------------------- + // Internal state classes + // ------------------------------------------------------------------------- + + private static final class DbScanState { + final String dbName; + boolean tableInfosComplete = true; + final Set activeTableIds = new LinkedHashSet(); + final List liveTables = new ArrayList(); + + DbScanState(String dbName) { + this.dbName = dbName; + } + } + + private static final class LiveTableScope { + final String dbName; + final String tableName; + final TablePath tablePath; + final long tableId; + final TableInfo tableInfo; + final boolean partitioned; + boolean partitionInfosComplete = true; + final List partitions = new ArrayList(); + final Set activePartitionIds = new LinkedHashSet(); + + LiveTableScope(String dbName, String tableName, TableInfo tableInfo) { + this.dbName = dbName; + this.tableName = tableName; + this.tablePath = tableInfo.getTablePath(); + this.tableId = tableInfo.getTableId(); + this.tableInfo = tableInfo; + this.partitioned = tableInfo.isPartitioned(); + } + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java new file mode 100644 index 0000000000..0e24fa7399 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.job; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.flink.action.orphan.audit.AuditLogger; +import org.apache.fluss.fs.FsPath; + +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.BoundedOneInput; +import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Stage 3 of the orphan files cleanup job. Runs at parallelism=1 to aggregate {@link CleanStats} + * from all Stage 2 subtasks and perform the final empty-directory sweep. + * + *

Implemented as a custom operator (not ProcessFunction) because {@code ProcessOperator} does + * not implement {@link BoundedOneInput} — the {@code endInput()} callback would never fire. This + * operator accumulates all incoming stats and performs the empty-dir sweep in {@code endInput()}. + */ +@Internal +public final class StatsAggregateOperator extends AbstractStreamOperator + implements OneInputStreamOperator, BoundedOneInput { + + private static final long serialVersionUID = 1L; + private static final Logger LOG = LoggerFactory.getLogger(StatsAggregateOperator.class); + + private final boolean dryRun; + + private transient CleanStats accumulated; + + public StatsAggregateOperator(boolean dryRun) { + this.dryRun = dryRun; + } + + @Override + public void processElement(StreamRecord element) { + if (accumulated == null) { + accumulated = CleanStats.empty(); + } + accumulated = accumulated.merge(element.getValue()); + } + + @Override + public void endInput() { + if (accumulated == null) { + accumulated = CleanStats.empty(); + } + + long emptyDirsRemoved = sweepEmptyDirs(accumulated.touchedDirs()); + long totalDeleted = accumulated.deleted() + emptyDirsRemoved; + + CleanStats finalStats = + new CleanStats( + accumulated.scanned(), + totalDeleted, + accumulated.deleteFailures(), + accumulated.bytesReclaimed(), + new ArrayList()); + + LOG.info( + "Orphan cleanup complete: scanned={}, deleted={} (files={}, emptyDirs={}), " + + "failures={}, bytesReclaimed={}", + finalStats.scanned(), + totalDeleted, + accumulated.deleted(), + emptyDirsRemoved, + finalStats.deleteFailures(), + finalStats.bytesReclaimed()); + + output.collect(new StreamRecord<>(finalStats)); + } + + private long sweepEmptyDirs(List touchedDirs) { + if (touchedDirs.isEmpty()) { + return 0L; + } + AuditLogger audit = new AuditLogger(); + EmptyDirSweeper sweeper = new EmptyDirSweeper(dryRun, audit); + for (String dir : touchedDirs) { + sweeper.registerTouched(new FsPath(dir)); + } + try { + return sweeper.sweep(); + } catch (IOException e) { + LOG.warn("Empty directory sweep encountered errors", e); + return 0L; + } + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/BucketActiveRefs.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/BucketActiveRefs.java new file mode 100644 index 0000000000..73a847dd75 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/BucketActiveRefs.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.rule; + +import org.apache.fluss.annotation.Internal; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** Immutable view of all active references for a single bucket / table partition. */ +@Internal +public final class BucketActiveRefs { + + private static final BucketActiveRefs EMPTY = + new BucketActiveRefs( + Collections.emptySet(), Collections.emptySet(), Collections.emptySet()); + + private final Set logSegmentRelativePaths; + private final Set kvActiveSnapDirs; + private final Set logActiveManifestPaths; + + public BucketActiveRefs( + Set logSegmentRelativePaths, + Set kvActiveSnapDirs, + Set logActiveManifestPaths) { + this.logSegmentRelativePaths = + Collections.unmodifiableSet(new HashSet<>(logSegmentRelativePaths)); + this.kvActiveSnapDirs = Collections.unmodifiableSet(new HashSet<>(kvActiveSnapDirs)); + this.logActiveManifestPaths = + Collections.unmodifiableSet(new HashSet<>(logActiveManifestPaths)); + } + + public static BucketActiveRefs empty() { + return EMPTY; + } + + public Set logSegmentRelativePaths() { + return logSegmentRelativePaths; + } + + /** + * Returns the set of active {@code snap-} directory names for the bucket. + * + *

The set is the union of two server-side categories the {@code ListKvSnapshots} RPC emits + * as one flat list (client does not distinguish): + * + *

    + *
  • RETAINED — the most recent N completed snapshots kept per the retention window. + *
  • STILL_IN_USE — snapshots pinned by an active lease; emitted unconditionally even when + * the corresponding ZK znode has been removed, on the principle "may over-count active, + * must never under-count." + *
+ * + *

A KV snap-private file is preserved iff its parent directory's name is in this set. + */ + public Set kvActiveSnapDirs() { + return kvActiveSnapDirs; + } + + /** + * Returns the set of active log manifest paths reported by {@code ListRemoteLogManifests}. The + * "current" manifest for a bucket is always also a member of this set, so {@link + * LogManifestRule} only needs to check this single collection. + */ + public Set logActiveManifestPaths() { + return logActiveManifestPaths; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/Decision.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/Decision.java new file mode 100644 index 0000000000..491281a22e --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/Decision.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.rule; + +import org.apache.fluss.annotation.Internal; + +/** Decision returned by a {@link FileRule} for a given file. */ +@Internal +public enum Decision { + + /** File is orphan and should be deleted. */ + DELETE, + + /** File is referenced by an active object (manifest, snapshot, etc.). */ + KEEP_ACTIVE, + + /** + * File is not in the active set but its age is under the {@code --older-than} threshold; the + * deletion verdict is deferred to a future cleanup round, by which time the file will either + * have entered the active set (KEEP_ACTIVE) or aged past the threshold (DELETE). The grace + * window prevents racing in-flight writes whose manifest entry has not yet been committed. + */ + DEFER, + + /** File path or extension is not recognized; skip without deletion. */ + SKIP_UNKNOWN +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/FileMeta.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/FileMeta.java new file mode 100644 index 0000000000..74072de4fa --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/FileMeta.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.rule; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.fs.FsPath; + +/** Immutable metadata describing a candidate file evaluated by {@link FileRule}. */ +@Internal +public final class FileMeta { + + private final FsPath path; + private final long size; + private final long modificationTime; + + public FileMeta(FsPath path, long size, long modificationTime) { + this.path = path; + this.size = size; + this.modificationTime = modificationTime; + } + + public FsPath path() { + return path; + } + + public long size() { + return size; + } + + public long modificationTime() { + return modificationTime; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/FileRule.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/FileRule.java new file mode 100644 index 0000000000..af9a01468a --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/FileRule.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.rule; + +import org.apache.fluss.annotation.Internal; + +/** Rule that decides whether a single file is orphan. */ +@Internal +public interface FileRule { + + /** Stable identifier used in audit logs. */ + RuleId id(); + + /** + * Decide what to do with the given file. + * + * @param cutoffMillis absolute epoch-ms cutoff: a file whose mtime is {@code < cutoffMillis} is + * age-eligible for deletion (a {@link Decision#DELETE}); a file whose mtime is {@code >= + * cutoffMillis} is {@link Decision#DEFER}red. Pre-frozen at action start; does not slide + * during a run. + */ + Decision evaluate(FileMeta file, BucketActiveRefs activeRefs, long cutoffMillis); +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/KvSharedSstRule.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/KvSharedSstRule.java new file mode 100644 index 0000000000..8fc1e5b2c0 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/KvSharedSstRule.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.rule; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.utils.FlussPaths; + +/** + * Rule for shared SST files under the {@code shared/} KV directory. + * + *

Always returns {@link Decision#KEEP_ACTIVE}. The true active set for shared SSTs lives inside + * the engine's {@code SharedKvFileRegistry}; orphan cleanup has no read path into that registry, so + * any deletion here would be a guess. Per the action's hard constraint "prefer leak over + * mis-delete," the rule never deletes, and as a consequence orphan PK-table / orphan-partition + * directories permanently retain their {@code shared/} subtree as accepted residue (recovering that + * residue would require a registry-backed GC channel that is out of scope for this action). + */ +@Internal +public final class KvSharedSstRule implements FileRule { + + @Override + public RuleId id() { + return RuleId.KV_SHARED_SST; + } + + @Override + public Decision evaluate(FileMeta file, BucketActiveRefs activeRefs, long cutoffMillis) { + FsPath parent = file.path().getParent(); + if (parent == null || !FlussPaths.REMOTE_KV_SNAPSHOT_SHARED_DIR.equals(parent.getName())) { + return Decision.SKIP_UNKNOWN; + } + if (!file.path().getName().endsWith(".sst")) { + return Decision.SKIP_UNKNOWN; + } + return Decision.KEEP_ACTIVE; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/KvSnapshotFileRule.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/KvSnapshotFileRule.java new file mode 100644 index 0000000000..0700b9563f --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/KvSnapshotFileRule.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.rule; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.utils.FlussPaths; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** + * Rule for files under a {@code snap-/} KV snapshot directory. + * + *

Match key is the file's parent {@code snap-} directory name: if that name is in {@link + * BucketActiveRefs#kvActiveSnapDirs()} (which carries the per-bucket union of RETAINED + + * STILL_IN_USE entries from {@code ListKvSnapshots}, see that getter's javadoc) the file is {@link + * Decision#KEEP_ACTIVE}. + * + *

The set-based check is what prevents retained non-latest snapshots from being misclassified as + * orphan — e.g. with {@code kv.snapshot.num-retained=2}, {@code snap-9} is still active while + * {@code snap-10} is the latest. + */ +@Internal +public final class KvSnapshotFileRule implements FileRule { + + private static final String SNAP_DIR_PREFIX = FlussPaths.REMOTE_KV_SNAPSHOT_DIR_PREFIX; + + private static final Set KNOWN_FIXED_NAMES = + new HashSet(Arrays.asList("_METADATA", "CURRENT", "LOG", "IDENTITY")); + + @Override + public RuleId id() { + return RuleId.KV_SNAPSHOT_FILE; + } + + @Override + public Decision evaluate(FileMeta file, BucketActiveRefs activeRefs, long cutoffMillis) { + FsPath parent = file.path().getParent(); + if (parent == null) { + return Decision.SKIP_UNKNOWN; + } + + String parentName = parent.getName(); + if (!parentName.startsWith(SNAP_DIR_PREFIX)) { + return Decision.SKIP_UNKNOWN; + } + + // Parent must be snap-; reject e.g. snap-, snap-abc. + String snapIdPart = parentName.substring(SNAP_DIR_PREFIX.length()); + if (snapIdPart.isEmpty()) { + return Decision.SKIP_UNKNOWN; + } + for (int i = 0; i < snapIdPart.length(); i++) { + if (!Character.isDigit(snapIdPart.charAt(i))) { + return Decision.SKIP_UNKNOWN; + } + } + + if (!isKnownSnapshotFile(file.path().getName())) { + return Decision.SKIP_UNKNOWN; + } + + if (activeRefs.kvActiveSnapDirs().contains(parentName)) { + return Decision.KEEP_ACTIVE; + } + + return file.modificationTime() < cutoffMillis ? Decision.DELETE : Decision.DEFER; + } + + private static boolean isKnownSnapshotFile(String fileName) { + if (KNOWN_FIXED_NAMES.contains(fileName)) { + return true; + } + if (fileName.startsWith("MANIFEST-") || fileName.startsWith("OPTIONS-")) { + return true; + } + return fileName.endsWith(".sst") || fileName.endsWith(".log"); + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/LogManifestRule.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/LogManifestRule.java new file mode 100644 index 0000000000..23fb5d5edd --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/LogManifestRule.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.rule; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.utils.FlussPaths; + +/** + * Rule for manifest files under the {@code metadata/} directory of a log bucket. + * + *

Default behavior is to return {@link Decision#KEEP_ACTIVE} for every manifest. The asymmetry + * is the reason: mis-deleting an active manifest leaves the coordinator's manifest pointer dangling + * and breaks the bucket's metadata chain entirely, while keeping orphan manifests is structurally + * harmless (KB-sized files). Operators opt into the destructive path via {@code + * allowDeleteManifest=true} (driven by the {@code --allow-delete-manifest} CLI flag); only then + * does the rule consult the active-manifest set and apply the file-level age threshold. + */ +@Internal +public final class LogManifestRule implements FileRule { + + private final boolean allowDeleteManifest; + + /** Default-conservative constructor: {@code allowDeleteManifest=false}. */ + public LogManifestRule() { + this(false); + } + + public LogManifestRule(boolean allowDeleteManifest) { + this.allowDeleteManifest = allowDeleteManifest; + } + + @Override + public RuleId id() { + return RuleId.LOG_MANIFEST; + } + + @Override + public Decision evaluate(FileMeta file, BucketActiveRefs activeRefs, long cutoffMillis) { + FsPath path = file.path(); + FsPath parent = path.getParent(); + if (parent == null + || !FlussPaths.REMOTE_LOG_METADATA_DIR_NAME.equals(parent.getName()) + || !path.getName().endsWith(".manifest")) { + return Decision.SKIP_UNKNOWN; + } + + // Default-conservative: never delete a manifest. Keeping orphans is harmless; deleting an + // active manifest leaves the coordinator's manifest pointer dangling and breaks the + // bucket's metadata chain. + if (!allowDeleteManifest) { + return Decision.KEEP_ACTIVE; + } + + // Opt-in path: preserve the original active-set + cutoff semantics. The "current" bucket + // manifest is always present in logActiveManifestPaths (the server emits one path per + // bucket in ListRemoteLogManifests), so a single set lookup suffices. + String pathString = path.toString(); + if (activeRefs.logActiveManifestPaths().contains(pathString)) { + return Decision.KEEP_ACTIVE; + } + + return file.modificationTime() < cutoffMillis ? Decision.DELETE : Decision.DEFER; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/LogSegmentRule.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/LogSegmentRule.java new file mode 100644 index 0000000000..1ac4156e8f --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/LogSegmentRule.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.rule; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.utils.FlussPaths; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * Rule for log-segment files under a remote log bucket. + * + *

{@code .writer_snapshot} files are only eligible for deletion in orphan-directory mode. In + * active-bucket mode the engine's own TTL cleanup handles them; the orphan tool conservatively + * keeps them to avoid any risk of racing a concurrent write. + */ +@Internal +public final class LogSegmentRule implements FileRule { + + private static final Pattern SEGMENT_DIR_PATTERN = + Pattern.compile( + "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}" + + "-[0-9a-fA-F]{12}"); + + private static final Set KNOWN_SUFFIXES = + new HashSet(Arrays.asList(".log", ".index", ".timeindex", ".writer_snapshot")); + + private final boolean orphanDirMode; + + public LogSegmentRule() { + this(false); + } + + public LogSegmentRule(boolean orphanDirMode) { + this.orphanDirMode = orphanDirMode; + } + + @Override + public RuleId id() { + return RuleId.LOG_SEGMENT; + } + + @Override + public Decision evaluate(FileMeta file, BucketActiveRefs activeRefs, long cutoffMillis) { + FsPath path = file.path(); + FsPath parent = path.getParent(); + if (parent == null || !isSegmentDir(parent.getName()) || !hasKnownSuffix(path.getName())) { + return Decision.SKIP_UNKNOWN; + } + + String relativePath = parent.getName() + "/" + path.getName(); + if (activeRefs.logSegmentRelativePaths().contains(relativePath)) { + return Decision.KEEP_ACTIVE; + } + + if (path.getName().endsWith(FlussPaths.WRITER_SNAPSHOT_FILE_SUFFIX) && !orphanDirMode) { + return Decision.KEEP_ACTIVE; + } + + return file.modificationTime() < cutoffMillis ? Decision.DELETE : Decision.DEFER; + } + + static boolean isSegmentDir(String dirName) { + return SEGMENT_DIR_PATTERN.matcher(dirName).matches(); + } + + private static boolean hasKnownSuffix(String fileName) { + String name = fileName; + if (name.endsWith(FlussPaths.DELETED_FILE_SUFFIX)) { + name = name.substring(0, name.length() - FlussPaths.DELETED_FILE_SUFFIX.length()); + } + for (String suffix : KNOWN_SUFFIXES) { + if (name.endsWith(suffix)) { + return true; + } + } + return false; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/OrphanDirDetector.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/OrphanDirDetector.java new file mode 100644 index 0000000000..5762ff51c2 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/OrphanDirDetector.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.rule; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.annotation.VisibleForTesting; + +import javax.annotation.Nullable; + +import java.util.Set; + +/** + * Detects orphan table and partition directories by ID guard. + * + *

A directory is an orphan candidate iff its parsed ID is not in the active set and does not + * exceed the last-known maximum (conservatively treating IDs above the max as freshly allocated). + * Unrecognizable directory names are never flagged. + */ +@Internal +public final class OrphanDirDetector { + + private OrphanDirDetector() {} + + /** + * Returns {@code true} if the directory name matches {@code {name}-{tableId}} and the parsed ID + * is not in {@code activeTableIds} and is {@code <= maxKnownTableId}. + */ + public static boolean isOrphanTable( + String dirName, Set activeTableIds, long maxKnownTableId) { + Long parsed = parseTableId(dirName); + if (parsed == null) { + return false; + } + if (activeTableIds.contains(parsed)) { + return false; + } + return parsed <= maxKnownTableId; + } + + /** + * Returns {@code true} if the directory name matches {@code {name}-p{partitionId}} and the + * parsed ID is not in {@code activePartitionIds} and is {@code <= maxKnownPartitionId}. + */ + public static boolean isOrphanPartition( + String dirName, Set activePartitionIds, long maxKnownPartitionId) { + Long parsed = parsePartitionId(dirName); + if (parsed == null) { + return false; + } + if (activePartitionIds.contains(parsed)) { + return false; + } + return parsed <= maxKnownPartitionId; + } + + @VisibleForTesting + @Nullable + static Long parseTableId(String dirName) { + int dash = dirName.lastIndexOf('-'); + if (dash <= 0 || dash == dirName.length() - 1) { + return null; + } + String idPart = dirName.substring(dash + 1); + for (int i = 0; i < idPart.length(); i++) { + if (!Character.isDigit(idPart.charAt(i))) { + return null; + } + } + try { + return Long.parseLong(idPart); + } catch (NumberFormatException e) { + return null; + } + } + + @VisibleForTesting + @Nullable + static Long parsePartitionId(String dirName) { + int dashP = dirName.lastIndexOf("-p"); + if (dashP <= 0 || dashP == dirName.length() - 2) { + return null; + } + String idPart = dirName.substring(dashP + 2); + for (int i = 0; i < idPart.length(); i++) { + if (!Character.isDigit(idPart.charAt(i))) { + return null; + } + } + try { + return Long.parseLong(idPart); + } catch (NumberFormatException e) { + return null; + } + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/RuleDispatcher.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/RuleDispatcher.java new file mode 100644 index 0000000000..9880c6e64d --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/RuleDispatcher.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.rule; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.utils.FlussPaths; + +/** Dispatches a candidate file to the matching orphan-cleanup rule. */ +@Internal +public final class RuleDispatcher { + + private static final FileRule UNKNOWN_RULE = + new FileRule() { + @Override + public RuleId id() { + return RuleId.UNKNOWN; + } + + @Override + public Decision evaluate( + FileMeta file, BucketActiveRefs activeRefs, long cutoffMillis) { + return Decision.SKIP_UNKNOWN; + } + }; + + private final FileRule logSegmentRule; + private final FileRule logManifestRule; + private final FileRule kvSnapshotFileRule = new KvSnapshotFileRule(); + private final FileRule kvSharedSstRule = new KvSharedSstRule(); + + public RuleDispatcher() { + this(false, false); + } + + public RuleDispatcher(boolean allowDeleteManifest) { + this(allowDeleteManifest, false); + } + + public RuleDispatcher(boolean allowDeleteManifest, boolean orphanDirMode) { + this.logSegmentRule = new LogSegmentRule(orphanDirMode); + this.logManifestRule = new LogManifestRule(allowDeleteManifest); + } + + public FileRule dispatch(FileMeta file) { + FsPath path = file.path(); + FsPath parent = path.getParent(); + if (parent == null) { + return UNKNOWN_RULE; + } + + String parentName = parent.getName(); + if (FlussPaths.REMOTE_LOG_METADATA_DIR_NAME.equals(parentName)) { + return logManifestRule; + } + if (FlussPaths.REMOTE_KV_SNAPSHOT_SHARED_DIR.equals(parentName)) { + return kvSharedSstRule; + } + if (parentName.startsWith(FlussPaths.REMOTE_KV_SNAPSHOT_DIR_PREFIX)) { + return kvSnapshotFileRule; + } + if (LogSegmentRule.isSegmentDir(parentName)) { + return logSegmentRule; + } + return UNKNOWN_RULE; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/RuleId.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/RuleId.java new file mode 100644 index 0000000000..ab2de83db2 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/rule/RuleId.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.rule; + +import org.apache.fluss.annotation.Internal; + +/** Enumeration of all file-level rule identifiers used in orphan cleanup audit logs. */ +@Internal +public enum RuleId { + LOG_SEGMENT("log-segment"), + LOG_MANIFEST("log-manifest"), + KV_SNAPSHOT_FILE("kv-snapshot-file"), + KV_SHARED_SST("kv-shared-sst"), + UNKNOWN("unknown"); + + private final String auditTag; + + RuleId(String auditTag) { + this.auditTag = auditTag; + } + + /** Stable string written to audit logs. */ + public String auditTag() { + return auditTag; + } + + @Override + public String toString() { + return auditTag; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory b/fluss-flink/fluss-flink-common/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory new file mode 100644 index 0000000000..c30c9dd5ab --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.fluss.flink.action.orphan.OrphanFilesCleanActionFactory diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java new file mode 100644 index 0000000000..b139614136 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java @@ -0,0 +1,1157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.flink.action.orphan.config.OrphanCleanConfig; +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.PhysicalTablePath; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.server.zk.ZooKeeperClient; +import org.apache.fluss.server.zk.data.BucketSnapshot; +import org.apache.fluss.server.zk.data.RemoteLogManifestHandle; +import org.apache.fluss.server.zk.data.ZkData.BucketSnapshotIdZNode; +import org.apache.fluss.server.zk.data.ZkData.PartitionZNode; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.FlussPaths; + +import org.apache.flink.api.java.utils.MultipleParameterTool; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.LoggerConfig; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileTime; +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.assertj.core.api.Assertions.assertThat; + +/** End-to-end tests for orphan files cleanup safety scenarios. */ +class OrphanFilesCleanITCase { + + @RegisterExtension + static final FlussClusterExtension FLUSS_CLUSTER_EXTENSION = + FlussClusterExtension.builder() + .setClusterConf(buildClusterConf()) + .setNumOfTabletServers(1) + .build(); + + private static Configuration buildClusterConf() { + Configuration clusterConf = new Configuration(); + clusterConf.set(ConfigOptions.KV_MAX_RETAINED_SNAPSHOTS, 2); + return clusterConf; + } + + private static Connection connection; + private static Admin admin; + private static String bootstrapServers; + + private CapturingAppender auditAppender; + private LoggerConfig auditLoggerConfig; + private Level previousAuditLevel; + + @BeforeAll + static void beforeAll() { + bootstrapServers = FLUSS_CLUSTER_EXTENSION.getBootstrapServers(); + Configuration clientConfig = new Configuration(); + clientConfig.setString(ConfigOptions.BOOTSTRAP_SERVERS.key(), bootstrapServers); + connection = ConnectionFactory.createConnection(clientConfig); + admin = connection.getAdmin(); + } + + @AfterAll + static void afterAll() throws Exception { + if (admin != null) { + admin.close(); + admin = null; + } + if (connection != null) { + connection.close(); + connection = null; + } + } + + @BeforeEach + void setUp() { + attachAuditAppender(); + } + + @AfterEach + void tearDown() { + detachAuditAppender(); + } + + private Path remoteDataRoot() { + return Paths.get(URI.create(FLUSS_CLUSTER_EXTENSION.getRemoteDataDir())); + } + + private List auditMessages() { + return auditAppender.messages(); + } + + private void attachAuditAppender() { + LoggerContext context = (LoggerContext) LogManager.getContext(false); + org.apache.logging.log4j.core.config.Configuration config = context.getConfiguration(); + auditAppender = new CapturingAppender("orphan-clean-it-audit"); + auditAppender.start(); + auditLoggerConfig = config.getLoggerConfig("fluss.orphan.audit"); + previousAuditLevel = auditLoggerConfig.getLevel(); + auditLoggerConfig.setLevel(Level.DEBUG); + auditLoggerConfig.addAppender(auditAppender, Level.DEBUG, null); + context.updateLoggers(); + } + + private void detachAuditAppender() { + if (auditLoggerConfig != null && auditAppender != null) { + auditLoggerConfig.removeAppender(auditAppender.getName()); + auditLoggerConfig.setLevel(previousAuditLevel); + ((LoggerContext) LogManager.getContext(false)).updateLoggers(); + auditAppender.stop(); + } + } + + private static final Duration OLD_ENOUGH = Duration.ofDays(2); + + @Test + void happyPathDeletesOrphanSegment() throws Exception { + String dbName = newDatabaseName("happy"); + TablePath tablePath = createLogTable(dbName, "happy_path"); + Path activeSegment = seedActiveBucketManifest(tablePath); + Path orphan = createOldSegmentFile(tablePath, "99999999999999999999.log"); + + runCleanerForDatabase(false, dbName); + + assertThat(Files.exists(orphan)).isFalse(); + assertThat(Files.exists(activeSegment)).isTrue(); + assertThat(auditMessages()) + .anyMatch( + m -> + m.contains("action=deleted") + && m.contains("rule=log-segment") + && m.contains(orphan.toString())); + } + + @Test + void dryRunDoesNotDeleteFiles() throws Exception { + String dbName = newDatabaseName("dryrun"); + TablePath tablePath = createLogTable(dbName, "dry_run"); + Path activeSegment = seedActiveBucketManifest(tablePath); + Path orphan = createOldSegmentFile(tablePath, "99999999999999999999.log"); + + runCleanerForDatabase(true, dbName); + + assertThat(Files.exists(orphan)).isTrue(); + assertThat(Files.exists(activeSegment)).isTrue(); + assertThat(auditMessages()) + .anyMatch( + m -> + m.contains("action=would_delete") + && m.contains("rule=log-segment") + && m.contains(orphan.toString())); + assertThat(auditMessages()).noneMatch(m -> m.contains("action=deleted")); + // Catch a regression that targets the active segment with a would_delete intent: the + // file-existence checks above would silently pass under dry-run even if the planner + // mis-marked the active segment, because dry-run never touches disk. + assertThat(auditMessages()) + .noneMatch( + m -> + m.contains("action=would_delete") + && m.contains(activeSegment.toString())); + } + + @Test + void unknownExtensionFilePreserved() throws Exception { + String dbName = newDatabaseName("unknown"); + TablePath tablePath = createLogTable(dbName, "unknown_file"); + Path activeSegment = seedActiveBucketManifest(tablePath); + Path orphan = createOldSegmentFile(tablePath, "99999999999999999999.log"); + Path unknown = orphan.getParent().resolve("data.bloomfilter"); + Files.write(unknown, new byte[] {0x24}); + makeOld(unknown); + + runCleanerForDatabase(false, dbName); + + assertThat(Files.exists(orphan)).isFalse(); + assertThat(Files.exists(unknown)).isTrue(); + assertThat(Files.exists(activeSegment)).isTrue(); + assertThat(auditMessages()) + .anyMatch( + m -> + m.contains("action=deleted") + && m.contains("rule=log-segment") + && m.contains(orphan.toString())); + assertThat(auditMessages()) + .anyMatch(m -> m.contains("action=skip_unknown") && m.contains(unknown.toString())); + } + + /** + * Seeds a remote log manifest + matching active segment under a freshly-allocated UUID so the + * active-file cleanup reaches {@code ManifestReadStatus.RESOLVED} for bucket 0 of the given log + * table. Returns the active segment's {@code .log} path so callers can assert it survives + * cleanup. + * + *

Without a manifest the bucket falls back to {@code ManifestReadStatus.NOT_LISTED} and the + * active-file cleanup skips the entire bucket (see §4.3.1 of the design doc) — which would + * prevent any orphan file under the bucket from being visited at all. + */ + private Path seedActiveBucketManifest(TablePath tablePath) throws Exception { + TableInfo tableInfo = admin.getTableInfo(tablePath).get(); + TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), 0); + FsPath remoteLogTabletDir = + FlussPaths.remoteLogTabletDir( + new FsPath(remoteDataRoot().resolve("log").toUri().toString()), + PhysicalTablePath.of(tablePath), + tableBucket); + FsPath manifestPath = + new FsPath( + localPath(remoteLogTabletDir) + .resolve("metadata/p0.manifest") + .toUri() + .toString()); + String activeSegmentId = UUID.randomUUID().toString(); + Path activeSegment = + seedManifestAndSegment(remoteLogTabletDir, manifestPath, activeSegmentId, 0L, 0L); + upsertManifest(tableBucket, manifestPath, 0L); + return activeSegment; + } + + @Test + void defaultDoesNotEnterOrphanTableDir() throws Exception { + String dbName = newDatabaseName("defaultskip"); + long tableId = allocateDroppedTableId(dbName, "seed_table"); + createLogTable(dbName, "live_anchor"); + OrphanTableLayout layout = + createOldOrphanTableLayout( + remoteDataRoot(), + dbName, + tableId, + "ghost_table", + "99999999999999999999.log"); + + runCleanerForAllDatabases(false); + + assertThat(Files.exists(layout.orphanFile)).isTrue(); + assertThat(Files.exists(layout.tableDir)).isTrue(); + assertThat(auditMessages()) + .anyMatch( + m -> + m.contains("action=skip_orphan_table") + && m.contains("default-conservative") + && m.contains(layout.tableDir.toString())); + } + + @Test + void optInCleansOrphanTableDirWhenEnabled() throws Exception { + String dbName = newDatabaseName("optin"); + long tableId = allocateDroppedTableId(dbName, "seed_table"); + createLogTable(dbName, "live_anchor"); + OrphanTableLayout layout = + createOldOrphanTableLayout( + remoteDataRoot(), + dbName, + tableId, + "ghost_table", + "99999999999999999999.log"); + + runCleanerForAllDatabases(false, "--allow-clean-orphan-tables"); + + assertThat(Files.exists(layout.orphanFile)).isFalse(); + assertThat(Files.exists(layout.tableDir)).isFalse(); + assertThat(auditMessages()) + .anyMatch( + m -> + m.contains("action=deleted") + && m.contains("rule=log-segment") + && m.contains(layout.orphanFile.toString())); + } + + @Test + void scanRootIncludesAdditionalRemoteRootWhenOrphanTableCleanupEnabled(@TempDir Path extraRoot) + throws Exception { + String dbName = newDatabaseName("scanroot"); + long tableId = allocateDroppedTableId(dbName, "seed_table"); + createLogTable(dbName, "live_anchor"); + OrphanTableLayout layout = + createOldOrphanTableLayout( + extraRoot, dbName, tableId, "external_table", "99999999999999999999.log"); + + runCleanerForDatabase( + false, + dbName, + "--scan-root", + extraRoot.toUri().toString(), + "--allow-clean-orphan-tables"); + + assertThat(Files.exists(layout.orphanFile)).isFalse(); + assertThat(Files.exists(layout.tableDir)).isFalse(); + assertThat(auditMessages()) + .anyMatch( + m -> + m.contains("action=deleted") + && m.contains("rule=log-segment") + && m.contains(layout.orphanFile.toString())); + } + + @Test + void livePrimaryKeyTableDoesNotCleanKvSharedFiles() throws Exception { + String dbName = newDatabaseName("livepk"); + TablePath tablePath = createPrimaryKeyTable(dbName, "live_pk_table"); + Path orphanKvFile = + createOldKvSharedSstFile( + tablePath, "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa-orphan.sst"); + + runCleanerForDatabase(false, dbName); + + assertThat(Files.exists(orphanKvFile)).isTrue(); + assertThat(auditMessages()) + .noneMatch( + m -> + m.contains("rule=kv-shared-sst") + && m.contains(orphanKvFile.toString())); + } + + @Test + void pkOrphanTableRetainsSharedSstEvenWithOptIn() throws Exception { + String dbName = newDatabaseName("orphankv"); + long tableId = allocateDroppedPrimaryKeyTableId(dbName, "seed_pk_table"); + createLogTable(dbName, "live_anchor"); + OrphanTableLayout layout = + createOldOrphanKvTableLayout( + remoteDataRoot(), + dbName, + tableId, + "ghost_pk_table", + "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa-orphan.sst"); + + runCleanerForDatabase(false, dbName, "--allow-clean-orphan-tables"); + + assertThat(Files.exists(layout.orphanFile)).isTrue(); + assertThat(Files.exists(layout.tableDir)).isTrue(); + assertThat(auditMessages()) + .noneMatch( + m -> + m.contains("rule=kv-shared-sst") + && m.contains(layout.orphanFile.toString())); + } + + @Test + void manifestPreservedByDefault() throws Exception { + String dbName = newDatabaseName("manifest"); + TablePath tablePath = createLogTable(dbName, "manifest_default"); + Path orphanManifest = createOldLogManifestFile(tablePath, "orphan.manifest"); + + runCleanerForDatabase(false, dbName); + + assertThat(Files.exists(orphanManifest)).isTrue(); + assertThat(auditMessages()) + .noneMatch( + m -> + m.contains("rule=log-manifest") + && m.contains(orphanManifest.toString())); + } + + @Test + void retainedNonLatestSnapshotPreserved() throws Exception { + String dbName = newDatabaseName("retained"); + TablePath tablePath = createPrimaryKeyTable(dbName, "retained_pk"); + TableInfo tableInfo = admin.getTableInfo(tablePath).get(); + TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), 0); + FsPath remoteKvTabletDir = + FlussPaths.remoteKvTabletDir( + new FsPath(remoteDataRoot().resolve("kv").toUri().toString()), + PhysicalTablePath.of(tablePath), + tableBucket); + + seedKvSnapshots(tableBucket, remoteKvTabletDir, new long[] {1L, 2L, 3L, 4L}); + + runCleanerForDatabase(false, dbName); + + assertThat(Files.exists(localPath(FlussPaths.remoteKvSnapshotDir(remoteKvTabletDir, 1L)))) + .isFalse(); + assertThat(Files.exists(localPath(FlussPaths.remoteKvSnapshotDir(remoteKvTabletDir, 2L)))) + .isFalse(); + assertThat(Files.exists(localPath(FlussPaths.remoteKvSnapshotDir(remoteKvTabletDir, 3L)))) + .isTrue(); + assertThat(Files.exists(localPath(FlussPaths.remoteKvSnapshotDir(remoteKvTabletDir, 4L)))) + .isTrue(); + } + + @Test + void listPartitionInfosFailureScopesToSingleTable() throws Exception { + String dbName = newDatabaseName("partfail"); + PartitionedTableLayout tableA = createPartitionedLogTable(dbName, "table_a", "pa"); + PartitionedTableLayout tableB = createPartitionedLogTable(dbName, "table_b", "pb"); + + long orphanPartitionIdForA = + Math.max( + tableA.partitionInfo.getPartitionId(), + tableB.partitionInfo.getPartitionId()); + long orphanPartitionIdForB = + Math.min( + tableA.partitionInfo.getPartitionId(), + tableB.partitionInfo.getPartitionId()); + + OrphanPartitionLayout orphanA = + createOldOrphanPartitionLayout( + remoteDataRoot(), + tableA.tablePath, + tableA.tableId, + "ghost-a", + orphanPartitionIdForA, + "99999999999999999999.log"); + OrphanPartitionLayout orphanB = + createOldOrphanPartitionLayout( + remoteDataRoot(), + tableB.tablePath, + tableB.tableId, + "ghost-b", + orphanPartitionIdForB, + "99999999999999999999.log"); + + ZooKeeperClient zk = FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(); + String brokenPartitionPath = + PartitionZNode.path(tableA.tablePath, tableA.partitionInfo.getPartitionName()); + byte[] originalPartitionBytes = + zk.getCuratorClient().getData().forPath(brokenPartitionPath); + zk.getCuratorClient() + .setData() + .forPath(brokenPartitionPath, "not-json".getBytes(StandardCharsets.UTF_8)); + try { + runCleanerForDatabase(false, dbName, "--allow-clean-orphan-partitions"); + } finally { + zk.getCuratorClient().setData().forPath(brokenPartitionPath, originalPartitionBytes); + } + + assertThat(Files.exists(orphanA.partitionDir)).isTrue(); + assertThat(Files.exists(orphanA.orphanFile)).isTrue(); + assertThat(Files.exists(orphanB.partitionDir)).isFalse(); + assertThat(Files.exists(orphanB.orphanFile)).isFalse(); + assertThat(auditMessages()) + .anyMatch( + m -> + m.contains("action=skip_partition_list") + && m.contains("table=" + tableA.tablePath.getTableName())); + } + + @Test + void multipleRoundsConvergeAfterManifestUpsert() throws Exception { + String dbName = newDatabaseName("converge"); + TablePath tablePath = createLogTable(dbName, "converge_log"); + TableInfo tableInfo = admin.getTableInfo(tablePath).get(); + TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), 0); + FsPath remoteLogTabletDir = + FlussPaths.remoteLogTabletDir( + new FsPath(remoteDataRoot().resolve("log").toUri().toString()), + PhysicalTablePath.of(tablePath), + tableBucket); + + String segmentId = UUID.randomUUID().toString(); + FsPath manifest0 = + new FsPath( + localPath(remoteLogTabletDir) + .resolve("metadata/p0.manifest") + .toUri() + .toString()); + Path oldSegment = seedManifestAndSegment(remoteLogTabletDir, manifest0, segmentId, 0L, 0L); + upsertManifest(tableBucket, manifest0, 0L); + + runCleanerForDatabase(false, dbName); + + assertThat(Files.exists(oldSegment)).isTrue(); + + FsPath manifest1 = + new FsPath( + localPath(remoteLogTabletDir) + .resolve("metadata/p1.manifest") + .toUri() + .toString()); + Path newSegment = + seedManifestAndSegment(remoteLogTabletDir, manifest1, segmentId, 100L, 100L); + upsertManifest(tableBucket, manifest1, 100L); + + runCleanerForDatabase(false, dbName); + + assertThat(Files.exists(oldSegment)).isFalse(); + assertThat(Files.exists(newSegment)).isTrue(); + assertThat(auditMessages()) + .anyMatch( + m -> + m.contains("action=deleted") + && m.contains("rule=log-segment") + && m.contains(oldSegment.toString())); + } + + @Test + void logBucketSkippedOnNoRemoteManifest() throws Exception { + String dbName = newDatabaseName("logbucketskip"); + TablePath tablePath = createLogTable(dbName, "no_manifest_yet"); + TableInfo tableInfo = admin.getTableInfo(tablePath).get(); + + runCleanerForDatabase(false, dbName); + + assertThat(auditMessages()) + .anyMatch( + m -> + m.contains("action=skip_log_bucket") + && m.contains("reason=no_remote_manifest") + && m.contains("table_id=" + tableInfo.getTableId()) + && m.contains("bucket_id=0")); + } + + @Test + void kvBucketSkippedOnEmptyBucketActiveRefs() throws Exception { + String dbName = newDatabaseName("kvbucketskip"); + TablePath tablePath = createPrimaryKeyTable(dbName, "no_snapshots_yet"); + TableInfo tableInfo = admin.getTableInfo(tablePath).get(); + + runCleanerForDatabase(false, dbName); + + assertThat(auditMessages()) + .anyMatch( + m -> + m.contains("action=skip_kv_bucket") + && m.contains("reason=empty_active_set") + && m.contains("table_id=" + tableInfo.getTableId()) + && m.contains("bucket_id=0")); + } + + @Test + void singleTableModeSkipsOrphanTableScan() throws Exception { + String dbName = newDatabaseName("singletable"); + long orphanTableId = allocateDroppedTableId(dbName, "orphan_seed"); + TablePath liveTable = createLogTable(dbName, "live_target"); + OrphanTableLayout layout = + createOldOrphanTableLayout( + remoteDataRoot(), + dbName, + orphanTableId, + "ghost_table", + "99999999999999999999.log"); + + runCleanerForDatabase( + false, dbName, "--table", liveTable.getTableName(), "--allow-clean-orphan-tables"); + + // The orphan-table scan must skip because tableInfosComplete=false in --table + // single-table mode. + // Sibling orphan must be preserved even with --allow-clean-orphan-tables set. + assertThat(Files.exists(layout.orphanFile)).isTrue(); + assertThat(Files.exists(layout.tableDir)).isTrue(); + assertThat(auditMessages()) + .anyMatch( + m -> + m.contains("action=skip_orphan_table_scan") + && m.contains("reason=tableInfos-incomplete") + && m.contains("db=" + dbName)); + // Must use the dedicated event, not the older skip_db. + assertThat(auditMessages()) + .noneMatch(m -> m.contains("action=skip_db") && m.contains("db=" + dbName)); + } + + @Test + void kvUnitFailureDoesNotBlockLogCleanup() throws Exception { + String dbName = newDatabaseName("crossflow"); + TablePath tablePath = createPrimaryKeyTable(dbName, "fail_kv_keep_log"); + TableInfo tableInfo = admin.getTableInfo(tablePath).get(); + TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), 0); + + // Seed a valid KV snapshot in ZK so listBucketSnapshots returns a child to decode. + FsPath remoteKvTabletDir = + FlussPaths.remoteKvTabletDir( + new FsPath(remoteDataRoot().resolve("kv").toUri().toString()), + PhysicalTablePath.of(tablePath), + tableBucket); + long activeSnapshotId = 1L; + seedKvSnapshots(tableBucket, remoteKvTabletDir, new long[] {activeSnapshotId}); + + // Seed a log manifest + active segment so the log bucket reaches RESOLVED in the + // active-file cleanup. + Path activeLogSegment = seedActiveBucketManifest(tablePath); + + // ----------------------------------------------------------------- + // Step 1 — baseline (no fault injection) + // Plant an orphan KV snapshot dir under snap-99 (NOT registered in ZK) plus an + // orphan log segment. With the cluster wired normally, cleanup MUST delete them: + // this establishes the negative control that proves the phase-2 preservation + // claim is meaningful and not just an accidental no-op. + // ----------------------------------------------------------------- + long baselineOrphanSnapshotId = 99L; + FsPath baselineOrphanKvDir = + FlussPaths.remoteKvSnapshotDir(remoteKvTabletDir, baselineOrphanSnapshotId); + Path baselineOrphanKvMetadata = localPath(baselineOrphanKvDir).resolve("_METADATA"); + Path baselineOrphanKvSst = + localPath(baselineOrphanKvDir).resolve(baselineOrphanSnapshotId + ".sst"); + Files.createDirectories(localPath(baselineOrphanKvDir)); + Files.write(baselineOrphanKvMetadata, new byte[] {0x55}); + Files.write(baselineOrphanKvSst, new byte[] {0x66}); + makeOld(baselineOrphanKvMetadata); + makeOld(baselineOrphanKvSst); + + Path baselineOrphanLogSegment = createOldSegmentFile(tablePath, "99999999999999999999.log"); + + runCleanerForDatabase(false, dbName); + + // Baseline: snap-99 files were DELETED, proving normal cleanup would have killed + // them. Path-specific assertions guarantee these audit events refer to phase 1. + assertThat(Files.exists(baselineOrphanKvMetadata)) + .as( + "phase 1 baseline: snap-99/_METADATA must be DELETED " + + "(cleanup would normally remove orphan KV files)") + .isFalse(); + assertThat(Files.exists(baselineOrphanKvSst)) + .as("phase 1 baseline: snap-99/.sst must be DELETED") + .isFalse(); + assertThat(auditMessages()) + .anyMatch( + m -> + m.contains("action=deleted") + && m.contains("rule=kv-snapshot-file") + && m.contains(baselineOrphanKvMetadata.toString())); + assertThat(auditMessages()) + .anyMatch( + m -> + m.contains("action=deleted") + && m.contains("rule=kv-snapshot-file") + && m.contains(baselineOrphanKvSst.toString())); + // Baseline: orphan log segment was DELETED and the active segment survived. Phase 1's + // log deletion is asserted both via Files.exists and via the audit stream so the final + // phase-2 assertion can require TWO deletion events on the same path (one per phase). + assertThat(Files.exists(baselineOrphanLogSegment)) + .as("phase 1 baseline: orphan log segment must be DELETED") + .isFalse(); + assertThat(Files.exists(activeLogSegment)) + .as("phase 1: active log segment must survive cleanup") + .isTrue(); + assertThat(auditMessages()) + .filteredOn( + m -> + m.contains("action=deleted") + && m.contains("rule=log-segment") + && m.contains(baselineOrphanLogSegment.toString())) + .as("phase 1 baseline: orphan log segment deletion must appear in audit stream") + .hasSizeGreaterThanOrEqualTo(1); + + // ----------------------------------------------------------------- + // Step 2 — fault injection + // Re-plant orphan KV files under a DIFFERENT snap-77 dir so path-specific audit + // assertions are unambiguous (phase-1 audits target snap-99, phase-2 audits + // target snap-77). Re-plant the orphan log segment at its original path (phase 1 + // deleted it) so we can verify log cleanup still proceeds when the KV unit fails. + // ----------------------------------------------------------------- + long faultInjectionOrphanSnapshotId = 77L; + FsPath faultInjectionOrphanKvDir = + FlussPaths.remoteKvSnapshotDir(remoteKvTabletDir, faultInjectionOrphanSnapshotId); + Path faultInjectionOrphanKvMetadata = + localPath(faultInjectionOrphanKvDir).resolve("_METADATA"); + Path faultInjectionOrphanKvSst = + localPath(faultInjectionOrphanKvDir) + .resolve(faultInjectionOrphanSnapshotId + ".sst"); + Files.createDirectories(localPath(faultInjectionOrphanKvDir)); + Files.write(faultInjectionOrphanKvMetadata, new byte[] {0x55}); + Files.write(faultInjectionOrphanKvSst, new byte[] {0x66}); + makeOld(faultInjectionOrphanKvMetadata); + makeOld(faultInjectionOrphanKvSst); + + // Re-planted at the SAME path as baselineOrphanLogSegment (createOldSegmentFile uses a + // fixed UUID + filename), so the audit stream will contain TWO delete events targeting + // this path -- one from each phase. The final + // filteredOn(...).hasSizeGreaterThanOrEqualTo(2) + // assertion below verifies both. + Path faultInjectionOrphanLogSegment = + createOldSegmentFile(tablePath, "99999999999999999999.log"); + + // Corrupt the BucketSnapshot znode bytes so server-side listBucketSnapshots throws on + // decode. Client-side fetchKvActiveSnapDirs propagates the exception and + // cleanActiveTableFiles catches it to emit skip_kv_target. + ZooKeeperClient zk = FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(); + String snapshotZnodePath = BucketSnapshotIdZNode.path(tableBucket, activeSnapshotId); + byte[] originalSnapshotBytes = zk.getCuratorClient().getData().forPath(snapshotZnodePath); + zk.getCuratorClient() + .setData() + .forPath(snapshotZnodePath, "not-json".getBytes(StandardCharsets.UTF_8)); + try { + runCleanerForDatabase(false, dbName); + } finally { + zk.getCuratorClient().setData().forPath(snapshotZnodePath, originalSnapshotBytes); + } + + // KV target was skipped: skip_kv_target audit fires AND snap-77 orphan files preserved. + assertThat(auditMessages()) + .as("phase 2: skip_kv_target audit must fire when LIST_KV_SNAPSHOTS RPC fails") + .anyMatch( + m -> + m.contains("action=skip_kv_target") + && m.contains("table_id=" + tableInfo.getTableId())); + assertThat(Files.exists(faultInjectionOrphanKvMetadata)) + .as( + "phase 2: snap-77/_METADATA must be PRESERVED " + + "(KV target failure must short-circuit cleanup)") + .isTrue(); + assertThat(Files.exists(faultInjectionOrphanKvSst)) + .as("phase 2: snap-77/.sst must be PRESERVED") + .isTrue(); + // Defensive: nothing in the audit stream ever marked snap-77 files for deletion. + assertThat(auditMessages()) + .noneMatch( + m -> + m.contains("action=deleted") + && m.contains("rule=kv-snapshot-file") + && m.contains(faultInjectionOrphanKvMetadata.toString())); + assertThat(auditMessages()) + .noneMatch( + m -> + m.contains("action=deleted") + && m.contains("rule=kv-snapshot-file") + && m.contains(faultInjectionOrphanKvSst.toString())); + + // Log cleanup proceeded independently: orphan log segment DELETED, active preserved. + // The re-planted segment lives at the same path as baselineOrphanLogSegment, so the audit + // stream must contain >=2 deletion events for this path: one from phase 1, one from + // phase 2. anyMatch alone could be satisfied by phase 1's event in isolation, which is + // why we count instead. + assertThat(Files.exists(faultInjectionOrphanLogSegment)) + .as("phase 2: orphan log segment must be re-deleted (log cleanup is independent)") + .isFalse(); + assertThat(Files.exists(activeLogSegment)) + .as("phase 2: active log segment must still survive cleanup") + .isTrue(); + assertThat(auditMessages()) + .filteredOn( + m -> + m.contains("action=deleted") + && m.contains("rule=log-segment") + && m.contains(faultInjectionOrphanLogSegment.toString())) + .as( + "orphan log segment must be deleted in both phase 1 (baseline) and " + + "phase 2 (with KV fault) -- two events on the same path") + .hasSizeGreaterThanOrEqualTo(2); + } + + private TablePath createLogTable(String databaseName, String tableName) throws Exception { + admin.createDatabase(databaseName, DatabaseDescriptor.EMPTY, true).get(); + TablePath tablePath = TablePath.of(databaseName, tableName); + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("value", DataTypes.STRING()) + .build(); + TableDescriptor descriptor = + TableDescriptor.builder().schema(schema).distributedBy(1, "id").build(); + admin.createTable(tablePath, descriptor, true).get(); + return tablePath; + } + + private TablePath createPrimaryKeyTable(String databaseName, String tableName) + throws Exception { + admin.createDatabase(databaseName, DatabaseDescriptor.EMPTY, true).get(); + TablePath tablePath = TablePath.of(databaseName, tableName); + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("value", DataTypes.STRING()) + .primaryKey("id") + .build(); + TableDescriptor descriptor = + TableDescriptor.builder().schema(schema).distributedBy(1, "id").build(); + admin.createTable(tablePath, descriptor, true).get(); + return tablePath; + } + + private long allocateDroppedTableId(String databaseName, String tableName) throws Exception { + TablePath tablePath = createLogTable(databaseName, tableName); + long tableId = admin.getTableInfo(tablePath).get().getTableId(); + admin.dropTable(tablePath, false).get(); + return tableId; + } + + private long allocateDroppedPrimaryKeyTableId(String databaseName, String tableName) + throws Exception { + TablePath tablePath = createPrimaryKeyTable(databaseName, tableName); + long tableId = admin.getTableInfo(tablePath).get().getTableId(); + admin.dropTable(tablePath, false).get(); + return tableId; + } + + private Path createOldSegmentFile(TablePath tablePath, String fileName) throws Exception { + TableInfo tableInfo = admin.getTableInfo(tablePath).get(); + org.apache.fluss.fs.FsPath tabletDir = + FlussPaths.remoteLogTabletDir( + new org.apache.fluss.fs.FsPath( + FLUSS_CLUSTER_EXTENSION.getRemoteDataDir() + + "/" + + FlussPaths.REMOTE_LOG_DIR_NAME), + PhysicalTablePath.of(tablePath), + new TableBucket(tableInfo.getTableId(), 0)); + Path segmentDir = + Paths.get(java.net.URI.create(tabletDir.toString())) + .resolve( + UUID.fromString("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa").toString()); + Files.createDirectories(segmentDir); + Path file = segmentDir.resolve(fileName); + Files.write(file, new byte[] {0x42}); + makeOld(file); + return file; + } + + private Path createOldLogManifestFile(TablePath tablePath, String fileName) throws Exception { + TableInfo tableInfo = admin.getTableInfo(tablePath).get(); + org.apache.fluss.fs.FsPath tabletDir = + FlussPaths.remoteLogTabletDir( + new org.apache.fluss.fs.FsPath( + FLUSS_CLUSTER_EXTENSION.getRemoteDataDir() + + "/" + + FlussPaths.REMOTE_LOG_DIR_NAME), + PhysicalTablePath.of(tablePath), + new TableBucket(tableInfo.getTableId(), 0)); + Path metadataDir = Paths.get(java.net.URI.create(tabletDir.toString())).resolve("metadata"); + Files.createDirectories(metadataDir); + Path file = metadataDir.resolve(fileName); + Files.write(file, new byte[] {0x11}); + makeOld(file); + return file; + } + + private Path createOldKvSharedSstFile(TablePath tablePath, String fileName) throws Exception { + TableInfo tableInfo = admin.getTableInfo(tablePath).get(); + org.apache.fluss.fs.FsPath kvTabletDir = + FlussPaths.remoteKvTabletDir( + new org.apache.fluss.fs.FsPath( + FLUSS_CLUSTER_EXTENSION.getRemoteDataDir() + + "/" + + FlussPaths.REMOTE_KV_DIR_NAME), + PhysicalTablePath.of(tablePath), + new TableBucket(tableInfo.getTableId(), 0)); + org.apache.fluss.fs.FsPath sharedDir = FlussPaths.remoteKvSharedDir(kvTabletDir); + Path localSharedDir = Paths.get(java.net.URI.create(sharedDir.toString())); + Files.createDirectories(localSharedDir); + Path file = localSharedDir.resolve(fileName); + Files.write(file, new byte[] {0x24}); + makeOld(file); + return file; + } + + private PartitionedTableLayout createPartitionedLogTable( + String databaseName, String tableName, String partitionValue) throws Exception { + admin.createDatabase(databaseName, DatabaseDescriptor.EMPTY, true).get(); + TablePath tablePath = TablePath.of(databaseName, tableName); + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("value", DataTypes.STRING()) + .column("pt", DataTypes.STRING()) + .build(); + TableDescriptor descriptor = + TableDescriptor.builder() + .schema(schema) + .distributedBy(1, "id") + .partitionedBy("pt") + .build(); + admin.createTable(tablePath, descriptor, true).get(); + admin.createPartition(tablePath, partitionSpec("pt", partitionValue), false).get(); + + Map partitionIds = + FLUSS_CLUSTER_EXTENSION.waitUntilPartitionAllReady(tablePath, 1); + TableInfo tableInfo = admin.getTableInfo(tablePath).get(); + long partitionId = partitionIds.get(partitionValue); + FLUSS_CLUSTER_EXTENSION.waitUntilTablePartitionReady(tableInfo.getTableId(), partitionId); + List partitionInfos = admin.listPartitionInfos(tablePath).get(); + assertThat(partitionInfos).hasSize(1); + return new PartitionedTableLayout(tablePath, tableInfo.getTableId(), partitionInfos.get(0)); + } + + private void seedKvSnapshots( + TableBucket tableBucket, FsPath remoteKvTabletDir, long[] snapshotIds) + throws Exception { + ZooKeeperClient zk = FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(); + for (long snapshotId : snapshotIds) { + FsPath snapshotDir = FlussPaths.remoteKvSnapshotDir(remoteKvTabletDir, snapshotId); + Path localSnapshotDir = localPath(snapshotDir); + Files.createDirectories(localSnapshotDir); + + Path metadataFile = localSnapshotDir.resolve("_METADATA"); + Files.write(metadataFile, new byte[] {0x33}); + makeOld(metadataFile); + + Path dataFile = localSnapshotDir.resolve(snapshotId + ".sst"); + Files.write(dataFile, new byte[] {0x44}); + makeOld(dataFile); + + makeOld(localSnapshotDir); + + zk.registerTableBucketSnapshot( + tableBucket, + new BucketSnapshot( + snapshotId, snapshotId, snapshotDir.toString() + "/_METADATA")); + } + } + + private Path seedManifestAndSegment( + FsPath remoteLogTabletDir, + FsPath manifestPath, + String segmentId, + long startOffset, + long endOffset) + throws Exception { + Path manifest = localPath(manifestPath); + Files.createDirectories(manifest.getParent()); + Files.write( + manifest, + manifestJson(segmentId, startOffset, endOffset).getBytes(StandardCharsets.UTF_8)); + makeOld(manifest); + + FsPath segmentDir = new FsPath(remoteLogTabletDir, segmentId); + Path localSegmentDir = localPath(segmentDir); + Files.createDirectories(localSegmentDir); + Path logFile = + localSegmentDir.resolve(FlussPaths.filenamePrefixFromOffset(startOffset) + ".log"); + Files.write(logFile, new byte[] {0x55}); + makeOld(logFile); + return logFile; + } + + private void upsertManifest(TableBucket tableBucket, FsPath manifestPath, long endOffset) + throws Exception { + FLUSS_CLUSTER_EXTENSION + .getZooKeeperClient() + .upsertRemoteLogManifestHandle( + tableBucket, new RemoteLogManifestHandle(manifestPath, endOffset)); + } + + private void runCleanerForDatabase(boolean dryRun, String databaseName, String... extraArgs) + throws Exception { + List args = new ArrayList(); + args.add("--bootstrap-server"); + args.add(bootstrapServers); + args.add("--database"); + args.add(databaseName); + appendCommonArgs(args, dryRun, extraArgs); + OrphanCleanConfig config = + OrphanCleanConfig.fromParams( + MultipleParameterTool.fromArgs(args.toArray(new String[args.size()]))); + new OrphanFilesCleanAction(config).run(); + } + + private void runCleanerForAllDatabases(boolean dryRun, String... extraArgs) throws Exception { + List args = new ArrayList(); + args.add("--bootstrap-server"); + args.add(bootstrapServers); + args.add("--all-databases"); + appendCommonArgs(args, dryRun, extraArgs); + OrphanCleanConfig config = + OrphanCleanConfig.fromParams( + MultipleParameterTool.fromArgs(args.toArray(new String[args.size()]))); + new OrphanFilesCleanAction(config).run(); + } + + private static final DateTimeFormatter CUTOFF_FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + private static void appendCommonArgs(List args, boolean dryRun, String... extraArgs) { + // Tests back-date their orphan files to now - 2d via makeOld(); a cutoff at now - 1d + // safely puts those files strictly before the cutoff (mtime < cutoff → DELETE-eligible). + String cutoff = LocalDateTime.now().minusDays(1).format(CUTOFF_FORMATTER); + args.add("--older-than"); + args.add(cutoff); + for (String extraArg : extraArgs) { + args.add(extraArg); + } + if (dryRun) { + args.add("--dry-run"); + } + } + + private OrphanPartitionLayout createOldOrphanPartitionLayout( + Path remoteRoot, + TablePath tablePath, + long tableId, + String partitionName, + long partitionId, + String fileName) + throws Exception { + Path tableDir = + remoteRoot + .resolve("log") + .resolve(tablePath.getDatabaseName()) + .resolve(tablePath.getTableName() + "-" + tableId); + Path partitionDir = tableDir.resolve(partitionName + "-p" + partitionId); + Path segmentDir = + partitionDir + .resolve("0") + .resolve( + UUID.fromString("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb").toString()); + Files.createDirectories(segmentDir); + Path orphanFile = segmentDir.resolve(fileName); + Files.write(orphanFile, new byte[] {0x66}); + makeOld(orphanFile); + makeOld(segmentDir); + makeOld(segmentDir.getParent()); + makeOld(partitionDir); + return new OrphanPartitionLayout(partitionDir, orphanFile); + } + + private OrphanTableLayout createOldOrphanTableLayout( + Path remoteRoot, String dbName, long tableId, String tableName, String fileName) + throws Exception { + Path tableDir = + remoteRoot.resolve("log").resolve(dbName).resolve(tableName + "-" + tableId); + Path segmentDir = + tableDir.resolve("0") + .resolve( + UUID.fromString("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa").toString()); + Files.createDirectories(segmentDir); + Path orphanFile = segmentDir.resolve(fileName); + Files.write(orphanFile, new byte[] {0x42}); + makeOld(orphanFile); + makeOld(segmentDir); + makeOld(segmentDir.getParent()); + makeOld(tableDir); + return new OrphanTableLayout(tableDir, orphanFile); + } + + private OrphanTableLayout createOldOrphanKvTableLayout( + Path remoteRoot, String dbName, long tableId, String tableName, String fileName) + throws Exception { + Path tableDir = remoteRoot.resolve("kv").resolve(dbName).resolve(tableName + "-" + tableId); + Path sharedDir = tableDir.resolve("0").resolve("shared"); + Files.createDirectories(sharedDir); + Path orphanFile = sharedDir.resolve(fileName); + Files.write(orphanFile, new byte[] {0x24}); + makeOld(orphanFile); + makeOld(sharedDir); + makeOld(sharedDir.getParent()); + makeOld(tableDir); + return new OrphanTableLayout(tableDir, orphanFile); + } + + private static String newDatabaseName(String prefix) { + return prefix + Long.toString(System.nanoTime()); + } + + private static PartitionSpec partitionSpec(String key, String value) { + return new PartitionSpec(Collections.singletonMap(key, value)); + } + + private static Path localPath(FsPath path) { + return Paths.get(java.net.URI.create(path.toString())); + } + + private static String manifestJson(String segmentId, long startOffset, long endOffset) { + return "{\"remote_log_segments\":[{" + + "\"segment_id\":\"" + + segmentId + + "\",\"start_offset\":" + + startOffset + + ",\"end_offset\":" + + endOffset + + "}]}"; + } + + private void makeOld(Path path) throws Exception { + Files.setLastModifiedTime( + path, FileTime.fromMillis(System.currentTimeMillis() - OLD_ENOUGH.toMillis())); + } + + private static final class PartitionedTableLayout { + private final TablePath tablePath; + private final long tableId; + private final PartitionInfo partitionInfo; + + private PartitionedTableLayout( + TablePath tablePath, long tableId, PartitionInfo partitionInfo) { + this.tablePath = tablePath; + this.tableId = tableId; + this.partitionInfo = partitionInfo; + } + } + + private static final class OrphanPartitionLayout { + private final Path partitionDir; + private final Path orphanFile; + + private OrphanPartitionLayout(Path partitionDir, Path orphanFile) { + this.partitionDir = partitionDir; + this.orphanFile = orphanFile; + } + } + + private static final class OrphanTableLayout { + private final Path tableDir; + private final Path orphanFile; + + private OrphanTableLayout(Path tableDir, Path orphanFile) { + this.tableDir = tableDir; + this.orphanFile = orphanFile; + } + } + + private static final class CapturingAppender extends AbstractAppender { + + private final List messages = new CopyOnWriteArrayList(); + + CapturingAppender(String name) { + super( + name, + null, + null, + true, + org.apache.logging.log4j.core.config.Property.EMPTY_ARRAY); + } + + @Override + public void append(LogEvent event) { + messages.add(event.getMessage().getFormattedMessage()); + } + + List messages() { + return new ArrayList(messages); + } + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/RpcErrorClassifierTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/RpcErrorClassifierTest.java new file mode 100644 index 0000000000..8746be4ae5 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/RpcErrorClassifierTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan; + +import org.apache.fluss.exception.FlussRuntimeException; +import org.apache.fluss.exception.PartitionNotExistException; +import org.apache.fluss.exception.TableNotExistException; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.concurrent.CompletionException; +import java.util.concurrent.TimeoutException; + +import static org.apache.fluss.flink.action.orphan.RpcErrorClassifier.Category.NOT_FOUND; +import static org.apache.fluss.flink.action.orphan.RpcErrorClassifier.Category.SERVER_ERROR; +import static org.apache.fluss.flink.action.orphan.RpcErrorClassifier.Category.TRANSIENT; +import static org.apache.fluss.flink.action.orphan.RpcErrorClassifier.Category.UNKNOWN; +import static org.assertj.core.api.Assertions.assertThat; + +class RpcErrorClassifierTest { + + @Test + void tableNotExistIsNotFound() { + assertThat(RpcErrorClassifier.classify(new TableNotExistException("x"))) + .isEqualTo(NOT_FOUND); + } + + @Test + void partitionNotExistIsNotFound() { + assertThat(RpcErrorClassifier.classify(new PartitionNotExistException("x"))) + .isEqualTo(NOT_FOUND); + } + + @Test + void ioExceptionIsTransient() { + assertThat(RpcErrorClassifier.classify(new IOException("conn reset"))).isEqualTo(TRANSIENT); + } + + @Test + void timeoutIsTransient() { + assertThat(RpcErrorClassifier.classify(new TimeoutException("rpc"))).isEqualTo(TRANSIENT); + } + + @Test + void unwrapsCompletionException() { + assertThat( + RpcErrorClassifier.classify( + new CompletionException(new TableNotExistException("x")))) + .isEqualTo(NOT_FOUND); + } + + @Test + void flussServerErrorIsServerError() { + assertThat(RpcErrorClassifier.classify(new FlussRuntimeException("internal"))) + .isEqualTo(SERVER_ERROR); + } + + @Test + void otherRuntimeIsUnknown() { + assertThat(RpcErrorClassifier.classify(new IllegalStateException("?"))).isEqualTo(UNKNOWN); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcherTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcherTest.java new file mode 100644 index 0000000000..384d305324 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcherTest.java @@ -0,0 +1,428 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.build; + +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.rpc.messages.ListKvSnapshotsResponse; +import org.apache.fluss.rpc.messages.ListRemoteLogManifestsResponse; +import org.apache.fluss.rpc.messages.PbKvSnapshot; +import org.apache.fluss.rpc.messages.PbRemoteLogManifestEntry; +import org.apache.fluss.utils.FlussPaths; + +import org.junit.jupiter.api.Test; + +import javax.annotation.Nullable; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link ActiveRefsFetcher} — log active set sourced from coordinator metadata. */ +class ActiveRefsFetcherTest { + + @Test + void emptyManifestListReturnsEmptyResult() { + AtomicInteger rpcCalls = new AtomicInteger(0); + StubAdmin admin = new StubAdmin(rpcCalls); + admin.queueEmptyResponse(); + + StubManifestReader reader = new StubManifestReader(); + + ActiveRefsFetcher builder = new ActiveRefsFetcher(admin, reader, /* maxRetries= */ 3); + LogActiveRefsFetchResult result = builder.fetchLogActiveRefsByBucket(7L, null); + + assertThat(result.listOk()).isTrue(); + assertThat(result.respondedBucketIds()).isEmpty(); + // Empty success must NOT trigger a retry — lock down call count. + assertThat(rpcCalls.get()).isEqualTo(1); + } + + @Test + void fileNotFoundMarksBucketReadFailedWithoutRetry() { + // Locks down "no per-bucket retry": a single FileNotFound on the manifest second-read + // immediately marks the bucket READ_FAILED; recovery is left to the next cleanup round. + // This prevents N × retries × IO blow-up during cluster-wide manifest upsert turbulence. + FsPath p0 = new FsPath("oss://b/log/db/t-7/0/metadata/p0.manifest"); + AtomicInteger rpcCalls = new AtomicInteger(0); + StubAdmin admin = new StubAdmin(rpcCalls); + admin.queueResponse(p0); + + StubManifestReader reader = new StubManifestReader(); + reader.failWithNotFound(p0); + + ActiveRefsFetcher builder = new ActiveRefsFetcher(admin, reader, /* maxRetries= */ 3); + LogActiveRefsFetchResult result = builder.fetchLogActiveRefsByBucket(7L, null); + + assertThat(result.listOk()).isTrue(); + assertThat(result.statusFor(0)) + .isEqualTo(LogActiveRefsFetchResult.ManifestReadStatus.READ_FAILED); + assertThat(result.readFailureReason(0)) + .contains("Manifest not found (likely upserted concurrently)") + .contains("bucketId=0"); + // Per-target RPC issued exactly once; no per-bucket retry burst. + assertThat(rpcCalls.get()).isEqualTo(1); + } + + @Test + void fetchLogActiveRefsByBucket_abortsOnlyFailedBucket() throws Exception { + FsPath p0 = new FsPath("oss://b/log/db/t-7/0/metadata/p0.manifest"); + FsPath p1 = new FsPath("oss://b/log/db/t-7/1/metadata/p1.manifest"); + String manifestJson = + "{\"remote_log_segments\":[{" + + "\"segment_id\":\"11111111-1111-1111-1111-111111111111\"," + + "\"start_offset\":7," + + "\"end_offset\":9}]}"; + + AtomicInteger rpcCalls = new AtomicInteger(0); + StubAdmin admin = new StubAdmin(rpcCalls); + admin.queueMultiBucketResponse(p0, p1); + + StubManifestReader reader = new StubManifestReader(); + reader.returnBytes(p0, manifestJson.getBytes(StandardCharsets.UTF_8)); + reader.failWithNotFound(p1); + + ActiveRefsFetcher builder = new ActiveRefsFetcher(admin, reader, /* maxRetries= */ 3); + LogActiveRefsFetchResult result = builder.fetchLogActiveRefsByBucket(7L, null); + + assertThat(result.listOk()).isTrue(); + assertThat(result.statusFor(0)) + .isEqualTo(LogActiveRefsFetchResult.ManifestReadStatus.RESOLVED); + assertThat(result.statusFor(1)) + .isEqualTo(LogActiveRefsFetchResult.ManifestReadStatus.READ_FAILED); + assertThat(result.activeRefsOf(0).logSegmentRelativePaths()) + .containsExactlyInAnyOrder( + "11111111-1111-1111-1111-111111111111/" + + FlussPaths.filenamePrefixFromOffset(7L) + + ".log", + "11111111-1111-1111-1111-111111111111/" + + FlussPaths.filenamePrefixFromOffset(7L) + + ".index", + "11111111-1111-1111-1111-111111111111/" + + FlussPaths.filenamePrefixFromOffset(7L) + + ".timeindex", + "11111111-1111-1111-1111-111111111111/" + + FlussPaths.filenamePrefixFromOffset(9L) + + ".writer_snapshot"); + assertThat(result.readFailureReason(1)) + .contains("Manifest not found (likely upserted concurrently)") + .contains("bucketId=1"); + assertThat(result.statusFor(2)) + .isEqualTo(LogActiveRefsFetchResult.ManifestReadStatus.NOT_LISTED); + // Per-target RPC issued exactly once; per-bucket failure does not trigger any extra RPC. + assertThat(rpcCalls.get()).isEqualTo(1); + } + + @Test + void fetchLogActiveRefsByBucket_targetRpcFailure() { + AtomicInteger rpcCalls = new AtomicInteger(0); + StubAdmin admin = new StubAdmin(rpcCalls); + + ActiveRefsFetcher builder = + new ActiveRefsFetcher(admin, new StubManifestReader(), /* maxRetries= */ 3); + LogActiveRefsFetchResult result = builder.fetchLogActiveRefsByBucket(7L, null); + + assertThat(result.listOk()).isFalse(); + assertThat(result.listFailureReason()).contains("RPC failure for tableId=7"); + // Per-bucket queries are not meaningful when listOk=false. + assertThatThrownBy(() -> result.statusFor(0)).isInstanceOf(IllegalStateException.class); + // Per-target RPC is retried up to maxRetries times before giving up. + assertThat(rpcCalls.get()).isEqualTo(3); + } + + @Test + void manifestParseFailureMarksBucketReadFailed() { + FsPath p0 = new FsPath("oss://b/log/db/t-7/0/metadata/p0.manifest"); + StubAdmin admin = new StubAdmin(new AtomicInteger()); + admin.queueResponse(p0); + + StubManifestReader reader = new StubManifestReader(); + reader.returnBytes(p0, "{}".getBytes(StandardCharsets.UTF_8)); + + ActiveRefsFetcher builder = new ActiveRefsFetcher(admin, reader, /* maxRetries= */ 3); + LogActiveRefsFetchResult result = builder.fetchLogActiveRefsByBucket(7L, null); + + assertThat(result.listOk()).isTrue(); + assertThat(result.statusFor(0)) + .isEqualTo(LogActiveRefsFetchResult.ManifestReadStatus.READ_FAILED); + assertThat(result.readFailureReason(0)) + .contains("Manifest parse failure") + .contains("bucketId=0"); + } + + @Test + void ioErrorMarksBucketReadFailed() { + FsPath p0 = new FsPath("oss://b/log/db/t-7/0/metadata/p0.manifest"); + StubAdmin admin = new StubAdmin(new AtomicInteger()); + admin.queueResponse(p0); + + StubManifestReader reader = new StubManifestReader(); + reader.failWithIo(p0, new IOException("disk fault")); + + ActiveRefsFetcher builder = new ActiveRefsFetcher(admin, reader, /* maxRetries= */ 3); + LogActiveRefsFetchResult result = builder.fetchLogActiveRefsByBucket(7L, null); + + assertThat(result.listOk()).isTrue(); + assertThat(result.statusFor(0)) + .isEqualTo(LogActiveRefsFetchResult.ManifestReadStatus.READ_FAILED); + assertThat(result.readFailureReason(0)).contains("IO error reading manifest"); + } + + @Test + void fetchKvActiveSnapDirsAggregatesPerBucket() { + StubAdmin admin = new StubAdmin(new AtomicInteger()); + admin.queueKvResponse(Arrays.asList(kvSnapshot(0, 9), kvSnapshot(0, 10), kvSnapshot(1, 5))); + + ActiveRefsFetcher builder = + new ActiveRefsFetcher(admin, /* metadataReader */ null, /* maxRetries= */ 3); + KvActiveRefsFetchResult result = builder.fetchKvActiveSnapDirs(7L, null); + + assertThat(result.listOk()).isTrue(); + Map> perBucket = result.activeSnapDirsByBucket(); + assertThat(perBucket.get(0)).containsExactlyInAnyOrder("snap-9", "snap-10"); + assertThat(perBucket.get(1)).containsExactly("snap-5"); + } + + /** + * Symmetric with {@link #fetchLogActiveRefsByBucket_targetRpcFailure}: the KV per-target RPC + * retries up to {@code maxRetries} times and reports {@code listOk=false} on exhaustion. + */ + @Test + void fetchKvActiveSnapDirsRetriesThenReportsListFailure() { + AtomicInteger rpcCalls = new AtomicInteger(0); + StubAdmin admin = new StubAdmin(rpcCalls); + // No queued KV response → StubAdmin returns failed CompletableFutures on every attempt. + + ActiveRefsFetcher builder = + new ActiveRefsFetcher(admin, /* metadataReader */ null, /* maxRetries= */ 3); + KvActiveRefsFetchResult result = builder.fetchKvActiveSnapDirs(7L, null); + + assertThat(result.listOk()).isFalse(); + // Reason is classified via RpcErrorClassifier for audit compatibility. + assertThat(result.listFailureReason()).isNotEmpty(); + // Per-target RPC is retried up to maxRetries times before giving up. + assertThat(rpcCalls.get()).isEqualTo(3); + } + + /** + * Verifies that a non-null {@code partitionId} is forwarded to the underlying {@code + * listRemoteLogManifests} RPC by {@link ActiveRefsFetcher#fetchLogActiveRefsByBucket}. + */ + @Test + void fetchLogActiveRefsByBucketWithPartitionIdRoutesCorrectly() throws Exception { + FsPath p0 = new FsPath("oss://b/log/db/t-7/0/metadata/p0.manifest"); + String manifestJson = + "{\"remote_log_segments\":[{" + + "\"segment_id\":\"11111111-1111-1111-1111-111111111111\"," + + "\"start_offset\":7," + + "\"end_offset\":9}]}"; + + AtomicInteger rpcCalls = new AtomicInteger(0); + StubAdmin admin = new StubAdmin(rpcCalls); + admin.queueResponse(p0); + + StubManifestReader reader = new StubManifestReader(); + reader.returnBytes(p0, manifestJson.getBytes(StandardCharsets.UTF_8)); + + ActiveRefsFetcher builder = new ActiveRefsFetcher(admin, reader, /* maxRetries= */ 3); + LogActiveRefsFetchResult result = builder.fetchLogActiveRefsByBucket(7L, 42L); + + assertThat(result.listOk()).isTrue(); + assertThat(result.statusFor(0)) + .isEqualTo(LogActiveRefsFetchResult.ManifestReadStatus.RESOLVED); + // Proves partitionId=42 was forwarded to the RPC (sentinel Long.MIN_VALUE would mean + // the stub was never invoked). + assertThat(admin.lastLogPartitionId.get()) + .as("partitionId must be forwarded to listRemoteLogManifests RPC") + .isEqualTo(42L); + assertThat(rpcCalls.get()) + .as("happy path must issue exactly one listRemoteLogManifests RPC") + .isEqualTo(1); + } + + /** + * Verifies that a non-null {@code partitionId} is forwarded to the underlying {@code + * listKvSnapshots} RPC by {@link ActiveRefsFetcher#fetchKvActiveSnapDirs}. + */ + @Test + void fetchKvActiveSnapDirsWithPartitionIdRoutesCorrectly() { + AtomicInteger rpcCalls = new AtomicInteger(0); + StubAdmin admin = new StubAdmin(rpcCalls); + admin.queueKvResponse(Arrays.asList(kvSnapshot(0, 5))); + + ActiveRefsFetcher builder = + new ActiveRefsFetcher(admin, /* metadataReader */ null, /* maxRetries= */ 3); + KvActiveRefsFetchResult result = builder.fetchKvActiveSnapDirs(7L, 99L); + + assertThat(result.listOk()).isTrue(); + Map> perBucket = result.activeSnapDirsByBucket(); + assertThat(perBucket).containsOnlyKeys(0); + assertThat(perBucket.get(0)).containsExactly("snap-5"); + // Proves partitionId=99 was forwarded to the RPC. + assertThat(admin.lastKvPartitionId.get()) + .as("partitionId must be forwarded to listKvSnapshots RPC") + .isEqualTo(99L); + assertThat(rpcCalls.get()) + .as("happy path must issue exactly one listKvSnapshots RPC") + .isEqualTo(1); + } + + private static PbKvSnapshot kvSnapshot(int bucketId, long snapshotId) { + return new PbKvSnapshot().setBucketId(bucketId).setSnapshotId(snapshotId); + } + + // ------------------------------------------------------------------------- + // Test fixtures + // ------------------------------------------------------------------------- + + /** Queues per-call responses for ListRemoteLogManifests / ListKvSnapshots and tracks calls. */ + private static final class StubAdmin implements ActiveRefsFetcher.AdminFacade { + + private final Deque responses = new ArrayDeque<>(); + private final Deque kvResponses = new ArrayDeque<>(); + private final AtomicInteger callCounter; + // Sentinel Long.MIN_VALUE differentiates "never invoked" from "invoked with null". + private final AtomicReference lastLogPartitionId = + new AtomicReference<>(Long.MIN_VALUE); + private final AtomicReference lastKvPartitionId = + new AtomicReference<>(Long.MIN_VALUE); + + StubAdmin(AtomicInteger callCounter) { + this.callCounter = callCounter; + } + + void queueResponse(FsPath manifestPath) { + queueResponse(manifestPath, 0); + } + + void queueResponse(FsPath manifestPath, int bucketId) { + ListRemoteLogManifestsResponse response = new ListRemoteLogManifestsResponse(); + PbRemoteLogManifestEntry entry = response.addManifest(); + entry.setTableBucket().setTableId(7L).setBucketId(bucketId); + entry.setRemoteLogManifestPath(manifestPath.toString()); + entry.setRemoteLogEndOffset(0L); + responses.add(response); + } + + void queueMultiBucketResponse(FsPath manifestPath0, FsPath manifestPath1) { + ListRemoteLogManifestsResponse response = new ListRemoteLogManifestsResponse(); + PbRemoteLogManifestEntry entry0 = response.addManifest(); + entry0.setTableBucket().setTableId(7L).setBucketId(0); + entry0.setRemoteLogManifestPath(manifestPath0.toString()); + entry0.setRemoteLogEndOffset(0L); + PbRemoteLogManifestEntry entry1 = response.addManifest(); + entry1.setTableBucket().setTableId(7L).setBucketId(1); + entry1.setRemoteLogManifestPath(manifestPath1.toString()); + entry1.setRemoteLogEndOffset(0L); + responses.add(response); + } + + void queueEmptyResponse() { + responses.add(new ListRemoteLogManifestsResponse()); + } + + void queueKvResponse(List snapshots) { + ListKvSnapshotsResponse response = new ListKvSnapshotsResponse().setTableId(7L); + for (PbKvSnapshot snapshot : snapshots) { + response.addActiveSnapshot().copyFrom(snapshot); + } + kvResponses.add(response); + } + + @Override + public CompletableFuture listRemoteLogManifests( + long tableId, @Nullable Long partitionId) { + callCounter.incrementAndGet(); + lastLogPartitionId.set(partitionId); + ListRemoteLogManifestsResponse next = responses.poll(); + if (next == null) { + CompletableFuture failed = + new CompletableFuture<>(); + failed.completeExceptionally( + new IllegalStateException("StubAdmin: no more queued responses")); + return failed; + } + return CompletableFuture.completedFuture(next); + } + + @Override + public CompletableFuture listKvSnapshots( + long tableId, @Nullable Long partitionId) { + callCounter.incrementAndGet(); + lastKvPartitionId.set(partitionId); + ListKvSnapshotsResponse next = kvResponses.poll(); + if (next == null) { + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally( + new IllegalStateException("StubAdmin: no more queued kv responses")); + return failed; + } + return CompletableFuture.completedFuture(next); + } + } + + /** Per-path file-content / failure registry for the second-read step. */ + private static final class StubManifestReader implements ActiveRefsFetcher.MetadataReader { + + private final Map bytesByPath = new HashMap<>(); + private final Set notFoundPaths = new HashSet<>(); + private final Map ioFailuresByPath = new HashMap<>(); + + void returnBytes(FsPath path, byte[] data) { + bytesByPath.put(path.toString(), data); + } + + void failWithNotFound(FsPath path) { + notFoundPaths.add(path.toString()); + } + + void failWithIo(FsPath path, IOException e) { + ioFailuresByPath.put(path.toString(), e); + } + + @Override + public byte[] read(FsPath path) throws IOException { + String key = path.toString(); + if (notFoundPaths.contains(key)) { + throw new FileNotFoundException(key); + } + IOException io = ioFailuresByPath.get(key); + if (io != null) { + throw io; + } + byte[] data = bytesByPath.get(key); + if (data == null) { + throw new FileNotFoundException(key); + } + return data; + } + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/MaxKnownIdsTrackerTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/MaxKnownIdsTrackerTest.java new file mode 100644 index 0000000000..46a3814a04 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/MaxKnownIdsTrackerTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.build; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class MaxKnownIdsTrackerTest { + + @Test + void initialValuesAreNegativeOne() { + MaxKnownIdsTracker t = new MaxKnownIdsTracker(); + assertThat(t.maxKnownTableId()).isEqualTo(-1L); + assertThat(t.maxKnownPartitionId()).isEqualTo(-1L); + } + + @Test + void observeTableIdMonotonicallyIncreases() { + MaxKnownIdsTracker t = new MaxKnownIdsTracker(); + t.observeTableId(5L); + assertThat(t.maxKnownTableId()).isEqualTo(5L); + t.observeTableId(3L); + assertThat(t.maxKnownTableId()).isEqualTo(5L); // never decreases + t.observeTableId(10L); + assertThat(t.maxKnownTableId()).isEqualTo(10L); + } + + @Test + void observePartitionIdMonotonicallyIncreases() { + MaxKnownIdsTracker t = new MaxKnownIdsTracker(); + t.observePartitionId(7L); + t.observePartitionId(2L); + assertThat(t.maxKnownPartitionId()).isEqualTo(7L); + } + + @Test + void independentTracking() { + MaxKnownIdsTracker t = new MaxKnownIdsTracker(); + t.observeTableId(100L); + assertThat(t.maxKnownPartitionId()).isEqualTo(-1L); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfigTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfigTest.java new file mode 100644 index 0000000000..734ac682a0 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfigTest.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.config; + +import org.apache.flink.api.java.utils.MultipleParameterTool; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link OrphanCleanConfig}. */ +class OrphanCleanConfigTest { + + private static final DateTimeFormatter CUTOFF_FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + @Test + void parsesAllDatabasesWithDefaults() { + long beforeParse = System.currentTimeMillis(); + OrphanCleanConfig config = + OrphanCleanConfig.fromParams( + MultipleParameterTool.fromArgs( + new String[] {"--bootstrap-server", "h:9123", "--all-databases"})); + long afterParse = System.currentTimeMillis(); + + assertThat(config.allDatabases()).isTrue(); + assertThat(config.database()).isEmpty(); + long olderThanLow = beforeParse - Duration.ofDays(3).toMillis(); + long olderThanHigh = afterParse - Duration.ofDays(3).toMillis(); + assertThat(config.olderThanMillis()).isBetween(olderThanLow, olderThanHigh); + assertThat(config.dryRun()).isFalse(); + assertThat(config.deleteRateLimitPerSecond()).isEqualTo(100L); + assertThat(config.allowDeleteManifest()).isFalse(); + assertThat(config.allowCleanOrphanTables()).isFalse(); + assertThat(config.allowCleanOrphanPartitions()).isFalse(); + } + + @Test + void databaseAndAllDatabasesAreMutuallyExclusive() { + assertThatThrownBy( + () -> + OrphanCleanConfig.fromParams( + MultipleParameterTool.fromArgs( + new String[] { + "--bootstrap-server", + "h:9123", + "--database", + "x", + "--all-databases" + }))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("mutually exclusive"); + } + + @Test + void cutoffCloserThanOneDayRejected() { + LocalDateTime tooClose = LocalDateTime.now().minusMinutes(30); + assertThatThrownBy( + () -> + OrphanCleanConfig.fromParams( + MultipleParameterTool.fromArgs( + new String[] { + "--bootstrap-server", + "h:9123", + "--all-databases", + "--older-than", + tooClose.format(CUTOFF_FORMATTER) + }))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("at least 1d before now"); + } + + @Test + void tableCannotBeUsedWithAllDatabases() { + assertThatThrownBy( + () -> + OrphanCleanConfig.fromParams( + MultipleParameterTool.fromArgs( + new String[] { + "--bootstrap-server", + "h:9123", + "--all-databases", + "--table", + "t1" + }))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("--table requires --database"); + } + + @Test + void bootstrapServerRequired() { + assertThatThrownBy( + () -> + OrphanCleanConfig.fromParams( + MultipleParameterTool.fromArgs( + new String[] {"--all-databases"}))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("bootstrap-server"); + } + + @Test + void optInFlagsParsed() { + OrphanCleanConfig cfg = + OrphanCleanConfig.fromParams( + MultipleParameterTool.fromArgs( + new String[] { + "--bootstrap-server", + "x:1", + "--all-databases", + "--allow-delete-manifest", + "--allow-clean-orphan-tables", + "--allow-clean-orphan-partitions" + })); + assertThat(cfg.allowDeleteManifest()).isTrue(); + assertThat(cfg.allowCleanOrphanTables()).isTrue(); + assertThat(cfg.allowCleanOrphanPartitions()).isTrue(); + } + + @Test + void extraConfigsParsed() { + OrphanCleanConfig cfg = + OrphanCleanConfig.fromParams( + MultipleParameterTool.fromArgs( + new String[] { + "--bootstrap-server", + "h:9123", + "--all-databases", + "--conf", + "fs.oss.accessKeyId=myKey", + "--conf", + "fs.oss.accessKeySecret=mySecret", + "--conf", + "fs.oss.endpoint=oss-cn-hangzhou.aliyuncs.com" + })); + assertThat(cfg.extraConfigs()).hasSize(3); + assertThat(cfg.extraConfigs().get("fs.oss.accessKeyId")).isEqualTo("myKey"); + assertThat(cfg.extraConfigs().get("fs.oss.accessKeySecret")).isEqualTo("mySecret"); + assertThat(cfg.extraConfigs().get("fs.oss.endpoint")) + .isEqualTo("oss-cn-hangzhou.aliyuncs.com"); + } + + @Test + void extraConfigsEmptyWhenNotProvided() { + OrphanCleanConfig cfg = + OrphanCleanConfig.fromParams( + MultipleParameterTool.fromArgs( + new String[] {"--bootstrap-server", "h:9123", "--all-databases"})); + assertThat(cfg.extraConfigs()).isEmpty(); + } + + @Test + void extraConfigsRejectsMalformedEntry() { + assertThatThrownBy( + () -> + OrphanCleanConfig.fromParams( + MultipleParameterTool.fromArgs( + new String[] { + "--bootstrap-server", + "h:9123", + "--all-databases", + "--conf", + "noEqualsSign" + }))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("key=value"); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleterTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleterTest.java new file mode 100644 index 0000000000..42022164c7 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleterTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.fs; + +import org.apache.fluss.flink.action.orphan.audit.AuditLogger; +import org.apache.fluss.flink.action.orphan.rule.Decision; +import org.apache.fluss.flink.action.orphan.rule.RuleId; +import org.apache.fluss.fs.FileSystem; +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.fs.local.LocalFileSystem; +import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.RateLimiter; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link SafeDeleter} against the local filesystem. */ +class SafeDeleterTest { + + @TempDir Path tmp; + + @Test + void deleteFileRespectsDryRun() throws IOException { + Path target = Files.createFile(tmp.resolve("orphan.log")); + SafeDeleter d = new SafeDeleter(localFs(), true, new AuditLogger()); + d.deleteFile(new FsPath(target.toString()), Decision.DELETE, RuleId.LOG_SEGMENT); + assertThat(Files.exists(target)).isTrue(); + } + + @Test + void deleteFileActuallyDeletesWhenNotDryRun() throws IOException { + Path target = Files.createFile(tmp.resolve("orphan.log")); + SafeDeleter d = new SafeDeleter(localFs(), false, new AuditLogger()); + d.deleteFile(new FsPath(target.toString()), Decision.DELETE, RuleId.LOG_SEGMENT); + assertThat(Files.exists(target)).isFalse(); + } + + @Test + void deleteFileRejectsNonDeleteDecision() { + SafeDeleter d = new SafeDeleter(null, false, new AuditLogger()); + assertThatThrownBy( + () -> + d.deleteFile( + new FsPath("/tmp/x"), Decision.KEEP_ACTIVE, RuleId.UNKNOWN)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void deleteEmptyDirNoOpsOnNonEmpty() throws IOException { + Path dir = Files.createDirectory(tmp.resolve("d")); + Files.createFile(dir.resolve("child")); + SafeDeleter d = new SafeDeleter(localFs(), false, new AuditLogger()); + d.deleteEmptyDir(new FsPath(dir.toString())); + assertThat(Files.exists(dir)).isTrue(); + } + + @Test + void deleteEmptyDirActuallyDeletes() throws IOException { + Path dir = Files.createDirectory(tmp.resolve("d")); + SafeDeleter d = new SafeDeleter(localFs(), false, new AuditLogger()); + d.deleteEmptyDir(new FsPath(dir.toString())); + assertThat(Files.exists(dir)).isFalse(); + } + + @Test + void multipleDeletesAllSucceed() throws IOException { + Path a = Files.createFile(tmp.resolve("a.log")); + Path b = Files.createFile(tmp.resolve("b.log")); + Path c = Files.createFile(tmp.resolve("c.log")); + Files.write(a, new byte[] {1}); + Files.write(b, new byte[] {2}); + Files.write(c, new byte[] {3}); + Path emptyDir = Files.createDirectory(tmp.resolve("emptyDir")); + + RateLimiter limiter = RateLimiter.create(Double.MAX_VALUE); + SafeDeleter deleter = new SafeDeleter(localFs(), false, new AuditLogger(), limiter); + + deleter.deleteFile(new FsPath(a.toString()), Decision.DELETE, RuleId.LOG_SEGMENT); + deleter.deleteFile(new FsPath(b.toString()), Decision.DELETE, RuleId.LOG_SEGMENT); + deleter.deleteFile(new FsPath(c.toString()), Decision.DELETE, RuleId.LOG_SEGMENT); + deleter.deleteEmptyDir(new FsPath(emptyDir.toString())); + + assertThat(Files.exists(a)).isFalse(); + assertThat(Files.exists(b)).isFalse(); + assertThat(Files.exists(c)).isFalse(); + assertThat(Files.exists(emptyDir)).isFalse(); + } + + @Test + void dryRunPreservesAllFiles() throws IOException { + Path file = Files.createFile(tmp.resolve("orphan.log")); + Path emptyDir = Files.createDirectory(tmp.resolve("emptyDir")); + + RateLimiter limiter = RateLimiter.create(Double.MAX_VALUE); + SafeDeleter deleter = new SafeDeleter(localFs(), true, new AuditLogger(), limiter); + + deleter.deleteFile(new FsPath(file.toString()), Decision.DELETE, RuleId.LOG_SEGMENT); + deleter.deleteEmptyDir(new FsPath(emptyDir.toString())); + + assertThat(Files.exists(file)).isTrue(); + assertThat(Files.exists(emptyDir)).isTrue(); + } + + private static FileSystem localFs() { + return LocalFileSystem.getSharedInstance(); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeperTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeperTest.java new file mode 100644 index 0000000000..cc47f95671 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeperTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.job; + +import org.apache.fluss.flink.action.orphan.audit.AuditLogger; +import org.apache.fluss.fs.FsPath; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +class EmptyDirSweeperTest { + + @Test + void deletesEmptyDirsBottomUp(@TempDir Path tmp) throws IOException { + Path a = Files.createDirectories(tmp.resolve("a")); + Path b = Files.createDirectories(a.resolve("b")); + Path c = Files.createDirectories(b.resolve("c")); + + EmptyDirSweeper sweeper = new EmptyDirSweeper(false, new AuditLogger()); + sweeper.registerTouched(new FsPath(a.toString())); + long removed = sweeper.sweep(); + + assertThat(removed).isEqualTo(3L); + assertThat(Files.exists(c)).isFalse(); + assertThat(Files.exists(b)).isFalse(); + assertThat(Files.exists(a)).isFalse(); + } + + @Test + void leavesNonEmptyDirsAlone(@TempDir Path tmp) throws IOException { + Path a = Files.createDirectories(tmp.resolve("a")); + Path b = Files.createDirectories(a.resolve("b")); + Files.write(b.resolve("keep.txt"), new byte[] {0x42}); + + EmptyDirSweeper sweeper = new EmptyDirSweeper(false, new AuditLogger()); + sweeper.registerTouched(new FsPath(a.toString())); + long removed = sweeper.sweep(); + + assertThat(removed).isEqualTo(0L); + assertThat(Files.exists(b)).isTrue(); + assertThat(Files.exists(a)).isTrue(); + } + + @Test + void dryRunCountsWouldDeleteButDoesNotActuallyDelete(@TempDir Path tmp) throws IOException { + Path a = Files.createDirectories(tmp.resolve("a")); + Path b = Files.createDirectories(a.resolve("b")); + + EmptyDirSweeper sweeper = new EmptyDirSweeper(true /* dryRun */, new AuditLogger()); + sweeper.registerTouched(new FsPath(a.toString())); + long removed = sweeper.sweep(); + + // dry-run leaves both directories on disk, but reports the would-delete count. + assertThat(removed).isEqualTo(2L); + assertThat(Files.exists(b)).isTrue(); + assertThat(Files.exists(a)).isTrue(); + } + + @Test + void nonExistentRootIsNoOp(@TempDir Path tmp) throws IOException { + EmptyDirSweeper sweeper = new EmptyDirSweeper(false, new AuditLogger()); + sweeper.registerTouched(new FsPath(tmp.resolve("does-not-exist").toString())); + assertThat(sweeper.sweep()).isEqualTo(0L); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/KvSharedSstRuleTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/KvSharedSstRuleTest.java new file mode 100644 index 0000000000..c6267d31c8 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/KvSharedSstRuleTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.rule; + +import org.apache.fluss.fs.FsPath; + +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link KvSharedSstRule}. */ +class KvSharedSstRuleTest { + + private static final long NOW = 1_700_000_000_000L; + private static final long DAY_MS = 24L * 60L * 60L * 1000L; + private static final long CUTOFF_MS = NOW - DAY_MS; + + private final KvSharedSstRule rule = new KvSharedSstRule(); + + @Test + void keepsExpiredUnreferencedSharedSst() { + FileMeta file = file("/kv/db/t-1/0/shared/abc-001.sst", NOW - 2 * DAY_MS); + + assertThat(rule.evaluate(file, BucketActiveRefs.empty(), CUTOFF_MS)) + .isEqualTo(Decision.KEEP_ACTIVE); + } + + @Test + void keepsReferencedSharedSst() { + FileMeta file = file("/kv/db/t-1/0/shared/abc-001.sst", NOW - 2 * DAY_MS); + Set sharedFiles = new HashSet(); + sharedFiles.add("abc-001.sst"); + BucketActiveRefs activeRefs = + new BucketActiveRefs( + Collections.emptySet(), + Collections.emptySet(), + sharedFiles); + + assertThat(rule.evaluate(file, activeRefs, CUTOFF_MS)).isEqualTo(Decision.KEEP_ACTIVE); + } + + @Test + void skipsUnknownNonSstFileUnderSharedDirectory() { + FileMeta file = file("/kv/db/t-1/0/shared/abc-001.meta", NOW - 2 * DAY_MS); + + assertThat(rule.evaluate(file, BucketActiveRefs.empty(), CUTOFF_MS)) + .isEqualTo(Decision.SKIP_UNKNOWN); + } + + @Test + void skipsSstOutsideSharedDirectory() { + FileMeta file = file("/kv/db/t-1/0/snap-5/abc-001.sst", NOW - 2 * DAY_MS); + + assertThat(rule.evaluate(file, BucketActiveRefs.empty(), CUTOFF_MS)) + .isEqualTo(Decision.SKIP_UNKNOWN); + } + + private static FileMeta file(String path, long modificationTime) { + return new FileMeta(new FsPath(path), 1L, modificationTime); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/KvSnapshotFileRuleTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/KvSnapshotFileRuleTest.java new file mode 100644 index 0000000000..c056d8e538 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/KvSnapshotFileRuleTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.rule; + +import org.apache.fluss.fs.FsPath; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link KvSnapshotFileRule}. */ +class KvSnapshotFileRuleTest { + + private static final long NOW = 1_700_000_000_000L; + private static final long DAY_MS = 24L * 60L * 60L * 1000L; + + /** Absolute cutoff = NOW - 1d. Files with mtime strictly less are deletion-eligible. */ + private static final long CUTOFF_MS = NOW - DAY_MS; + + private final KvSnapshotFileRule rule = new KvSnapshotFileRule(); + + @Test + void deletesExpiredSnapshotFileOutsideBucketActiveRefs() { + FileMeta file = file("/kv/db/t-1/0/snap-5/001.sst", NOW - 2 * DAY_MS); + + assertThat(rule.evaluate(file, kvActiveSnapDirs("snap-7", "snap-9"), CUTOFF_MS)) + .isEqualTo(Decision.DELETE); + } + + @Test + void keepsActiveSnapshotFile() { + FileMeta file = file("/kv/db/t-1/0/snap-5/001.sst", NOW - 2 * DAY_MS); + + assertThat(rule.evaluate(file, kvActiveSnapDirs("snap-5"), CUTOFF_MS)) + .isEqualTo(Decision.KEEP_ACTIVE); + } + + @Test + void defersSnapshotWhenMtimeAtOrAfterCutoff() { + FileMeta file = file("/kv/db/t-1/0/snap-5/001.sst", NOW - DAY_MS / 2); + + assertThat(rule.evaluate(file, BucketActiveRefs.empty(), CUTOFF_MS)) + .isEqualTo(Decision.DEFER); + } + + @Test + void skipsUnknownFileNameInsideSnapshotDirectory() { + FileMeta file = file("/kv/db/t-1/0/snap-5/data.bloom", NOW - 2 * DAY_MS); + + assertThat(rule.evaluate(file, BucketActiveRefs.empty(), CUTOFF_MS)) + .isEqualTo(Decision.SKIP_UNKNOWN); + } + + @Test + void skipsUnknownWhenParentIsNotSnapshotDirectory() { + FileMeta file = file("/kv/db/t-1/0/random/001.sst", NOW - 2 * DAY_MS); + + assertThat(rule.evaluate(file, BucketActiveRefs.empty(), CUTOFF_MS)) + .isEqualTo(Decision.SKIP_UNKNOWN); + } + + @Test + void recognizesExactPrefixAndSuffixBasedSnapshotFiles() { + String[] fileNames = { + "_METADATA", "MANIFEST-001", "OPTIONS-002", "CURRENT", "LOG", "IDENTITY", "001.log" + }; + + for (String fileName : fileNames) { + FileMeta file = file("/kv/db/t-1/0/snap-5/" + fileName, NOW - 2 * DAY_MS); + assertThat(rule.evaluate(file, BucketActiveRefs.empty(), CUTOFF_MS)) + .as("file=%s", fileName) + .isEqualTo(Decision.DELETE); + } + } + + @Test + void retainedNonLatestSnapshotIsActive() { + // Simulates kv.snapshot.num-retained=2, latest snapId=10, retained={9,10}: the active set + // is the full retained set (server emits RETAINED ∪ STILL_IN_USE), so a file under snap-9 + // MUST be classified as KEEP_ACTIVE even if it's old enough to clear the cutoff. Cutoff is + // set to NOW (an aggressive value) to prove the active-set check short-circuits before the + // age check. + FileMeta file = + new FileMeta(new FsPath("oss://b/kv/db/t-7/0/snap-9/_METADATA"), 1024L, 200L); + + Decision decision = rule.evaluate(file, kvActiveSnapDirs("snap-9", "snap-10"), NOW); + + assertThat(decision).isEqualTo(Decision.KEEP_ACTIVE); + } + + private static BucketActiveRefs kvActiveSnapDirs(String... snapDirs) { + Set activeDirs = new HashSet(Arrays.asList(snapDirs)); + return new BucketActiveRefs( + Collections.emptySet(), activeDirs, Collections.emptySet()); + } + + private static FileMeta file(String path, long modificationTime) { + return new FileMeta(new FsPath(path), 1L, modificationTime); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/LogManifestRuleTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/LogManifestRuleTest.java new file mode 100644 index 0000000000..b8d166059a --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/LogManifestRuleTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.rule; + +import org.apache.fluss.fs.FsPath; + +import org.junit.jupiter.api.Test; + +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link LogManifestRule}. */ +class LogManifestRuleTest { + + private static final long NOW = 1_700_000_000_000L; + private static final long DAY_MS = 24L * 60L * 60L * 1000L; + private static final long CUTOFF_MS = NOW - DAY_MS; + + /** Default-conservative rule (allowDeleteManifest=false): never deletes manifests. */ + private final LogManifestRule defaultRule = new LogManifestRule(); + + /** Opt-in rule (allowDeleteManifest=true): uses active-set + cutoff semantics. */ + private final LogManifestRule optInRule = new LogManifestRule(true); + + @Test + void deletesExpiredNonActiveManifest() { + FileMeta file = file("/log/db/t-1/0/metadata/old.manifest", NOW - 2 * DAY_MS); + BucketActiveRefs activeRefs = + new BucketActiveRefs( + Collections.emptySet(), + Collections.emptySet(), + Collections.singleton("/log/db/t-1/0/metadata/current.manifest")); + + assertThat(optInRule.evaluate(file, activeRefs, CUTOFF_MS)).isEqualTo(Decision.DELETE); + } + + @Test + void keepsManifestListedInActiveManifestPaths() { + FileMeta file = file("/log/db/t-1/0/metadata/active.manifest", NOW - 2 * DAY_MS); + BucketActiveRefs activeRefs = + new BucketActiveRefs( + Collections.emptySet(), + Collections.emptySet(), + Collections.singleton("/log/db/t-1/0/metadata/active.manifest")); + + assertThat(optInRule.evaluate(file, activeRefs, CUTOFF_MS)).isEqualTo(Decision.KEEP_ACTIVE); + } + + @Test + void defersManifestWhenMtimeAtOrAfterCutoff() { + FileMeta file = file("/log/db/t-1/0/metadata/fresh.manifest", NOW - DAY_MS / 2); + + assertThat(optInRule.evaluate(file, BucketActiveRefs.empty(), CUTOFF_MS)) + .isEqualTo(Decision.DEFER); + } + + @Test + void skipsUnknownFileInMetadataDirectory() { + FileMeta file = file("/log/db/t-1/0/metadata/readme.txt", NOW - 2 * DAY_MS); + + assertThat(defaultRule.evaluate(file, BucketActiveRefs.empty(), CUTOFF_MS)) + .isEqualTo(Decision.SKIP_UNKNOWN); + assertThat(optInRule.evaluate(file, BucketActiveRefs.empty(), CUTOFF_MS)) + .isEqualTo(Decision.SKIP_UNKNOWN); + } + + @Test + void skipsManifestOutsideMetadataDirectory() { + FileMeta file = + file( + "/log/db/t-1/0/11111111-1111-1111-1111-111111111111/file.manifest", + NOW - 2 * DAY_MS); + + assertThat(defaultRule.evaluate(file, BucketActiveRefs.empty(), CUTOFF_MS)) + .isEqualTo(Decision.SKIP_UNKNOWN); + assertThat(optInRule.evaluate(file, BucketActiveRefs.empty(), CUTOFF_MS)) + .isEqualTo(Decision.SKIP_UNKNOWN); + } + + @Test + void defaultRuleNeverDeletesEvenWhenStaleAndOrphan() { + // mtime=0L (very old); active-set lists a different manifest as active; under the + // default-conservative branch the rule MUST still return KEEP_ACTIVE rather than DELETE. + FileMeta file = file("/log/db/t-1/0/metadata/orphan.manifest", 0L); + BucketActiveRefs activeRefs = + new BucketActiveRefs( + Collections.emptySet(), + Collections.emptySet(), + Collections.singleton("/log/db/t-1/0/metadata/current.manifest")); + + assertThat(defaultRule.evaluate(file, activeRefs, CUTOFF_MS)) + .isEqualTo(Decision.KEEP_ACTIVE); + } + + private static FileMeta file(String path, long modificationTime) { + return new FileMeta(new FsPath(path), 1L, modificationTime); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/LogSegmentRuleTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/LogSegmentRuleTest.java new file mode 100644 index 0000000000..bb8249e55d --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/LogSegmentRuleTest.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.rule; + +import org.apache.fluss.fs.FsPath; + +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link LogSegmentRule}. */ +class LogSegmentRuleTest { + + private static final String SEGMENT_ID = "11111111-1111-1111-1111-111111111111"; + private static final long NOW = 1_700_000_000_000L; + private static final long DAY_MS = 24L * 60L * 60L * 1000L; + + /** + * Absolute cutoff = NOW - 1d. Files with mtime strictly less than this are deletion-eligible. + */ + private static final long CUTOFF_MS = NOW - DAY_MS; + + private final LogSegmentRule rule = new LogSegmentRule(); + + @Test + void deleteWhenKnownExpiredAndNotInBucketActiveRefs() { + FileMeta file = + file("/log/db/t-1/0/" + SEGMENT_ID + "/00000000000000000000.log", NOW - 2 * DAY_MS); + + Decision decision = rule.evaluate(file, BucketActiveRefs.empty(), CUTOFF_MS); + + assertThat(decision).isEqualTo(Decision.DELETE); + } + + @Test + void keepActiveWhenInBucketActiveRefs() { + FileMeta file = + file("/log/db/t-1/0/" + SEGMENT_ID + "/00000000000000000000.log", NOW - 2 * DAY_MS); + Set liveFiles = new HashSet(); + liveFiles.add(SEGMENT_ID + "/00000000000000000000.log"); + BucketActiveRefs activeRefs = + new BucketActiveRefs( + liveFiles, Collections.emptySet(), Collections.emptySet()); + + Decision decision = rule.evaluate(file, activeRefs, CUTOFF_MS); + + assertThat(decision).isEqualTo(Decision.KEEP_ACTIVE); + } + + @Test + void deferWhenMtimeAtOrAfterCutoff() { + FileMeta file = + file("/log/db/t-1/0/" + SEGMENT_ID + "/00000000000000000000.log", NOW - DAY_MS / 2); + + Decision decision = rule.evaluate(file, BucketActiveRefs.empty(), CUTOFF_MS); + + assertThat(decision).isEqualTo(Decision.DEFER); + } + + @Test + void skipUnknownExtension() { + FileMeta file = + file( + "/log/db/t-1/0/" + SEGMENT_ID + "/00000000000000000000.bloom", + NOW - 2 * DAY_MS); + + Decision decision = rule.evaluate(file, BucketActiveRefs.empty(), CUTOFF_MS); + + assertThat(decision).isEqualTo(Decision.SKIP_UNKNOWN); + } + + @Test + void skipUnknownWhenParentIsNotSegmentUuid() { + FileMeta file = file("/log/db/t-1/0/not-a-uuid/00000000000000000000.log", NOW - 2 * DAY_MS); + + Decision decision = rule.evaluate(file, BucketActiveRefs.empty(), CUTOFF_MS); + + assertThat(decision).isEqualTo(Decision.SKIP_UNKNOWN); + } + + @Test + void deletedSuffixIsRecognizedAsKnownType() { + FileMeta file = + file( + "/log/db/t-1/0/" + SEGMENT_ID + "/00000000000000000000.log.deleted", + NOW - 2 * DAY_MS); + + Decision decision = rule.evaluate(file, BucketActiveRefs.empty(), CUTOFF_MS); + + assertThat(decision).isEqualTo(Decision.DELETE); + } + + private static FileMeta file(String path, long modificationTime) { + return new FileMeta(new FsPath(path), 100L, modificationTime); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/OrphanDirDetectorTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/OrphanDirDetectorTest.java new file mode 100644 index 0000000000..aa874a4520 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/OrphanDirDetectorTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.rule; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Unit tests for {@link OrphanDirDetector}. */ +class OrphanDirDetectorTest { + + // --- Table directory detection --- + + @Test + void tableOrphanWhenIdLeMaxKnown() { + assertThat(OrphanDirDetector.isOrphanTable("foo-15", ids(10L, 20L), 30L)).isTrue(); + } + + @Test + void tableNotOrphanWhenIdGreaterThanMaxKnown() { + assertThat(OrphanDirDetector.isOrphanTable("foo-99", ids(10L, 20L), 30L)).isFalse(); + } + + @Test + void tableNotOrphanWhenInActiveSet() { + assertThat(OrphanDirDetector.isOrphanTable("foo-10", ids(10L, 20L), 30L)).isFalse(); + } + + @Test + void tableNotOrphanWhenNameFormatBad() { + assertThat(OrphanDirDetector.isOrphanTable("no_id_here", Collections.emptySet(), 10L)) + .isFalse(); + } + + // --- Partition directory detection --- + + @Test + void partitionOrphanWhenIdLeMaxKnown() { + assertThat(OrphanDirDetector.isOrphanPartition("dt=2024-p150", ids(101L, 102L), 200L)) + .isTrue(); + } + + @Test + void partitionNotOrphanWhenIdGreaterThanMaxKnown() { + assertThat( + OrphanDirDetector.isOrphanPartition( + "dt=2024-p250", Collections.emptySet(), 200L)) + .isFalse(); + } + + @Test + void partitionNotOrphanWhenInActiveSet() { + assertThat(OrphanDirDetector.isOrphanPartition("dt=2024-p150", ids(150L), 200L)).isFalse(); + } + + @Test + void partitionNotOrphanWhenMissingPPrefix() { + assertThat(OrphanDirDetector.isOrphanPartition("0", Collections.emptySet(), 200L)) + .isFalse(); + } + + private static Set ids(Long... values) { + return new HashSet(Arrays.asList(values)); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/RuleDispatcherTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/RuleDispatcherTest.java new file mode 100644 index 0000000000..1527361d74 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/RuleDispatcherTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.rule; + +import org.apache.fluss.fs.FsPath; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link RuleDispatcher}. */ +class RuleDispatcherTest { + + private static final String SEGMENT_ID = "11111111-1111-1111-1111-111111111111"; + + private final RuleDispatcher dispatcher = new RuleDispatcher(); + + @Test + void dispatchesLogSegmentRule() { + assertThat(dispatcher.dispatch(file("/log/db/t-1/0/" + SEGMENT_ID + "/000.log")).id()) + .isEqualTo(RuleId.LOG_SEGMENT); + } + + @Test + void dispatchesLogManifestRule() { + assertThat(dispatcher.dispatch(file("/log/db/t-1/0/metadata/current.manifest")).id()) + .isEqualTo(RuleId.LOG_MANIFEST); + } + + @Test + void dispatchesKvSnapshotFileRule() { + assertThat(dispatcher.dispatch(file("/kv/db/t-1/0/snap-5/001.sst")).id()) + .isEqualTo(RuleId.KV_SNAPSHOT_FILE); + } + + @Test + void dispatchesKvSharedSstRule() { + assertThat(dispatcher.dispatch(file("/kv/db/t-1/0/shared/abc-001.sst")).id()) + .isEqualTo(RuleId.KV_SHARED_SST); + } + + @Test + void fallsBackToUnknownRule() { + assertThat(dispatcher.dispatch(file("/random/path/file.bin")).id()) + .isEqualTo(RuleId.UNKNOWN); + } + + private static FileMeta file(String path) { + return new FileMeta(new FsPath(path), 0L, 0L); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/testutils/TestAdminAdapter.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/testutils/TestAdminAdapter.java index e8120c83d2..d3e16fbd3c 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/testutils/TestAdminAdapter.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/testutils/TestAdminAdapter.java @@ -50,6 +50,8 @@ import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.metadata.TableStats; +import org.apache.fluss.rpc.messages.ListKvSnapshotsResponse; +import org.apache.fluss.rpc.messages.ListRemoteLogManifestsResponse; import org.apache.fluss.security.acl.AclBinding; import org.apache.fluss.security.acl.AclBindingFilter; @@ -337,4 +339,16 @@ public CompletableFuture listKvSnapshots( long tableId, @Nullable Long partitionId) { throw new UnsupportedOperationException("Not implemented in TestAdminAdapter"); } + + @Override + public CompletableFuture listRemoteLogManifests( + long tableId, @Nullable Long partitionId) { + throw new UnsupportedOperationException("Not implemented in TestAdminAdapter"); + } + + @Override + public CompletableFuture listKvSnapshots( + long tableId, @Nullable Long partitionId) { + throw new UnsupportedOperationException("Not implemented in TestAdminAdapter"); + } } diff --git a/fluss-flink/pom.xml b/fluss-flink/pom.xml index 4f65374352..b66643a90a 100644 --- a/fluss-flink/pom.xml +++ b/fluss-flink/pom.xml @@ -38,6 +38,7 @@ fluss-flink-1.18 fluss-flink-2.2 fluss-flink-tiering + fluss-flink-action + org.apache.fluss.flink.action.Action + org.apache.fluss.flink.action.ActionFactory + org.apache.fluss.flink.action.ActionLoader + org.apache.fluss.flink.action.FlussFlinkActionEntrypoint + org.apache.fluss.flink.action.orphan.OrphanFilesCleanActionFactory + org.apache.flink.table.catalog.* From 15de1476b2228c557bc0e8887ad507f406cd6598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Tue, 2 Jun 2026 14:26:40 +0800 Subject: [PATCH 03/19] [flink][client][server] address PR #3404 review feedback --- ...va => Flink119OrphanFilesCleanITCase.java} | 2 +- ...va => Flink120OrphanFilesCleanITCase.java} | 2 +- fluss-flink/fluss-flink-action/pom.xml | 2 +- ...ypoint.java => FlussActionEntrypoint.java} | 2 +- .../flink/action/orphan/OrphanCleanUtils.java | 16 +++- .../orphan/build/ActiveRefsFetcher.java | 46 +++++----- .../orphan/config/OrphanCleanConfig.java | 6 +- .../flink/action/orphan/fs/SafeDeleter.java | 34 ++++++-- .../action/orphan/job/BucketCleaner.java | 12 ++- .../orphan/job/OrphanFilesCleanJob.java | 6 +- .../orphan/job/ScanAndCleanFunction.java | 3 +- .../orphan/job/ScopeEnumeratorFunction.java | 21 +---- .../orphan/job/StatsAggregateOperator.java | 15 +++- .../orphan/build/ActiveRefsFetcherTest.java | 85 ++++++++++--------- .../rpc/gateway/AdminReadOnlyGateway.java | 25 ------ .../rpc/TestingTabletGatewayService.java | 16 ---- .../fluss/server/tablet/TabletService.java | 28 ------ .../tablet/TestTabletServerGateway.java | 16 ---- fluss-test-coverage/pom.xml | 2 +- 19 files changed, 147 insertions(+), 192 deletions(-) rename fluss-flink/fluss-flink-1.19/src/test/java/org/apache/fluss/flink/action/orphan/{Flink19OrphanFilesCleanITCase.java => Flink119OrphanFilesCleanITCase.java} (92%) rename fluss-flink/fluss-flink-1.20/src/test/java/org/apache/fluss/flink/action/orphan/{Flink20OrphanFilesCleanITCase.java => Flink120OrphanFilesCleanITCase.java} (92%) rename fluss-flink/fluss-flink-action/src/main/java/org/apache/fluss/flink/action/{FlussFlinkActionEntrypoint.java => FlussActionEntrypoint.java} (97%) diff --git a/fluss-flink/fluss-flink-1.19/src/test/java/org/apache/fluss/flink/action/orphan/Flink19OrphanFilesCleanITCase.java b/fluss-flink/fluss-flink-1.19/src/test/java/org/apache/fluss/flink/action/orphan/Flink119OrphanFilesCleanITCase.java similarity index 92% rename from fluss-flink/fluss-flink-1.19/src/test/java/org/apache/fluss/flink/action/orphan/Flink19OrphanFilesCleanITCase.java rename to fluss-flink/fluss-flink-1.19/src/test/java/org/apache/fluss/flink/action/orphan/Flink119OrphanFilesCleanITCase.java index 7a177185f3..d775605170 100644 --- a/fluss-flink/fluss-flink-1.19/src/test/java/org/apache/fluss/flink/action/orphan/Flink19OrphanFilesCleanITCase.java +++ b/fluss-flink/fluss-flink-1.19/src/test/java/org/apache/fluss/flink/action/orphan/Flink119OrphanFilesCleanITCase.java @@ -19,4 +19,4 @@ package org.apache.fluss.flink.action.orphan; /** The IT case for orphan files cleanup in Flink 1.19. */ -class Flink19OrphanFilesCleanITCase extends OrphanFilesCleanITCase {} +class Flink119OrphanFilesCleanITCase extends OrphanFilesCleanITCase {} diff --git a/fluss-flink/fluss-flink-1.20/src/test/java/org/apache/fluss/flink/action/orphan/Flink20OrphanFilesCleanITCase.java b/fluss-flink/fluss-flink-1.20/src/test/java/org/apache/fluss/flink/action/orphan/Flink120OrphanFilesCleanITCase.java similarity index 92% rename from fluss-flink/fluss-flink-1.20/src/test/java/org/apache/fluss/flink/action/orphan/Flink20OrphanFilesCleanITCase.java rename to fluss-flink/fluss-flink-1.20/src/test/java/org/apache/fluss/flink/action/orphan/Flink120OrphanFilesCleanITCase.java index 3b5b1997b2..0dc35613f9 100644 --- a/fluss-flink/fluss-flink-1.20/src/test/java/org/apache/fluss/flink/action/orphan/Flink20OrphanFilesCleanITCase.java +++ b/fluss-flink/fluss-flink-1.20/src/test/java/org/apache/fluss/flink/action/orphan/Flink120OrphanFilesCleanITCase.java @@ -19,4 +19,4 @@ package org.apache.fluss.flink.action.orphan; /** The IT case for orphan files cleanup in Flink 1.20. */ -class Flink20OrphanFilesCleanITCase extends OrphanFilesCleanITCase {} +class Flink120OrphanFilesCleanITCase extends OrphanFilesCleanITCase {} diff --git a/fluss-flink/fluss-flink-action/pom.xml b/fluss-flink/fluss-flink-action/pom.xml index e7e3979692..51e0852089 100644 --- a/fluss-flink/fluss-flink-action/pom.xml +++ b/fluss-flink/fluss-flink-action/pom.xml @@ -68,7 +68,7 @@ - org.apache.fluss.flink.action.FlussFlinkActionEntrypoint + org.apache.fluss.flink.action.FlussActionEntrypoint diff --git a/fluss-flink/fluss-flink-action/src/main/java/org/apache/fluss/flink/action/FlussFlinkActionEntrypoint.java b/fluss-flink/fluss-flink-action/src/main/java/org/apache/fluss/flink/action/FlussActionEntrypoint.java similarity index 97% rename from fluss-flink/fluss-flink-action/src/main/java/org/apache/fluss/flink/action/FlussFlinkActionEntrypoint.java rename to fluss-flink/fluss-flink-action/src/main/java/org/apache/fluss/flink/action/FlussActionEntrypoint.java index c83ea3b304..b1dbdecd97 100644 --- a/fluss-flink/fluss-flink-action/src/main/java/org/apache/fluss/flink/action/FlussFlinkActionEntrypoint.java +++ b/fluss-flink/fluss-flink-action/src/main/java/org/apache/fluss/flink/action/FlussActionEntrypoint.java @@ -20,7 +20,7 @@ import java.util.Optional; /** Main entrypoint for the Fluss Flink action jar. Delegates to {@link ActionLoader}. */ -public class FlussFlinkActionEntrypoint { +public class FlussActionEntrypoint { public static void main(String[] args) throws Exception { Optional action; diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanCleanUtils.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanCleanUtils.java index acf2dc7214..e8ef3d74fd 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanCleanUtils.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanCleanUtils.java @@ -39,6 +39,8 @@ import java.util.List; import java.util.Map; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + /** Shared utility methods for the orphan files cleanup action. */ @Internal public final class OrphanCleanUtils { @@ -78,9 +80,10 @@ public static List enumerateBuckets( /** * Resolves the effective remote data directory for a table/partition target using the - * three-level fallback: partition-level → table-level → cluster-level. + * three-level fallback: partition-level → table-level → cluster-level. At least one level is + * always set because the coordinator assigns a {@code remoteDataDir} to every table at creation + * time via {@code RemoteDirSelector.nextDataDir()}. */ - @Nullable public static String resolveRemoteDataDir( TableInfo tableInfo, @Nullable PartitionInfo partitionInfo, @@ -91,12 +94,17 @@ public static String resolveRemoteDataDir( if (tableInfo.getRemoteDataDir() != null) { return tableInfo.getRemoteDataDir(); } - return clusterRemoteDataDir; + return checkNotNull( + clusterRemoteDataDir, + "No remote data directory resolvable: partition, table, " + + "and cluster levels are all null. This should not happen because the " + + "coordinator requires remote.data.dir or remote.data.dirs at startup."); } /** * Resolves the cluster-level {@code remote.data.dir} by querying the coordinator's runtime - * configuration. + * configuration. Returns {@code null} when the cluster uses {@code remote.data.dirs} + * (multi-directory mode) without the legacy single {@code remote.data.dir}. */ @Nullable public static String resolveClusterRemoteDataDir(Admin admin) throws Exception { diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcher.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcher.java index 8abf1184ce..a11c46e157 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcher.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcher.java @@ -20,14 +20,12 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.annotation.VisibleForTesting; import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.metadata.ActiveKvSnapshots; +import org.apache.fluss.client.metadata.RemoteLogManifestInfo; import org.apache.fluss.flink.action.orphan.RpcErrorClassifier; import org.apache.fluss.flink.action.orphan.rule.BucketActiveRefs; import org.apache.fluss.fs.FSDataInputStream; import org.apache.fluss.fs.FsPath; -import org.apache.fluss.rpc.messages.ListKvSnapshotsResponse; -import org.apache.fluss.rpc.messages.ListRemoteLogManifestsResponse; -import org.apache.fluss.rpc.messages.PbKvSnapshot; -import org.apache.fluss.rpc.messages.PbRemoteLogManifestEntry; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; @@ -131,13 +129,13 @@ public ActiveRefsFetcher(Admin admin, MetadataReader metadataReader, int maxRetr private static AdminFacade wrap(Admin admin) { return new AdminFacade() { @Override - public CompletableFuture listRemoteLogManifests( + public CompletableFuture> listRemoteLogManifests( long tableId, @Nullable Long partitionId) { return admin.listRemoteLogManifests(tableId, partitionId); } @Override - public CompletableFuture listKvSnapshots( + public CompletableFuture listKvSnapshots( long tableId, @Nullable Long partitionId) { return admin.listKvSnapshots(tableId, partitionId); } @@ -154,9 +152,9 @@ public CompletableFuture listKvSnapshots( */ public LogActiveRefsFetchResult fetchLogActiveRefsByBucket( long tableId, @Nullable Long partitionId) { - ListRemoteLogManifestsResponse rpc; + List manifests; try { - rpc = + manifests = RetryUtils.executeWithRetry( () -> admin.listRemoteLogManifests(tableId, partitionId).get(), "listRemoteLogManifests", @@ -171,15 +169,15 @@ public LogActiveRefsFetchResult fetchLogActiveRefsByBucket( formatRpcFailureReason(tableId, partitionId, e.getCause())); } - Map> entriesByBucket = new HashMap<>(); - for (PbRemoteLogManifestEntry entry : rpc.getManifestsList()) { - int bucketId = entry.getTableBucket().getBucketId(); + Map> entriesByBucket = new HashMap<>(); + for (RemoteLogManifestInfo entry : manifests) { + int bucketId = entry.getTableBucket().getBucket(); entriesByBucket.computeIfAbsent(bucketId, id -> new ArrayList<>()).add(entry); } Map resolved = new HashMap<>(); Map readFailures = new HashMap<>(); - for (Map.Entry> bucketEntries : + for (Map.Entry> bucketEntries : entriesByBucket.entrySet()) { int bucketId = bucketEntries.getKey(); try { @@ -223,9 +221,9 @@ public LogActiveRefsFetchResult fetchLogActiveRefsByBucket( * with the log path. */ public KvActiveRefsFetchResult fetchKvActiveSnapDirs(long tableId, @Nullable Long partitionId) { - ListKvSnapshotsResponse rpc; + ActiveKvSnapshots activeKvSnapshots; try { - rpc = + activeKvSnapshots = RetryUtils.executeWithRetry( () -> admin.listKvSnapshots(tableId, partitionId).get(), "listKvSnapshots", @@ -240,10 +238,14 @@ public KvActiveRefsFetchResult fetchKvActiveSnapDirs(long tableId, @Nullable Lon formatRpcFailureReason(tableId, partitionId, e.getCause())); } Map> dirsByBucket = new HashMap<>(); - for (PbKvSnapshot snapshot : rpc.getActiveSnapshotsList()) { - int bucketId = snapshot.getBucketId(); - String dirName = FlussPaths.REMOTE_KV_SNAPSHOT_DIR_PREFIX + snapshot.getSnapshotId(); - dirsByBucket.computeIfAbsent(bucketId, b -> new HashSet<>()).add(dirName); + for (Map.Entry> entry : + activeKvSnapshots.getSnapshotIdsByBucket().entrySet()) { + int bucketId = entry.getKey(); + Set dirNames = new HashSet<>(); + for (Long snapshotId : entry.getValue()) { + dirNames.add(FlussPaths.REMOTE_KV_SNAPSHOT_DIR_PREFIX + snapshotId); + } + dirsByBucket.put(bucketId, dirNames); } return KvActiveRefsFetchResult.ok(dirsByBucket); } @@ -274,11 +276,11 @@ private static String formatBucketReadFailureReason( return reason; } - private BucketActiveRefs buildBucketActiveRefs(List entries) + private BucketActiveRefs buildBucketActiveRefs(List entries) throws IOException { Set manifestPaths = new HashSet<>(); Set segmentRelpaths = new HashSet<>(); - for (PbRemoteLogManifestEntry entry : entries) { + for (RemoteLogManifestInfo entry : entries) { String path = entry.getRemoteLogManifestPath(); manifestPaths.add(path); byte[] manifestBytes = metadataReader.read(new FsPath(path)); @@ -337,10 +339,10 @@ static final class ManifestParseException extends IOException { */ @VisibleForTesting interface AdminFacade { - CompletableFuture listRemoteLogManifests( + CompletableFuture> listRemoteLogManifests( long tableId, @Nullable Long partitionId); - CompletableFuture listKvSnapshots( + CompletableFuture listKvSnapshots( long tableId, @Nullable Long partitionId); } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfig.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfig.java index 82d2a5fbf0..d8ffb7051e 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfig.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfig.java @@ -159,9 +159,9 @@ public static OrphanCleanConfig fromParams(MultipleParameterToolAdapter params) /** * Parses a CLI cutoff value into an absolute epoch-ms timestamp. Empty input falls back to * {@code now - defaultGap}. Explicit input must parse as {@code yyyy-MM-dd HH:mm:ss} in the - * server's local time zone and must be at least {@link #HARD_LOWER_BOUND} earlier than {@code - * now} — closer-to-now cutoffs would race with active writes (see {@code HARD_LOWER_BOUND} - * javadoc). + * Flink action JVM's local time zone and must be at least {@link #HARD_LOWER_BOUND} earlier + * than {@code now} — closer-to-now cutoffs would race with active writes (see {@code + * HARD_LOWER_BOUND} javadoc). */ private static long parseCutoff( String flag, @Nullable String value, long now, Duration defaultGap) { diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleter.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleter.java index 81875974a0..b83bdea913 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleter.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleter.java @@ -26,6 +26,9 @@ import org.apache.fluss.fs.FsPath; import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.RateLimiter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.IOException; import static org.apache.fluss.utils.Preconditions.checkArgument; @@ -47,6 +50,8 @@ @Internal public final class SafeDeleter { + private static final Logger LOG = LoggerFactory.getLogger(SafeDeleter.class); + private final FileSystem fs; private final boolean dryRun; private final AuditLogger audit; @@ -71,7 +76,7 @@ public SafeDeleter(FileSystem fs, boolean dryRun, AuditLogger audit, RateLimiter * (deletion silently failed — e.g. permissions, transient remote-store error). Callers * should track {@code false} returns as delete failures in their run summary. */ - public boolean deleteFile(FsPath file, Decision decision, RuleId ruleId) throws IOException { + public boolean deleteFile(FsPath file, Decision decision, RuleId ruleId) { checkArgument( decision == Decision.DELETE, "deleteFile must only be called for Decision.DELETE, got %s", @@ -81,9 +86,15 @@ public boolean deleteFile(FsPath file, Decision decision, RuleId ruleId) throws return true; } rateLimiter.acquire(); - boolean ok = fs.delete(file, false); - audit.logDeleted(file, ruleId, ok); - return ok; + try { + boolean ok = fs.delete(file, false); + audit.logDeleted(file, ruleId, ok); + return ok; + } catch (IOException e) { + LOG.warn("Failed to delete file: {}", file, e); + audit.logDeleted(file, ruleId, false); + return false; + } } /** @@ -94,7 +105,7 @@ public boolean deleteFile(FsPath file, Decision decision, RuleId ruleId) throws * {@link FileSystem#delete} returned {@code false}. Callers should not increment a "deleted * directory" counter when this returns {@code false}. */ - public boolean deleteEmptyDir(FsPath dir) throws IOException { + public boolean deleteEmptyDir(FsPath dir) { FileStatus[] children = listChildrenSilently(dir); if (children == null || children.length > 0) { return false; @@ -104,11 +115,16 @@ public boolean deleteEmptyDir(FsPath dir) throws IOException { return true; } rateLimiter.acquire(); - boolean ok = fs.delete(dir, false); - if (ok) { - audit.logDirDeleted(dir); + try { + boolean ok = fs.delete(dir, false); + if (ok) { + audit.logDirDeleted(dir); + } + return ok; + } catch (IOException e) { + LOG.warn("Failed to delete empty directory: {}", dir, e); + return false; } - return ok; } private FileStatus[] listChildrenSilently(FsPath dir) { diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java index ea7d9ea161..d1ea20a6bf 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java @@ -28,6 +28,10 @@ import org.apache.fluss.fs.FileStatus; import org.apache.fluss.fs.FileSystem; import org.apache.fluss.fs.FsPath; +import org.apache.fluss.utils.FlussPaths; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayDeque; @@ -43,6 +47,8 @@ @Internal public final class BucketCleaner { + private static final Logger LOG = LoggerFactory.getLogger(BucketCleaner.class); + private final RuleDispatcher dispatcher; private final SafeDeleter safeDeleter; private final AuditLogger audit; @@ -84,7 +90,8 @@ private void walkAndCleanDir(FsPath root, BucketActiveRefs activeRefs, BucketCle FileStatus[] children; try { children = fs.listStatus(dir); - } catch (IOException ignored) { + } catch (IOException e) { + LOG.warn("Failed to list directory: {}", dir, e); continue; } if (children == null) { @@ -93,6 +100,9 @@ private void walkAndCleanDir(FsPath root, BucketActiveRefs activeRefs, BucketCle for (FileStatus child : children) { FsPath childPath = child.getPath(); if (child.isDir()) { + if (FlussPaths.REMOTE_KV_SNAPSHOT_SHARED_DIR.equals(childPath.getName())) { + continue; + } stack.push(childPath); continue; } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java index 62f780e096..f93b73ec36 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java @@ -67,6 +67,7 @@ public static CleanStats execute( trigger.process(new ScopeEnumeratorFunction(config)) .returns(TypeInformation.of(new TypeHint() {})) .setParallelism(1) + .setMaxParallelism(1) .name("ScopeEnumerator"); // Stage 2: ScanAndClean (parallelism=N) @@ -86,8 +87,9 @@ public static CleanStats execute( stats.transform( "StatsAggregate", TypeInformation.of(new TypeHint() {}), - new StatsAggregateOperator(config.dryRun())) - .setParallelism(1); + new StatsAggregateOperator(config.dryRun(), config.extraConfigs())) + .setParallelism(1) + .setMaxParallelism(1); // Execute and collect the single result List collected = collectResults(result); diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java index c6e2cae92d..4f3f179921 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java @@ -161,7 +161,8 @@ private CleanStats processOrphanDirTask(OrphanDirCleanTask task) throws IOExcept FileStatus[] children; try { children = fs.listStatus(dir); - } catch (IOException ignored) { + } catch (IOException e) { + LOG.warn("Failed to list directory: {}", dir, e); continue; } if (children == null) { diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java index b9e2ac4279..84383110cd 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java @@ -206,7 +206,7 @@ private void resolveTable( List partitions = admin.listPartitionInfos(tablePath).get(); TableInfo confirm = admin.getTableInfo(tablePath).get(); if (confirm.getTableId() != tableInfo.getTableId()) { - audit.logSkipTable(dbState.dbName, tableName, "ABA"); + audit.logSkipTable(dbState.dbName, tableName, "table-recreated-during-enumeration"); liveTable.partitionInfosComplete = false; return; } @@ -274,13 +274,6 @@ private void emitBucketTasksForTarget( String remoteDataDir = resolveRemoteDataDir(liveTable.tableInfo, partitionInfo, clusterRemoteDataDir); - if (remoteDataDir == null) { - LOG.warn( - "Table {} partition {} has no resolvable remote.data.dir; skipping", - liveTable.tablePath, - partitionId); - return; - } FsPath remoteLogDir = remoteSubDir(remoteDataDir, FlussPaths.REMOTE_LOG_DIR_NAME); FsPath remoteKvDir = remoteSubDir(remoteDataDir, FlussPaths.REMOTE_KV_DIR_NAME); @@ -482,16 +475,10 @@ private List rootsToScan(@Nullable String clusterRemoteDataDir) { private List rootsForLiveTable( LiveTableScope liveTable, @Nullable String clusterRemoteDataDir) { LinkedHashSet roots = new LinkedHashSet(rootsToScan(clusterRemoteDataDir)); - String tableRoot = resolveRemoteDataDir(liveTable.tableInfo, null, clusterRemoteDataDir); - if (tableRoot != null) { - roots.add(tableRoot); - } + roots.add(resolveRemoteDataDir(liveTable.tableInfo, null, clusterRemoteDataDir)); for (PartitionInfo partitionInfo : liveTable.partitions) { - String partitionRoot = - resolveRemoteDataDir(liveTable.tableInfo, partitionInfo, clusterRemoteDataDir); - if (partitionRoot != null) { - roots.add(partitionRoot); - } + roots.add( + resolveRemoteDataDir(liveTable.tableInfo, partitionInfo, clusterRemoteDataDir)); } return new ArrayList(roots); } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java index 0e24fa7399..e109d37491 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java @@ -18,7 +18,9 @@ package org.apache.fluss.flink.action.orphan.job; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.Configuration; import org.apache.fluss.flink.action.orphan.audit.AuditLogger; +import org.apache.fluss.fs.FileSystem; import org.apache.fluss.fs.FsPath; import org.apache.flink.streaming.api.operators.AbstractStreamOperator; @@ -31,6 +33,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; /** * Stage 3 of the orphan files cleanup job. Runs at parallelism=1 to aggregate {@link CleanStats} @@ -48,11 +51,21 @@ public final class StatsAggregateOperator extends AbstractStreamOperator extraConfigs; private transient CleanStats accumulated; - public StatsAggregateOperator(boolean dryRun) { + public StatsAggregateOperator(boolean dryRun, Map extraConfigs) { this.dryRun = dryRun; + this.extraConfigs = extraConfigs; + } + + @Override + public void open() throws Exception { + super.open(); + if (!extraConfigs.isEmpty()) { + FileSystem.initialize(Configuration.fromMap(extraConfigs), null); + } } @Override diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcherTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcherTest.java index 10c40e7f48..12c66e5aee 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcherTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcherTest.java @@ -17,11 +17,10 @@ package org.apache.fluss.flink.action.orphan.build; +import org.apache.fluss.client.metadata.ActiveKvSnapshots; +import org.apache.fluss.client.metadata.RemoteLogManifestInfo; import org.apache.fluss.fs.FsPath; -import org.apache.fluss.rpc.messages.ListKvSnapshotsResponse; -import org.apache.fluss.rpc.messages.ListRemoteLogManifestsResponse; -import org.apache.fluss.rpc.messages.PbKvSnapshot; -import org.apache.fluss.rpc.messages.PbRemoteLogManifestEntry; +import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.utils.FlussPaths; import org.junit.jupiter.api.Test; @@ -32,7 +31,9 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayDeque; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Deque; import java.util.HashMap; import java.util.HashSet; @@ -200,7 +201,10 @@ void ioErrorMarksBucketReadFailed() { @Test void fetchKvActiveSnapDirsAggregatesPerBucket() { StubAdmin admin = new StubAdmin(new AtomicInteger()); - admin.queueKvResponse(Arrays.asList(kvSnapshot(0, 9), kvSnapshot(0, 10), kvSnapshot(1, 5))); + Map> snapshotIds = new HashMap<>(); + snapshotIds.put(0, new HashSet<>(Arrays.asList(9L, 10L))); + snapshotIds.put(1, new HashSet<>(Arrays.asList(5L))); + admin.queueKvResponseMultiBucket(snapshotIds); ActiveRefsFetcher builder = new ActiveRefsFetcher(admin, /* metadataReader */ null, /* maxRetries= */ 3); @@ -277,7 +281,7 @@ void fetchLogActiveRefsByBucketWithPartitionIdRoutesCorrectly() throws Exception void fetchKvActiveSnapDirsWithPartitionIdRoutesCorrectly() { AtomicInteger rpcCalls = new AtomicInteger(0); StubAdmin admin = new StubAdmin(rpcCalls); - admin.queueKvResponse(Arrays.asList(kvSnapshot(0, 5))); + admin.queueKvResponse(0, 5L); ActiveRefsFetcher builder = new ActiveRefsFetcher(admin, /* metadataReader */ null, /* maxRetries= */ 3); @@ -296,10 +300,6 @@ void fetchKvActiveSnapDirsWithPartitionIdRoutesCorrectly() { .isEqualTo(1); } - private static PbKvSnapshot kvSnapshot(int bucketId, long snapshotId) { - return new PbKvSnapshot().setBucketId(bucketId).setSnapshotId(snapshotId); - } - // ------------------------------------------------------------------------- // Test fixtures // ------------------------------------------------------------------------- @@ -307,10 +307,9 @@ private static PbKvSnapshot kvSnapshot(int bucketId, long snapshotId) { /** Queues per-call responses for ListRemoteLogManifests / ListKvSnapshots and tracks calls. */ private static final class StubAdmin implements ActiveRefsFetcher.AdminFacade { - private final Deque responses = new ArrayDeque<>(); - private final Deque kvResponses = new ArrayDeque<>(); + private final Deque> responses = new ArrayDeque<>(); + private final Deque kvResponses = new ArrayDeque<>(); private final AtomicInteger callCounter; - // Sentinel Long.MIN_VALUE differentiates "never invoked" from "invoked with null". private final AtomicReference lastLogPartitionId = new AtomicReference<>(Long.MIN_VALUE); private final AtomicReference lastKvPartitionId = @@ -325,48 +324,50 @@ void queueResponse(FsPath manifestPath) { } void queueResponse(FsPath manifestPath, int bucketId) { - ListRemoteLogManifestsResponse response = new ListRemoteLogManifestsResponse(); - PbRemoteLogManifestEntry entry = response.addManifest(); - entry.setTableBucket().setTableId(7L).setBucketId(bucketId); - entry.setRemoteLogManifestPath(manifestPath.toString()); - entry.setRemoteLogEndOffset(0L); - responses.add(response); + List list = new ArrayList<>(); + list.add( + new RemoteLogManifestInfo( + new TableBucket(7L, bucketId), manifestPath.toString(), 0L)); + responses.add(list); } void queueMultiBucketResponse(FsPath manifestPath0, FsPath manifestPath1) { - ListRemoteLogManifestsResponse response = new ListRemoteLogManifestsResponse(); - PbRemoteLogManifestEntry entry0 = response.addManifest(); - entry0.setTableBucket().setTableId(7L).setBucketId(0); - entry0.setRemoteLogManifestPath(manifestPath0.toString()); - entry0.setRemoteLogEndOffset(0L); - PbRemoteLogManifestEntry entry1 = response.addManifest(); - entry1.setTableBucket().setTableId(7L).setBucketId(1); - entry1.setRemoteLogManifestPath(manifestPath1.toString()); - entry1.setRemoteLogEndOffset(0L); - responses.add(response); + List list = new ArrayList<>(); + list.add( + new RemoteLogManifestInfo( + new TableBucket(7L, 0), manifestPath0.toString(), 0L)); + list.add( + new RemoteLogManifestInfo( + new TableBucket(7L, 1), manifestPath1.toString(), 0L)); + responses.add(list); } void queueEmptyResponse() { - responses.add(new ListRemoteLogManifestsResponse()); + responses.add(Collections.emptyList()); } - void queueKvResponse(List snapshots) { - ListKvSnapshotsResponse response = new ListKvSnapshotsResponse().setTableId(7L); - for (PbKvSnapshot snapshot : snapshots) { - response.addActiveSnapshot().copyFrom(snapshot); + void queueKvResponse(int bucketId, long... snapshotIds) { + Map> snapshotIdsByBucket = new HashMap<>(); + Set ids = new HashSet<>(); + for (long id : snapshotIds) { + ids.add(id); } - kvResponses.add(response); + snapshotIdsByBucket.put(bucketId, ids); + kvResponses.add(new ActiveKvSnapshots(7L, null, snapshotIdsByBucket)); + } + + void queueKvResponseMultiBucket(Map> snapshotIdsByBucket) { + kvResponses.add(new ActiveKvSnapshots(7L, null, snapshotIdsByBucket)); } @Override - public CompletableFuture listRemoteLogManifests( + public CompletableFuture> listRemoteLogManifests( long tableId, @Nullable Long partitionId) { callCounter.incrementAndGet(); lastLogPartitionId.set(partitionId); - ListRemoteLogManifestsResponse next = responses.poll(); + List next = responses.poll(); if (next == null) { - CompletableFuture failed = - new CompletableFuture<>(); + CompletableFuture> failed = new CompletableFuture<>(); failed.completeExceptionally( new IllegalStateException("StubAdmin: no more queued responses")); return failed; @@ -375,13 +376,13 @@ public CompletableFuture listRemoteLogManifests( } @Override - public CompletableFuture listKvSnapshots( + public CompletableFuture listKvSnapshots( long tableId, @Nullable Long partitionId) { callCounter.incrementAndGet(); lastKvPartitionId.set(partitionId); - ListKvSnapshotsResponse next = kvResponses.poll(); + ActiveKvSnapshots next = kvResponses.poll(); if (next == null) { - CompletableFuture failed = new CompletableFuture<>(); + CompletableFuture failed = new CompletableFuture<>(); failed.completeExceptionally( new IllegalStateException("StubAdmin: no more queued kv responses")); return failed; diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/gateway/AdminReadOnlyGateway.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/gateway/AdminReadOnlyGateway.java index 7d27c1786c..574e1a510d 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/gateway/AdminReadOnlyGateway.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/gateway/AdminReadOnlyGateway.java @@ -42,12 +42,8 @@ import org.apache.fluss.rpc.messages.ListAclsResponse; import org.apache.fluss.rpc.messages.ListDatabasesRequest; import org.apache.fluss.rpc.messages.ListDatabasesResponse; -import org.apache.fluss.rpc.messages.ListKvSnapshotsRequest; -import org.apache.fluss.rpc.messages.ListKvSnapshotsResponse; import org.apache.fluss.rpc.messages.ListPartitionInfosRequest; import org.apache.fluss.rpc.messages.ListPartitionInfosResponse; -import org.apache.fluss.rpc.messages.ListRemoteLogManifestsRequest; -import org.apache.fluss.rpc.messages.ListRemoteLogManifestsResponse; import org.apache.fluss.rpc.messages.ListTablesRequest; import org.apache.fluss.rpc.messages.ListTablesResponse; import org.apache.fluss.rpc.messages.MetadataRequest; @@ -136,27 +132,6 @@ public interface AdminReadOnlyGateway extends RpcGateway { @RPC(api = ApiKeys.GET_METADATA) CompletableFuture metadata(MetadataRequest request); - /** - * List remote log manifest entries for all buckets of a table or single partition. - * - * @param request request with table_id and optional partition_id - * @return per-bucket manifest path and end offset - */ - @RPC(api = ApiKeys.LIST_REMOTE_LOG_MANIFESTS) - CompletableFuture listRemoteLogManifests( - ListRemoteLogManifestsRequest request); - - /** - * List active KV snapshot ids for a unit. The response is the union of (a) the top-N retained - * snapshots for each bucket (controlled by {@code kv.snapshot.num-retained}) and (b) any - * lease-pinned snapshots still in use beyond that retention window. The server merges both - * sources and emits one entry per active {@code (bucket_id, snapshot_id)} pair with no source - * discriminator — callers should treat the entire response as the active set and never under- - * count (a missed entry risks misdeleting an active snapshot). - */ - @RPC(api = ApiKeys.LIST_KV_SNAPSHOTS) - CompletableFuture listKvSnapshots(ListKvSnapshotsRequest request); - /** * Get the latest kv snapshots of a primary key table. A kv snapshot is a snapshot of a kv * tablet, so a table can have multiple kv snapshots. diff --git a/fluss-rpc/src/test/java/org/apache/fluss/rpc/TestingTabletGatewayService.java b/fluss-rpc/src/test/java/org/apache/fluss/rpc/TestingTabletGatewayService.java index 2b0ef6e444..f465bb4a69 100644 --- a/fluss-rpc/src/test/java/org/apache/fluss/rpc/TestingTabletGatewayService.java +++ b/fluss-rpc/src/test/java/org/apache/fluss/rpc/TestingTabletGatewayService.java @@ -51,14 +51,10 @@ import org.apache.fluss.rpc.messages.ListAclsResponse; import org.apache.fluss.rpc.messages.ListDatabasesRequest; import org.apache.fluss.rpc.messages.ListDatabasesResponse; -import org.apache.fluss.rpc.messages.ListKvSnapshotsRequest; -import org.apache.fluss.rpc.messages.ListKvSnapshotsResponse; import org.apache.fluss.rpc.messages.ListOffsetsRequest; import org.apache.fluss.rpc.messages.ListOffsetsResponse; import org.apache.fluss.rpc.messages.ListPartitionInfosRequest; import org.apache.fluss.rpc.messages.ListPartitionInfosResponse; -import org.apache.fluss.rpc.messages.ListRemoteLogManifestsRequest; -import org.apache.fluss.rpc.messages.ListRemoteLogManifestsResponse; import org.apache.fluss.rpc.messages.ListTablesRequest; import org.apache.fluss.rpc.messages.ListTablesResponse; import org.apache.fluss.rpc.messages.LookupRequest; @@ -249,18 +245,6 @@ public CompletableFuture listPartitionInfos( return null; } - @Override - public CompletableFuture listRemoteLogManifests( - ListRemoteLogManifestsRequest request) { - return null; - } - - @Override - public CompletableFuture listKvSnapshots( - ListKvSnapshotsRequest request) { - return null; - } - @Override public CompletableFuture getLakeSnapshot( GetLakeSnapshotRequest request) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java index dd676e378a..892bcbc348 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java @@ -47,12 +47,8 @@ import org.apache.fluss.rpc.messages.InitWriterResponse; import org.apache.fluss.rpc.messages.LimitScanRequest; import org.apache.fluss.rpc.messages.LimitScanResponse; -import org.apache.fluss.rpc.messages.ListKvSnapshotsRequest; -import org.apache.fluss.rpc.messages.ListKvSnapshotsResponse; import org.apache.fluss.rpc.messages.ListOffsetsRequest; import org.apache.fluss.rpc.messages.ListOffsetsResponse; -import org.apache.fluss.rpc.messages.ListRemoteLogManifestsRequest; -import org.apache.fluss.rpc.messages.ListRemoteLogManifestsResponse; import org.apache.fluss.rpc.messages.LookupRequest; import org.apache.fluss.rpc.messages.LookupResponse; import org.apache.fluss.rpc.messages.MetadataRequest; @@ -385,30 +381,6 @@ public CompletableFuture metadata(MetadataRequest request) { return CompletableFuture.completedFuture(metadataResponse); } - @Override - public CompletableFuture listRemoteLogManifests( - ListRemoteLogManifestsRequest request) { - // listRemoteLogManifests is served by CoordinatorService; TabletService rejects the call. - CompletableFuture failed = new CompletableFuture<>(); - failed.completeExceptionally( - new UnsupportedOperationException( - "listRemoteLogManifests is not supported on the tablet server; " - + "send this request to the coordinator.")); - return failed; - } - - @Override - public CompletableFuture listKvSnapshots( - ListKvSnapshotsRequest request) { - // listKvSnapshots is served by CoordinatorService; TabletService rejects the call. - CompletableFuture failed = new CompletableFuture<>(); - failed.completeExceptionally( - new UnsupportedOperationException( - "listKvSnapshots is not supported on the tablet server; " - + "send this request to the coordinator.")); - return failed; - } - @Override public CompletableFuture updateMetadata(UpdateMetadataRequest request) { int coordinatorEpoch = diff --git a/fluss-server/src/test/java/org/apache/fluss/server/tablet/TestTabletServerGateway.java b/fluss-server/src/test/java/org/apache/fluss/server/tablet/TestTabletServerGateway.java index bf597aa341..c78270ea5c 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/tablet/TestTabletServerGateway.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/tablet/TestTabletServerGateway.java @@ -56,14 +56,10 @@ import org.apache.fluss.rpc.messages.ListAclsResponse; import org.apache.fluss.rpc.messages.ListDatabasesRequest; import org.apache.fluss.rpc.messages.ListDatabasesResponse; -import org.apache.fluss.rpc.messages.ListKvSnapshotsRequest; -import org.apache.fluss.rpc.messages.ListKvSnapshotsResponse; import org.apache.fluss.rpc.messages.ListOffsetsRequest; import org.apache.fluss.rpc.messages.ListOffsetsResponse; import org.apache.fluss.rpc.messages.ListPartitionInfosRequest; import org.apache.fluss.rpc.messages.ListPartitionInfosResponse; -import org.apache.fluss.rpc.messages.ListRemoteLogManifestsRequest; -import org.apache.fluss.rpc.messages.ListRemoteLogManifestsResponse; import org.apache.fluss.rpc.messages.ListTablesRequest; import org.apache.fluss.rpc.messages.ListTablesResponse; import org.apache.fluss.rpc.messages.LookupRequest; @@ -166,18 +162,6 @@ public CompletableFuture listPartitionInfos( throw new UnsupportedOperationException(); } - @Override - public CompletableFuture listRemoteLogManifests( - ListRemoteLogManifestsRequest request) { - throw new UnsupportedOperationException(); - } - - @Override - public CompletableFuture listKvSnapshots( - ListKvSnapshotsRequest request) { - throw new UnsupportedOperationException(); - } - @Override public CompletableFuture getLakeSnapshot( GetLakeSnapshotRequest request) { diff --git a/fluss-test-coverage/pom.xml b/fluss-test-coverage/pom.xml index 58fc932fe2..d0654b971c 100644 --- a/fluss-test-coverage/pom.xml +++ b/fluss-test-coverage/pom.xml @@ -506,7 +506,7 @@ org.apache.fluss.flink.action.Action org.apache.fluss.flink.action.ActionFactory org.apache.fluss.flink.action.ActionLoader - org.apache.fluss.flink.action.FlussFlinkActionEntrypoint + org.apache.fluss.flink.action.FlussActionEntrypoint org.apache.fluss.flink.action.orphan.OrphanFilesCleanActionFactory From 407bd575d5ca04e23c18e9d2c9a3aa46de00f3da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Tue, 2 Jun 2026 22:09:56 +0800 Subject: [PATCH 04/19] [flink][test] fix Flink 2.x open() compatibility and flaky snapshot trigger - ScanAndCleanFunction: change open(Configuration) to open(OpenContext) for Flink 2.x compatibility (Flink 2.x removed the Configuration overload) - FlussClusterExtension: triggerSnapshot() returns null on no-op instead of failing hard when snapshot ID does not advance (initSnapshot skips when logOffset <= lastSnapshotOffset) - triggerAndWaitSnapshots() silently skips null buckets (original behavior) --- .../orphan/job/ScanAndCleanFunction.java | 4 ++- .../testutils/FlussClusterExtension.java | 34 +++++++++---------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java index 4f3f179921..69fd5797d1 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java @@ -77,7 +77,9 @@ public ScanAndCleanFunction(long deleteRateLimitPerSecond, Map e } @Override - public void open(org.apache.flink.configuration.Configuration parameters) { + public void open(org.apache.flink.api.common.functions.OpenContext openContext) + throws Exception { + super.open(openContext); if (!extraConfigs.isEmpty()) { FileSystem.initialize(Configuration.fromMap(extraConfigs), null); } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java b/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java index ddf52b8bab..0da413385e 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java @@ -772,33 +772,33 @@ public CompletedSnapshot triggerAndWaitSnapshot(TableBucket tableBucket) { } private Long triggerSnapshot(TableBucket tableBucket) { - Long snapshotId = null; - Long nextSnapshotId = null; for (TabletServer ts : tabletServers.values()) { ReplicaManager.HostedReplica replica = ts.getReplicaManager().getReplica(tableBucket); if (replica instanceof ReplicaManager.OnlineReplica) { Replica r = ((ReplicaManager.OnlineReplica) replica).getReplica(); PeriodicSnapshotManager kvSnapshotManager = r.getKvSnapshotManager(); if (r.isLeader() && kvSnapshotManager != null) { - snapshotId = kvSnapshotManager.currentSnapshotId(); + long snapshotId = kvSnapshotManager.currentSnapshotId(); kvSnapshotManager.triggerSnapshot(); - nextSnapshotId = kvSnapshotManager.currentSnapshotId(); - break; + // triggerSnapshot() submits to guardedExecutor asynchronously. + // Wait for the counter to advance; if it does not, initSnapshot() + // determined there is no new data (logOffset <= lastSnapshotOffset) + // and the trigger was a legitimate no-op — return null. + try { + waitUntil( + () -> kvSnapshotManager.currentSnapshotId() > snapshotId, + Duration.ofSeconds(3), + Duration.ofMillis(50), + ""); + } catch (AssertionError e) { + return null; + } + return snapshotId; } } } - - if (snapshotId != null) { - if (nextSnapshotId > snapshotId) { - // only there is a new snapshot triggered, we return the snapshot id - return snapshotId; - } else { - return null; - } - } else { - fail("No KV snapshot manager found for table bucket " + tableBucket); - return null; - } + fail("No KV snapshot manager found for table bucket " + tableBucket); + return null; } public CompletedSnapshot waitUntilSnapshotFinished(TableBucket tableBucket, long snapshotId) { From 7a26c4e73d5b1a7058ee2373b64935a6548f8fef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Thu, 11 Jun 2026 23:04:52 +0800 Subject: [PATCH 05/19] [server][flink][test] polish orphan cleanup and broaden active snapshot set --- .../flink/action/orphan/OrphanCleanUtils.java | 59 ++++++- .../orphan/OrphanFilesCleanActionFactory.java | 16 +- .../action/orphan/audit/AuditLogger.java | 11 ++ .../orphan/config/OrphanCleanConfig.java | 55 ++----- .../flink/action/orphan/job/CleanStats.java | 22 +-- .../action/orphan/job/EmptyDirSweeper.java | 4 - .../orphan/job/OrphanFilesCleanJob.java | 5 +- .../orphan/job/ScanAndCleanFunction.java | 34 ++-- .../orphan/job/ScopeEnumeratorFunction.java | 76 +++++---- .../orphan/job/StatsAggregateOperator.java | 71 +++++---- .../action/orphan/OrphanFilesCleanITCase.java | 147 +++++++++++++----- .../action/orphan/audit/AuditLoggerTest.java | 78 ++++++++++ .../orphan/config/OrphanCleanConfigTest.java | 40 ++++- .../orphan/job/EmptyDirSweeperTest.java | 13 +- .../testutils/FlussClusterExtension.java | 21 +-- 15 files changed, 443 insertions(+), 209 deletions(-) create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/audit/AuditLoggerTest.java diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanCleanUtils.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanCleanUtils.java index e8ef3d74fd..1a1a1475f0 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanCleanUtils.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanCleanUtils.java @@ -20,6 +20,7 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.client.admin.Admin; import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; import org.apache.fluss.config.cluster.ConfigEntry; import org.apache.fluss.fs.FileStatus; import org.apache.fluss.fs.FileSystem; @@ -36,6 +37,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -108,12 +110,65 @@ public static String resolveRemoteDataDir( */ @Nullable public static String resolveClusterRemoteDataDir(Admin admin) throws Exception { + return resolveClusterRemoteDataDir(fetchClusterConfigMap(admin)); + } + + /** Extracts the single-root {@code remote.data.dir} from a pre-fetched config map. */ + @Nullable + public static String resolveClusterRemoteDataDir(Map configMap) { + return configMap.get(ConfigOptions.REMOTE_DATA_DIR.key()); + } + + /** + * Resolves all cluster-level remote data directories by querying the coordinator's runtime + * configuration. Reads both the single-root {@code remote.data.dir} and the multi-root {@code + * remote.data.dirs}, deduplicates by normalized form, and returns the union as the canonical + * root list. + * + *

This is the authoritative source for determining what storage roots the cleanup action is + * allowed to touch. + * + * @return list of normalized roots (no trailing slash); never {@code null}, may be empty if the + * cluster has neither config set (which should not happen because the coordinator requires + * at least one remote data dir at startup). + */ + public static List resolveClusterRemoteDataDirs(Admin admin) throws Exception { + return resolveClusterRemoteDataDirs(fetchClusterConfigMap(admin)); + } + + /** Extracts all remote data roots from a pre-fetched config map. */ + public static List resolveClusterRemoteDataDirs(Map configMap) { + Configuration conf = Configuration.fromMap(configMap); + LinkedHashSet roots = new LinkedHashSet(); + String singleDir = conf.get(ConfigOptions.REMOTE_DATA_DIR); + if (singleDir != null && !singleDir.isEmpty()) { + roots.add(normalizeRoot(singleDir)); + } + List multiDirs = conf.get(ConfigOptions.REMOTE_DATA_DIRS); + if (multiDirs != null) { + for (String dir : multiDirs) { + if (dir != null && !dir.isEmpty()) { + roots.add(normalizeRoot(dir)); + } + } + } + return new ArrayList(roots); + } + + /** + * Fetches the coordinator's runtime configuration as a key-value map. Use this once and pass + * the result to the map-based overloads of {@link #resolveClusterRemoteDataDir(Map)} and {@link + * #resolveClusterRemoteDataDirs(Map)} to avoid duplicate RPCs. + */ + public static Map fetchClusterConfigMap(Admin admin) throws Exception { Collection entries = admin.describeClusterConfigs().get(); Map map = new HashMap(); for (ConfigEntry entry : entries) { - map.put(entry.key(), entry.value()); + if (entry.value() != null) { + map.put(entry.key(), entry.value()); + } } - return map.get(ConfigOptions.REMOTE_DATA_DIR.key()); + return map; } /** Constructs a remote sub-directory path, normalizing trailing slashes on the root. */ diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanActionFactory.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanActionFactory.java index a77378f781..56c7c681e8 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanActionFactory.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanActionFactory.java @@ -44,8 +44,7 @@ public Optional create(MultipleParameterToolAdapter params) { public String help() { return "Usage: orphan_files_clean --bootstrap-server \n" + " (--database [--table ] | --all-databases)\n" - + " [--scan-root ]...\n" - + " [--older-than 'yyyy-MM-dd HH:mm:ss']\n" + + " [--older-than '']\n" + " [--delete-rate-limit-per-second 100] [--dry-run]\n" + " [--allow-delete-manifest]\n" + " [--allow-clean-orphan-tables]\n" @@ -53,12 +52,13 @@ public String help() { + " [--conf =]...\n" + "\n" + "Notes:\n" - + " --older-than is an absolute wall-clock cutoff (server local timezone). Files\n" - + " with mtime strictly less than the cutoff are deletion-eligible. Default:\n" - + " now - 3d, computed once at startup. The cutoff is frozen for the run, so a\n" - + " long scan cannot accidentally pull in files written after the action started.\n" - + " The cutoff must be at least 1d before now (closer cutoffs would race with\n" - + " mid-write files).\n" + + " --older-than is an absolute wall-clock cutoff in ISO-8601 with explicit\n" + + " offset (e.g. '2024-01-01T00:00:00+08:00' or '2024-01-01T00:00:00Z').\n" + + " Files with mtime strictly less than the cutoff are deletion-eligible.\n" + + " Default: now - 3d, computed once at startup. The cutoff is frozen for the\n" + + " run, so a long scan cannot accidentally pull in files written after the\n" + + " action started. The cutoff must be at least 1d before now (closer cutoffs\n" + + " would race with mid-write files).\n" + " Orphan directory detection (table/partition) relies solely on ID guards\n" + " (maxKnownTableId / maxKnownPartitionId), not mtime.\n" + " --table also disables the orphan-table scan (no sibling orphan-table scan in\n" diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/audit/AuditLogger.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/audit/AuditLogger.java index 4e18f54be9..e073135890 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/audit/AuditLogger.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/audit/AuditLogger.java @@ -196,4 +196,15 @@ public void logSkipOrphanPartition(FsPath dir, String reason) { AUDIT.info( "action=skip_orphan_partition reason={} path={} ts={}", reason, dir, Instant.now()); } + + /** Skip a bucket target because its metadata-resolved root is outside cluster config. */ + public void logSkipBucketOutOfScope(long tableId, Long partitionId, String resolvedRoot) { + AUDIT.info( + "action=skip_bucket_target reason=out-of-scope-root table_id={} partition_id={}" + + " resolved_root={} ts={}", + tableId, + partitionId, + resolvedRoot, + Instant.now()); + } } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfig.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfig.java index d8ffb7051e..6676c04e52 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfig.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfig.java @@ -26,15 +26,11 @@ import java.io.Serializable; import java.time.Duration; import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; +import java.time.OffsetDateTime; import java.time.format.DateTimeParseException; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Optional; @@ -57,14 +53,6 @@ public final class OrphanCleanConfig implements Serializable { private static final long DEFAULT_DELETE_RATE_LIMIT_PER_SECOND = 100L; - /** - * Wall-clock timestamp format accepted on the CLI ({@code yyyy-MM-dd HH:mm:ss}, interpreted in - * the server's local time zone). Matches Apache Paimon's {@code orphan_files_clean older_than} - * grammar to minimize operator context-switching between systems. - */ - private static final DateTimeFormatter CUTOFF_FORMATTER = - DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - private final String bootstrapServer; private final boolean allDatabases; private final @Nullable String database; @@ -73,7 +61,6 @@ public final class OrphanCleanConfig implements Serializable { private final boolean dryRun; private final long deleteRateLimitPerSecond; private final @Nullable Integer parallelism; - private final List scanRoots; private final boolean allowDeleteManifest; private final boolean allowCleanOrphanTables; private final boolean allowCleanOrphanPartitions; @@ -88,7 +75,6 @@ private OrphanCleanConfig( boolean dryRun, long deleteRateLimitPerSecond, @Nullable Integer parallelism, - List scanRoots, boolean allowDeleteManifest, boolean allowCleanOrphanTables, boolean allowCleanOrphanPartitions, @@ -101,7 +87,6 @@ private OrphanCleanConfig( this.dryRun = dryRun; this.deleteRateLimitPerSecond = deleteRateLimitPerSecond; this.parallelism = parallelism; - this.scanRoots = Collections.unmodifiableList(new ArrayList(scanRoots)); this.allowDeleteManifest = allowDeleteManifest; this.allowCleanOrphanTables = allowCleanOrphanTables; this.allowCleanOrphanPartitions = allowCleanOrphanPartitions; @@ -149,7 +134,6 @@ public static OrphanCleanConfig fromParams(MultipleParameterToolAdapter params) params.has("dry-run"), deleteRateLimitPerSecond, parallelism, - parseScanRoots(params.getMultiParameter("scan-root")), allowDeleteManifest, allowCleanOrphanTables, allowCleanOrphanPartitions, @@ -158,27 +142,28 @@ public static OrphanCleanConfig fromParams(MultipleParameterToolAdapter params) /** * Parses a CLI cutoff value into an absolute epoch-ms timestamp. Empty input falls back to - * {@code now - defaultGap}. Explicit input must parse as {@code yyyy-MM-dd HH:mm:ss} in the - * Flink action JVM's local time zone and must be at least {@link #HARD_LOWER_BOUND} earlier - * than {@code now} — closer-to-now cutoffs would race with active writes (see {@code - * HARD_LOWER_BOUND} javadoc). + * {@code now - defaultGap}. Explicit input must be ISO-8601 with an explicit offset (e.g. + * {@code 2024-01-01T00:00:00+08:00} or {@code 2024-01-01T00:00:00Z}) and must be at least + * {@link #HARD_LOWER_BOUND} earlier than {@code now} — closer-to-now cutoffs would race with + * active writes (see {@code HARD_LOWER_BOUND} javadoc). */ private static long parseCutoff( String flag, @Nullable String value, long now, Duration defaultGap) { if (StringUtils.isNullOrWhitespaceOnly(value)) { return now - defaultGap.toMillis(); } - LocalDateTime parsed; + OffsetDateTime parsed; try { - parsed = LocalDateTime.parse(value, CUTOFF_FORMATTER); + parsed = OffsetDateTime.parse(value); } catch (DateTimeParseException e) { throw new IllegalArgumentException( flag - + " must be a timestamp in 'yyyy-MM-dd HH:mm:ss' (server local TZ), got: " + + " must be an ISO-8601 timestamp with an explicit offset (e.g." + + " '2024-01-01T00:00:00+08:00' or '2024-01-01T00:00:00Z'); got: " + value, e); } - long parsedMillis = parsed.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + long parsedMillis = parsed.toInstant().toEpochMilli(); long maxAllowed = now - HARD_LOWER_BOUND.toMillis(); if (parsedMillis > maxAllowed) { throw new IllegalArgumentException( @@ -215,21 +200,6 @@ private static Integer parseParallelism(@Nullable String value) { return p; } - private static List parseScanRoots(@Nullable Collection values) { - if (values == null || values.isEmpty()) { - return Collections.emptyList(); - } - - List scanRoots = new ArrayList(values.size()); - for (String value : values) { - if (StringUtils.isNullOrWhitespaceOnly(value)) { - throw new IllegalArgumentException("--scan-root must not be blank"); - } - scanRoots.add(value); - } - return scanRoots; - } - private static Map parseExtraConfigs(@Nullable Collection values) { if (values == null || values.isEmpty()) { return Collections.emptyMap(); @@ -291,11 +261,6 @@ public Optional parallelism() { return Optional.ofNullable(parallelism); } - /** Returns additional remote.data.dir roots to scan. */ - public List scanRoots() { - return scanRoots; - } - /** * Opt-in to delete {@code .manifest} files. Default {@code false}: mis-deleting an active * manifest leaves the coordinator's manifest pointer dangling and breaks the bucket's metadata diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/CleanStats.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/CleanStats.java index 31e1e66f10..0cb97d5678 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/CleanStats.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/CleanStats.java @@ -24,9 +24,10 @@ import java.util.List; /** - * Aggregatable cleanup statistics emitted by each {@link ScanAndCleanFunction} subtask. The {@code - * touchedDirs} list is collected by the final aggregator for empty-directory sweeping after all - * subtasks complete. + * Per-task cleanup statistics emitted by each {@link ScanAndCleanFunction} subtask. The scalar + * counters are accumulated by {@link StatsAggregateOperator} via simple addition; the short {@code + * touchedDirs} list (typically 1–2 entries per task) is inserted into a {@code HashSet} for O(1) + * deduplication — no list concatenation or O(n²) merge is needed. */ @Internal public final class CleanStats implements Serializable { @@ -49,11 +50,11 @@ public CleanStats( this.deleted = deleted; this.deleteFailures = deleteFailures; this.bytesReclaimed = bytesReclaimed; - this.touchedDirs = new ArrayList<>(touchedDirs); + this.touchedDirs = new ArrayList(touchedDirs); } public static CleanStats empty() { - return new CleanStats(0L, 0L, 0L, 0L, new ArrayList()); + return new CleanStats(0L, 0L, 0L, 0L, new ArrayList(0)); } public long scanned() { @@ -75,15 +76,4 @@ public long bytesReclaimed() { public List touchedDirs() { return touchedDirs; } - - public CleanStats merge(CleanStats other) { - List mergedDirs = new ArrayList<>(this.touchedDirs); - mergedDirs.addAll(other.touchedDirs); - return new CleanStats( - this.scanned + other.scanned, - this.deleted + other.deleted, - this.deleteFailures + other.deleteFailures, - this.bytesReclaimed + other.bytesReclaimed, - mergedDirs); - } } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeper.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeper.java index 191ba87638..30c193cf6d 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeper.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeper.java @@ -54,10 +54,6 @@ public final class EmptyDirSweeper { private final RateLimiter rateLimiter; private final Set touchedRoots = new HashSet(); - public EmptyDirSweeper(boolean dryRun, AuditLogger audit) { - this(dryRun, audit, RateLimiter.create(100.0)); - } - public EmptyDirSweeper(boolean dryRun, AuditLogger audit, RateLimiter rateLimiter) { this.dryRun = dryRun; this.audit = audit; diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java index f93b73ec36..1375452f74 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java @@ -87,7 +87,10 @@ public static CleanStats execute( stats.transform( "StatsAggregate", TypeInformation.of(new TypeHint() {}), - new StatsAggregateOperator(config.dryRun(), config.extraConfigs())) + new StatsAggregateOperator( + config.dryRun(), + config.extraConfigs(), + config.deleteRateLimitPerSecond())) .setParallelism(1) .setMaxParallelism(1); diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java index 69fd5797d1..15e244a763 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java @@ -39,6 +39,7 @@ import java.io.IOException; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Arrays; import java.util.Deque; import java.util.List; import java.util.Map; @@ -56,9 +57,10 @@ * older than the cutoff. * * - *

Each task emits its own {@link CleanStats} immediately upon completion. Delete rate is limited - * per-subtask: {@code configuredRate / runtimeParallelism}. The serial processing within each - * subtask guarantees no concurrent throttler access. + *

Each task emits a single {@link CleanStats} containing scalar counters and the short list of + * directories walked. Delete rate is limited per-subtask: {@code configuredRate / + * runtimeParallelism}. The serial processing within each subtask guarantees no concurrent throttler + * access. */ @Internal public final class ScanAndCleanFunction extends ProcessFunction { @@ -70,6 +72,7 @@ public final class ScanAndCleanFunction extends ProcessFunction extraConfigs; private transient AuditLogger audit; + private transient RateLimiter rateLimiter; public ScanAndCleanFunction(long deleteRateLimitPerSecond, Map extraConfigs) { this.deleteRateLimitPerSecond = deleteRateLimitPerSecond; @@ -84,6 +87,16 @@ public void open(org.apache.flink.api.common.functions.OpenContext openContext) FileSystem.initialize(Configuration.fromMap(extraConfigs), null); } audit = new AuditLogger(); + int parallelism = getRuntimeContext().getTaskInfo().getNumberOfParallelSubtasks(); + int subtaskIndex = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask(); + // Distribute the configured rate as base + 1 extra for the first `remainder` subtasks so + // that the per-subtask rates sum back to the configured aggregate. Each subtask gets at + // least 1/s (hard floor) — when parallelism exceeds the configured rate, the aggregate + // may theoretically exceed it; in practice Batch scheduling limits actual concurrency. + long base = deleteRateLimitPerSecond / parallelism; + long remainder = deleteRateLimitPerSecond % parallelism; + long quota = base + (subtaskIndex < remainder ? 1L : 0L); + rateLimiter = RateLimiter.create(Math.max(1.0, (double) quota)); } @Override @@ -121,7 +134,7 @@ private CleanStats processBucketTask(BucketCleanTask task) throws IOException { BucketCleaner.BucketCleanStats bucketStats = cleaner.clean(activeRefs, logDir, kvDir); - List touchedDirs = new ArrayList(); + List touchedDirs = new ArrayList(2); if (logDir != null) { touchedDirs.add(logDir.toString()); } @@ -208,9 +221,12 @@ private CleanStats processOrphanDirTask(OrphanDirCleanTask task) throws IOExcept } } - List touchedDirs = new ArrayList(); - touchedDirs.add(dirPath.toString()); - return new CleanStats(scanned, deleted, deleteFailures, bytesReclaimed, touchedDirs); + return new CleanStats( + scanned, + deleted, + deleteFailures, + bytesReclaimed, + Arrays.asList(dirPath.toString())); } // ------------------------------------------------------------------------- @@ -218,8 +234,6 @@ private CleanStats processOrphanDirTask(OrphanDirCleanTask task) throws IOExcept // ------------------------------------------------------------------------- private SafeDeleter createSafeDeleter(FileSystem fs, boolean dryRun) { - int parallelism = getRuntimeContext().getTaskInfo().getNumberOfParallelSubtasks(); - double perSubtaskRate = Math.max(1.0, (double) deleteRateLimitPerSecond / parallelism); - return new SafeDeleter(fs, dryRun, audit, RateLimiter.create(perSubtaskRate)); + return new SafeDeleter(fs, dryRun, audit, rateLimiter); } } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java index 84383110cd..44c96dc9f6 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java @@ -60,11 +60,14 @@ import java.util.function.Predicate; import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.enumerateBuckets; +import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.fetchClusterConfigMap; import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.getFileSystemIfExists; import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.listStatuses; +import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.normalizeRoot; import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.physicalPath; import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.remoteSubDir; import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.resolveClusterRemoteDataDir; +import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.resolveClusterRemoteDataDirs; import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.resolveRemoteDataDir; /** @@ -107,21 +110,33 @@ public void processElement(Integer trigger, Context ctx, Collector ou ActiveRefsFetcher fetcher = new ActiveRefsFetcher(admin, 3); MaxKnownIdsTracker tracker = new MaxKnownIdsTracker(); - String clusterRemoteDataDir = resolveClusterRemoteDataDir(admin); + Map clusterConfigMap = fetchClusterConfigMap(admin); + String clusterRemoteDataDir = resolveClusterRemoteDataDir(clusterConfigMap); + List clusterRoots = + normalizeRoots(resolveClusterRemoteDataDirs(clusterConfigMap)); Map dbStates = enumerateActiveScope(admin, audit, tracker); for (DbScanState dbState : dbStates.values()) { for (LiveTableScope liveTable : dbState.liveTables) { - emitBucketTasks(liveTable, fetcher, audit, clusterRemoteDataDir, out); - emitOrphanPartitionDirTasks( - liveTable, tracker, clusterRemoteDataDir, audit, out); + emitBucketTasks( + liveTable, fetcher, audit, clusterRemoteDataDir, clusterRoots, out); + emitOrphanPartitionDirTasks(liveTable, tracker, clusterRoots, audit, out); } - emitOrphanTableDirTasks(dbState, tracker, clusterRemoteDataDir, audit, out); + emitOrphanTableDirTasks(dbState, tracker, clusterRoots, audit, out); } } } + /** Normalizes each root in the list and returns a deduplicated ordered list. */ + private static List normalizeRoots(List roots) { + LinkedHashSet normalized = new LinkedHashSet(); + for (String root : roots) { + normalized.add(normalizeRoot(root)); + } + return new ArrayList(normalized); + } + // ------------------------------------------------------------------------- // Scope enumeration (coordinator RPCs only) // ------------------------------------------------------------------------- @@ -230,6 +245,7 @@ private void emitBucketTasks( ActiveRefsFetcher fetcher, AuditLogger audit, @Nullable String clusterRemoteDataDir, + List clusterRoots, Collector out) { if (liveTable.partitioned && !liveTable.partitionInfosComplete) { return; @@ -240,7 +256,13 @@ private void emitBucketTasks( : Collections.singletonList(null); for (PartitionInfo partitionInfo : partitionTargets) { emitBucketTasksForTarget( - liveTable, partitionInfo, fetcher, audit, clusterRemoteDataDir, out); + liveTable, + partitionInfo, + fetcher, + audit, + clusterRemoteDataDir, + clusterRoots, + out); } } @@ -250,9 +272,20 @@ private void emitBucketTasksForTarget( ActiveRefsFetcher fetcher, AuditLogger audit, @Nullable String clusterRemoteDataDir, + List clusterRoots, Collector out) { Long partitionId = partitionInfo == null ? null : partitionInfo.getPartitionId(); + String remoteDataDir = + resolveRemoteDataDir(liveTable.tableInfo, partitionInfo, clusterRemoteDataDir); + + // Scope guard: skip this target if its metadata-resolved root is not part of the + // cluster's configured remote data directories. + if (!clusterRoots.contains(normalizeRoot(remoteDataDir))) { + audit.logSkipBucketOutOfScope(liveTable.tableId, partitionId, remoteDataDir); + return; + } + LogActiveRefsFetchResult logResult = fetcher.fetchLogActiveRefsByBucket(liveTable.tableId, partitionId); if (!logResult.listOk()) { @@ -272,9 +305,6 @@ private void emitBucketTasksForTarget( } } - String remoteDataDir = - resolveRemoteDataDir(liveTable.tableInfo, partitionInfo, clusterRemoteDataDir); - FsPath remoteLogDir = remoteSubDir(remoteDataDir, FlussPaths.REMOTE_LOG_DIR_NAME); FsPath remoteKvDir = remoteSubDir(remoteDataDir, FlussPaths.REMOTE_KV_DIR_NAME); @@ -353,7 +383,7 @@ private void emitBucketTasksForTarget( private void emitOrphanTableDirTasks( DbScanState dbState, MaxKnownIdsTracker tracker, - @Nullable String clusterRemoteDataDir, + List clusterRoots, AuditLogger audit, Collector out) throws IOException { @@ -364,7 +394,7 @@ private void emitOrphanTableDirTasks( Set activeTableIds = dbState.activeTableIds; long maxKnownTableId = tracker.maxKnownTableId(); boolean emit = config.allowCleanOrphanTables(); - for (String root : rootsToScan(clusterRemoteDataDir)) { + for (String root : clusterRoots) { for (String topLevel : TOP_LEVEL_DIRS) { FsPath dbDir = remoteSubDir(root, topLevel + "/" + dbState.dbName); if (emit) { @@ -395,7 +425,7 @@ private void emitOrphanTableDirTasks( private void emitOrphanPartitionDirTasks( LiveTableScope liveTable, MaxKnownIdsTracker tracker, - @Nullable String clusterRemoteDataDir, + List clusterRoots, AuditLogger audit, Collector out) throws IOException { @@ -405,7 +435,7 @@ private void emitOrphanPartitionDirTasks( Set activePartitionIds = liveTable.activePartitionIds; long maxKnownPartitionId = tracker.maxKnownPartitionId(); boolean emit = config.allowCleanOrphanPartitions(); - for (String root : rootsForLiveTable(liveTable, clusterRemoteDataDir)) { + for (String root : clusterRoots) { for (String topLevel : TOP_LEVEL_DIRS) { FsPath tableDir = FlussPaths.remoteTableDir( @@ -463,26 +493,6 @@ private void forEachOrphanDirUnderParent( // Helpers // ------------------------------------------------------------------------- - private List rootsToScan(@Nullable String clusterRemoteDataDir) { - LinkedHashSet roots = new LinkedHashSet(); - if (clusterRemoteDataDir != null) { - roots.add(clusterRemoteDataDir); - } - roots.addAll(config.scanRoots()); - return new ArrayList(roots); - } - - private List rootsForLiveTable( - LiveTableScope liveTable, @Nullable String clusterRemoteDataDir) { - LinkedHashSet roots = new LinkedHashSet(rootsToScan(clusterRemoteDataDir)); - roots.add(resolveRemoteDataDir(liveTable.tableInfo, null, clusterRemoteDataDir)); - for (PartitionInfo partitionInfo : liveTable.partitions) { - roots.add( - resolveRemoteDataDir(liveTable.tableInfo, partitionInfo, clusterRemoteDataDir)); - } - return new ArrayList(roots); - } - private static String classifyName(Throwable e) { return RpcErrorClassifier.classify(e).name(); } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java index e109d37491..556b80c2fa 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java @@ -22,6 +22,7 @@ import org.apache.fluss.flink.action.orphan.audit.AuditLogger; import org.apache.fluss.fs.FileSystem; import org.apache.fluss.fs.FsPath; +import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.RateLimiter; import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.BoundedOneInput; @@ -32,32 +33,44 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.List; +import java.util.HashSet; import java.util.Map; +import java.util.Set; /** - * Stage 3 of the orphan files cleanup job. Runs at parallelism=1 to aggregate {@link CleanStats} - * from all Stage 2 subtasks and perform the final empty-directory sweep. + * Stage 3 of the orphan files cleanup job. Runs at parallelism=1 to aggregate per-subtask {@link + * CleanStats} records and perform the final empty-directory sweep. * *

Implemented as a custom operator (not ProcessFunction) because {@code ProcessOperator} does - * not implement {@link BoundedOneInput} — the {@code endInput()} callback would never fire. This - * operator accumulates all incoming stats and performs the empty-dir sweep in {@code endInput()}. + * not implement {@link BoundedOneInput} — the {@code endInput()} callback would never fire. + * + *

Scalar counters are accumulated into four longs; directory paths from each incoming {@link + * CleanStats#touchedDirs()} are inserted into a {@link HashSet} for O(1) deduplication. The final + * empty-dir sweep happens in {@link #endInput()}. */ @Internal public final class StatsAggregateOperator extends AbstractStreamOperator implements OneInputStreamOperator, BoundedOneInput { - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 2L; private static final Logger LOG = LoggerFactory.getLogger(StatsAggregateOperator.class); private final boolean dryRun; private final Map extraConfigs; + private final long deleteRateLimitPerSecond; - private transient CleanStats accumulated; + private transient long scanned; + private transient long deleted; + private transient long deleteFailures; + private transient long bytesReclaimed; + private transient Set touchedDirs; + private transient RateLimiter sweepRateLimiter; - public StatsAggregateOperator(boolean dryRun, Map extraConfigs) { + public StatsAggregateOperator( + boolean dryRun, Map extraConfigs, long deleteRateLimitPerSecond) { this.dryRun = dryRun; this.extraConfigs = extraConfigs; + this.deleteRateLimitPerSecond = deleteRateLimitPerSecond; } @Override @@ -66,39 +79,43 @@ public void open() throws Exception { if (!extraConfigs.isEmpty()) { FileSystem.initialize(Configuration.fromMap(extraConfigs), null); } + scanned = 0L; + deleted = 0L; + deleteFailures = 0L; + bytesReclaimed = 0L; + touchedDirs = new HashSet(); + sweepRateLimiter = RateLimiter.create((double) deleteRateLimitPerSecond); } @Override public void processElement(StreamRecord element) { - if (accumulated == null) { - accumulated = CleanStats.empty(); - } - accumulated = accumulated.merge(element.getValue()); + CleanStats stats = element.getValue(); + scanned += stats.scanned(); + deleted += stats.deleted(); + deleteFailures += stats.deleteFailures(); + bytesReclaimed += stats.bytesReclaimed(); + touchedDirs.addAll(stats.touchedDirs()); } @Override public void endInput() { - if (accumulated == null) { - accumulated = CleanStats.empty(); - } - - long emptyDirsRemoved = sweepEmptyDirs(accumulated.touchedDirs()); - long totalDeleted = accumulated.deleted() + emptyDirsRemoved; + long emptyDirsRemoved = sweepEmptyDirs(touchedDirs); + long totalDeleted = deleted + emptyDirsRemoved; CleanStats finalStats = new CleanStats( - accumulated.scanned(), + scanned, totalDeleted, - accumulated.deleteFailures(), - accumulated.bytesReclaimed(), - new ArrayList()); + deleteFailures, + bytesReclaimed, + new ArrayList(0)); LOG.info( "Orphan cleanup complete: scanned={}, deleted={} (files={}, emptyDirs={}), " + "failures={}, bytesReclaimed={}", finalStats.scanned(), totalDeleted, - accumulated.deleted(), + deleted, emptyDirsRemoved, finalStats.deleteFailures(), finalStats.bytesReclaimed()); @@ -106,13 +123,13 @@ public void endInput() { output.collect(new StreamRecord<>(finalStats)); } - private long sweepEmptyDirs(List touchedDirs) { - if (touchedDirs.isEmpty()) { + private long sweepEmptyDirs(Set dirs) { + if (dirs.isEmpty()) { return 0L; } AuditLogger audit = new AuditLogger(); - EmptyDirSweeper sweeper = new EmptyDirSweeper(dryRun, audit); - for (String dir : touchedDirs) { + EmptyDirSweeper sweeper = new EmptyDirSweeper(dryRun, audit, sweepRateLimiter); + for (String dir : dirs) { sweeper.registerTouched(new FsPath(dir)); } try { diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java index c50f0321ed..118ee78903 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java @@ -38,7 +38,7 @@ import org.apache.fluss.server.zk.ZooKeeperClient; import org.apache.fluss.server.zk.data.BucketSnapshot; import org.apache.fluss.server.zk.data.RemoteLogManifestHandle; -import org.apache.fluss.server.zk.data.ZkData.BucketSnapshotIdZNode; +import org.apache.fluss.server.zk.data.ZkData.BucketSnapshotsZNode; import org.apache.fluss.server.zk.data.ZkData.PartitionZNode; import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.FlussPaths; @@ -55,7 +55,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.api.io.TempDir; import java.net.URI; import java.nio.charset.StandardCharsets; @@ -64,7 +63,8 @@ import java.nio.file.Paths; import java.nio.file.attribute.FileTime; import java.time.Duration; -import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collections; @@ -390,33 +390,6 @@ void optInCleansOrphanTableDirWhenEnabled() throws Exception { && m.contains(layout.orphanFile.toString())); } - @Test - void scanRootIncludesAdditionalRemoteRootWhenOrphanTableCleanupEnabled(@TempDir Path extraRoot) - throws Exception { - String dbName = newDatabaseName("scanroot"); - long tableId = allocateDroppedTableId(dbName, "seed_table"); - createLogTable(dbName, "live_anchor"); - OrphanTableLayout layout = - createOldOrphanTableLayout( - extraRoot, dbName, tableId, "external_table", "99999999999999999999.log"); - - runCleanerForDatabase( - false, - dbName, - "--scan-root", - extraRoot.toUri().toString(), - "--allow-clean-orphan-tables"); - - assertThat(Files.exists(layout.orphanFile)).isFalse(); - assertThat(Files.exists(layout.tableDir)).isFalse(); - assertThat(auditMessages()) - .anyMatch( - m -> - m.contains("action=deleted") - && m.contains("rule=log-segment") - && m.contains(layout.orphanFile.toString())); - } - @Test void livePrimaryKeyTableDoesNotCleanKvSharedFiles() throws Exception { String dbName = newDatabaseName("livepk"); @@ -489,16 +462,32 @@ void retainedNonLatestSnapshotPreserved() throws Exception { seedKvSnapshots(tableBucket, remoteKvTabletDir, new long[] {1L, 2L, 3L, 4L}); + // Drop a snapshot directory locally without registering it in ZK to model a + // crash-leftover. The active set is derived from ZK references, so this + // unreferenced snapshot must still be cleaned — guarding the assertions below + // from passing trivially when the cleaner fails to scan at all. + long unreferencedSnapshotId = 99L; + Path unreferencedSnapshotDir = + localPath( + FlussPaths.remoteKvSnapshotDir(remoteKvTabletDir, unreferencedSnapshotId)); + Files.createDirectories(unreferencedSnapshotDir); + Path unreferencedMeta = unreferencedSnapshotDir.resolve("_METADATA"); + Files.write(unreferencedMeta, new byte[] {0x33}); + makeOld(unreferencedMeta); + makeOld(unreferencedSnapshotDir); + runCleanerForDatabase(false, dbName); + // Every snapshot still referenced in ZK is preserved, regardless of recency. assertThat(Files.exists(localPath(FlussPaths.remoteKvSnapshotDir(remoteKvTabletDir, 1L)))) - .isFalse(); + .isTrue(); assertThat(Files.exists(localPath(FlussPaths.remoteKvSnapshotDir(remoteKvTabletDir, 2L)))) - .isFalse(); + .isTrue(); assertThat(Files.exists(localPath(FlussPaths.remoteKvSnapshotDir(remoteKvTabletDir, 3L)))) .isTrue(); assertThat(Files.exists(localPath(FlussPaths.remoteKvSnapshotDir(remoteKvTabletDir, 4L)))) .isTrue(); + assertThat(Files.exists(unreferencedSnapshotDir)).isFalse(); } @Test @@ -784,19 +773,17 @@ void kvUnitFailureDoesNotBlockLogCleanup() throws Exception { Path faultInjectionOrphanLogSegment = createOldSegmentFile(tablePath, "99999999999999999999.log"); - // Corrupt the BucketSnapshot znode bytes so server-side listBucketSnapshots throws on - // decode. Client-side fetchKvActiveSnapDirs propagates the exception and - // cleanActiveTableFiles catches it to emit skip_kv_target. + // Inject a non-numeric child znode under BucketSnapshotsZNode so server-side + // listBucketSnapshotIds throws NumberFormatException on Long.parseLong. Client-side + // fetchKvActiveSnapDirs propagates the exception and cleanActiveTableFiles catches it + // to emit skip_kv_target. ZooKeeperClient zk = FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(); - String snapshotZnodePath = BucketSnapshotIdZNode.path(tableBucket, activeSnapshotId); - byte[] originalSnapshotBytes = zk.getCuratorClient().getData().forPath(snapshotZnodePath); - zk.getCuratorClient() - .setData() - .forPath(snapshotZnodePath, "not-json".getBytes(StandardCharsets.UTF_8)); + String invalidChildPath = BucketSnapshotsZNode.path(tableBucket) + "/not-a-long"; + zk.getCuratorClient().create().forPath(invalidChildPath, new byte[0]); try { runCleanerForDatabase(false, dbName); } finally { - zk.getCuratorClient().setData().forPath(snapshotZnodePath, originalSnapshotBytes); + zk.getCuratorClient().delete().forPath(invalidChildPath); } // KV target was skipped: skip_kv_target audit fires AND snap-77 orphan files preserved. @@ -851,6 +838,80 @@ void kvUnitFailureDoesNotBlockLogCleanup() throws Exception { .hasSizeGreaterThanOrEqualTo(2); } + @Test + void optInCleansOrphanPartitionDir() throws Exception { + String dbName = newDatabaseName("orphanpart"); + // Create two partitioned tables so the tracker observes both partition IDs. + // The second table's partition ID is higher. We plant an orphan under the second + // table using the first table's (lower) ID so the guard passes: + // orphanId <= maxKnownPartitionId. + PartitionedTableLayout tableA = createPartitionedLogTable(dbName, "table_a", "pa"); + PartitionedTableLayout tableB = createPartitionedLogTable(dbName, "table_b", "pb"); + + long orphanPartitionId = + Math.min( + tableA.partitionInfo.getPartitionId(), + tableB.partitionInfo.getPartitionId()); + // Plant orphan under whichever table does NOT own the lower-ID partition. + PartitionedTableLayout targetTable = + (tableA.partitionInfo.getPartitionId() == orphanPartitionId) ? tableB : tableA; + + OrphanPartitionLayout orphan = + createOldOrphanPartitionLayout( + remoteDataRoot(), + targetTable.tablePath, + targetTable.tableId, + "ghost", + orphanPartitionId, + "99999999999999999999.log"); + + runCleanerForDatabase(false, dbName, "--allow-clean-orphan-partitions"); + + assertThat(Files.exists(orphan.orphanFile)) + .as("orphan partition file must be deleted") + .isFalse(); + assertThat(Files.exists(orphan.partitionDir)) + .as("orphan partition dir must be removed") + .isFalse(); + assertThat(auditMessages()) + .anyMatch( + m -> + m.contains("action=deleted") + && m.contains("rule=log-segment") + && m.contains(orphan.orphanFile.toString())); + } + + @Test + void emptyDirsSweptAfterOrphanFileDeletion() throws Exception { + String dbName = newDatabaseName("emptydir"); + TablePath tablePath = createLogTable(dbName, "emptydir_table"); + Path activeSegment = seedActiveBucketManifest(tablePath); + + // Create an orphan file as the sole content of its UUID directory. + Path orphan = createOldSegmentFile(tablePath, "99999999999999999999.log"); + Path orphanSegmentDir = orphan.getParent(); + + // Pre-condition: the segment directory exists before cleanup. + assertThat(Files.exists(orphanSegmentDir)).isTrue(); + + runCleanerForDatabase(false, dbName); + + // The orphan file must be deleted. + assertThat(Files.exists(orphan)).as("orphan file must be deleted").isFalse(); + // The now-empty UUID directory must also be swept. + assertThat(Files.exists(orphanSegmentDir)) + .as("empty segment dir must be swept after cleanup") + .isFalse(); + // Active segment and its directory survive. + assertThat(Files.exists(activeSegment)).as("active segment must survive").isTrue(); + assertThat(auditMessages()) + .anyMatch( + m -> + m.contains("action=deleted") + && m.contains("rule=log-segment") + && m.contains(orphan.toString())); + } + private TablePath createLogTable(String databaseName, String tableName) throws Exception { admin.createDatabase(databaseName, DatabaseDescriptor.EMPTY, true).get(); TablePath tablePath = TablePath.of(databaseName, tableName); @@ -1082,12 +1143,12 @@ private void runCleanerForAllDatabases(boolean dryRun, String... extraArgs) thro } private static final DateTimeFormatter CUTOFF_FORMATTER = - DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + DateTimeFormatter.ISO_OFFSET_DATE_TIME; private static void appendCommonArgs(List args, boolean dryRun, String... extraArgs) { // Tests back-date their orphan files to now - 2d via makeOld(); a cutoff at now - 1d // safely puts those files strictly before the cutoff (mtime < cutoff → DELETE-eligible). - String cutoff = LocalDateTime.now().minusDays(1).format(CUTOFF_FORMATTER); + String cutoff = OffsetDateTime.now(ZoneOffset.UTC).minusDays(1).format(CUTOFF_FORMATTER); args.add("--older-than"); args.add(cutoff); for (String extraArg : extraArgs) { diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/audit/AuditLoggerTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/audit/AuditLoggerTest.java new file mode 100644 index 0000000000..0495470a5a --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/audit/AuditLoggerTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.audit; + +import org.apache.fluss.flink.action.orphan.rule.RuleId; +import org.apache.fluss.fs.FsPath; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThatCode; + +/** + * Smoke tests covering every {@link AuditLogger} entry point. The audit logger has no return + * values, no in-memory state, and only forwards structured key/value lines to SLF4J — these tests + * therefore only verify each emit path is reachable without throwing, which is exactly the + * production contract callers rely on. + */ +class AuditLoggerTest { + + private final AuditLogger audit = new AuditLogger(); + private final FsPath path = new FsPath("file:///tmp/orphan/file"); + private final FsPath dir = new FsPath("file:///tmp/orphan/dir"); + + @Test + void allFileAndDirEventsEmitWithoutThrowing() { + assertThatCode( + () -> { + audit.logCutoff(1700000000000L); + audit.logDeleted(path, RuleId.LOG_SEGMENT, true); + audit.logDeleted(path, RuleId.LOG_SEGMENT, false); + audit.logWouldDelete(path, RuleId.KV_SNAPSHOT_FILE); + audit.logDirDeleted(dir); + audit.logWouldDeleteDir(dir); + audit.logSkipUnknown(path, RuleId.UNKNOWN); + }) + .doesNotThrowAnyException(); + } + + @Test + void allScopeAndBucketSkipEventsEmitWithoutThrowing() { + assertThatCode( + () -> { + audit.logBucketAborted("tb=1,bucket=0", "manifest-parse-failed"); + audit.logSkipDb("db", "list-tables-failed"); + audit.logSkipTable("db", "t", "get-table-info-failed"); + audit.logSkipPartitionList("db", "t", "list-partitions-failed"); + audit.logSkipKvTarget(7L, 11L, "list-kv-snapshots-failed"); + audit.logSkipKvTarget(7L, null, "list-kv-snapshots-failed"); + audit.logSkipKvBucket(7L, 11L, 0, "empty-active-set"); + audit.logSkipKvBucket(7L, null, 0, "empty-active-set"); + audit.logSkipLogTarget(7L, 11L, "list-remote-manifests-failed"); + audit.logSkipLogTarget(7L, null, "list-remote-manifests-failed"); + audit.logSkipLogBucket(7L, 11L, 0, "no-remote-manifest"); + audit.logSkipLogBucket(7L, null, 0, "no-remote-manifest"); + audit.logSkipOrphanTable(dir, "opt-in-not-set"); + audit.logSkipOrphanTableScan("db", "incomplete-table-id-set"); + audit.logSkipOrphanPartition(dir, "opt-in-not-set"); + audit.logSkipBucketOutOfScope(7L, 11L, "file:///other/root"); + audit.logSkipBucketOutOfScope(7L, null, "file:///other/root"); + }) + .doesNotThrowAnyException(); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfigTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfigTest.java index dc41066e50..b564f90b90 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfigTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfigTest.java @@ -22,7 +22,8 @@ import org.junit.jupiter.api.Test; import java.time.Duration; -import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import static org.assertj.core.api.Assertions.assertThat; @@ -32,7 +33,7 @@ class OrphanCleanConfigTest { private static final DateTimeFormatter CUTOFF_FORMATTER = - DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + DateTimeFormatter.ISO_OFFSET_DATE_TIME; @Test void parsesAllDatabasesWithDefaults() { @@ -74,7 +75,7 @@ void databaseAndAllDatabasesAreMutuallyExclusive() { @Test void cutoffCloserThanOneDayRejected() { - LocalDateTime tooClose = LocalDateTime.now().minusMinutes(30); + OffsetDateTime tooClose = OffsetDateTime.now(ZoneOffset.UTC).minusMinutes(30); assertThatThrownBy( () -> OrphanCleanConfig.fromParams( @@ -90,6 +91,39 @@ void cutoffCloserThanOneDayRejected() { .hasMessageContaining("at least 1d before now"); } + @Test + void cutoffWithoutExplicitOffsetRejected() { + assertThatThrownBy( + () -> + OrphanCleanConfig.fromParams( + MultipleParameterToolAdapter.fromArgs( + new String[] { + "--bootstrap-server", + "h:9123", + "--all-databases", + "--older-than", + "2024-01-01 00:00:00" + }))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("ISO-8601"); + } + + @Test + void cutoffWithExplicitOffsetParsed() { + OffsetDateTime cutoff = OffsetDateTime.now(ZoneOffset.UTC).minusDays(2).withNano(0); + OrphanCleanConfig cfg = + OrphanCleanConfig.fromParams( + MultipleParameterToolAdapter.fromArgs( + new String[] { + "--bootstrap-server", + "h:9123", + "--all-databases", + "--older-than", + cutoff.format(CUTOFF_FORMATTER) + })); + assertThat(cfg.olderThanMillis()).isEqualTo(cutoff.toInstant().toEpochMilli()); + } + @Test void tableCannotBeUsedWithAllDatabases() { assertThatThrownBy( diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeperTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeperTest.java index cc47f95671..fdfd60acb7 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeperTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeperTest.java @@ -19,6 +19,7 @@ import org.apache.fluss.flink.action.orphan.audit.AuditLogger; import org.apache.fluss.fs.FsPath; +import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.RateLimiter; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -31,13 +32,17 @@ class EmptyDirSweeperTest { + private static EmptyDirSweeper newSweeper(boolean dryRun) { + return new EmptyDirSweeper(dryRun, new AuditLogger(), RateLimiter.create(1000.0)); + } + @Test void deletesEmptyDirsBottomUp(@TempDir Path tmp) throws IOException { Path a = Files.createDirectories(tmp.resolve("a")); Path b = Files.createDirectories(a.resolve("b")); Path c = Files.createDirectories(b.resolve("c")); - EmptyDirSweeper sweeper = new EmptyDirSweeper(false, new AuditLogger()); + EmptyDirSweeper sweeper = newSweeper(false); sweeper.registerTouched(new FsPath(a.toString())); long removed = sweeper.sweep(); @@ -53,7 +58,7 @@ void leavesNonEmptyDirsAlone(@TempDir Path tmp) throws IOException { Path b = Files.createDirectories(a.resolve("b")); Files.write(b.resolve("keep.txt"), new byte[] {0x42}); - EmptyDirSweeper sweeper = new EmptyDirSweeper(false, new AuditLogger()); + EmptyDirSweeper sweeper = newSweeper(false); sweeper.registerTouched(new FsPath(a.toString())); long removed = sweeper.sweep(); @@ -67,7 +72,7 @@ void dryRunCountsWouldDeleteButDoesNotActuallyDelete(@TempDir Path tmp) throws I Path a = Files.createDirectories(tmp.resolve("a")); Path b = Files.createDirectories(a.resolve("b")); - EmptyDirSweeper sweeper = new EmptyDirSweeper(true /* dryRun */, new AuditLogger()); + EmptyDirSweeper sweeper = newSweeper(true /* dryRun */); sweeper.registerTouched(new FsPath(a.toString())); long removed = sweeper.sweep(); @@ -79,7 +84,7 @@ void dryRunCountsWouldDeleteButDoesNotActuallyDelete(@TempDir Path tmp) throws I @Test void nonExistentRootIsNoOp(@TempDir Path tmp) throws IOException { - EmptyDirSweeper sweeper = new EmptyDirSweeper(false, new AuditLogger()); + EmptyDirSweeper sweeper = newSweeper(false); sweeper.registerTouched(new FsPath(tmp.resolve("does-not-exist").toString())); assertThat(sweeper.sweep()).isEqualTo(0L); } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java b/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java index 0da413385e..aeb8a848df 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java @@ -779,21 +779,16 @@ private Long triggerSnapshot(TableBucket tableBucket) { PeriodicSnapshotManager kvSnapshotManager = r.getKvSnapshotManager(); if (r.isLeader() && kvSnapshotManager != null) { long snapshotId = kvSnapshotManager.currentSnapshotId(); + // KvTablet#getGuardedExecutor runs the submitted task synchronously + // on the calling thread inside the kv write lock, so initSnapshot() + // has already completed by the time triggerSnapshot() returns. The + // counter is either bumped (a new snapshot was scheduled) or left + // unchanged (no new data since the last snapshot — legitimate no-op). kvSnapshotManager.triggerSnapshot(); - // triggerSnapshot() submits to guardedExecutor asynchronously. - // Wait for the counter to advance; if it does not, initSnapshot() - // determined there is no new data (logOffset <= lastSnapshotOffset) - // and the trigger was a legitimate no-op — return null. - try { - waitUntil( - () -> kvSnapshotManager.currentSnapshotId() > snapshotId, - Duration.ofSeconds(3), - Duration.ofMillis(50), - ""); - } catch (AssertionError e) { - return null; + if (kvSnapshotManager.currentSnapshotId() > snapshotId) { + return snapshotId; } - return snapshotId; + return null; } } } From 7565d27c1e8a7bebb944ecb3e179022b77f943d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Mon, 15 Jun 2026 17:25:23 +0800 Subject: [PATCH 06/19] [flink] Polish orphan cleanup action --- ...rg.apache.fluss.flink.action.ActionFactory | 0 ...rg.apache.fluss.flink.action.ActionFactory | 19 +++ ...rg.apache.fluss.flink.action.ActionFactory | 19 +++ ...rg.apache.fluss.flink.action.ActionFactory | 19 +++ .../action/orphan/OrphanFilesCleanAction.java | 2 +- .../orphan/OrphanFilesCleanActionFactory.java | 6 +- .../action/orphan/OrphanFilesCleanITCase.java | 115 ------------------ .../action/orphan/audit/AuditLoggerTest.java | 78 ------------ .../orphan/build/MaxKnownIdsTrackerTest.java | 58 --------- .../action/orphan/fs/SafeDeleterTest.java | 40 ------ .../orphan/rule/OrphanDirDetectorTest.java | 85 ------------- .../orphan/rule/RuleDispatcherTest.java | 73 ----------- 12 files changed, 61 insertions(+), 453 deletions(-) rename fluss-flink/{fluss-flink-common => fluss-flink-1.18}/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory (100%) create mode 100644 fluss-flink/fluss-flink-1.19/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory create mode 100644 fluss-flink/fluss-flink-1.20/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory create mode 100644 fluss-flink/fluss-flink-2.2/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory delete mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/audit/AuditLoggerTest.java delete mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/MaxKnownIdsTrackerTest.java delete mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/OrphanDirDetectorTest.java delete mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/RuleDispatcherTest.java diff --git a/fluss-flink/fluss-flink-common/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory b/fluss-flink/fluss-flink-1.18/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory similarity index 100% rename from fluss-flink/fluss-flink-common/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory rename to fluss-flink/fluss-flink-1.18/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory diff --git a/fluss-flink/fluss-flink-1.19/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory b/fluss-flink/fluss-flink-1.19/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory new file mode 100644 index 0000000000..c30c9dd5ab --- /dev/null +++ b/fluss-flink/fluss-flink-1.19/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.fluss.flink.action.orphan.OrphanFilesCleanActionFactory diff --git a/fluss-flink/fluss-flink-1.20/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory b/fluss-flink/fluss-flink-1.20/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory new file mode 100644 index 0000000000..c30c9dd5ab --- /dev/null +++ b/fluss-flink/fluss-flink-1.20/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.fluss.flink.action.orphan.OrphanFilesCleanActionFactory diff --git a/fluss-flink/fluss-flink-2.2/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory b/fluss-flink/fluss-flink-2.2/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory new file mode 100644 index 0000000000..c30c9dd5ab --- /dev/null +++ b/fluss-flink/fluss-flink-2.2/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.fluss.flink.action.orphan.OrphanFilesCleanActionFactory diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanAction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanAction.java index c5b49944bc..4622f41c0f 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanAction.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanAction.java @@ -54,7 +54,7 @@ public void run() throws Exception { CleanStats stats = OrphanFilesCleanJob.execute(env, config, config.parallelism().orElse(null)); LOG.info( - "orphan_files_clean done: scope={} scanned={} deleted={} failures={}" + "remove_orphan_files done: scope={} scanned={} deleted={} failures={}" + " bytesReclaimed={} dryRun={}", scopeDescription(), stats.scanned(), diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanActionFactory.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanActionFactory.java index 56c7c681e8..e63dfcede1 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanActionFactory.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanActionFactory.java @@ -31,7 +31,7 @@ public class OrphanFilesCleanActionFactory implements ActionFactory { @Override public String identifier() { - return "orphan_files_clean"; + return "remove_orphan_files"; } @Override @@ -42,7 +42,7 @@ public Optional create(MultipleParameterToolAdapter params) { @Override public String help() { - return "Usage: orphan_files_clean --bootstrap-server \n" + return "Usage: remove_orphan_files --bootstrap-server \n" + " (--database [--table ] | --all-databases)\n" + " [--older-than '']\n" + " [--delete-rate-limit-per-second 100] [--dry-run]\n" @@ -68,7 +68,7 @@ public String help() { + " fs.oss.accessKeySecret, fs.oss.endpoint, fs.oss.region). Repeatable.\n" + "\n" + "Examples:\n" - + " orphan_files_clean --bootstrap-server host:9123 --all-databases\n" + + " remove_orphan_files --bootstrap-server host:9123 --all-databases\n" + " --conf fs.oss.accessKeyId=XXXX --conf fs.oss.accessKeySecret=YYYY\n" + " --conf fs.oss.endpoint=oss-cn-hangzhou-internal.aliyuncs.com\n" + " --conf fs.oss.region=cn-hangzhou"; diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java index 118ee78903..061204c5c1 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java @@ -237,25 +237,6 @@ void mixedOrphanAndActiveFilesInSameBucket() throws Exception { .noneMatch(m -> m.contains("action=deleted") && m.contains(activeFile2.toString())); } - @Test - void happyPathDeletesOrphanSegment() throws Exception { - String dbName = newDatabaseName("happy"); - TablePath tablePath = createLogTable(dbName, "happy_path"); - Path activeSegment = seedActiveBucketManifest(tablePath); - Path orphan = createOldSegmentFile(tablePath, "99999999999999999999.log"); - - runCleanerForDatabase(false, dbName); - - assertThat(Files.exists(orphan)).isFalse(); - assertThat(Files.exists(activeSegment)).isTrue(); - assertThat(auditMessages()) - .anyMatch( - m -> - m.contains("action=deleted") - && m.contains("rule=log-segment") - && m.contains(orphan.toString())); - } - @Test void dryRunDoesNotDeleteFiles() throws Exception { String dbName = newDatabaseName("dryrun"); @@ -284,31 +265,6 @@ void dryRunDoesNotDeleteFiles() throws Exception { && m.contains(activeSegment.toString())); } - @Test - void unknownExtensionFilePreserved() throws Exception { - String dbName = newDatabaseName("unknown"); - TablePath tablePath = createLogTable(dbName, "unknown_file"); - Path activeSegment = seedActiveBucketManifest(tablePath); - Path orphan = createOldSegmentFile(tablePath, "99999999999999999999.log"); - Path unknown = orphan.getParent().resolve("data.bloomfilter"); - Files.write(unknown, new byte[] {0x24}); - makeOld(unknown); - - runCleanerForDatabase(false, dbName); - - assertThat(Files.exists(orphan)).isFalse(); - assertThat(Files.exists(unknown)).isTrue(); - assertThat(Files.exists(activeSegment)).isTrue(); - assertThat(auditMessages()) - .anyMatch( - m -> - m.contains("action=deleted") - && m.contains("rule=log-segment") - && m.contains(orphan.toString())); - assertThat(auditMessages()) - .anyMatch(m -> m.contains("action=skip_unknown") && m.contains(unknown.toString())); - } - /** * Seeds a remote log manifest + matching active segment under a freshly-allocated UUID so the * active-file cleanup reaches {@code ManifestReadStatus.RESOLVED} for bucket 0 of the given log @@ -390,24 +346,6 @@ void optInCleansOrphanTableDirWhenEnabled() throws Exception { && m.contains(layout.orphanFile.toString())); } - @Test - void livePrimaryKeyTableDoesNotCleanKvSharedFiles() throws Exception { - String dbName = newDatabaseName("livepk"); - TablePath tablePath = createPrimaryKeyTable(dbName, "live_pk_table"); - Path orphanKvFile = - createOldKvSharedSstFile( - tablePath, "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa-orphan.sst"); - - runCleanerForDatabase(false, dbName); - - assertThat(Files.exists(orphanKvFile)).isTrue(); - assertThat(auditMessages()) - .noneMatch( - m -> - m.contains("rule=kv-shared-sst") - && m.contains(orphanKvFile.toString())); - } - @Test void pkOrphanTableRetainsSharedSstEvenWithOptIn() throws Exception { String dbName = newDatabaseName("orphankv"); @@ -595,40 +533,6 @@ void multipleRoundsConvergeAfterManifestUpsert() throws Exception { && m.contains(oldSegment.toString())); } - @Test - void logBucketSkippedOnNoRemoteManifest() throws Exception { - String dbName = newDatabaseName("logbucketskip"); - TablePath tablePath = createLogTable(dbName, "no_manifest_yet"); - TableInfo tableInfo = admin.getTableInfo(tablePath).get(); - - runCleanerForDatabase(false, dbName); - - assertThat(auditMessages()) - .anyMatch( - m -> - m.contains("action=skip_log_bucket") - && m.contains("reason=no_remote_manifest") - && m.contains("table_id=" + tableInfo.getTableId()) - && m.contains("bucket_id=0")); - } - - @Test - void kvBucketSkippedOnEmptyBucketActiveRefs() throws Exception { - String dbName = newDatabaseName("kvbucketskip"); - TablePath tablePath = createPrimaryKeyTable(dbName, "no_snapshots_yet"); - TableInfo tableInfo = admin.getTableInfo(tablePath).get(); - - runCleanerForDatabase(false, dbName); - - assertThat(auditMessages()) - .anyMatch( - m -> - m.contains("action=skip_kv_bucket") - && m.contains("reason=empty_active_set") - && m.contains("table_id=" + tableInfo.getTableId()) - && m.contains("bucket_id=0")); - } - @Test void singleTableModeSkipsOrphanTableScan() throws Exception { String dbName = newDatabaseName("singletable"); @@ -996,25 +900,6 @@ private Path createOldLogManifestFile(TablePath tablePath, String fileName) thro return file; } - private Path createOldKvSharedSstFile(TablePath tablePath, String fileName) throws Exception { - TableInfo tableInfo = admin.getTableInfo(tablePath).get(); - org.apache.fluss.fs.FsPath kvTabletDir = - FlussPaths.remoteKvTabletDir( - new org.apache.fluss.fs.FsPath( - FLUSS_CLUSTER_EXTENSION.getRemoteDataDir() - + "/" - + FlussPaths.REMOTE_KV_DIR_NAME), - PhysicalTablePath.of(tablePath), - new TableBucket(tableInfo.getTableId(), 0)); - org.apache.fluss.fs.FsPath sharedDir = FlussPaths.remoteKvSharedDir(kvTabletDir); - Path localSharedDir = Paths.get(java.net.URI.create(sharedDir.toString())); - Files.createDirectories(localSharedDir); - Path file = localSharedDir.resolve(fileName); - Files.write(file, new byte[] {0x24}); - makeOld(file); - return file; - } - private PartitionedTableLayout createPartitionedLogTable( String databaseName, String tableName, String partitionValue) throws Exception { admin.createDatabase(databaseName, DatabaseDescriptor.EMPTY, true).get(); diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/audit/AuditLoggerTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/audit/AuditLoggerTest.java deleted file mode 100644 index 0495470a5a..0000000000 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/audit/AuditLoggerTest.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.flink.action.orphan.audit; - -import org.apache.fluss.flink.action.orphan.rule.RuleId; -import org.apache.fluss.fs.FsPath; - -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThatCode; - -/** - * Smoke tests covering every {@link AuditLogger} entry point. The audit logger has no return - * values, no in-memory state, and only forwards structured key/value lines to SLF4J — these tests - * therefore only verify each emit path is reachable without throwing, which is exactly the - * production contract callers rely on. - */ -class AuditLoggerTest { - - private final AuditLogger audit = new AuditLogger(); - private final FsPath path = new FsPath("file:///tmp/orphan/file"); - private final FsPath dir = new FsPath("file:///tmp/orphan/dir"); - - @Test - void allFileAndDirEventsEmitWithoutThrowing() { - assertThatCode( - () -> { - audit.logCutoff(1700000000000L); - audit.logDeleted(path, RuleId.LOG_SEGMENT, true); - audit.logDeleted(path, RuleId.LOG_SEGMENT, false); - audit.logWouldDelete(path, RuleId.KV_SNAPSHOT_FILE); - audit.logDirDeleted(dir); - audit.logWouldDeleteDir(dir); - audit.logSkipUnknown(path, RuleId.UNKNOWN); - }) - .doesNotThrowAnyException(); - } - - @Test - void allScopeAndBucketSkipEventsEmitWithoutThrowing() { - assertThatCode( - () -> { - audit.logBucketAborted("tb=1,bucket=0", "manifest-parse-failed"); - audit.logSkipDb("db", "list-tables-failed"); - audit.logSkipTable("db", "t", "get-table-info-failed"); - audit.logSkipPartitionList("db", "t", "list-partitions-failed"); - audit.logSkipKvTarget(7L, 11L, "list-kv-snapshots-failed"); - audit.logSkipKvTarget(7L, null, "list-kv-snapshots-failed"); - audit.logSkipKvBucket(7L, 11L, 0, "empty-active-set"); - audit.logSkipKvBucket(7L, null, 0, "empty-active-set"); - audit.logSkipLogTarget(7L, 11L, "list-remote-manifests-failed"); - audit.logSkipLogTarget(7L, null, "list-remote-manifests-failed"); - audit.logSkipLogBucket(7L, 11L, 0, "no-remote-manifest"); - audit.logSkipLogBucket(7L, null, 0, "no-remote-manifest"); - audit.logSkipOrphanTable(dir, "opt-in-not-set"); - audit.logSkipOrphanTableScan("db", "incomplete-table-id-set"); - audit.logSkipOrphanPartition(dir, "opt-in-not-set"); - audit.logSkipBucketOutOfScope(7L, 11L, "file:///other/root"); - audit.logSkipBucketOutOfScope(7L, null, "file:///other/root"); - }) - .doesNotThrowAnyException(); - } -} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/MaxKnownIdsTrackerTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/MaxKnownIdsTrackerTest.java deleted file mode 100644 index 46a3814a04..0000000000 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/MaxKnownIdsTrackerTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.flink.action.orphan.build; - -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; - -class MaxKnownIdsTrackerTest { - - @Test - void initialValuesAreNegativeOne() { - MaxKnownIdsTracker t = new MaxKnownIdsTracker(); - assertThat(t.maxKnownTableId()).isEqualTo(-1L); - assertThat(t.maxKnownPartitionId()).isEqualTo(-1L); - } - - @Test - void observeTableIdMonotonicallyIncreases() { - MaxKnownIdsTracker t = new MaxKnownIdsTracker(); - t.observeTableId(5L); - assertThat(t.maxKnownTableId()).isEqualTo(5L); - t.observeTableId(3L); - assertThat(t.maxKnownTableId()).isEqualTo(5L); // never decreases - t.observeTableId(10L); - assertThat(t.maxKnownTableId()).isEqualTo(10L); - } - - @Test - void observePartitionIdMonotonicallyIncreases() { - MaxKnownIdsTracker t = new MaxKnownIdsTracker(); - t.observePartitionId(7L); - t.observePartitionId(2L); - assertThat(t.maxKnownPartitionId()).isEqualTo(7L); - } - - @Test - void independentTracking() { - MaxKnownIdsTracker t = new MaxKnownIdsTracker(); - t.observeTableId(100L); - assertThat(t.maxKnownPartitionId()).isEqualTo(-1L); - } -} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleterTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleterTest.java index 42022164c7..147192411e 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleterTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleterTest.java @@ -23,7 +23,6 @@ import org.apache.fluss.fs.FileSystem; import org.apache.fluss.fs.FsPath; import org.apache.fluss.fs.local.LocalFileSystem; -import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.RateLimiter; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -83,45 +82,6 @@ void deleteEmptyDirActuallyDeletes() throws IOException { assertThat(Files.exists(dir)).isFalse(); } - @Test - void multipleDeletesAllSucceed() throws IOException { - Path a = Files.createFile(tmp.resolve("a.log")); - Path b = Files.createFile(tmp.resolve("b.log")); - Path c = Files.createFile(tmp.resolve("c.log")); - Files.write(a, new byte[] {1}); - Files.write(b, new byte[] {2}); - Files.write(c, new byte[] {3}); - Path emptyDir = Files.createDirectory(tmp.resolve("emptyDir")); - - RateLimiter limiter = RateLimiter.create(Double.MAX_VALUE); - SafeDeleter deleter = new SafeDeleter(localFs(), false, new AuditLogger(), limiter); - - deleter.deleteFile(new FsPath(a.toString()), Decision.DELETE, RuleId.LOG_SEGMENT); - deleter.deleteFile(new FsPath(b.toString()), Decision.DELETE, RuleId.LOG_SEGMENT); - deleter.deleteFile(new FsPath(c.toString()), Decision.DELETE, RuleId.LOG_SEGMENT); - deleter.deleteEmptyDir(new FsPath(emptyDir.toString())); - - assertThat(Files.exists(a)).isFalse(); - assertThat(Files.exists(b)).isFalse(); - assertThat(Files.exists(c)).isFalse(); - assertThat(Files.exists(emptyDir)).isFalse(); - } - - @Test - void dryRunPreservesAllFiles() throws IOException { - Path file = Files.createFile(tmp.resolve("orphan.log")); - Path emptyDir = Files.createDirectory(tmp.resolve("emptyDir")); - - RateLimiter limiter = RateLimiter.create(Double.MAX_VALUE); - SafeDeleter deleter = new SafeDeleter(localFs(), true, new AuditLogger(), limiter); - - deleter.deleteFile(new FsPath(file.toString()), Decision.DELETE, RuleId.LOG_SEGMENT); - deleter.deleteEmptyDir(new FsPath(emptyDir.toString())); - - assertThat(Files.exists(file)).isTrue(); - assertThat(Files.exists(emptyDir)).isTrue(); - } - private static FileSystem localFs() { return LocalFileSystem.getSharedInstance(); } diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/OrphanDirDetectorTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/OrphanDirDetectorTest.java deleted file mode 100644 index aa874a4520..0000000000 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/OrphanDirDetectorTest.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.flink.action.orphan.rule; - -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; - -import static org.assertj.core.api.Assertions.assertThat; - -/** Unit tests for {@link OrphanDirDetector}. */ -class OrphanDirDetectorTest { - - // --- Table directory detection --- - - @Test - void tableOrphanWhenIdLeMaxKnown() { - assertThat(OrphanDirDetector.isOrphanTable("foo-15", ids(10L, 20L), 30L)).isTrue(); - } - - @Test - void tableNotOrphanWhenIdGreaterThanMaxKnown() { - assertThat(OrphanDirDetector.isOrphanTable("foo-99", ids(10L, 20L), 30L)).isFalse(); - } - - @Test - void tableNotOrphanWhenInActiveSet() { - assertThat(OrphanDirDetector.isOrphanTable("foo-10", ids(10L, 20L), 30L)).isFalse(); - } - - @Test - void tableNotOrphanWhenNameFormatBad() { - assertThat(OrphanDirDetector.isOrphanTable("no_id_here", Collections.emptySet(), 10L)) - .isFalse(); - } - - // --- Partition directory detection --- - - @Test - void partitionOrphanWhenIdLeMaxKnown() { - assertThat(OrphanDirDetector.isOrphanPartition("dt=2024-p150", ids(101L, 102L), 200L)) - .isTrue(); - } - - @Test - void partitionNotOrphanWhenIdGreaterThanMaxKnown() { - assertThat( - OrphanDirDetector.isOrphanPartition( - "dt=2024-p250", Collections.emptySet(), 200L)) - .isFalse(); - } - - @Test - void partitionNotOrphanWhenInActiveSet() { - assertThat(OrphanDirDetector.isOrphanPartition("dt=2024-p150", ids(150L), 200L)).isFalse(); - } - - @Test - void partitionNotOrphanWhenMissingPPrefix() { - assertThat(OrphanDirDetector.isOrphanPartition("0", Collections.emptySet(), 200L)) - .isFalse(); - } - - private static Set ids(Long... values) { - return new HashSet(Arrays.asList(values)); - } -} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/RuleDispatcherTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/RuleDispatcherTest.java deleted file mode 100644 index 68d38d6170..0000000000 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/rule/RuleDispatcherTest.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.flink.action.orphan.rule; - -import org.apache.fluss.fs.FsPath; - -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; - -/** Tests for {@link RuleDispatcher}. */ -class RuleDispatcherTest { - - private static final String SEGMENT_ID = "11111111-1111-1111-1111-111111111111"; - - private final RuleDispatcher dispatcher = new RuleDispatcher(); - - @Test - void dispatchesLogSegmentRule() { - assertThat(dispatcher.dispatch(file("/log/db/t-1/0/" + SEGMENT_ID + "/000.log")).id()) - .isEqualTo(RuleId.LOG_SEGMENT); - } - - @Test - void dispatchesLogManifestRule() { - assertThat(dispatcher.dispatch(file("/log/db/t-1/0/metadata/current.manifest")).id()) - .isEqualTo(RuleId.LOG_MANIFEST); - } - - @Test - void dispatchesKvSnapshotFileRule() { - assertThat(dispatcher.dispatch(file("/kv/db/t-1/0/snap-5/001.sst")).id()) - .isEqualTo(RuleId.KV_SNAPSHOT_FILE); - } - - @Test - void dispatchesKvSharedSstRule() { - assertThat(dispatcher.dispatch(file("/kv/db/t-1/0/shared/abc-001.sst")).id()) - .isEqualTo(RuleId.KV_SHARED_SST); - } - - @Test - void fallsBackToUnknownRule() { - assertThat(dispatcher.dispatch(file("/random/path/file.bin")).id()) - .isEqualTo(RuleId.UNKNOWN); - } - - @Test - void unknownRuleSkipsConservatively() { - FileRule rule = dispatcher.dispatch(file("/random/path/file.bin")); - assertThat(rule.evaluate(file("/random/path/file.bin"), BucketActiveRefs.empty(), 0L)) - .isEqualTo(Decision.SKIP_UNKNOWN); - } - - private static FileMeta file(String path) { - return new FileMeta(new FsPath(path), 0L, 0L); - } -} From 744085eb5dc9755cf0332d2c97ff0eb96bcf022d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Mon, 15 Jun 2026 19:02:13 +0800 Subject: [PATCH 07/19] [server][flink][common] move RemoteLogManifest to fluss-common Move RemoteLogManifest and RemoteLogManifestJsonSerde from fluss-server to fluss-common (org.apache.fluss.remote) so the orphan-files cleanup client can reuse the shared serde via RemoteLogManifest.fromJsonBytes() instead of maintaining a parallel hand-written JSON parser. --- .../fluss}/remote/RemoteLogManifest.java | 7 +-- .../remote/RemoteLogManifestJsonSerde.java | 3 +- .../RemoteLogManifestJsonSerdeTest.java | 5 +- .../orphan/build/ActiveRefsFetcher.java | 55 +++++++------------ .../action/orphan/OrphanFilesCleanITCase.java | 22 ++++++-- .../orphan/build/ActiveRefsFetcherTest.java | 30 ++++++---- .../log/remote/DefaultRemoteLogStorage.java | 1 + .../server/log/remote/LogTieringTask.java | 1 + .../server/log/remote/RemoteLogManager.java | 1 + .../server/log/remote/RemoteLogStorage.java | 1 + .../server/log/remote/RemoteLogTablet.java | 1 + .../rebalance/RebalanceManagerITCase.java | 2 +- .../remote/DefaultRemoteLogStorageTest.java | 1 + .../server/log/remote/RemoteLogITCase.java | 1 + .../log/remote/TestingRemoteLogStorage.java | 1 + 15 files changed, 72 insertions(+), 60 deletions(-) rename {fluss-server/src/main/java/org/apache/fluss/server/log => fluss-common/src/main/java/org/apache/fluss}/remote/RemoteLogManifest.java (96%) rename {fluss-server/src/main/java/org/apache/fluss/server/log => fluss-common/src/main/java/org/apache/fluss}/remote/RemoteLogManifestJsonSerde.java (98%) rename {fluss-server/src/test/java/org/apache/fluss/server/log => fluss-common/src/test/java/org/apache/fluss}/remote/RemoteLogManifestJsonSerdeTest.java (97%) diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManifest.java b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifest.java similarity index 96% rename from fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManifest.java rename to fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifest.java index b255b8718d..bc856e361d 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManifest.java +++ b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifest.java @@ -15,12 +15,10 @@ * limitations under the License. */ -package org.apache.fluss.server.log.remote; +package org.apache.fluss.remote; -import org.apache.fluss.annotation.VisibleForTesting; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; -import org.apache.fluss.remote.RemoteLogSegment; import java.util.ArrayList; import java.util.Collections; @@ -33,7 +31,7 @@ /** * A remote log manifest is an immutable list of current {@link RemoteLogSegment} which can - * represent a snapshot of {@link RemoteLogTablet}. + * represent a snapshot of a remote log tablet. */ public class RemoteLogManifest { private final PhysicalTablePath physicalTablePath; @@ -122,7 +120,6 @@ public TableBucket getTableBucket() { return tableBucket; } - @VisibleForTesting public List getRemoteLogSegmentList() { return remoteLogSegmentList; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManifestJsonSerde.java b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestJsonSerde.java similarity index 98% rename from fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManifestJsonSerde.java rename to fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestJsonSerde.java index 27c5488490..c90a85ea02 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManifestJsonSerde.java +++ b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestJsonSerde.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.fluss.server.log.remote; +package org.apache.fluss.remote; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; -import org.apache.fluss.remote.RemoteLogSegment; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.fluss.utils.json.JsonDeserializer; diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManifestJsonSerdeTest.java b/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestJsonSerdeTest.java similarity index 97% rename from fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManifestJsonSerdeTest.java rename to fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestJsonSerdeTest.java index da4024ffc4..e095132158 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManifestJsonSerdeTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestJsonSerdeTest.java @@ -15,18 +15,17 @@ * limitations under the License. */ -package org.apache.fluss.server.log.remote; +package org.apache.fluss.remote; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TablePath; -import org.apache.fluss.remote.RemoteLogSegment; import org.apache.fluss.utils.json.JsonSerdeTestBase; import java.util.Arrays; import java.util.UUID; -/** Tests of {@link org.apache.fluss.server.log.remote.RemoteLogManifestJsonSerde}. */ +/** Tests of {@link RemoteLogManifestJsonSerde}. */ class RemoteLogManifestJsonSerdeTest extends JsonSerdeTestBase { private static final PhysicalTablePath TABLE_PATH1 = PhysicalTablePath.of(TablePath.of("db", "mytable")); diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcher.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcher.java index a11c46e157..26b01d7a62 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcher.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcher.java @@ -26,9 +26,8 @@ import org.apache.fluss.flink.action.orphan.rule.BucketActiveRefs; import org.apache.fluss.fs.FSDataInputStream; import org.apache.fluss.fs.FsPath; -import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; -import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; -import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.fluss.remote.RemoteLogManifest; +import org.apache.fluss.remote.RemoteLogSegment; import org.apache.fluss.utils.FlussPaths; import org.apache.fluss.utils.IOUtils; import org.apache.fluss.utils.RetryUtils; @@ -42,7 +41,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -68,13 +66,6 @@ @Internal public final class ActiveRefsFetcher { - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - - private static final String REMOTE_LOG_SEGMENTS_FIELD = "remote_log_segments"; - private static final String SEGMENT_ID_FIELD = "segment_id"; - private static final String START_OFFSET_FIELD = "start_offset"; - private static final String END_OFFSET_FIELD = "end_offset"; - /** * Retry backoff base used by {@link RetryUtils} for per-target RPCs. With the default 3 retries * and exponential backoff (200 → 400 → cap) this caps total retry delay at ~600ms — negligible @@ -191,10 +182,9 @@ public LogActiveRefsFetchResult fetchLogActiveRefsByBucket( partitionId, bucketId, e)); - } catch (ManifestParseException | JsonProcessingException e) { - // Manifest payload is unreadable as JSON or violates the expected shape — corrupt - // or schema-skewed, not a transient FS hiccup. Distinct reason so operators triage - // separately (re-running the action will not recover). + } catch (ManifestParseException e) { + // Manifest payload is unreadable or violates the shared manifest serde schema. + // Distinct reason so operators triage separately from transient FS hiccups. readFailures.put( bucketId, formatBucketReadFailureReason( @@ -289,16 +279,20 @@ private BucketActiveRefs buildBucketActiveRefs(List entri return new BucketActiveRefs(segmentRelpaths, Collections.emptySet(), manifestPaths); } - private Set parseLogSegmentRelativePaths(byte[] manifestBytes) throws IOException { - JsonNode root = OBJECT_MAPPER.readTree(manifestBytes); - JsonNode segmentsNode = requiredNode(root, REMOTE_LOG_SEGMENTS_FIELD); + private Set parseLogSegmentRelativePaths(byte[] manifestBytes) + throws ManifestParseException { + RemoteLogManifest manifest; + try { + manifest = RemoteLogManifest.fromJsonBytes(manifestBytes); + } catch (RuntimeException e) { + throw new ManifestParseException("Failed to parse remote log manifest", e); + } + Set relativePaths = new HashSet<>(); - Iterator iterator = segmentsNode.elements(); - while (iterator.hasNext()) { - JsonNode segmentNode = iterator.next(); - String segmentId = requiredNode(segmentNode, SEGMENT_ID_FIELD).asText(); - long startOffset = requiredNode(segmentNode, START_OFFSET_FIELD).asLong(); - long endOffset = requiredNode(segmentNode, END_OFFSET_FIELD).asLong(); + for (RemoteLogSegment segment : manifest.getRemoteLogSegmentList()) { + String segmentId = segment.remoteLogSegmentId().toString(); + long startOffset = segment.remoteLogStartOffset(); + long endOffset = segment.remoteLogEndOffset(); String baseOffset = FlussPaths.filenamePrefixFromOffset(startOffset); String writerOffset = FlussPaths.filenamePrefixFromOffset(endOffset); @@ -311,15 +305,6 @@ private Set parseLogSegmentRelativePaths(byte[] manifestBytes) throws IO return relativePaths; } - private static JsonNode requiredNode(JsonNode node, String fieldName) - throws ManifestParseException { - JsonNode field = node.get(fieldName); - if (field == null) { - throw new ManifestParseException("Missing required field: " + fieldName); - } - return field; - } - /** * Thrown when a remote-log manifest payload is structurally invalid (missing required field, * wrong shape). Distinct from {@link IOException} so the bucket-read failure handler can route @@ -327,8 +312,8 @@ private static JsonNode requiredNode(JsonNode node, String fieldName) * bucket — same skip-this-round outcome, different operator triage. */ static final class ManifestParseException extends IOException { - ManifestParseException(String message) { - super(message); + ManifestParseException(String message, Throwable cause) { + super(message, cause); } } diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java index 061204c5c1..c7ae62dadb 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java @@ -185,13 +185,20 @@ void mixedOrphanAndActiveFilesInSameBucket() throws Exception { Path manifest = localPath(manifestPath); Files.createDirectories(manifest.getParent()); String manifestContent = - "{\"remote_log_segments\":[" + "{\"version\":1," + + "\"database\":\"db\"," + + "\"table\":\"t\"," + + "\"table_id\":0," + + "\"bucket_id\":0," + + "\"remote_log_segments\":[" + "{\"segment_id\":\"" + activeId1 - + "\",\"start_offset\":0,\"end_offset\":99}," + + "\",\"start_offset\":0,\"end_offset\":99," + + "\"max_timestamp\":0,\"size_in_bytes\":1}," + "{\"segment_id\":\"" + activeId2 - + "\",\"start_offset\":100,\"end_offset\":199}" + + "\",\"start_offset\":100,\"end_offset\":199," + + "\"max_timestamp\":0,\"size_in_bytes\":1}" + "]}"; Files.write(manifest, manifestContent.getBytes(StandardCharsets.UTF_8)); makeOld(manifest); @@ -1120,13 +1127,20 @@ private static Path localPath(FsPath path) { } private static String manifestJson(String segmentId, long startOffset, long endOffset) { - return "{\"remote_log_segments\":[{" + return "{\"version\":1," + + "\"database\":\"db\"," + + "\"table\":\"t\"," + + "\"table_id\":0," + + "\"bucket_id\":0," + + "\"remote_log_segments\":[{" + "\"segment_id\":\"" + segmentId + "\",\"start_offset\":" + startOffset + ",\"end_offset\":" + endOffset + + ",\"max_timestamp\":0," + + "\"size_in_bytes\":1" + "}]}"; } diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcherTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcherTest.java index 12c66e5aee..7144b4f031 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcherTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcherTest.java @@ -98,11 +98,7 @@ void fileNotFoundMarksBucketReadFailedWithoutRetry() { void fetchLogActiveRefsByBucket_abortsOnlyFailedBucket() throws Exception { FsPath p0 = new FsPath("oss://b/log/db/t-7/0/metadata/p0.manifest"); FsPath p1 = new FsPath("oss://b/log/db/t-7/1/metadata/p1.manifest"); - String manifestJson = - "{\"remote_log_segments\":[{" - + "\"segment_id\":\"11111111-1111-1111-1111-111111111111\"," - + "\"start_offset\":7," - + "\"end_offset\":9}]}"; + String manifestJson = manifestJson("11111111-1111-1111-1111-111111111111", 7L, 9L); AtomicInteger rpcCalls = new AtomicInteger(0); StubAdmin admin = new StubAdmin(rpcCalls); @@ -244,11 +240,7 @@ void fetchKvActiveSnapDirsRetriesThenReportsListFailure() { @Test void fetchLogActiveRefsByBucketWithPartitionIdRoutesCorrectly() throws Exception { FsPath p0 = new FsPath("oss://b/log/db/t-7/0/metadata/p0.manifest"); - String manifestJson = - "{\"remote_log_segments\":[{" - + "\"segment_id\":\"11111111-1111-1111-1111-111111111111\"," - + "\"start_offset\":7," - + "\"end_offset\":9}]}"; + String manifestJson = manifestJson("11111111-1111-1111-1111-111111111111", 7L, 9L); AtomicInteger rpcCalls = new AtomicInteger(0); StubAdmin admin = new StubAdmin(rpcCalls); @@ -304,6 +296,24 @@ void fetchKvActiveSnapDirsWithPartitionIdRoutesCorrectly() { // Test fixtures // ------------------------------------------------------------------------- + private static String manifestJson(String segmentId, long startOffset, long endOffset) { + return "{\"version\":1," + + "\"database\":\"db\"," + + "\"table\":\"t\"," + + "\"table_id\":7," + + "\"bucket_id\":0," + + "\"remote_log_segments\":[{" + + "\"segment_id\":\"" + + segmentId + + "\",\"start_offset\":" + + startOffset + + ",\"end_offset\":" + + endOffset + + ",\"max_timestamp\":0," + + "\"size_in_bytes\":1" + + "}]}"; + } + /** Queues per-call responses for ListRemoteLogManifests / ListKvSnapshots and tracks calls. */ private static final class StubAdmin implements ActiveRefsFetcher.AdminFacade { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/DefaultRemoteLogStorage.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/DefaultRemoteLogStorage.java index 33f29f2830..56e8e24091 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/DefaultRemoteLogStorage.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/DefaultRemoteLogStorage.java @@ -25,6 +25,7 @@ import org.apache.fluss.fs.FsPath; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.remote.RemoteLogManifest; import org.apache.fluss.remote.RemoteLogSegment; import org.apache.fluss.utils.CloseableRegistry; import org.apache.fluss.utils.ExceptionUtils; diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java index 8c7d0d8832..c9cb32215e 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java @@ -22,6 +22,7 @@ import org.apache.fluss.fs.FsPath; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.remote.RemoteLogManifest; import org.apache.fluss.remote.RemoteLogSegment; import org.apache.fluss.rpc.gateway.CoordinatorGateway; import org.apache.fluss.rpc.messages.CommitRemoteLogManifestRequest; diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManager.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManager.java index 143ae251a7..f57a6ea81c 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManager.java @@ -25,6 +25,7 @@ import org.apache.fluss.fs.FsPath; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.remote.RemoteLogManifest; import org.apache.fluss.remote.RemoteLogSegment; import org.apache.fluss.rpc.gateway.CoordinatorGateway; import org.apache.fluss.server.log.LogManager; diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogStorage.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogStorage.java index 1a410fcb2c..6e1de16cf3 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogStorage.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogStorage.java @@ -22,6 +22,7 @@ import org.apache.fluss.fs.FsPath; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.remote.RemoteLogManifest; import org.apache.fluss.remote.RemoteLogSegment; import java.io.Closeable; diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogTablet.java index c840a0a028..9f0ae6e949 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogTablet.java @@ -22,6 +22,7 @@ import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metrics.MetricNames; import org.apache.fluss.metrics.groups.MetricGroup; +import org.apache.fluss.remote.RemoteLogManifest; import org.apache.fluss.remote.RemoteLogSegment; import org.apache.fluss.server.metrics.group.BucketMetricGroup; diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/rebalance/RebalanceManagerITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/rebalance/RebalanceManagerITCase.java index a99370908b..425b5463e1 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/rebalance/RebalanceManagerITCase.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/rebalance/RebalanceManagerITCase.java @@ -28,6 +28,7 @@ import org.apache.fluss.metadata.TableBucketReplica; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.remote.RemoteLogManifest; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.messages.AddServerTagRequest; import org.apache.fluss.server.coordinator.CoordinatorEventProcessor; @@ -35,7 +36,6 @@ import org.apache.fluss.server.coordinator.rebalance.model.ClusterModel; import org.apache.fluss.server.coordinator.statemachine.ReplicaState; import org.apache.fluss.server.log.remote.RemoteLogManager; -import org.apache.fluss.server.log.remote.RemoteLogManifest; import org.apache.fluss.server.log.remote.RemoteLogTablet; import org.apache.fluss.server.replica.ReplicaManager; import org.apache.fluss.server.tablet.TabletServer; diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/DefaultRemoteLogStorageTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/DefaultRemoteLogStorageTest.java index a450295a7f..a8ec544510 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/DefaultRemoteLogStorageTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/DefaultRemoteLogStorageTest.java @@ -21,6 +21,7 @@ import org.apache.fluss.fs.FsPath; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.remote.RemoteLogManifest; import org.apache.fluss.remote.RemoteLogSegment; import org.apache.fluss.server.log.LogTablet; import org.apache.fluss.server.log.remote.RemoteLogStorage.IndexType; diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java index 90dfa0914e..872682ad26 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java @@ -27,6 +27,7 @@ import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.remote.RemoteLogManifest; import org.apache.fluss.remote.RemoteLogSegment; import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.gateway.CoordinatorGateway; diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TestingRemoteLogStorage.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TestingRemoteLogStorage.java index a946e9dd0c..eeba26a54c 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TestingRemoteLogStorage.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TestingRemoteLogStorage.java @@ -20,6 +20,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.RemoteStorageException; import org.apache.fluss.fs.FsPath; +import org.apache.fluss.remote.RemoteLogManifest; import org.apache.fluss.remote.RemoteLogSegment; import java.io.IOException; From 937e54a3f535bb20c87840727c9e917eb2bf1be7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Mon, 15 Jun 2026 21:39:11 +0800 Subject: [PATCH 08/19] [flink][test] make OrphanFilesCleanITCase extend AbstractTestBase Reuse Flink minicluster across tests in the same class via Flink's AbstractTestBase. Cuts Flink120OrphanFilesCleanITCase wall time from ~78s to ~66s and prevents the package-private parent from being picked up as a standalone test in fluss-flink-common. --- .../fluss/flink/action/orphan/OrphanFilesCleanITCase.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java index c7ae62dadb..1f9b522e54 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java @@ -43,6 +43,7 @@ import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.FlussPaths; +import org.apache.flink.test.util.AbstractTestBase; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.LogEvent; @@ -76,7 +77,7 @@ import static org.assertj.core.api.Assertions.assertThat; /** End-to-end tests for orphan files cleanup safety scenarios. */ -class OrphanFilesCleanITCase { +abstract class OrphanFilesCleanITCase extends AbstractTestBase { @RegisterExtension static final FlussClusterExtension FLUSS_CLUSTER_EXTENSION = From e6efd17d52d8903bd26d50d07918824a28e33abf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Mon, 15 Jun 2026 22:24:10 +0800 Subject: [PATCH 09/19] [flink] fail-fast on incompatible Fluss server in orphan cleanup The action depends on ListRemoteLogManifests and ListKvSnapshots, both new server RPCs. The action jar may be deployed against an older cluster, so probe both RPCs once at the start of stage 1 and abort with a clear error if the server raises UnsupportedVersionException. Without this guard the job would silently emit skip_log_target / skip_kv_target audit events for every bucket and exit with deleted=0, masking the incompatibility. --- .../orphan/job/ScopeEnumeratorFunction.java | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java index 44c96dc9f6..170325e602 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java @@ -23,6 +23,7 @@ import org.apache.fluss.client.admin.Admin; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.UnsupportedVersionException; import org.apache.fluss.flink.action.orphan.OrphanCleanUtils; import org.apache.fluss.flink.action.orphan.RpcErrorClassifier; import org.apache.fluss.flink.action.orphan.audit.AuditLogger; @@ -105,6 +106,13 @@ public void processElement(Integer trigger, Context ctx, Collector ou try (Connection connection = ConnectionFactory.createConnection(flussConfig); Admin admin = connection.getAdmin()) { + // Fail fast on incompatible servers: the action jar may be deployed against an + // older cluster that does not implement ListRemoteLogManifests / ListKvSnapshots. + // Without this guard, every per-target fetch would degrade to skip_log_target / + // skip_kv_target audit events and the job would exit "successfully" with + // deleted=0, masking the incompatibility. + verifyServerSupportsRequiredApis(admin); + AuditLogger audit = new AuditLogger(); audit.logCutoff(config.olderThanMillis()); @@ -137,6 +145,56 @@ private static List normalizeRoots(List roots) { return new ArrayList(normalized); } + /** + * Probes the two RPCs this action depends on and throws if the connected server does not + * implement them. A sentinel {@code tableId} of {@link Long#MAX_VALUE} is used so that on a + * compatible server the call simply fails with a benign error (typically table-not-found), + * whereas an incompatible server raises {@link UnsupportedVersionException} during ApiVersions + * negotiation. Any non-{@code UnsupportedVersionException} outcome is treated as proof that the + * RPC is recognized. + */ + private static void verifyServerSupportsRequiredApis(Admin admin) { + long sentinelTableId = Long.MAX_VALUE; + probeApi( + "ListRemoteLogManifests", + () -> admin.listRemoteLogManifests(sentinelTableId, null).get()); + probeApi("ListKvSnapshots", () -> admin.listKvSnapshots(sentinelTableId, null).get()); + } + + private static void probeApi(String apiName, ThrowingProbe probe) { + try { + probe.run(); + } catch (Throwable t) { + if (isUnsupportedVersion(t)) { + throw new UnsupportedOperationException( + "Orphan files cleanup requires the Fluss server to support the " + + apiName + + " RPC, which the connected cluster does not. Upgrade the" + + " cluster to a version that exposes this RPC, or run an" + + " older orphan-files-cleanup action that targets this server.", + t); + } + // Any other failure means the RPC is recognized; the call merely failed because of + // the sentinel target id. Compatibility is satisfied. + } + } + + private static boolean isUnsupportedVersion(Throwable t) { + Throwable cause = t; + while (cause != null) { + if (cause instanceof UnsupportedVersionException) { + return true; + } + cause = cause.getCause(); + } + return false; + } + + @FunctionalInterface + private interface ThrowingProbe { + void run() throws Exception; + } + // ------------------------------------------------------------------------- // Scope enumeration (coordinator RPCs only) // ------------------------------------------------------------------------- From 26154d3c718fac4d1ba5d356aa7b16dd39a00195 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Mon, 15 Jun 2026 22:41:02 +0800 Subject: [PATCH 10/19] [flink] emit orphan cleanup summary through AuditLogger Route the final scanned/deleted/failures/bytes-reclaimed counters through the dedicated fluss.orphan.audit logger so the run-level outcome lands in the same sink as per-file action= events and can be queried alongside deletes and skips. The application-side LOG.info line is kept for local debugging. --- .../action/orphan/audit/AuditLogger.java | 26 +++++++++++++++++++ .../orphan/job/StatsAggregateOperator.java | 9 ++++--- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/audit/AuditLogger.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/audit/AuditLogger.java index e073135890..26adf5f00e 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/audit/AuditLogger.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/audit/AuditLogger.java @@ -207,4 +207,30 @@ public void logSkipBucketOutOfScope(long tableId, Long partitionId, String resol resolvedRoot, Instant.now()); } + + /** + * Final summary event emitted once at the end of a run, carrying the headline counters that + * operators query most often ("how many files were removed and how much space was reclaimed"). + * Routed through the dedicated audit logger so the result is queryable from the same sink as + * the per-file {@code action=deleted} / {@code action=skip_*} lines. + */ + public void logSummary( + long scanned, + long deletedFiles, + long emptyDirsRemoved, + long deleteFailures, + long bytesReclaimed, + boolean dryRun) { + AUDIT.info( + "action=summary scanned={} deleted_total={} deleted_files={} empty_dirs_removed={}" + + " delete_failures={} bytes_reclaimed={} dry_run={} ts={}", + scanned, + deletedFiles + emptyDirsRemoved, + deletedFiles, + emptyDirsRemoved, + deleteFailures, + bytesReclaimed, + dryRun, + Instant.now()); + } } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java index 556b80c2fa..f91185ed7d 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java @@ -99,7 +99,8 @@ public void processElement(StreamRecord element) { @Override public void endInput() { - long emptyDirsRemoved = sweepEmptyDirs(touchedDirs); + AuditLogger audit = new AuditLogger(); + long emptyDirsRemoved = sweepEmptyDirs(touchedDirs, audit); long totalDeleted = deleted + emptyDirsRemoved; CleanStats finalStats = @@ -110,6 +111,9 @@ public void endInput() { bytesReclaimed, new ArrayList(0)); + audit.logSummary( + scanned, deleted, emptyDirsRemoved, deleteFailures, bytesReclaimed, dryRun); + LOG.info( "Orphan cleanup complete: scanned={}, deleted={} (files={}, emptyDirs={}), " + "failures={}, bytesReclaimed={}", @@ -123,11 +127,10 @@ public void endInput() { output.collect(new StreamRecord<>(finalStats)); } - private long sweepEmptyDirs(Set dirs) { + private long sweepEmptyDirs(Set dirs, AuditLogger audit) { if (dirs.isEmpty()) { return 0L; } - AuditLogger audit = new AuditLogger(); EmptyDirSweeper sweeper = new EmptyDirSweeper(dryRun, audit, sweepRateLimiter); for (String dir : dirs) { sweeper.registerTouched(new FsPath(dir)); From 6c0a56c3ec2acda38a551c10f5b442659149d6d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Mon, 15 Jun 2026 22:45:11 +0800 Subject: [PATCH 11/19] [flink] drop duplicate LOG.info in StatsAggregateOperator The application-side completion line duplicated the structured summary written by AuditLogger.logSummary. Remove it; the audit logger inherits root appenders so local runs still surface the result. --- .../action/orphan/job/StatsAggregateOperator.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java index f91185ed7d..1f798182c0 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java @@ -114,16 +114,6 @@ public void endInput() { audit.logSummary( scanned, deleted, emptyDirsRemoved, deleteFailures, bytesReclaimed, dryRun); - LOG.info( - "Orphan cleanup complete: scanned={}, deleted={} (files={}, emptyDirs={}), " - + "failures={}, bytesReclaimed={}", - finalStats.scanned(), - totalDeleted, - deleted, - emptyDirsRemoved, - finalStats.deleteFailures(), - finalStats.bytesReclaimed()); - output.collect(new StreamRecord<>(finalStats)); } From c7fd80d6bd55a6f42b1920569889ef5fa6dccec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Tue, 16 Jun 2026 11:19:18 +0800 Subject: [PATCH 12/19] [flink] Clean empty dirs during scan --- .../action/orphan/OrphanFilesCleanAction.java | 7 +- .../action/orphan/job/BucketCleaner.java | 37 +++- .../flink/action/orphan/job/CleanStats.java | 28 +-- .../action/orphan/job/EmptyDirSweeper.java | 162 ------------------ .../orphan/job/OrphanFilesCleanJob.java | 9 +- .../orphan/job/ScanAndCleanFunction.java | 77 +++++---- .../orphan/job/StatsAggregateOperator.java | 71 ++------ .../action/orphan/job/BucketCleanerTest.java | 113 ++++++++++++ .../orphan/job/EmptyDirSweeperTest.java | 91 ---------- 9 files changed, 225 insertions(+), 370 deletions(-) delete mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeper.java create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/BucketCleanerTest.java delete mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeperTest.java diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanAction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanAction.java index 4622f41c0f..716c514cdf 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanAction.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanAction.java @@ -34,7 +34,7 @@ *

    *
  1. ScopeEnumerator (p=1): coordinator RPCs to enumerate scope and emit work items. *
  2. ScanAndClean (p=N): parallel FS scan + rate-limited delete. - *
  3. StatsAggregate (p=1): merge stats + empty-directory sweep. + *
  4. StatsAggregate (p=1): merge stats. *
*/ @Internal @@ -54,11 +54,12 @@ public void run() throws Exception { CleanStats stats = OrphanFilesCleanJob.execute(env, config, config.parallelism().orElse(null)); LOG.info( - "remove_orphan_files done: scope={} scanned={} deleted={} failures={}" - + " bytesReclaimed={} dryRun={}", + "remove_orphan_files done: scope={} scanned={} deletedTotal={}" + + " emptyDirsRemoved={} failures={} bytesReclaimed={} dryRun={}", scopeDescription(), stats.scanned(), stats.deleted(), + stats.emptyDirsRemoved(), stats.deleteFailures(), stats.bytesReclaimed(), config.dryRun()); diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java index d1ea20a6bf..e9f8329772 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java @@ -83,27 +83,39 @@ private void walkAndCleanDir(FsPath root, BucketActiveRefs activeRefs, BucketCle if (!fs.exists(root)) { return; } - Deque stack = new ArrayDeque(); - stack.push(root); + Deque stack = new ArrayDeque(); + stack.push(new DirVisit(root, false, false)); while (!stack.isEmpty()) { - FsPath dir = stack.pop(); + DirVisit visit = stack.pop(); + if (visit.postOrder) { + if (visit.oldEnough && safeDeleter.deleteEmptyDir(visit.dir)) { + stats.deleted++; + stats.emptyDirsRemoved++; + } + continue; + } FileStatus[] children; try { - children = fs.listStatus(dir); + children = fs.listStatus(visit.dir); } catch (IOException e) { - LOG.warn("Failed to list directory: {}", dir, e); + LOG.warn("Failed to list directory: {}", visit.dir, e); continue; } if (children == null) { continue; } + if (!visit.dir.toString().equals(root.toString())) { + stack.push(new DirVisit(visit.dir, true, visit.oldEnough)); + } for (FileStatus child : children) { FsPath childPath = child.getPath(); if (child.isDir()) { if (FlussPaths.REMOTE_KV_SNAPSHOT_SHARED_DIR.equals(childPath.getName())) { continue; } - stack.push(childPath); + stack.push( + new DirVisit( + childPath, false, child.getModificationTime() < cutoffMillis)); continue; } if (childPath.getName().startsWith(".")) { @@ -142,6 +154,7 @@ private void walkAndCleanDir(FsPath root, BucketActiveRefs activeRefs, BucketCle public static final class BucketCleanStats { public long scanned; public long deleted; + public long emptyDirsRemoved; public long deleteFailures; public long bytesReclaimed; @@ -149,4 +162,16 @@ public static BucketCleanStats empty() { return new BucketCleanStats(); } } + + private static final class DirVisit { + private final FsPath dir; + private final boolean postOrder; + private final boolean oldEnough; + + private DirVisit(FsPath dir, boolean postOrder, boolean oldEnough) { + this.dir = dir; + this.postOrder = postOrder; + this.oldEnough = oldEnough; + } + } } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/CleanStats.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/CleanStats.java index 0cb97d5678..cfecb0096e 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/CleanStats.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/CleanStats.java @@ -20,14 +20,10 @@ import org.apache.fluss.annotation.Internal; import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; /** * Per-task cleanup statistics emitted by each {@link ScanAndCleanFunction} subtask. The scalar - * counters are accumulated by {@link StatsAggregateOperator} via simple addition; the short {@code - * touchedDirs} list (typically 1–2 entries per task) is inserted into a {@code HashSet} for O(1) - * deduplication — no list concatenation or O(n²) merge is needed. + * counters are accumulated by {@link StatsAggregateOperator} via simple addition. */ @Internal public final class CleanStats implements Serializable { @@ -36,25 +32,29 @@ public final class CleanStats implements Serializable { private final long scanned; private final long deleted; + private final long emptyDirsRemoved; private final long deleteFailures; private final long bytesReclaimed; - private final List touchedDirs; + + public CleanStats(long scanned, long deleted, long deleteFailures, long bytesReclaimed) { + this(scanned, deleted, 0L, deleteFailures, bytesReclaimed); + } public CleanStats( long scanned, long deleted, + long emptyDirsRemoved, long deleteFailures, - long bytesReclaimed, - List touchedDirs) { + long bytesReclaimed) { this.scanned = scanned; this.deleted = deleted; + this.emptyDirsRemoved = emptyDirsRemoved; this.deleteFailures = deleteFailures; this.bytesReclaimed = bytesReclaimed; - this.touchedDirs = new ArrayList(touchedDirs); } public static CleanStats empty() { - return new CleanStats(0L, 0L, 0L, 0L, new ArrayList(0)); + return new CleanStats(0L, 0L, 0L, 0L); } public long scanned() { @@ -65,6 +65,10 @@ public long deleted() { return deleted; } + public long emptyDirsRemoved() { + return emptyDirsRemoved; + } + public long deleteFailures() { return deleteFailures; } @@ -72,8 +76,4 @@ public long deleteFailures() { public long bytesReclaimed() { return bytesReclaimed; } - - public List touchedDirs() { - return touchedDirs; - } } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeper.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeper.java deleted file mode 100644 index 30c193cf6d..0000000000 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeper.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.flink.action.orphan.job; - -import org.apache.fluss.annotation.Internal; -import org.apache.fluss.flink.action.orphan.audit.AuditLogger; -import org.apache.fluss.flink.action.orphan.fs.SafeDeleter; -import org.apache.fluss.fs.FileStatus; -import org.apache.fluss.fs.FileSystem; -import org.apache.fluss.fs.FsPath; -import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.RateLimiter; - -import java.io.IOException; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Deque; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * End-of-run empty-directory reclaim. Walks every registered "touched" directory tree depth-first - * and asks {@link SafeDeleter} to remove any empty directories encountered, bottom up; non-empty - * directories are no-ops via {@code SafeDeleter}'s contract. - * - *

Run exactly once, after all per-table / per-db cleanup has finished. Any sub-flow that touched - * a tablet dir or descended into an orphan dir is expected to register that root via {@link - * #registerTouched(FsPath)} during its own pass, so this single end-of-run sweep can collect the - * leftover empties without re-walking the live cleanup paths. - * - *

The sweeper deliberately does not own a {@link FileSystem} — it derives one per-root from the - * given {@link FsPath} so different remote stores can coexist. - */ -@Internal -public final class EmptyDirSweeper { - - private final boolean dryRun; - private final AuditLogger audit; - private final RateLimiter rateLimiter; - private final Set touchedRoots = new HashSet(); - - public EmptyDirSweeper(boolean dryRun, AuditLogger audit, RateLimiter rateLimiter) { - this.dryRun = dryRun; - this.audit = audit; - this.rateLimiter = rateLimiter; - } - - /** - * Register a directory root whose subtree should be considered by the final empty-dir sweep. - * Call sites: every cleanup sub-flow that may have removed files under {@code root} (live log / - * KV tablet dirs, orphan partition / orphan table dirs). Multiple registrations of the same - * root are deduplicated; the actual sweep is deferred until {@link #sweep()} runs at end of - * action. - */ - public void registerTouched(FsPath root) { - if (root != null) { - touchedRoots.add(root); - } - } - - /** - * Sweeps every registered subtree, removing empty leaf directories first and propagating up to - * the registered root. - * - * @return the number of empty directories deleted, or that would be deleted in dry-run mode - */ - public long sweep() throws IOException { - long removed = 0L; - for (FsPath root : touchedRoots) { - removed += sweepOne(root); - } - return removed; - } - - private long sweepOne(FsPath root) throws IOException { - FileSystem fs = root.getFileSystem(); - SafeDeleter safeDeleter = new SafeDeleter(fs, dryRun, audit, rateLimiter); - if (!fs.exists(root)) { - return 0L; - } - // First, gather all directories (root and descendants) in pre-order; then process in - // reverse order so deeper directories are visited before their parents. - List dirs = new ArrayList(); - Deque stack = new ArrayDeque(); - stack.push(root); - while (!stack.isEmpty()) { - FsPath dir = stack.pop(); - dirs.add(dir); - FileStatus[] children; - try { - children = fs.listStatus(dir); - } catch (IOException ignored) { - continue; - } - if (children == null) { - continue; - } - for (FileStatus child : children) { - if (child.isDir()) { - stack.push(child.getPath()); - } - } - } - long deleted = 0L; - if (dryRun) { - Set virtuallyDeletedDirs = new HashSet(); - for (int i = dirs.size() - 1; i >= 0; i--) { - FsPath dir = dirs.get(i); - if (!fs.exists(dir)) { - continue; - } - if (!isEffectivelyEmpty(fs, dir, virtuallyDeletedDirs)) { - continue; - } - audit.logWouldDeleteDir(dir); - virtuallyDeletedDirs.add(dir.toString()); - deleted++; - } - } else { - for (int i = dirs.size() - 1; i >= 0; i--) { - if (safeDeleter.deleteEmptyDir(dirs.get(i))) { - deleted++; - } - } - } - return deleted; - } - - private boolean isEffectivelyEmpty( - FileSystem fs, FsPath dir, Set virtuallyDeletedDirs) { - FileStatus[] remaining; - try { - remaining = fs.listStatus(dir); - } catch (IOException ignored) { - return false; - } - if (remaining == null) { - return false; - } - for (FileStatus child : remaining) { - if (!child.isDir() || !virtuallyDeletedDirs.contains(child.getPath().toString())) { - return false; - } - } - return true; - } -} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java index 1375452f74..2cc0b5a5bf 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java @@ -37,7 +37,7 @@ * 

  * Stage 1: ScopeEnumerator (p=1)   — coordinator RPCs, emits CleanTask
  * Stage 2: ScanAndClean (p=N)      — FS scan + rate-limited delete, emits CleanStats
- * Stage 3: StatsAggregate (p=1)    — merge stats + empty-dir sweep, emits final CleanStats
+ * Stage 3: StatsAggregate (p=1)    — merge stats, emits final CleanStats
  *
*/ @Internal @@ -82,15 +82,12 @@ public static CleanStats execute( stats = stats.setParallelism(parallelism); } - // Stage 3: StatsAggregate + EmptyDirSweep (parallelism=1) + // Stage 3: StatsAggregate (parallelism=1) SingleOutputStreamOperator result = stats.transform( "StatsAggregate", TypeInformation.of(new TypeHint() {}), - new StatsAggregateOperator( - config.dryRun(), - config.extraConfigs(), - config.deleteRateLimitPerSecond())) + new StatsAggregateOperator(config.dryRun())) .setParallelism(1) .setMaxParallelism(1); diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java index 15e244a763..6058299eb4 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java @@ -38,10 +38,7 @@ import java.io.IOException; import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Deque; -import java.util.List; import java.util.Map; /** @@ -52,15 +49,15 @@ * *
    *
  • {@link BucketCleanTask}: second-reads manifests from object storage to build the active - * reference set, then walks log/kv directories and deletes orphan files. + * reference set, then walks log/kv directories and deletes orphan files and old empty child + * directories. *
  • {@link OrphanDirCleanTask}: recursively walks the orphan directory and deletes all files - * older than the cutoff. + * older than the cutoff, then removes old empty directories bottom-up. *
* - *

Each task emits a single {@link CleanStats} containing scalar counters and the short list of - * directories walked. Delete rate is limited per-subtask: {@code configuredRate / - * runtimeParallelism}. The serial processing within each subtask guarantees no concurrent throttler - * access. + *

Each task emits a single {@link CleanStats} containing scalar counters. Delete rate is limited + * per-subtask: {@code configuredRate / runtimeParallelism}. The serial processing within each + * subtask guarantees no concurrent throttler access. */ @Internal public final class ScanAndCleanFunction extends ProcessFunction { @@ -134,20 +131,12 @@ private CleanStats processBucketTask(BucketCleanTask task) throws IOException { BucketCleaner.BucketCleanStats bucketStats = cleaner.clean(activeRefs, logDir, kvDir); - List touchedDirs = new ArrayList(2); - if (logDir != null) { - touchedDirs.add(logDir.toString()); - } - if (kvDir != null) { - touchedDirs.add(kvDir.toString()); - } - return new CleanStats( bucketStats.scanned, bucketStats.deleted, + bucketStats.emptyDirsRemoved, bucketStats.deleteFailures, - bucketStats.bytesReclaimed, - touchedDirs); + bucketStats.bytesReclaimed); } // ------------------------------------------------------------------------- @@ -166,27 +155,46 @@ private CleanStats processOrphanDirTask(OrphanDirCleanTask task) throws IOExcept long scanned = 0L; long deleted = 0L; + long emptyDirsRemoved = 0L; long deleteFailures = 0L; long bytesReclaimed = 0L; - Deque stack = new ArrayDeque(); - stack.push(dirPath); + FileStatus rootStatus = fs.getFileStatus(dirPath); + Deque stack = new ArrayDeque(); + stack.push( + new DirVisit( + dirPath, + false, + rootStatus.isDir() + && rootStatus.getModificationTime() < task.cutoffMillis())); while (!stack.isEmpty()) { - FsPath dir = stack.pop(); + DirVisit visit = stack.pop(); + if (visit.postOrder) { + if (visit.oldEnough && safeDeleter.deleteEmptyDir(visit.dir)) { + deleted++; + emptyDirsRemoved++; + } + continue; + } FileStatus[] children; try { - children = fs.listStatus(dir); + children = fs.listStatus(visit.dir); } catch (IOException e) { - LOG.warn("Failed to list directory: {}", dir, e); + LOG.warn("Failed to list directory: {}", visit.dir, e); continue; } if (children == null) { continue; } + stack.push(new DirVisit(visit.dir, true, visit.oldEnough)); for (FileStatus child : children) { FsPath childPath = child.getPath(); if (child.isDir()) { - stack.push(childPath); + stack.push( + new DirVisit( + childPath, + false, + child.getModificationTime() < task.cutoffMillis())); continue; } if (childPath.getName().startsWith(".")) { @@ -221,12 +229,7 @@ private CleanStats processOrphanDirTask(OrphanDirCleanTask task) throws IOExcept } } - return new CleanStats( - scanned, - deleted, - deleteFailures, - bytesReclaimed, - Arrays.asList(dirPath.toString())); + return new CleanStats(scanned, deleted, emptyDirsRemoved, deleteFailures, bytesReclaimed); } // ------------------------------------------------------------------------- @@ -236,4 +239,16 @@ private CleanStats processOrphanDirTask(OrphanDirCleanTask task) throws IOExcept private SafeDeleter createSafeDeleter(FileSystem fs, boolean dryRun) { return new SafeDeleter(fs, dryRun, audit, rateLimiter); } + + private static final class DirVisit { + private final FsPath dir; + private final boolean postOrder; + private final boolean oldEnough; + + private DirVisit(FsPath dir, boolean postOrder, boolean oldEnough) { + this.dir = dir; + this.postOrder = postOrder; + this.oldEnough = oldEnough; + } + } } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java index 1f798182c0..2e1686ddbf 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/StatsAggregateOperator.java @@ -18,73 +18,49 @@ package org.apache.fluss.flink.action.orphan.job; import org.apache.fluss.annotation.Internal; -import org.apache.fluss.config.Configuration; import org.apache.fluss.flink.action.orphan.audit.AuditLogger; -import org.apache.fluss.fs.FileSystem; -import org.apache.fluss.fs.FsPath; -import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.RateLimiter; import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.BoundedOneInput; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; /** * Stage 3 of the orphan files cleanup job. Runs at parallelism=1 to aggregate per-subtask {@link - * CleanStats} records and perform the final empty-directory sweep. + * CleanStats} records. * *

Implemented as a custom operator (not ProcessFunction) because {@code ProcessOperator} does * not implement {@link BoundedOneInput} — the {@code endInput()} callback would never fire. * - *

Scalar counters are accumulated into four longs; directory paths from each incoming {@link - * CleanStats#touchedDirs()} are inserted into a {@link HashSet} for O(1) deduplication. The final - * empty-dir sweep happens in {@link #endInput()}. + *

Scalar counters are accumulated into longs and the final summary is emitted in {@link + * #endInput()}. */ @Internal public final class StatsAggregateOperator extends AbstractStreamOperator implements OneInputStreamOperator, BoundedOneInput { private static final long serialVersionUID = 2L; - private static final Logger LOG = LoggerFactory.getLogger(StatsAggregateOperator.class); private final boolean dryRun; - private final Map extraConfigs; - private final long deleteRateLimitPerSecond; private transient long scanned; private transient long deleted; + private transient long emptyDirsRemoved; private transient long deleteFailures; private transient long bytesReclaimed; - private transient Set touchedDirs; - private transient RateLimiter sweepRateLimiter; - public StatsAggregateOperator( - boolean dryRun, Map extraConfigs, long deleteRateLimitPerSecond) { + public StatsAggregateOperator(boolean dryRun) { this.dryRun = dryRun; - this.extraConfigs = extraConfigs; - this.deleteRateLimitPerSecond = deleteRateLimitPerSecond; } @Override public void open() throws Exception { super.open(); - if (!extraConfigs.isEmpty()) { - FileSystem.initialize(Configuration.fromMap(extraConfigs), null); - } scanned = 0L; deleted = 0L; + emptyDirsRemoved = 0L; deleteFailures = 0L; bytesReclaimed = 0L; - touchedDirs = new HashSet(); - sweepRateLimiter = RateLimiter.create((double) deleteRateLimitPerSecond); } @Override @@ -92,44 +68,25 @@ public void processElement(StreamRecord element) { CleanStats stats = element.getValue(); scanned += stats.scanned(); deleted += stats.deleted(); + emptyDirsRemoved += stats.emptyDirsRemoved(); deleteFailures += stats.deleteFailures(); bytesReclaimed += stats.bytesReclaimed(); - touchedDirs.addAll(stats.touchedDirs()); } @Override public void endInput() { AuditLogger audit = new AuditLogger(); - long emptyDirsRemoved = sweepEmptyDirs(touchedDirs, audit); - long totalDeleted = deleted + emptyDirsRemoved; - CleanStats finalStats = - new CleanStats( - scanned, - totalDeleted, - deleteFailures, - bytesReclaimed, - new ArrayList(0)); + new CleanStats(scanned, deleted, emptyDirsRemoved, deleteFailures, bytesReclaimed); audit.logSummary( - scanned, deleted, emptyDirsRemoved, deleteFailures, bytesReclaimed, dryRun); + scanned, + deleted - emptyDirsRemoved, + emptyDirsRemoved, + deleteFailures, + bytesReclaimed, + dryRun); output.collect(new StreamRecord<>(finalStats)); } - - private long sweepEmptyDirs(Set dirs, AuditLogger audit) { - if (dirs.isEmpty()) { - return 0L; - } - EmptyDirSweeper sweeper = new EmptyDirSweeper(dryRun, audit, sweepRateLimiter); - for (String dir : dirs) { - sweeper.registerTouched(new FsPath(dir)); - } - try { - return sweeper.sweep(); - } catch (IOException e) { - LOG.warn("Empty directory sweep encountered errors", e); - return 0L; - } - } } diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/BucketCleanerTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/BucketCleanerTest.java new file mode 100644 index 0000000000..c0eccbd601 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/BucketCleanerTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.action.orphan.job; + +import org.apache.fluss.flink.action.orphan.audit.AuditLogger; +import org.apache.fluss.flink.action.orphan.fs.SafeDeleter; +import org.apache.fluss.flink.action.orphan.rule.BucketActiveRefs; +import org.apache.fluss.flink.action.orphan.rule.RuleDispatcher; +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.RateLimiter; +import org.apache.fluss.utils.FlussPaths; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; + +class BucketCleanerTest { + + @Test + void removesOldEmptySegmentDirAfterDeletingExpiredFiles(@TempDir Path tmp) throws IOException { + Path bucketRoot = Files.createDirectories(tmp.resolve("bucket")); + Path segmentDir = + Files.createDirectories(bucketRoot.resolve("11111111-1111-1111-1111-111111111111")); + Path logFile = + Files.write( + segmentDir.resolve( + FlussPaths.filenamePrefixFromOffset(0L) + + FlussPaths.LOG_FILE_SUFFIX), + new byte[] {0x42}); + long cutoff = System.currentTimeMillis() - 1000L; + makeOld(logFile, cutoff - 1000L); + makeOld(segmentDir, cutoff - 1000L); + makeOld(bucketRoot, cutoff - 1000L); + + BucketCleaner cleaner = + new BucketCleaner( + new RuleDispatcher(), + new SafeDeleter( + new FsPath(bucketRoot.toString()).getFileSystem(), + false, + new AuditLogger(), + RateLimiter.create(1000.0)), + new AuditLogger(), + cutoff); + + BucketCleaner.BucketCleanStats stats = + cleaner.clean(BucketActiveRefs.empty(), new FsPath(bucketRoot.toString())); + + assertThat(stats.scanned).isEqualTo(1L); + assertThat(stats.deleted).isEqualTo(2L); + assertThat(stats.emptyDirsRemoved).isEqualTo(1L); + assertThat(Files.exists(logFile)).isFalse(); + assertThat(Files.exists(segmentDir)).isFalse(); + assertThat(Files.exists(bucketRoot)).isTrue(); + } + + @Test + void keepsFreshEmptySegmentDir(@TempDir Path tmp) throws IOException { + Path bucketRoot = Files.createDirectories(tmp.resolve("bucket")); + Path segmentDir = + Files.createDirectories(bucketRoot.resolve("11111111-1111-1111-1111-111111111111")); + long cutoff = System.currentTimeMillis() - 1000L; + + BucketCleaner cleaner = + new BucketCleaner( + new RuleDispatcher(), + new SafeDeleter( + new FsPath(bucketRoot.toString()).getFileSystem(), + false, + new AuditLogger(), + RateLimiter.create(1000.0)), + new AuditLogger(), + cutoff); + + BucketCleaner.BucketCleanStats stats = + cleaner.clean( + new BucketActiveRefs( + Collections.emptySet(), + Collections.emptySet(), + Collections.emptySet()), + new FsPath(bucketRoot.toString())); + + assertThat(stats.deleted).isEqualTo(0L); + assertThat(stats.emptyDirsRemoved).isEqualTo(0L); + assertThat(Files.exists(segmentDir)).isTrue(); + } + + private static void makeOld(Path path, long timestampMillis) throws IOException { + Files.setLastModifiedTime(path, FileTime.fromMillis(timestampMillis)); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeperTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeperTest.java deleted file mode 100644 index fdfd60acb7..0000000000 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/EmptyDirSweeperTest.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.flink.action.orphan.job; - -import org.apache.fluss.flink.action.orphan.audit.AuditLogger; -import org.apache.fluss.fs.FsPath; -import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.RateLimiter; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; - -import static org.assertj.core.api.Assertions.assertThat; - -class EmptyDirSweeperTest { - - private static EmptyDirSweeper newSweeper(boolean dryRun) { - return new EmptyDirSweeper(dryRun, new AuditLogger(), RateLimiter.create(1000.0)); - } - - @Test - void deletesEmptyDirsBottomUp(@TempDir Path tmp) throws IOException { - Path a = Files.createDirectories(tmp.resolve("a")); - Path b = Files.createDirectories(a.resolve("b")); - Path c = Files.createDirectories(b.resolve("c")); - - EmptyDirSweeper sweeper = newSweeper(false); - sweeper.registerTouched(new FsPath(a.toString())); - long removed = sweeper.sweep(); - - assertThat(removed).isEqualTo(3L); - assertThat(Files.exists(c)).isFalse(); - assertThat(Files.exists(b)).isFalse(); - assertThat(Files.exists(a)).isFalse(); - } - - @Test - void leavesNonEmptyDirsAlone(@TempDir Path tmp) throws IOException { - Path a = Files.createDirectories(tmp.resolve("a")); - Path b = Files.createDirectories(a.resolve("b")); - Files.write(b.resolve("keep.txt"), new byte[] {0x42}); - - EmptyDirSweeper sweeper = newSweeper(false); - sweeper.registerTouched(new FsPath(a.toString())); - long removed = sweeper.sweep(); - - assertThat(removed).isEqualTo(0L); - assertThat(Files.exists(b)).isTrue(); - assertThat(Files.exists(a)).isTrue(); - } - - @Test - void dryRunCountsWouldDeleteButDoesNotActuallyDelete(@TempDir Path tmp) throws IOException { - Path a = Files.createDirectories(tmp.resolve("a")); - Path b = Files.createDirectories(a.resolve("b")); - - EmptyDirSweeper sweeper = newSweeper(true /* dryRun */); - sweeper.registerTouched(new FsPath(a.toString())); - long removed = sweeper.sweep(); - - // dry-run leaves both directories on disk, but reports the would-delete count. - assertThat(removed).isEqualTo(2L); - assertThat(Files.exists(b)).isTrue(); - assertThat(Files.exists(a)).isTrue(); - } - - @Test - void nonExistentRootIsNoOp(@TempDir Path tmp) throws IOException { - EmptyDirSweeper sweeper = newSweeper(false); - sweeper.registerTouched(new FsPath(tmp.resolve("does-not-exist").toString())); - assertThat(sweeper.sweep()).isEqualTo(0L); - } -} From e1a1b31eb8cec0f0ce02e7f6a9f15b4e3b47580f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Tue, 16 Jun 2026 14:39:59 +0800 Subject: [PATCH 13/19] [flink] Audit dot orphan files --- .../action/orphan/job/BucketCleaner.java | 3 -- .../orphan/job/ScanAndCleanFunction.java | 3 -- .../action/orphan/job/BucketCleanerTest.java | 32 +++++++++++++++++++ 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java index e9f8329772..2c047e1e4e 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java @@ -118,9 +118,6 @@ private void walkAndCleanDir(FsPath root, BucketActiveRefs activeRefs, BucketCle childPath, false, child.getModificationTime() < cutoffMillis)); continue; } - if (childPath.getName().startsWith(".")) { - continue; - } FileMeta meta = new FileMeta(childPath, child.getLen(), child.getModificationTime()); FileRule rule = dispatcher.dispatch(meta); diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java index 6058299eb4..33edd34d8c 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java @@ -197,9 +197,6 @@ private CleanStats processOrphanDirTask(OrphanDirCleanTask task) throws IOExcept child.getModificationTime() < task.cutoffMillis())); continue; } - if (childPath.getName().startsWith(".")) { - continue; - } scanned++; if (child.getModificationTime() >= task.cutoffMillis()) { continue; diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/BucketCleanerTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/BucketCleanerTest.java index c0eccbd601..d41f849c0f 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/BucketCleanerTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/BucketCleanerTest.java @@ -107,6 +107,38 @@ void keepsFreshEmptySegmentDir(@TempDir Path tmp) throws IOException { assertThat(Files.exists(segmentDir)).isTrue(); } + @Test + void scansButDoesNotDeleteUnknownDotFiles(@TempDir Path tmp) throws IOException { + Path bucketRoot = Files.createDirectories(tmp.resolve("bucket")); + Path segmentDir = + Files.createDirectories(bucketRoot.resolve("11111111-1111-1111-1111-111111111111")); + Path dotFile = Files.write(segmentDir.resolve(".unknown"), new byte[] {0x42}); + long cutoff = System.currentTimeMillis() - 1000L; + makeOld(dotFile, cutoff - 1000L); + makeOld(segmentDir, cutoff - 1000L); + makeOld(bucketRoot, cutoff - 1000L); + + BucketCleaner cleaner = + new BucketCleaner( + new RuleDispatcher(), + new SafeDeleter( + new FsPath(bucketRoot.toString()).getFileSystem(), + false, + new AuditLogger(), + RateLimiter.create(1000.0)), + new AuditLogger(), + cutoff); + + BucketCleaner.BucketCleanStats stats = + cleaner.clean(BucketActiveRefs.empty(), new FsPath(bucketRoot.toString())); + + assertThat(stats.scanned).isEqualTo(1L); + assertThat(stats.deleted).isEqualTo(0L); + assertThat(stats.emptyDirsRemoved).isEqualTo(0L); + assertThat(Files.exists(dotFile)).isTrue(); + assertThat(Files.exists(segmentDir)).isTrue(); + } + private static void makeOld(Path path, long timestampMillis) throws IOException { Files.setLastModifiedTime(path, FileTime.fromMillis(timestampMillis)); } From 26bb17889e5b0318b132caded40900aadb4ee7a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Tue, 16 Jun 2026 15:55:11 +0800 Subject: [PATCH 14/19] [flink] Throttle remote fs cleanup ops --- .../flink/action/orphan/OrphanCleanUtils.java | 27 ----------- .../orphan/OrphanFilesCleanActionFactory.java | 3 +- .../orphan/build/ActiveRefsFetcher.java | 27 ++++++++--- .../orphan/config/OrphanCleanConfig.java | 36 +++++++++----- .../flink/action/orphan/fs/SafeDeleter.java | 16 +++---- .../action/orphan/job/BucketCleaner.java | 8 +++- .../orphan/job/OrphanFilesCleanJob.java | 3 +- .../orphan/job/ScanAndCleanFunction.java | 45 ++++++++++------- .../orphan/job/ScopeEnumeratorFunction.java | 48 +++++++++++++++---- .../orphan/config/OrphanCleanConfigTest.java | 34 ++++++++++++- .../action/orphan/fs/SafeDeleterTest.java | 15 ++++-- .../action/orphan/job/BucketCleanerTest.java | 47 +++++++----------- 12 files changed, 190 insertions(+), 119 deletions(-) diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanCleanUtils.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanCleanUtils.java index 1a1a1475f0..24381ab752 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanCleanUtils.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanCleanUtils.java @@ -22,8 +22,6 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; import org.apache.fluss.config.cluster.ConfigEntry; -import org.apache.fluss.fs.FileStatus; -import org.apache.fluss.fs.FileSystem; import org.apache.fluss.fs.FsPath; import org.apache.fluss.metadata.PartitionInfo; import org.apache.fluss.metadata.PhysicalTablePath; @@ -33,7 +31,6 @@ import javax.annotation.Nullable; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -183,30 +180,6 @@ public static String normalizeRoot(String remoteDataDir) { : remoteDataDir; } - /** - * Lists the entries of a directory, returning {@code null} on {@link IOException} (directory - * does not exist or is inaccessible). - */ - @Nullable - public static FileStatus[] listStatuses(FileSystem fs, FsPath dir) { - try { - return fs.listStatus(dir); - } catch (IOException e) { - return null; - } - } - - /** - * Returns the {@link FileSystem} for a path if the path exists, or {@code null} otherwise. - * - * @throws IOException if resolving the filesystem itself fails - */ - @Nullable - public static FileSystem getFileSystemIfExists(FsPath dir) throws IOException { - FileSystem fs = dir.getFileSystem(); - return fs.exists(dir) ? fs : null; - } - /** Formats a bucket-scope key for audit/logging purposes. */ public static String bucketScopeKey(long tableId, Long partitionId, int bucketId) { return tableId + ":" + partitionId + ":" + bucketId; diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanActionFactory.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanActionFactory.java index e63dfcede1..ef6dc7bdc6 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanActionFactory.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanActionFactory.java @@ -45,7 +45,8 @@ public String help() { return "Usage: remove_orphan_files --bootstrap-server \n" + " (--database [--table ] | --all-databases)\n" + " [--older-than '']\n" - + " [--delete-rate-limit-per-second 100] [--dry-run]\n" + + " [--remote-fs-op-rate-limit-per-second 100]\n" + + " [--dry-run]\n" + " [--allow-delete-manifest]\n" + " [--allow-clean-orphan-tables]\n" + " [--allow-clean-orphan-partitions]\n" diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcher.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcher.java index 26b01d7a62..223c6b97c4 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcher.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/build/ActiveRefsFetcher.java @@ -28,6 +28,7 @@ import org.apache.fluss.fs.FsPath; import org.apache.fluss.remote.RemoteLogManifest; import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.RateLimiter; import org.apache.fluss.utils.FlussPaths; import org.apache.fluss.utils.IOUtils; import org.apache.fluss.utils.RetryUtils; @@ -91,13 +92,15 @@ public byte[] read(FsPath path) throws IOException { private final MetadataReader metadataReader; private final int maxRetries; private final long backoffMillis; + private final RateLimiter remoteFsOpRateLimiter; - public ActiveRefsFetcher(Admin admin, int maxRetries) { - this(wrap(admin), DEFAULT_METADATA_READER, maxRetries, DEFAULT_BACKOFF_MILLIS); - } - - public ActiveRefsFetcher(Admin admin, MetadataReader metadataReader, int maxRetries) { - this(wrap(admin), metadataReader, maxRetries, DEFAULT_BACKOFF_MILLIS); + public ActiveRefsFetcher(Admin admin, int maxRetries, RateLimiter remoteFsOpRateLimiter) { + this( + wrap(admin), + DEFAULT_METADATA_READER, + maxRetries, + DEFAULT_BACKOFF_MILLIS, + remoteFsOpRateLimiter); } /** Test constructor: defaults backoff to 0 so unit tests don't pay retry sleep. */ @@ -109,12 +112,23 @@ public ActiveRefsFetcher(Admin admin, MetadataReader metadataReader, int maxRetr @VisibleForTesting ActiveRefsFetcher( AdminFacade admin, MetadataReader metadataReader, int maxRetries, long backoffMillis) { + this(admin, metadataReader, maxRetries, backoffMillis, RateLimiter.create(1000.0)); + } + + @VisibleForTesting + ActiveRefsFetcher( + AdminFacade admin, + MetadataReader metadataReader, + int maxRetries, + long backoffMillis, + RateLimiter remoteFsOpRateLimiter) { checkArgument(maxRetries >= 1, "maxRetries must be >= 1, got %s", maxRetries); checkArgument(backoffMillis >= 0L, "backoffMillis must be >= 0, got %s", backoffMillis); this.admin = admin; this.metadataReader = metadataReader; this.maxRetries = maxRetries; this.backoffMillis = backoffMillis; + this.remoteFsOpRateLimiter = remoteFsOpRateLimiter; } private static AdminFacade wrap(Admin admin) { @@ -273,6 +287,7 @@ private BucketActiveRefs buildBucketActiveRefs(List entri for (RemoteLogManifestInfo entry : entries) { String path = entry.getRemoteLogManifestPath(); manifestPaths.add(path); + remoteFsOpRateLimiter.acquire(); byte[] manifestBytes = metadataReader.read(new FsPath(path)); segmentRelpaths.addAll(parseLogSegmentRelativePaths(manifestBytes)); } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfig.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfig.java index 6676c04e52..839ca7ccc1 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfig.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfig.java @@ -51,7 +51,7 @@ public final class OrphanCleanConfig implements Serializable { /** Default file-level cutoff: files written before {@code now - 3d} are deletion-eligible. */ private static final Duration DEFAULT_OLDER_THAN = Duration.ofDays(3); - private static final long DEFAULT_DELETE_RATE_LIMIT_PER_SECOND = 100L; + private static final long DEFAULT_REMOTE_FS_OP_RATE_LIMIT_PER_SECOND = 100L; private final String bootstrapServer; private final boolean allDatabases; @@ -59,7 +59,7 @@ public final class OrphanCleanConfig implements Serializable { private final @Nullable String table; private final long olderThanMillis; private final boolean dryRun; - private final long deleteRateLimitPerSecond; + private final long remoteFsOpRateLimitPerSecond; private final @Nullable Integer parallelism; private final boolean allowDeleteManifest; private final boolean allowCleanOrphanTables; @@ -73,7 +73,7 @@ private OrphanCleanConfig( @Nullable String table, long olderThanMillis, boolean dryRun, - long deleteRateLimitPerSecond, + long remoteFsOpRateLimitPerSecond, @Nullable Integer parallelism, boolean allowDeleteManifest, boolean allowCleanOrphanTables, @@ -85,7 +85,7 @@ private OrphanCleanConfig( this.table = table; this.olderThanMillis = olderThanMillis; this.dryRun = dryRun; - this.deleteRateLimitPerSecond = deleteRateLimitPerSecond; + this.remoteFsOpRateLimitPerSecond = remoteFsOpRateLimitPerSecond; this.parallelism = parallelism; this.allowDeleteManifest = allowDeleteManifest; this.allowCleanOrphanTables = allowCleanOrphanTables; @@ -118,8 +118,11 @@ public static OrphanCleanConfig fromParams(MultipleParameterToolAdapter params) long now = System.currentTimeMillis(); long olderThanMillis = parseCutoff("--older-than", params.get("older-than"), now, DEFAULT_OLDER_THAN); - long deleteRateLimitPerSecond = - parseDeleteRateLimit(params.get("delete-rate-limit-per-second")); + long remoteFsOpRateLimitPerSecond = + parsePositiveRateLimit( + "--remote-fs-op-rate-limit-per-second", + params.get("remote-fs-op-rate-limit-per-second"), + DEFAULT_REMOTE_FS_OP_RATE_LIMIT_PER_SECOND); Integer parallelism = parseParallelism(params.get("parallelism")); boolean allowDeleteManifest = params.has("allow-delete-manifest"); boolean allowCleanOrphanTables = params.has("allow-clean-orphan-tables"); @@ -132,7 +135,7 @@ public static OrphanCleanConfig fromParams(MultipleParameterToolAdapter params) params.get("table"), olderThanMillis, params.has("dry-run"), - deleteRateLimitPerSecond, + remoteFsOpRateLimitPerSecond, parallelism, allowDeleteManifest, allowCleanOrphanTables, @@ -177,13 +180,14 @@ private static long parseCutoff( return parsedMillis; } - private static long parseDeleteRateLimit(@Nullable String value) { + private static long parsePositiveRateLimit( + String flag, @Nullable String value, long defaultValue) { if (StringUtils.isNullOrWhitespaceOnly(value)) { - return DEFAULT_DELETE_RATE_LIMIT_PER_SECOND; + return defaultValue; } long rate = Long.parseLong(value); if (rate <= 0) { - throw new IllegalArgumentException("--delete-rate-limit-per-second must be positive"); + throw new IllegalArgumentException(flag + " must be positive"); } return rate; } @@ -251,9 +255,15 @@ public boolean dryRun() { return dryRun; } - /** Returns the maximum number of actual delete calls per second. */ - public long deleteRateLimitPerSecond() { - return deleteRateLimitPerSecond; + /** + * Returns the best-effort job-level target rate for remote filesystem operations per second. + * + *

The budget is shared by remote filesystem metadata reads, manifest reads, and deletes. + * Scan subtasks split this value by operator parallelism because Flink does not provide a + * cross-JVM limiter for this action. + */ + public long remoteFsOpRateLimitPerSecond() { + return remoteFsOpRateLimitPerSecond; } /** Returns the optional parallelism for the ScanAndClean stage. */ diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleter.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleter.java index b83bdea913..9b52fa43d8 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleter.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleter.java @@ -55,17 +55,14 @@ public final class SafeDeleter { private final FileSystem fs; private final boolean dryRun; private final AuditLogger audit; - private final RateLimiter rateLimiter; + private final RateLimiter remoteFsOpRateLimiter; - public SafeDeleter(FileSystem fs, boolean dryRun, AuditLogger audit) { - this(fs, dryRun, audit, RateLimiter.create(100.0)); - } - - public SafeDeleter(FileSystem fs, boolean dryRun, AuditLogger audit, RateLimiter rateLimiter) { + public SafeDeleter( + FileSystem fs, boolean dryRun, AuditLogger audit, RateLimiter remoteFsOpRateLimiter) { this.fs = fs; this.dryRun = dryRun; this.audit = audit; - this.rateLimiter = rateLimiter; + this.remoteFsOpRateLimiter = remoteFsOpRateLimiter; } /** @@ -85,7 +82,7 @@ public boolean deleteFile(FsPath file, Decision decision, RuleId ruleId) { audit.logWouldDelete(file, ruleId); return true; } - rateLimiter.acquire(); + remoteFsOpRateLimiter.acquire(); try { boolean ok = fs.delete(file, false); audit.logDeleted(file, ruleId, ok); @@ -114,7 +111,7 @@ public boolean deleteEmptyDir(FsPath dir) { audit.logWouldDeleteDir(dir); return true; } - rateLimiter.acquire(); + remoteFsOpRateLimiter.acquire(); try { boolean ok = fs.delete(dir, false); if (ok) { @@ -129,6 +126,7 @@ public boolean deleteEmptyDir(FsPath dir) { private FileStatus[] listChildrenSilently(FsPath dir) { try { + remoteFsOpRateLimiter.acquire(); return fs.listStatus(dir); } catch (IOException ignored) { return null; diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java index 2c047e1e4e..a1e13cf424 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/BucketCleaner.java @@ -28,6 +28,7 @@ import org.apache.fluss.fs.FileStatus; import org.apache.fluss.fs.FileSystem; import org.apache.fluss.fs.FsPath; +import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.RateLimiter; import org.apache.fluss.utils.FlussPaths; import org.slf4j.Logger; @@ -53,16 +54,19 @@ public final class BucketCleaner { private final SafeDeleter safeDeleter; private final AuditLogger audit; private final long cutoffMillis; + private final RateLimiter remoteFsOpRateLimiter; public BucketCleaner( RuleDispatcher dispatcher, SafeDeleter safeDeleter, AuditLogger audit, - long cutoffMillis) { + long cutoffMillis, + RateLimiter remoteFsOpRateLimiter) { this.dispatcher = dispatcher; this.safeDeleter = safeDeleter; this.audit = audit; this.cutoffMillis = cutoffMillis; + this.remoteFsOpRateLimiter = remoteFsOpRateLimiter; } /** Cleans one bucket's log/kv subtrees using the caller-supplied active reference set. */ @@ -80,6 +84,7 @@ public BucketCleanStats clean(BucketActiveRefs activeRefs, FsPath... bucketDirs) private void walkAndCleanDir(FsPath root, BucketActiveRefs activeRefs, BucketCleanStats stats) throws IOException { FileSystem fs = root.getFileSystem(); + remoteFsOpRateLimiter.acquire(); if (!fs.exists(root)) { return; } @@ -96,6 +101,7 @@ private void walkAndCleanDir(FsPath root, BucketActiveRefs activeRefs, BucketCle } FileStatus[] children; try { + remoteFsOpRateLimiter.acquire(); children = fs.listStatus(visit.dir); } catch (IOException e) { LOG.warn("Failed to list directory: {}", visit.dir, e); diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java index 2cc0b5a5bf..3008715c14 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/OrphanFilesCleanJob.java @@ -75,7 +75,8 @@ public static CleanStats execute( tasks.rebalance() .process( new ScanAndCleanFunction( - config.deleteRateLimitPerSecond(), config.extraConfigs())) + config.remoteFsOpRateLimitPerSecond(), + config.extraConfigs())) .returns(TypeInformation.of(new TypeHint() {})) .name("ScanAndClean"); if (parallelism != null) { diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java index 33edd34d8c..85a64d349b 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScanAndCleanFunction.java @@ -55,9 +55,9 @@ * older than the cutoff, then removes old empty directories bottom-up. * * - *

Each task emits a single {@link CleanStats} containing scalar counters. Delete rate is limited - * per-subtask: {@code configuredRate / runtimeParallelism}. The serial processing within each - * subtask guarantees no concurrent throttler access. + *

Each task emits a single {@link CleanStats} containing scalar counters. Remote filesystem + * operation rate is limited per-subtask: {@code configuredRate / runtimeParallelism}. The serial + * processing within each subtask guarantees no concurrent throttler access. */ @Internal public final class ScanAndCleanFunction extends ProcessFunction { @@ -65,14 +65,15 @@ public final class ScanAndCleanFunction extends ProcessFunction extraConfigs; private transient AuditLogger audit; - private transient RateLimiter rateLimiter; + private transient RateLimiter remoteFsOpRateLimiter; - public ScanAndCleanFunction(long deleteRateLimitPerSecond, Map extraConfigs) { - this.deleteRateLimitPerSecond = deleteRateLimitPerSecond; + public ScanAndCleanFunction( + long remoteFsOpRateLimitPerSecond, Map extraConfigs) { + this.remoteFsOpRateLimitPerSecond = remoteFsOpRateLimitPerSecond; this.extraConfigs = extraConfigs; } @@ -86,14 +87,13 @@ public void open(org.apache.flink.api.common.functions.OpenContext openContext) audit = new AuditLogger(); int parallelism = getRuntimeContext().getTaskInfo().getNumberOfParallelSubtasks(); int subtaskIndex = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask(); - // Distribute the configured rate as base + 1 extra for the first `remainder` subtasks so - // that the per-subtask rates sum back to the configured aggregate. Each subtask gets at - // least 1/s (hard floor) — when parallelism exceeds the configured rate, the aggregate - // may theoretically exceed it; in practice Batch scheduling limits actual concurrency. - long base = deleteRateLimitPerSecond / parallelism; - long remainder = deleteRateLimitPerSecond % parallelism; - long quota = base + (subtaskIndex < remainder ? 1L : 0L); - rateLimiter = RateLimiter.create(Math.max(1.0, (double) quota)); + // Distribute the configured rate as base + 1 extra for the first `remainder` subtasks. + // Flink does not provide a cross-JVM limiter here, so this is a best-effort job-level + // target. Each subtask gets at least 1/s; if parallelism exceeds the configured rate, the + // effective aggregate can exceed the target by that floor. + remoteFsOpRateLimiter = + RateLimiter.create( + perSubtaskRate(remoteFsOpRateLimitPerSecond, parallelism, subtaskIndex)); } @Override @@ -127,7 +127,8 @@ private CleanStats processBucketTask(BucketCleanTask task) throws IOException { RuleDispatcher dispatcher = new RuleDispatcher(task.allowDeleteManifest()); SafeDeleter safeDeleter = createSafeDeleter(anyDir.getFileSystem(), task.dryRun()); BucketCleaner cleaner = - new BucketCleaner(dispatcher, safeDeleter, audit, task.cutoffMillis()); + new BucketCleaner( + dispatcher, safeDeleter, audit, task.cutoffMillis(), remoteFsOpRateLimiter); BucketCleaner.BucketCleanStats bucketStats = cleaner.clean(activeRefs, logDir, kvDir); @@ -146,6 +147,7 @@ private CleanStats processBucketTask(BucketCleanTask task) throws IOException { private CleanStats processOrphanDirTask(OrphanDirCleanTask task) throws IOException { FsPath dirPath = new FsPath(task.dirPath()); FileSystem fs = dirPath.getFileSystem(); + remoteFsOpRateLimiter.acquire(); if (!fs.exists(dirPath)) { return CleanStats.empty(); } @@ -159,6 +161,7 @@ private CleanStats processOrphanDirTask(OrphanDirCleanTask task) throws IOExcept long deleteFailures = 0L; long bytesReclaimed = 0L; + remoteFsOpRateLimiter.acquire(); FileStatus rootStatus = fs.getFileStatus(dirPath); Deque stack = new ArrayDeque(); stack.push( @@ -178,6 +181,7 @@ private CleanStats processOrphanDirTask(OrphanDirCleanTask task) throws IOExcept } FileStatus[] children; try { + remoteFsOpRateLimiter.acquire(); children = fs.listStatus(visit.dir); } catch (IOException e) { LOG.warn("Failed to list directory: {}", visit.dir, e); @@ -234,7 +238,14 @@ private CleanStats processOrphanDirTask(OrphanDirCleanTask task) throws IOExcept // ------------------------------------------------------------------------- private SafeDeleter createSafeDeleter(FileSystem fs, boolean dryRun) { - return new SafeDeleter(fs, dryRun, audit, rateLimiter); + return new SafeDeleter(fs, dryRun, audit, remoteFsOpRateLimiter); + } + + private static double perSubtaskRate(long totalRate, int parallelism, int subtaskIndex) { + long base = totalRate / parallelism; + long remainder = totalRate % parallelism; + long quota = base + (subtaskIndex < remainder ? 1L : 0L); + return Math.max(1.0, (double) quota); } private static final class DirVisit { diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java index 170325e602..5c5654c80c 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java @@ -40,6 +40,7 @@ import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.RateLimiter; import org.apache.fluss.utils.FlussPaths; import org.apache.flink.streaming.api.functions.ProcessFunction; @@ -62,8 +63,6 @@ import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.enumerateBuckets; import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.fetchClusterConfigMap; -import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.getFileSystemIfExists; -import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.listStatuses; import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.normalizeRoot; import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.physicalPath; import static org.apache.fluss.flink.action.orphan.OrphanCleanUtils.remoteSubDir; @@ -116,7 +115,9 @@ public void processElement(Integer trigger, Context ctx, Collector ou AuditLogger audit = new AuditLogger(); audit.logCutoff(config.olderThanMillis()); - ActiveRefsFetcher fetcher = new ActiveRefsFetcher(admin, 3); + RateLimiter remoteFsOpRateLimiter = + RateLimiter.create((double) config.remoteFsOpRateLimitPerSecond()); + ActiveRefsFetcher fetcher = new ActiveRefsFetcher(admin, 3, remoteFsOpRateLimiter); MaxKnownIdsTracker tracker = new MaxKnownIdsTracker(); Map clusterConfigMap = fetchClusterConfigMap(admin); String clusterRemoteDataDir = resolveClusterRemoteDataDir(clusterConfigMap); @@ -129,9 +130,11 @@ public void processElement(Integer trigger, Context ctx, Collector ou for (LiveTableScope liveTable : dbState.liveTables) { emitBucketTasks( liveTable, fetcher, audit, clusterRemoteDataDir, clusterRoots, out); - emitOrphanPartitionDirTasks(liveTable, tracker, clusterRoots, audit, out); + emitOrphanPartitionDirTasks( + liveTable, tracker, clusterRoots, audit, remoteFsOpRateLimiter, out); } - emitOrphanTableDirTasks(dbState, tracker, clusterRoots, audit, out); + emitOrphanTableDirTasks( + dbState, tracker, clusterRoots, audit, remoteFsOpRateLimiter, out); } } } @@ -443,6 +446,7 @@ private void emitOrphanTableDirTasks( MaxKnownIdsTracker tracker, List clusterRoots, AuditLogger audit, + RateLimiter remoteFsOpRateLimiter, Collector out) throws IOException { if (!dbState.tableInfosComplete) { @@ -461,6 +465,7 @@ private void emitOrphanTableDirTasks( dirName -> OrphanDirDetector.isOrphanTable( dirName, activeTableIds, maxKnownTableId), + remoteFsOpRateLimiter, dir -> out.collect( new OrphanDirCleanTask( @@ -474,6 +479,7 @@ private void emitOrphanTableDirTasks( dirName -> OrphanDirDetector.isOrphanTable( dirName, activeTableIds, maxKnownTableId), + remoteFsOpRateLimiter, dir -> audit.logSkipOrphanTable(dir, "default-conservative")); } } @@ -485,6 +491,7 @@ private void emitOrphanPartitionDirTasks( MaxKnownIdsTracker tracker, List clusterRoots, AuditLogger audit, + RateLimiter remoteFsOpRateLimiter, Collector out) throws IOException { if (!liveTable.partitioned || !liveTable.partitionInfosComplete) { @@ -506,6 +513,7 @@ private void emitOrphanPartitionDirTasks( dirName -> OrphanDirDetector.isOrphanPartition( dirName, activePartitionIds, maxKnownPartitionId), + remoteFsOpRateLimiter, dir -> out.collect( new OrphanDirCleanTask( @@ -519,6 +527,7 @@ private void emitOrphanPartitionDirTasks( dirName -> OrphanDirDetector.isOrphanPartition( dirName, activePartitionIds, maxKnownPartitionId), + remoteFsOpRateLimiter, dir -> audit.logSkipOrphanPartition(dir, "default-conservative")); } } @@ -526,13 +535,16 @@ private void emitOrphanPartitionDirTasks( } private void forEachOrphanDirUnderParent( - FsPath parentDir, Predicate isOrphan, Consumer action) + FsPath parentDir, + Predicate isOrphan, + RateLimiter remoteFsOpRateLimiter, + Consumer action) throws IOException { - FileSystem fs = getFileSystemIfExists(parentDir); + FileSystem fs = getFileSystemIfExists(parentDir, remoteFsOpRateLimiter); if (fs == null) { return; } - FileStatus[] entries = listStatuses(fs, parentDir); + FileStatus[] entries = listStatuses(fs, parentDir, remoteFsOpRateLimiter); if (entries == null) { return; } @@ -555,6 +567,26 @@ private static String classifyName(Throwable e) { return RpcErrorClassifier.classify(e).name(); } + @Nullable + private static FileSystem getFileSystemIfExists(FsPath dir, RateLimiter remoteFsOpRateLimiter) + throws IOException { + FileSystem fs = dir.getFileSystem(); + remoteFsOpRateLimiter.acquire(); + return fs.exists(dir) ? fs : null; + } + + @Nullable + private static FileStatus[] listStatuses( + FileSystem fs, FsPath dir, RateLimiter remoteFsOpRateLimiter) { + try { + remoteFsOpRateLimiter.acquire(); + return fs.listStatus(dir); + } catch (IOException e) { + LOG.warn("Failed to list directory: {}", dir, e); + return null; + } + } + // ------------------------------------------------------------------------- // Internal state classes // ------------------------------------------------------------------------- diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfigTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfigTest.java index b564f90b90..222d4743e2 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfigTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/config/OrphanCleanConfigTest.java @@ -50,12 +50,44 @@ void parsesAllDatabasesWithDefaults() { long olderThanHigh = afterParse - Duration.ofDays(3).toMillis(); assertThat(config.olderThanMillis()).isBetween(olderThanLow, olderThanHigh); assertThat(config.dryRun()).isFalse(); - assertThat(config.deleteRateLimitPerSecond()).isEqualTo(100L); + assertThat(config.remoteFsOpRateLimitPerSecond()).isEqualTo(100L); assertThat(config.allowDeleteManifest()).isFalse(); assertThat(config.allowCleanOrphanTables()).isFalse(); assertThat(config.allowCleanOrphanPartitions()).isFalse(); } + @Test + void remoteFsOpRateLimitParsed() { + OrphanCleanConfig cfg = + OrphanCleanConfig.fromParams( + MultipleParameterToolAdapter.fromArgs( + new String[] { + "--bootstrap-server", + "h:9123", + "--all-databases", + "--remote-fs-op-rate-limit-per-second", + "42" + })); + assertThat(cfg.remoteFsOpRateLimitPerSecond()).isEqualTo(42L); + } + + @Test + void remoteFsOpRateLimitMustBePositive() { + assertThatThrownBy( + () -> + OrphanCleanConfig.fromParams( + MultipleParameterToolAdapter.fromArgs( + new String[] { + "--bootstrap-server", + "h:9123", + "--all-databases", + "--remote-fs-op-rate-limit-per-second", + "0" + }))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("--remote-fs-op-rate-limit-per-second must be positive"); + } + @Test void databaseAndAllDatabasesAreMutuallyExclusive() { assertThatThrownBy( diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleterTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleterTest.java index 147192411e..8be4bd3d11 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleterTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/fs/SafeDeleterTest.java @@ -23,6 +23,7 @@ import org.apache.fluss.fs.FileSystem; import org.apache.fluss.fs.FsPath; import org.apache.fluss.fs.local.LocalFileSystem; +import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.RateLimiter; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -42,7 +43,7 @@ class SafeDeleterTest { @Test void deleteFileRespectsDryRun() throws IOException { Path target = Files.createFile(tmp.resolve("orphan.log")); - SafeDeleter d = new SafeDeleter(localFs(), true, new AuditLogger()); + SafeDeleter d = newDeleter(localFs(), true); d.deleteFile(new FsPath(target.toString()), Decision.DELETE, RuleId.LOG_SEGMENT); assertThat(Files.exists(target)).isTrue(); } @@ -50,14 +51,14 @@ void deleteFileRespectsDryRun() throws IOException { @Test void deleteFileActuallyDeletesWhenNotDryRun() throws IOException { Path target = Files.createFile(tmp.resolve("orphan.log")); - SafeDeleter d = new SafeDeleter(localFs(), false, new AuditLogger()); + SafeDeleter d = newDeleter(localFs(), false); d.deleteFile(new FsPath(target.toString()), Decision.DELETE, RuleId.LOG_SEGMENT); assertThat(Files.exists(target)).isFalse(); } @Test void deleteFileRejectsNonDeleteDecision() { - SafeDeleter d = new SafeDeleter(null, false, new AuditLogger()); + SafeDeleter d = newDeleter(null, false); assertThatThrownBy( () -> d.deleteFile( @@ -69,7 +70,7 @@ void deleteFileRejectsNonDeleteDecision() { void deleteEmptyDirNoOpsOnNonEmpty() throws IOException { Path dir = Files.createDirectory(tmp.resolve("d")); Files.createFile(dir.resolve("child")); - SafeDeleter d = new SafeDeleter(localFs(), false, new AuditLogger()); + SafeDeleter d = newDeleter(localFs(), false); d.deleteEmptyDir(new FsPath(dir.toString())); assertThat(Files.exists(dir)).isTrue(); } @@ -77,11 +78,15 @@ void deleteEmptyDirNoOpsOnNonEmpty() throws IOException { @Test void deleteEmptyDirActuallyDeletes() throws IOException { Path dir = Files.createDirectory(tmp.resolve("d")); - SafeDeleter d = new SafeDeleter(localFs(), false, new AuditLogger()); + SafeDeleter d = newDeleter(localFs(), false); d.deleteEmptyDir(new FsPath(dir.toString())); assertThat(Files.exists(dir)).isFalse(); } + private static SafeDeleter newDeleter(FileSystem fs, boolean dryRun) { + return new SafeDeleter(fs, dryRun, new AuditLogger(), RateLimiter.create(1000.0)); + } + private static FileSystem localFs() { return LocalFileSystem.getSharedInstance(); } diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/BucketCleanerTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/BucketCleanerTest.java index d41f849c0f..b0fc5484f5 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/BucketCleanerTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/job/BucketCleanerTest.java @@ -54,16 +54,7 @@ void removesOldEmptySegmentDirAfterDeletingExpiredFiles(@TempDir Path tmp) throw makeOld(segmentDir, cutoff - 1000L); makeOld(bucketRoot, cutoff - 1000L); - BucketCleaner cleaner = - new BucketCleaner( - new RuleDispatcher(), - new SafeDeleter( - new FsPath(bucketRoot.toString()).getFileSystem(), - false, - new AuditLogger(), - RateLimiter.create(1000.0)), - new AuditLogger(), - cutoff); + BucketCleaner cleaner = createCleaner(bucketRoot, cutoff); BucketCleaner.BucketCleanStats stats = cleaner.clean(BucketActiveRefs.empty(), new FsPath(bucketRoot.toString())); @@ -83,16 +74,7 @@ void keepsFreshEmptySegmentDir(@TempDir Path tmp) throws IOException { Files.createDirectories(bucketRoot.resolve("11111111-1111-1111-1111-111111111111")); long cutoff = System.currentTimeMillis() - 1000L; - BucketCleaner cleaner = - new BucketCleaner( - new RuleDispatcher(), - new SafeDeleter( - new FsPath(bucketRoot.toString()).getFileSystem(), - false, - new AuditLogger(), - RateLimiter.create(1000.0)), - new AuditLogger(), - cutoff); + BucketCleaner cleaner = createCleaner(bucketRoot, cutoff); BucketCleaner.BucketCleanStats stats = cleaner.clean( @@ -118,16 +100,7 @@ void scansButDoesNotDeleteUnknownDotFiles(@TempDir Path tmp) throws IOException makeOld(segmentDir, cutoff - 1000L); makeOld(bucketRoot, cutoff - 1000L); - BucketCleaner cleaner = - new BucketCleaner( - new RuleDispatcher(), - new SafeDeleter( - new FsPath(bucketRoot.toString()).getFileSystem(), - false, - new AuditLogger(), - RateLimiter.create(1000.0)), - new AuditLogger(), - cutoff); + BucketCleaner cleaner = createCleaner(bucketRoot, cutoff); BucketCleaner.BucketCleanStats stats = cleaner.clean(BucketActiveRefs.empty(), new FsPath(bucketRoot.toString())); @@ -142,4 +115,18 @@ void scansButDoesNotDeleteUnknownDotFiles(@TempDir Path tmp) throws IOException private static void makeOld(Path path, long timestampMillis) throws IOException { Files.setLastModifiedTime(path, FileTime.fromMillis(timestampMillis)); } + + private static BucketCleaner createCleaner(Path bucketRoot, long cutoff) throws IOException { + RateLimiter remoteFsOpRateLimiter = RateLimiter.create(1000.0); + return new BucketCleaner( + new RuleDispatcher(), + new SafeDeleter( + new FsPath(bucketRoot.toString()).getFileSystem(), + false, + new AuditLogger(), + remoteFsOpRateLimiter), + new AuditLogger(), + cutoff, + remoteFsOpRateLimiter); + } } From 577c159ff8497409101bf1f244a5755fe72e0a6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Tue, 16 Jun 2026 17:01:42 +0800 Subject: [PATCH 15/19] [flink] Use versioned action jars --- fluss-flink/fluss-flink-1.18/pom.xml | 10 ++- ...rg.apache.fluss.flink.action.ActionFactory | 19 ----- fluss-flink/fluss-flink-1.19/pom.xml | 7 +- fluss-flink/fluss-flink-1.20/pom.xml | 7 +- fluss-flink/fluss-flink-2.2/pom.xml | 5 ++ fluss-flink/fluss-flink-action/pom.xml | 82 ------------------- .../fluss/flink/action/ActionLoader.java | 21 ++++- .../flink/action/FlussActionEntrypoint.java | 2 +- fluss-flink/pom.xml | 3 +- 9 files changed, 47 insertions(+), 109 deletions(-) delete mode 100644 fluss-flink/fluss-flink-1.18/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory delete mode 100644 fluss-flink/fluss-flink-action/pom.xml rename fluss-flink/{fluss-flink-action => fluss-flink-common}/src/main/java/org/apache/fluss/flink/action/FlussActionEntrypoint.java (93%) diff --git a/fluss-flink/fluss-flink-1.18/pom.xml b/fluss-flink/fluss-flink-1.18/pom.xml index 1636f25569..9f67b6ce9b 100644 --- a/fluss-flink/fluss-flink-1.18/pom.xml +++ b/fluss-flink/fluss-flink-1.18/pom.xml @@ -219,6 +219,14 @@ org.apache.fluss:fluss-client + + + org.apache.fluss:fluss-flink-common + + org/apache/fluss/flink/action/** + + + @@ -226,4 +234,4 @@ - \ No newline at end of file + diff --git a/fluss-flink/fluss-flink-1.18/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory b/fluss-flink/fluss-flink-1.18/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory deleted file mode 100644 index c30c9dd5ab..0000000000 --- a/fluss-flink/fluss-flink-1.18/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -org.apache.fluss.flink.action.orphan.OrphanFilesCleanActionFactory diff --git a/fluss-flink/fluss-flink-1.19/pom.xml b/fluss-flink/fluss-flink-1.19/pom.xml index a9df2c830a..d16e6e46a8 100644 --- a/fluss-flink/fluss-flink-1.19/pom.xml +++ b/fluss-flink/fluss-flink-1.19/pom.xml @@ -213,6 +213,11 @@ org.apache.fluss:fluss-client + + + org.apache.fluss.flink.action.FlussActionEntrypoint + + @@ -220,4 +225,4 @@ - \ No newline at end of file + diff --git a/fluss-flink/fluss-flink-1.20/pom.xml b/fluss-flink/fluss-flink-1.20/pom.xml index 25d867b398..ab0915f6e8 100644 --- a/fluss-flink/fluss-flink-1.20/pom.xml +++ b/fluss-flink/fluss-flink-1.20/pom.xml @@ -234,6 +234,11 @@ org.apache.fluss:fluss-client + + + org.apache.fluss.flink.action.FlussActionEntrypoint + + @@ -241,4 +246,4 @@ - \ No newline at end of file + diff --git a/fluss-flink/fluss-flink-2.2/pom.xml b/fluss-flink/fluss-flink-2.2/pom.xml index f2ea4cb597..3337797d4c 100644 --- a/fluss-flink/fluss-flink-2.2/pom.xml +++ b/fluss-flink/fluss-flink-2.2/pom.xml @@ -258,6 +258,11 @@ org.apache.fluss:fluss-client + + + org.apache.fluss.flink.action.FlussActionEntrypoint + + diff --git a/fluss-flink/fluss-flink-action/pom.xml b/fluss-flink/fluss-flink-action/pom.xml deleted file mode 100644 index 51e0852089..0000000000 --- a/fluss-flink/fluss-flink-action/pom.xml +++ /dev/null @@ -1,82 +0,0 @@ - - - - - 4.0.0 - - org.apache.fluss - fluss-flink - 1.0-SNAPSHOT - - - jar - - fluss-flink-action - Fluss : Flink : Action - - - 1.20.3 - - - - - org.apache.fluss - fluss-flink-common - ${project.version} - provided - - - - org.apache.flink - flink-streaming-java - ${flink.minor.version} - provided - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - shade-fluss - package - - shade - - - - - org.apache.fluss.flink.action.FlussActionEntrypoint - - - - - - - - - - - diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/ActionLoader.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/ActionLoader.java index 9d0eb1b93a..91599e7510 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/ActionLoader.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/ActionLoader.java @@ -45,6 +45,10 @@ public static Optional createAction(String[] args) { printDefaultHelp(); return Optional.empty(); } + if (isHelp(args[0])) { + printDefaultHelp(); + return Optional.empty(); + } String name = args[0].toLowerCase().replace('-', '_'); ActionFactory factory = findFactory(name) @@ -55,14 +59,27 @@ public static Optional createAction(String[] args) { + args[0] + ". Run with --help for available actions.")); String[] remaining = Arrays.copyOfRange(args, 1, args.length); - MultipleParameterToolAdapter params = MultipleParameterToolAdapter.fromArgs(remaining); - if (params.has("help")) { + if (hasHelp(remaining)) { System.out.println(factory.help()); return Optional.empty(); } + MultipleParameterToolAdapter params = MultipleParameterToolAdapter.fromArgs(remaining); return factory.create(params); } + private static boolean isHelp(String arg) { + return "--help".equals(arg) || "-h".equals(arg); + } + + private static boolean hasHelp(String[] args) { + for (String arg : args) { + if (isHelp(arg)) { + return true; + } + } + return false; + } + private static Optional findFactory(String identifier) { for (ActionFactory f : ServiceLoader.load(ActionFactory.class)) { if (f.identifier().equals(identifier)) { diff --git a/fluss-flink/fluss-flink-action/src/main/java/org/apache/fluss/flink/action/FlussActionEntrypoint.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/FlussActionEntrypoint.java similarity index 93% rename from fluss-flink/fluss-flink-action/src/main/java/org/apache/fluss/flink/action/FlussActionEntrypoint.java rename to fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/FlussActionEntrypoint.java index b1dbdecd97..dda7d4cf93 100644 --- a/fluss-flink/fluss-flink-action/src/main/java/org/apache/fluss/flink/action/FlussActionEntrypoint.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/FlussActionEntrypoint.java @@ -19,7 +19,7 @@ import java.util.Optional; -/** Main entrypoint for the Fluss Flink action jar. Delegates to {@link ActionLoader}. */ +/** Main entrypoint for Fluss Flink action jars. Delegates to {@link ActionLoader}. */ public class FlussActionEntrypoint { public static void main(String[] args) throws Exception { diff --git a/fluss-flink/pom.xml b/fluss-flink/pom.xml index b66643a90a..a0fae789b9 100644 --- a/fluss-flink/pom.xml +++ b/fluss-flink/pom.xml @@ -38,7 +38,6 @@ fluss-flink-1.18 fluss-flink-2.2 fluss-flink-tiering - fluss-flink-action org.apache.flink.table.catalog.* From fb43e3b4dbc13b9baba89a7eb6f79347164df13d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Tue, 30 Jun 2026 10:18:06 +0800 Subject: [PATCH 19/19] Pass client.* extra configs to Fluss connection for auth support --- .../orphan/job/ScopeEnumeratorFunction.java | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java index 5c5654c80c..eede19cc68 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/job/ScopeEnumeratorFunction.java @@ -23,6 +23,8 @@ import org.apache.fluss.client.admin.Admin; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.DisconnectException; +import org.apache.fluss.exception.NetworkException; import org.apache.fluss.exception.UnsupportedVersionException; import org.apache.fluss.flink.action.orphan.OrphanCleanUtils; import org.apache.fluss.flink.action.orphan.RpcErrorClassifier; @@ -41,6 +43,7 @@ import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.RateLimiter; +import org.apache.fluss.utils.ExceptionUtils; import org.apache.fluss.utils.FlussPaths; import org.apache.flink.streaming.api.functions.ProcessFunction; @@ -102,6 +105,12 @@ public void processElement(Integer trigger, Context ctx, Collector ou Configuration flussConfig = new Configuration(); flussConfig.setString(ConfigOptions.BOOTSTRAP_SERVERS.key(), config.bootstrapServer()); + // Pass through client-related extra configs (e.g. security/auth). + for (Map.Entry entry : config.extraConfigs().entrySet()) { + if (entry.getKey().startsWith("client.")) { + flussConfig.setString(entry.getKey(), entry.getValue()); + } + } try (Connection connection = ConnectionFactory.createConnection(flussConfig); Admin admin = connection.getAdmin()) { @@ -177,11 +186,31 @@ private static void probeApi(String apiName, ThrowingProbe probe) { + " older orphan-files-cleanup action that targets this server.", t); } + if (isConnectionFailure(t)) { + throw new IllegalStateException( + "Failed to connect to Fluss cluster while probing " + + apiName + + " RPC. The bootstrap server may be unreachable.", + t); + } // Any other failure means the RPC is recognized; the call merely failed because of // the sentinel target id. Compatibility is satisfied. } } + private static boolean isConnectionFailure(Throwable t) { + Throwable cause = ExceptionUtils.stripExecutionException(t); + while (cause != null) { + if (cause instanceof NetworkException + || cause instanceof DisconnectException + || cause instanceof IOException) { + return true; + } + cause = cause.getCause(); + } + return false; + } + private static boolean isUnsupportedVersion(Throwable t) { Throwable cause = t; while (cause != null) { @@ -235,7 +264,10 @@ private List resolveDatabasesToScan(Admin admin, AuditLogger audit) { return admin.listDatabases().get(); } catch (Exception e) { audit.logSkipDb("*", classifyName(e)); - return Collections.emptyList(); + throw new IllegalStateException( + "Failed to list databases from Fluss cluster. " + + "The coordinator server may be unreachable.", + e); } } String databaseName = config.database().get(); @@ -245,7 +277,12 @@ private List resolveDatabasesToScan(Admin admin, AuditLogger audit) { } } catch (Exception e) { audit.logSkipDb(databaseName, classifyName(e)); - return Collections.emptyList(); + throw new IllegalStateException( + "Failed to check existence of database '" + + databaseName + + "'. " + + "The coordinator server may be unreachable.", + e); } audit.logSkipDb(databaseName, RpcErrorClassifier.Category.NOT_FOUND.name()); return Collections.emptyList();