[flink-action][server][client] add orphan files cleanup action for remote storage

fluss-common/src/main/java/org/apache/fluss/fs/FileStatus.java

@@ -46,4 +46,18 @@ public interface FileStatus {
      * @return the corresponding Path to the FileStatus
      */
     FsPath getPath();
+
+    /**
+     * Returns the modification time of the file in milliseconds since the epoch.
+     *
+     * <p>The default implementation returns {@link Long#MAX_VALUE}, which is interpreted by
+     * time-based filters (e.g. orphan-files cleanup) as "always fresh" - effectively a fail-closed
+     * default that prevents deletion when modification time is unavailable. File system
+     * implementations that can expose modification time SHOULD override this.
+     *
+     * @return the modification time in epoch millis, or {@link Long#MAX_VALUE} when unavailable
+     */
+    default long getModificationTime() {
+        return Long.MAX_VALUE;
+    }
 }

fluss-common/src/main/java/org/apache/fluss/fs/local/LocalFileStatus.java

@@ -67,6 +67,11 @@ public FsPath getPath() {
         return this.path;
     }
 
+    @Override
+    public long getModificationTime() {
+        return this.file.lastModified();
+    }
+
     public File getFile() {
         return this.file;
     }

fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManifest.java → fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifest.java

@@ -15,12 +15,10 @@
  * limitations under the License.
  */
 
-package org.apache.fluss.server.log.remote;
+package org.apache.fluss.remote;
 
-import org.apache.fluss.annotation.VisibleForTesting;
 import org.apache.fluss.metadata.PhysicalTablePath;
 import org.apache.fluss.metadata.TableBucket;
-import org.apache.fluss.remote.RemoteLogSegment;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -33,7 +31,7 @@
 
 /**
  * A remote log manifest is an immutable list of current {@link RemoteLogSegment} which can
- * represent a snapshot of {@link RemoteLogTablet}.
+ * represent a snapshot of a remote log tablet.
  */
 public class RemoteLogManifest {
     private final PhysicalTablePath physicalTablePath;
@@ -122,7 +120,6 @@ public TableBucket getTableBucket() {
         return tableBucket;
     }
 
-    @VisibleForTesting
     public List<RemoteLogSegment> getRemoteLogSegmentList() {
         return remoteLogSegmentList;
     }

fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManifestJsonSerde.java → fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestJsonSerde.java

@@ -15,11 +15,10 @@
  * limitations under the License.
  */
 
-package org.apache.fluss.server.log.remote;
+package org.apache.fluss.remote;
 
 import org.apache.fluss.metadata.PhysicalTablePath;
 import org.apache.fluss.metadata.TableBucket;
-import org.apache.fluss.remote.RemoteLogSegment;
 import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator;
 import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode;
 import org.apache.fluss.utils.json.JsonDeserializer;

fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java

@@ -74,7 +74,7 @@ public class FlussPaths {
     public static final String REMOTE_LOG_DIR_NAME = "log";
 
     /** The directory name for storing metadata files (e.g., manifest) for a log tablet. */
-    private static final String REMOTE_LOG_METADATA_DIR_NAME = "metadata";
+    public static final String REMOTE_LOG_METADATA_DIR_NAME = "metadata";
 
     /** Suffix of a manifest file. */
     private static final String REMOTE_LOG_MANIFEST_FILE_SUFFIX = ".manifest";

fluss-common/src/test/java/org/apache/fluss/fs/FileStatusTest.java

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.fs;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for default methods of {@link FileStatus}. */
+class FileStatusTest {
+
+    /**
+     * An implementation that does not override {@link FileStatus#getModificationTime()} must
+     * inherit the fail-safe default of {@link Long#MAX_VALUE}, so time-based filters treat the file
+     * as "always fresh" and never delete it when modification time is unavailable.
+     */
+    @Test
+    void defaultModificationTimeIsMaxValueFailSafe() {
+        FileStatus status =
+                new FileStatus() {
+                    @Override
+                    public long getLen() {
+                        return 0L;
+                    }
+
+                    @Override
+                    public boolean isDir() {
+                        return false;
+                    }
+
+                    @Override
+                    public FsPath getPath() {
+                        return new FsPath("/tmp/x");
+                    }
+                };
+
+        assertThat(status.getModificationTime()).isEqualTo(Long.MAX_VALUE);
+    }
+}

fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManifestJsonSerdeTest.java → fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestJsonSerdeTest.java

@@ -15,18 +15,17 @@
  * limitations under the License.
  */
 
-package org.apache.fluss.server.log.remote;
+package org.apache.fluss.remote;
 
 import org.apache.fluss.metadata.PhysicalTablePath;
 import org.apache.fluss.metadata.TableBucket;
 import org.apache.fluss.metadata.TablePath;
-import org.apache.fluss.remote.RemoteLogSegment;
 import org.apache.fluss.utils.json.JsonSerdeTestBase;
 
 import java.util.Arrays;
 import java.util.UUID;
 
-/** Tests of {@link org.apache.fluss.server.log.remote.RemoteLogManifestJsonSerde}. */
+/** Tests of {@link RemoteLogManifestJsonSerde}. */
 class RemoteLogManifestJsonSerdeTest extends JsonSerdeTestBase<RemoteLogManifest> {
     private static final PhysicalTablePath TABLE_PATH1 =
             PhysicalTablePath.of(TablePath.of("db", "mytable"));

fluss-filesystems/fluss-fs-hadoop/src/main/java/org/apache/fluss/fs/hdfs/HadoopFileStatus.java

@@ -52,6 +52,11 @@ public boolean isDir() {
         return fileStatus.isDirectory();
     }
 
+    @Override
+    public long getModificationTime() {
+        return fileStatus.getModificationTime();
+    }
+
     // ------------------------------------------------------------------------
 
     /**

fluss-flink/fluss-flink-1.18/pom.xml

@@ -219,11 +219,19 @@
                                     <include>org.apache.fluss:fluss-client</include>
                                 </includes>
                             </artifactSet>
+                            <filters combine.children="append">
+                                <filter>
+                                    <artifact>org.apache.fluss:fluss-flink-common</artifact>
+                                    <excludes>
+                                        <exclude>org/apache/fluss/flink/action/**</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
                         </configuration>
                     </execution>
                 </executions>
             </plugin>
         </plugins>
     </build>
 
-</project>
+</project>

fluss-flink/fluss-flink-1.19/pom.xml

@@ -213,11 +213,16 @@
                                     <include>org.apache.fluss:fluss-client</include>
                                 </includes>
                             </artifactSet>
+                            <transformers combine.children="append">
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>org.apache.fluss.flink.action.FlussActionEntrypoint</mainClass>
+                                </transformer>
+                            </transformers>
                         </configuration>
                     </execution>
                 </executions>
             </plugin>
         </plugins>
     </build>
 
-</project>
+</project>

fluss-flink/fluss-flink-1.19/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory

@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.fluss.flink.action.orphan.OrphanFilesCleanActionFactory

fluss-flink/fluss-flink-1.19/src/test/java/org/apache/fluss/flink/action/orphan/Flink119OrphanFilesCleanITCase.java

@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.flink.action.orphan;
+
+/** The IT case for orphan files cleanup in Flink 1.19. */
+class Flink119OrphanFilesCleanITCase extends OrphanFilesCleanITCase {}

fluss-flink/fluss-flink-1.20/pom.xml

@@ -234,11 +234,16 @@
                                     <include>org.apache.fluss:fluss-client</include>
                                 </includes>
                             </artifactSet>
+                            <transformers combine.children="append">
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>org.apache.fluss.flink.action.FlussActionEntrypoint</mainClass>
+                                </transformer>
+                            </transformers>
                         </configuration>
                     </execution>
                 </executions>
             </plugin>
         </plugins>
     </build>
 
-</project>
+</project>

fluss-flink/fluss-flink-1.20/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory

@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.fluss.flink.action.orphan.OrphanFilesCleanActionFactory

fluss-flink/fluss-flink-1.20/src/test/java/org/apache/fluss/flink/action/orphan/Flink120OrphanFilesCleanITCase.java

@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.flink.action.orphan;
+
+/** The IT case for orphan files cleanup in Flink 1.20. */
+class Flink120OrphanFilesCleanITCase extends OrphanFilesCleanITCase {}

fluss-flink/fluss-flink-2.2/pom.xml

@@ -258,6 +258,11 @@
                                     <include>org.apache.fluss:fluss-client</include>
                                 </includes>
                             </artifactSet>
+                            <transformers combine.children="append">
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>org.apache.fluss.flink.action.FlussActionEntrypoint</mainClass>
+                                </transformer>
+                            </transformers>
                         </configuration>
                     </execution>
                 </executions>

fluss-flink/fluss-flink-2.2/src/main/java/org/apache/fluss/flink/adapter/MultipleParameterToolAdapter.java

@@ -19,6 +19,9 @@
 
 import org.apache.flink.util.MultipleParameterTool;
 
+import javax.annotation.Nullable;
+
+import java.util.Collection;
 import java.util.Map;
 
 /**
@@ -43,4 +46,23 @@ public static MultipleParameterToolAdapter fromArgs(String[] args) {
     public Map<String, String> toMap() {
         return this.multipleParameterTool.toMap();
     }
+
+    /** Returns whether the given key is present in the parsed arguments. */
+    public boolean has(String key) {
+        return this.multipleParameterTool.has(key);
+    }
+
+    /** Returns the value for the given key, or {@code null} if the key is not found. */
+    @Nullable
+    public String get(String key) {
+        return this.multipleParameterTool.get(key);
+    }
+
+    /**
+     * Returns all values associated with the given key, or {@code null} if the key is not found.
+     */
+    @Nullable
+    public Collection<String> getMultiParameter(String key) {
+        return this.multipleParameterTool.getMultiParameter(key);
+    }
 }

fluss-flink/fluss-flink-2.2/src/main/resources/META-INF/services/org.apache.fluss.flink.action.ActionFactory

@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.fluss.flink.action.orphan.OrphanFilesCleanActionFactory