Skip to content

Use FileSystem.openFile with FileStatus to reduce NameNode RPCs #6460

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dlmarion wants to merge 5 commits into
base: 2.1
Choose a base branch
from
Open

Use FileSystem.openFile with FileStatus to reduce NameNode RPCs #6460

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
00b3067
Use FileSystem.openFile with FileStatus to reduce NameNode RPCs
dlmarion Jul 2, 2026
9aff821
Merge branch '2.1' into fs-open-use-file-status
dlmarion Jul 6, 2026
159d789
Merge branch '2.1' into fs-open-use-file-status
dlmarion Jul 10, 2026
1c580c2
Added overloaded method definition to reduce passing null
dlmarion Jul 10, 2026
0ee17c5
Added another optimization in RFileScannerBuilder and RFileOperations
dlmarion Jul 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 27 additions & 2 deletions core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScannerBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,17 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;

import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.rfile.RFile.ScannerFSOptions;
import org.apache.accumulo.core.client.rfile.RFile.ScannerOptions;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

Expand All @@ -56,8 +60,29 @@ RFileSource[] getSources() throws IOException {
if (sources == null) {
sources = new RFileSource[paths.length];
for (int i = 0; i < paths.length; i++) {
sources[i] = new RFileSource(getFileSystem(paths[i]).open(paths[i]),
getFileSystem(paths[i]).getFileStatus(paths[i]).getLen());
FileSystem fs = getFileSystem(paths[i]);
FileStatus status = fs.getFileStatus(paths[i]);
CompletableFuture<FSDataInputStream> future =
fs.openFile(paths[i]).withFileStatus(status).build();
while (!future.isDone()) {
try {
Thread.sleep(10);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IOException("Interrupted while opening file: " + paths[i], e);
}
}
try {
FSDataInputStream is = future.get();
sources[i] = new RFileSource(is, status.getLen());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IOException("Interrupted while opening file: " + paths[i], e);
} catch (CancellationException e) {
throw new IOException("Cancelled while opening file: " + paths[i], e);
} catch (ExecutionException e) {
throw new IOException("Error trying to open file: " + paths[i], e);
}
}
} else {
for (int i = 0; i < sources.length; i++) {
Expand Down
18 changes: 10 additions & 8 deletions core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkImport.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,8 +266,10 @@ public MLong(long i) {
}

public static Map<KeyExtent,Long> estimateSizes(AccumuloConfiguration acuConf, Path mapFile,
long fileSize, Collection<KeyExtent> extents, FileSystem ns, Cache<String,Long> fileLenCache,
CryptoService cs) throws IOException {
FileStatus status, Collection<KeyExtent> extents, FileSystem ns,
Cache<String,Long> fileLenCache, CryptoService cs) throws IOException {

final long fileSize = status.getLen();

if (extents.size() == 1) {
return Collections.singletonMap(extents.iterator().next(), fileSize);
Expand All @@ -282,7 +284,7 @@ public static Map<KeyExtent,Long> estimateSizes(AccumuloConfiguration acuConf, P
Text row = new Text();

FileSKVIterator index = FileOperations.getInstance().newIndexReaderBuilder()
.forFile(mapFile.toString(), ns, ns.getConf(), cs).withTableConfiguration(acuConf)
.forFile(mapFile.toString(), ns, ns.getConf(), cs, status).withTableConfiguration(acuConf)
.withFileLenCache(fileLenCache).build();

try {
Expand Down Expand Up @@ -365,9 +367,9 @@ private static Text nextRow(Text row) {

public static List<KeyExtent> findOverlappingTablets(ClientContext context,
KeyExtentCache keyExtentCache, Path file, FileSystem fs, Cache<String,Long> fileLenCache,
CryptoService cs) throws IOException {
CryptoService cs, FileStatus status) throws IOException {
try (FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder()
.forFile(file.toString(), fs, fs.getConf(), cs)
.forFile(file.toString(), fs, fs.getConf(), cs, status)
.withTableConfiguration(context.getConfiguration()).withFileLenCache(fileLenCache)
.seekToBeginning().build()) {

Expand Down Expand Up @@ -574,12 +576,12 @@ public SortedMap<KeyExtent,Bulk.Files> computeFileToTabletMappings(FileSystem fs
CompletableFuture<Map<KeyExtent,Bulk.FileInfo>> future = CompletableFuture.supplyAsync(() -> {
try {
long t1 = System.currentTimeMillis();
List<KeyExtent> extents =
findOverlappingTablets(context, extentCache, filePath, fs, fileLensCache, cs);
List<KeyExtent> extents = findOverlappingTablets(context, extentCache, filePath, fs,
fileLensCache, cs, fileStatus);
// make sure file isn't going to too many tablets
checkTabletCount(maxTablets, extents.size(), filePath.toString());
Map<KeyExtent,Long> estSizes = estimateSizes(context.getConfiguration(), filePath,
fileStatus.getLen(), extents, fs, fileLensCache, cs);
fileStatus, extents, fs, fileLensCache, cs);
Map<KeyExtent,Bulk.FileInfo> pathLocations = new HashMap<>();
for (KeyExtent ke : extents) {
pathLocations.put(ke, new Bulk.FileInfo(filePath, estSizes.getOrDefault(ke, 0L)));
Expand Down
38 changes: 33 additions & 5 deletions core/src/main/java/org/apache/accumulo/core/file/FileOperations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.apache.accumulo.core.util.ratelimit.RateLimiter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapred.FileOutputCommitter;

Expand Down Expand Up @@ -191,12 +192,14 @@ public static class FileOptions {
public final Set<ByteSequence> columnFamilies;
public final boolean inclusive;
public final boolean dropCacheBehind;
public final FileStatus status;

public FileOptions(TableId tableId, AccumuloConfiguration tableConfiguration, String filename,
FileSystem fs, Configuration fsConf, RateLimiter rateLimiter, String compression,
FSDataOutputStream outputStream, boolean enableAccumuloStart, CacheProvider cacheProvider,
Cache<String,Long> fileLenCache, boolean seekToBeginning, CryptoService cryptoService,
Range range, Set<ByteSequence> columnFamilies, boolean inclusive, boolean dropCacheBehind) {
Range range, Set<ByteSequence> columnFamilies, boolean inclusive, boolean dropCacheBehind,
FileStatus status) {
this.tableId = tableId;
this.tableConfiguration = tableConfiguration;
this.filename = filename;
Expand All @@ -214,6 +217,7 @@ public FileOptions(TableId tableId, AccumuloConfiguration tableConfiguration, St
this.columnFamilies = columnFamilies;
this.inclusive = inclusive;
this.dropCacheBehind = dropCacheBehind;
this.status = status;
}

public TableId getTableId() {
Expand Down Expand Up @@ -293,6 +297,7 @@ public static class FileHelper {
private RateLimiter rateLimiter;
private CryptoService cryptoService;
private boolean dropCacheBehind = false;
private FileStatus status;

protected FileHelper table(TableId tid) {
this.tableId = tid;
Expand Down Expand Up @@ -334,31 +339,36 @@ protected FileHelper dropCacheBehind(boolean drop) {
return this;
}

protected FileHelper fileStatus(FileStatus status) {
this.status = status;
return this;
}

protected FileOptions toWriterBuilderOptions(String compression,
FSDataOutputStream outputStream, boolean startEnabled) {
return new FileOptions(tableId, tableConfiguration, filename, fs, fsConf, rateLimiter,
compression, outputStream, startEnabled, NULL_PROVIDER, null, false, cryptoService, null,
null, true, dropCacheBehind);
null, true, dropCacheBehind, status);
}

protected FileOptions toReaderBuilderOptions(CacheProvider cacheProvider,
Cache<String,Long> fileLenCache, boolean seekToBeginning) {
return new FileOptions(tableId, tableConfiguration, filename, fs, fsConf, rateLimiter, null,
null, false, cacheProvider == null ? NULL_PROVIDER : cacheProvider, fileLenCache,
seekToBeginning, cryptoService, null, null, true, dropCacheBehind);
seekToBeginning, cryptoService, null, null, true, dropCacheBehind, status);
}

protected FileOptions toIndexReaderBuilderOptions(Cache<String,Long> fileLenCache) {
return new FileOptions(tableId, tableConfiguration, filename, fs, fsConf, rateLimiter, null,
null, false, NULL_PROVIDER, fileLenCache, false, cryptoService, null, null, true,
dropCacheBehind);
dropCacheBehind, status);
}

protected FileOptions toScanReaderBuilderOptions(Range range, Set<ByteSequence> columnFamilies,
boolean inclusive) {
return new FileOptions(tableId, tableConfiguration, filename, fs, fsConf, rateLimiter, null,
null, false, NULL_PROVIDER, null, false, cryptoService, range, columnFamilies, inclusive,
dropCacheBehind);
dropCacheBehind, status);
}

protected AccumuloConfiguration getTableConfiguration() {
Expand Down Expand Up @@ -441,6 +451,12 @@ public ReaderTableConfiguration forFile(String filename, FileSystem fs, Configur
return this;
}

public ReaderTableConfiguration forFile(String filename, FileSystem fs, Configuration fsConf,
CryptoService cs, FileStatus status) {
filename(filename).fs(fs).fsConf(fsConf).cryptoService(cs).fileStatus(status);
return this;
}

@Override
public ReaderBuilder withTableConfiguration(AccumuloConfiguration tableConfiguration) {
tableConfiguration(tableConfiguration);
Expand Down Expand Up @@ -509,6 +525,12 @@ public IndexReaderTableConfiguration forFile(String filename, FileSystem fs,
return this;
}

public IndexReaderTableConfiguration forFile(String filename, FileSystem fs,
Configuration fsConf, CryptoService cs, FileStatus status) {
filename(filename).fs(fs).fsConf(fsConf).cryptoService(cs).fileStatus(status);
return this;
}

@Override
public IndexReaderBuilder withTableConfiguration(AccumuloConfiguration tableConfiguration) {
tableConfiguration(tableConfiguration);
Expand Down Expand Up @@ -541,6 +563,12 @@ public ScanReaderTableConfiguration forFile(String filename, FileSystem fs,
return this;
}

public ScanReaderTableConfiguration forFile(String filename, FileSystem fs,
Configuration fsConf, CryptoService cs, FileStatus status) {
filename(filename).fs(fs).fsConf(fsConf).cryptoService(cs).fileStatus(status);
return this;
}

@Override
public ScanReaderBuilder withTableConfiguration(AccumuloConfiguration tableConfiguration) {
tableConfiguration(tableConfiguration);
Expand Down
62 changes: 47 additions & 15 deletions core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import java.util.Collections;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Supplier;
Expand All @@ -45,7 +47,9 @@
import org.apache.accumulo.core.util.ratelimit.RateLimiter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FutureDataInputStreamBuilder;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.Seekable;
import org.slf4j.Logger;
Expand Down Expand Up @@ -87,29 +91,57 @@ public CachableBuilder conf(Configuration hadoopConf) {
}

public CachableBuilder fsPath(FileSystem fs, Path dataFile) {
return fsPath(fs, dataFile, false);
return fsPath(fs, dataFile, false, null);
}

public CachableBuilder fsPath(FileSystem fs, Path dataFile, boolean dropCacheBehind) {
public CachableBuilder fsPath(FileSystem fs, Path dataFile, FileStatus status) {
return fsPath(fs, dataFile, false, status);
}

public CachableBuilder fsPath(FileSystem fs, Path dataFile, boolean dropCacheBehind,
FileStatus status) {
this.cacheId = pathToCacheId(dataFile);
this.inputSupplier = () -> {
FSDataInputStream is = fs.open(dataFile);
if (dropCacheBehind) {
// Tell the DataNode that the write ahead log does not need to be cached in the OS page
// cache
FutureDataInputStreamBuilder builder = fs.openFile(dataFile);
if (status != null) {
builder.withFileStatus(status);
}
CompletableFuture<FSDataInputStream> future = builder.build();
while (!future.isDone()) {
try {
is.setDropBehind(Boolean.TRUE);
log.trace("Called setDropBehind(TRUE) for stream reading file {}", dataFile);
} catch (UnsupportedOperationException e) {
log.debug("setDropBehind not enabled for wal file: {}", dataFile);
} catch (IOException e) {
log.debug("IOException setting drop behind for file: {}, msg: {}", dataFile,
e.getMessage());
Thread.sleep(10);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IOException("Interrupted while opening file: " + dataFile, e);
}
}
try {
FSDataInputStream is = future.get();
if (dropCacheBehind) {
// Tell the DataNode that the write ahead log does not need to be cached in the OS page
// cache
try {
is.setDropBehind(Boolean.TRUE);
log.trace("Called setDropBehind(TRUE) for stream reading file {}", dataFile);
} catch (UnsupportedOperationException e) {
log.debug("setDropBehind not enabled for wal file: {}", dataFile);
} catch (IOException e) {
log.debug("IOException setting drop behind for file: {}, msg: {}", dataFile,
e.getMessage());
}
}
return is;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IOException("Interrupted while opening file: " + dataFile, e);
} catch (CancellationException e) {
throw new IOException("Cancelled while opening file: " + dataFile, e);
} catch (ExecutionException e) {
throw new IOException("Error trying to open file: " + dataFile, e);
}
return is;
};
this.lengthSupplier = () -> fs.getFileStatus(dataFile).getLen();
this.lengthSupplier =
() -> status == null ? fs.getFileStatus(dataFile).getLen() : status.getLen();
return this;
}

Expand Down
9 changes: 7 additions & 2 deletions core/src/main/java/org/apache/accumulo/core/file/rfile/RFileOperations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,8 @@ public class RFileOperations extends FileOperations {

private static RFile.Reader getReader(FileOptions options) throws IOException {
CachableBuilder cb = new CachableBuilder()
.fsPath(options.getFileSystem(), new Path(options.getFilename()), options.dropCacheBehind)
.fsPath(options.getFileSystem(), new Path(options.getFilename()), options.dropCacheBehind,
options.status)
.conf(options.getConfiguration()).fileLen(options.getFileLenCache())
.cacheProvider(options.cacheProvider).readLimiter(options.getRateLimiter())
.cryptoService(options.getCryptoService());
Expand All @@ -71,7 +72,11 @@ private static RFile.Reader getReader(FileOptions options) throws IOException {

@Override
protected long getFileSize(FileOptions options) throws IOException {
return options.getFileSystem().getFileStatus(new Path(options.getFilename())).getLen();
if (options.status == null) {
return options.getFileSystem().getFileStatus(new Path(options.getFilename())).getLen();
} else {
return options.status.getLen();
}
}

@Override
Expand Down
Loading