Skip to content

Provide a better initial size for collections if possible #6464

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dlmarion merged 1 commit into from
Jul 9, 2026
Merged

Provide a better initial size for collections if possible #6464

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion core/src/main/java/org/apache/accumulo/core/clientImpl/ActiveCompactionImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ public long getPausedCount() {

@Override
public List<IteratorSetting> getIterators() {
ArrayList<IteratorSetting> ret = new ArrayList<>();
ArrayList<IteratorSetting> ret = new ArrayList<>(tac.getSsiListSize());

for (IterInfo ii : tac.getSsiList()) {
IteratorSetting settings =
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ public CachedTablet findTablet(ClientContext context, Text row, boolean skipRow,
}

public abstract <T extends Mutation> void binMutations(ClientContext context, List<T> mutations,
Map<String,TabletServerMutations<T>> binnedMutations, List<T> failures)
Map<String,TabletServerMutations<T>> binnedMutations, ArrayList<T> failures)
throws AccumuloException, AccumuloSecurityException, TableNotFoundException,
InvalidTabletHostingRequestException;

Expand Down
6 changes: 4 additions & 2 deletions core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ public ClientTabletCacheImpl(TableId tableId, ClientTabletCache parent, CachedTa

@Override
public <T extends Mutation> void binMutations(ClientContext context, List<T> mutations,
Map<String,TabletServerMutations<T>> binnedMutations, List<T> failures)
Map<String,TabletServerMutations<T>> binnedMutations, ArrayList<T> failures)
throws AccumuloException, AccumuloSecurityException, TableNotFoundException,
InvalidTabletHostingRequestException {

Expand Down Expand Up @@ -257,6 +257,8 @@ public <T extends Mutation> void binMutations(ClientContext context, List<T> mut
notInCache.sort((o1, o2) -> WritableComparator.compareBytes(o1.getRow(), 0,
o1.getRow().length, o2.getRow(), 0, o2.getRow().length));

failures.ensureCapacity(failures.size() + notInCache.size());

// Want to ignore any entries in the cache w/o a location that were created before the
// following time. Entries created after the following time may have been populated by the
// following loop, and we want to use those.
Expand Down Expand Up @@ -635,7 +637,7 @@ private void requestTabletHosting(ClientContext context,
return;
}

List<TKeyExtent> extentsToBringOnline = new ArrayList<>();
List<TKeyExtent> extentsToBringOnline = new ArrayList<>(tabletsWithNoLocation.size());
for (var cachedTablet : tabletsWithNoLocation) {
if (cachedTablet.getCreationTimer().elapsed().compareTo(STALE_DURATION) < 0) {
if (cachedTablet.getAvailability() == TabletAvailability.ONDEMAND) {
Expand Down
21 changes: 12 additions & 9 deletions core/src/main/java/org/apache/accumulo/core/clientImpl/ConditionalWriterImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,7 @@ private void queueRetry(List<QCMutation> mutations, HostAndPort server) {
}

private void queue(List<QCMutation> mutations) {
List<QCMutation> failures = new ArrayList<>();
ArrayList<QCMutation> failures = new ArrayList<>();
Map<String,TabletServerMutations<QCMutation>> binnedMutations = new HashMap<>();

try {
Expand Down Expand Up @@ -349,7 +349,7 @@ private void reschedule(SendTask task) {
private TabletServerMutations<QCMutation> dequeue(String location) {
var queue = getServerQueue(location).queue;

var mutations = new ArrayList<TabletServerMutations<QCMutation>>();
var mutations = new ArrayList<TabletServerMutations<QCMutation>>(queue.size());
queue.drainTo(mutations);

if (mutations.isEmpty()) {
Expand All @@ -363,8 +363,10 @@ private TabletServerMutations<QCMutation> dequeue(String location) {
TabletServerMutations<QCMutation> tsm = mutations.get(0);

for (int i = 1; i < mutations.size(); i++) {
mutations.get(i).getMutations().forEach((keyExtent, mutationList) -> tsm.getMutations()
.computeIfAbsent(keyExtent, k -> new ArrayList<>()).addAll(mutationList));
mutations.get(i).getMutations()
.forEach((keyExtent, mutationList) -> tsm.getMutations()
.computeIfAbsent(keyExtent, k -> new ArrayList<>(mutationList.size()))
.addAll(mutationList));
}

return tsm;
Expand All @@ -388,7 +390,7 @@ private TabletServerMutations<QCMutation> dequeue(String location) {
this.classLoaderContext = config.getClassLoaderContext();

Runnable failureHandler = () -> {
List<QCMutation> mutations = new ArrayList<>();
List<QCMutation> mutations = new ArrayList<>(failedMutations.size());
failedMutations.drainTo(mutations);
if (!mutations.isEmpty()) {
queue(mutations);
Expand Down Expand Up @@ -545,7 +547,7 @@ private void unreserveSessionID(HostAndPort location) {
}

List<SessionID> getActiveSessions() {
ArrayList<SessionID> activeSessions = new ArrayList<>();
ArrayList<SessionID> activeSessions = new ArrayList<>(cachedSessionIDs.values().size() / 2);
for (SessionID sid : cachedSessionIDs.values()) {
if (sid.isActive()) {
activeSessions.add(sid);
Expand Down Expand Up @@ -638,8 +640,9 @@ private void sendToServer(HostAndPort location, TabletServerMutations<QCMutation
}

private void queueRetry(Map<Long,CMK> cmidToCm, HostAndPort location) {
ArrayList<QCMutation> ignored = new ArrayList<>();
for (CMK cmk : cmidToCm.values()) {
var values = cmidToCm.values();
ArrayList<QCMutation> ignored = new ArrayList<>(values.size());
for (CMK cmk : values) {
ignored.add(cmk.cm);
}
queueRetry(ignored, location);
Expand Down Expand Up @@ -742,7 +745,7 @@ private void convertMutations(TabletServerMutations<QCMutation> mutations, Map<L
CompressedIterators compressedIters) {

mutations.getMutations().forEach((keyExtent, mutationList) -> {
var tcondMutaions = new ArrayList<TConditionalMutation>();
var tcondMutaions = new ArrayList<TConditionalMutation>(mutationList.size());

for (var cm : mutationList) {
var id = cmid.longValue();
Expand Down
47 changes: 29 additions & 18 deletions core/src/main/java/org/apache/accumulo/core/clientImpl/InstanceOperationsImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,28 +240,30 @@ public List<String> getManagerLocations() {
@Override
@Deprecated(since = "4.0.0")
public Set<String> getCompactors() {
Set<String> results = new HashSet<>();
context.getServerPaths().getCompactor(ResourceGroupPredicate.ANY, AddressSelector.all(), true)
.forEach(t -> results.add(t.getServer()));
var compactors = context.getServerPaths().getCompactor(ResourceGroupPredicate.ANY,
AddressSelector.all(), true);
Set<String> results = new HashSet<>(compactors.size(), 1.0f);
compactors.forEach(t -> results.add(t.getServer()));
return results;
}

@Override
@Deprecated(since = "4.0.0")
public Set<String> getScanServers() {
Set<String> results = new HashSet<>();
context.getServerPaths().getScanServer(ResourceGroupPredicate.ANY, AddressSelector.all(), true)
.forEach(t -> results.add(t.getServer()));
var sservers = context.getServerPaths().getScanServer(ResourceGroupPredicate.ANY,
AddressSelector.all(), true);
Set<String> results = new HashSet<>(sservers.size(), 1.f);
sservers.forEach(t -> results.add(t.getServer()));
return results;
}

@Override
@Deprecated(since = "4.0.0")
public List<String> getTabletServers() {
List<String> results = new ArrayList<>();
context.getServerPaths()
.getTabletServer(ResourceGroupPredicate.ANY, AddressSelector.all(), true)
.forEach(t -> results.add(t.getServer()));
var tserverLocks = context.getServerPaths().getTabletServer(ResourceGroupPredicate.ANY,
AddressSelector.all(), true);
List<String> results = new ArrayList<>(tserverLocks.size());
tserverLocks.forEach(t -> results.add(t.getServer()));
return results;
}

Expand Down Expand Up @@ -556,18 +558,22 @@ public Set<ServerId> getServers(ServerId.Type type,
private Set<ServerId> getServers(ServerId.Type type,
Predicate<ResourceGroupId> resourceGroupPredicate, AddressSelector addressSelector) {

final Set<ServerId> results = new HashSet<>();
final Set<ServerId> results;

switch (type) {
case COMPACTOR:
context.getServerPaths().getCompactor(resourceGroupPredicate::test, addressSelector, true)
.forEach(c -> results.add(createServerId(type, c)));
var compactors = context.getServerPaths().getCompactor(resourceGroupPredicate::test,
addressSelector, true);
results = new HashSet<>(compactors.size(), 1.0f);
compactors.forEach(c -> results.add(createServerId(type, c)));
break;
case MANAGER:
results = new HashSet<>();
context.getServerPaths().getAssistantManagers(addressSelector, true)
.forEach(s -> results.add(createServerId(type, s)));
break;
case MONITOR:
results = new HashSet<>();
ServiceLockPath mon = context.getServerPaths().getMonitor(true);
if (mon != null) {
Optional<ServiceLockData> sld = context.getZooCache().getLockData(mon);
Expand All @@ -582,6 +588,7 @@ private Set<ServerId> getServers(ServerId.Type type,
}
break;
case GARBAGE_COLLECTOR:
results = new HashSet<>();
ServiceLockPath gc = context.getServerPaths().getGarbageCollector(true);
if (gc != null) {
Optional<ServiceLockData> sld = context.getZooCache().getLockData(gc);
Expand All @@ -596,15 +603,19 @@ private Set<ServerId> getServers(ServerId.Type type,
}
break;
case SCAN_SERVER:
context.getServerPaths().getScanServer(resourceGroupPredicate::test, addressSelector, true)
.forEach(s -> results.add(createServerId(type, s)));
var sservers = context.getServerPaths().getScanServer(resourceGroupPredicate::test,
addressSelector, true);
results = new HashSet<>(sservers.size(), 1.0f);
sservers.forEach(s -> results.add(createServerId(type, s)));
break;
case TABLET_SERVER:
context.getServerPaths()
.getTabletServer(resourceGroupPredicate::test, addressSelector, true)
.forEach(t -> results.add(createServerId(type, t)));
var tservers = context.getServerPaths().getTabletServer(resourceGroupPredicate::test,
addressSelector, true);
results = new HashSet<>(tservers.size(), 1.0f);
tservers.forEach(t -> results.add(createServerId(type, t)));
break;
default:
results = new HashSet<>();
break;
}

Expand Down
1 change: 1 addition & 0 deletions core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,7 @@ private SortedKeyValueIterator<Key,Value> createIterator(KeyExtent extent,
if (scannerSamplerConfigImpl != null && !scannerSamplerConfigImpl.equals(samplerConfImpl)) {
throw new SampleNotPresentException();
}
readers.ensureCapacity(absFiles.size());
for (StoredTabletFile file : absFiles) {
var cs = CryptoFactoryLoader.getServiceForClientWithTable(systemConf, tableConf, tableId);
FileSystem fs = VolumeConfiguration.fileSystemForPath(file.getNormalizedPathStr(), conf);
Expand Down
3 changes: 2 additions & 1 deletion core/src/main/java/org/apache/accumulo/core/clientImpl/RootClientTabletCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.concurrent.TimeUnit.MILLISECONDS;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
Expand Down Expand Up @@ -67,7 +68,7 @@ public class RootClientTabletCache extends ClientTabletCache {

@Override
public <T extends Mutation> void binMutations(ClientContext context, List<T> mutations,
Map<String,TabletServerMutations<T>> binnedMutations, List<T> failures) {
Map<String,TabletServerMutations<T>> binnedMutations, ArrayList<T> failures) {
CachedTablet rootCachedTablet = getRootTabletLocation(context);
if (rootCachedTablet != null && rootCachedTablet.getTserverLocation().isPresent()) {
var tsm = new TabletServerMutations<T>(rootCachedTablet.getTserverSession().orElseThrow());
Expand Down
3 changes: 2 additions & 1 deletion core/src/main/java/org/apache/accumulo/core/clientImpl/SyncingClientTabletCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
*/
package org.apache.accumulo.core.clientImpl;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -76,7 +77,7 @@ public CachedTablet findTablet(ClientContext context, Text row, boolean skipRow,

@Override
public <T extends Mutation> void binMutations(ClientContext context, List<T> mutations,
Map<String,TabletServerMutations<T>> binnedMutations, List<T> failures)
Map<String,TabletServerMutations<T>> binnedMutations, ArrayList<T> failures)
throws AccumuloException, AccumuloSecurityException, TableNotFoundException,
InvalidTabletHostingRequestException {
syncLocator().binMutations(context, mutations, binnedMutations, failures);
Expand Down
6 changes: 3 additions & 3 deletions core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -509,7 +509,7 @@ public void putSplits(String tableName, SortedMap<Text,TabletMergeability> split
Map<KeyExtent,List<SplitMergeability>> tabletSplits = splitsToTablets.newSplits;
Map<KeyExtent,TabletMergeability> existingSplits = splitsToTablets.existingSplits;

List<CompletableFuture<Void>> futures = new ArrayList<>();
List<CompletableFuture<Void>> futures = new ArrayList<>(existingSplits.size());

// Handle existing updates
if (!existingSplits.isEmpty()) {
Expand Down Expand Up @@ -881,7 +881,7 @@ public void flush(String tableName, Text start, Text end, boolean wait)
@Override
public void compact(String tableName, Text start, Text end, boolean flush, boolean wait)
throws AccumuloSecurityException, TableNotFoundException, AccumuloException {
compact(tableName, start, end, new ArrayList<>(), flush, wait);
compact(tableName, start, end, List.of(), flush, wait);
}

@Override
Expand Down Expand Up @@ -1616,7 +1616,7 @@ public List<DiskUsage> getDiskUsage(Set<String> tableNames)
}
}

List<DiskUsage> finalUsages = new ArrayList<>();
List<DiskUsage> finalUsages = new ArrayList<>(diskUsages.size());
for (TDiskUsage diskUsage : diskUsages) {
finalUsages.add(new DiskUsage(new TreeSet<>(diskUsage.getTables()), diskUsage.getUsage()));
}
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/apache/accumulo/core/clientImpl/TabletInformationCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ public static Stream<TabletInformation> getTabletInformation(ClientContext conte
}
});

List<Stream<TabletInformation>> tabletStreams = new ArrayList<>();
List<Stream<TabletInformation>> tabletStreams = new ArrayList<>(mergedRanges.size());
for (RowRange rowRange : mergedRanges) {

Text startRow = rowRange.getLowerBound();
Expand Down
12 changes: 6 additions & 6 deletions core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchReaderIterator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,9 +323,9 @@ private ScanServerData binRanges(ClientTabletCache clientTabletCache, List<Range

// truncate the ranges to within the tablets... this makes it easier to know what work
// needs to be redone when failures occurs and tablets have merged or split
Map<String,Map<KeyExtent,List<Range>>> binnedRanges2 = new HashMap<>();
Map<String,Map<KeyExtent,List<Range>>> binnedRanges2 = new HashMap<>(binnedRanges.size(), 1.0f);
for (Entry<String,Map<KeyExtent,List<Range>>> entry : binnedRanges.entrySet()) {
Map<KeyExtent,List<Range>> tabletMap = new HashMap<>();
Map<KeyExtent,List<Range>> tabletMap = new HashMap<>(entry.getValue().size(), 1.0f);
binnedRanges2.put(entry.getKey(), tabletMap);
for (Entry<KeyExtent,List<Range>> tabletRanges : entry.getValue().entrySet()) {
Range tabletRange = tabletRanges.getKey().toDataRange();
Expand Down Expand Up @@ -663,10 +663,10 @@ private ScanServerData binRangesForScanServers(ClientTabletCache clientTabletCac
AccumuloSecurityException, InvalidTabletHostingRequestException {
ScanServerSelector ecsm = context.getScanServerSelector();

Map<KeyExtent,String> extentToTserverMap = new HashMap<>();
Map<KeyExtent,List<Range>> extentToRangesMap = new HashMap<>();
Map<KeyExtent,String> extentToTserverMap = new HashMap<>(ranges.size());
Map<KeyExtent,List<Range>> extentToRangesMap = new HashMap<>(ranges.size());

Set<TabletIdImpl> tabletIds = new HashSet<>();
Set<TabletIdImpl> tabletIds = new HashSet<>(ranges.size());

List<Range> failures = clientTabletCache.findTablets(context, ranges, (cachedTablet, range) -> {
if (cachedTablet.getTserverLocation().isPresent()) {
Expand Down Expand Up @@ -712,7 +712,7 @@ public <T> Optional<T> waitUntil(Supplier<Optional<T>> condition, Duration maxWa

var actions = ecsm.selectServers(params);

Map<String,BatchAttemptReporter> reporters = new HashMap<>();
Map<String,BatchAttemptReporter> reporters = new HashMap<>(tabletIds.size());

failures = new ArrayList<>();

Expand Down
6 changes: 3 additions & 3 deletions core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportPool.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,9 +144,9 @@ public TTransport getTransport(ThriftClientTypes<?> type, HostAndPort location,

public Pair<String,TTransport> getAnyCachedTransport(ThriftClientTypes<?> type) {

final List<ThriftTransportKey> serversSet = new ArrayList<>();

for (ThriftTransportKey ttk : connectionPool.getThriftTransportKeys()) {
var keys = connectionPool.getThriftTransportKeys();
final List<ThriftTransportKey> serversSet = new ArrayList<>(keys.size());
for (ThriftTransportKey ttk : keys) {
if (ttk.getType().equals(type)) {
serversSet.add(ttk);
}
Expand Down
3 changes: 2 additions & 1 deletion core/src/main/java/org/apache/accumulo/core/clientImpl/TimeoutClientTabletCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
*/
package org.apache.accumulo.core.clientImpl;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;

Expand Down Expand Up @@ -84,7 +85,7 @@ public CachedTablet findTablet(ClientContext context, Text row, boolean skipRow,

@Override
public <T extends Mutation> void binMutations(ClientContext context, List<T> mutations,
Map<String,TabletServerMutations<T>> binnedMutations, List<T> failures)
Map<String,TabletServerMutations<T>> binnedMutations, ArrayList<T> failures)
throws AccumuloException, AccumuloSecurityException, TableNotFoundException,
InvalidTabletHostingRequestException {
try {
Expand Down
7 changes: 5 additions & 2 deletions core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/Bulk.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,13 +167,16 @@ public int hashCode() {
}

public static class Files implements Iterable<FileInfo> {
final Map<String,FileInfo> files = new HashMap<>();
final Map<String,FileInfo> files;

public Files(Collection<FileInfo> files) {
this.files = new HashMap<>(files.size(), 1.0f);

@ddanielr ddanielr Jul 8, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why the change from the default 0.75 load factor to 1.0?

@ctubbsii ctubbsii Jul 9, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the 1.0 load factor prevents resizing until you try to exceed the current size. So this avoids an unnecessary resize as we're filling it up with a known quantity.

@dlmarion dlmarion Jul 9, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Exactly. If we know the size before-hand, then we can set the collection size appropriately so that it won't resize.

files.forEach(this::add);
}

public Files() {}
public Files() {
this.files = new HashMap<>();
}

public void add(FileInfo fi) {
if (files.putIfAbsent(fi.name, fi) != null) {
Expand Down
Loading