Skip to content

Modified VisibilityFilter to handle multiple Authorizations objects #6402

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dlmarion wants to merge 2 commits into
base: main
Choose a base branch
from
+139 −14
Open

Modified VisibilityFilter to handle multiple Authorizations objects #6402

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
62cf5e0
Added MultiAuthVisibilityFilter to user iterator package
dlmarion May 29, 2026
8cc6649
Merged multi-auth feature into user VisibilityFilter
dlmarion Jun 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions core/src/main/java/org/apache/accumulo/core/clientImpl/access/BytesAccess.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@

import static java.nio.charset.StandardCharsets.ISO_8859_1;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
Expand Down Expand Up @@ -93,6 +95,19 @@ public boolean canAccess(byte[] expression) {
}
}

public static BytesEvaluator newEvaluator(Collection<Authorizations> authsSet) {
Collection<Set<String>> convertedAuths = new ArrayList<>();
for (Authorizations auths : authsSet) {
List<byte[]> bytesAuths = auths.getAuthorizations();
Set<String> stringAuths = new HashSet<>(bytesAuths.size());
for (var auth : bytesAuths) {
stringAuths.add(new String(auth, ISO_8859_1));

@ctubbsii ctubbsii Jul 9, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A comment is warranted here to explain why ISO-8859-1 is used.

}
convertedAuths.add(stringAuths);
}
return new BytesEvaluator(ACCESS.newEvaluator(convertedAuths));
}

public static BytesEvaluator newEvaluator(Authorizations auths) {
List<byte[]> bytesAuths = auths.getAuthorizations();
Set<String> stringAuths = new HashSet<>(bytesAuths.size());
Expand Down
52 changes: 42 additions & 10 deletions core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@
import static java.nio.charset.StandardCharsets.UTF_8;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Map;
import java.util.Objects;

import org.apache.accumulo.access.InvalidAccessExpressionException;
import org.apache.accumulo.core.client.IteratorSetting;
Expand All @@ -39,6 +42,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Preconditions;

/**
* A SortedKeyValueIterator that filters based on ColumnVisibility.
*/
Expand All @@ -50,7 +55,8 @@ public class VisibilityFilter extends Filter implements OptionDescriber {

private static final Logger log = LoggerFactory.getLogger(VisibilityFilter.class);

private static final String AUTHS = "auths";
private static final String NUM_AUTHS = "numAuths";
private static final String AUTH_PREFIX = "auth_";
private static final String FILTER_INVALID_ONLY = "filterInvalid";

private boolean filterInvalid;
Expand All @@ -63,11 +69,26 @@ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> op
this.filterInvalid = Boolean.parseBoolean(options.get(FILTER_INVALID_ONLY));

if (!filterInvalid) {
String auths = options.get(AUTHS);
Authorizations authObj = auths == null || auths.isEmpty() ? new Authorizations()
: new Authorizations(auths.getBytes(UTF_8));

this.accessEvaluator = BytesAccess.newEvaluator(authObj);
String numAuthsParameter = options.get(NUM_AUTHS);
Objects.requireNonNull(numAuthsParameter, "NUM_AUTHS option not set.");
int numAuths = Integer.parseInt(numAuthsParameter);
Preconditions.checkArgument(numAuths >= 0, NUM_AUTHS + " must be a positive integer");

Collection<Authorizations> authSet = new ArrayList<>();
if (numAuths == 0) {
authSet.add(new Authorizations());
} else {
for (int idx = 0; idx < numAuths; idx++) {
String auths = options.get(AUTH_PREFIX + idx);
Authorizations authObj = auths == null || auths.isEmpty() ? new Authorizations()
: new Authorizations(auths.getBytes(UTF_8));
authSet.add(authObj);
}
String auths = options.get(AUTH_PREFIX + numAuths);
Preconditions.checkArgument(auths == null,
"NUM_AUTHS is set incorrectly, should be at least: " + NUM_AUTHS + " = " + 1);
}
this.accessEvaluator = BytesAccess.newEvaluator(authSet);
}
this.cache = new LRUMap<>(1000);
}
Expand Down Expand Up @@ -136,14 +157,25 @@ public IteratorOptions describeOptions() {
io.addNamedOption(FILTER_INVALID_ONLY,
"if 'true', the iterator is instructed to ignore the authorizations and"
+ " only filter invalid visibility labels (default: false)");
io.addNamedOption(AUTHS,
"the serialized set of authorizations to filter against (default: empty"
+ " string, accepts only entries visible by all)");
Comment on lines -139 to -141

@ctubbsii ctubbsii Jul 9, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can't remove this option. The configuration parameter is the main API for this class. Removing or renaming it breaks it.

io.addNamedOption(NUM_AUTHS,
"The number of serialized authorizations to filter against (default 0)");
io.addUnnamedOption(AUTH_PREFIX
+ "N, where the value is a serialized set of authorizations. N must be between zero and NUM_AUTHS.");
Comment on lines +160 to +163

@ctubbsii ctubbsii Jul 9, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure these descriptions really communicate how these are used. Is the idea that you specify some config like:

numAuths = 3
auth_0 = a,b,c
auth_1 = c,b,d
auth_2 = a,c,z

And then it does something like:

if (entry.evaluateUsing(auth_0) && entry..evaluateUsing(auth_1) && entry..evaluateUsing(auth_2)) {
  return entry;
}

Is the idea to replace multiple executions of the iterator with a single one?

Instead, I think it might make sense to have a config that preserves the existing behavior, and extends the existing config:

auths = a,b,c;c,b,d;a,c,z

This is simpler, and backwards compatible. The description can be extended to say: "if multiple serialized sets are configured, separated by semi-colons, then entries are accepted only if they would be accepted by each of the sets independently"

return io;
}

public static void setAuthorizations(IteratorSetting setting, Authorizations auths) {
setting.addOption(AUTHS, auths.serialize());
setting.addOption(NUM_AUTHS, "1");
setting.addOption(AUTH_PREFIX + 0, auths.serialize());
}

public static void setAuthorizations(IteratorSetting setting, Collection<Authorizations> auths) {
setting.addOption(NUM_AUTHS, Integer.toString(auths.size()));
int idx = 0;
for (Authorizations auth : auths) {
setting.addOption(AUTH_PREFIX + idx, auth.serialize());
idx++;
}
}

public static void filterInvalidLabelsOnly(IteratorSetting setting, boolean featureEnabled) {
Expand Down
41 changes: 37 additions & 4 deletions core/src/test/java/org/apache/accumulo/core/iterators/user/VisibilityFilterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
Expand Down Expand Up @@ -155,6 +155,22 @@ public void testAllowAuthorizedLabelsOnly() throws IOException {
verify(source, 1500, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1000);
}

@Test
public void testMulitAllowAuthorizedLabelsOnly() throws IOException {
IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class);
VisibilityFilter.setAuthorizations(is,
List.of(new Authorizations("abc"), new Authorizations("def")));

TreeMap<Key,Value> source = createSourceWithHiddenData(1, 2);
verify(source, 3, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1);

source = createSourceWithHiddenData(30, 500);
verify(source, 530, is.getOptions(), GOOD, GOOD, GOOD_VIS, 30);

source = createSourceWithHiddenData(1000, 500);
verify(source, 1500, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1000);
}

@Test
public void testAllowUnauthorizedLabelsOnly() throws IOException {
IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class);
Expand All @@ -171,6 +187,23 @@ public void testAllowUnauthorizedLabelsOnly() throws IOException {
verify(source, 1500, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500);
}

@Test
public void testMultiAllowUnauthorizedLabelsOnly() throws IOException {
IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class);
VisibilityFilter.setNegate(is, true);
VisibilityFilter.setAuthorizations(is,
List.of(new Authorizations("abc"), new Authorizations("def")));

TreeMap<Key,Value> source = createSourceWithHiddenData(1, 2);
verify(source, 3, is.getOptions(), BAD, BAD, HIDDEN_VIS, 2);

source = createSourceWithHiddenData(30, 500);
verify(source, 530, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500);

source = createSourceWithHiddenData(1000, 500);
verify(source, 1500, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500);
}

@Test
public void testNoLabels() throws IOException {
IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class);
Expand Down Expand Up @@ -203,7 +236,7 @@ public void testFilterUnauthorizedAndBad() throws IOException {

@Test
public void testCommaSeparatedAuthorizations() throws IOException {
Map<String,String> options = Collections.singletonMap("auths", "x,def,y");
Map<String,String> options = Map.of("numAuths", "1", "auth_0", "x,def,y");

TreeMap<Key,Value> source = createSourceWithHiddenData(1, 2);
verify(source, 3, options, GOOD, GOOD, GOOD_VIS, 1);
Expand All @@ -218,7 +251,7 @@ public void testCommaSeparatedAuthorizations() throws IOException {
@Test
public void testSerializedAuthorizations() throws IOException {
Map<String,String> options =
Collections.singletonMap("auths", new Authorizations("x", "def", "y").serialize());
Map.of("numAuths", "1", "auth_0", new Authorizations("x", "def", "y").serialize());

TreeMap<Key,Value> source = createSourceWithHiddenData(1, 2);
verify(source, 3, options, GOOD, GOOD, GOOD_VIS, 1);
Expand All @@ -240,7 +273,7 @@ public void testStaticConfigurators() {
Map<String,String> opts = is.getOptions();
assertEquals("false", opts.get("filterInvalid"));
assertEquals("true", opts.get("negate"));
assertEquals(new Authorizations("abc", "def").serialize(), opts.get("auths"));
assertEquals(new Authorizations("abc", "def").serialize(), opts.get("auth_0"));
}

@Test
Expand Down
45 changes: 45 additions & 0 deletions test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,14 @@
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.user.VisibilityFilter;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.core.util.ByteArraySet;
Expand Down Expand Up @@ -92,6 +94,7 @@ public void run() throws Exception {

insertData(c, table);
queryData(c, table);
queryDataMultiAuth(c, table);
deleteData(c, table);

insertDefaultData(c, table2);
Expand Down Expand Up @@ -222,6 +225,48 @@ private void queryData(AccumuloClient c, String tableName) throws Exception {
queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss(), expected);
}

/**
* Configures Scanners with the users default authorizations, then it adds a
* MultiAuthVisibilityFilter with different sets of Authorizations
*/
private void queryDataMultiAuth(AccumuloClient c, String tableName) throws Exception {

c.securityOperations().changeUserAuthorizations(getAdminPrincipal(),
new Authorizations("A", "B", "FOO", "L", "M", "Z"));

Authorizations userAuths = c.securityOperations().getUserAuthorizations(c.whoami());

Set<String> expectedUserAuths =
Set.of("v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13");
try (Scanner scanner = c.createScanner(tableName, userAuths);
BatchScanner bs = c.createBatchScanner(tableName, userAuths, 3)) {
verify(scanner.iterator(), expectedUserAuths.toArray(new String[] {}));

bs.setRanges(Collections.singleton(new Range()));
verify(bs.iterator(), expectedUserAuths.toArray(new String[] {}));
}

Authorizations entity1 = new Authorizations("A", "B", "FOO", "L", "M");
Authorizations entity2 = new Authorizations("B", "FOO", "Z");
// should only see entries with no column visibility, B and/or FOO
Set<String> expectedAuths = Set.of("v1", "v3", "v11");

IteratorSetting is = new IteratorSetting(100, "userAuths", VisibilityFilter.class);
VisibilityFilter.setAuthorizations(is, Set.of(entity1, entity2));

try (Scanner scanner = c.createScanner(tableName, userAuths);
BatchScanner bs = c.createBatchScanner(tableName, userAuths, 3)) {

scanner.addScanIterator(is);
verify(scanner.iterator(), expectedAuths.toArray(new String[] {}));

bs.setRanges(Collections.singleton(new Range()));
bs.addScanIterator(is);
verify(bs.iterator(), expectedAuths.toArray(new String[] {}));
}

}

private void queryData(AccumuloClient c, String tableName, Set<String> allAuths,
Set<String> userAuths, Map<Set<String>,Set<String>> expected) throws Exception {

Expand Down