Skip to content

feat(bigquery-jdbc): add EnableProjectDiscovery connection property for metadata methods #13344

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
keshavdandeva wants to merge 8 commits into
base: main
Choose a base branch
from
Open

feat(bigquery-jdbc): add EnableProjectDiscovery connection property for metadata methods #13344

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8f75b30
feat: add `EnableProjectDiscovery` connection property for metadata m…
keshavdandeva Jun 3, 2026
81e80a0
address pr feedback
keshavdandeva Jun 3, 2026
86d62b1
address pr feedback
keshavdandeva Jun 3, 2026
f440367
chore: address pr feedback
keshavdandeva Jun 5, 2026
c45cec9
Merge branch 'main' into jdbc/get-all-projects
keshavdandeva Jun 18, 2026
3387cf5
use the SDK `listProjects` API with connection-scoped caching
keshavdandeva Jun 18, 2026
b0aa5cd
Merge branch 'main' into jdbc/get-all-projects
keshavdandeva Jun 18, 2026
47a47ac
Merge branch 'main' into jdbc/get-all-projects
keshavdandeva Jun 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 41 additions & 0 deletions java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,21 @@
import com.google.api.gax.core.CredentialsProvider;
import com.google.api.gax.core.FixedCredentialsProvider;
import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider;
import com.google.api.gax.paging.Page;
import com.google.api.gax.retrying.RetrySettings;
import com.google.api.gax.rpc.FixedHeaderProvider;
import com.google.api.gax.rpc.HeaderProvider;
import com.google.api.gax.rpc.TransportChannelProvider;
import com.google.auth.Credentials;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQuery.ProjectListOption;
import com.google.cloud.bigquery.BigQueryException;
import com.google.cloud.bigquery.BigQueryOptions;
import com.google.cloud.bigquery.ConnectionProperty;
import com.google.cloud.bigquery.DatasetId;
import com.google.cloud.bigquery.Job;
import com.google.cloud.bigquery.JobInfo;
import com.google.cloud.bigquery.Project;
import com.google.cloud.bigquery.QueryJobConfiguration;
import com.google.cloud.bigquery.QueryJobConfiguration.JobCreationMode;
import com.google.cloud.bigquery.exception.BigQueryJdbcException;
Expand All @@ -41,6 +44,7 @@
import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient;
import com.google.cloud.bigquery.storage.v1.BigQueryWriteSettings;
import com.google.cloud.http.HttpTransportOptions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSortedSet;
import java.io.IOException;
import java.io.InputStream;
Expand Down Expand Up @@ -121,6 +125,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
BigQueryJdbcUrlUtility.SWA_APPEND_ROW_COUNT_PROPERTY_NAME,
BigQueryJdbcUrlUtility.SWA_ACTIVATION_ROW_COUNT_PROPERTY_NAME,
BigQueryJdbcUrlUtility.FILTER_TABLES_ON_DEFAULT_DATASET_PROPERTY_NAME,
BigQueryJdbcUrlUtility.ENABLE_PROJECT_DISCOVERY_PROPERTY_NAME,
BigQueryJdbcUrlUtility.REQUEST_GOOGLE_DRIVE_SCOPE_PROPERTY_NAME,
BigQueryJdbcUrlUtility.SSL_TRUST_STORE_PROPERTY_NAME,
BigQueryJdbcUrlUtility.MAX_BYTES_BILLED_PROPERTY_NAME,
Expand Down Expand Up @@ -170,6 +175,8 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
int highThroughputMinTableSize;
int highThroughputActivationRatio;
boolean enableSession;
boolean enableProjectDiscovery;
private List<String> discoveredProjectsCache;
boolean unsupportedHTAPIFallback;
boolean useQueryCache;
String queryDialect;
Expand Down Expand Up @@ -338,6 +345,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
this.additionalProjects = ds.getAdditionalProjects();

this.filterTablesOnDefaultDataset = ds.getFilterTablesOnDefaultDataset();
this.enableProjectDiscovery = ds.getEnableProjectDiscovery();
this.requestGoogleDriveScope = ds.getRequestGoogleDriveScope();
this.metadataFetchThreadCount = ds.getMetadataFetchThreadCount();
this.requestReason = ds.getRequestReason();
Expand Down Expand Up @@ -1312,6 +1320,39 @@ private boolean checkIsReadOnlyTokenUsed(Map<String, String> authProps) {
return false;
}

public boolean isEnableProjectDiscovery() {
return this.enableProjectDiscovery;
}

public synchronized List<String> getDiscoveredProjects() {
if (this.discoveredProjectsCache != null) {
return this.discoveredProjectsCache;
}

try {
BigQuery bigQuery = getBigQuery();
List<String> projects = new ArrayList<>();
Page<Project> projectPage =
bigQuery.listProjects(ProjectListOption.pageSize(getMaxResults()));
for (Project project : projectPage.iterateAll()) {
projects.add(project.getProjectId());
}
this.discoveredProjectsCache = ImmutableList.copyOf(projects);
} catch (BigQueryException e) {
LOG.warning(e, "Failed to list all accessible projects due to BigQuery error.");
Comment thread
keshavdandeva marked this conversation as resolved.
int statusCode = e.getCode();
// Only cache empty list for non-transient auth/permission errors (400, 401, 403)
if (statusCode == 400 || statusCode == 401 || statusCode == 403) {
this.discoveredProjectsCache = ImmutableList.of();
}
return ImmutableList.of();
} catch (Exception e) {
LOG.warning(e, "Failed to list all accessible projects, falling back to connection default.");
Comment thread
keshavdandeva marked this conversation as resolved.
return ImmutableList.of();
}
Comment thread
keshavdandeva marked this conversation as resolved.
return this.discoveredProjectsCache;
}
Comment thread
keshavdandeva marked this conversation as resolved.

@Override
public <T> T unwrap(Class<T> iface) throws SQLException {
if (iface.isInstance(this)) {
Expand Down
4 changes: 4 additions & 0 deletions ...-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5199,6 +5199,10 @@ private List<String> getAccessibleCatalogNames() {
}
}

if (this.connection.isEnableProjectDiscovery()) {
accessibleCatalogs.addAll(this.connection.getDiscoveredProjects());
}

List<String> sortedCatalogs = new ArrayList<>(accessibleCatalogs);
Collections.sort(sortedCatalogs);
return sortedCatalogs;
Expand Down
9 changes: 9 additions & 0 deletions java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,8 @@ protected boolean removeEldestEntry(Map.Entry<String, Map<String, String>> eldes
static final String FILTER_TABLES_ON_DEFAULT_DATASET_PROPERTY_NAME =
"FilterTablesOnDefaultDataset";
static final boolean DEFAULT_FILTER_TABLES_ON_DEFAULT_DATASET_VALUE = false;
static final String ENABLE_PROJECT_DISCOVERY_PROPERTY_NAME = "EnableProjectDiscovery";
static final boolean DEFAULT_ENABLE_PROJECT_DISCOVERY_VALUE = false;
static final String REQUEST_GOOGLE_DRIVE_SCOPE_PROPERTY_NAME = "RequestGoogleDriveScope";
static final String SSL_TRUST_STORE_PROPERTY_NAME = "SSLTrustStore";
static final String SSL_TRUST_STORE_PWD_PROPERTY_NAME = "SSLTrustStorePwd";
Expand Down Expand Up @@ -577,6 +579,13 @@ protected boolean removeEldestEntry(Map.Entry<String, Map<String, String>> eldes
.setDefaultValue(
String.valueOf(DEFAULT_FILTER_TABLES_ON_DEFAULT_DATASET_VALUE))
.build(),
BigQueryConnectionProperty.newBuilder()
.setName(ENABLE_PROJECT_DISCOVERY_PROPERTY_NAME)
.setDescription(
"Enables or disables automatic discovery of all accessible Google Cloud projects. "
+ "When disabled, only the default ProjectId and AdditionalProjects are listed as catalogs.")
.setDefaultValue(String.valueOf(DEFAULT_ENABLE_PROJECT_DISCOVERY_VALUE))
.build(),
BigQueryConnectionProperty.newBuilder()
.setName(REQUEST_GOOGLE_DRIVE_SCOPE_PROPERTY_NAME)
.setDescription(
Expand Down
22 changes: 22 additions & 0 deletions java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ public class DataSource implements javax.sql.DataSource {
private Boolean enableWriteAPI;
private String additionalProjects;
private Boolean filterTablesOnDefaultDataset;
private Boolean enableProjectDiscovery;
private Integer requestGoogleDriveScope;
private Integer metadataFetchThreadCount;
private String sslTrustStorePath;
Expand Down Expand Up @@ -242,6 +243,12 @@ public class DataSource implements javax.sql.DataSource {
BigQueryJdbcUrlUtility.convertIntToBoolean(
val,
BigQueryJdbcUrlUtility.FILTER_TABLES_ON_DEFAULT_DATASET_PROPERTY_NAME)))
.put(
BigQueryJdbcUrlUtility.ENABLE_PROJECT_DISCOVERY_PROPERTY_NAME,
(ds, val) ->
ds.setEnableProjectDiscovery(
BigQueryJdbcUrlUtility.convertIntToBoolean(
val, BigQueryJdbcUrlUtility.ENABLE_PROJECT_DISCOVERY_PROPERTY_NAME)))
.put(
BigQueryJdbcUrlUtility.REQUEST_GOOGLE_DRIVE_SCOPE_PROPERTY_NAME,
(ds, val) -> ds.setRequestGoogleDriveScope(Integer.parseInt(val)))
Expand Down Expand Up @@ -555,6 +562,11 @@ Properties createProperties() {
BigQueryJdbcUrlUtility.FILTER_TABLES_ON_DEFAULT_DATASET_PROPERTY_NAME,
String.valueOf(this.filterTablesOnDefaultDataset));
}
if (this.enableProjectDiscovery != null) {
connectionProperties.setProperty(
BigQueryJdbcUrlUtility.ENABLE_PROJECT_DISCOVERY_PROPERTY_NAME,
String.valueOf(this.enableProjectDiscovery));
}
if (this.requestGoogleDriveScope != null) {
connectionProperties.setProperty(
BigQueryJdbcUrlUtility.REQUEST_GOOGLE_DRIVE_SCOPE_PROPERTY_NAME,
Expand Down Expand Up @@ -1060,6 +1072,16 @@ public void setFilterTablesOnDefaultDataset(Boolean filterTablesOnDefaultDataset
this.filterTablesOnDefaultDataset = filterTablesOnDefaultDataset;
}

public Boolean getEnableProjectDiscovery() {
return enableProjectDiscovery != null
? enableProjectDiscovery
: BigQueryJdbcUrlUtility.DEFAULT_ENABLE_PROJECT_DISCOVERY_VALUE;
}

public void setEnableProjectDiscovery(Boolean enableProjectDiscovery) {
this.enableProjectDiscovery = enableProjectDiscovery;
}

public Integer getRequestGoogleDriveScope() {
return requestGoogleDriveScope != null
? requestGoogleDriveScope
Expand Down
75 changes: 75 additions & 0 deletions java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,18 +23,28 @@
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider;
import com.google.api.gax.paging.Page;
import com.google.api.gax.rpc.HeaderProvider;
import com.google.api.gax.rpc.TransportChannelProvider;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQueryException;
import com.google.cloud.bigquery.Project;
import com.google.cloud.bigquery.QueryJobConfiguration.JobCreationMode;
import com.google.cloud.bigquery.exception.BigQueryJdbcException;
import com.google.cloud.bigquery.storage.v1.BigQueryReadClient;
import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient;
import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.Properties;
import java.util.logging.Level;
Expand Down Expand Up @@ -519,4 +529,69 @@ public void testWrapperMethods() throws Exception {
assertTrue(e.getMessage().contains("Cannot unwrap to java.sql.Statement"));
}
}

@Test
public void testGetDiscoveredProjects_Success() throws Exception {
try (BigQueryConnection connection = new BigQueryConnection(BASE_URL)) {
BigQuery mockBigQuery = mock(BigQuery.class);
connection.bigQuery = mockBigQuery;

Page<Project> mockPage = mock(Page.class);
Project project1 = mock(Project.class);
when(project1.getProjectId()).thenReturn("discovered-p1");
Project project2 = mock(Project.class);
when(project2.getProjectId()).thenReturn("discovered-p2");

when(mockPage.iterateAll()).thenReturn(Arrays.asList(project1, project2));
when(mockBigQuery.listProjects(any(BigQuery.ProjectListOption.class))).thenReturn(mockPage);

List<String> discovered = connection.getDiscoveredProjects();
assertEquals(Arrays.asList("discovered-p1", "discovered-p2"), discovered);

// Verify caching: second call should not invoke listProjects again
List<String> discoveredCached = connection.getDiscoveredProjects();
assertSame(discovered, discoveredCached);
verify(mockBigQuery, times(1)).listProjects(any(BigQuery.ProjectListOption.class));
}
}

@Test
public void testGetDiscoveredProjects_NonTransientError() throws Exception {
try (BigQueryConnection connection = new BigQueryConnection(BASE_URL)) {
BigQuery mockBigQuery = mock(BigQuery.class);
connection.bigQuery = mockBigQuery;

// 403 Forbidden (Non-transient error)
BigQueryException exception = new BigQueryException(403, "Access Denied");
when(mockBigQuery.listProjects(any(BigQuery.ProjectListOption.class))).thenThrow(exception);

List<String> discovered = connection.getDiscoveredProjects();
assertTrue(discovered.isEmpty());

// Verify that it caches the empty list for 403, so it does not retry.
List<String> discoveredCached = connection.getDiscoveredProjects();
assertTrue(discoveredCached.isEmpty());
verify(mockBigQuery, times(1)).listProjects(any(BigQuery.ProjectListOption.class));
}
}

@Test
public void testGetDiscoveredProjects_TransientError() throws Exception {
try (BigQueryConnection connection = new BigQueryConnection(BASE_URL)) {
BigQuery mockBigQuery = mock(BigQuery.class);
connection.bigQuery = mockBigQuery;

// 500 Internal Error (Transient error, should not cache empty list)
BigQueryException exception = new BigQueryException(500, "Internal Server Error");
when(mockBigQuery.listProjects(any(BigQuery.ProjectListOption.class))).thenThrow(exception);

List<String> discovered = connection.getDiscoveredProjects();
assertTrue(discovered.isEmpty());

// Since it's a transient error, it should NOT cache the empty list and should try again.
List<String> discoveredCached = connection.getDiscoveredProjects();
assertTrue(discoveredCached.isEmpty());
verify(mockBigQuery, times(2)).listProjects(any(BigQuery.ProjectListOption.class));
}
}
}
Loading
Loading