Skip to content

feat(bigquery-jdbc): add EnableProjectDiscovery connection property for metadata methods #13344

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
keshavdandeva merged 13 commits into from
Jun 25, 2026
Merged

feat(bigquery-jdbc): add EnableProjectDiscovery connection property for metadata methods #13344

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
8f75b30
feat: add `EnableProjectDiscovery` connection property for metadata m…
keshavdandeva Jun 3, 2026
81e80a0
address pr feedback
keshavdandeva Jun 3, 2026
86d62b1
address pr feedback
keshavdandeva Jun 3, 2026
f440367
chore: address pr feedback
keshavdandeva Jun 5, 2026
c45cec9
Merge branch 'main' into jdbc/get-all-projects
keshavdandeva Jun 18, 2026
3387cf5
use the SDK `listProjects` API with connection-scoped caching
keshavdandeva Jun 18, 2026
b0aa5cd
Merge branch 'main' into jdbc/get-all-projects
keshavdandeva Jun 18, 2026
47a47ac
Merge branch 'main' into jdbc/get-all-projects
keshavdandeva Jun 19, 2026
bf7319c
throw error instead of logging warning for `getDiscoveredProjects`
keshavdandeva Jun 23, 2026
f816861
Merge branch 'main' into jdbc/get-all-projects
keshavdandeva Jun 23, 2026
902f799
remove `getMaxResults` for `listProjects`
keshavdandeva Jun 23, 2026
c52e956
remove extra catch block
keshavdandeva Jun 25, 2026
fd5c2cc
Merge branch 'main' into jdbc/get-all-projects
keshavdandeva Jun 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 62 additions & 0 deletions java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,19 @@

package com.google.cloud.bigquery.jdbc;

import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.gson.GsonFactory;
import com.google.api.gax.core.CredentialsProvider;
import com.google.api.gax.core.FixedCredentialsProvider;
import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider;
import com.google.api.gax.retrying.RetrySettings;
import com.google.api.gax.rpc.FixedHeaderProvider;
import com.google.api.gax.rpc.HeaderProvider;
import com.google.api.gax.rpc.TransportChannelProvider;
import com.google.api.services.bigquery.Bigquery;
import com.google.api.services.bigquery.model.ProjectList;
import com.google.api.services.bigquery.model.ProjectList.Projects;
import com.google.auth.Credentials;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQueryException;
Expand All @@ -41,6 +47,7 @@
import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient;
import com.google.cloud.bigquery.storage.v1.BigQueryWriteSettings;
import com.google.cloud.http.HttpTransportOptions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSortedSet;
import java.io.IOException;
import java.io.InputStream;
Expand Down Expand Up @@ -77,6 +84,10 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
private final String connectionId;
private static final String DEFAULT_JDBC_TOKEN_VALUE = "Google-BigQuery-JDBC-Driver";
private static final String DEFAULT_VERSION = "0.0.0";
private static final String BIGQUERY_SERVICE_NAME = "bigquery";
private static final long MAX_PROJECTS_PER_PAGE = 10000L;
Comment thread
keshavdandeva marked this conversation as resolved.
Outdated
private static final String PROJECT_LIST_FIELDS =
"projects/projectReference/projectId,nextPageToken";
Comment thread
keshavdandeva marked this conversation as resolved.
Outdated
private static final Set<String> SAFE_TO_LOG_PROPERTIES =
ImmutableSortedSet.orderedBy(String.CASE_INSENSITIVE_ORDER)
.add(
Expand Down Expand Up @@ -120,6 +131,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
BigQueryJdbcUrlUtility.SWA_APPEND_ROW_COUNT_PROPERTY_NAME,
BigQueryJdbcUrlUtility.SWA_ACTIVATION_ROW_COUNT_PROPERTY_NAME,
BigQueryJdbcUrlUtility.FILTER_TABLES_ON_DEFAULT_DATASET_PROPERTY_NAME,
BigQueryJdbcUrlUtility.ENABLE_PROJECT_DISCOVERY_PROPERTY_NAME,
BigQueryJdbcUrlUtility.REQUEST_GOOGLE_DRIVE_SCOPE_PROPERTY_NAME,
BigQueryJdbcUrlUtility.SSL_TRUST_STORE_PROPERTY_NAME,
BigQueryJdbcUrlUtility.MAX_BYTES_BILLED_PROPERTY_NAME,
Expand Down Expand Up @@ -169,6 +181,8 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
int highThroughputMinTableSize;
int highThroughputActivationRatio;
boolean enableSession;
boolean enableProjectDiscovery;
private List<String> discoveredProjectsCache;
boolean unsupportedHTAPIFallback;
boolean useQueryCache;
String queryDialect;
Expand Down Expand Up @@ -335,6 +349,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
this.additionalProjects = ds.getAdditionalProjects();

this.filterTablesOnDefaultDataset = ds.getFilterTablesOnDefaultDataset();
this.enableProjectDiscovery = ds.getEnableProjectDiscovery();
this.requestGoogleDriveScope = ds.getRequestGoogleDriveScope();
this.metadataFetchThreadCount = ds.getMetadataFetchThreadCount();
this.requestReason = ds.getRequestReason();
Expand Down Expand Up @@ -1221,6 +1236,53 @@ private boolean checkIsReadOnlyTokenUsed(Map<String, String> authProps) {
return false;
}

public boolean isEnableProjectDiscovery() {
return this.enableProjectDiscovery;
}

public synchronized List<String> getDiscoveredProjects() {
if (this.discoveredProjectsCache != null) {
return this.discoveredProjectsCache;
}

try {
BigQueryOptions options = (BigQueryOptions) getBigQuery().getOptions();
Comment thread
keshavdandeva marked this conversation as resolved.
Outdated
HttpTransportOptions transportOptions = (HttpTransportOptions) options.getTransportOptions();
HttpTransport transport = transportOptions.getHttpTransportFactory().create();
HttpRequestInitializer initializer = transportOptions.getHttpRequestInitializer(options);
Bigquery lowLevelBq =
Comment thread
keshavdandeva marked this conversation as resolved.
Outdated
new Bigquery.Builder(transport, new GsonFactory(), initializer)
Comment thread
keshavdandeva marked this conversation as resolved.
Outdated
.setRootUrl(options.getResolvedApiaryHost(BIGQUERY_SERVICE_NAME))
.setApplicationName(DEFAULT_JDBC_TOKEN_VALUE)
.build();

List<String> projects = new ArrayList<>();
String pageToken = null;
do {
ProjectList projectList =
lowLevelBq
.projects()
.list()
.setPageToken(pageToken)
.setMaxResults(MAX_PROJECTS_PER_PAGE)
.setFields(PROJECT_LIST_FIELDS)
.execute();
if (projectList.getProjects() != null) {
for (Projects p : projectList.getProjects()) {
projects.add(p.getProjectReference().getProjectId());
}
}
pageToken = projectList.getNextPageToken();
} while (pageToken != null);

this.discoveredProjectsCache = ImmutableList.copyOf(projects);
} catch (Exception e) {
Comment thread
keshavdandeva marked this conversation as resolved.
LOG.warning(e, "Failed to list all accessible projects, falling back to connection default.");
Comment thread
keshavdandeva marked this conversation as resolved.
Outdated
this.discoveredProjectsCache = ImmutableList.of();
}
Comment thread
keshavdandeva marked this conversation as resolved.
return this.discoveredProjectsCache;
}

@Override
public <T> T unwrap(Class<T> iface) throws SQLException {
if (iface.isInstance(this)) {
Expand Down
98 changes: 55 additions & 43 deletions ...-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3652,60 +3652,61 @@ public ResultSet getSchemas(String catalog, String schemaPattern) {
return;
}

ExecutorService apiExecutor = null;
final List<Future<List<Dataset>>> apiFutures = new ArrayList<>();
try {
apiExecutor = Executors.newFixedThreadPool(API_EXECUTOR_POOL_SIZE);
Comment thread
keshavdandeva marked this conversation as resolved.
Outdated
Comment thread
keshavdandeva marked this conversation as resolved.
Outdated
for (String currentProjectToScan : projectsToScanList) {
if (Thread.currentThread().isInterrupted()) {
LOG.warning(
"Schema fetcher interrupted during project iteration for project: "
+ currentProjectToScan);
break;
}
LOG.info("Fetching schemas for project: " + currentProjectToScan);
List<Dataset> datasetsInProject =
findMatchingBigQueryObjects(
"Dataset",
() ->
bigquery.listDatasets(
currentProjectToScan,
BigQuery.DatasetListOption.pageSize(DEFAULT_PAGE_SIZE)),
(name) -> bigquery.getDataset(DatasetId.of(currentProjectToScan, name)),
(ds) -> ds.getDatasetId().getDataset(),
schemaPattern,
schemaRegex,
LOG);

if (datasetsInProject.isEmpty() || Thread.currentThread().isInterrupted()) {
LOG.info(
"Fetcher thread found no matching datasets in project: "
+ currentProjectToScan);
continue;
}
checkInterrupted(apiFutures);
Callable<List<Dataset>> apiCallable =
() ->
findMatchingBigQueryObjects(
"Dataset",
() ->
bigquery.listDatasets(
currentProjectToScan,
BigQuery.DatasetListOption.pageSize(DEFAULT_PAGE_SIZE)),
(name) -> bigquery.getDataset(DatasetId.of(currentProjectToScan, name)),
(ds) -> ds.getDatasetId().getDataset(),
schemaPattern,
schemaRegex,
LOG);
apiFutures.add(apiExecutor.submit(apiCallable));
}
apiExecutor.shutdown();

LOG.fine("Processing found datasets for project: " + currentProjectToScan);
for (Dataset dataset : datasetsInProject) {
if (Thread.currentThread().isInterrupted()) {
LOG.warning(
"Schema fetcher interrupted during dataset iteration for project: "
+ currentProjectToScan);
break;
for (Future<List<Dataset>> apiFuture : apiFutures) {
checkInterrupted(apiFutures);
try {
List<Dataset> datasetsInProject = apiFuture.get();
if (datasetsInProject != null) {
for (Dataset dataset : datasetsInProject) {
processSchemaInfo(dataset, collectedResults, localResultSchemaFields);
}
}
processSchemaInfo(dataset, collectedResults, localResultSchemaFields);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
checkInterrupted(apiFutures);
} catch (ExecutionException e) {
LOG.warning("Error executing findMatchingDatasets task: " + e.getMessage());
Comment thread
keshavdandeva marked this conversation as resolved.
Outdated
} catch (CancellationException e) {
LOG.warning("A findMatchingDatasets task was cancelled.");
}
}

if (!Thread.currentThread().isInterrupted()) {
Comparator<FieldValueList> comparator =
defineGetSchemasComparator(localResultSchemaFields);
sortResults(collectedResults, comparator, "getSchemas", LOG);
}

if (!Thread.currentThread().isInterrupted()) {
populateQueue(collectedResults, queue, localResultSchemaFields);
}
checkInterrupted(apiFutures);
Comparator<FieldValueList> comparator =
defineGetSchemasComparator(localResultSchemaFields);
sortResults(collectedResults, comparator, "getSchemas", LOG);
populateQueue(collectedResults, queue, localResultSchemaFields);

} catch (CancellationException e) {
LOG.warning("Schema fetcher task was cancelled/interrupted.");
} catch (Throwable t) {
LOG.severe("Unexpected error in schema fetcher runnable: " + t.getMessage());
apiFutures.forEach(f -> f.cancel(true));
} finally {
shutdownExecutor(apiExecutor);
signalEndOfData(queue, localResultSchemaFields);
LOG.info("Schema fetcher thread finished.");
}
Expand Down Expand Up @@ -5147,6 +5148,13 @@ private void signalEndOfData(
}
}

private void checkInterrupted(List<? extends Future<?>> futures) {
if (Thread.currentThread().isInterrupted()) {
futures.forEach(f -> f.cancel(true));
throw new CancellationException("Fetcher thread was interrupted.");
}
}

private void shutdownExecutor(ExecutorService executor) {
if (executor == null || executor.isShutdown()) {
return;
Expand Down Expand Up @@ -5197,6 +5205,10 @@ private List<String> getAccessibleCatalogNames() {
}
}

if (this.connection.isEnableProjectDiscovery()) {
accessibleCatalogs.addAll(this.connection.getDiscoveredProjects());
}

List<String> sortedCatalogs = new ArrayList<>(accessibleCatalogs);
Collections.sort(sortedCatalogs);
return sortedCatalogs;
Expand Down
9 changes: 9 additions & 0 deletions java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,8 @@ protected boolean removeEldestEntry(Map.Entry<String, Map<String, String>> eldes
static final String FILTER_TABLES_ON_DEFAULT_DATASET_PROPERTY_NAME =
"FilterTablesOnDefaultDataset";
static final boolean DEFAULT_FILTER_TABLES_ON_DEFAULT_DATASET_VALUE = false;
static final String ENABLE_PROJECT_DISCOVERY_PROPERTY_NAME = "EnableProjectDiscovery";
static final boolean DEFAULT_ENABLE_PROJECT_DISCOVERY_VALUE = false;
static final String REQUEST_GOOGLE_DRIVE_SCOPE_PROPERTY_NAME = "RequestGoogleDriveScope";
static final String SSL_TRUST_STORE_PROPERTY_NAME = "SSLTrustStore";
static final String SSL_TRUST_STORE_PWD_PROPERTY_NAME = "SSLTrustStorePwd";
Expand Down Expand Up @@ -576,6 +578,13 @@ protected boolean removeEldestEntry(Map.Entry<String, Map<String, String>> eldes
.setDefaultValue(
String.valueOf(DEFAULT_FILTER_TABLES_ON_DEFAULT_DATASET_VALUE))
.build(),
BigQueryConnectionProperty.newBuilder()
.setName(ENABLE_PROJECT_DISCOVERY_PROPERTY_NAME)
.setDescription(
"Enables or disables automatic discovery of all accessible Google Cloud projects. "
+ "When disabled, only the default ProjectId and AdditionalProjects are listed as catalogs.")
.setDefaultValue(String.valueOf(DEFAULT_ENABLE_PROJECT_DISCOVERY_VALUE))
.build(),
BigQueryConnectionProperty.newBuilder()
.setName(REQUEST_GOOGLE_DRIVE_SCOPE_PROPERTY_NAME)
.setDescription(
Expand Down
22 changes: 22 additions & 0 deletions java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ public class DataSource implements javax.sql.DataSource {
private Boolean enableWriteAPI;
private String additionalProjects;
private Boolean filterTablesOnDefaultDataset;
private Boolean enableProjectDiscovery;
private Integer requestGoogleDriveScope;
private Integer metadataFetchThreadCount;
private String sslTrustStorePath;
Expand Down Expand Up @@ -242,6 +243,12 @@ public class DataSource implements javax.sql.DataSource {
BigQueryJdbcUrlUtility.convertIntToBoolean(
val,
BigQueryJdbcUrlUtility.FILTER_TABLES_ON_DEFAULT_DATASET_PROPERTY_NAME)))
.put(
BigQueryJdbcUrlUtility.ENABLE_PROJECT_DISCOVERY_PROPERTY_NAME,
(ds, val) ->
ds.setEnableProjectDiscovery(
BigQueryJdbcUrlUtility.convertIntToBoolean(
val, BigQueryJdbcUrlUtility.ENABLE_PROJECT_DISCOVERY_PROPERTY_NAME)))
.put(
BigQueryJdbcUrlUtility.REQUEST_GOOGLE_DRIVE_SCOPE_PROPERTY_NAME,
(ds, val) -> ds.setRequestGoogleDriveScope(Integer.parseInt(val)))
Expand Down Expand Up @@ -555,6 +562,11 @@ Properties createProperties() {
BigQueryJdbcUrlUtility.FILTER_TABLES_ON_DEFAULT_DATASET_PROPERTY_NAME,
String.valueOf(this.filterTablesOnDefaultDataset));
}
if (this.enableProjectDiscovery != null) {
connectionProperties.setProperty(
BigQueryJdbcUrlUtility.ENABLE_PROJECT_DISCOVERY_PROPERTY_NAME,
String.valueOf(this.enableProjectDiscovery));
}
if (this.requestGoogleDriveScope != null) {
connectionProperties.setProperty(
BigQueryJdbcUrlUtility.REQUEST_GOOGLE_DRIVE_SCOPE_PROPERTY_NAME,
Expand Down Expand Up @@ -1059,6 +1071,16 @@ public void setFilterTablesOnDefaultDataset(Boolean filterTablesOnDefaultDataset
this.filterTablesOnDefaultDataset = filterTablesOnDefaultDataset;
}

public Boolean getEnableProjectDiscovery() {
return enableProjectDiscovery != null
? enableProjectDiscovery
: BigQueryJdbcUrlUtility.DEFAULT_ENABLE_PROJECT_DISCOVERY_VALUE;
}

public void setEnableProjectDiscovery(Boolean enableProjectDiscovery) {
this.enableProjectDiscovery = enableProjectDiscovery;
}

public Integer getRequestGoogleDriveScope() {
return requestGoogleDriveScope != null
? requestGoogleDriveScope
Expand Down
Loading
Loading