From 63c0692c771d9f31d503b95588e009bb4033f872 Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Wed, 15 Jul 2026 14:59:55 -0700 Subject: [PATCH 1/3] Add GCP support for RP SQL (BYOC) --- .../pages/sql/sql-statements/copy-to.adoc | 14 ++++- .../sql/sql-statements/create-storage.adoc | 57 +++++++++++++++++-- .../pages/connect-to-sql/authenticate.adoc | 2 +- .../pages/get-started/deploy-sql-cluster.adoc | 16 ++++-- .../sql/pages/get-started/sql-quickstart.adoc | 2 +- 5 files changed, 78 insertions(+), 13 deletions(-) diff --git a/modules/reference/pages/sql/sql-statements/copy-to.adoc b/modules/reference/pages/sql/sql-statements/copy-to.adoc index 5e68d7c47..3b5ec63bd 100644 --- a/modules/reference/pages/sql/sql-statements/copy-to.adoc +++ b/modules/reference/pages/sql/sql-statements/copy-to.adoc @@ -162,6 +162,9 @@ WITH (AWS_CRED(AWS_REGION 'us-west-1', AWS_KEY_ID 'key_id', AWS_PRIVATE_KEY 'acc ==== Google Cloud Storage +// TODO: This example passes a file path to GCS_CRED, but GCS_CRED (like CREATE STORAGE's +// service_account_key) accepts the JSON key file's *contents*, not a path. Verify and fix +// in a follow-up pass on this PR. Pass the path to your credentials JSON file: [source,sql] @@ -176,7 +179,14 @@ If you cannot use a path to the credentials file, pass its contents as a string: COPY default_redpanda_catalog=>project TO 'gs://your-bucket/project_file' WITH (GCS_CRED('')); ---- -You can also use `AWS_CRED` with GCS by setting the endpoint: +`GCS_CRED` is optional. If you omit it, Redpanda SQL authenticates using Application Default Credentials (ADC), which includes Workload Identity Federation on GKE: + +[source,sql] +---- +COPY default_redpanda_catalog=>project TO 'gs://your-bucket/project_file'; +---- + +`GCS_CRED` has no option to override the GCS endpoint. To target a custom endpoint (for example, a GCS emulator or the XML API), use `AWS_CRED` instead, since Google Cloud Storage supports an S3-compatible API: [source,sql] ---- @@ -186,7 +196,7 @@ WITH (AWS_CRED(AWS_REGION 'region1', AWS_KEY_ID 'key_id', AWS_PRIVATE_KEY 'acces [TIP] ==== -For Google Cloud Storage, use HMAC keys for authentication. See the link:https://cloud.google.com/storage/docs/authentication/hmackeys[HMAC keys - Cloud Storage^] page for details. +The `AWS_CRED` approach requires HMAC keys for authentication. See the link:https://cloud.google.com/storage/docs/authentication/hmackeys[HMAC keys - Cloud Storage^] page for details. ==== ==== Azure Blob Storage diff --git a/modules/reference/pages/sql/sql-statements/create-storage.adoc b/modules/reference/pages/sql/sql-statements/create-storage.adoc index cbd100f34..25f88dfb7 100644 --- a/modules/reference/pages/sql/sql-statements/create-storage.adoc +++ b/modules/reference/pages/sql/sql-statements/create-storage.adoc @@ -2,22 +2,22 @@ :description: The CREATE STORAGE statement creates a connection to external object storage for use with Redpanda catalogs. :page-topic-type: reference -The `CREATE STORAGE` statement creates a named connection to external object storage such as Amazon S3. +The `CREATE STORAGE` statement creates a named connection to external object storage such as Amazon S3 or Google Cloud Storage (GCS). == Syntax [source,sql] ---- CREATE STORAGE [IF NOT EXISTS] storage_name -TYPE = S3 +TYPE = S3 | GCS WITH (option = 'value' [, ...]); ---- * `storage_name`: Name for the new storage connection. -* `TYPE`: Storage type. Redpanda SQL currently supports only `S3`. +* `TYPE`: Storage type. Redpanda SQL supports `S3` and `GCS`. * `IF NOT EXISTS`: Optional. Prevents an error if a storage connection with the same name already exists. -== Options +== S3 options [cols="<30%,<15%,<10%,<45%",options="header"] |=== @@ -39,8 +39,32 @@ WITH (option = 'value' [, ...]); |AWS secret access key. |=== +== GCS options + +[cols="<30%,<15%,<10%,<45%",options="header"] +|=== +|Option |Type |Required |Description + +|`url` +|STRING +|No +|GCS bucket URL, in the form `gs:///`. If omitted, the connection holds credentials only and has no associated bucket. + +|`service_account_key` +|STRING +|No +|Contents of a GCP service account key JSON file. If omitted, Redpanda SQL authenticates using Application Default Credentials (ADC), which includes Workload Identity Federation on GKE. + +|`endpoint` +|STRING +|No +|Override the default GCS endpoint. +|=== + == Examples +=== Create an S3 storage connection + [source,sql] ---- CREATE STORAGE archive_storage @@ -51,3 +75,28 @@ WITH ( secret_access_key = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' ); ---- + +=== Create a GCS storage connection with a service account key + +[source,sql] +---- +CREATE STORAGE archive_storage +TYPE = GCS +WITH ( + url = 'gs://archive-bucket/redpanda-sql', + service_account_key = '{"type": "service_account", "project_id": "my-project", ...}' +); +---- + +=== Create a GCS storage connection using Application Default Credentials + +Omit `service_account_key` to authenticate with Application Default Credentials (ADC). On GKE, this includes Workload Identity Federation. + +[source,sql] +---- +CREATE STORAGE archive_storage +TYPE = GCS +WITH ( + url = 'gs://archive-bucket/redpanda-sql' +); +---- diff --git a/modules/sql/pages/connect-to-sql/authenticate.adoc b/modules/sql/pages/connect-to-sql/authenticate.adoc index 99b6a2019..af23d49e6 100644 --- a/modules/sql/pages/connect-to-sql/authenticate.adoc +++ b/modules/sql/pages/connect-to-sql/authenticate.adoc @@ -14,7 +14,7 @@ After completing these steps, you will be able to: == Prerequisites -* A Redpanda BYOC cluster on AWS with Redpanda SQL enabled. See xref:sql:get-started/deploy-sql-cluster.adoc[Enable Redpanda SQL]. +* A Redpanda BYOC cluster on AWS or GCP with Redpanda SQL enabled. See xref:sql:get-started/deploy-sql-cluster.adoc[Enable Redpanda SQL]. * https://www.postgresql.org/download/[`psql`^] or another PostgreSQL-compatible client. * A user or service account assigned a role with the *SQL: Access* or *SQL: Manage* permission in Redpanda Cloud's data-plane RBAC. Redpanda Cloud provisions a corresponding user in the SQL engine when the role is assigned. To assign roles, go to *Organization IAM > Roles*; SQL permissions are under the *Data Plane* tab when you create or edit a role. See xref:security:authorization/rbac/rbac_dp.adoc[Configure RBAC in the Data Plane]. diff --git a/modules/sql/pages/get-started/deploy-sql-cluster.adoc b/modules/sql/pages/get-started/deploy-sql-cluster.adoc index 22bd946de..19df2b039 100644 --- a/modules/sql/pages/get-started/deploy-sql-cluster.adoc +++ b/modules/sql/pages/get-started/deploy-sql-cluster.adoc @@ -6,7 +6,9 @@ :learning-objective-2: Scale or disable the SQL engine :learning-objective-3: Verify that the SQL engine is running and ready to accept connections -Enable Redpanda SQL on a Bring Your Own Cloud (BYOC) cluster so you can query streaming data in Redpanda topics using standard PostgreSQL syntax. For Iceberg-enabled topics, queries can span both the streaming topic and its Iceberg history. See xref:sql:query-data/query-iceberg-topics.adoc[Query Iceberg-enabled topics] for that workflow. +// TODO: Drop the "On AWS" scoping once RPSQL Deployment with Iceberg on GCP +// (CIAINFRA-3422/M9) and BigLake REST catalog support (OXLA-9522, cloudv2 PR #27525) ship. +Enable Redpanda SQL on a Bring Your Own Cloud (BYOC) cluster so you can query streaming data in Redpanda topics using standard PostgreSQL syntax. On AWS, Iceberg-enabled topics also support queries that span both the streaming topic and its Iceberg history. See xref:sql:query-data/query-iceberg-topics.adoc[Query Iceberg-enabled topics] for that workflow. After reading this page, you will be able to: @@ -14,7 +16,7 @@ After reading this page, you will be able to: * [ ] {learning-objective-2} * [ ] {learning-objective-3} -NOTE: Redpanda SQL is currently available only on BYOC clusters running on AWS. +NOTE: Redpanda SQL is currently available on BYOC clusters running on AWS and GCP. TIP: If you manage your infrastructure with Terraform, see xref:manage:terraform-provider.adoc#enable-redpanda-sql-on-a-byoc-cluster[Enable Redpanda SQL on a BYOC cluster] in the Terraform provider guide. @@ -43,7 +45,7 @@ Cloud Console:: + -- . Log in to https://cloud.redpanda.com[Redpanda Cloud^]. -. Start creating a new BYOC cluster on AWS. For details and prerequisites, see xref:get-started:cluster-types/byoc/aws/create-byoc-cluster-aws.adoc[]. +. Start creating a new BYOC cluster on AWS or GCP. For details and prerequisites, see xref:get-started:cluster-types/byoc/aws/create-byoc-cluster-aws.adoc[] or xref:get-started:cluster-types/byoc/gcp/create-byoc-cluster-gcp.adoc[]. . In the cluster creation form, locate the *Redpanda SQL engine* card. Toggle the engine on and use the *RPU* slider to set the compute size. + For more on RPUs, compute, and cost calculations, see xref:billing:billing.adoc#redpanda-sql-billing-metrics[Redpanda SQL billing metrics]. @@ -64,7 +66,7 @@ curl -X POST "https://api.redpanda.com/v1/clusters" \ -d '{ "cluster": { "name": "", - "cloud_provider": "CLOUD_PROVIDER_AWS", + "cloud_provider": "", "type": "TYPE_BYOC", "region": "", "zones": [ ], @@ -79,8 +81,9 @@ curl -X POST "https://api.redpanda.com/v1/clusters" \ }' ---- + -Replace the `rpsql` placeholders with your own values: +Replace the placeholders with your own values: + +* ``: `CLOUD_PROVIDER_AWS` or `CLOUD_PROVIDER_GCP`. * ``: Set the initial number of SQL compute nodes (minimum 1). * `` (optional): For multi-AZ clusters, specify which of the cluster's availability zones to deploy for the SQL engine. Provide a single zone string from the cluster's `zones` list. + @@ -90,6 +93,9 @@ Replace the `rpsql` placeholders with your own values: === On an existing cluster +// TODO: Confirm with SME whether GCP BYOC clusters created before Redpanda SQL launched on GCP +// (M7/CIAINFRA-3420) need an equivalent agent-permission refresh step before enabling SQL, similar +// to the AWS `rpk cloud byoc aws apply` requirement below. NOTE: If your BYOC cluster on AWS was created before May 28, 2026, rerun `rpk cloud byoc aws apply --redpanda-id=` to update the agent's IAM permissions before enabling Redpanda SQL. [tabs] diff --git a/modules/sql/pages/get-started/sql-quickstart.adoc b/modules/sql/pages/get-started/sql-quickstart.adoc index 989b2807d..703a1ddd8 100644 --- a/modules/sql/pages/get-started/sql-quickstart.adoc +++ b/modules/sql/pages/get-started/sql-quickstart.adoc @@ -14,7 +14,7 @@ After reading this page, you will be able to: == Prerequisites -* A Redpanda BYOC cluster on AWS with Redpanda SQL enabled. See xref:sql:get-started/deploy-sql-cluster.adoc[]. +* A Redpanda BYOC cluster on AWS or GCP with Redpanda SQL enabled. See xref:sql:get-started/deploy-sql-cluster.adoc[]. * Admin access to your cluster in the Redpanda Cloud Console, or a role with the *SQL: Manage* permission. You need one of these to view SQL connection details and to create catalogs, tables, and grants in the SQL engine. For more information on authorization in Redpanda Cloud, see xref:security:authorization/rbac/index.adoc[]. * A Redpanda topic with a schema registered in Schema Registry. If you don't have one, follow the optional <> section below to create a sample `orders` topic. * xref:manage:rpk/rpk-install.adoc[`rpk` v26.1.6] or later installed on your local machine to generate an authentication token. From 2ea7fb709d1392203a2ea26ea98d67d48a03c8eb Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Wed, 15 Jul 2026 15:00:28 -0700 Subject: [PATCH 2/3] TODOs for Iceberg/BigLake --- .../pages/sql/sql-statements/create-iceberg-catalog.adoc | 3 +++ modules/reference/pages/sql/system-virtual-tables.adoc | 2 ++ modules/sql/pages/query-data/query-iceberg-topics.adoc | 2 ++ 3 files changed, 7 insertions(+) diff --git a/modules/reference/pages/sql/sql-statements/create-iceberg-catalog.adoc b/modules/reference/pages/sql/sql-statements/create-iceberg-catalog.adoc index 749fd12ae..fc96ecb65 100644 --- a/modules/reference/pages/sql/sql-statements/create-iceberg-catalog.adoc +++ b/modules/reference/pages/sql/sql-statements/create-iceberg-catalog.adoc @@ -36,6 +36,9 @@ NOTE: Catalogs are created in the current schema (`public` by default). To creat |No |Iceberg warehouse identifier or location. +// TODO: Add `gcp` auth_type (keyless/Workload Identity, requires gcp_project_id) once BigLake +// REST catalog support ships (OXLA-9522, cloudv2 PR #27525). Add a corresponding example +// and a `gcp_project_id` option row below. |`auth_type` |STRING |No diff --git a/modules/reference/pages/sql/system-virtual-tables.adoc b/modules/reference/pages/sql/system-virtual-tables.adoc index bbaad3d8b..7232123af 100644 --- a/modules/reference/pages/sql/system-virtual-tables.adoc +++ b/modules/reference/pages/sql/system-virtual-tables.adoc @@ -76,6 +76,8 @@ Iceberg catalog connections, with their REST endpoint and authentication details |`name` |`text` |No |Iceberg catalog name. |`uri` |`text` |No |REST catalog endpoint URI. |`warehouse` |`text` |No |Iceberg warehouse identifier or location. +// TODO: Add `gcp` to the auth_type list once BigLake REST catalog support ships +// (OXLA-9522, cloudv2 PR #27525) — see also create-iceberg-catalog.adoc auth_type option. |`auth_type` |`text` |No |Authentication type for the REST catalog. One of `oauth2`, `basic`, `aws_sigv4`, or empty if the catalog connects without authentication. |`namespace_name` |`text` |No |Schema containing the catalog. |`database_name` |`text` |No |Database containing the catalog. diff --git a/modules/sql/pages/query-data/query-iceberg-topics.adoc b/modules/sql/pages/query-data/query-iceberg-topics.adoc index d028c8ae5..af1c25fd5 100644 --- a/modules/sql/pages/query-data/query-iceberg-topics.adoc +++ b/modules/sql/pages/query-data/query-iceberg-topics.adoc @@ -18,6 +18,8 @@ After reading this page, you will be able to: == Prerequisites +// TODO: Update prerequisites to include GCP once RPSQL Deployment with Iceberg on GCP +// (CIAINFRA-3422/M9) and BigLake REST catalog support (OXLA-9522, cloudv2 PR #27525) ship. * A Redpanda BYOC cluster on AWS with Redpanda SQL enabled. See xref:sql:get-started/deploy-sql-cluster.adoc[Enable Redpanda SQL]. * The cluster's xref:reference:properties/cluster-properties.adoc#iceberg_catalog_type[`iceberg_catalog_type`] property is set to `rest`. The `object_storage` catalog type does not support Iceberg queries from Redpanda SQL. * The cluster's Iceberg REST catalog is configured. See xref:manage:iceberg/rest-catalog/index.adoc[Integrate with REST Catalogs] for the supported REST catalog options (AWS Glue, Snowflake, Databricks Unity, Polaris, and others) and their configuration steps. From 9a826db20af0bf668f5f635183a37e17fb197984 Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Wed, 15 Jul 2026 15:15:02 -0700 Subject: [PATCH 3/3] Review pass --- modules/reference/pages/sql/sql-statements/copy-to.adoc | 2 +- .../reference/pages/sql/sql-statements/create-storage.adoc | 2 +- modules/sql/pages/get-started/deploy-sql-cluster.adoc | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/reference/pages/sql/sql-statements/copy-to.adoc b/modules/reference/pages/sql/sql-statements/copy-to.adoc index 3b5ec63bd..fdb25f424 100644 --- a/modules/reference/pages/sql/sql-statements/copy-to.adoc +++ b/modules/reference/pages/sql/sql-statements/copy-to.adoc @@ -179,7 +179,7 @@ If you cannot use a path to the credentials file, pass its contents as a string: COPY default_redpanda_catalog=>project TO 'gs://your-bucket/project_file' WITH (GCS_CRED('')); ---- -`GCS_CRED` is optional. If you omit it, Redpanda SQL authenticates using Application Default Credentials (ADC), which includes Workload Identity Federation on GKE: +`GCS_CRED` is optional. If you omit it, Redpanda SQL authenticates using Application Default Credentials (ADC), which includes Workload Identity Federation on Google Kubernetes Engine (GKE): [source,sql] ---- diff --git a/modules/reference/pages/sql/sql-statements/create-storage.adoc b/modules/reference/pages/sql/sql-statements/create-storage.adoc index 25f88dfb7..b622d744f 100644 --- a/modules/reference/pages/sql/sql-statements/create-storage.adoc +++ b/modules/reference/pages/sql/sql-statements/create-storage.adoc @@ -53,7 +53,7 @@ WITH (option = 'value' [, ...]); |`service_account_key` |STRING |No -|Contents of a GCP service account key JSON file. If omitted, Redpanda SQL authenticates using Application Default Credentials (ADC), which includes Workload Identity Federation on GKE. +|Contents of a GCP service account key JSON file. If omitted, Redpanda SQL authenticates using Application Default Credentials (ADC), which includes Workload Identity Federation on Google Kubernetes Engine (GKE). |`endpoint` |STRING diff --git a/modules/sql/pages/get-started/deploy-sql-cluster.adoc b/modules/sql/pages/get-started/deploy-sql-cluster.adoc index 19df2b039..7a6272631 100644 --- a/modules/sql/pages/get-started/deploy-sql-cluster.adoc +++ b/modules/sql/pages/get-started/deploy-sql-cluster.adoc @@ -8,7 +8,7 @@ // TODO: Drop the "On AWS" scoping once RPSQL Deployment with Iceberg on GCP // (CIAINFRA-3422/M9) and BigLake REST catalog support (OXLA-9522, cloudv2 PR #27525) ship. -Enable Redpanda SQL on a Bring Your Own Cloud (BYOC) cluster so you can query streaming data in Redpanda topics using standard PostgreSQL syntax. On AWS, Iceberg-enabled topics also support queries that span both the streaming topic and its Iceberg history. See xref:sql:query-data/query-iceberg-topics.adoc[Query Iceberg-enabled topics] for that workflow. +Enable Redpanda SQL on a Bring Your Own Cloud (BYOC) cluster so you can query streaming data in Redpanda topics using standard PostgreSQL syntax. You can also run queries that span both the streaming topic and its Iceberg history for Iceberg-enabled topics. See xref:sql:query-data/query-iceberg-topics.adoc[Query Iceberg-enabled topics] for that workflow. After reading this page, you will be able to: @@ -16,7 +16,7 @@ After reading this page, you will be able to: * [ ] {learning-objective-2} * [ ] {learning-objective-3} -NOTE: Redpanda SQL is currently available on BYOC clusters running on AWS and GCP. +NOTE: Redpanda SQL is available for BYOC clusters on AWS and GCP. TIP: If you manage your infrastructure with Terraform, see xref:manage:terraform-provider.adoc#enable-redpanda-sql-on-a-byoc-cluster[Enable Redpanda SQL on a BYOC cluster] in the Terraform provider guide.