diff --git a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts index ccbb06a95ddce..1f16c321f6476 100644 --- a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts +++ b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts @@ -1196,12 +1196,10 @@ export const database: NavMenuConstant = { { name: 'Setting up', url: '/guides/database/replication/pipelines' as `/${string}`, - items: [ - { - name: 'BigQuery', - url: '/guides/database/replication/bigquery' as `/${string}`, - }, - ], + }, + { + name: 'BigQuery', + url: '/guides/database/replication/bigquery' as `/${string}`, }, { name: 'Monitoring', diff --git a/apps/docs/content/_partials/billing/pricing/pricing_pipelines.mdx b/apps/docs/content/_partials/billing/pricing/pricing_pipelines.mdx index be36a6cd76a6f..48072976687d8 100644 --- a/apps/docs/content/_partials/billing/pricing/pricing_pipelines.mdx +++ b/apps/docs/content/_partials/billing/pricing/pricing_pipelines.mdx @@ -1,8 +1,7 @@ {/* prettier-ignore */} - per hour (~ per month) for each pipeline. per Gigabyte backfill -data. per Gigabyte replicated data. + per hour for each configured pipeline. per Gigabyte of initial sync data. per Gigabyte of ongoing replication data. -| Plan | Pipeline | Backfill/Initial Copy Data | Replicated Data | +| Plan | Configured Pipeline | Initial Sync Data | Ongoing Replication Data | | ---------- | -------------------------- | ----------------------------- | ----------------------------- | | Free | - | - | - | | Pro | /hr | per GB | per GB | diff --git a/apps/docs/content/_partials/pipelines-public-alpha.mdx b/apps/docs/content/_partials/pipelines-public-alpha.mdx new file mode 100644 index 0000000000000..7e769687aa7be --- /dev/null +++ b/apps/docs/content/_partials/pipelines-public-alpha.mdx @@ -0,0 +1,5 @@ + + +Supabase Pipelines is currently in public alpha. Features and behavior may change as we continue developing the product. + + diff --git a/apps/docs/content/guides/database/replication.mdx b/apps/docs/content/guides/database/replication.mdx index 7ad4408b84ee7..8e9863b01f2f7 100644 --- a/apps/docs/content/guides/database/replication.mdx +++ b/apps/docs/content/guides/database/replication.mdx @@ -6,7 +6,7 @@ subtitle: 'An introduction to database replication and change data capture.' sidebar_label: 'Overview' --- -Replication is the process of copying changes from your database to another location. It's also referred to as change data capture (CDC): capturing all the changes that occur to your data. +Replication keeps data synchronized with another location. Logical replication products such as Supabase Pipelines use change data capture (CDC) to read database changes and apply them to a destination. ## Use cases @@ -14,7 +14,7 @@ You might use database replication for: - **Analytics and data warehousing**: Replicate your operational database to analytics platforms for complex analysis without impacting your application's performance. - **Data integration**: Keep your data synchronized across different systems and services in your tech stack. -- **Backup and disaster recovery**: Maintain up-to-date copies of your data in different locations. +- **Operational reporting**: Maintain a copy of selected application data that you can query in another system. ## Replication methods @@ -30,23 +30,22 @@ Read replicas are additional Supabase Postgres databases kept in sync with your ### Pipelines - +<$Partial path="pipelines-public-alpha.mdx" /> -Supabase Pipelines is currently in private alpha. Private alpha features can be unstable and may introduce breaking changes while we evaluate the product direction, refine the feature set, and incorporate customer feedback. - - - -Pipelines is a managed CDC feature for creating replication pipelines from Supabase Postgres to destination systems. It uses Postgres logical replication under the hood with the open-source [Supabase ETL engine](https://github.com/supabase/etl). A destination is where your replicated data is stored; the pipeline is the managed process that continuously streams database changes to that destination. +Supabase Pipelines is a managed CDC product for moving data from Supabase Postgres to supported destination systems. It uses Postgres logical replication with the open-source [Supabase ETL engine](https://github.com/supabase/etl). A destination is where your replicated data is stored; a pipeline first performs an initial sync of existing rows, then uses ongoing replication (CDC) to send subsequent database changes to that destination. - [Set up Pipelines](/docs/guides/database/replication/pipelines) #### Supported destinations -Pipelines currently supports BigQuery as the managed destination. We are working on new destinations, and this table will be updated as support expands. +{/* supa-mdx-lint-disable-next-line Rule003Spelling */} +Pipelines currently supports BigQuery as the managed destination. You can [request early access to ClickHouse, Snowflake, and DuckLake](/go/supabase-pipelines-new-destinations) while we expand destination support. + +Managed Pipelines run in **AWS `eu-central-1` (Frankfurt)**. Choose destination resources as close as possible to Frankfurt to reduce network latency and replication lag. -| Destination | Insert | Update | Delete | Truncate | Schema change | Description | -| ------------------------------------------------------ | ------------ | ------------ | ------------ | ------------ | ------------- | ------------------------------------------------------------------- | -| [BigQuery](/docs/guides/database/replication/bigquery) | ✅ Supported | ✅ Supported | ✅ Supported | ✅ Supported | ✅ Supported | Managed replication to Google BigQuery for analytics and reporting. | +| Destination | Insert | Update | Delete | Truncate | Schema change | Description | +| ------------------------------------------------------ | ------------ | ------------ | ------------ | ------------ | -------------- | ------------------------------------------------------------------- | +| [BigQuery](/docs/guides/database/replication/bigquery) | ✅ Supported | ✅ Supported | ✅ Supported | ✅ Supported | Beta (limited) | Managed replication to Google BigQuery for analytics and reporting. | ### Manual replication diff --git a/apps/docs/content/guides/database/replication/bigquery.mdx b/apps/docs/content/guides/database/replication/bigquery.mdx index ca96330f84401..b0c28364c6436 100644 --- a/apps/docs/content/guides/database/replication/bigquery.mdx +++ b/apps/docs/content/guides/database/replication/bigquery.mdx @@ -6,11 +6,7 @@ subtitle: 'Replicate Supabase Postgres tables to BigQuery.' sidebar_label: 'BigQuery' --- - - -Supabase Pipelines is currently in private alpha. Private alpha features can be unstable and may introduce breaking changes while we evaluate the product direction, refine the feature set, and incorporate customer feedback. - - +<$Partial path="pipelines-public-alpha.mdx" /> [BigQuery](https://cloud.google.com/bigquery) is Google's fully managed data warehouse. You can replicate your database tables to BigQuery for analytics and reporting. @@ -18,21 +14,25 @@ Supabase Pipelines is currently in private alpha. Private alpha features can be Before configuring BigQuery as a destination, set up the following in Google Cloud Platform: -1. **Google Cloud Platform (GCP) account**: [Sign up for GCP](https://cloud.google.com/gcp) if you don't have one +1. **Google Cloud Platform (GCP) account**: [Sign up for GCP](https://cloud.google.com/gcp) if you don't have one. In the destination project, make sure the [BigQuery API and BigQuery Storage API](https://cloud.google.com/bigquery/docs/service-dependencies) are enabled. 2. **BigQuery dataset**: Create a [BigQuery dataset](https://cloud.google.com/bigquery/docs/datasets-intro) in your GCP project - Open the BigQuery console in GCP - Select your project - Click **Create Dataset** - Provide a dataset ID, for example `supabase_replication` + - Choose the [dataset location](https://cloud.google.com/bigquery/docs/locations) intentionally. Supabase Pipelines run in **AWS `eu-central-1` (Frankfurt)**, so choose the closest available BigQuery location to reduce network latency. You can't change a dataset's location after it is created, and Pipelines doesn't infer it from your Supabase project or copy Postgres partitioning settings. 3. **GCP service account key**: Create a [service account](https://cloud.google.com/iam/docs/keys-create-delete) with appropriate permissions - Go to **IAM & Admin > Service Accounts** - Click **Create Service Account** - - Grant the "BigQuery Data Editor" and "BigQuery Job User" roles + - Grant **BigQuery Data Editor** on the destination dataset + - Grant **BigQuery Job User** on the GCP project - Create and download the JSON key file -Required permissions: + Treat the downloaded JSON as a secret. Don't commit or share it, and [rotate or revoke the key](https://cloud.google.com/iam/docs/key-rotation) if it is exposed. + +These roles provide the permissions Pipelines needs to inspect and manage destination tables, write data through the Storage Write API, and run BigQuery jobs. If you use a custom IAM role, it must provide: - `bigquery.datasets.get` - `bigquery.jobs.create` @@ -43,20 +43,20 @@ Required permissions: - `bigquery.tables.list` - `bigquery.tables.update` - `bigquery.tables.updateData` -- `bigquery.routines.get` -- `bigquery.routines.list` ## Configure BigQuery as a destination 1. Navigate to the [**Database > Replication**](/dashboard/project/_/database/replication) section of the Dashboard 2. Click **Add destination** -3. Configure the general settings: +3. Select **BigQuery** as the destination type + +4. Configure the destination details: - **Destination name**: A name to identify this destination, for example "BigQuery Warehouse" - **Publication**: The publication to replicate data from - - **Destination type**: Select **BigQuery** + - **Region**: Managed Pipelines run in the fixed **AWS `eu-central-1` (Frankfurt)** region. This is separate from your BigQuery dataset location and can't be changed. -4. Configure BigQuery-specific settings: +5. Configure the BigQuery settings: - **Project ID**: Your BigQuery project identifier, found in the GCP Console - **Dataset ID**: The name of your BigQuery dataset, without the project ID @@ -68,26 +68,41 @@ Required permissions: - **Service Account Key**: Your GCP service account key in JSON format -5. Optionally expand **Advanced settings** for BigQuery-specific performance tuning: +BigQuery destination form with pipeline details and BigQuery credentials + +6. Optionally expand **Advanced settings** for pipeline and BigQuery-specific tuning: + + The general batch, initial-sync concurrency, and invalidated-slot settings are described in [Set up Pipelines](/docs/guides/database/replication/pipelines#step-3-configure-a-destination). BigQuery adds these settings: + + | Setting | Default | Description | + | ------------------------ | ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | + | **Connection pool size** | `4` connections | Number of BigQuery Storage Write API connections used for destination writes. More connections can increase write throughput, but consume more pipeline and BigQuery resources. | + | **Maximum staleness** | Freshest results | Maximum acceptable staleness, in whole minutes, of table data returned by queries while BigQuery applies CDC `UPSERT` and `DELETE` changes in the background. Leave unset for the freshest table data. A larger number of minutes allows BigQuery to return older data, which can reduce query-time CDC merge cost and latency. For example, `15` allows data to be up to 15 minutes stale. This value is applied when Pipelines creates or recreates a table; changing it doesn't alter existing destination tables. | - | Setting | Default | Description | - | ------------------------ | ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | - | **Connection pool size** | `4` connections | Size of the BigQuery Storage Write API connection pool. More connections allow more parallel writes, but consume more resources. | - | **Maximum staleness** | No staleness limit | Maximum allowed age, in minutes, for BigQuery cached metadata before reading base tables. Lower values improve freshness. Higher values can reduce query cost and latency. | +7. Review the [source table requirements](#source-table-requirements), then click **Create and start pipeline** to begin replication -6. Click **Create and start pipeline** to begin replication +The pipeline begins the initial sync from your database to BigQuery. -Your replication pipeline now starts copying data from your database to BigQuery. +Supabase Pipelines charges and Google Cloud charges are separate. BigQuery can charge for Storage Write API ingestion, storage, and the compute used to apply CDC changes. See [BigQuery CDC pricing](https://cloud.google.com/bigquery/docs/change-data-capture#pricing). ## How it works Once configured, replication to BigQuery: -1. Captures changes from your Postgres database (INSERT, UPDATE, DELETE, TRUNCATE operations) +1. Captures the `INSERT`, `UPDATE`, `DELETE`, and `TRUNCATE` operations included by your Postgres publication 2. Optimizes delivery automatically -3. Creates BigQuery tables automatically to match your Postgres schema +3. Creates destination tables from the replicated source schema using BigQuery-compatible names and types 4. Streams data to BigQuery +Pipelines keeps a current-state table that you can query for each replicated source table and may replace its destination data during a truncate or new initial sync. It does not provide a history of every row version that you can query. + ## Source table requirements BigQuery replication requires each source table to have a primary key, and the publication must include the primary-key columns. Pipelines declares those columns as the BigQuery destination primary key so BigQuery change data capture (CDC) can apply `UPSERT` and `DELETE` rows. @@ -100,15 +115,15 @@ Source tables must also use a BigQuery-compatible Postgres `REPLICA IDENTITY` se | ------------------------------------------------ | ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `REPLICA IDENTITY DEFAULT` with a primary key | Supported | Recommended for most tables. BigQuery uses the replicated source primary key to apply upserts and deletes. | | `REPLICA IDENTITY FULL` | Supported | Recommended for tables with large `text`, `jsonb`, `bytea`, or other values that Postgres may store out-of-line using TOAST, especially when those rows update. | -| `REPLICA IDENTITY USING INDEX` | Not supported | BigQuery change data capture (CDC) rows are keyed by the source primary key, not by an alternative unique index. | -| `REPLICA IDENTITY NOTHING` | Not supported | Updates and deletes do not include enough row identity for BigQuery to apply them safely. | +| `REPLICA IDENTITY USING INDEX` | Limited | Supported only when the selected unique index contains exactly the source primary-key columns. An alternative unique-key identity is not supported. | +| `REPLICA IDENTITY NOTHING` | Insert-only | Inserts can be replicated, but updates and deletes do not include enough row identity for BigQuery to apply them safely. | | `REPLICA IDENTITY DEFAULT` without a primary key | Not supported | BigQuery requires a source primary key. | For a general explanation of how replica identity affects update and delete events, see [How does replica identity affect updates and deletes?](/docs/guides/database/replication/pipelines-faq#how-does-replica-identity-affect-updates-and-deletes). For updates, Postgres does not always send a complete old row through logical replication. It can also mark unchanged toasted values as `unchanged toast` instead of resending the value. BigQuery change data capture (CDC) upserts require a complete new row because omitted columns are not preserved in the destination. The replication pipeline can reconstruct a complete update when the old row image contains the missing value, which is reliable with `REPLICA IDENTITY FULL`. -If a BigQuery pipeline fails with an error about a partial update row, set `REPLICA IDENTITY FULL` on the affected source table and restart the pipeline. Changing replica identity only affects new WAL records, so a retained update that was written before the change may still need to be skipped by recreating the pipeline or re-copying the affected table. +If a BigQuery pipeline fails with an error about a partial update row, set `REPLICA IDENTITY FULL` on the affected source table and restart the pipeline. Changing replica identity only affects new WAL records, so a retained update that was written before the change may still need to be skipped by recreating the pipeline or restarting the affected table's initial sync. Check a table's current replica identity: @@ -133,23 +148,13 @@ alter table public.your_table replica identity full; `REPLICA IDENTITY FULL` increases WAL volume because Postgres logs the full old row for updates and deletes. Use it on tables where update correctness is more important than the extra replication overhead. -## How tables are structured - -Due to BigQuery limitations, replicated tables use a versioned structure: - -- The table you query is a **view**, for example `users` -- The actual data is stored in versioned tables with a `_version` suffix, for example `users_version` -- When a table is truncated in your database, a new version is created and the view automatically points to the latest version - -This structure handles table truncations while maintaining query compatibility. - ## Schema change support Schema change support for BigQuery is currently in beta. Pipelines supports a limited set of schema changes for BigQuery while the feature is developed further. Supported schema changes: -- Adding a nullable column +- Adding a scalar, top-level column (created as `NULLABLE` in BigQuery) - Removing a column - Renaming a column - Dropping a `NOT NULL` constraint @@ -162,16 +167,24 @@ Unsupported or limited schema changes: - Filling existing rows for `ADD COLUMN ... DEFAULT` - Unsupported default expressions -BigQuery requires added columns to be nullable. When a replicated `ADD COLUMN` includes a default, Pipelines can apply supported default metadata for future rows, but BigQuery does not backfill existing rows through that DDL. Existing destination rows remain `NULL` unless you run a separate backfill. +When the initial sync creates a BigQuery table, Pipelines preserves whether each scalar, non-array source column allows `NULL`: Postgres `NOT NULL` columns become `REQUIRED`, and nullable columns become `NULLABLE`. BigQuery represents Postgres arrays as `REPEATED` fields instead of using `REQUIRED` or `NULLABLE` mode. + +After the table exists, BigQuery requires every newly added scalar, top-level column to be `NULLABLE`. If Postgres adds a `NOT NULL` column, Pipelines adds it as `NULLABLE` in BigQuery and logs a warning. Postgres remains the source of truth and rejects new `NULL` values before they reach the destination. + +For Postgres `DROP NOT NULL`, Pipelines relaxes an existing BigQuery column from `REQUIRED` to `NULLABLE`. For Postgres `SET NOT NULL`, BigQuery cannot change an existing `NULLABLE` column to `REQUIRED`, so Pipelines leaves the destination column nullable and logs a warning. -Supported defaults are best-effort translations to BigQuery SQL. Unsupported defaults are skipped with a warning instead of failing replication. +Column defaults are handled independently from whether the column allows `NULL`. BigQuery does not support `ADD COLUMN ... DEFAULT` on an existing table, so Pipelines first adds the nullable column and then applies supported default metadata with a separate statement. This is destination metadata for future BigQuery writes that omit the column; pipeline writes already contain the value evaluated by Postgres, and the metadata doesn't populate existing destination rows. Unsupported defaults, including defaults on destination primary-key columns, are skipped with a warning instead of failing replication. ## Limitations - **Row size**: Limited to 10 MB per row due to BigQuery Storage Write API constraints - **Primary keys**: Source tables must have a primary key, the replicated primary key can contain at most 16 columns, and BigQuery does not enforce key uniqueness -- **Replica identity**: Source tables must use primary-key replica identity or `REPLICA IDENTITY FULL` -- **Table names**: Source table names cannot start or end with `_` when replicating to BigQuery +- **Columns**: BigQuery CDC supports at most 2,000 top-level columns +- **Replica identity**: Updates and deletes require a supported primary-key identity or `REPLICA IDENTITY FULL` +- **Schema and table names**: Source schema and table names can't start or end with `_` or contain `"` or `;` when replicating to BigQuery +- **Arrays and numeric values**: Arrays can't contain `NULL` elements. Numeric values with more than 38 fractional digits and exact JSON integer values outside BigQuery's supported range can't be replicated. +- **BigQuery CDC tables**: While CDC is active, BigQuery doesn't support mutating DML (`UPDATE`, `DELETE`, or `MERGE`), wildcard table queries, or search indexes on the destination table. See [BigQuery CDC limitations](https://cloud.google.com/bigquery/docs/change-data-capture#limitations) for the complete list. +- **Managed destination objects**: Don't delete or modify tables or views created by Pipelines. Doing so can stop replication and may require a new, billable initial sync. - **Schema changes**: Limited to the supported schema changes listed above ## Additional resources diff --git a/apps/docs/content/guides/database/replication/manual-replication-setup.mdx b/apps/docs/content/guides/database/replication/manual-replication-setup.mdx index e2d414c8c6233..b751ef6ce4794 100644 --- a/apps/docs/content/guides/database/replication/manual-replication-setup.mdx +++ b/apps/docs/content/guides/database/replication/manual-replication-setup.mdx @@ -48,10 +48,6 @@ You can follow those steps with the following modifications: 1. Use the `postgres` user 2. Select `logical replication` as the replication method (`xmin` is possible, but not recommended) -### Troubleshooting - -Airbyte has a known [issue](https://discuss.airbyte.io/t/postgres-source-replication-slot-safe-wal-size-only-reset-when-a-change-occurs/3263/7) where it does not clear WAL files on each successful sync. The recommended workaround is to have a `heartbeat` table that you write changes to once an hour.> - diff --git a/apps/docs/content/guides/database/replication/pipelines-faq.mdx b/apps/docs/content/guides/database/replication/pipelines-faq.mdx index f82686959836a..f01524856aa6e 100644 --- a/apps/docs/content/guides/database/replication/pipelines-faq.mdx +++ b/apps/docs/content/guides/database/replication/pipelines-faq.mdx @@ -2,27 +2,24 @@ id: 'pipelines-faq' title: 'Pipelines FAQ' description: 'Frequently asked questions about Supabase Pipelines.' -subtitle: 'Common questions and answers about managed replication pipelines.' +subtitle: 'Common questions and answers about Supabase Pipelines.' sidebar_label: 'FAQ' --- - - -Supabase Pipelines is currently in private alpha. Private alpha features can be unstable and may introduce breaking changes while we evaluate the product direction, refine the feature set, and incorporate customer feedback. - - +<$Partial path="pipelines-public-alpha.mdx" /> ## What destinations are supported? Pipelines currently supports **BigQuery** as the managed destination. See the [BigQuery destination guide](/docs/guides/database/replication/bigquery) for configuration details. -We are working on new destinations. Supported destinations can be Supabase-managed or third-party systems as support expands. Availability may continue to vary based on the planned roll-out strategy. +{/* supa-mdx-lint-disable-next-line Rule003Spelling */} +You can [request early access to ClickHouse, Snowflake, and DuckLake](/go/supabase-pipelines-new-destinations). Supported destinations can be Supabase-managed or third-party systems as support expands. ## Does the destination's region affect performance? Yes. The further apart your source database, the pipeline, and your destination are, the more network latency is added to replication, which increases replication lag and reduces throughput. -Pipelines run out of **eu-central-1 (Frankfurt)**. For the best performance, place both your source project and your destination as close to this region as possible. +Managed Pipelines run in **AWS `eu-central-1` (Frankfurt)**. For the best performance, place both your source project and destination as close to Frankfurt as possible. When configuring a destination provider such as BigQuery, choose its closest available region. If you can only optimize one side, prioritize placing your **destination** close to the pipeline's region. Replicated data continuously streams out to the destination, so latency on that leg has a bigger impact on overall replication lag than latency between the pipeline and the source. @@ -30,7 +27,7 @@ If you can only optimize one side, prioritize placing your **destination** close ## Which plans support Pipelines? -Pipelines is available on the Pro, Team, and Enterprise plans. +Pipelines requires a Pro, Team, or Enterprise plan. During public alpha, access is being rolled out gradually, so an eligible plan does not guarantee that Pipelines is enabled for every organization yet. If it is not available, request access from the **Database > Replication** page or contact your account manager. {/* supa-mdx-lint-disable-next-line Rule001HeadingCase */} @@ -49,7 +46,7 @@ When you enable Pipelines, Supabase installs database objects that help track re - An event trigger that runs on every `ALTER TABLE` statement. Supabase uses this to support schema change handling. - A set of tables in the `etl` schema. These tables track replication state for your pipelines. -The replication state tables are not updated very often, especially after the initial copy phase is complete. +The replication state tables are not updated very often, especially after the initial sync is complete. ## What schema changes are supported? @@ -57,13 +54,13 @@ Schema change support is currently in beta and limited to the BigQuery destinati Supported schema changes: -- Adding a nullable column +- Adding a scalar, top-level column (created as `NULLABLE` in BigQuery) - Removing a column - Renaming a column - Dropping a `NOT NULL` constraint - Setting or dropping supported column default metadata -Pipelines does not currently support changing column data types, adding `NOT NULL` with `SET NOT NULL`, or filling existing destination rows for `ADD COLUMN ... DEFAULT`. Supported defaults are applied as destination metadata for future rows where BigQuery can represent them. See [BigQuery schema change support](/docs/guides/database/replication/bigquery#schema-change-support) for details. +Initial BigQuery table creation preserves whether each scalar, non-array Postgres column allows `NULL`. Arrays use BigQuery's `REPEATED` mode. For later changes, `DROP NOT NULL` relaxes a BigQuery `REQUIRED` column to `NULLABLE`, while `SET NOT NULL` leaves an existing BigQuery column nullable and logs a warning. Newly added scalar, top-level columns are always nullable in BigQuery. Supported defaults are applied separately as destination metadata and do not populate existing destination rows. Pipelines does not currently support changing column data types. See [BigQuery schema change support](/docs/guides/database/replication/bigquery#schema-change-support) for details. {/* supa-mdx-lint-disable-next-line Rule001HeadingCase */} @@ -71,7 +68,7 @@ Pipelines does not currently support changing column data types, adding `NOT NUL Disabling Pipelines removes the database objects that were installed in your source database, including the replication state tables in the `etl` schema and the DDL event trigger. -To remove Pipelines from a project, delete all replication pipelines first. After all pipelines are deleted, the disable action becomes available. For the Dashboard steps, see [Disabling Pipelines](/docs/guides/database/replication/pipelines#disabling-pipelines). +To remove Pipelines from a project, delete all Pipelines destinations first. After all destinations are deleted, the disable action becomes available. For the Dashboard steps, see [Disabling Pipelines](/docs/guides/database/replication/pipelines#disabling-pipelines). Disabling Pipelines stops Supabase from managing replication for the project. It does not delete tables or data that were already written to your destination. @@ -79,7 +76,7 @@ Disabling Pipelines stops Supabase from managing replication for the project. It Common reasons: -- **Missing primary key**: Tables must have a primary key to be replicated (Postgres logical replication requirement) +- **Missing primary key**: BigQuery requires each source table to have a primary key and requires the publication to include its columns. This is a BigQuery Pipelines requirement, not a general requirement for publishing Postgres inserts. - **Not in publication**: Ensure the table is included in your Postgres publication - **Generated columns**: Generated columns are skipped during replication @@ -97,51 +94,15 @@ See [Partitioned tables](/docs/guides/database/replication/pipelines#partitioned ## How does replica identity affect updates and deletes? -If inserts replicate but updates or deletes fail, check the table's `REPLICA IDENTITY` setting. - -Postgres logical replication uses `REPLICA IDENTITY` to decide what old-row data to include in WAL for `UPDATE` and `DELETE` events. This is based on logical WAL being enabled, not whether a replication consumer is connected at that moment. Replica identity does not affect the row data Postgres logs for `INSERT` events. - -Downstream destinations use old-row data to find the row that changed, handle primary-key changes, and apply deletes. - -Postgres supports these replica identity modes: - -- `DEFAULT`: uses the table's primary key. Updates and deletes require a primary key when those operations are published. -- `FULL`: logs the full old row for updates and deletes. Postgres flattens toasted old values when needed. -- `USING INDEX`: uses a selected unique index as the replica identity. -- `NOTHING`: logs no old-row identity. If the table is in a publication that publishes updates or deletes, Postgres rejects those statements because there is no row identity to publish. - -With `DEFAULT` or `USING INDEX`, Postgres logs only the replica-identity columns. For updates, Postgres includes that old identity when it needs it, such as when an identity column changes or an identity column contains external toasted data. For deletes, Postgres includes the old identity so consumers can identify the deleted row. - -Large values such as `text`, `jsonb`, and `bytea` can be stored out-of-line by Postgres using TOAST. During an update, if a toasted value did not change, Postgres may send a marker that means "the value is unchanged" instead of sending the value again. The replication pipeline can only reconstruct that value when the logical replication event includes enough old-row data. If the pipeline cannot reconstruct an unchanged toasted value, the update is treated as partial. - -You can check a table's current setting with: - -```sql -select - n.nspname as schema_name, - c.relname as table_name, - c.relreplident as replica_identity -from - pg_class as c - join pg_namespace as n on n.oid = c.relnamespace -where n.nspname = 'public' and c.relname = 'your_table'; -``` - -The `replica_identity` value is `d` for default, `f` for full, `i` for index, and `n` for nothing. +If inserts replicate but updates or deletes fail, the source table might not be sending enough old-row identity through Postgres logical replication. -For tables with toasted columns that need reliable updates, or for destinations that need complete old-row images, set `REPLICA IDENTITY FULL`: +Every table sent to BigQuery must have a primary key, and every primary-key column must be included in the publication. For updates and deletes, use the primary key replica identity or `REPLICA IDENTITY FULL`. Full replica identity is also recommended for tables with large `text`, `jsonb`, `bytea`, or other values that Postgres can store out of line. ```sql alter table public.your_table replica identity full; ``` -`REPLICA IDENTITY FULL` increases WAL volume because Postgres logs the full old row for updates and deletes. Use it when correctness for updates and deletes is more important than the extra replication overhead. - -Changing replica identity only affects WAL records generated after the change. If a retained update was written while the table used `DEFAULT`, restarting replication does not decode that older record as `FULL`. Postgres logical decoding uses historic catalog snapshots, and the old tuple or key data must already be present in the WAL record. - -Destination-specific requirements can be stricter than Postgres itself. For BigQuery, see [BigQuery source table requirements](/docs/guides/database/replication/bigquery#source-table-requirements). - -After changing replica identity, restart each pipeline that includes the affected tables so the pipeline refreshes relation metadata and applies the new setting to future changes. If the pipeline is already failed, change the replica identity and then restart the pipeline. +Full replica identity increases WAL volume and only affects new WAL records. Fix the setting before generating more changes. If the failing update is already retained in WAL, restarting may fail again; recreate the pipeline or restart the affected table's initial sync. See [BigQuery source table requirements](/docs/guides/database/replication/bigquery#source-table-requirements) for details. ## Why aren't publication changes reflected after adding or removing tables? @@ -149,7 +110,7 @@ After modifying your Postgres publication, you must restart the replication pipe ## Why is a pipeline in failed state? -Pipeline failures occur during the streaming phase when an error happens while replicating live data. This prevents data loss. To recover: +A pipeline enters **Failed** when it encounters a non-retryable pipeline-level error during startup or ongoing replication. The pipeline stops instead of silently skipping the failure. To recover: 1. Check the error message by hovering over the **Failed** status 2. Click **View pipeline** for detailed information @@ -160,15 +121,14 @@ See [Handling errors](/docs/guides/database/replication/pipelines-monitoring#han ## Why was the pipeline stopped? -Supabase may automatically stop a pipeline if it encounters too many errors in a short period of time. This protects your source database, your destination, and the Pipelines system from a pipeline repeatedly retrying a failing operation. - -Automatic stopping is a safety mechanism. It prevents repeated pipeline errors from consuming resources or causing wider instability while preserving the pipeline's replication position so you can investigate the underlying issue. +A pipeline is stopped when you select **Stop pipeline**, when its project becomes inactive, or while an operation requires it to restart. A non-retryable configuration, schema, or data error can instead put the pipeline in **Failed** state. Transient connection and destination errors retry automatically when possible. To recover: -1. Check the pipeline error details and replication logs -2. Fix the underlying issue, such as destination connectivity, schema mismatches, permissions, or rate limits -3. Restart the pipeline manually from the [**Database > Replication**](/dashboard/project/_/database/replication) section of the Dashboard +1. Check whether the pipeline is **Stopped**, **Stopping**, or **Failed** +2. If it failed, review the error details and [replication logs](/dashboard/project/_/logs/replication-logs) +3. Fix the underlying issue, such as destination connectivity, schema mismatches, permissions, or rate limits +4. Start or restart the pipeline from [**Database > Replication**](/dashboard/project/_/database/replication) If the same error continues after restart, the pipeline may stop again. Review [Monitor pipeline status](/docs/guides/database/replication/pipelines-monitoring) for troubleshooting steps. @@ -176,19 +136,19 @@ If the same error continues after restart, the pipeline may stop again. Review [ Lag increases when Postgres produces WAL faster than the pipeline can confirm it has processed. Common causes include a slow or rate-limited destination, a pipeline issue, heavy source database activity, long transactions, network latency between the pipeline and source database, or a stopped/disconnected pipeline. -Open [**Database > Replication**](/dashboard/project/_/database/replication), click **View pipeline**, and check **Waiting to sync**, **Room before pausing**, **Last check-in**, **Connected**, and **Slot status**. See [Dealing with replication lag](/docs/guides/database/replication/pipelines-monitoring#dealing-with-replication-lag) for the full investigation and response flow. +Open [**Database > Replication**](/dashboard/project/_/database/replication), click **View pipeline**, and check **Waiting to sync**, **WAL retention remaining**, **Last check-in**, **Connected**, and **Slot status**. See [Dealing with replication lag](/docs/guides/database/replication/pipelines-monitoring#dealing-with-replication-lag) for the full investigation and response flow. ## What does a `Lost` slot status mean? `Lost` means Postgres has already removed WAL files that the pipeline's replication slot needed. The pipeline cannot continue from that slot. -You can recreate the pipeline, or open the pipeline's **Advanced settings**, set **Invalidated slot behavior** to **Recreate**, and restart the pipeline. On restart, the pipeline creates a new replication slot and starts replication from scratch for all tables. This is required for consistency because the old slot can no longer provide every change the pipeline missed. +You can recreate the pipeline, or open the pipeline's **Advanced settings**, set **Invalidated slot behavior** to **Recreate**, and start the pipeline again. The pipeline resets its saved table-sync state, creates a new replication slot, and replaces each destination table through a new initial sync. This destructive restart is required for consistency because the old slot can no longer provide every change the pipeline missed, and the new initial sync is billed as initial sync data. See [Slot statuses](/docs/guides/database/replication/pipelines-monitoring#slot-statuses) for all slot states and what to do next. ## Why is a table in error state? -Table errors occur during the copy phase. To recover, click **View pipeline**, find the affected table, and reset the table state. This will restart the table copy from the beginning. +Table errors occur during the initial sync. To recover, click **View pipeline**, find the affected table, and click its restart action. This restarts that table's initial sync from the beginning, deletes its existing destination data, and bills the copied data as new initial sync data. ## How to verify replication is working @@ -201,7 +161,7 @@ Check the [**Database > Replication**](/dashboard/project/_/database/replication See [Monitor pipeline status](/docs/guides/database/replication/pipelines-monitoring) for comprehensive monitoring instructions. -## How to stop or pause replication +## How to stop replication You can manage your pipeline using the actions menu in the destinations list. See [Managing your pipeline](/docs/guides/database/replication/pipelines#managing-your-pipeline) for details on available actions. @@ -209,6 +169,8 @@ You can manage your pipeline using the actions menu in the destinations list. Se Stopping replication causes changes to queue up in the WAL. +Stopping requests a graceful shutdown, so the pipeline can remain **Stopping** for up to five minutes while in-flight work finishes. Configured pipeline-hour billing continues while the pipeline is stopped. Delete the destination to end that charge. + ## What happens if a project becomes inactive? @@ -223,15 +185,15 @@ When a project is downgraded to the Free Plan, all replication pipelines created ## What happens if a table is deleted at the destination? -If a table is deleted downstream at the destination, the behavior depends on the destination. In general, the replication pipeline will try to recreate the missing table so that all tables in your publication can continue replicating. +Don't delete or modify tables or views managed by Pipelines. For BigQuery, deleting a managed object can stop replication and may require a new, billable initial sync. Pipelines does not guarantee that it will automatically repair or fully resynchronize a destination object that was removed manually. **To permanently remove a table from your destination you have two options:** -**Option 1: Pause the pipeline first** +**Option 1: End replication permanently** -1. Pause or delete your replication pipeline +1. Delete the destination to permanently end replication and pipeline-hour billing. Stopping the pipeline alone does not end pipeline-hour billing. 2. Delete the table at your destination -3. The table will not be recreated since the pipeline is not running +3. Don't restart the pipeline while the table remains in its publication **Option 2: Remove from publication first** @@ -245,29 +207,15 @@ Removing a table from the publication and restarting the pipeline does not delet -## Can data duplicates occur during pipeline operations? - -Yes, data duplicates can occur in certain scenarios when stopping a pipeline. - -When you stop a pipeline (for restarts or updates), the replication process tries to finish processing any transactions that are currently being sent to your destination. The pipeline waits up to a few minutes to allow these in-progress transactions to complete cleanly before stopping. +## Can data be processed more than once? -However, if a transaction in your database takes longer than this waiting period to complete, the pipeline will stop before that entire transaction has been fully processed. When the pipeline starts again, it must restart the incomplete transaction from the beginning to maintain transaction boundaries, which results in some data being sent twice to your destination. +Yes. Pipelines uses at-least-once processing. If a destination accepted a batch but the pipeline did not persist its checkpoint before a retry, restart, or interruption, that batch can be processed again. -### Understanding transaction boundaries - -A transaction is a group of database changes that happen together (for example, all changes within a `BEGIN...COMMIT` block). Postgres logical replication must process entire transactions - it cannot process part of a transaction, stop, and then continue from the middle. This means if a transaction is interrupted, the whole transaction must be replayed when the pipeline resumes. - -**Example scenario**: Suppose you have a large operation that updates 10,000 rows within a single transaction. If this operation takes 10 minutes to complete and you stop the pipeline after 5 minutes (when 5,000 rows have been processed), the pipeline cannot resume from row 5,001. Instead, when it restarts, it must reprocess all 10,000 rows from the beginning, resulting in the first 5,000 rows being sent to your destination twice. - - - -There are currently no plans to implement automatic deduplication. If your use case requires guaranteed exactly-once delivery, you should implement deduplication logic in your downstream systems based on primary keys or other unique identifiers. - - +BigQuery uses the replicated source primary key and CDC ordering metadata to converge on the current table state. Pipelines does not provide a history of each delivery attempt that you can query or guarantee exactly-once event processing. ## Where to find replication logs -Navigate to the [**Logs > Replication**](/dashboard/project/_/logs/explorer) section of the Dashboard to see all pipeline logs. Logs contain diagnostic information. If you're experiencing issues, contact support with your error details. +Navigate to the [**Logs > Replication**](/dashboard/project/_/logs/replication-logs) section of the Dashboard to see all pipeline logs. Logs contain diagnostic information. If you're experiencing issues, contact support with your error details. ## How to get help diff --git a/apps/docs/content/guides/database/replication/pipelines-monitoring.mdx b/apps/docs/content/guides/database/replication/pipelines-monitoring.mdx index f466c8108bdd0..1c7c132aa3697 100644 --- a/apps/docs/content/guides/database/replication/pipelines-monitoring.mdx +++ b/apps/docs/content/guides/database/replication/pipelines-monitoring.mdx @@ -6,51 +6,42 @@ subtitle: 'Track replication status, view logs, and troubleshoot issues.' sidebar_label: 'Monitoring' --- - +<$Partial path="pipelines-public-alpha.mdx" /> -Supabase Pipelines is currently in private alpha. Private alpha features can be unstable and may introduce breaking changes while we evaluate the product direction, refine the feature set, and incorporate customer feedback. - - - -After setting up Pipelines, you can monitor the status and health of your replication pipelines directly from the Dashboard. The pipeline is the active Postgres replication process that continuously streams changes from your database to your destination. +After setting up Supabase Pipelines, you can monitor the status and health of your pipelines directly from the Dashboard. A pipeline first performs an initial sync of existing rows, then uses ongoing replication (CDC) to send subsequent database changes to your destination. ### Viewing pipeline status -To monitor your replication pipelines: +To monitor your pipelines: 1. Navigate to the [**Database > Replication**](/dashboard/project/_/database/replication) section of the Dashboard 2. You'll see a list of all your destinations with their pipeline status -Destinations List - #### Pipeline states Each destination shows its pipeline in one of these states: -| State | Description | -| ------------ | ------------------------------------------------------------------------------- | -| **Stopped** | Pipeline is not running | -| **Starting** | Pipeline is being started | -| **Running** | Pipeline is actively replicating data | -| **Stopping** | Pipeline is being stopped | -| **Failed** | Pipeline has encountered an error (hover over the status to view error details) | +| State | Description | +| -------------- | ------------------------------------------------------------------------------- | +| **Stopped** | Pipeline is not running | +| **Starting** | Pipeline is being started | +| **Running** | Pipeline is actively replicating data | +| **Stopping** | Pipeline is being stopped | +| **Restarting** | Pipeline settings or table state are being applied before replication restarts | +| **Failed** | Pipeline has encountered an error (hover over the status to view error details) | +| **Unknown** | The Dashboard can't currently determine the pipeline status | ### Viewing detailed pipeline metrics For detailed information about a specific pipeline, click **View pipeline** on the destination. This opens the pipeline status page where you can monitor replication performance and table states. Pipeline status view #### Replication lag metrics @@ -61,17 +52,17 @@ The destinations list also shows a compact lag value. This value is byte-based: The detailed status page shows: -| Metric | What it means | What to watch for | -| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Waiting to sync** | Bytes of WAL between the pipeline's confirmed flush position and the current Postgres WAL position. This is the main byte-based replication lag. | A value that keeps growing means the pipeline is receiving changes more slowly than Postgres produces them. | -| **Room before pausing** | How much WAL can still accumulate before Postgres can no longer safely keep all WAL needed by the replication slot. This is controlled by `max_slot_wal_keep_size`. | A small or shrinking value means the slot is getting closer to being invalidated. `Unlimited` means Postgres is not reporting a slot WAL retention limit. | -| **Last check-in** | How long it has been since the pipeline last sent replication feedback to Postgres. | An old value can mean the pipeline is stopped, disconnected, overloaded, or unable to make progress. | -| **Connected** | Whether the pipeline's replication slot is active and currently being used. | `Not connected` while the pipeline should be running usually means you should check pipeline status and logs. | -| **Slot status** | How safely Postgres is keeping the WAL files the pipeline still needs. | `Unreserved` and `Lost` require action. See [Slot statuses](#slot-statuses). | +| Metric | What it means | What to watch for | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Waiting to sync** | Bytes of WAL between the pipeline's confirmed flush position and the current Postgres WAL position. This is the main byte-based replication lag. | A value that keeps growing means the pipeline is receiving changes more slowly than Postgres produces them. | +| **WAL retention remaining** | How much WAL can still accumulate before the replication slot is at risk of becoming unusable. This is controlled by `max_slot_wal_keep_size`. | A small or shrinking value means you should investigate before required WAL is removed. `Unlimited` means Postgres is not reporting a slot WAL retention limit. | +| **Last check-in** | How long it has been since the pipeline last sent replication feedback to Postgres. | An old value can mean the pipeline is stopped, disconnected, overloaded, or unable to make progress. | +| **Connected** | Whether the pipeline's replication slot is active and currently being used. | `Not connected` while the pipeline should be running usually means you should check pipeline status and logs. | +| **Slot status** | How safely Postgres is keeping the WAL files the pipeline still needs. | `Unreserved` and `Lost` require action. See [Slot statuses](#slot-statuses). | -Pipelines uses one main pipeline replication slot for ongoing changes. During the initial copy phase, it can also create temporary table-sync replication slots. These temporary slots let multiple tables copy in parallel, make large table copies faster, and allow individual tables to be retried or copied again without restarting the whole pipeline. +Pipelines uses one main pipeline replication slot for ongoing replication. During the initial sync, it can also create temporary table-sync replication slots. These temporary slots let multiple tables sync in parallel, make large initial syncs faster, and allow individual tables to be retried without restarting the whole pipeline. -Temporary table-sync slots show the same kind of lag and slot health metrics while they are active. After a table finishes copying and catches up, its temporary slot is removed and ongoing changes continue through the main pipeline slot. For overall replication health, focus first on the main pipeline slot. +Temporary table-sync slots show the same kind of lag and slot health metrics while they are active. After a table finishes its initial sync and catches up, its temporary slot is removed and ongoing replication continues through the main pipeline slot. For overall replication health, focus first on the main pipeline slot. #### Slot statuses @@ -89,17 +80,20 @@ Replication slot status tells you whether Postgres is still retaining the WAL th The pipeline status page also shows the state of individual tables being replicated. Each table can be in one of these states: -| State | Description | -| ----------- | ---------------------------------------------------------------------- | -| **Queue** | Table is getting ready to be copied | -| **Copying** | Initial snapshot of the table is being copied | -| **Copied** | Table snapshot is complete and getting ready for real-time replication | -| **Live** | Table is now replicating data in near real-time | -| **Error** | Table has experienced an error during replication | +| State | Description | +| ----------------- | ------------------------------------------------------------------------ | +| **Queued** | Table is waiting for the pipeline to begin its initial sync | +| **Copying** | Existing rows are being copied during the initial sync | +| **Copied** | Initial sync is complete and the table is preparing to replicate changes | +| **Live** | Table is now receiving ongoing replication | +| **Error** | Table has experienced an error during replication | +| **Restarting** | The table's initial sync is being restarted | +| **Not Available** | Table state is temporarily unavailable while the pipeline changes state | +| **Unknown** | The Dashboard received a table state it doesn't recognize | ### Dealing with replication lag -Replication lag means the pipeline is behind the source database. Some lag is expected during the initial copy phase, after a burst of writes, or after restarting a stopped pipeline. Lag becomes a problem when it keeps increasing, when **Room before pausing** is running low, or when the slot status moves to **Unreserved** or **Lost**. +Replication lag means the pipeline is behind the source database. Some lag is expected during the initial sync, after a burst of writes, or after restarting a stopped pipeline. Lag becomes a problem when it keeps increasing, when **WAL retention remaining** is running low, or when the slot status moves to **Unreserved** or **Lost**. Lag can come from several places: @@ -108,26 +102,26 @@ Lag can come from several places: - **Source database activity**: Postgres is producing WAL faster than the pipeline can consume it, often during bulk writes, migrations, or backfill jobs. - **Network latency**: Latency or instability between the pipeline and source database can slow down WAL streaming. - **Stopped or disconnected pipeline**: When a pipeline is stopped, disconnected, or failed, Postgres keeps WAL for the slot until the retention limit is reached. -- **Slow initial table copy**: A temporary table-sync slot can fall behind if a table is copied more slowly than new changes are written to that table. +- **Slow initial sync**: A temporary table-sync slot can fall behind if existing rows are copied more slowly than new changes are written to that table. -#### Initial copy and table-sync slots +#### Initial sync and table-sync slots -A common initial sync issue happens when a large or busy table is still in **Copying** while new rows keep being inserted or updated. The temporary table-sync slot needs to keep the changes that happen during the copy. If the copy is too slow compared to the table's write rate, the slot can move to **Unreserved** and then **Lost** if Postgres removes changes the copy still needs. +A common initial sync issue happens when a large or busy table is still in **Copying** while new rows keep being inserted or updated. The temporary table-sync slot retains changes that happen during the initial sync. If copying is too slow compared to the table's write rate, the slot can move to **Unreserved** and then **Lost** if Postgres removes changes the sync still needs. -When a table-sync slot is lost, the affected table needs to be copied again. Tune the copy settings, then retry the table copy: +When a table-sync slot is lost, the affected table needs to run its initial sync again. Tune the copy settings, then retry the table: -- Increase **Copy connections per table** when one large table is the bottleneck. This lets the pipeline copy chunks of that table in parallel, up to the point where the source database or network becomes the limit. +- Increase **Copy connections per table** when one large table is the bottleneck. This lets the pipeline copy chunks of that table over multiple source connections, up to the point where the source database, network, or destination becomes the limit. - Increase **Table sync workers** when several tables need to copy at the same time. Each worker can copy one table, and each worker uses an additional temporary replication slot during initial sync. -- If possible, run the initial copy during a quieter write period or reduce bulk writes until the table reaches **Live**. +- If possible, run the initial sync during a quieter write period or reduce bulk writes until the table reaches **Live**. After the affected table finishes copying and catches up, the temporary slot is deleted. The table then continues through the main pipeline replication slot. #### Investigate the lag 1. Open [**Database > Replication**](/dashboard/project/_/database/replication) and check the destination's lag column. -2. Click **View pipeline** and check **Waiting to sync**, **Room before pausing**, **Last check-in**, **Connected**, and **Slot status**. -3. Check table states. Tables in **Copying** can create temporary lag while the initial snapshot catches up to live changes. If a table-sync slot is **Unreserved** or **Lost**, tune copy parallelism and retry the affected table copy. -4. Open [**Logs > Replication**](/dashboard/project/_/logs/explorer) and look for destination errors, retries, rate limits, schema errors, or repeated restarts. +2. Click **View pipeline** and check **Waiting to sync**, **WAL retention remaining**, **Last check-in**, **Connected**, and **Slot status**. +3. Check table states. Tables in **Copying** can create temporary lag while the initial sync catches up to ongoing changes. If a table-sync slot is **Unreserved** or **Lost**, tune copy parallelism and retry the affected table's initial sync. +4. Open [**Logs > Replication**](/dashboard/project/_/logs/replication-logs) and look for destination errors, retries, rate limits, schema errors, or repeated restarts. 5. Compare the lag trend with recent database activity, such as imports, migrations, bulk updates, or long transactions. #### Respond based on the slot status @@ -146,7 +140,7 @@ After the affected table finishes copying and catches up, the temporary slot is - Avoid leaving pipelines stopped for long periods while the source database is still receiving writes. - Schedule bulk updates, imports, and migrations during lower-traffic windows when possible. - For BigQuery, verify that service account permissions, table requirements, and replica identity settings match the [BigQuery destination guide](/docs/guides/database/replication/bigquery). -- If the initial copy is the bottleneck, review **Table sync workers** and **Copy connections per table** in the pipeline's advanced settings. Increasing them can speed up copying, but it uses more database connections and replication slots. +- If the initial sync is the bottleneck, review **Table sync workers** and **Copy connections per table** in the pipeline's advanced settings. Increasing either can use more source database connections; increasing table sync workers can also use more temporary replication slots. ### Handling errors @@ -154,15 +148,7 @@ Errors can occur at two levels: per table or per pipeline. #### Table errors -Table errors occur during the copy phase and affect individual tables. These errors can be retried without stopping the entire pipeline. - -Table Error Details +Table errors occur during the initial sync and affect individual tables. These errors can be retried without stopping the entire pipeline. **Viewing table error details:** @@ -172,31 +158,17 @@ height={2146} **Recovering from table errors:** -When a table encounters an error during the copy phase, you can reset the table state. This will restart the table copy from the beginning. +When a table encounters an error during the initial sync, you can reset the table state. This restarts that table's initial sync from the beginning. #### Pipeline errors -Pipeline errors occur during the streaming phase (Live state) and affect the entire pipeline. When streaming data, if an error occurs, the entire pipeline will stop and enter a **Failed** state. This prevents data loss by ensuring no changes are skipped. - - - -When a pipeline error occurs, you'll receive an email notification immediately. This ensures you're promptly notified of any issues so you can take action to resolve them. - - - -Pipeline error details +Pipeline errors can occur during startup or ongoing replication and affect the entire pipeline. If a non-retryable pipeline-level error occurs, the entire pipeline stops and enters a **Failed** state instead of silently skipping the failure. **Viewing pipeline error details:** 1. Hover over the **Failed** status in the destinations list to see a quick error summary 2. Click **View pipeline** for comprehensive error information -3. Navigate to the [**Logs > Replication**](/dashboard/project/_/logs/explorer) section of the Dashboard for detailed error logs +3. Navigate to the [**Logs > Replication**](/dashboard/project/_/logs/replication-logs) section of the Dashboard for detailed error logs **Recovering from pipeline errors:** @@ -208,18 +180,11 @@ To recover from a pipeline error, you'll need to: ### Viewing logs -To see detailed logs for all your replication pipelines: +To see detailed logs for all your pipelines: -1. Navigate to the [**Logs > Replication**](/dashboard/project/_/logs/explorer) section of the Dashboard +1. Navigate to the [**Logs > Replication**](/dashboard/project/_/logs/replication-logs) section of the Dashboard 2. Select **Replication** from the log source filter -3. You'll see all logs from your replication pipelines - -Replication Logs +3. You'll see all logs from your pipelines @@ -243,7 +208,7 @@ If you see a **Failed** status: 1. Hover over the status to see the error summary 2. Click **View pipeline** to see detailed error information 3. Check table states to identify which tables are affected -4. Navigate to the [**Logs > Replication**](/dashboard/project/_/logs/explorer) section of the Dashboard for full error details +4. Navigate to the [**Logs > Replication**](/dashboard/project/_/logs/replication-logs) section of the Dashboard for full error details 5. For table errors, attempt to reset the affected tables #### Monitoring performance @@ -261,7 +226,7 @@ If you notice issues with your replication: 1. **Check pipeline state**: Ensure the pipeline is in **Running** state 2. **Review table states**: Identify tables in **Error** state -3. **Check logs**: Navigate to the [**Logs > Replication**](/dashboard/project/_/logs/explorer) section of the Dashboard for detailed error information +3. **Check logs**: Navigate to the [**Logs > Replication**](/dashboard/project/_/logs/replication-logs) section of the Dashboard for detailed error information 4. **Verify publication**: Ensure your Postgres publication is properly configured 5. **Monitor replication lag**: High lag may indicate performance issues diff --git a/apps/docs/content/guides/database/replication/pipelines.mdx b/apps/docs/content/guides/database/replication/pipelines.mdx index e84d8ec39bdab..75bee3a3dcff7 100644 --- a/apps/docs/content/guides/database/replication/pipelines.mdx +++ b/apps/docs/content/guides/database/replication/pipelines.mdx @@ -1,23 +1,28 @@ --- id: 'pipelines' title: 'Set up Pipelines' -description: 'Create a managed replication pipeline using Postgres logical replication.' -subtitle: 'Configure publications, destinations, and managed replication pipelines.' +description: 'Create a Supabase Pipeline using Postgres logical replication.' +subtitle: 'Configure publications, destinations, and Supabase Pipelines.' sidebar_label: 'Setting up' --- - +<$Partial path="pipelines-public-alpha.mdx" /> -Supabase Pipelines is currently in private alpha. Private alpha features can be unstable and may introduce breaking changes while we evaluate the product direction, refine the feature set, and incorporate customer feedback. +Supabase Pipelines is a managed CDC product for moving data from Supabase Postgres to supported destination systems. It uses **Postgres logical replication** with the open-source [Supabase ETL engine](https://github.com/supabase/etl). You choose a destination in the Dashboard, and Supabase runs the pipeline that sends database changes to that destination. - +Pipelines has two replication phases: + +- **Initial sync**: A one-time copy of the existing rows in the published tables. +- **Ongoing replication (CDC)**: Continuously captures and applies subsequent inserts, updates, deletes, and truncates. -Pipelines is a managed CDC feature for creating replication pipelines from Supabase Postgres to destination systems. It uses **Postgres logical replication** with the open-source [Supabase ETL engine](https://github.com/supabase/etl). You choose a destination in the Dashboard, and Supabase runs the pipeline that streams database changes to that destination. +Managed Pipelines run in **AWS `eu-central-1` (Frankfurt)**. Choose a destination region as close as possible to Frankfurt to reduce network latency and replication lag. ## Pricing <$Partial path="billing/pricing/pricing_pipelines.mdx" /> +For billing examples and optimization guidance, see [Manage Pipelines usage](/docs/guides/platform/manage-your-usage/pipelines). + ## Setup overview Pipelines requires two main components: a **Postgres publication** (defines what to replicate) and a **destination** (where data is sent). Supabase runs the managed pipeline that reads from the publication and writes to the destination. Follow these steps to set up your replication pipeline. @@ -30,9 +35,12 @@ If you already have a Postgres publication set up, you can skip to [Step 2: Enab ### Step 1: Create a Postgres publication -A Postgres publication defines which tables and change types will be replicated from your database. You create publications using SQL. +A Postgres publication defines which tables and change types will be replicated from your database. You can create a basic publication in the Dashboard while configuring the destination, or use SQL when you need column lists, row filters, schema-wide publications, or other advanced options. + +- **Dashboard**: Continue to [Step 2](#step-2-enable-pipelines). In Step 3, open the **Publication** selector, click **New publication**, enter a name, and select at least one table. +- **SQL**: Create the publication now using one of the examples below, then select it when you configure the destination. -#### Creating a publication +#### Creating a publication with SQL The following SQL examples assume you have `users` and `orders` tables in your database. @@ -64,6 +72,12 @@ create publication pub_all_tables for all tables; This tracks changes for all tables in your database. + + +`FOR ALL TABLES` includes tables in Supabase-managed schemas, including the internal `etl` tables that Pipelines creates. Prefer `FOR TABLES IN SCHEMA public` or list the application tables explicitly unless you intend to replicate every eligible table in the database. + + + #### Advanced publication options ##### Selecting specific columns @@ -122,11 +136,15 @@ Use `publish_via_partition_root = true` when you want analytics queries to read Publications created from the Dashboard replication flow use `publish_via_partition_root = true`. If you create or alter a publication manually with SQL, set this option explicitly so the destination shape matches what you expect. -On Postgres 15 and newer, row filters on partition publications apply during both the initial copy and streaming phases. Pipelines uses the row filter attached to the effective publication table entry: the published partition root when `publish_via_partition_root = true`, and the published leaf relation when `publish_via_partition_root = false`. +On Postgres 15 and newer, row filters on partition publications apply during both the initial sync and ongoing replication. Pipelines uses the row filter attached to the effective publication table entry: the published partition root when `publish_via_partition_root = true`, and the published leaf relation when `publish_via_partition_root = false`. + +The publication setting controls which Postgres relation becomes a destination table. It does not copy the source table's physical partitioning configuration, partition key, or partition bounds to BigQuery. -With `publish_via_partition_root = true`, truncating an individual leaf partition is not replicated as a truncate event for the published parent. If you need truncate replication, run `TRUNCATE` on the published partition root. +With `publish_via_partition_root = true`, truncating an individual leaf partition is not replicated as a truncate event for the published parent. This is useful for append-only data such as events: you can truncate old leaf partitions to keep Postgres storage bounded while retaining the rows already copied to the destination. If you want the destination to be truncated too, run `TRUNCATE` on the published partition root. + +Rows retained only in the destination aren't a permanent archive. A table reset or full pipeline initial sync rebuilds the destination from the rows that still exist in Postgres. @@ -148,61 +166,58 @@ Before creating a managed replication pipeline, enable Pipelines for your projec 3. Select a Pipelines destination, such as **BigQuery** 4. Click **Enable Pipelines** -Enable Pipelines - ### Step 3: Configure a destination Once Pipelines is enabled and you have a Postgres publication, configure a destination. The destination is where your replicated data will be stored, while the pipeline is the active Postgres replication process that continuously streams changes from your database to that destination. #### Choose and configure your destination -Follow these steps to configure your destination. Each destination has its own setup requirements and behavior. **BigQuery** is currently available, and we are working on new destinations. +{/* supa-mdx-lint-disable-next-line Rule003Spelling */} +Follow these steps to configure your destination. Each destination has its own setup requirements and behavior. **BigQuery** is currently available. You can [request early access to ClickHouse, Snowflake, and DuckLake](/go/supabase-pipelines-new-destinations). 1. Navigate to the [**Database > Replication**](/dashboard/project/_/database/replication) section of the Dashboard 2. Click **Add destination** if the destination side panel isn't already open -3. Configure the general settings: +3. Select the destination type + +4. Configure the destination details: - **Destination name**: A name to identify this destination - - **Publication**: The publication to replicate data from (created in [Step 1](#step-1-create-a-postgres-publication)) - - **Destination type**: Select the destination you want to use + - **Publication**: Select an existing publication, or click **New publication** to create one by choosing a name and at least one table + - **Region**: Managed Pipelines run in the fixed **AWS `eu-central-1` (Frankfurt)** region. This can't be changed. In your destination provider, choose a nearby dataset, warehouse, or storage bucket region. -4. Configure the destination-specific settings. See the destination guide for required credentials, permissions, and limitations: +5. Configure the destination-specific settings. See the destination guide for required credentials, permissions, and limitations: - [BigQuery](/docs/guides/database/replication/bigquery) -5. Optionally expand **Advanced settings** to tune pipeline behavior. These settings apply to the pipeline rather than the destination: +6. Optionally expand **Advanced settings** to tune pipeline behavior: + + | Setting | Default | Allowed values | Description | + | ------------------------------ | -------------------- | ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | + | **Batch wait time** | `10000` milliseconds | Whole milliseconds, `0` or greater | Maximum time after the first buffered initial-sync row or ongoing change before the pipeline flushes a partially filled batch. Internal size and memory limits can flush it earlier. Lower values can reduce batching delay; higher values can improve destination write efficiency. | + | **Table sync workers** | `4` workers | Whole number greater than `0` | Maximum number of tables synced in parallel during the initial sync. Each active table sync temporarily uses one additional replication slot, up to `N + 1` slots including the pipeline's main slot. | + | **Copy connections per table** | `4` connections | Whole number greater than `0` | Maximum source database connections used to copy one table in parallel. With multiple table sync workers, source connection usage can scale with both settings. More connections can speed up large tables until the source database, network, or destination becomes the bottleneck. | + | **Invalidated slot behavior** | `Error` | `Error` or `Recreate` | What happens when the main replication slot can no longer continue from retained WAL. **Error** blocks startup for manual recovery. **Recreate** resets table sync state, rebuilds the slot on the next start, and runs the initial sync again for every replicated table. | - | Setting | Default | Description | - | ------------------------------ | -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | - | **Batch wait time** | `10000` milliseconds | Maximum time the pipeline waits to collect additional changes before flushing them. Lower values reduce replication latency. Higher values can improve batching efficiency. | - | **Table sync workers** | `4` workers | Number of tables copied in parallel during the initial snapshot phase. Each worker uses one replication slot, up to `N + 1` total replication slots while syncing. | - | **Copy connections per table** | `2` connections | Number of parallel database connections each table copy can use during the initial sync. Increasing this can speed up large table copies, but uses more database connections. | - | **Invalidated slot behavior** | `Error` | What the pipeline does when its replication slot is invalidated. **Error** blocks startup so you can recover manually. **Recreate** rebuilds the slot on the next pipeline restart and starts replication from scratch for all tables. | + Leave these settings at their defaults unless you need to tune initial sync speed, latency, or recovery behavior. - Leave these settings at their defaults unless you need to tune initial copy speed, latency, or recovery behavior. + Use **Invalidated slot behavior** carefully. If **Recreate** is selected and the pipeline starts after Postgres has invalidated the main replication slot, the pipeline resets its saved table-sync state, creates a new slot, and replaces each destination table through a new initial sync. This destructive restart is required for consistency because the old slot can no longer provide every change the pipeline missed, and the new initial sync is billed as initial sync data. - Use **Invalidated slot behavior** carefully. If **Recreate** is selected and the pipeline restarts after Postgres has invalidated the main replication slot, the pipeline creates a new slot and copies all replicated tables again. This full restart is required for consistency because the old slot can no longer provide every change the pipeline missed. +7. Click **Create and start pipeline** to begin replication -6. Click **Create and start pipeline** to begin replication +Pipeline cost confirmation showing the initial sync estimate and ongoing replication prices -Your replication pipeline now starts copying data from your database to your destination. +The pipeline begins the initial sync from your database to your destination. ### Step 4: Monitor your pipeline After you create and start the pipeline, its destination appears in the destinations list. You can monitor the pipeline's status and performance from the Dashboard. -Replication pipelines destinations list - For comprehensive monitoring instructions including pipeline states, metrics, and logs, see [Monitor pipeline status](/docs/guides/database/replication/pipelines-monitoring). ### Managing your pipeline @@ -210,33 +225,28 @@ For comprehensive monitoring instructions including pipeline states, metrics, an You can manage your pipeline from the destinations list using the actions menu. Pipeline actions Available actions: - **Start pipeline**: Begin replication for a stopped pipeline -- **Stop**: Pause replication (changes will queue up in the WAL) +- **Update available**: Review and apply the latest managed pipeline version when an update is available +- **Stop pipeline**: Request a graceful stop. The pipeline can remain **Stopping** for up to five minutes while in-flight work finishes. New changes queue in the WAL, and configured pipeline-hour billing continues while stopped. - **Restart pipeline**: Stop and start the pipeline (required after publication changes) - **Edit destination**: Modify destination settings like credentials or advanced options -- **Delete**: Remove the destination and permanently stop replication +- **Delete destination**: Remove the destination and permanently stop replication {/* supa-mdx-lint-disable-next-line Rule001HeadingCase */} ### Disabling Pipelines -To turn off Pipelines for a project, delete all replication pipelines first. After all pipelines are removed, open the three-dot actions menu on the Replication page and click **Disable Pipelines**. - -Disable Pipelines from the Replication page actions menu +To turn off Pipelines for a project, delete all Pipelines destinations first. After all destinations are removed, open the three-dot actions menu on the Replication page and click **Disable Pipelines**. For cleanup details, see [What happens when you disable Pipelines?](/docs/guides/database/replication/pipelines-faq#what-happens-when-you-disable-pipelines). @@ -280,7 +290,7 @@ If your Postgres publication uses `FOR ALL TABLES` or `FOR TABLES IN SCHEMA`, ne -When a table is deleted at the destination, the behavior depends on the destination. In general, the pipeline tries to recreate the table so replication can continue. To permanently delete a table, pause the pipeline first or remove it from the publication before deleting. See the [Pipelines FAQ](/docs/guides/database/replication/pipelines-faq#what-happens-if-a-table-is-deleted-at-the-destination) for details. +When a table is deleted at the destination, the behavior depends on the destination. In general, the pipeline tries to recreate the table so replication can continue. To permanently delete a table, stop the pipeline first or remove it from the publication before deleting. See the [Pipelines FAQ](/docs/guides/database/replication/pipelines-faq#what-happens-if-a-table-is-deleted-at-the-destination) for details. @@ -293,17 +303,17 @@ Schema change support depends on the destination. BigQuery is currently the only Once configured, a replication pipeline: 1. **Captures** changes from your Postgres database using Postgres publications and logical replication -2. **Streams** the changes through the managed pipeline +2. **Sends** the changes through the managed pipeline 3. **Loads** the data to your destination -Pipelines automatically optimizes how changes are delivered to the destination. It currently performs data extraction and loading only, without transformation - your data is replicated as-is to the destination. +Pipelines automatically optimizes how changes are delivered to the destination. It maps published source columns and values to destination-compatible names and types, but doesn't provide user-defined transformations. ### Troubleshooting If you encounter issues during setup: - **Publication not appearing**: Ensure you created the Postgres publication via SQL and refresh the dashboard -- **Tables not showing in publication**: Verify your tables have primary keys (required for Postgres logical replication) +- **Tables not showing in publication**: Verify your tables meet the requirements of the selected destination. BigQuery requires a source primary key and requires the publication to include its columns. - **Pipeline failed to start**: Check the error message in the status view for specific details - **No data being replicated**: Verify your Postgres publication includes the correct tables and event types @@ -313,13 +323,13 @@ For more troubleshooting help, see the [Pipelines FAQ](/docs/guides/database/rep Pipelines has the following limitations: -- **Primary keys required**: Tables must have primary keys (Postgres logical replication requirement) +- **Primary keys**: Requirements are destination-specific. Postgres can publish inserts for a table without a primary key, but BigQuery Pipelines always requires a source primary key and requires the publication to include its columns. - **Custom data types**: Custom values replicate as strings. Check that your destination can interpret those string values correctly. - **Generated columns**: Generated columns are skipped. Use triggers to store derived values in regular columns if you need them in the destination. -- **Replica identity**: `REPLICA IDENTITY FULL` is strongly recommended. Updates and deletes need enough row data to apply changes correctly at the destination. +- **Replica identity**: Requirements are destination-specific. Updates and deletes need enough row identity to apply safely. See [BigQuery source table requirements](/docs/guides/database/replication/bigquery#source-table-requirements) for the supported modes. - **Schema changes**: Currently in beta and limited to BigQuery -- **No data transformation**: Data is replicated as-is without transformation -- **Data duplicates**: Duplicates can occur when stopping a pipeline if your database has transactions that take longer than a few minutes to complete. See [Can data duplicates occur during pipeline operations?](/docs/guides/database/replication/pipelines-faq#can-data-duplicates-occur-during-pipeline-operations) for details +- **No user-defined transformations**: Pipelines performs destination-compatible type and name mapping, but doesn't run custom transformations +- **At-least-once processing**: A retry, restart, or interruption can cause a batch to be processed again. BigQuery uses primary-key-based CDC to converge on the current table state. See [Can data be processed more than once?](/docs/guides/database/replication/pipelines-faq#can-data-be-processed-more-than-once) for details. Destination-specific limitations, such as BigQuery's row size limits, are documented in each destination guide. diff --git a/apps/docs/content/guides/getting-started/features.mdx b/apps/docs/content/guides/getting-started/features.mdx index 285be68ead412..c25c6cf7e8952 100644 --- a/apps/docs/content/guides/getting-started/features.mdx +++ b/apps/docs/content/guides/getting-started/features.mdx @@ -199,61 +199,61 @@ Features in Beta are tested by an external penetration tester for security issue In addition to the Beta requirements, features in GA are covered by the [uptime SLA](/sla). -| Product | Feature | Stage | Available on self-hosted | -| -------------- | -------------------------- | --------------- | ------------------------------------------- | -| Database | Postgres | `GA` | ✅ | -| Database | Vector Database | `GA` | ✅ | -| Database | Auto-generated Rest API | `GA` | ✅ | -| Database | Auto-generated GraphQL API | `GA` | ✅ | -| Database | Webhooks | `beta` | ✅ | -| Database | Vault | `public alpha` | ✅ | -| Database | Replication | `private alpha` | N/A | -| Platform | | `GA` | ✅ | -| Platform | Point-in-Time Recovery | `GA` | 🚧 [wal-g](https://github.com/wal-g/wal-g) | -| Platform | Custom Domains | `GA` | N/A | -| Platform | Network Restrictions | `GA` | N/A | -| Platform | SSL enforcement | `GA` | N/A | -| Platform | Branching | `beta` | N/A | -| Platform | Terraform Provider | `public alpha` | N/A | -| Platform | Read Replicas | `GA` | N/A | -| Platform | Log Drains | `public alpha` | ✅ | -| Platform | MCP | `public alpha` | ✅ | -| Platform | PrivateLink | `beta` | N/A | -| Studio | | `GA` | ✅ | -| Studio | SSO | `GA` | ✅ | -| Studio | Column Privileges | `public alpha` | ✅ | -| Realtime | Postgres Changes | `GA` | ✅ | -| Realtime | Broadcast | `GA` | ✅ | -| Realtime | Presence | `GA` | ✅ | -| Realtime | Broadcast Authorization | `public beta` | ✅ | -| Realtime | Presence Authorization | `public beta` | ✅ | -| Realtime | Broadcast from Database | `public beta` | ✅ | -| Storage | | `GA` | ✅ | -| Storage | CDN | `GA` | 🚧 [Cloudflare](https://www.cloudflare.com) | -| Storage | Smart CDN | `GA` | 🚧 [Cloudflare](https://www.cloudflare.com) | -| Storage | Image Transformations | `GA` | ✅ | -| Storage | Resumable Uploads | `GA` | ✅ | -| Storage | S3 compatibility | `GA` | ✅ | -| Edge Functions | | `GA` | ✅ | -| Edge Functions | Regional Invocations | `GA` | ✅ | -| Edge Functions | NPM compatibility | `GA` | ✅ | -| Auth | | `GA` | ✅ | -| Auth | Email login | `GA` | ✅ | -| Auth | Social login | `GA` | ✅ | -| Auth | Phone login | `GA` | ✅ | -| Auth | Passwordless login | `GA` | ✅ | -| Auth | SSO with SAML | `GA` | ✅ | -| Auth | Authorization via RLS | `GA` | ✅ | -| Auth | CAPTCHA protection | `GA` | ✅ | -| Auth | Server-side Auth | `beta` | ✅ | -| Auth | Third-Party Auth | `GA` | ✅ | -| Auth | Hooks | `beta` | ✅ | -| CLI | | `GA` | ✅ Works with self-hosted | -| Management API | | `GA` | N/A | -| Client Library | JavaScript | `GA` | N/A | -| Client Library | Flutter | `GA` | N/A | -| Client Library | Swift | `GA` | N/A | -| Client Library | Python | `beta` | N/A | +| Product | Feature | Stage | Available on self-hosted | +| -------------- | -------------------------- | -------------- | ------------------------------------------- | +| Database | Postgres | `GA` | ✅ | +| Database | Vector Database | `GA` | ✅ | +| Database | Auto-generated Rest API | `GA` | ✅ | +| Database | Auto-generated GraphQL API | `GA` | ✅ | +| Database | Webhooks | `beta` | ✅ | +| Database | Vault | `public alpha` | ✅ | +| Database | Supabase Pipelines | `public alpha` | N/A | +| Platform | | `GA` | ✅ | +| Platform | Point-in-Time Recovery | `GA` | 🚧 [wal-g](https://github.com/wal-g/wal-g) | +| Platform | Custom Domains | `GA` | N/A | +| Platform | Network Restrictions | `GA` | N/A | +| Platform | SSL enforcement | `GA` | N/A | +| Platform | Branching | `beta` | N/A | +| Platform | Terraform Provider | `public alpha` | N/A | +| Platform | Read Replicas | `GA` | N/A | +| Platform | Log Drains | `public alpha` | ✅ | +| Platform | MCP | `public alpha` | ✅ | +| Platform | PrivateLink | `beta` | N/A | +| Studio | | `GA` | ✅ | +| Studio | SSO | `GA` | ✅ | +| Studio | Column Privileges | `public alpha` | ✅ | +| Realtime | Postgres Changes | `GA` | ✅ | +| Realtime | Broadcast | `GA` | ✅ | +| Realtime | Presence | `GA` | ✅ | +| Realtime | Broadcast Authorization | `public beta` | ✅ | +| Realtime | Presence Authorization | `public beta` | ✅ | +| Realtime | Broadcast from Database | `public beta` | ✅ | +| Storage | | `GA` | ✅ | +| Storage | CDN | `GA` | 🚧 [Cloudflare](https://www.cloudflare.com) | +| Storage | Smart CDN | `GA` | 🚧 [Cloudflare](https://www.cloudflare.com) | +| Storage | Image Transformations | `GA` | ✅ | +| Storage | Resumable Uploads | `GA` | ✅ | +| Storage | S3 compatibility | `GA` | ✅ | +| Edge Functions | | `GA` | ✅ | +| Edge Functions | Regional Invocations | `GA` | ✅ | +| Edge Functions | NPM compatibility | `GA` | ✅ | +| Auth | | `GA` | ✅ | +| Auth | Email login | `GA` | ✅ | +| Auth | Social login | `GA` | ✅ | +| Auth | Phone login | `GA` | ✅ | +| Auth | Passwordless login | `GA` | ✅ | +| Auth | SSO with SAML | `GA` | ✅ | +| Auth | Authorization via RLS | `GA` | ✅ | +| Auth | CAPTCHA protection | `GA` | ✅ | +| Auth | Server-side Auth | `beta` | ✅ | +| Auth | Third-Party Auth | `GA` | ✅ | +| Auth | Hooks | `beta` | ✅ | +| CLI | | `GA` | ✅ Works with self-hosted | +| Management API | | `GA` | N/A | +| Client Library | JavaScript | `GA` | N/A | +| Client Library | Flutter | `GA` | N/A | +| Client Library | Swift | `GA` | N/A | +| Client Library | Python | `beta` | N/A | - ✅ = Fully Available - 🚧 = Available, but requires external tools or configuration diff --git a/apps/docs/content/guides/platform/manage-your-usage/pipelines.mdx b/apps/docs/content/guides/platform/manage-your-usage/pipelines.mdx index 76540f3b5c4d9..c468004e748f2 100644 --- a/apps/docs/content/guides/platform/manage-your-usage/pipelines.mdx +++ b/apps/docs/content/guides/platform/manage-your-usage/pipelines.mdx @@ -5,11 +5,17 @@ title: 'Manage Pipelines usage' ## What you are charged for -You are charged for all pipelines configured, initial/backfill data and replicated data. Pipelines are charged by the hour for as long as they are configured. +You are charged for configured pipelines, initial sync data, and ongoing replication data. Pipelines are charged by the hour for as long as they are configured, including while they are stopped. + +- **Pipeline hours** measure how long each pipeline remains configured. Delete a pipeline to end this charge. +- **Initial sync data** is the volume of destination-write data processed when a table is first synchronized or synchronized again. +- **Ongoing replication data** is the volume of destination-write data processed for subsequent database changes. It depends on how much your published data changes, not on the source table size, WAL size, or destination's compressed storage size. + +Destination-provider charges are separate. For example, Google Cloud can charge for BigQuery ingestion, storage, and CDC compute. ### Usage on your invoice -Usage is shown as "ETL Pipeline Hours", "ETL Copy Backfill Data GB" and "ETL Replicated Data GB" on your invoice. +Usage is shown as "ETL Pipeline Hours", "ETL Copy Backfill Data GB", and "ETL Replicated Data GB" on your invoice. ## Pricing @@ -17,9 +23,9 @@ Usage is shown as "ETL Pipeline Hours", "ETL Copy Backfill Data GB" and "ETL Rep ## Billing examples -### Single project with no backfill +### Billing period without an initial sync -The project has an active pipeline for the entire month. +The project has a configured pipeline for the entire month. | Line Item | Units | Costs | | ----------------------------- | --------- | ---------------------------- | @@ -31,9 +37,9 @@ The project has an active pipeline for the entire month. | Compute Credits | | - | | **Total** | | **** | -### Multiple projects with backfills +### Multiple projects with initial syncs -Multiple projects had an active pipeline for the entire month and had initial backfills and replicated data. +Multiple projects had a configured pipeline for the entire month and processed initial sync data and ongoing replication data. | Line Item | Units | Costs | | ----------------------------------- | --------- | ----------------------------- | @@ -53,10 +59,9 @@ Multiple projects had an active pipeline for the entire month and had initial ba | Compute Credits | | - | | **Total** | | **** | -### After disabling an add-on for a day +### After deleting a pipeline after one day -Project add-ons are billed in arrears based on how many hours you used them. -If you remove the add-on, you are no longer billed from the time of removal onward. +Pipeline hours are billed in arrears for as long as a pipeline is configured, including while it is stopped. After you delete the pipeline, pipeline-hour billing ends. | Line Item | Hours | Costs | | ----------------------------- | ----- | --------------------------- | @@ -71,7 +76,6 @@ If you remove the add-on, you are no longer billed from the time of removal onwa ## Optimize usage -- Only replicate tables you need to replicate -- If you no longer require replication, remove the pipeline through your [project's replication settings](/dashboard/project/_/database/replication) -- Reduce the number of writes to your table -- Exclude columns you do not need from replication - Pipelines let you configure which columns to replicate +- Include only the tables and columns that you need at the destination. +- Keep high-churn tables out of the publication when their changes are not needed for analytics. +- If you no longer require replication, delete the pipeline through your [project's replication settings](/dashboard/project/_/database/replication) to stop pipeline-hour charges. diff --git a/apps/docs/features/ui/CodeBlock/CodeBlock.client.tsx b/apps/docs/features/ui/CodeBlock/CodeBlock.client.tsx index 0d8f8fbb1f743..d26fc31ceabd3 100644 --- a/apps/docs/features/ui/CodeBlock/CodeBlock.client.tsx +++ b/apps/docs/features/ui/CodeBlock/CodeBlock.client.tsx @@ -1,10 +1,11 @@ 'use client' -import { Check, Copy, WrapText, ArrowRightFromLine } from 'lucide-react' -import { type MouseEvent, useCallback, useEffect, useState, useRef } from 'react' +import { ArrowRightFromLine, Check, Copy, WrapText } from 'lucide-react' +import { useCallback, useEffect, useRef, useState, type MouseEvent } from 'react' import { type ThemedToken } from 'shiki' import { type NodeHover } from 'twoslash' import { cn, copyToClipboard, Tooltip, TooltipContent, TooltipTrigger } from 'ui' + import { getFontStyle } from './CodeBlock.utils' export function AnnotatedSpan({ @@ -92,26 +93,41 @@ export function CodeCopyButton({ className, content }: { className?: string; con const handleCopy = async () => { copyToClipboard(content, () => { setCopied(true) - setTimeout(() => setCopied(false), 1000) }) } + const resetStatus = () => { + setCopied(false) + } + return ( - + <> + + {copied ? 'Code copied' : ''} + + + + + + Copy code + + ) } @@ -136,7 +152,10 @@ export function CodeBlockControls({ content }: { content: string }) { }, []) return ( -
+
) } diff --git a/apps/docs/features/ui/Tabs.tsx b/apps/docs/features/ui/Tabs.tsx index 7f8399280839a..c227a87062011 100644 --- a/apps/docs/features/ui/Tabs.tsx +++ b/apps/docs/features/ui/Tabs.tsx @@ -266,6 +266,7 @@ export const TabPanel = ({ children, id, className }: TabPanelProps) => { {children} diff --git a/apps/docs/public/img/database/replication/pipelines-actions-menu.png b/apps/docs/public/img/database/replication/pipelines-actions-menu.png new file mode 100644 index 0000000000000..8c7bf5939a2dd Binary files /dev/null and b/apps/docs/public/img/database/replication/pipelines-actions-menu.png differ diff --git a/apps/docs/public/img/database/replication/pipelines-bigquery-configuration.png b/apps/docs/public/img/database/replication/pipelines-bigquery-configuration.png new file mode 100644 index 0000000000000..59a56bfab75a7 Binary files /dev/null and b/apps/docs/public/img/database/replication/pipelines-bigquery-configuration.png differ diff --git a/apps/docs/public/img/database/replication/pipelines-cost-confirmation.png b/apps/docs/public/img/database/replication/pipelines-cost-confirmation.png new file mode 100644 index 0000000000000..0c954b19af5f9 Binary files /dev/null and b/apps/docs/public/img/database/replication/pipelines-cost-confirmation.png differ diff --git a/apps/docs/public/img/database/replication/pipelines-metrics.png b/apps/docs/public/img/database/replication/pipelines-metrics.png new file mode 100644 index 0000000000000..56a9e49ce5fd6 Binary files /dev/null and b/apps/docs/public/img/database/replication/pipelines-metrics.png differ diff --git a/apps/docs/public/img/database/replication/replication-add-destination.png b/apps/docs/public/img/database/replication/replication-add-destination.png deleted file mode 100644 index 88154e6767763..0000000000000 Binary files a/apps/docs/public/img/database/replication/replication-add-destination.png and /dev/null differ diff --git a/apps/docs/public/img/database/replication/replication-bigquery-details.png b/apps/docs/public/img/database/replication/replication-bigquery-details.png deleted file mode 100644 index 587ddca2bf2ed..0000000000000 Binary files a/apps/docs/public/img/database/replication/replication-bigquery-details.png and /dev/null differ diff --git a/apps/docs/public/img/database/replication/replication-destinations-list.png b/apps/docs/public/img/database/replication/replication-destinations-list.png deleted file mode 100644 index b33f6c20f1f98..0000000000000 Binary files a/apps/docs/public/img/database/replication/replication-destinations-list.png and /dev/null differ diff --git a/apps/docs/public/img/database/replication/replication-disable-external-replication.png b/apps/docs/public/img/database/replication/replication-disable-external-replication.png deleted file mode 100644 index cc09c2d198587..0000000000000 Binary files a/apps/docs/public/img/database/replication/replication-disable-external-replication.png and /dev/null differ diff --git a/apps/docs/public/img/database/replication/replication-enable-replication.png b/apps/docs/public/img/database/replication/replication-enable-replication.png deleted file mode 100644 index d30519f58cf8d..0000000000000 Binary files a/apps/docs/public/img/database/replication/replication-enable-replication.png and /dev/null differ diff --git a/apps/docs/public/img/database/replication/replication-iceberg-bucket-details.png b/apps/docs/public/img/database/replication/replication-iceberg-bucket-details.png deleted file mode 100644 index 367d7080dce70..0000000000000 Binary files a/apps/docs/public/img/database/replication/replication-iceberg-bucket-details.png and /dev/null differ diff --git a/apps/docs/public/img/database/replication/replication-iceberg-details.png b/apps/docs/public/img/database/replication/replication-iceberg-details.png deleted file mode 100644 index ba51c58c3a56b..0000000000000 Binary files a/apps/docs/public/img/database/replication/replication-iceberg-details.png and /dev/null differ diff --git a/apps/docs/public/img/database/replication/replication-iceberg-new-bucket.png b/apps/docs/public/img/database/replication/replication-iceberg-new-bucket.png deleted file mode 100644 index fbd860ef20af0..0000000000000 Binary files a/apps/docs/public/img/database/replication/replication-iceberg-new-bucket.png and /dev/null differ diff --git a/apps/docs/public/img/database/replication/replication-logs.png b/apps/docs/public/img/database/replication/replication-logs.png deleted file mode 100644 index 449ae39077cbc..0000000000000 Binary files a/apps/docs/public/img/database/replication/replication-logs.png and /dev/null differ diff --git a/apps/docs/public/img/database/replication/replication-pipeline-actions.png b/apps/docs/public/img/database/replication/replication-pipeline-actions.png deleted file mode 100644 index 870e96fab51d5..0000000000000 Binary files a/apps/docs/public/img/database/replication/replication-pipeline-actions.png and /dev/null differ diff --git a/apps/docs/public/img/database/replication/replication-pipeline-error.png b/apps/docs/public/img/database/replication/replication-pipeline-error.png deleted file mode 100644 index a1c672c94a5f6..0000000000000 Binary files a/apps/docs/public/img/database/replication/replication-pipeline-error.png and /dev/null differ diff --git a/apps/docs/public/img/database/replication/replication-pipeline-table-error.png b/apps/docs/public/img/database/replication/replication-pipeline-table-error.png deleted file mode 100644 index 667a6a5b1ceb3..0000000000000 Binary files a/apps/docs/public/img/database/replication/replication-pipeline-table-error.png and /dev/null differ diff --git a/apps/docs/public/img/database/replication/replication-view-status.png b/apps/docs/public/img/database/replication/replication-view-status.png deleted file mode 100644 index 3c34ebf4cd63e..0000000000000 Binary files a/apps/docs/public/img/database/replication/replication-view-status.png and /dev/null differ diff --git a/apps/studio/components/interfaces/Auth/Users/Users.utils.tsx b/apps/studio/components/interfaces/Auth/Users/Users.utils.tsx index 3e0756fdfe586..3f7536348977c 100644 --- a/apps/studio/components/interfaces/Auth/Users/Users.utils.tsx +++ b/apps/studio/components/interfaces/Auth/Users/Users.utils.tsx @@ -1,6 +1,6 @@ import dayjs from 'dayjs' import { SqlEditor, TableEditor } from 'icons' -import { Copy, Trash, UserIcon } from 'lucide-react' +import { Copy, ScrollText, Trash, UserIcon } from 'lucide-react' import { Column, useRowSelection } from 'react-data-grid' import { Checkbox, @@ -283,6 +283,7 @@ export const formatUserColumns = ({ setSortByValue, onSelectDeleteUser, onSelectImpersonateUser, + onSelectViewLogs, }: { specificFilterColumn: string columns: UsersTableColumn[] @@ -292,6 +293,7 @@ export const formatUserColumns = ({ setSortByValue: (val: string) => void onSelectDeleteUser: (user: User) => void onSelectImpersonateUser: (user: User, destination: 'sql' | 'table-editor') => Promise + onSelectViewLogs: (user: User) => void }) => { const columnOrder = config.map((c) => c.id) ?? columns.map((c) => c.id) @@ -464,6 +466,17 @@ export const formatUserColumns = ({ Run SQL as user + e.stopPropagation()} + onSelect={() => { + if (user) onSelectViewLogs(user) + }} + > + + View user logs + + { 'show', parseAsString.withOptions({ history: 'push', clearOnDefault: true }) ) - const [improvedSearchDismissed, setImprovedSearchDismissed] = useLocalStorageQuery( LOCAL_STORAGE_KEYS.AUTH_USERS_IMPROVED_SEARCH_DISMISSED(projectRef ?? ''), false @@ -373,6 +373,12 @@ export const UsersV2 = () => { } } + const onSelectViewLogs = (user: User) => { + const identifier = user.id || user.email + if (!projectRef || !identifier) return + router.push(buildUnifiedLogsUrl({ projectRef, user: identifier })) + } + const handleScroll = (event: UIEvent) => { const isScrollingHorizontally = xScroll.current !== event.currentTarget.scrollLeft xScroll.current = event.currentTarget.scrollLeft @@ -482,6 +488,7 @@ export const UsersV2 = () => { setSortByValue: updateSortByValue, onSelectDeleteUser: setSelectedUserToDelete, onSelectImpersonateUser, + onSelectViewLogs, }) setColumns(columns) if (columns.length < userTableColumns.length) { @@ -716,6 +723,7 @@ export const UsersV2 = () => { setSortByValue: updateSortByValue, onSelectDeleteUser: setSelectedUserToDelete, onSelectImpersonateUser, + onSelectViewLogs, }) setSelectedColumns(value) diff --git a/apps/studio/components/interfaces/Database/Replication/BatchRestartDialog.tsx b/apps/studio/components/interfaces/Database/Replication/BatchRestartDialog.tsx index 28f2a25f1c5e6..5ee178156e630 100644 --- a/apps/studio/components/interfaces/Database/Replication/BatchRestartDialog.tsx +++ b/apps/studio/components/interfaces/Database/Replication/BatchRestartDialog.tsx @@ -13,6 +13,7 @@ import { } from 'ui' import { PipelineStatusName } from './Replication.constants' +import { RestartCostEstimate } from './RestartCostEstimate' import { ReplicationPipelineTableStatus } from '@/data/replication/pipeline-replication-status-query' import { useRollbackTablesMutation } from '@/data/replication/rollback-tables-mutation' @@ -20,9 +21,9 @@ interface BatchRestartDialogProps { open: boolean onOpenChange: (open: boolean) => void mode: 'all' | 'errored' - totalTables: number - erroredTablesCount: number tables: ReplicationPipelineTableStatus[] + sourceId?: number + publicationName?: string pipelineStatusName?: PipelineStatusName onRestartStart?: (tableIds: number[]) => void onRestartComplete?: (tableIds: number[]) => void @@ -32,32 +33,38 @@ export const BatchRestartDialog = ({ open, onOpenChange, mode, - totalTables, - erroredTablesCount, tables, + sourceId, + publicationName, pipelineStatusName, onRestartStart, onRestartComplete, }: BatchRestartDialogProps) => { const { ref: projectRef, pipelineId: _pipelineId } = useParams() const pipelineId = Number(_pipelineId) - // Calculate which table IDs will be restarted based on mode (memoized) - const affectedTableIds = useMemo(() => { + const affectedTables = useMemo(() => { if (mode === 'all') { - return tables.map((t) => t.table_id) - } else { return tables - .filter( - (t) => - t.state.name === 'error' && - 'retry_policy' in t.state && - t.state.retry_policy?.policy === 'manual_retry' - ) - .map((t) => t.table_id) + } else { + return tables.filter( + (t) => + t.state.name === 'error' && + 'retry_policy' in t.state && + t.state.retry_policy?.policy === 'manual_retry' + ) } }, [mode, tables]) + const affectedTableIds = useMemo( + () => affectedTables.map((table) => table.table_id), + [affectedTables] + ) + + const affectedTableNames = useMemo( + () => affectedTables.map((table) => table.table_name), + [affectedTables] + ) - const { mutate: rollbackTables, isPending: isResetting } = useRollbackTablesMutation({ + const { mutateAsync: rollbackTables, isPending: isResetting } = useRollbackTablesMutation({ onSuccess: (data) => { const count = data.tables.length toast.success( @@ -73,18 +80,20 @@ export const BatchRestartDialog = ({ }, }) - const handleReset = () => { + const handleReset = async () => { if (!projectRef) return toast.error('Project ref is required') onRestartStart?.(affectedTableIds) - rollbackTables({ - projectRef, - pipelineId, - target: mode === 'all' ? { type: 'all_tables' } : { type: 'all_errored_tables' }, - rollbackType: 'full', - pipelineStatusName, - }) + try { + await rollbackTables({ + projectRef, + pipelineId, + target: mode === 'all' ? { type: 'all_tables' } : { type: 'all_errored_tables' }, + rollbackType: 'full', + pipelineStatusName, + }) + } catch (error) {} } const dialogContent = @@ -94,14 +103,14 @@ export const BatchRestartDialog = ({ description: (

- This will restart replication for all - {totalTables === 0 ? '' : totalTables} table{totalTables > 1 ? 's' : ''} in this - pipeline from scratch: + This will restart replication for all {affectedTables.length} table + {affectedTables.length === 1 ? '' : 's'} in this pipeline from scratch:

  • - All table copies will be re-initialized. Every table will be - copied again from the source. + Every table's initial sync will restart. All table data will be + copied again from the source. This initial sync is billed in addition to previous + initial syncs.
  • All downstream data will be deleted. All replicated data will be @@ -122,12 +131,13 @@ export const BatchRestartDialog = ({

    This will restart replication for{' '} - all {erroredTablesCount} failed tables from scratch: + all {affectedTables.length} failed tables from scratch:

    • - Failed table copies will be re-initialized. These tables will be - copied again from the source. + Each failed table's initial sync will restart. These tables will + be copied again from the source. This initial sync is billed in addition to + previous initial syncs.
    • Existing downstream data will be deleted. Replicated data for @@ -154,6 +164,13 @@ export const BatchRestartDialog = ({ {dialogContent.title} {dialogContent.description} + Cancel diff --git a/apps/studio/components/interfaces/Database/Replication/DeleteDestination.tsx b/apps/studio/components/interfaces/Database/Replication/DeleteDestination.tsx index 11bcb9ece7379..edbf73f510ada 100644 --- a/apps/studio/components/interfaces/Database/Replication/DeleteDestination.tsx +++ b/apps/studio/components/interfaces/Database/Replication/DeleteDestination.tsx @@ -24,8 +24,8 @@ export const DeleteDestination = ({ confirmLabel={isLoading ? 'Deleting...' : `Delete destination`} confirmPlaceholder="Type in name of destination" confirmString={name ?? 'Unknown'} - text={`This will delete the destination "${name}" and stop its replication pipeline.`} - alert={{ title: 'You cannot recover this destination or pipeline once deleted.' }} + text={`This will delete the destination "${name}" and stop its replication pipeline. Already replicated data remains at the destination, and pipeline-hour billing ends when deletion completes.`} + alert={{ title: 'You cannot recover this destination or pipeline after deletion.' }} onCancel={() => setVisible(!visible)} onConfirm={onDelete} /> diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx index 2f95430ab63b6..65945d1106836 100644 --- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx @@ -62,24 +62,15 @@ export const AdvancedSettings = ({ -

      - Maximum time pipeline waits to collect additional changes before flushing a - batch. -

      -

      - Lower values reduce replication latency, higher values improve batching - efficiency. -

      - - } + description="How long the pipeline waits before sending a partially filled batch." > -

      Number of tables copied in parallel during the initial snapshot phase.

      -

      - Each worker uses one replication slot (up to N + 1 total while syncing). -

      - - } + description="Maximum number of tables synced at the same time." > -

      - Number of parallel connections each table copy can use during initial sync. -

      -

      - More connections speed up large table copies, but use more database - connections. -

      - - } + description="Maximum number of connections used to sync each table." > diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsx index de5192ff12c78..d124377e79f18 100644 --- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsx +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsx @@ -87,6 +87,11 @@ export const DestinationPanel = ({ onSuccessCreateReadReplica }: DestinationPane setEdit(null) } + const docsUrl = + urlDestinationType === 'BigQuery' + ? `${DOCS_URL}/guides/database/replication/bigquery#configure-bigquery-as-a-destination` + : `${DOCS_URL}/guides/database/replication/pipelines#step-3-configure-a-destination` + useEffect(() => { if (edit !== null && invalidExistingDestination) { toast(`Unable to find destination ID ${edit}`) @@ -99,13 +104,16 @@ export const DestinationPanel = ({ onSuccessCreateReadReplica }: DestinationPane
      - - {editMode ? 'Edit destination' : 'Add destination'} - - {editMode - ? 'Update the configuration for this destination.' - : 'A destination can be a read replica or an external destination that receives replicated data in near real time.'} - + +
      + {editMode ? 'Edit destination' : 'Add new destination'} + + {editMode + ? 'Update the configuration for this destination.' + : 'Set up a read replica or Supabase Pipelines destination for near real-time replication'} + +
      +       @@ -120,9 +128,9 @@ export const DestinationPanel = ({ onSuccessCreateReadReplica }: DestinationPane

      Request Pipelines access

      - Pipelines is in alpha and being - rolled out gradually. Request access below to join the waitlist. Read replicas - are available now. + Pipelines is in public alpha and + being rolled out gradually. Request access below to join the waitlist. Read + replicas are available now.

      @@ -136,7 +144,7 @@ export const DestinationPanel = ({ onSuccessCreateReadReplica }: DestinationPane rel="noreferrer" href="https://forms.supabase.com/pg_replicate" > - Request alpha access + Request Pipelines access diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.test.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.test.tsx index f5d2d4427850f..0b3253063cef8 100644 --- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.test.tsx +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.test.tsx @@ -113,7 +113,7 @@ describe('DestinationTypeSelection', () => { expect(screen.queryByText('Pipelines')).not.toBeInTheDocument() }) - test('shows alpha warning when an alpha destination type is selected', async () => { + test('shows the public alpha warning for a Pipelines destination', async () => { mockBigQueryEnabled.mockReturnValue(true) mockIcebergEnabled.mockReturnValue(false) mockDucklakeEnabled.mockReturnValue(false) @@ -125,7 +125,7 @@ describe('DestinationTypeSelection', () => { fireEvent.click(await screen.findByRole('combobox')) fireEvent.click(await screen.findByText('BigQuery')) - expect(await screen.findByText(/This destination type is in alpha/)).toBeInTheDocument() + expect(await screen.findByText(/This destination type is in public alpha/)).toBeInTheDocument() }) test('disables the selector in edit mode so the destination type cannot be changed', async () => { diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.tsx index c528ff2e08e28..47b1acb094972 100644 --- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.tsx +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.tsx @@ -29,7 +29,7 @@ interface DestinationTypeOption { label: string description: string icon: typeof Database - isAlpha: boolean + stage: 'Public Alpha' | 'Early Access' | 'Deprecated' | null enabled: boolean } @@ -83,7 +83,7 @@ export const DestinationTypeSelection = () => { description: 'Deploy a read-only database in another region for lower latency and workload isolation', icon: Database, - isAlpha: false, + stage: null, enabled: isOptionVisible('Read Replica', infrastructureReadReplicas), }, ], @@ -96,32 +96,32 @@ export const DestinationTypeSelection = () => { label: 'Analytics Bucket', description: 'Write Apache Iceberg tables to Supabase Storage for analytics workflows', icon: AnalyticsBucket, - isAlpha: true, + stage: 'Deprecated', enabled: isOptionVisible('Analytics Bucket', etlEnableIceberg), }, { value: 'BigQuery', label: 'BigQuery', - description: "Stream changes to Google Cloud's data warehouse for analytics and BI", + description: "Replicate changes to Google Cloud's data warehouse for analytics and BI", icon: BigQuery, - isAlpha: true, + stage: 'Public Alpha', enabled: isOptionVisible('BigQuery', etlEnableBigQuery), }, { value: 'DuckLake', label: 'DuckLake', - description: 'Stream changes to a DuckLake catalog backed by S3-compatible storage', + description: 'Replicate changes to a DuckLake catalog backed by S3-compatible storage', icon: Database, - isAlpha: true, + stage: 'Early Access', enabled: isOptionVisible('DuckLake', etlEnableDucklake), }, { value: 'Snowflake', label: 'Snowflake', description: - 'Stream changes to Snowflake for warehouse analytics and downstream data workflows', + 'Replicate changes to Snowflake for warehouse analytics and downstream data workflows', icon: Snowflake, - isAlpha: true, + stage: 'Early Access', enabled: isOptionVisible('Snowflake', etlEnableSnowflake), }, ], @@ -140,17 +140,22 @@ export const DestinationTypeSelection = () => { - This destination type is in alpha and may be unstable or introduce breaking changes - while we iterate based on customer feedback.{' '} - - Leave feedback - + selectedOption?.stage && ( + + {selectedOption.stage === 'Public Alpha' + ? 'This destination type is in public alpha and may change as we iterate based on customer feedback. ' + : selectedOption.stage === 'Deprecated' + ? 'This destination type is deprecated.' + : 'This destination type is available through early access and may change as we iterate based on customer feedback. '} + {selectedOption.stage !== 'Deprecated' && ( + + Leave feedback + + )} ) } @@ -166,7 +171,19 @@ export const DestinationTypeSelection = () => {
      {selectedOption.label} - {selectedOption.isAlpha && Alpha} + {selectedOption.stage && ( + + {selectedOption.stage} + + )}
      ) : ( @@ -185,7 +202,19 @@ export const DestinationTypeSelection = () => {
      {option.label} - {option.isAlpha && Alpha} + {option.stage && ( + + {option.stage} + + )}
      {option.description}
      diff --git a/apps/studio/components/interfaces/Database/Replication/Destinations.tsx b/apps/studio/components/interfaces/Database/Replication/Destinations.tsx index 398b08c23e005..a94000c3e1d82 100644 --- a/apps/studio/components/interfaces/Database/Replication/Destinations.tsx +++ b/apps/studio/components/interfaces/Database/Replication/Destinations.tsx @@ -264,7 +264,9 @@ export const Destinations = () => { {organization?.slug && ( )} diff --git a/apps/studio/components/interfaces/Database/Replication/DisablePipelinesDialog.tsx b/apps/studio/components/interfaces/Database/Replication/DisablePipelinesDialog.tsx index d9bab23243ede..9b666d6bf01f8 100644 --- a/apps/studio/components/interfaces/Database/Replication/DisablePipelinesDialog.tsx +++ b/apps/studio/components/interfaces/Database/Replication/DisablePipelinesDialog.tsx @@ -54,8 +54,8 @@ export const DisablePipelinesDialog = ({ open, setOpen }: DisablePipelinesDialog

      This will remove the etl schema and all - connected resources from your database. Any active replication pipelines sending - changes to external destinations will stop. + Pipelines-managed resources from your database. Data already written to destination + systems is not deleted.

      Read replicas are not affected.

       diff --git a/apps/studio/components/interfaces/Database/Replication/EnablePipelinesCallout.tsx b/apps/studio/components/interfaces/Database/Replication/EnablePipelinesCallout.tsx index 602bda1509780..6dcc2c0769b53 100644 --- a/apps/studio/components/interfaces/Database/Replication/EnablePipelinesCallout.tsx +++ b/apps/studio/components/interfaces/Database/Replication/EnablePipelinesCallout.tsx @@ -17,6 +17,7 @@ import { Admonition } from 'ui-patterns/admonition' import { DestinationType } from './DestinationPanel/DestinationPanel.types' import { DocsButton } from '@/components/ui/DocsButton' +import { InlineLink } from '@/components/ui/InlineLink' import { UpgradePlanButton } from '@/components/ui/UpgradePlanButton' import { useCreateTenantSourceMutation } from '@/data/replication/create-tenant-source-mutation' import { DOCS_URL } from '@/lib/constants' @@ -57,15 +58,19 @@ const EnablePipelinesModal = () => {

      - Alpha features can be unstable and may introduce breaking changes while we evaluate - the product direction, refine the feature set, and incorporate customer feedback. + Public alpha features may change as we refine the product and incorporate customer + feedback.

      - Pricing is not finalized. You can enable Pipelines now; we'll announce pricing later - and notify you before any charges apply. + Pipelines is billed for configured pipeline hours, initial sync data, and ongoing + replication data. Review the{' '} + + Pipelines pricing + {' '} + before enabling it.

      @@ -96,9 +101,8 @@ export const EnablePipelinesCallout = ({

      Enable Pipelines

      - Pipelines creates managed replication pipelines that stream database changes to - destination systems.{' '} - {hasAccess ? 'Enable Pipelines for your project' : 'Upgrade to the Pro plan'} to stream + Supabase Pipelines replicates database changes to supported destination systems.{' '} + {hasAccess ? 'Enable Pipelines for your project' : 'Upgrade to the Pro plan'} to replicate database changes to {type ?? 'data warehouses and analytics platforms'}.

      diff --git a/apps/studio/components/interfaces/Database/Replication/Pipeline.utils.ts b/apps/studio/components/interfaces/Database/Replication/Pipeline.utils.ts index 9064e4fd2f15b..a2c834dbc2c8e 100644 --- a/apps/studio/components/interfaces/Database/Replication/Pipeline.utils.ts +++ b/apps/studio/components/interfaces/Database/Replication/Pipeline.utils.ts @@ -59,7 +59,7 @@ const PIPELINE_DISPLAY_STATES: Record { onOpenChange={setShowRestartDialog} tableId={selectedTableForRestart.tableId} tableName={selectedTableForRestart.tableName} + sourceId={pipeline?.source_id} + publicationName={pipeline?.config.publication_name} pipelineStatusName={statusName} onRestartStart={() => { setRestartingTableIds((prev) => new Set(prev).add(selectedTableForRestart.tableId)) @@ -614,9 +616,9 @@ export const ReplicationPipelineStatus = () => { open={showBatchRestartDialog} onOpenChange={setShowBatchRestartDialog} mode={batchRestartMode} - totalTables={tableStatuses.length} - erroredTablesCount={erroredTables.length} tables={tableStatuses} + sourceId={pipeline?.source_id} + publicationName={pipeline?.config.publication_name} pipelineStatusName={statusName} onRestartStart={(tableIds) => { setRestartingTableIds((prev) => new Set([...prev, ...tableIds])) diff --git a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.utils.tsx b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.utils.tsx index 9ab2ebcd4c2a5..293251ae5daa5 100644 --- a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.utils.tsx +++ b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.utils.tsx @@ -19,22 +19,22 @@ export const getStatusConfig = (state: TableState['state']) => { case 'copying_table': return { badge: Copying, - description: "Table's existing rows are being copied before live streaming begins.", - tooltip: "Table's existing rows are being copied before live streaming begins.", + description: "Table's existing rows are being copied during the initial sync.", + tooltip: "Table's existing rows are being copied during the initial sync.", color: 'text-brand-600', } case 'copied_table': return { badge: Copied, - description: "Table copy is complete and it's preparing to follow WAL changes.", - tooltip: "Table copy is complete and it's preparing to follow WAL changes.", + description: 'Initial sync is complete and the table is preparing for ongoing replication.', + tooltip: 'Initial sync is complete and the table is preparing for ongoing replication.', color: 'text-success-600', } case 'following_wal': return { badge: Live, - description: 'Table is streaming new changes in real time from the WAL.', - tooltip: 'Table is streaming new changes in real time from the WAL.', + description: 'Table is receiving ongoing changes from the WAL.', + tooltip: 'Table is receiving ongoing changes from the WAL.', color: 'text-success-600', } case 'error': diff --git a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/SlotLagMetrics.tsx b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/SlotLagMetrics.tsx index cede6f71a1da9..2c1f88748db9a 100644 --- a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/SlotLagMetrics.tsx +++ b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/SlotLagMetrics.tsx @@ -30,12 +30,12 @@ const SLOT_LAG_FIELDS: SlotLagField[] = [ }, { key: 'safe_wal_size_bytes', - label: 'Room before pausing', + label: 'WAL retention remaining', type: 'bytes', description: ( <> - How much more can pile up before the pipeline has to be set up again. Controlled by the{' '} - max_slot_wal_keep_size setting. + How much more WAL can accumulate before the replication slot is at risk of being lost. + Controlled by the max_slot_wal_keep_size setting. ), nullLabel: 'Unlimited', diff --git a/apps/studio/components/interfaces/Database/Replication/RestartCostEstimate.test.ts b/apps/studio/components/interfaces/Database/Replication/RestartCostEstimate.test.ts new file mode 100644 index 0000000000000..c63198b7cb5fd --- /dev/null +++ b/apps/studio/components/interfaces/Database/Replication/RestartCostEstimate.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from 'vitest' + +import { calculateRestartCostEstimate } from './RestartCostEstimate' + +const tables = [ + { + schema: 'public', + name: 'orders', + estimated_bytes: 2_000_000_000, + estimated_cost: 1.25, + is_row_filtered: false, + }, + { + schema: 'public', + name: 'customers', + estimated_bytes: 1_000_000_000, + estimated_cost: 0.75, + is_row_filtered: true, + }, + { + schema: 'internal', + name: 'audit_log', + estimated_bytes: 500_000_000, + estimated_cost: 0.3, + is_row_filtered: false, + }, +] + +describe('calculateRestartCostEstimate', () => { + it('sums only the tables being restarted', () => { + expect(calculateRestartCostEstimate(tables, ['public.orders', 'public.customers'])).toEqual({ + isComplete: true, + estimatedBytes: 3_000_000_000, + estimatedCost: 2, + hasRowFilteredTables: true, + }) + }) + + it('marks the estimate incomplete when a restarted table is missing', () => { + expect( + calculateRestartCostEstimate(tables, ['public.orders', 'public.missing']).isComplete + ).toBe(false) + }) +}) diff --git a/apps/studio/components/interfaces/Database/Replication/RestartCostEstimate.tsx b/apps/studio/components/interfaces/Database/Replication/RestartCostEstimate.tsx new file mode 100644 index 0000000000000..2558a0b6b9a46 --- /dev/null +++ b/apps/studio/components/interfaces/Database/Replication/RestartCostEstimate.tsx @@ -0,0 +1,84 @@ +import { useMemo } from 'react' +import { GenericSkeletonLoader } from 'ui-patterns/ShimmeringLoader' + +import { useReplicationCostEstimateQuery } from '@/data/replication/cost-estimate-query' +import { formatBytes, formatCurrency } from '@/lib/helpers' + +interface RestartCostEstimateProps { + open: boolean + projectRef?: string + sourceId?: number + publicationName?: string + tableNames: string[] +} + +interface EstimatedTable { + schema: string + name: string + estimated_bytes: number + estimated_cost: number + is_row_filtered: boolean +} + +export const calculateRestartCostEstimate = (tables: EstimatedTable[], tableNames: string[]) => { + const tableNameSet = new Set(tableNames) + const matchedTables = tables.filter((table) => tableNameSet.has(`${table.schema}.${table.name}`)) + + return { + isComplete: matchedTables.length === tableNames.length, + estimatedBytes: matchedTables.reduce((total, table) => total + table.estimated_bytes, 0), + estimatedCost: matchedTables.reduce((total, table) => total + table.estimated_cost, 0), + hasRowFilteredTables: matchedTables.some((table) => table.is_row_filtered), + } +} + +export const RestartCostEstimate = ({ + open, + projectRef, + sourceId, + publicationName, + tableNames, +}: RestartCostEstimateProps) => { + const { data: estimate, isFetching } = useReplicationCostEstimateQuery( + { projectRef, sourceId, publicationName }, + { enabled: open } + ) + const restartEstimate = useMemo( + () => + estimate === undefined + ? undefined + : calculateRestartCostEstimate(estimate.table_copy.tables, tableNames), + [estimate, tableNames] + ) + return ( +
      + {isFetching ? ( + + ) : restartEstimate?.isComplete ? ( +
      +
      +

      Estimated additional initial sync

      +

      + {formatBytes(restartEstimate.estimatedBytes)} across {tableNames.length}{' '} + {tableNames.length === 1 ? 'table' : 'tables'} +

      +
      + + {formatCurrency(restartEstimate.estimatedCost)} + {restartEstimate.hasRowFilteredTables ? '*' : null} + +
      + ) : ( +

      + A cost estimate is unavailable. You can still restart the{' '} + {tableNames.length === 1 ? 'table' : 'tables'}. +

      + )} + {restartEstimate?.isComplete && restartEstimate.hasRowFilteredTables && ( +

      + *Tables with row filters may cost less than shown. +

      + )} +
      + ) +} diff --git a/apps/studio/components/interfaces/Database/Replication/RestartTableDialog.tsx b/apps/studio/components/interfaces/Database/Replication/RestartTableDialog.tsx index d8bc697e96f87..17561ae774ff8 100644 --- a/apps/studio/components/interfaces/Database/Replication/RestartTableDialog.tsx +++ b/apps/studio/components/interfaces/Database/Replication/RestartTableDialog.tsx @@ -12,6 +12,7 @@ import { } from 'ui' import { PipelineStatusName } from './Replication.constants' +import { RestartCostEstimate } from './RestartCostEstimate' import { useRollbackTablesMutation } from '@/data/replication/rollback-tables-mutation' interface RestartTableDialogProps { @@ -19,6 +20,8 @@ interface RestartTableDialogProps { onOpenChange: (open: boolean) => void tableId: number tableName: string + sourceId?: number + publicationName?: string pipelineStatusName?: PipelineStatusName onRestartStart?: () => void onRestartComplete?: () => void @@ -29,6 +32,8 @@ export const RestartTableDialog = ({ onOpenChange, tableId, tableName, + sourceId, + publicationName, pipelineStatusName, onRestartStart, onRestartComplete, @@ -80,8 +85,9 @@ export const RestartTableDialog = ({

      • - The table copy will be re-initialized. All data will be copied - again from the source. + The table's initial sync will restart. All existing data will be + copied again from the source. This initial sync is billed in addition to previous + initial syncs.
      • Existing downstream data will be deleted. Any replicated data for @@ -98,6 +104,13 @@ export const RestartTableDialog = ({
      + Cancel diff --git a/apps/studio/components/interfaces/Organization/BillingSettings/BillingBreakdown/BillingBreakdown.constants.ts b/apps/studio/components/interfaces/Organization/BillingSettings/BillingBreakdown/BillingBreakdown.constants.ts index 015c7352085b8..dafa999fdc2f2 100644 --- a/apps/studio/components/interfaces/Organization/BillingSettings/BillingBreakdown/BillingBreakdown.constants.ts +++ b/apps/studio/components/interfaces/Organization/BillingSettings/BillingBreakdown/BillingBreakdown.constants.ts @@ -180,18 +180,18 @@ export const BILLING_BREAKDOWN_METRICS: Metric[] = [ }, { key: PricingMetric.ETL_COPY_BACKFILL_DATA, - name: 'Pipeline Backfill Data', + name: 'Pipeline Initial Sync Data', units: 'bytes', unitName: 'GB', category: 'Pipelines', - anchor: 'pipeline-backfill-data', + anchor: 'pipeline-initial-sync-data', }, { key: PricingMetric.ETL_REPLICATED_DATA, - name: 'Pipeline Replicated Data', + name: 'Pipeline Ongoing Replication Data', units: 'bytes', unitName: 'GB', category: 'Pipelines', - anchor: 'pipeline-replicated-data', + anchor: 'pipeline-ongoing-replication-data', }, ] diff --git a/apps/studio/components/interfaces/Organization/Usage/Usage.constants.tsx b/apps/studio/components/interfaces/Organization/Usage/Usage.constants.tsx index bdbc8cb3a4ac5..8d5eb804fce0b 100644 --- a/apps/studio/components/interfaces/Organization/Usage/Usage.constants.tsx +++ b/apps/studio/components/interfaces/Organization/Usage/Usage.constants.tsx @@ -410,34 +410,34 @@ export const USAGE_CATEGORIES: (subscription?: OrgSubscription) => CategoryMeta[ description: 'Usage statistics related to your pipelines', attributes: [ { - anchor: 'pipeline-backfill-data', + anchor: 'pipeline-initial-sync-data', key: PricingMetric.ETL_COPY_BACKFILL_DATA, attributes: [{ key: PricingMetric.ETL_COPY_BACKFILL_DATA.toLowerCase(), color: 'white' }], - name: 'Pipeline Backfill Data', + name: 'Pipeline Initial Sync Data', unit: 'bytes', description: - 'Total amount of backfill/initial copy data processed across all projects.\nBilling is based on the total amount of backfill/initial copy data processed in Gigabyte.', + 'Total amount of initial sync data processed across all projects.\nBilling is based on the total amount of initial sync data processed in gigabytes.', chartDescription: 'The data refreshes every hour.', links: [ { name: 'Pipelines', - url: `${DOCS_URL}/guides/database/replication/pipelines`, + url: `${DOCS_URL}/guides/platform/manage-your-usage/pipelines`, }, ], }, { - anchor: 'pipeline-replicated-data', + anchor: 'pipeline-ongoing-replication-data', key: PricingMetric.ETL_REPLICATED_DATA, attributes: [{ key: PricingMetric.ETL_REPLICATED_DATA.toLowerCase(), color: 'white' }], - name: 'Pipeline Replicated Data', + name: 'Pipeline Ongoing Replication Data', unit: 'bytes', description: - 'Total amount of replicated data processed across all projects.\nBilling is based on the total amount of replicated data processed in Gigabyte.', + 'Total amount of ongoing replication data processed across all projects.\nBilling is based on the total amount of ongoing replication data processed in gigabytes.', chartDescription: 'The data refreshes every hour.', links: [ { name: 'Pipelines', - url: `${DOCS_URL}/guides/database/replication/pipelines`, + url: `${DOCS_URL}/guides/platform/manage-your-usage/pipelines`, }, ], }, diff --git a/apps/studio/components/interfaces/ProjectCreation/DataSeeding.tsx b/apps/studio/components/interfaces/ProjectCreation/DataSeeding.tsx new file mode 100644 index 0000000000000..54ab2be0dd737 --- /dev/null +++ b/apps/studio/components/interfaces/ProjectCreation/DataSeeding.tsx @@ -0,0 +1,46 @@ +import { UseFormReturn } from 'react-hook-form' +import { Checkbox, FormControl, FormDescription, FormField, FormItem, FormLabel } from 'ui' +import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout' + +import { CreateProjectForm } from './ProjectCreation.schema' +import Panel from '@/components/ui/Panel' + +interface DataSeedingProps { + form: UseFormReturn + layout?: 'vertical' | 'horizontal' +} + +export const DataSeeding = ({ form, layout = 'horizontal' }: DataSeedingProps) => { + return ( + + +
      + ( + + + field.onChange(value === true)} + /> + +
      + + Create sample tables with seed data + + + To get you started quickly, we can create new tables for you with seed (sample) + data. You can delete these tables later. + +
      +       + )} + /> +
      +       +       + ) +} diff --git a/apps/studio/components/interfaces/ProjectCreation/OrganizationSelector.tsx b/apps/studio/components/interfaces/ProjectCreation/OrganizationSelector.tsx index 070ab1292e2b2..13effc013d9c0 100644 --- a/apps/studio/components/interfaces/ProjectCreation/OrganizationSelector.tsx +++ b/apps/studio/components/interfaces/ProjectCreation/OrganizationSelector.tsx @@ -28,9 +28,13 @@ import { useSelectedOrganizationQuery } from '@/hooks/misc/useSelectedOrganizati interface OrganizationSelectorProps { form: UseFormReturn + disableOrganizationSelection: boolean } -export const OrganizationSelector = ({ form }: OrganizationSelectorProps) => { +export const OrganizationSelector = ({ + form, + disableOrganizationSelection, +}: OrganizationSelectorProps) => { const router = useRouter() const { slug } = useParams() const queryClient = useQueryClient() @@ -65,6 +69,7 @@ export const OrganizationSelector = ({ form }: OrganizationSelectorProps) => { }} value={field.value} defaultValue={field.value} + disabled={disableOrganizationSelection} > diff --git a/apps/studio/components/interfaces/ProjectCreation/ProjectCreation.schema.ts b/apps/studio/components/interfaces/ProjectCreation/ProjectCreation.schema.ts index 1d3948c47c2b5..38b3369873a2e 100644 --- a/apps/studio/components/interfaces/ProjectCreation/ProjectCreation.schema.ts +++ b/apps/studio/components/interfaces/ProjectCreation/ProjectCreation.schema.ts @@ -42,6 +42,7 @@ export const FormSchema = z enableRlsEventTrigger: z.boolean(), postgresVersionSelection: z.string(), useOrioleDb: z.boolean(), + shouldRunMigrations: z.boolean(), }) .superRefine( ( diff --git a/apps/studio/components/interfaces/ProjectCreation/ProjectCreationFooter.tsx b/apps/studio/components/interfaces/ProjectCreation/ProjectCreationFooter.tsx index 8d237ed15258f..e01c2be3dd6d6 100644 --- a/apps/studio/components/interfaces/ProjectCreation/ProjectCreationFooter.tsx +++ b/apps/studio/components/interfaces/ProjectCreation/ProjectCreationFooter.tsx @@ -29,6 +29,7 @@ interface ProjectCreationFooterProps { organizationProjects: OrgProject[] isCreatingNewProject: boolean isSuccessNewProject: boolean + hideCancelButton: boolean } export const ProjectCreationFooter = ({ @@ -38,6 +39,7 @@ export const ProjectCreationFooter = ({ organizationProjects, isCreatingNewProject, isSuccessNewProject, + hideCancelButton, }: ProjectCreationFooterProps) => { const router = useRouter() const { data: currentOrg } = useSelectedOrganizationQuery() @@ -167,16 +169,18 @@ export const ProjectCreationFooter = ({
    - + {!hideCancelButton && ( + + )} + ) : null} +
    + +
    + } + containerClassName="h-8 rounded-sm" + value={search} + trailing={isFetching ? : undefined} + onChange={(e) => setSearch(e.target.value)} + /> + +
    + {isPending ? ( +
    + +
    + ) : options.length === 0 ? ( +
    +

    No users found

    +
    + ) : ( + options.map((option, index) => { + const checked = option.id === user + + return ( +
    + toggle(option.id)} + /> + +
    + ) + }) + )} +
    +
    +     + + ) +} diff --git a/apps/studio/components/interfaces/UserJourneys/UserJourneys.queries.ts b/apps/studio/components/interfaces/UserJourneys/UserJourneys.queries.ts new file mode 100644 index 0000000000000..111c0a1b59682 --- /dev/null +++ b/apps/studio/components/interfaces/UserJourneys/UserJourneys.queries.ts @@ -0,0 +1,44 @@ +import { getPaginatedUsersSQL } from '@supabase/pg-meta' + +import { authKeys } from '@/data/auth/keys' +import { getQueryClient } from '@/data/query-client' +import { executeSql } from '@/data/sql/execute-sql-mutation' + +export interface JourneyUser { + id: string + email: string | null + createdAt: string | null + lastSignInAt: string | null +} + +export async function searchAuthUsers( + projectRef: string, + connectionString: string | null, + keywords: string, + signal?: AbortSignal +): Promise { + const queryClient = getQueryClient() + const result = await queryClient.fetchQuery< + { id: string; email: string | null; created_at: string; last_sign_in_at: string | null }[] + >({ + // eslint-disable-next-line @tanstack/query/exhaustive-deps + queryKey: authKeys.usersSearch(projectRef, keywords), + queryFn: () => { + const sql = getPaginatedUsersSQL({ + keywords, + limit: 20, + sort: 'id', + order: 'asc', + }) + return executeSql< + { id: string; email: string | null; created_at: string; last_sign_in_at: string | null }[] + >({ projectRef, connectionString, sql }, signal).then((res) => res.result) + }, + }) + return result.map((row) => ({ + id: row.id, + email: row.email, + createdAt: row.created_at, + lastSignInAt: row.last_sign_in_at, + })) +} diff --git a/apps/studio/components/layouts/LogsLayout/LogsSidebarMenuV2.tsx b/apps/studio/components/layouts/LogsLayout/LogsSidebarMenuV2.tsx index 109431cb146ea..56608b50c2bc1 100644 --- a/apps/studio/components/layouts/LogsLayout/LogsSidebarMenuV2.tsx +++ b/apps/studio/components/layouts/LogsLayout/LogsSidebarMenuV2.tsx @@ -24,6 +24,7 @@ import { useReplicationSourcesQuery } from '@/data/replication/sources-query' import { useCheckEntitlements } from '@/hooks/misc/useCheckEntitlements' import { useIsFeatureEnabled } from '@/hooks/misc/useIsFeatureEnabled' import { useShowMultigresLogs } from '@/hooks/misc/useShowMultigresLogs' +import { useShowPostgresUpgradeLogs } from '@/hooks/misc/useShowPostgresUpgradeLogs' export function SidebarCollapsible({ children, @@ -87,6 +88,7 @@ export function LogsSidebarMenuV2() { const { hasAccess: hasDedicatedPooler } = useCheckEntitlements('dedicated_pooler') const showMultigresLogs = useShowMultigresLogs() + const showPostgresUpgradeLogs = useShowPostgresUpgradeLogs() const { data: savedQueriesRes, isPending: savedQueriesLoading } = useContentQuery({ projectRef: ref, @@ -190,16 +192,17 @@ export function LogsSidebarMenuV2() { : null, ].filter((x) => x !== null) - const OPERATIONAL_COLLECTIONS = IS_PLATFORM - ? [ - { - name: 'Postgres Version Upgrade', - key: 'pg-upgrade-logs', - url: `/project/${ref}/logs/pg-upgrade-logs`, - items: [], - }, - ] - : [] + const OPERATIONAL_COLLECTIONS = + IS_PLATFORM && showPostgresUpgradeLogs + ? [ + { + name: 'Postgres Version Upgrade', + key: 'pg-upgrade-logs', + url: `/project/${ref}/logs/pg-upgrade-logs`, + items: [], + }, + ] + : [] const filteredLogs = BASE_COLLECTIONS.filter((collection) => { return collection?.name.toLowerCase().includes(searchText.toLowerCase()) diff --git a/apps/studio/components/ui/DataTable/DataTableFilters/DataTableFilterControls.tsx b/apps/studio/components/ui/DataTable/DataTableFilters/DataTableFilterControls.tsx index e13a8906643c1..e5d299a0b96c2 100644 --- a/apps/studio/components/ui/DataTable/DataTableFilters/DataTableFilterControls.tsx +++ b/apps/studio/components/ui/DataTable/DataTableFilters/DataTableFilterControls.tsx @@ -1,3 +1,4 @@ +import { Fragment, type ReactNode } from 'react' import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from 'ui' import { DateRangeDisabled } from '../DataTable.types' @@ -17,10 +18,13 @@ import { DataTableFilterTimerange } from './DataTableFilterTimerange' interface DataTableFilterControls { dateRangeDisabled?: DateRangeDisabled + /** Extra accordion items to render immediately after the field whose `value` is the key. */ + itemsAfter?: Record } export function DataTableFilterControls({ dateRangeDisabled: _dateRangeDisabled, + itemsAfter, }: DataTableFilterControls) { const { filterFields, isLoadingCounts } = useDataTable() return ( @@ -35,46 +39,49 @@ export function DataTableFilterControls({ .map((field) => { const value = field.value as string return ( - -
    - -
    -

    {field.label}

    -
    -     - -
    - - {/* REMINDER: avoid the focus state to be cut due to overflow-hidden */} - {/* REMINDER: need to move within here because of accordion height animation */} -
    - {(() => { - switch (field.type) { - case 'checkbox': { - // [Joshen] Loader here so that CheckboxAsync can retrieve the data - // immediately to be set in its react query state - if (field.hasDynamicOptions && isLoadingCounts) { - return - } else if (field.hasAsyncSearch) { - return - } else { - return + + +
    + +
    +

    {field.label}

    +
    +     + +
    + + {/* REMINDER: avoid the focus state to be cut due to overflow-hidden */} + {/* REMINDER: need to move within here because of accordion height animation */} +
    + {(() => { + switch (field.type) { + case 'checkbox': { + // [Joshen] Loader here so that CheckboxAsync can retrieve the data + // immediately to be set in its react query state + if (field.hasDynamicOptions && isLoadingCounts) { + return + } else if (field.hasAsyncSearch) { + return + } else { + return + } + } + case 'slider': { + return + } + case 'input': { + return + } + case 'timerange': { + return } } - case 'slider': { - return - } - case 'input': { - return - } - case 'timerange': { - return - } - } - })()} -
    -     -     + })()} +
    +     +     + {itemsAfter?.[value] ?? null} + ) })} diff --git a/apps/studio/components/ui/DataTable/DataTableFilters/DataTableFilterControlsDrawer.tsx b/apps/studio/components/ui/DataTable/DataTableFilters/DataTableFilterControlsDrawer.tsx index 1e89501fc941d..71700a6f8a985 100644 --- a/apps/studio/components/ui/DataTable/DataTableFilters/DataTableFilterControlsDrawer.tsx +++ b/apps/studio/components/ui/DataTable/DataTableFilters/DataTableFilterControlsDrawer.tsx @@ -19,6 +19,7 @@ import { import { useMediaQuery } from '../hooks/useMediaQuery' import { Kbd } from '../primitives/Kbd' import { DataTableFilterControls } from './DataTableFilterControls' +import { UserLogFilterControl } from '@/components/interfaces/UnifiedLogs/components/UserLogFilterControl' import { SHORTCUT_IDS } from '@/state/shortcuts/registry' import { useShortcut } from '@/state/shortcuts/useShortcut' @@ -63,7 +64,7 @@ export function DataTableFilterControlsDrawer() {
    - + }} />
    diff --git a/apps/studio/components/ui/DataTable/DataTableInfinite.tsx b/apps/studio/components/ui/DataTable/DataTableInfinite.tsx index 43dddc7263466..179558cd4c0fb 100644 --- a/apps/studio/components/ui/DataTable/DataTableInfinite.tsx +++ b/apps/studio/components/ui/DataTable/DataTableInfinite.tsx @@ -3,7 +3,7 @@ import type { ColumnDef, Row, Table as TTable, VisibilityState } from '@tanstack import { flexRender } from '@tanstack/react-table' import { LoaderCircle } from 'lucide-react' import { useQueryState } from 'nuqs' -import { Fragment, UIEvent, useCallback, useRef } from 'react' +import { Fragment, ReactNode, UIEvent, useCallback, useRef } from 'react' import { Button, cn, Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from 'ui' import { ShimmeringLoader } from 'ui-patterns/ShimmeringLoader' @@ -27,6 +27,8 @@ export interface DataTableInfiniteProps { fetchNextPage: (options?: FetchNextPageOptions | undefined) => Promise setColumnOrder: (columnOrder: string[]) => void setColumnVisibility: (columnVisibility: VisibilityState) => void + /** Overrides the "No results found" copy shown when the current filters can't match any row. */ + emptyStateMessage?: string | ReactNode // [Joshen] See if we can type this properly searchParamsParser: any @@ -43,6 +45,7 @@ export function DataTableInfinite({ totalRowsFetched = 0, setColumnOrder, setColumnVisibility, + emptyStateMessage = 'No results found', searchParamsParser, }: DataTableInfiniteProps) { const tableRef = useRef(null) @@ -193,7 +196,11 @@ export function DataTableInfinite({ className={cn(TableCellClassName, 'text-center')} >
    -

    No results found

    + {typeof emptyStateMessage === 'string' ? ( +

    {emptyStateMessage}

    + ) : ( + emptyStateMessage + )}
    diff --git a/apps/studio/components/ui/DataTable/DataTableResetButton.tsx b/apps/studio/components/ui/DataTable/DataTableResetButton.tsx index 7cc8a49fd2618..158326bb76fb0 100644 --- a/apps/studio/components/ui/DataTable/DataTableResetButton.tsx +++ b/apps/studio/components/ui/DataTable/DataTableResetButton.tsx @@ -6,9 +6,19 @@ import { ShortcutTooltip } from '@/components/ui/ShortcutTooltip' import { SHORTCUT_IDS } from '@/state/shortcuts/registry' import { useShortcut } from '@/state/shortcuts/useShortcut' -export function DataTableResetButton() { +export interface DataTableResetButtonProps { + /** Called alongside `table.resetColumnFilters()` — for filters that live outside TanStack Table's columnFilters state (e.g. a cross-cutting URL param). */ + onReset?: () => void +} + +export function DataTableResetButton({ onReset }: DataTableResetButtonProps) { const { table } = useDataTable() - useShortcut(SHORTCUT_IDS.DATA_TABLE_RESET_FILTERS, () => table.resetColumnFilters(), { + const reset = () => { + table.resetColumnFilters() + onReset?.() + } + + useShortcut(SHORTCUT_IDS.DATA_TABLE_RESET_FILTERS, reset, { registerInCommandMenu: true, }) @@ -18,7 +28,7 @@ export function DataTableResetButton() { label="Reset filters" side="left" > - diff --git a/apps/studio/components/ui/DataTable/FilterSideBar.tsx b/apps/studio/components/ui/DataTable/FilterSideBar.tsx index 71fb110655c0c..09f408c004c00 100644 --- a/apps/studio/components/ui/DataTable/FilterSideBar.tsx +++ b/apps/studio/components/ui/DataTable/FilterSideBar.tsx @@ -1,5 +1,6 @@ import { useParams } from 'common' import Link from 'next/link' +import { parseAsString, useQueryState } from 'nuqs' import { cloneElement, Dispatch, SetStateAction, useEffect } from 'react' import { Badge, Button, cn, ResizablePanel, usePanelRef } from 'ui' @@ -9,6 +10,7 @@ import { DataTableFilterControls } from './DataTableFilters/DataTableFilterContr import { DataTableResetButton } from './DataTableResetButton' import { useDataTable } from './providers/DataTableProvider' import { LOG_DRAIN_TYPES } from '@/components/interfaces/LogDrains/LogDrains.constants' +import { UserLogFilterControl } from '@/components/interfaces/UnifiedLogs/components/UserLogFilterControl' import { UnifiedLogsBanner } from '@/components/interfaces/UnifiedLogs/UnifiedLogsBanner' interface FilterSideBarProps { @@ -24,6 +26,7 @@ export function FilterSideBar({ }: FilterSideBarProps) { const { ref } = useParams() const { table } = useDataTable() + const [user, setUser] = useQueryState('user', parseAsString) const panelRef = usePanelRef() @@ -58,14 +61,19 @@ export function FilterSideBar({

    Logs

    Beta
- {table.getState().columnFilters.length ? : null} + {table.getState().columnFilters.length || user ? ( + setUser(null)} /> + ) : null}
- + }} + /> { const { ref } = useParams() const { data } = useProjectUpgradingStatusQuery({ projectRef: ref }, { enabled: IS_PLATFORM }) const { status, initiated_at, latest_status_at, error } = data?.databaseUpgradeStatus ?? {} + const showPostgresUpgradeLogs = useShowPostgresUpgradeLogs() const isFailed = status === DatabaseUpgradeStatus.Failed const initiatedAt = dayjs @@ -64,13 +66,15 @@ export const ProjectUpgradeFailedBanner = () => { Your project and its data are not affected. Please reach out to us via our support form for assistance with the upgrade.
-
- You may also view logs related to the failed upgrade in your{' '} - - project's logs - - . -
+ {showPostgresUpgradeLogs && ( +
+ You may also view logs related to the failed upgrade in your{' '} + + project's logs + + . +
+ )} ) diff --git a/apps/studio/data/auth/keys.ts b/apps/studio/data/auth/keys.ts index baf29ce6dc18a..b1b4cb6cd3783 100644 --- a/apps/studio/data/auth/keys.ts +++ b/apps/studio/data/auth/keys.ts @@ -14,6 +14,8 @@ export const authKeys = { column?: OptimizedSearchColumns } ) => ['projects', projectRef, 'users-infinite', params].filter(Boolean), + usersSearch: (projectRef: string | undefined, keywords: string) => + ['projects', projectRef, 'users-search', keywords] as const, usersCount: ( projectRef: string | undefined, params?: { diff --git a/apps/studio/hooks/misc/useShowPostgresUpgradeLogs.ts b/apps/studio/hooks/misc/useShowPostgresUpgradeLogs.ts new file mode 100644 index 0000000000000..9294bc979639a --- /dev/null +++ b/apps/studio/hooks/misc/useShowPostgresUpgradeLogs.ts @@ -0,0 +1,5 @@ +import { useFlag } from 'common' + +export const useShowPostgresUpgradeLogs = () => { + return useFlag('showPostgresUpgradeLogs') +} diff --git a/apps/studio/package.json b/apps/studio/package.json index 2b403ab6629a5..72da599debab9 100644 --- a/apps/studio/package.json +++ b/apps/studio/package.json @@ -112,6 +112,7 @@ "ip-num": "^1.5.1", "json-logic-js": "^2.0.2", "lodash": "catalog:", + "lodash-es": "catalog:", "lucide-react": "^0.436.0", "markdown-table": "^3.0.3", "mime-db": "^1.53.0", diff --git a/apps/studio/pages/integrations/vercel/[slug]/deploy-button/new-project.tsx b/apps/studio/pages/integrations/vercel/[slug]/deploy-button/new-project.tsx index 1b55a32e4b929..2e1f3b8d2778a 100644 --- a/apps/studio/pages/integrations/vercel/[slug]/deploy-button/new-project.tsx +++ b/apps/studio/pages/integrations/vercel/[slug]/deploy-button/new-project.tsx @@ -1,223 +1,36 @@ import { useParams } from 'common' -import { ChangeEvent, useEffect, useRef, useState } from 'react' -import { AWS_REGIONS } from 'shared-data' -import { toast } from 'sonner' -import { - Button, - Checkbox, - Input, - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from 'ui' +import { useEffect, useState } from 'react' import { Admonition } from 'ui-patterns/admonition' -import { Input as PasswordInput } from 'ui-patterns/DataInputs/Input' -import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout' import { isVercelUrl } from '@/components/interfaces/Integrations/Vercel/VercelIntegration.utils' -import { Markdown } from '@/components/interfaces/Markdown' +import { ProjectCreationForm } from '@/components/interfaces/ProjectCreation/ProjectCreationForm' import VercelIntegrationWindowLayout from '@/components/layouts/IntegrationsLayout/VercelIntegrationWindowLayout' import { ScaffoldColumn, ScaffoldContainer } from '@/components/layouts/Scaffold' -import { PasswordStrengthBar } from '@/components/ui/PasswordStrengthBar' import { useProjectSettingsV2Query } from '@/data/config/project-settings-v2-query' import { useIntegrationsQuery } from '@/data/integrations/integrations-query' import { useIntegrationVercelConnectionsCreateMutation } from '@/data/integrations/integrations-vercel-connections-create-mutation' import { useVercelProjectsQuery } from '@/data/integrations/integrations-vercel-projects-query' import { useOrganizationsQuery } from '@/data/organizations/organizations-query' -import { useProjectCreateMutation } from '@/data/projects/project-create-mutation' -import { - isInDataApiRevokeTreatment, - useDataApiRevokeOnCreateDefaultEnabled, - useTrackDefaultPrivilegesExposure, -} from '@/hooks/misc/useDataApiRevokeOnCreateDefault' -import { useSelectedOrganizationQuery } from '@/hooks/misc/useSelectedOrganization' -import { usePHFlag } from '@/hooks/ui/useFlag' -import { BASE_PATH, PROVIDERS } from '@/lib/constants' -import { getInitialMigrationSQLFromGitHubRepo } from '@/lib/integration-utils' -import { passwordStrength, PasswordStrengthScore } from '@/lib/password-strength' -import { generateStrongPassword } from '@/lib/project' -import { useTrack } from '@/lib/telemetry/track' import { useIntegrationInstallationSnapshot } from '@/state/integration-installation' import type { NextPageWithLayout } from '@/types' const VercelIntegration: NextPageWithLayout = () => { - return ( - <> - - -
-

New project

- -
- - - - - - ) -} - -VercelIntegration.getLayout = (page) => ( - {page} -) - -const CreateProject = () => { - const { data: selectedOrganization } = useSelectedOrganizationQuery() - const [projectName, setProjectName] = useState('') - const [dbPass, setDbPass] = useState('') - const [passwordStrengthMessage, setPasswordStrengthMessage] = useState('') - const [passwordStrengthScore, setPasswordStrengthScore] = useState(-1) - const [shouldRunMigrations, setShouldRunMigrations] = useState(true) - const [dbRegion, setDbRegion] = useState(PROVIDERS.AWS.default_region.displayName) - - const track = useTrack() + const { slug, next, currentProjectId: foreignProjectId } = useParams() const snapshot = useIntegrationInstallationSnapshot() - const isDataApiRevokeOnCreateDefault = useDataApiRevokeOnCreateDefaultEnabled() - const dataApiRevokeOnCreateDefaultFlag = usePHFlag( - 'dataApiRevokeOnCreateDefault' - ) - const [dataApiDefaultPrivileges, setDataApiDefaultPrivileges] = useState( - !isDataApiRevokeOnCreateDefault - ) - const hasUserModifiedDataApiDefaultPrivileges = useRef(false) - useEffect(() => { - if (dataApiRevokeOnCreateDefaultFlag === undefined) return - if (hasUserModifiedDataApiDefaultPrivileges.current) return - setDataApiDefaultPrivileges(!isInDataApiRevokeTreatment(dataApiRevokeOnCreateDefaultFlag)) - }, [dataApiRevokeOnCreateDefaultFlag]) + const [newProjectRef, setNewProjectRef] = useState() - const { slug, next, currentProjectId: foreignProjectId, externalId } = useParams() - - useTrackDefaultPrivilegesExposure({ - surface: 'vercel', - orgSlug: slug, - dataApiDefaultPrivileges, - hasUserModified: hasUserModifiedDataApiDefaultPrivileges.current, - }) - - async function checkPasswordStrength(value: string) { - const { message, strength } = await passwordStrength(value) - setPasswordStrengthScore(strength) - setPasswordStrengthMessage(message) - } - - const { mutateAsync: createConnections } = useIntegrationVercelConnectionsCreateMutation() + const { data: integrationData } = useIntegrationsQuery() + const organizationIntegration = integrationData?.find((x) => x.organization.slug === slug) const { data: organizationData } = useOrganizationsQuery() const organization = organizationData?.find((x) => x.slug === slug) - const hasTrackedFormExposed = useRef(false) - useEffect(() => { - if (hasTrackedFormExposed.current) return - if (!slug) return - hasTrackedFormExposed.current = true - track('project_creation_form_exposed', { surface: 'vercel' }, { organization: slug }) - }, [slug, track]) - - /** - * array of integrations installed - */ - const { data: integrationData } = useIntegrationsQuery() - - /** - * the vercel integration installed for organization chosen - */ - const organizationIntegration = integrationData?.find((x) => x.organization.slug === slug) - - /** - * Vercel projects available for this integration - */ const { data: vercelProjects } = useVercelProjectsQuery( - { - organization_integration_id: organizationIntegration?.id, - }, + { organization_integration_id: organizationIntegration?.id }, { enabled: organizationIntegration !== undefined } ) - function onProjectNameChange(e: ChangeEvent) { - e.target.value = e.target.value.replace(/\./g, '') - setProjectName(e.target.value) - } - - function onDbPassChange(e: ChangeEvent) { - const value = e.target.value - setDbPass(value) - if (value == '') { - setPasswordStrengthScore(-1) - setPasswordStrengthMessage('') - } else checkPasswordStrength(value) - } - - function generatePassword() { - const password = generateStrongPassword() - setDbPass(password) - checkPasswordStrength(password) - } - - const [newProjectRef, setNewProjectRef] = useState(undefined) - - const { mutate: createProject } = useProjectCreateMutation({ - onSuccess: (res) => { - setNewProjectRef(res.ref) - track( - 'project_creation_simple_version_submitted', - { - surface: 'vercel', - dataApiEnabled: true, - dataApiDefaultPrivilegesGranted: dataApiDefaultPrivileges, - ...(dataApiRevokeOnCreateDefaultFlag !== undefined && { - dataApiRevokeOnCreateDefaultEnabled: dataApiRevokeOnCreateDefaultFlag, - }), - }, - { - project: res.ref, - organization: res.organization_slug, - } - ) - }, - onError: (error) => { - toast.error(error.message) - snapshot.setLoading(false) - }, - }) - - async function onCreateProject() { - if (!organizationIntegration) return console.error('No organization installation details found') - if (!organizationIntegration?.id) return console.error('No organization installation ID found') - if (!foreignProjectId) return console.error('No foreignProjectId set') - if (!organization) return console.error('No organization set') - - snapshot.setLoading(true) - - let dbSql: string | undefined - if (shouldRunMigrations) { - const id = toast(`Fetching initial migrations from GitHub repo`) - const migrationSql = await getInitialMigrationSQLFromGitHubRepo(externalId) - if (migrationSql) dbSql = migrationSql - toast.success(`Done fetching initial migrations`, { id }) - } - - createProject({ - organizationSlug: organization.slug, - name: projectName, - dbPass, - dbRegion, - dbSql, - dataApiRevokeDefaultPrivileges: !dataApiDefaultPrivileges, - }) - } - // Wait for the new project to be created before creating the connection const { data, isSuccess } = useProjectSettingsV2Query( { projectRef: newProjectRef }, @@ -230,8 +43,12 @@ const CreateProject = () => { }, } ) + + const { mutateAsync: createConnections } = useIntegrationVercelConnectionsCreateMutation() + useEffect(() => { if (!isSuccess) return + const onSuccessFunc = async () => { const isReady = (data.service_api_keys ?? []).length > 0 @@ -239,7 +56,7 @@ const CreateProject = () => { return } - const projectDetails = vercelProjects?.find((x: any) => x.id === foreignProjectId) + const projectDetails = vercelProjects?.find((x) => x.id === foreignProjectId) try { await createConnections({ @@ -257,7 +74,7 @@ const CreateProject = () => { }, }, }, - orgSlug: selectedOrganization?.slug, + orgSlug: organization?.slug, }) } catch (error) { console.error('An error occurred during createConnections:', error) @@ -266,157 +83,31 @@ const CreateProject = () => { snapshot.setLoading(false) - if (next && isVercelUrl(next)) { - window.location.href = next - } + if (next && isVercelUrl(next)) window.location.href = next } + onSuccessFunc() + // eslint-disable-next-line react-hooks/exhaustive-deps }, [data, isSuccess]) return ( -
-

Supabase project details

-
- - - -
-
- - } - > - 0} - onChange={onDbPassChange} - /> - -
-
-
- - - -
-
-
-
- setShouldRunMigrations(!!checked)} - /> -
- -

- To get you started quickly, we can create new tables for you with seed (sample) data. - You can delete these tables later. -

-
-
-
-
-
- { - hasUserModifiedDataApiDefaultPrivileges.current = true - setDataApiDefaultPrivileges(!!checked) - }} - /> -
- -

- Grants privileges to Data API roles by default, exposing new tables. We recommend - disabling this to control access manually. -

-
-
-
-
- -
-
+ + + + + + + ) } +VercelIntegration.getLayout = (page) => ( + {page} +) + export default VercelIntegration diff --git a/apps/studio/routes/__root.tsx b/apps/studio/routes/__root.tsx index dd80a116015a9..6766027df6280 100644 --- a/apps/studio/routes/__root.tsx +++ b/apps/studio/routes/__root.tsx @@ -57,6 +57,7 @@ import { } from 'react' import { ErrorBoundary } from 'react-error-boundary' import { TooltipProvider } from 'ui' +import { TimestampInfoProvider } from 'ui-patterns/TimestampInfo' import { StudioCommandMenu } from '@/components/interfaces/App/CommandMenu' import { StudioCommandProvider as CommandProvider } from '@/components/interfaces/App/CommandMenu/StudioCommandProvider' @@ -67,11 +68,13 @@ import { RouteValidationWrapper } from '@/components/interfaces/App/RouteValidat import { MainScrollContainerProvider } from '@/components/layouts/MainScrollContainerContext' import { BannerStackProvider } from '@/components/ui/BannerStack/BannerStackProvider' import { GlobalErrorBoundaryState } from '@/components/ui/ErrorBoundary/GlobalErrorBoundaryState' +import { GlobalShortcuts } from '@/components/ui/GlobalShortcuts/GlobalShortcuts' import { useCustomContent } from '@/hooks/custom-content/useCustomContent' import { useSelectedOrganizationQuery } from '@/hooks/misc/useSelectedOrganization' import { AuthProvider } from '@/lib/auth' import { configureMonacoLoader } from '@/lib/configure-monaco-loader' import { API_URL, BASE_PATH, IS_PLATFORM, useDefaultProvider } from '@/lib/constants' +import { TimezoneProvider, useTimezone } from '@/lib/datetime' // Custom adapter instead of `nuqs/adapters/tanstack-router` — the stock one // injects a trailing slash before the query on every nuqs write (see module). import { NuqsAdapter } from '@/lib/nuqs-tanstack-adapter' @@ -99,14 +102,36 @@ const FeatureFlagProviderWithOrgContext = ({ ...props }: ComponentProps) => { const { data: selectedOrganization } = useSelectedOrganizationQuery({ enabled: IS_PLATFORM }) + const cloudProvider = useDefaultProvider() + + const getConfigCatFlags = useCallback( + (userEmail?: string) => { + const customAttributes: Record = {} + if (cloudProvider) customAttributes.cloud_provider = cloudProvider + if (selectedOrganization?.plan?.id) customAttributes.plan = selectedOrganization.plan.id + return getFlags(userEmail, customAttributes) + }, + [cloudProvider, selectedOrganization?.plan?.id] + ) return ( - + {children} ) } +// Bridges the user's stored timezone preference into TimestampInfoProvider so +// dayjs.tz.setDefault runs app-wide (see @/lib/datetime). +const TimestampInfoTimezoneBridge = ({ children }: { children: ReactNode }) => { + const { timezone } = useTimezone() + return {children} +} + const IS_NON_PROD_ENV = process.env.NEXT_PUBLIC_ENVIRONMENT === 'local' || process.env.NEXT_PUBLIC_ENVIRONMENT === 'staging' @@ -190,7 +215,7 @@ function buildRootHead() { const meta: Array> = [ { charSet: 'utf-8' }, { name: 'viewport', content: 'initial-scale=1.0, width=device-width' }, - { property: 'og:image', content: `${BASE_PATH}/img/supabase-logo.png` }, + { property: 'og:image', content: `${BASE_PATH}/img/supabase-og.png` }, { name: 'googlebot', content: 'notranslate' }, { name: 'application-name', content: APPLICATION_NAME }, { name: 'msapplication-TileColor', content: `#${THEME_COLOR}` }, @@ -316,59 +341,50 @@ export const Route = createRootRouteWithContext()({ function RootComponent() { useThemeSandbox() - const cloudProvider = useDefaultProvider() - - const getConfigCatFlags = useCallback( - (userEmail?: string) => { - const customAttributes = cloudProvider ? { cloud_provider: cloudProvider } : undefined - return getFlags(userEmail, customAttributes) - }, - [cloudProvider] - ) - return ( - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/apps/studio/routes/_auth.tsx b/apps/studio/routes/_auth.tsx index b3fc40e8a4926..d6f5d77f24143 100644 --- a/apps/studio/routes/_auth.tsx +++ b/apps/studio/routes/_auth.tsx @@ -1,15 +1,14 @@ import { createFileRoute, Outlet } from '@tanstack/react-router' -import { AuthenticationLayout } from '@/components/layouts/AuthenticationLayout' - export const Route = createFileRoute('/_auth')({ component: AuthShell, }) +// No shared layout here. In Next only pages/sign-in.tsx wraps itself in +// AuthenticationLayout via getLayout; every other auth page renders just its +// inner layout (SignInLayout / ForgotPasswordLayout / InterstitialLayout) with +// no AuthenticationLayout. The sign-in route wraps AuthenticationLayout at the +// leaf to match. function AuthShell() { - return ( - - - - ) + return } diff --git a/apps/studio/routes/_auth/cli/login.tsx b/apps/studio/routes/_auth/cli/login.tsx index ea50cb77a3ff5..97be430653d4c 100644 --- a/apps/studio/routes/_auth/cli/login.tsx +++ b/apps/studio/routes/_auth/cli/login.tsx @@ -7,6 +7,7 @@ export const Route = createFileRoute('/_auth/cli/login')({ }) function CliLogin() { - // Next page default export is already wrapped in withAuth and inlines APIAuthorizationLayout. + // Next page default export is already wrapped in withAuth and inlines InterstitialLayout + // (via a local CliLoginInterstitial wrapper). return } diff --git a/apps/studio/routes/_auth/partners/stripe/projects/login.tsx b/apps/studio/routes/_auth/partners/stripe/projects/login.tsx index 642d4438277dd..1dcb68b4d7093 100644 --- a/apps/studio/routes/_auth/partners/stripe/projects/login.tsx +++ b/apps/studio/routes/_auth/partners/stripe/projects/login.tsx @@ -7,6 +7,6 @@ export const Route = createFileRoute('/_auth/partners/stripe/projects/login')({ }) function StripeProjectsLogin() { - // Next page default export is already wrapped in withAuth and inlines APIAuthorizationLayout. + // Next page default export is already wrapped in withAuth and inlines InterstitialLayout. return } diff --git a/apps/studio/routes/_auth/sign-in.tsx b/apps/studio/routes/_auth/sign-in.tsx index 6a1da95e7dfd3..b562d5bad013d 100644 --- a/apps/studio/routes/_auth/sign-in.tsx +++ b/apps/studio/routes/_auth/sign-in.tsx @@ -1,5 +1,6 @@ import { createFileRoute } from '@tanstack/react-router' +import { AuthenticationLayout } from '@/components/layouts/AuthenticationLayout' import { SignInLayout } from '@/components/layouts/SignInLayout/SignInLayout' import SignInPage from '@/pages/sign-in' @@ -9,13 +10,15 @@ export const Route = createFileRoute('/_auth/sign-in')({ function SignIn() { return ( - - - + + + + + ) } diff --git a/apps/studio/routes/api/v1/projects/$ref/functions/$slug/body.ts b/apps/studio/routes/api/v1/projects/$ref/functions/$slug/body.ts index e3af4ac10b950..a471c38546d53 100644 --- a/apps/studio/routes/api/v1/projects/$ref/functions/$slug/body.ts +++ b/apps/studio/routes/api/v1/projects/$ref/functions/$slug/body.ts @@ -1,5 +1,6 @@ import { createReadStream } from 'node:fs' import { Readable } from 'node:stream' +import * as Sentry from '@sentry/nextjs' import { createFileRoute } from '@tanstack/react-router' import { getFunctionsArtifactStore } from '@/lib/api/self-hosted/functions' @@ -19,75 +20,90 @@ import { uuidv4 } from '@/lib/helpers' // effective no-op when `IS_PLATFORM` is false). const GET = async ({ params }: { params: { ref?: string; slug?: string } }) => { - const { slug } = params - if (!slug) { - return new Response( - JSON.stringify({ error: { message: `Missing function 'slug' parameter` } }), - { status: 404, headers: { 'Content-Type': 'application/json' } } - ) - } - - const store = getFunctionsArtifactStore() - const fileEntries = await store.getFileEntriesBySlug(slug) + // Mirrors `apiWrapper`'s global catch (lib/api/apiWrapper.ts): capture to + // Sentry and return a 500 error body. Only errors thrown before the + // stream `Response` is returned are catchable here — a failure mid-stream + // (e.g. a file read) can't roll back an already-sent 200 and is left to the + // stream's own error propagation, which destroys the response. + try { + const { slug } = params + if (!slug) { + return new Response( + JSON.stringify({ error: { message: `Missing function 'slug' parameter` } }), + { status: 404, headers: { 'Content-Type': 'application/json' } } + ) + } - const boundary = `----FormBoundary${uuidv4().replace(/-/g, '')}` - const totalSize = fileEntries.reduce((sum, entry) => sum + entry.size, 0) - const metadata = { - // mock id, should be "__" - deployment_id: uuidv4(), - original_size: totalSize, - compressed_size: totalSize, - module_count: fileEntries.length, - } + const store = getFunctionsArtifactStore() + const fileEntries = await store.getFileEntriesBySlug(slug) - const encoder = new TextEncoder() + const boundary = `----FormBoundary${uuidv4().replace(/-/g, '')}` + const totalSize = fileEntries.reduce((sum, entry) => sum + entry.size, 0) + const metadata = { + // mock id, should be "__" + deployment_id: uuidv4(), + original_size: totalSize, + compressed_size: totalSize, + module_count: fileEntries.length, + } - // An async generator lets `Readable.from` drive the file reads on demand: - // it pauses the generator when the consumer is slow (backpressure) and runs - // the generator's cleanup (closing the file handle) if the stream is - // destroyed on client disconnect — neither of which an eager - // `ReadableStream.start()` loop does. - async function* multipartBody() { - yield encoder.encode( - `--${boundary}\r\n` + - `Content-Disposition: form-data; name="metadata"\r\n` + - `Content-Type: application/json\r\n` + - `\r\n` + - JSON.stringify(metadata) + - `\r\n` - ) + const encoder = new TextEncoder() - for (const entry of fileEntries) { - const safeName = entry.relativePath - .replace(/[\r\n]/g, '') - .replace(/\\/g, '\\\\') - .replace(/"/g, '\\"') - const encodedName = encodeURIComponent(entry.relativePath) + // An async generator lets `Readable.from` drive the file reads on demand: + // it pauses the generator when the consumer is slow (backpressure) and runs + // the generator's cleanup (closing the file handle) if the stream is + // destroyed on client disconnect — neither of which an eager + // `ReadableStream.start()` loop does. + async function* multipartBody() { yield encoder.encode( `--${boundary}\r\n` + - `Content-Disposition: form-data; name="file"; filename="${safeName}"; filename*=UTF-8''${encodedName}\r\n` + - `Content-Type: text/plain\r\n` + + `Content-Disposition: form-data; name="metadata"\r\n` + + `Content-Type: application/json\r\n` + + `\r\n` + + JSON.stringify(metadata) + `\r\n` ) - for await (const chunk of createReadStream(entry.absolutePath)) { - yield chunk as Uint8Array + for (const entry of fileEntries) { + const safeName = entry.relativePath + .replace(/[\r\n]/g, '') + .replace(/\\/g, '\\\\') + .replace(/"/g, '\\"') + const encodedName = encodeURIComponent(entry.relativePath) + yield encoder.encode( + `--${boundary}\r\n` + + `Content-Disposition: form-data; name="file"; filename="${safeName}"; filename*=UTF-8''${encodedName}\r\n` + + `Content-Type: text/plain\r\n` + + `\r\n` + ) + + for await (const chunk of createReadStream(entry.absolutePath)) { + yield chunk as Uint8Array + } + + yield encoder.encode(`\r\n`) } - yield encoder.encode(`\r\n`) + yield encoder.encode(`--${boundary}--\r\n`) } - yield encoder.encode(`--${boundary}--\r\n`) - } + const stream = Readable.toWeb( + Readable.from(multipartBody(), { objectMode: false }) + ) as ReadableStream - const stream = Readable.toWeb( - Readable.from(multipartBody(), { objectMode: false }) - ) as ReadableStream - - return new Response(stream, { - status: 200, - headers: { 'Content-Type': `multipart/form-data; boundary=${boundary}` }, - }) + return new Response(stream, { + status: 200, + headers: { 'Content-Type': `multipart/form-data; boundary=${boundary}` }, + }) + } catch (error) { + Sentry.captureException(error) + // Generic message only — echoing the caught error can leak stack/internal + // details to the client (CodeQL js/stack-trace-exposure). + return new Response(JSON.stringify({ error: { message: 'Internal Server Error' } }), { + status: 500, + headers: { 'Content-Type': 'application/json' }, + }) + } } export const Route = createFileRoute('/api/v1/projects/$ref/functions/$slug/body')({ diff --git a/apps/studio/routes/integrations/vercel.tsx b/apps/studio/routes/integrations/vercel.tsx index dc2607f0f384e..e31fc665ab917 100644 --- a/apps/studio/routes/integrations/vercel.tsx +++ b/apps/studio/routes/integrations/vercel.tsx @@ -1,19 +1,14 @@ import { createFileRoute, Outlet } from '@tanstack/react-router' -import VercelIntegrationWindowLayout from '@/components/layouts/IntegrationsLayout/VercelIntegrationWindowLayout' - export const Route = createFileRoute('/integrations/vercel')({ - component: VercelIntegrationShell, + component: VercelIntegrationPassthrough, }) -// Placed at top-level rather than under _app — Next getLayout for all -// three leaves only wraps in VercelIntegrationWindowLayout (no AppLayout -// or DefaultLayout), so adding _app's chrome here would be a behaviour -// change. -function VercelIntegrationShell() { - return ( - - - - ) +// No shared layout here. Since #47623, the install and +// marketplace/choose-project pages render their own InterstitialLayout and +// have no Next getLayout, so the Next runtime shows them without any window +// chrome. Only deploy-button/new-project still uses +// VercelIntegrationWindowLayout, and its route wraps it at the leaf. +function VercelIntegrationPassthrough() { + return } diff --git a/apps/studio/routes/integrations/vercel/$slug/deploy-button/new-project.tsx b/apps/studio/routes/integrations/vercel/$slug/deploy-button/new-project.tsx index 11f685c82081e..0b09a4cf7e9c7 100644 --- a/apps/studio/routes/integrations/vercel/$slug/deploy-button/new-project.tsx +++ b/apps/studio/routes/integrations/vercel/$slug/deploy-button/new-project.tsx @@ -1,11 +1,18 @@ import { createFileRoute } from '@tanstack/react-router' +import VercelIntegrationWindowLayout from '@/components/layouts/IntegrationsLayout/VercelIntegrationWindowLayout' import VercelIntegration from '@/pages/integrations/vercel/[slug]/deploy-button/new-project' export const Route = createFileRoute('/integrations/vercel/$slug/deploy-button/new-project')({ component: VercelDeployButtonNewProjectRoute, }) +// Mirrors the page's Next getLayout, which wraps this leaf (and only this +// leaf) in VercelIntegrationWindowLayout. function VercelDeployButtonNewProjectRoute() { - return + return ( + + + + ) } diff --git a/apps/studio/routes/join.tsx b/apps/studio/routes/join.tsx index a7daab6490d57..1e989a11be62e 100644 --- a/apps/studio/routes/join.tsx +++ b/apps/studio/routes/join.tsx @@ -1,5 +1,4 @@ import { createFileRoute } from '@tanstack/react-router' -import { cn } from 'ui' import JoinOrganizationPage from '@/pages/join' @@ -8,15 +7,5 @@ export const Route = createFileRoute('/join')({ }) function Join() { - return ( -
- -
- ) + return } diff --git a/apps/studio/routes/project/$ref/integrations.tsx b/apps/studio/routes/project/$ref/integrations.tsx index a219f87b212a0..25dd7ae2c2fdb 100644 --- a/apps/studio/routes/project/$ref/integrations.tsx +++ b/apps/studio/routes/project/$ref/integrations.tsx @@ -1,6 +1,6 @@ import { createFileRoute, Outlet } from '@tanstack/react-router' -import { ProjectIntegrationsLayout } from '@/components/layouts/ProjectIntegrationsLayout' +import { ProjectIntegrationsLayoutDispatch } from '@/components/layouts/ProjectIntegrationsLayoutDispatch' export const Route = createFileRoute('/project/$ref/integrations')({ component: IntegrationsShell, @@ -8,8 +8,8 @@ export const Route = createFileRoute('/project/$ref/integrations')({ function IntegrationsShell() { return ( - + - + ) } diff --git a/apps/studio/tests/features/logs/Logs.Datepickers.test.tsx b/apps/studio/tests/features/logs/Logs.Datepickers.test.tsx index 9d61d27c80fb7..654a4d52a8c0c 100644 --- a/apps/studio/tests/features/logs/Logs.Datepickers.test.tsx +++ b/apps/studio/tests/features/logs/Logs.Datepickers.test.tsx @@ -296,6 +296,19 @@ test('passing a value prop shows the correct dates in the label', async () => { ) }) +test('opening with an unparseable date value (legacy epoch-ms its/ite) does not crash', async () => { + render( + + ) + + await userEvent.click(await screen.findByRole('button')) + expect(await screen.findByText('Apply')).toBeInTheDocument() +}) + test('passing a helper as a value prop shows the helper text in the label', async () => { const helper = { text: 'Last 7 days', diff --git a/apps/studio/tests/features/logs/LogsPreviewer.test.tsx b/apps/studio/tests/features/logs/LogsPreviewer.test.tsx index 8fc1356ea5cb7..fb0255f262c01 100644 --- a/apps/studio/tests/features/logs/LogsPreviewer.test.tsx +++ b/apps/studio/tests/features/logs/LogsPreviewer.test.tsx @@ -140,6 +140,20 @@ test('can click load older', async () => { expect(handleClick).toHaveBeenCalled() }) +test('loads its/ite from the URL into the date picker and opens without crashing', async () => { + const its = dayjs('2024-01-15T12:00:00.000Z') + const ite = dayjs('2024-01-15T12:02:00.000Z') + + customRender( + , + { nuqs: { searchParams: { its: its.toISOString(), ite: ite.toISOString() } } } + ) + + const label = `${its.format('DD MMM, HH:mm')} - ${ite.format('DD MMM, HH:mm')}` + await userEvent.click(await screen.findByText(label)) + expect(await screen.findByText('Apply')).toBeInTheDocument() +}) + describe('calculateBarClickTimeRange', () => { const clickedTime = '2024-01-15T12:30:00.000Z' diff --git a/apps/studio/vercel.ts b/apps/studio/vercel.ts index 712c5358ccf02..fe12d2d64f34d 100644 --- a/apps/studio/vercel.ts +++ b/apps/studio/vercel.ts @@ -74,6 +74,13 @@ function routesFor(prefix: string) { maxAge: '1year', immutable: true, }), + // Static images and favicons aren't content-hashed, so they can't be + // `immutable`, but they change rarely — mirror next.config's + // `cache-control` for these paths (img: max-age=2592000 = 30 days, + // favicon: max-age=86400 = 1 day). Prefixed like the other rules so + // `/dashboard/img/x.png` gets the header too when a basePath is set. + routes.cacheControl(`${prefix}/img/(.*)`, { public: true, maxAge: '30days' }), + routes.cacheControl(`${prefix}/favicon/(.*)`, { public: true, maxAge: '1day' }), ], } } @@ -116,13 +123,30 @@ function buildTanstackConfig(): VercelConfig { // a fallback for bare-domain traffic). const ruleSets = (basePath ? [basePath, ''] : ['']).map(routesFor) + // Vercel's Flags Explorer probes `/.well-known/vercel/flags` and expects + // JSON. next.config.ts proxies it to supabase.com's endpoint and forces + // `content-type: application/json`; the TanStack build has no equivalent, so + // without these the path falls through to the extensionless catch-all shell + // rule and the browser gets HTML. This lives at the domain root (once, NOT + // inside routesFor) because next.config's rewrite sets `basePath: false`, so + // the path is never basePath-prefixed — well-known URLs are by convention at + // the root regardless of the app's basePath. The rewrite must be listed + // BEFORE the routesFor rewrites so it wins over the shell catch-all. + const wellKnownFlags = '/.well-known/vercel/flags' + return { framework: null, outputDirectory: 'dist/client', cleanUrls: true, redirects: buildRedirects(), - rewrites: ruleSets.flatMap((r) => r.rewrites), - headers: ruleSets.flatMap((r) => r.headers), + rewrites: [ + routes.rewrite(wellKnownFlags, `https://supabase.com${wellKnownFlags}`), + ...ruleSets.flatMap((r) => r.rewrites), + ], + headers: [ + routes.header(wellKnownFlags, [{ key: 'content-type', value: 'application/json' }]), + ...ruleSets.flatMap((r) => r.headers), + ], // `api/server.js` imports the TanStack SSR bundle via a computed // path so Vercel's function bundler doesn't try to statically // resolve `dist/server/server.js` during the Next.js prod build @@ -131,6 +155,16 @@ function buildTanstackConfig(): VercelConfig { // runtime import resolves. functions: { 'api/server.js': { + // Every API + server-function request is rewritten onto this single + // function (see the `/api/*` and `/_serverFn/*` rewrites), so its + // timeout must cover the LONGEST per-route `maxDuration` the Next + // build declared — otherwise long AI streams and Stripe sync hit the + // platform default (~15s) and get killed. Next set these per route via + // `export const maxDuration`: generate-attachment-url (120), + // ai/code/complete (60), ai/onboarding/design (60), ai/sql/generate-v4 + // (120), ai/feedback/rate (30), and integrations/stripe-sync (300). + // TanStack collapses them into one function, so we take the max: 300s. + maxDuration: 300, // Ship the SSR output, plus libpg-query's wasm. libpg-query is // externalized for SSR and loads its `.wasm` relative to its own // dir (`__dirname`) at import time; Vercel's function bundler diff --git a/apps/studio/vite.config.ts b/apps/studio/vite.config.ts index 58ac32eb907ed..1d4d09879f6dc 100644 --- a/apps/studio/vite.config.ts +++ b/apps/studio/vite.config.ts @@ -1,6 +1,7 @@ /* eslint-disable no-restricted-exports */ import fs from 'node:fs' +import { createRequire } from 'node:module' import path from 'node:path' import { fileURLToPath } from 'node:url' import remapping, { type SourceMapInput } from '@jridgewell/remapping' @@ -15,6 +16,13 @@ import { defineConfig, loadEnv, type Plugin } from 'vite' const rootDir = path.dirname(fileURLToPath(import.meta.url)) const compatRoot = path.resolve(rootDir, 'compat/next') +// Absolute dir of lodash-es, for the SSR-only lodash alias below. Resolved +// here (not left as a bare 'lodash-es' replacement) because rollup-alias +// rewrites the id but resolution still runs from the ORIGINAL importer — +// under pnpm's strict node_modules, workspace packages that don't declare +// lodash-es (ui, common) would fail to resolve the bare specifier. +const lodashEsDir = path.dirname(createRequire(import.meta.url).resolve('lodash-es/package.json')) + // Map of Next imports we've shimmed to their TanStack-backed replacement. // Add an entry here + a file under compat/next/ when a new Next surface is // needed by app source. @@ -216,6 +224,30 @@ function umdAmdShortCircuit(): Plugin { // exports so Rolldown's static analysis is satisfied. Easier to stub the one // internal consumer — `GraphiQL.tsx` only exposes a default-export component, // and no SSR-reachable route renders it (the GraphiQL tab is client-only). +// SSR-only lodash → lodash-es rewrite, for the whole SSR module graph (app +// source, workspace packages, and node_modules deps alike). The CJS lodash +// in `ssr.noExternal` evaluates as pure ESM in the dev module runner — no +// `module`/`exports`/`require` — so its UMD wrapper silently attaches `_` to +// the global and every named import binds to `undefined`, exploding only when +// first CALLED during SSR render ("(0, __vite_ssr_import_0__.isEqual) is not +// a function"). Deep imports (`lodash/isEqual`) fail harder: their plain-CJS +// `require` throws "require is not defined". lodash-es is the same version as +// real ESM, so both import styles just work. Client bundles are untouched — +// resolution is gated on `options.ssr`, and the Next build doesn't read this +// config. +function ssrLodashEs(): Plugin { + return { + name: 'studio-ssr-lodash-es', + enforce: 'pre', + resolveId(source, _importer, options) { + if (!options?.ssr) return + if (source === 'lodash') return path.join(lodashEsDir, 'lodash.js') + const subpath = source.match(/^lodash\/(.+?)(\.js)?$/) + if (subpath) return path.join(lodashEsDir, `${subpath[1]}.js`) + }, + } +} + function ssrStubGraphiql(): Plugin { return { name: 'studio-ssr-stub-graphiql', @@ -718,7 +750,11 @@ export default defineConfig(({ command, mode }) => { postcss: { plugins: [] }, }, ssr: { - // `lodash` is CJS; its named-export interop fails in Node ESM unless bundled. + // `lodash` must stay inlined so its ids flow through the plugin + // pipeline, where ssrLodashEs (above) rewrites them to lodash-es. + // Externalized bare ids skip user plugins in the dev module runner, so + // dropping this entry resurfaces Node's CJS named-export failure + // ("Named export 'debounce' not found") on every `from 'lodash'` import. // `next/*` must be bundled so our nextCompat shim wins — otherwise Vite's // SSR externalizer leaves `next/router` as a runtime package import and // Node resolves it to Next's real module. @@ -749,6 +785,7 @@ export default defineConfig(({ command, mode }) => { rawTextLoader(), graphiqlViteWorkers(), ssrStubGraphiql(), + ssrLodashEs(), umdAmdShortCircuit(), assertNoChunkCycles(), devtools(), diff --git a/apps/www/_blog/2025-12-02-introducing-supabase-pipelines.mdx b/apps/www/_blog/2025-12-02-introducing-supabase-pipelines.mdx index ca850205ce27b..eaea807b1a263 100644 --- a/apps/www/_blog/2025-12-02-introducing-supabase-pipelines.mdx +++ b/apps/www/_blog/2025-12-02-introducing-supabase-pipelines.mdx @@ -13,7 +13,7 @@ toc_depth: 2 Today we're introducing **Supabase Pipelines**: managed change-data-capture pipelines that replicate your Postgres tables to analytical destinations in near real time. -Pipelines reads changes from your Postgres database and writes them to external destinations. It uses logical replication to capture inserts, updates, deletes, and truncates as they happen. **Setup takes minutes in the Supabase Dashboard.** +Pipelines reads changes from your Postgres database and writes them to supported destination systems. It uses logical replication to capture inserts, updates, deletes, and truncates as they happen. **Setup takes minutes in the Supabase Dashboard.** The first supported destination is Google BigQuery. @@ -29,7 +29,7 @@ Analytical systems are designed for this. They can aggregate massive datasets ** ## How it works -Pipelines captures every change in your Postgres database and delivers it to your analytics destination in near real time. +Pipelines captures the changes selected by your Postgres publication and delivers them to your analytics destination in near real time. Here's how: @@ -43,7 +43,7 @@ Here's how: 5. Your data is available for querying in the destination -The pipeline starts with an initial copy of your selected tables, then switches to streaming mode. **Your analytics stay fresh with latency measured in milliseconds to seconds.** +The pipeline starts with an initial sync of your selected tables, then begins ongoing replication. Changes are batched, so replication latency depends on source activity, network conditions, and destination throughput. ## Setting up Pipelines @@ -109,7 +109,7 @@ Read replicas and Pipelines solve different problems. Read replicas help when you need to scale concurrent queries, but they're still Postgres. They don't make analytics faster. -Pipelines moves your data to systems built for analytics. You get **faster queries on large datasets**, **lower storage costs** through compression, and **complete separation** between your production workload and analytics. +Pipelines moves your data to systems built for analytics, keeping heavy analytical queries away from your production database. Query performance, storage format, compression, and destination costs depend on the destination you choose. You can use both: read replicas for application read scaling, Pipelines for analytics. @@ -117,22 +117,22 @@ You can use both: read replicas for application read scaling, Pipelines for anal Replication with Pipelines has a few constraints to be aware of: -- Tables must have primary keys (this is a Postgres logical replication requirement) +- BigQuery requires every source table to have a primary key, and the publication must include its columns - Generated columns are not supported - Custom data types are replicated as strings -- Schema changes are not automatically propagated to destinations +- Schema change support is currently in beta and limited to the supported BigQuery changes - Data is replicated as-is, without transformation -- During the initial copy phase, changes accumulate in the WAL and are replayed once streaming begins +- During the initial sync, changes continue accumulating in the WAL and are applied as each table catches up -We're working on schema change support and additional destinations, and evaluating different streaming techniques to improve flexibility and performance. +BigQuery is currently available. You can [request early access to ClickHouse, Snowflake, and DuckLake](/go/supabase-pipelines-new-destinations) while we expand destination support. ## Pricing -Pipelines pricing is still being finalized during alpha. You can request access now; we'll announce pricing later and notify you before any charges apply. +Pipelines is billed for configured pipeline hours, initial sync data, and ongoing replication data. See [Pipelines pricing](/docs/guides/platform/manage-your-usage/pipelines) for current rates and billing examples. Destination-provider charges, such as BigQuery charges from Google Cloud, are separate. ## Get started -**Pipelines is currently in alpha.** Alpha features can be unstable and may introduce breaking changes while we evaluate the product direction, refine the feature set, and incorporate customer feedback. To request access, contact your account manager or fill out the form in the Dashboard. +**Supabase Pipelines is currently in public alpha.** Features and behavior may change as we continue developing the product. If Pipelines isn't available for your organization yet, request access in the Dashboard or contact your account manager. If you want to dive into the code, **the underlying Supabase ETL engine is open source** and written in Rust. Check out the repository at [github.com/supabase/etl](https://github.com/supabase/etl). diff --git a/apps/www/_blog/2026-05-05-realtime-or-pipelines-how-to-choose-the-right-tool.mdx b/apps/www/_blog/2026-05-05-realtime-or-pipelines-how-to-choose-the-right-tool.mdx index 9aff7c84fb061..611ca94bdd659 100644 --- a/apps/www/_blog/2026-05-05-realtime-or-pipelines-how-to-choose-the-right-tool.mdx +++ b/apps/www/_blog/2026-05-05-realtime-or-pipelines-how-to-choose-the-right-tool.mdx @@ -58,42 +58,42 @@ Postgres Changes uses temporary replication slots. When no clients are subscribe The Realtime team built it this way on purpose. Guaranteed delivery requires persistent state tracking, message queuing, and acknowledgment protocols. Those things add latency and complexity that would make Realtime worse at its actual job: delivering live updates as fast as possible. -If you need every change to arrive at its destination, no matter what, Realtime is not the right tool. +If another system needs a durable, recoverable copy of selected database data, Realtime is not the right tool. ## What Pipelines does -Pipelines creates managed change-data-capture (CDC) pipelines. A pipeline reads every INSERT, UPDATE, DELETE, and TRUNCATE from your Postgres tables and writes them to a supported destination. +Pipelines creates managed change-data-capture (CDC) pipelines. A pipeline reads the operations, rows, and columns selected by your Postgres publication and writes them to a supported destination. -Pipelines replicates your data 1-to-1 in near real time. If your destination disconnects or has problems, the pipeline does not skip over data. +Pipelines keeps the current state of selected Postgres data synchronized with a supported destination in near real time. It maps source names and values to destination-compatible representations and retries transient delivery failures. ### How Pipelines works -When you create a pipeline, it connects to your database through a permanent replication slot. First, it performs a full copy of your existing data. Then it switches to streaming mode and captures every change as it happens, with latency measured in milliseconds to seconds based on configuration parameters, data size, and destination type. +When you create a pipeline, it connects to your database through a persistent replication slot. First, it performs an initial sync of the existing rows in the publication. It then begins ongoing replication, processing changes in batches. Latency depends on source activity, network conditions, pipeline configuration, and destination throughput. -It's important to note that Pipelines doesn't respect Row-Level Security. A pipeline reads every piece of data in its publication. If you need to filter data, you should use publication filters. +Row Level Security does not apply to Pipelines. A pipeline can read all rows and columns selected by its publication. Use publication row filters and column lists to limit the replicated data. -Changes are batched and written to your destination. If the pipeline crashes, it restarts from the last acknowledged position. No data changes are lost. Note that schema changes (adding or removing columns) do not propagate automatically and require manual handling. +Changes are batched and written to your destination. After a restart, the pipeline resumes from its last acknowledged position while Postgres still retains the required WAL. BigQuery supports a limited set of schema changes in beta; other changes require manual handling. ### What Pipelines guarantees -Pipelines provides at-least-once delivery. Every change that happens in your database will reach the destination at least once. In rare cases (like a crash during a long-running transaction), a change might be delivered more than once. Exactly-once processing is handled by the destination. +Pipelines uses at-least-once processing. A retry or restart can process the same change more than once, so destination writes must be idempotent. BigQuery uses the replicated source primary key to apply current-state upserts and deletes. -Pipelines uses permanent replication slots. This means Postgres holds onto WAL data until the pipeline confirms it has been processed. If you stop the pipeline for maintenance and restart it later, it picks up exactly where it left off. Be aware that while the pipeline is paused, Postgres continues to retain WAL data. Extended pauses can lead to significant disk growth, and depending on your Postgres configuration, the pipeline may fail if the WAL retention limit is exceeded. +Pipelines uses persistent replication slots. Postgres retains WAL until the pipeline confirms it has been processed, subject to the database's WAL retention limit. If you stop and restart a pipeline while its slot remains valid, it continues from the retained WAL. A long stop can increase disk usage or invalidate the slot; recovering from a lost slot requires a new initial sync. -This is the opposite of Realtime's approach. Pipelines trades speed for reliability. It may not deliver changes to your warehouse in the same millisecond they happen, but it will deliver every single one. +This is the opposite of Realtime's approach. Pipelines prioritizes durable progress tracking and recovery over millisecond delivery to end users. ## The key differences ### Delivery guarantees -| | **Realtime** | **Pipelines** | -| ----------------------- | ------------ | --------------------- | -| Guarantee | Best effort | At-least-once | -| Missed changes | Lost forever | Replayed on reconnect | -| Replication slot | Temporary | Permanent | -| Resume after disconnect | No | Yes | +| | **Realtime** | **Pipelines** | +| ----------------------- | ------------ | --------------------------------------------------- | +| Guarantee | Best effort | At-least-once | +| Disconnected changes | Not replayed | Read from retained WAL while the slot remains valid | +| Replication slot | Temporary | Permanent | +| Resume after disconnect | No | Yes, while the replication slot remains valid | -This is the most important difference. If you need every change to arrive, use Pipelines. If you need changes to arrive fast and can tolerate occasional gaps, use Realtime. +This is the most important difference. Use Pipelines when another system needs a durable, monitored copy of your selected data. Use Realtime when client applications need live updates and can tolerate gaps while disconnected. ### Where data goes @@ -135,13 +135,11 @@ The common thread: a human is watching and needs to see changes as they happen. Use Pipelines when you need reliable data movement to analytical systems: - **Analytics and reporting.** Move production data to an analytical destination for querying without impacting your production database. -- **Audit trails.** Maintain a complete changelog of every INSERT, UPDATE, and DELETE. Nothing is lost. - **Data warehousing.** Replicate your operational data to a columnar format optimized for analytical queries. -- **Compliance.** Maintain a complete, verifiable history of all data changes. - **ML pipelines.** Feed fresh data to training or feature stores without querying production. - **Workload isolation.** Run heavy analytical queries against your warehouse instead of your production database. -The common thread: a system needs a complete, reliable copy of your data. +The common thread: a system needs the current state of selected production data in an analytical destination. Pipelines does not automatically create a queryable history of every row version. ## The mistake we see most often @@ -151,7 +149,7 @@ It works great in development. It even works fine in production for a while. The The problem is not that Realtime is broken. The problem is that Realtime was not designed for this job. -If you are piping database changes into another system and you need every change to arrive, use Pipelines. That is exactly what it was built for. +If you are maintaining a durable copy of database data in another system, use Pipelines. It persists replication progress and resumes from Postgres WAL after reconnecting. ## Can I use both? @@ -175,7 +173,7 @@ Realtime handles the live experience. Pipelines handles the analytical pipeline. | Database required | Only for Postgres Changes and Broadcast from Database | Yes, always | | Processing | Sequential per-change with per-subscriber authorization | Batched with configurable parallelism | | Latency | Typically under 100ms | Seconds (batched) | -| Best for | Human users watching live data | Systems consuming complete data | +| Best for | Human users watching live data | Systems consuming current replicated data | | Built with | Elixir (Phoenix) | Rust | | Open source | [github.com/supabase/realtime](https://github.com/supabase/realtime) | [Supabase ETL engine](https://github.com/supabase/etl) | @@ -183,4 +181,4 @@ Realtime handles the live experience. Pipelines handles the analytical pipeline. Realtime is available on all Supabase projects. Check out the [Realtime documentation](/docs/guides/realtime) to get started. -Pipelines is currently in alpha. Alpha features can be unstable and may introduce breaking changes while we evaluate the product direction, refine the feature set, and incorporate customer feedback. You can request access through the Supabase Dashboard or contact your account manager. Read the [Pipelines announcement](/blog/introducing-supabase-pipelines) for more details on how it works. +Supabase Pipelines is currently in public alpha. Features and behavior may change as we continue developing the product. If Pipelines isn't available for your organization yet, request access through the Supabase Dashboard or contact your account manager. Read the [Pipelines announcement](/blog/introducing-supabase-pipelines) for more details on how it works. diff --git a/apps/www/_go/pre-release/supabase-pipelines-new-destinations.tsx b/apps/www/_go/pre-release/supabase-pipelines-new-destinations.tsx index 9b21f110438d4..deb6bfa6078e6 100644 --- a/apps/www/_go/pre-release/supabase-pipelines-new-destinations.tsx +++ b/apps/www/_go/pre-release/supabase-pipelines-new-destinations.tsx @@ -39,13 +39,13 @@ const page: GoPageInput = { className: 'border-y border-muted bg-surface-75 py-16 sm:py-24', title: 'Bring Supabase Postgres into your analytics stack', description: - 'Pipelines keeps your application workload on Postgres while streaming production changes into systems built for analytics, reporting, and lakehouse workflows.', + 'Pipelines keeps your application workload on Postgres while ongoing replication sends production changes to systems built for analytics, reporting, and lakehouse workflows.', columns: 3, items: [ { title: 'ClickHouse', description: - 'Stream operational data into ClickHouse for high-volume analytical queries, dashboards, and event workloads.', + 'Replicate operational data into ClickHouse for high-volume analytical queries, dashboards, and event workloads.', }, { title: 'Snowflake', @@ -77,7 +77,7 @@ const page: GoPageInput = { { type: 'text', name: 'org_slug', - label: 'Supabase Organization Slug', + label: 'Supabase organization slug', placeholder: 'acme-inc', description: 'Use the slug from your Supabase Dashboard URL, for example /org/acme-inc.', required: true, diff --git a/apps/www/components/Pricing/PricingTableRow.tsx b/apps/www/components/Pricing/PricingTableRow.tsx index e1868d3b26434..9d497e48feed8 100644 --- a/apps/www/components/Pricing/PricingTableRow.tsx +++ b/apps/www/components/Pricing/PricingTableRow.tsx @@ -40,7 +40,7 @@ export const pricingTooltips: PricingTooltips = { main: 'Billing is based on the total sum of all outgoing traffic (includes Database, Storage, Realtime, Auth, API, Edge Functions, Supavisor, Log Drains) in GB throughout your billing period. Excludes cache hits.', }, 'database.replication': { - main: 'Supabase Pipelines is available as an add-on for paid plans. It provides change-data-capture pipelines that replicate your Postgres tables to analytical destinations in near real time.\nYou are billed for each active pipeline, the replicated bytes, and initial copies/backfills. Initial copies/backfills are billed at a lower rate.', + main: 'Supabase Pipelines is available as an add-on for paid plans. It provides change-data-capture pipelines that replicate your Postgres tables to analytical destinations in near real time.\nYou are billed for configured pipeline hours, initial sync data, and ongoing replication data. Pipeline hours continue while a pipeline is stopped and end when the pipeline is deleted.', }, 'storage.cachedEgress': { main: 'Billing is based on the total sum of outgoing Storage traffic in GB throughout your billing period that is served from our CDN cache.', diff --git a/apps/www/content/md/database.md b/apps/www/content/md/database.md index 83c071abf4e40..bbb74e7d910e6 100644 --- a/apps/www/content/md/database.md +++ b/apps/www/content/md/database.md @@ -17,7 +17,7 @@ Supabase Database gives you a full Postgres database with no compromises. It is - **Read Replicas**: distribute read traffic across replicas in multiple regions for lower latency and higher throughput - **Realtime**: subscribe to INSERT, UPDATE, DELETE, and other changes via WebSockets - **Database Webhooks**: trigger Edge Functions or external HTTP endpoints on table events -- **Supabase Pipelines**: create managed replication pipelines that stream Postgres changes to external data warehouses in near real time +- **Supabase Pipelines**: move published Postgres data to supported analytical destinations in near real time ## Technical Details diff --git a/apps/www/data/features.tsx b/apps/www/data/features.tsx index a8046ee69f721..d6ad46cdc6007 100644 --- a/apps/www/data/features.tsx +++ b/apps/www/data/features.tsx @@ -643,30 +643,33 @@ Foreign Data Wrappers simplify data integration by bringing external data into y }, { title: 'Supabase Pipelines', - subtitle: 'Real-time data replication to analytical destinations.', - description: `Supabase Pipelines creates managed change-data-capture pipelines that replicate your Postgres tables to analytical destinations in near real time. Reading directly from the Postgres Write Ahead Log, each pipeline keeps your analytics data synchronized with your production database. + subtitle: 'Replicate Postgres data to analytical destinations.', + description: `Supabase Pipelines is a managed change data capture (CDC) product that uses Postgres logical replication to deliver published data to analytical destinations in near real time. ## Key benefits -1. Real-time replication: Near real-time data synchronization using Postgres logical replication. -2. BigQuery integration: Direct replication to Google's data warehouse. -3. Managed destinations: Start with BigQuery while more destinations are evaluated. -4. Complete change history: Captures INSERT, UPDATE, DELETE, and TRUNCATE operations. -5. Optimized for analytics: Faster queries and lower storage costs through compression. -6. Production isolation: Complete separation of analytics and production workloads. +1. Initial sync: Copy existing rows from published tables. +2. Ongoing replication: Apply subsequent INSERT, UPDATE, DELETE, and TRUNCATE operations selected by the publication. +3. Managed operation: Monitor pipeline status, lag, table state, and errors in the Dashboard. +4. Workload isolation: Keep analytical queries away from the primary database. -## How it works -Pipelines uses Postgres logical replication to capture changes. For BigQuery, a view is created for each table backed by versioned tables. +## Destinations +BigQuery is currently available. [Request early access](/go/supabase-pipelines-new-destinations) to ClickHouse, Snowflake, and DuckLake while destination support expands. + +## Setup +Create a Postgres publication for the tables to replicate. In Database > Replication, add a Pipelines destination, configure its settings, and monitor the pipeline from the Dashboard. + +## Requirements +Requirements depend on the destination. BigQuery requires source tables to have primary keys and requires the publication to include those columns. ## Pipelines is valuable for: -- Data warehousing and business intelligence -- Historical analysis and audit trails -- Large-scale analytics requiring separation from production -- Compliance scenarios requiring complete data history +- Near real-time analytics data movement +- Analytics separation from production +- Managed replication to supported destination systems ## Limitations -Tables require primary keys. DDL support (schema changes) is currently in development. +Schema change support is currently in beta and limited to supported BigQuery changes. Destination-specific constraints apply. -Pipelines provides a powerful alternative to Read Replicas for analytics workloads, optimizing performance while reducing costs.`, +Pipelines keeps the current destination table state synchronized. It does not automatically create a queryable history of every row version.`, icon: CloudCog, products: [PRODUCT_SHORTNAMES.DATABASE], heroImage: '', @@ -2832,48 +2835,6 @@ OrioleDB is a PostgreSQL storage extension built on its pluggable storage framew availableOnSelfHosted: true, }, }, - { - title: 'Replication', - subtitle: 'Replicate database changes to external destinations.', - description: `Supabase Pipelines uses Postgres logical replication to replicate database changes to external destinations like BigQuery. Changes are captured from the Write Ahead Log and delivered in near real time to analytical systems. - -## Key benefits -1. Near real-time sync: Changes replicated as they occur using WAL reading. -2. BigQuery integration: Direct replication to Google's data warehouse. -3. Managed pipeline: Monitor status, lag, and errors in dashboard. -4. Complete change capture: INSERT, UPDATE, DELETE, and TRUNCATE operations. -5. Production isolation: Keep analytical workloads away from your primary database. - -## Destinations -BigQuery creates views backed by versioned tables for efficient querying. Additional managed destinations are being evaluated as Pipelines develops. - -## Setup -Create Postgres publication for tables to replicate. Add destination in Replication section of dashboard. Configure destination-specific settings. Monitor pipeline in dashboard. - -## Requirements -Tables must have primary keys. Logical replication must be enabled. - -## Replication is valuable for: -- Real-time data warehousing -- Analytics separation from production -- Historical data archival -- Multi-destination data sync -- Compliance and audit trails - -## Limitations -No DDL support yet (ALTER TABLE, ADD COLUMN). Destination-specific constraints may apply. - -Pipelines provides the real-time data pipeline required for modern analytics architectures.`, - icon: DatabaseZap, - products: [PRODUCT_SHORTNAMES.DATABASE], - heroImage: '', - docsUrl: 'https://supabase.com/docs/guides/database/replication/pipelines', - slug: 'replication', - status: { - stage: PRODUCT_STAGES.PRIVATE_ALPHA, - availableOnSelfHosted: false, - }, - }, { title: 'Queues', subtitle: 'Durable messages with guaranteed delivery.', diff --git a/packages/shared-data/pricing.ts b/packages/shared-data/pricing.ts index ea94e7380e8d6..e4ddb8c6b12f5 100644 --- a/packages/shared-data/pricing.ts +++ b/packages/shared-data/pricing.ts @@ -201,14 +201,14 @@ export const pricing: Pricing = { plans: { free: false, pro: [ - '$39 per pipeline per month', - '$3.00 per GB replicated data', - '$0.60 per GB backfill data', + '$0.053 per pipeline per hour', + '$3.00 per GB ongoing replication data', + '$0.60 per GB initial sync data', ], team: [ - '$39 per pipeline per month', - '$3.00 per GB replicated data', - '$0.60 per GB backfill data', + '$0.053 per pipeline per hour', + '$3.00 per GB ongoing replication data', + '$0.60 per GB initial sync data', ], enterprise: 'Custom', }, diff --git a/packages/ui-patterns/src/FilterBar/CommandListItem.tsx b/packages/ui-patterns/src/FilterBar/CommandListItem.tsx index ec450b2f84e50..774d92e5ae49a 100644 --- a/packages/ui-patterns/src/FilterBar/CommandListItem.tsx +++ b/packages/ui-patterns/src/FilterBar/CommandListItem.tsx @@ -26,7 +26,7 @@ export function CommandListItem({ onClick={() => onSelect(item)} className={cn( 'relative flex items-center justify-between gap-2 px-2 h-[28px] text-xs cursor-pointer select-none outline-hidden text-foreground', - isHighlighted && 'bg-surface-300', + isHighlighted && 'bg-overlay-hover', !isHighlighted && 'hover:bg-surface-200' )} data-testid={`filter-menu-item-${item.value}`} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 41aa3f38b54a0..d816ed3c30eb0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1107,6 +1107,9 @@ importers: lodash: specifier: ^4.18.1 version: 4.18.1 + lodash-es: + specifier: ^4.18.1 + version: 4.18.1 lucide-react: specifier: ^0.436.0 version: 0.436.0(react@19.2.6)