From 8962308215a0b05f35ffa36002478931c5acf98e Mon Sep 17 00:00:00 2001 From: Joshen Lim Date: Mon, 20 Jul 2026 11:02:15 +0800 Subject: [PATCH 1/2] Add support for multiple custom auth providers in custom-content (#48030) ## Context Adds support for multiple custom auth providers in custom-content ## Summary by CodeRabbit * **New Features** * Added support for multiple custom sign-in providers via a new plural configuration. * Updated the sign-in page to render all configured custom provider options while maintaining compatibility with the legacy single-provider setting. * Improved the custom provider button display to remove internal prefixes from provider names. * **Documentation** * Updated the configuration schema, examples, and sample data to document the new multi-provider setting. * Marked the legacy single-provider configuration as deprecated in favor of the plural option. --- .../interfaces/SignIn/SignInWithCustom.tsx | 6 +++++- .../custom-content/CustomContent.types.ts | 2 ++ .../hooks/custom-content/custom-content.json | 2 ++ .../custom-content/custom-content.sample.json | 2 ++ .../custom-content/custom-content.schema.json | 10 ++++++++- .../hooks/custom-content/useCustomContent.ts | 8 ++----- apps/studio/pages/sign-in.tsx | 21 +++++++++++++------ 7 files changed, 37 insertions(+), 14 deletions(-) diff --git a/apps/studio/components/interfaces/SignIn/SignInWithCustom.tsx b/apps/studio/components/interfaces/SignIn/SignInWithCustom.tsx index 05ea9935e2d98..19feedfb1c1c2 100644 --- a/apps/studio/components/interfaces/SignIn/SignInWithCustom.tsx +++ b/apps/studio/components/interfaces/SignIn/SignInWithCustom.tsx @@ -10,6 +10,10 @@ interface SignInWithCustomProps { providerName: string } +const formatProviderName = (name: string) => { + return name.replace('custom:', '') +} + export const SignInWithCustom = ({ providerName }: SignInWithCustomProps) => { const [loading, setLoading] = useState(false) @@ -42,7 +46,7 @@ export const SignInWithCustom = ({ providerName }: SignInWithCustomProps) => { return ( ) } diff --git a/apps/studio/hooks/custom-content/CustomContent.types.ts b/apps/studio/hooks/custom-content/CustomContent.types.ts index b62e124743d80..d965508912dbc 100644 --- a/apps/studio/hooks/custom-content/CustomContent.types.ts +++ b/apps/studio/hooks/custom-content/CustomContent.types.ts @@ -5,6 +5,8 @@ export type CustomContentTypes = { dashboardAuthCustomProvider: string + dashboardAuthCustomProviders: string[] + docsRowLevelSecurityGuidePath: string organizationLegalDocuments: { diff --git a/apps/studio/hooks/custom-content/custom-content.json b/apps/studio/hooks/custom-content/custom-content.json index 717ecb5852b0f..a3ffff7d1d0d4 100644 --- a/apps/studio/hooks/custom-content/custom-content.json +++ b/apps/studio/hooks/custom-content/custom-content.json @@ -5,6 +5,8 @@ "dashboard_auth:custom_provider": null, + "dashboard_auth:custom_providers": null, + "docs:row_level_security_guide_path": "/guides/auth/row-level-security", "organization:legal_documents": null, diff --git a/apps/studio/hooks/custom-content/custom-content.sample.json b/apps/studio/hooks/custom-content/custom-content.sample.json index 6cc90b3f621cc..23ac2df49b501 100644 --- a/apps/studio/hooks/custom-content/custom-content.sample.json +++ b/apps/studio/hooks/custom-content/custom-content.sample.json @@ -5,6 +5,8 @@ "dashboard_auth:custom_provider": "Nimbus", + "dashboard_auth:custom_providers": ["Nimbus"], + "docs:row_level_security_guide_path": "/guides/database/postgres/row-level-security", "organization:legal_documents": [ diff --git a/apps/studio/hooks/custom-content/custom-content.schema.json b/apps/studio/hooks/custom-content/custom-content.schema.json index f6f80dd2722a6..3fed50e746c86 100644 --- a/apps/studio/hooks/custom-content/custom-content.schema.json +++ b/apps/studio/hooks/custom-content/custom-content.schema.json @@ -13,7 +13,15 @@ "dashboard_auth:custom_provider": { "type": ["string", "null"], - "description": "Show a custom provider on the sign in page (Continue with X)" + "description": "Show a custom provider on the sign in page (Continue with X). Will be deprecated by dashboard_auth:custom_providers." + }, + + "dashboard_auth:custom_providers": { + "type": ["array", "null"], + "description": "Show a custom provider on the sign in page (Continue with X)", + "items": { + "type": "string" + } }, "docs:row_level_security_guide_path": { diff --git a/apps/studio/hooks/custom-content/useCustomContent.ts b/apps/studio/hooks/custom-content/useCustomContent.ts index f125025178ed2..bbf69aec53e0f 100644 --- a/apps/studio/hooks/custom-content/useCustomContent.ts +++ b/apps/studio/hooks/custom-content/useCustomContent.ts @@ -27,16 +27,12 @@ function contentToCamelCase(feature: CustomContent) { export const useCustomContent = ( contents: T ): { - [key in CustomContentToCamelCase]: - | CustomContentTypes[CustomContentToCamelCase] - | null + [key in CustomContentToCamelCase]: CustomContentTypes[key] | null } => { // [Joshen] Running into some TS errors without the `as` here - must be overlooking something super simple return Object.fromEntries( contents.map((content) => [contentToCamelCase(content), customContentStaticObj[content]]) ) as { - [key in CustomContentToCamelCase]: - | CustomContentTypes[CustomContentToCamelCase] - | null + [key in CustomContentToCamelCase]: CustomContentTypes[key] | null } } diff --git a/apps/studio/pages/sign-in.tsx b/apps/studio/pages/sign-in.tsx index aeffd266dc3e0..9cf0b1c6ea513 100644 --- a/apps/studio/pages/sign-in.tsx +++ b/apps/studio/pages/sign-in.tsx @@ -32,9 +32,14 @@ const SignInPage: NextPageWithLayout = () => { 'dashboard_auth:sign_up', ]) - const { dashboardAuthCustomProvider: customProvider } = useCustomContent([ - 'dashboard_auth:custom_provider', - ]) + const { + dashboardAuthCustomProvider: customProvider, + dashboardAuthCustomProviders: customProvidersNew, + } = useCustomContent(['dashboard_auth:custom_provider', 'dashboard_auth:custom_providers']) + + // [Joshen] This is just for backward compatibility - singular customProvider needs to be deprecated subsequently + // Just need to remove customProvider and rename customProvidersNew to customProviders + const customProviders = customProvidersNew ?? (customProvider ? [customProvider] : []) const { focusProvider } = useInboundBranding('sign-in') const signInProviders = useEnabledIdentityProviders().filter((provider) => provider.showOnSignIn) @@ -44,11 +49,15 @@ const SignInPage: NextPageWithLayout = () => { dividerBgClass = 'bg-studio' ) => { const showOrDivider = - (providers.length > 0 || signInWithSsoEnabled || !!customProvider) && signInWithEmailEnabled + (providers.length > 0 || signInWithSsoEnabled || customProviders.length > 0) && + signInWithEmailEnabled return ( <> - {customProvider && } + {Array.isArray(customProviders) && + customProviders.map((providerName: string) => ( + + ))} {providers.map((provider) => ( ))} @@ -90,7 +99,7 @@ const SignInPage: NextPageWithLayout = () => { const hasOtherOptions = otherProviders.length > 0 || signInWithSsoEnabled || - !!customProvider || + customProviders.length > 0 || signInWithEmailEnabled return ( From fa1e4c4bbfa1e5304b9f099ccdc8143747494ede Mon Sep 17 00:00:00 2001 From: Jordan McQueen Date: Mon, 20 Jul 2026 13:09:13 +0900 Subject: [PATCH 2/2] feat(studio): add ClickHouse replication destination (#46870) Adds ClickHouse as a replication destination type in Studio. - New ClickHouse option in the destination type selector, gated behind the `etlEnableClickHousePrivateAlpha` organization feature flag (off by default). - ClickHouse settings form: URL, user, password (optional), database, and - Client-side URL validation requires HTTPS and rejects URLs targeting internal addresses (loopback, RFC 1918, link-local, CGNAT, IPv6 loopback/link-local/ULA, and IPv4-mapped/NAT64 forms). Server-side validation remains authoritative. ## Summary by CodeRabbit ## New Features - Added **ClickHouse** as a replication destination option (private alpha), including support in destination selection/panel, replication diagram rendering, and destination icons. - Introduced a ClickHouse destination form with fields for URL, user, optional password (masked toggle), database, and engine selection. - Added ClickHouse destination config handling for create/update flows, with normalization and engine support. ## Tests - Expanded unit tests to cover ClickHouse validation and destination config building/normalization, including HTTPS-only and blocking localhost/internal targets. --------- Co-authored-by: Joshen Lim --- .../AnalyticsBucket/AnalyticsBucket.schema.ts | 12 ++ .../BigQuery/BigQuery.schema.ts | 17 ++ .../ClickHouse/ClickHouse.schema.ts | 9 + .../ClickHouse/ClickHouse.utils.ts | 102 +++++++++++ .../DestinationForm/ClickHouse/Fields.tsx | 150 ++++++++++++++++ .../DestinationForm/DestinationForm.schema.ts | 71 ++------ .../DestinationForm.utils.test.ts | 169 ++++++++++++++++++ .../DestinationForm/DestinationForm.utils.ts | 30 ++++ .../DuckLake/DuckLake.schema.ts | 27 +++ .../DestinationForm/NewPublicationPanel.tsx | 2 +- .../DestinationForm/Snowflake/Fields.tsx | 12 +- .../Snowflake/Snowflake.schema.ts | 11 ++ .../DestinationForm/index.tsx | 30 +++- .../DestinationPanel/DestinationPanel.tsx | 1 + .../DestinationPanel.types.ts | 1 + .../DestinationTypeSelection.test.tsx | 8 + .../DestinationTypeSelection.tsx | 13 +- .../Database/Replication/DestinationRow.tsx | 4 +- .../Database/Replication/Destinations.tsx | 7 +- .../Replication/ReplicationDiagram/Nodes.tsx | 3 +- .../ReplicationDiagram/Nodes.utils.test.ts | 4 + .../ReplicationDiagram/Nodes.utils.ts | 8 +- .../Replication/useIsETLPrivateAlpha.ts | 8 +- .../create-destination-pipeline-mutation.ts | 25 ++- .../update-destination-pipeline-mutation.ts | 15 +- .../validate-destination-mutation.ts | 16 +- packages/icons/__registry__/index.tsx | 10 ++ packages/icons/src/icons/click-house.ts | 30 ++++ packages/icons/src/icons/index.ts | 1 + packages/icons/src/raw-icons/click-house.svg | 3 + 30 files changed, 724 insertions(+), 75 deletions(-) create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AnalyticsBucket/AnalyticsBucket.schema.ts create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/BigQuery.schema.ts create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ClickHouse/ClickHouse.schema.ts create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ClickHouse/ClickHouse.utils.ts create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ClickHouse/Fields.tsx create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DuckLake/DuckLake.schema.ts create mode 100644 apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/Snowflake/Snowflake.schema.ts create mode 100644 packages/icons/src/icons/click-house.ts create mode 100644 packages/icons/src/raw-icons/click-house.svg diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AnalyticsBucket/AnalyticsBucket.schema.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AnalyticsBucket/AnalyticsBucket.schema.ts new file mode 100644 index 0000000000000..abef0c9138fa9 --- /dev/null +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AnalyticsBucket/AnalyticsBucket.schema.ts @@ -0,0 +1,12 @@ +import * as z from 'zod' + +// Only warehouse name and namespace are visible + editable fields +export const AnalyticsBucketFormSchema = z.object({ + warehouseName: z.string().optional(), + namespace: z.string().optional(), + newNamespaceName: z.string().optional(), + catalogToken: z.string().optional(), + s3AccessKeyId: z.string().optional(), + s3SecretAccessKey: z.string().optional(), + s3Region: z.string().optional(), +}) diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/BigQuery.schema.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/BigQuery.schema.ts new file mode 100644 index 0000000000000..abd86b79faf9d --- /dev/null +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/BigQuery.schema.ts @@ -0,0 +1,17 @@ +import * as z from 'zod' + +export const BigQueryFormSchema = z.object({ + projectId: z.string().optional(), + datasetId: z.string().optional(), + serviceAccountKey: z.string().optional(), + connectionPoolSize: z + .number() + .int() + .min(1, 'Connection pool size must be greater than 0') + .optional(), + maxStalenessMins: z + .number() + .int('Maximum staleness must be a whole number of minutes') + .min(0, 'Maximum staleness must be 0 or greater') + .optional(), +}) diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ClickHouse/ClickHouse.schema.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ClickHouse/ClickHouse.schema.ts new file mode 100644 index 0000000000000..880e2b97789da --- /dev/null +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ClickHouse/ClickHouse.schema.ts @@ -0,0 +1,9 @@ +import * as z from 'zod' + +export const ClickHouseFormSchema = z.object({ + clickhouseUrl: z.string().optional(), + clickhouseUser: z.string().optional(), + clickhousePassword: z.string().optional(), + clickhouseDatabase: z.string().optional(), + clickhouseEngine: z.enum(['merge_tree', 'replacing_merge_tree']).optional(), +}) diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ClickHouse/ClickHouse.utils.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ClickHouse/ClickHouse.utils.ts new file mode 100644 index 0000000000000..e36d47f4c1860 --- /dev/null +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ClickHouse/ClickHouse.utils.ts @@ -0,0 +1,102 @@ +import { type DestinationPanelSchemaType } from '../DestinationForm.schema' + +export type ClickHouseApiConfig = { + url: string + user: string + password?: string + database: string + engine?: 'merge_tree' | 'replacing_merge_tree' +} + +type ClickHouseFieldPath = 'clickhouseUrl' | 'clickhouseUser' | 'clickhouseDatabase' + +export type ClickHouseValidationIssue = { + path: ClickHouseFieldPath + message: string +} + +/** + * Client-side check that the URL does not target an obviously internal address. + * This is a UX-level guard to surface mistakes before the validate round-trip; + * server-side validation remains authoritative. + */ +const isClickHouseHostInternal = (host: string): boolean => { + if (host === '' || host === 'localhost' || host.endsWith('.localhost')) return true + + const ipv4 = host.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/) + + if (ipv4) { + const a = Number(ipv4[1]) + const b = Number(ipv4[2]) + + return ( + a === 10 || + a === 127 || + a === 0 || + (a === 100 && b >= 64 && b <= 127) || // CGNAT (RFC 6598) + (a === 169 && b === 254) || + (a === 172 && b >= 16 && b <= 31) || + (a === 192 && b === 168) || + (a === 198 && (b === 18 || b === 19)) || // benchmarking (RFC 2544) + a >= 224 + ) + } + + if (host.includes(':')) { + const lower = host.toLowerCase() + + return ( + lower === '::1' || + lower === '::' || + /^fe[89ab][0-9a-f]?:/.test(lower) || + /^f[cd][0-9a-f]{2}:/.test(lower) || + lower.startsWith('::ffff:') || + lower.startsWith('64:ff9b:') // NAT64 (RFC 6052 well-known + RFC 8215 local-use) + ) + } + + return false +} + +export const getClickHouseValidationIssues = ( + data: Pick +): ClickHouseValidationIssue[] => { + const issues: ClickHouseValidationIssue[] = [] + + if (!data.clickhouseUrl?.length) { + issues.push({ path: 'clickhouseUrl', message: 'URL is required' }) + } else { + let parsed: URL | undefined + + try { + parsed = new URL(data.clickhouseUrl) + } catch { + issues.push({ path: 'clickhouseUrl', message: 'ClickHouse URL must be a valid URL' }) + } + + if (parsed) { + if (parsed.protocol !== 'https:') { + issues.push({ path: 'clickhouseUrl', message: 'ClickHouse URL must use https://' }) + } else { + const host = parsed.hostname.replace(/^\[|\]$/g, '') + + if (isClickHouseHostInternal(host)) { + issues.push({ + path: 'clickhouseUrl', + message: 'ClickHouse URL must not target an internal address', + }) + } + } + } + } + + if (!data.clickhouseUser?.length) { + issues.push({ path: 'clickhouseUser', message: 'User is required' }) + } + + if (!data.clickhouseDatabase?.length) { + issues.push({ path: 'clickhouseDatabase', message: 'Database is required' }) + } + + return issues +} diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ClickHouse/Fields.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ClickHouse/Fields.tsx new file mode 100644 index 0000000000000..c1b1097736167 --- /dev/null +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ClickHouse/Fields.tsx @@ -0,0 +1,150 @@ +import { Eye, EyeOff } from 'lucide-react' +import { useState } from 'react' +import type { UseFormReturn } from 'react-hook-form' +import { + Button, + FormControl, + FormField, + Input, + Select, + SelectContent, + SelectItem, + SelectTrigger, +} from 'ui' +import { Input as PasswordInput } from 'ui-patterns/DataInputs/Input' +import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout' + +import { STORED_SECRET_PLACEHOLDER } from '../DestinationForm.constants' +import type { DestinationPanelSchemaType } from '../DestinationForm.schema' + +export const ClickHouseFields = ({ + form, + hasStoredPassword, +}: { + form: UseFormReturn + hasStoredPassword: boolean +}) => { + const [showPassword, setShowPassword] = useState(false) + + return ( +
+

ClickHouse settings

+ +
+ ( + + + + + + )} + /> + + ( + + + + + + )} + /> + + ( + + + field.onChange(event.target.value)} + actions={ +
+
+ } + /> + + + )} + /> + + ( + + + + + + )} + /> + + ( + + + + + + )} + /> +
+
+ ) +} diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts index b1d1577fc02e9..6832876589edd 100644 --- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts @@ -1,9 +1,12 @@ import * as z from 'zod' -// [Joshen] JFYI if we plan to add another type here, I reckon we split this out into smaller components -// then as FormSchema is getting quite complex with some fields that aren't necessary if the type is one or the other -export const DestinationPanelFormSchema = z.object({ - // Common fields +import { AnalyticsBucketFormSchema } from './AnalyticsBucket/AnalyticsBucket.schema' +import { BigQueryFormSchema } from './BigQuery/BigQuery.schema' +import { ClickHouseFormSchema } from './ClickHouse/ClickHouse.schema' +import { DuckLakeFormSchema } from './DuckLake/DuckLake.schema' +import { SnowflakeFormSchema } from './Snowflake/Snowflake.schema' + +const CommonFormSchema = z.object({ name: z.string().min(1, 'Name is required'), publicationName: z.string().min(1, 'Publication is required'), maxFillMs: z @@ -22,60 +25,12 @@ export const DestinationPanelFormSchema = z.object({ .min(1, 'Max copy connections per table must be greater than 0') .optional(), invalidatedSlotBehavior: z.enum(['error', 'recreate']).optional(), - // BigQuery fields - projectId: z.string().optional(), - datasetId: z.string().optional(), - serviceAccountKey: z.string().optional(), - connectionPoolSize: z - .number() - .int() - .min(1, 'Connection pool size must be greater than 0') - .optional(), - maxStalenessMins: z - .number() - .int('Maximum staleness must be a whole number of minutes') - .min(0, 'Maximum staleness must be 0 or greater') - .optional(), - // Analytics Bucket fields, only warehouse name and namespace are visible + editable fields - warehouseName: z.string().optional(), - namespace: z.string().optional(), - newNamespaceName: z.string().optional(), - catalogToken: z.string().optional(), - s3AccessKeyId: z.string().optional(), - s3SecretAccessKey: z.string().optional(), - s3Region: z.string().optional(), - // DuckLake fields - // `supabase` mode picks Supabase projects for catalog + storage (managed), while `custom` - // mode keeps the manual PostgreSQL catalog URL + S3-compatible credentials. - ducklakeMode: z.enum(['supabase', 'custom']).optional(), - // DuckLake "Use Supabase" fields - ducklakeCatalogProjectRef: z.string().optional(), - ducklakeStorageProjectRef: z.string().optional(), - ducklakeStorageBucket: z.string().optional(), - // DuckLake "Custom parameters" fields - ducklakeCatalogUrl: z.string().optional(), - ducklakeDataPath: z.string().optional(), - ducklakePoolSize: z - .number() - .int() - .min(1, 'Pool size must be greater than 0') - .max(6, 'Pool size must be 6 or less') - .optional(), - ducklakeS3AccessKeyId: z.string().optional(), - ducklakeS3SecretAccessKey: z.string().optional(), - ducklakeS3Region: z.string().optional(), - ducklakeS3Endpoint: z.string().optional(), - ducklakeS3UrlStyle: z.enum(['path', 'vhost']).optional(), - ducklakeS3UseSsl: z.boolean().optional(), - ducklakeMetadataSchema: z.string().optional(), - // Snowflake fields - snowflakeAccountId: z.string().optional(), - snowflakeUser: z.string().optional(), - snowflakePrivateKey: z.string().optional(), - snowflakePrivateKeyPassphrase: z.string().optional(), - snowflakeDatabase: z.string().optional(), - snowflakeSchema: z.string().optional(), - snowflakeRole: z.string().optional(), }) +export const DestinationPanelFormSchema = CommonFormSchema.extend(BigQueryFormSchema.shape) + .extend(AnalyticsBucketFormSchema.shape) + .extend(DuckLakeFormSchema.shape) + .extend(SnowflakeFormSchema.shape) + .extend(ClickHouseFormSchema.shape) + export type DestinationPanelSchemaType = z.infer diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.ts index a1ca8a6837111..a23088a14a363 100644 --- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.ts +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest' import { getAnalyticsBucketValidationIssues } from './AnalyticsBucket/AnalyticsBucket.utils' import { getBigQueryValidationIssues } from './BigQuery/BigQuery.utils' +import { getClickHouseValidationIssues } from './ClickHouse/ClickHouse.utils' import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationForm.constants' import { buildDestinationConfig, @@ -79,6 +80,42 @@ const baseSnowflakeFormData = { snowflakeRole: ' PIPELINES_ROLE ', } +const baseClickHouseFormData = { + name: 'ClickHouse Destination', + publicationName: 'pub', + maxFillMs: undefined, + maxTableSyncWorkers: undefined, + maxCopyConnectionsPerTable: undefined, + invalidatedSlotBehavior: undefined, + projectId: undefined, + datasetId: undefined, + serviceAccountKey: undefined, + connectionPoolSize: undefined, + maxStalenessMins: undefined, + warehouseName: undefined, + namespace: undefined, + newNamespaceName: undefined, + catalogToken: undefined, + s3AccessKeyId: undefined, + s3SecretAccessKey: undefined, + s3Region: undefined, + ducklakeCatalogUrl: undefined, + ducklakeDataPath: undefined, + ducklakePoolSize: undefined, + ducklakeS3AccessKeyId: undefined, + ducklakeS3SecretAccessKey: undefined, + ducklakeS3Region: undefined, + ducklakeS3Endpoint: undefined, + ducklakeS3UrlStyle: undefined, + ducklakeS3UseSsl: undefined, + ducklakeMetadataSchema: undefined, + clickhouseUrl: ' https://your-cluster.clickhouse.cloud:8443 ', + clickhouseUser: ' default ', + clickhousePassword: ' secret password ', + clickhouseDatabase: ' analytics ', + clickhouseEngine: 'replacing_merge_tree' as const, +} + describe('DestinationForm.utils DuckLake', () => { it('builds DuckLake validation config with required fields trimmed and blank optionals removed', () => { const config = buildDestinationConfigForValidation({ @@ -440,6 +477,138 @@ describe('DestinationForm.utils Snowflake', () => { }) }) +describe('DestinationForm.utils ClickHouse', () => { + it('builds ClickHouse validation config with required fields trimmed and blank optionals removed', () => { + const config = buildDestinationConfigForValidation({ + projectRef: 'project-ref', + selectedType: 'ClickHouse', + data: { + ...baseClickHouseFormData, + clickhousePassword: '', + }, + }) + + expect(config).toEqual({ + clickHouse: { + url: 'https://your-cluster.clickhouse.cloud:8443', + user: 'default', + password: undefined, + database: 'analytics', + engine: 'replacing_merge_tree', + }, + }) + }) + + it('builds ClickHouse submit config with normalized values', async () => { + const createS3AccessKey = vi.fn() + const resolveNamespace = vi.fn() + + const config = await buildDestinationConfig({ + projectRef: 'project-ref', + selectedType: 'ClickHouse', + data: baseClickHouseFormData, + createS3AccessKey, + resolveNamespace, + }) + + expect(config).toEqual({ + clickHouse: { + url: 'https://your-cluster.clickhouse.cloud:8443', + user: 'default', + password: ' secret password ', + database: 'analytics', + engine: 'replacing_merge_tree', + }, + }) + expect(createS3AccessKey).not.toHaveBeenCalled() + expect(resolveNamespace).not.toHaveBeenCalled() + }) + + it('returns required-field errors for missing ClickHouse settings', () => { + const issues = getClickHouseValidationIssues({ + clickhouseUrl: '', + clickhouseUser: '', + clickhouseDatabase: '', + }) + + expect(issues).toEqual([ + { path: 'clickhouseUrl', message: 'URL is required' }, + { path: 'clickhouseUser', message: 'User is required' }, + { path: 'clickhouseDatabase', message: 'Database is required' }, + ]) + }) + + it('rejects ClickHouse URLs that are not https or target internal addresses', () => { + expect( + getClickHouseValidationIssues({ + clickhouseUrl: 'http://example.clickhouse.cloud:8443', + clickhouseUser: 'default', + clickhouseDatabase: 'analytics', + }) + ).toEqual([{ path: 'clickhouseUrl', message: 'ClickHouse URL must use https://' }]) + + expect( + getClickHouseValidationIssues({ + clickhouseUrl: 'https://127.0.0.1:8443', + clickhouseUser: 'default', + clickhouseDatabase: 'analytics', + }) + ).toEqual([ + { path: 'clickhouseUrl', message: 'ClickHouse URL must not target an internal address' }, + ]) + }) + + it.each([ + ['loopback hostname', 'https://localhost:8443'], + ['loopback subdomain', 'https://foo.localhost:8443'], + ['IPv4 loopback', 'https://127.0.0.1:8443'], + ['IPv4 unspecified', 'https://0.0.0.0:8443'], + ['RFC 1918 10.0.0.0/8', 'https://10.1.2.3:8443'], + ['RFC 1918 172.16.0.0/12', 'https://172.16.0.1:8443'], + ['RFC 1918 172.16.0.0/12 upper bound', 'https://172.31.255.254:8443'], + ['RFC 1918 192.168.0.0/16', 'https://192.168.1.1:8443'], + ['link-local 169.254.0.0/16', 'https://169.254.1.1:8443'], + ['CGNAT 100.64.0.0/10', 'https://100.64.0.1:8443'], + ['CGNAT 100.64.0.0/10 upper bound', 'https://100.127.255.254:8443'], + ['benchmarking 198.18.0.0/15', 'https://198.18.0.1:8443'], + ['multicast/reserved 224.0.0.0/4+', 'https://224.0.0.1:8443'], + ['broadcast', 'https://255.255.255.255:8443'], + ['IPv6 loopback', 'https://[::1]:8443'], + ['IPv6 unspecified', 'https://[::]:8443'], + ['IPv6 link-local', 'https://[fe80::1]:8443'], + ['IPv6 unique local (fc00::/7)', 'https://[fc00::1]:8443'], + ['IPv6 unique local (fd00::/8)', 'https://[fd12:3456::1]:8443'], + ['IPv4-mapped IPv6', 'https://[::ffff:127.0.0.1]:8443'], + ['NAT64', 'https://[64:ff9b::127.0.0.1]:8443'], + ['decimal-encoded IPv4 loopback', 'https://2130706433:8443'], + ['hex-encoded IPv4 loopback', 'https://0x7f000001:8443'], + ])('rejects ClickHouse URLs targeting an internal address: %s', (_label, clickhouseUrl) => { + expect( + getClickHouseValidationIssues({ + clickhouseUrl, + clickhouseUser: 'default', + clickhouseDatabase: 'analytics', + }) + ).toEqual([ + { path: 'clickhouseUrl', message: 'ClickHouse URL must not target an internal address' }, + ]) + }) + + it.each([ + ['public IPv4 address', 'https://8.8.8.8:8443'], + ['public hostname', 'https://your-cluster.clickhouse.cloud:8443'], + ['public IPv6 address', 'https://[2606:4700:4700::1111]:8443'], + ])('accepts ClickHouse URLs targeting a public address: %s', (_label, clickhouseUrl) => { + expect( + getClickHouseValidationIssues({ + clickhouseUrl, + clickhouseUser: 'default', + clickhouseDatabase: 'analytics', + }) + ).toEqual([]) + }) +}) + describe('DestinationForm.utils BigQuery', () => { it('returns required-field errors for missing BigQuery settings', () => { const issues = getBigQueryValidationIssues({ diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts index dbbd98a2c0b9f..e62f614071809 100644 --- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts @@ -3,6 +3,7 @@ import { snakeCase } from 'lodash' import z from 'zod' import { DestinationType } from '../DestinationPanel.types' +import { type ClickHouseApiConfig } from './ClickHouse/ClickHouse.utils' import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE, @@ -25,6 +26,7 @@ import { type DucklakeApiConfig } from './DuckLake/DuckLake.utils' import { type SnowflakeApiConfig } from './Snowflake/Snowflake.utils' import { BigQueryDestinationConfig, + ClickHouseDestinationConfig, DestinationConfig, DucklakeDestinationConfig, DucklakeManualDestinationConfig, @@ -88,6 +90,14 @@ export const generateDefaultValues = ({ snowflakeConfigValue && typeof snowflakeConfigValue === 'object' ? (snowflakeConfigValue as SnowflakeApiConfig) : undefined + const clickhouseConfigValue = + config && 'clickhouse' in (config as Record) + ? (config as Record).clickhouse + : undefined + const clickhouseConfig = + clickhouseConfigValue && typeof clickhouseConfigValue === 'object' + ? (clickhouseConfigValue as ClickHouseApiConfig) + : undefined return { // Common fields @@ -144,6 +154,12 @@ export const generateDefaultValues = ({ snowflakeDatabase: snowflakeConfig?.database ?? '', snowflakeSchema: snowflakeConfig?.schema ?? '', snowflakeRole: snowflakeConfig?.role ?? '', + // ClickHouse fields + clickhouseUrl: clickhouseConfig?.url ?? '', + clickhouseUser: clickhouseConfig?.user ?? '', + clickhousePassword: clickhouseConfig?.password ?? '', + clickhouseDatabase: clickhouseConfig?.database ?? '', + clickhouseEngine: clickhouseConfig?.engine ?? 'replacing_merge_tree', } } @@ -169,6 +185,16 @@ const buildSnowflakeConfig = ( role: normalizeOptionalString(data.snowflakeRole), }) +const buildClickHouseConfig = ( + data: z.infer +): ClickHouseDestinationConfig => ({ + url: normalizeRequiredString(data.clickhouseUrl), + user: normalizeRequiredString(data.clickhouseUser), + password: normalizeOptionalUntrimmedString(data.clickhousePassword), + database: normalizeRequiredString(data.clickhouseDatabase), + engine: data.clickhouseEngine, +}) + // Builds the studio-side DuckLake config from form data, picking the right shape for the // selected mode. The create / update / validate mutations convert this to the API payload. const buildDucklakeConfig = ( @@ -243,6 +269,8 @@ export const buildDestinationConfigForValidation = ({ return { ducklake: buildDucklakeConfig(data) } } else if (selectedType === 'Snowflake') { return { snowflake: buildSnowflakeConfig(data) } + } else if (selectedType === 'ClickHouse') { + return { clickHouse: buildClickHouseConfig(data) } } else { throw new Error('Invalid destination type') } @@ -302,6 +330,8 @@ export const buildDestinationConfig = async ({ destinationConfig = { ducklake: buildDucklakeConfig(data) } } else if (selectedType === 'Snowflake') { destinationConfig = { snowflake: buildSnowflakeConfig(data) } + } else if (selectedType === 'ClickHouse') { + destinationConfig = { clickHouse: buildClickHouseConfig(data) } } return destinationConfig diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DuckLake/DuckLake.schema.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DuckLake/DuckLake.schema.ts new file mode 100644 index 0000000000000..37da7aec648a4 --- /dev/null +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DuckLake/DuckLake.schema.ts @@ -0,0 +1,27 @@ +import * as z from 'zod' + +// `supabase` mode picks Supabase projects for catalog + storage (managed), while `custom` +// mode keeps the manual PostgreSQL catalog URL + S3-compatible credentials. +export const DuckLakeFormSchema = z.object({ + ducklakeMode: z.enum(['supabase', 'custom']).optional(), + // DuckLake "Use Supabase" fields + ducklakeCatalogProjectRef: z.string().optional(), + ducklakeStorageProjectRef: z.string().optional(), + ducklakeStorageBucket: z.string().optional(), + // DuckLake "Custom parameters" fields + ducklakeCatalogUrl: z.string().optional(), + ducklakeDataPath: z.string().optional(), + ducklakePoolSize: z + .number() + .int() + .min(1, 'Pool size must be greater than 0') + .max(6, 'Pool size must be 6 or less') + .optional(), + ducklakeS3AccessKeyId: z.string().optional(), + ducklakeS3SecretAccessKey: z.string().optional(), + ducklakeS3Region: z.string().optional(), + ducklakeS3Endpoint: z.string().optional(), + ducklakeS3UrlStyle: z.enum(['path', 'vhost']).optional(), + ducklakeS3UseSsl: z.boolean().optional(), + ducklakeMetadataSchema: z.string().optional(), +}) diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/NewPublicationPanel.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/NewPublicationPanel.tsx index 09b291a0e4868..8cb49e591e39e 100644 --- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/NewPublicationPanel.tsx +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/NewPublicationPanel.tsx @@ -150,7 +150,7 @@ export const NewPublicationPanel = ({ visible, sourceId, onClose }: NewPublicati - diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/Snowflake/Fields.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/Snowflake/Fields.tsx index b5eaccceef851..f77d1af47960f 100644 --- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/Snowflake/Fields.tsx +++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/Snowflake/Fields.tsx @@ -11,9 +11,11 @@ import type { DestinationPanelSchemaType } from '../DestinationForm.schema' export const SnowflakeFields = ({ form, editMode, + hasStoredPrivateKeyPassphrase, }: { form: UseFormReturn editMode: boolean + hasStoredPrivateKeyPassphrase: boolean }) => { const [showPrivateKeyPassphrase, setShowPrivateKeyPassphrase] = useState(false) @@ -157,7 +159,7 @@ export const SnowflakeFields = ({ layout="horizontal" label="Private key passphrase" description={ - editMode + hasStoredPrivateKeyPassphrase ? 'Stored passphrase setting is hidden. Enter a new passphrase to replace it.' : 'Optional passphrase for encrypted private keys' } @@ -166,13 +168,19 @@ export const SnowflakeFields = ({ field.onChange(event.target.value)} actions={