Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
import * as z from 'zod'

// Only warehouse name and namespace are visible + editable fields
export const AnalyticsBucketFormSchema = z.object({
warehouseName: z.string().optional(),
namespace: z.string().optional(),
newNamespaceName: z.string().optional(),
catalogToken: z.string().optional(),
s3AccessKeyId: z.string().optional(),
s3SecretAccessKey: z.string().optional(),
s3Region: z.string().optional(),
})
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import * as z from 'zod'

export const BigQueryFormSchema = z.object({
projectId: z.string().optional(),
datasetId: z.string().optional(),
serviceAccountKey: z.string().optional(),
connectionPoolSize: z
.number()
.int()
.min(1, 'Connection pool size must be greater than 0')
.optional(),
maxStalenessMins: z
.number()
.int('Maximum staleness must be a whole number of minutes')
.min(0, 'Maximum staleness must be 0 or greater')
.optional(),
})
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import * as z from 'zod'

export const ClickHouseFormSchema = z.object({
clickhouseUrl: z.string().optional(),
clickhouseUser: z.string().optional(),
clickhousePassword: z.string().optional(),
clickhouseDatabase: z.string().optional(),
clickhouseEngine: z.enum(['merge_tree', 'replacing_merge_tree']).optional(),
})
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
import { type DestinationPanelSchemaType } from '../DestinationForm.schema'

export type ClickHouseApiConfig = {
url: string
user: string
password?: string
database: string
engine?: 'merge_tree' | 'replacing_merge_tree'
}

type ClickHouseFieldPath = 'clickhouseUrl' | 'clickhouseUser' | 'clickhouseDatabase'

export type ClickHouseValidationIssue = {
path: ClickHouseFieldPath
message: string
}

/**
* Client-side check that the URL does not target an obviously internal address.
* This is a UX-level guard to surface mistakes before the validate round-trip;
* server-side validation remains authoritative.
*/
const isClickHouseHostInternal = (host: string): boolean => {
if (host === '' || host === 'localhost' || host.endsWith('.localhost')) return true

const ipv4 = host.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/)

if (ipv4) {
const a = Number(ipv4[1])
const b = Number(ipv4[2])

return (
a === 10 ||
a === 127 ||
a === 0 ||
(a === 100 && b >= 64 && b <= 127) || // CGNAT (RFC 6598)
(a === 169 && b === 254) ||
(a === 172 && b >= 16 && b <= 31) ||
(a === 192 && b === 168) ||
(a === 198 && (b === 18 || b === 19)) || // benchmarking (RFC 2544)
a >= 224
)
}

if (host.includes(':')) {
const lower = host.toLowerCase()

return (
lower === '::1' ||
lower === '::' ||
/^fe[89ab][0-9a-f]?:/.test(lower) ||
/^f[cd][0-9a-f]{2}:/.test(lower) ||
lower.startsWith('::ffff:') ||
lower.startsWith('64:ff9b:') // NAT64 (RFC 6052 well-known + RFC 8215 local-use)
)
}

return false
}

export const getClickHouseValidationIssues = (
data: Pick<DestinationPanelSchemaType, ClickHouseFieldPath>
): ClickHouseValidationIssue[] => {
const issues: ClickHouseValidationIssue[] = []

if (!data.clickhouseUrl?.length) {
issues.push({ path: 'clickhouseUrl', message: 'URL is required' })
} else {
let parsed: URL | undefined

try {
parsed = new URL(data.clickhouseUrl)
} catch {
issues.push({ path: 'clickhouseUrl', message: 'ClickHouse URL must be a valid URL' })
}

if (parsed) {
if (parsed.protocol !== 'https:') {
issues.push({ path: 'clickhouseUrl', message: 'ClickHouse URL must use https://' })
} else {
const host = parsed.hostname.replace(/^\[|\]$/g, '')

if (isClickHouseHostInternal(host)) {
issues.push({
path: 'clickhouseUrl',
message: 'ClickHouse URL must not target an internal address',
})
}
}
}
}

if (!data.clickhouseUser?.length) {
issues.push({ path: 'clickhouseUser', message: 'User is required' })
}

if (!data.clickhouseDatabase?.length) {
issues.push({ path: 'clickhouseDatabase', message: 'Database is required' })
}

return issues
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
import { Eye, EyeOff } from 'lucide-react'
import { useState } from 'react'
import type { UseFormReturn } from 'react-hook-form'
import {
Button,
FormControl,
FormField,
Input,
Select,
SelectContent,
SelectItem,
SelectTrigger,
} from 'ui'
import { Input as PasswordInput } from 'ui-patterns/DataInputs/Input'
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'

import { STORED_SECRET_PLACEHOLDER } from '../DestinationForm.constants'
import type { DestinationPanelSchemaType } from '../DestinationForm.schema'

export const ClickHouseFields = ({
form,
hasStoredPassword,
}: {
form: UseFormReturn<DestinationPanelSchemaType>
hasStoredPassword: boolean
}) => {
const [showPassword, setShowPassword] = useState(false)

return (
<div className="flex flex-col gap-y-6 p-5">
<p className="text-sm font-medium text-foreground">ClickHouse settings</p>

<div className="flex flex-col gap-y-4">
<FormField
control={form.control}
name="clickhouseUrl"
render={({ field }) => (
<FormItemLayout
layout="horizontal"
label="URL"
description="HTTPS endpoint for your ClickHouse server, including port"
>
<FormControl>
<Input
{...field}
value={field.value ?? ''}
placeholder="https://your-cluster.clickhouse.cloud:8443"
/>
</FormControl>
</FormItemLayout>
)}
/>

<FormField
control={form.control}
name="clickhouseUser"
render={({ field }) => (
<FormItemLayout
layout="horizontal"
label="User"
description="ClickHouse user with permission to write to the target database"
>
<FormControl>
<Input {...field} value={field.value ?? ''} placeholder="default" />
</FormControl>
</FormItemLayout>
)}
/>

<FormField
control={form.control}
name="clickhousePassword"
render={({ field }) => (
<FormItemLayout
layout="horizontal"
label="Password"
description={
hasStoredPassword
? 'Stored password is hidden. Enter a new password to replace it.'
: 'Omit for passwordless access'
}
>
<FormControl>
<PasswordInput
value={field.value ?? ''}
type={showPassword ? 'text' : 'password'}
placeholder={hasStoredPassword ? STORED_SECRET_PLACEHOLDER : 'Optional'}
onChange={(event) => field.onChange(event.target.value)}
actions={
<div className="flex items-center justify-center">
<Button
variant="default"
className="w-7"
title={showPassword ? 'Hide password' : 'Show password'}
aria-label={showPassword ? 'Hide password' : 'Show password'}
icon={showPassword ? <Eye /> : <EyeOff />}
onClick={() => setShowPassword(!showPassword)}
/>
</div>
}
/>
</FormControl>
</FormItemLayout>
)}
/>

<FormField
control={form.control}
name="clickhouseDatabase"
render={({ field }) => (
<FormItemLayout
layout="horizontal"
label="Database"
description="The ClickHouse database where replicated tables will be created"
>
<FormControl>
<Input {...field} value={field.value ?? ''} placeholder="default" />
</FormControl>
</FormItemLayout>
)}
/>

<FormField
control={form.control}
name="clickhouseEngine"
render={({ field }) => (
<FormItemLayout
layout="horizontal"
label="Table engine"
description="Server defaults to replacing_merge_tree when unset"
>
<FormControl>
<Select
value={field.value ?? 'replacing_merge_tree'}
onValueChange={field.onChange}
>
<SelectTrigger>{field.value ?? 'replacing_merge_tree'}</SelectTrigger>
<SelectContent>
<SelectItem value="replacing_merge_tree">replacing_merge_tree</SelectItem>
<SelectItem value="merge_tree">merge_tree</SelectItem>
</SelectContent>
</Select>
</FormControl>
</FormItemLayout>
)}
/>
</div>
</div>
)
}
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
import * as z from 'zod'

// [Joshen] JFYI if we plan to add another type here, I reckon we split this out into smaller components
// then as FormSchema is getting quite complex with some fields that aren't necessary if the type is one or the other
export const DestinationPanelFormSchema = z.object({
// Common fields
import { AnalyticsBucketFormSchema } from './AnalyticsBucket/AnalyticsBucket.schema'
import { BigQueryFormSchema } from './BigQuery/BigQuery.schema'
import { ClickHouseFormSchema } from './ClickHouse/ClickHouse.schema'
import { DuckLakeFormSchema } from './DuckLake/DuckLake.schema'
import { SnowflakeFormSchema } from './Snowflake/Snowflake.schema'

const CommonFormSchema = z.object({
name: z.string().min(1, 'Name is required'),
publicationName: z.string().min(1, 'Publication is required'),
maxFillMs: z
Expand All @@ -22,60 +25,12 @@ export const DestinationPanelFormSchema = z.object({
.min(1, 'Max copy connections per table must be greater than 0')
.optional(),
invalidatedSlotBehavior: z.enum(['error', 'recreate']).optional(),
// BigQuery fields
projectId: z.string().optional(),
datasetId: z.string().optional(),
serviceAccountKey: z.string().optional(),
connectionPoolSize: z
.number()
.int()
.min(1, 'Connection pool size must be greater than 0')
.optional(),
maxStalenessMins: z
.number()
.int('Maximum staleness must be a whole number of minutes')
.min(0, 'Maximum staleness must be 0 or greater')
.optional(),
// Analytics Bucket fields, only warehouse name and namespace are visible + editable fields
warehouseName: z.string().optional(),
namespace: z.string().optional(),
newNamespaceName: z.string().optional(),
catalogToken: z.string().optional(),
s3AccessKeyId: z.string().optional(),
s3SecretAccessKey: z.string().optional(),
s3Region: z.string().optional(),
// DuckLake fields
// `supabase` mode picks Supabase projects for catalog + storage (managed), while `custom`
// mode keeps the manual PostgreSQL catalog URL + S3-compatible credentials.
ducklakeMode: z.enum(['supabase', 'custom']).optional(),
// DuckLake "Use Supabase" fields
ducklakeCatalogProjectRef: z.string().optional(),
ducklakeStorageProjectRef: z.string().optional(),
ducklakeStorageBucket: z.string().optional(),
// DuckLake "Custom parameters" fields
ducklakeCatalogUrl: z.string().optional(),
ducklakeDataPath: z.string().optional(),
ducklakePoolSize: z
.number()
.int()
.min(1, 'Pool size must be greater than 0')
.max(6, 'Pool size must be 6 or less')
.optional(),
ducklakeS3AccessKeyId: z.string().optional(),
ducklakeS3SecretAccessKey: z.string().optional(),
ducklakeS3Region: z.string().optional(),
ducklakeS3Endpoint: z.string().optional(),
ducklakeS3UrlStyle: z.enum(['path', 'vhost']).optional(),
ducklakeS3UseSsl: z.boolean().optional(),
ducklakeMetadataSchema: z.string().optional(),
// Snowflake fields
snowflakeAccountId: z.string().optional(),
snowflakeUser: z.string().optional(),
snowflakePrivateKey: z.string().optional(),
snowflakePrivateKeyPassphrase: z.string().optional(),
snowflakeDatabase: z.string().optional(),
snowflakeSchema: z.string().optional(),
snowflakeRole: z.string().optional(),
})

export const DestinationPanelFormSchema = CommonFormSchema.extend(BigQueryFormSchema.shape)
.extend(AnalyticsBucketFormSchema.shape)
.extend(DuckLakeFormSchema.shape)
.extend(SnowflakeFormSchema.shape)
.extend(ClickHouseFormSchema.shape)

export type DestinationPanelSchemaType = z.infer<typeof DestinationPanelFormSchema>
Loading
Loading