fix(tables): sniff CSV delimiter and capture the real header row on import

Table CSV import derived the delimiter purely from the file extension, so
semicolon-delimited exports (European-locale Excel) parsed as a single column.
Header derivation also read Object.keys(rows[0]), so with relax_column_count a
ragged/sparse first data row dropped its trailing columns from schema inference.

- Add detectCsvDelimiter: trial-parse the head against ,/;/tab/| and pick the
  candidate with the most columns (tie-break on row-width consistency), quote-aware
  so separators inside quoted cells don't skew the guess; extension is now only the
  fallback
- Add sniffCsvDelimiterFromStream: memory-bounded (64KB) peek-then-replay for the
  streaming import paths so multi-GB files sniff without buffering
- Capture the true header row via the csv-parse columns callback on every path,
  fixing dropped columns when the first row is short
- Wire into the create route, append/replace route, background runner, client
  dialog preview, and parseFileRows (copilot)

Author: waleedlatif1

apps/sim/app/api/table/[tableId]/import/route.ts

@@ -36,6 +36,7 @@ import {
   validateMapping,
   wouldExceedRowLimit,
 } from '@/lib/table'
+import { sniffCsvDelimiterFromStream } from '@/lib/table/csv-delimiter-stream'
 import { importAppendRows, importReplaceRows } from '@/lib/table/import-data'
 import { getUserSettings } from '@/lib/users/queries'
 import {
@@ -176,19 +177,26 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro
       timezone = timezoneValidation.data
     }
 
-    const delimiter = extensionValidation.data === 'tsv' ? '\t' : ','
-    const parser = createCsvParser(delimiter)
+    // The extension only picks the fallback — the separator is sniffed from the file's
+    // head so semicolon/pipe exports (European-locale Excel) don't land in one column.
+    const { delimiter, stream: csvStream } = await sniffCsvDelimiterFromStream(
+      file.stream,
+      extensionValidation.data === 'tsv' ? '\t' : ','
+    )
+    let headers: string[] = []
+    const parser = createCsvParser(delimiter, (parsedHeaders) => {
+      headers = parsedHeaders
+    })
     // `.pipe` doesn't forward source errors; forward them so the iterator throws.
-    file.stream.on('error', (streamErr) => parser.destroy(streamErr))
-    file.stream.pipe(parser)
+    csvStream.on('error', (streamErr) => parser.destroy(streamErr))
+    csvStream.pipe(parser)
     const rows: Record<string, unknown>[] = []
     for await (const record of parser as AsyncIterable<Record<string, unknown>>) {
       rows.push(record)
     }
     if (rows.length === 0) {
       return NextResponse.json({ error: 'CSV file has no data rows' }, { status: 400 })
     }
-    const headers = Object.keys(rows[0])
 
     let effectiveMapping = mapping ?? buildAutoMapping(headers, table.schema)
     let prospectiveTable: TableDefinition = table

apps/sim/app/api/table/import-csv/route.ts

@@ -26,6 +26,7 @@ import {
   type TableDefinition,
   type TableSchema,
 } from '@/lib/table'
+import { sniffCsvDelimiterFromStream } from '@/lib/table/csv-delimiter-stream'
 import { getUserSettings } from '@/lib/users/queries'
 import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
 import {
@@ -107,12 +108,20 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
         { status: 400 }
       )
     }
-    const delimiter = extensionResult.data === 'tsv' ? '\t' : ','
+    // The extension only picks the fallback — the separator is sniffed from the file's
+    // head so semicolon/pipe exports (European-locale Excel) don't land in one column.
+    const { delimiter, stream: csvStream } = await sniffCsvDelimiterFromStream(
+      file.stream,
+      extensionResult.data === 'tsv' ? '\t' : ','
+    )
 
-    const parser = createCsvParser(delimiter)
+    let csvHeaders: string[] = []
+    const parser = createCsvParser(delimiter, (headers) => {
+      csvHeaders = headers
+    })
     // `.pipe` doesn't forward source errors; forward them so the iterator throws.
-    file.stream.on('error', (err) => parser.destroy(err))
-    file.stream.pipe(parser)
+    csvStream.on('error', (err) => parser.destroy(err))
+    csvStream.pipe(parser)
 
     interface ImportState {
       table: TableDefinition
@@ -139,7 +148,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
 
     /** Infer the schema from the buffered sample and create the (empty) table. */
     const buildTable = async (sampleRows: Record<string, unknown>[]): Promise<ImportState> => {
-      const inferred = inferSchemaFromCsv(Object.keys(sampleRows[0]), sampleRows)
+      const inferred = inferSchemaFromCsv(csvHeaders, sampleRows)
       const schema: TableSchema = { columns: inferred.columns.map(normalizeColumn) }
       const planLimits = await getWorkspaceTableLimits(workspaceId)
       const tableName = sanitizeName(file.filename.replace(/\.[^.]+$/, ''), 'imported_table').slice(

apps/sim/app/workspace/[workspaceId]/tables/components/import-csv-dialog/import-csv-dialog.tsx

@@ -25,7 +25,13 @@ import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
 import { truncate } from '@sim/utils/string'
 import { CSV_ASYNC_IMPORT_THRESHOLD_BYTES } from '@/lib/table/constants'
-import { buildAutoMapping, parseCsvBuffer } from '@/lib/table/import'
+import {
+  buildAutoMapping,
+  CSV_DELIMITER_SNIFF_BYTES,
+  type CsvDelimiter,
+  detectCsvDelimiter,
+  parseCsvBuffer,
+} from '@/lib/table/import'
 import type { TableDefinition } from '@/lib/table/types'
 import {
   type CsvImportMode,
@@ -107,15 +113,24 @@ interface ParsedCsv {
   sampleRows: Record<string, unknown>[]
 }
 
-/** Parses the head of a CSV/TSV for the mapping + sample, dropping any truncated final line. */
-async function parseCsvPreview(file: File, delimiter: ',' | '\t') {
+/**
+ * Parses the head of a CSV/TSV for the mapping + sample, dropping any truncated final line.
+ *
+ * The separator is sniffed from the same leading bytes the server sniffs, so the mapping shown
+ * here always matches the columns the import will actually produce.
+ */
+async function parseCsvPreview(file: File, fallbackDelimiter: CsvDelimiter) {
   const sliced = file.size > CSV_PREVIEW_BYTES
   const blob = sliced ? file.slice(0, CSV_PREVIEW_BYTES) : file
   let bytes = new Uint8Array(await blob.arrayBuffer())
   if (sliced) {
     const lastNewline = bytes.lastIndexOf(0x0a)
     if (lastNewline > 0) bytes = bytes.subarray(0, lastNewline + 1)
   }
+  const delimiter = await detectCsvDelimiter(
+    bytes.subarray(0, CSV_DELIMITER_SNIFF_BYTES),
+    fallbackDelimiter
+  )
   return parseCsvBuffer(bytes, delimiter)
 }
 
@@ -180,8 +195,7 @@ export function ImportCsvDialog({
     setParsing(true)
     setParseError(null)
     try {
-      const delimiter: ',' | '\t' = ext === 'tsv' ? '\t' : ','
-      const { headers, rows } = await parseCsvPreview(file, delimiter)
+      const { headers, rows } = await parseCsvPreview(file, ext === 'tsv' ? '\t' : ',')
       const autoMapping = buildAutoMapping(headers, table.schema)
       setParsed({
         file,

apps/sim/lib/table/csv-delimiter-stream.ts

@@ -0,0 +1,72 @@
+import { Readable } from 'node:stream'
+import {
+  CSV_DELIMITER_SNIFF_BYTES,
+  type CsvDelimiter,
+  detectCsvDelimiter,
+} from '@/lib/table/import'
+
+export interface SniffedCsvStream {
+  delimiter: CsvDelimiter
+  /**
+   * The full file contents, replayed from byte zero. Use this in place of the
+   * source stream — the source has already been partially consumed.
+   */
+  stream: Readable
+}
+
+/**
+ * Reads the head of a CSV/TSV stream, sniffs its field separator, then returns a
+ * stream that replays the buffered head followed by the untouched remainder.
+ *
+ * Only {@link CSV_DELIMITER_SNIFF_BYTES} are ever held in memory, so this stays
+ * safe on multi-GB imports. The buffered head is trimmed to its last newline
+ * before sniffing so a mid-record cut can't skew the column counts.
+ */
+export async function sniffCsvDelimiterFromStream(
+  source: Readable,
+  fallback: CsvDelimiter = ','
+): Promise<SniffedCsvStream> {
+  const reader = source[Symbol.asyncIterator]()
+  const chunks: Buffer[] = []
+  let size = 0
+  let exhausted = false
+
+  while (size < CSV_DELIMITER_SNIFF_BYTES) {
+    const next = await reader.next()
+    if (next.done) {
+      exhausted = true
+      break
+    }
+    const chunk = Buffer.isBuffer(next.value) ? next.value : Buffer.from(next.value as Uint8Array)
+    chunks.push(chunk)
+    size += chunk.length
+  }
+
+  const head = Buffer.concat(chunks)
+  let sample = head
+  if (!exhausted) {
+    const lastNewline = head.lastIndexOf(0x0a)
+    if (lastNewline > 0) sample = head.subarray(0, lastNewline + 1)
+  }
+
+  const delimiter = await detectCsvDelimiter(sample, fallback)
+
+  const stream = Readable.from(
+    (async function* replay() {
+      if (head.length > 0) yield head
+      if (exhausted) return
+      while (true) {
+        const next = await reader.next()
+        if (next.done) return
+        yield next.value
+      }
+    })()
+  )
+
+  // `Readable.from` closes the generator on destroy, which returns the source
+  // iterator — but an early destroy of the wrapper before the generator is
+  // pulled would otherwise leak the source's socket.
+  stream.on('close', () => source.destroy())
+
+  return { delimiter, stream }
+}

apps/sim/lib/table/import-runner.ts

@@ -19,6 +19,7 @@ import {
 } from '@/lib/table'
 import { assertRowCapacity, notifyTableRowUsage } from '@/lib/table/billing'
 import { withGeneratedColumnIds } from '@/lib/table/column-keys'
+import { sniffCsvDelimiterFromStream } from '@/lib/table/csv-delimiter-stream'
 import { appendTableEvent } from '@/lib/table/events'
 import {
   addImportColumns,
@@ -103,6 +104,11 @@ export async function runTableImport(payload: TableImportPayload): Promise<void>
     // Stream the file rather than buffering it — a ~1M-row import must never be held in memory.
     source = await downloadFileStream({ key: fileKey, context: 'workspace' })
 
+    // The kickoff route's extension-derived delimiter is only the fallback — the separator is
+    // sniffed from the file's head so semicolon/pipe exports don't collapse into one column.
+    const sniffed = await sniffCsvDelimiterFromStream(source, delimiter)
+    const csvStream = sniffed.stream
+
     // Append must continue after the existing rows; create/replace start empty. Read once up
     // front (the import is the table's sole writer) and assign contiguous positions / threaded
     // order keys from it.
@@ -123,11 +129,14 @@ export async function runTableImport(payload: TableImportPayload): Promise<void>
       },
     })
 
-    const parser = createCsvParser(delimiter)
+    let csvHeaders: string[] = []
+    const parser = createCsvParser(sniffed.delimiter, (headers) => {
+      csvHeaders = headers
+    })
     // `.pipe` doesn't forward source errors; forward so the iterator throws.
-    source.on('error', (err) => parser.destroy(err))
+    csvStream.on('error', (err) => parser.destroy(err))
     byteCounter.on('error', (err) => parser.destroy(err))
-    source.pipe(byteCounter).pipe(parser)
+    csvStream.pipe(byteCounter).pipe(parser)
 
     let schema: TableSchema | null = null
     let headerToColumn: Map<string, string> | null = null
@@ -142,7 +151,7 @@ export async function runTableImport(payload: TableImportPayload): Promise<void>
      * map onto the existing schema, optionally auto-creating `createColumns` first.
      */
     const resolveSetup = async () => {
-      const headers = Object.keys(sample[0])
+      const headers = csvHeaders
 
       if (mode === 'create') {
         const inferred = inferSchemaFromCsv(headers, sample)