From 52c94cd0aba4195f28c42e8938b1cc5890d85297 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Thu, 16 Apr 2026 12:08:04 -0700 Subject: [PATCH 1/4] feat(tables): import csv into existing tables --- .../table/[tableId]/import-csv/route.test.ts | 295 ++++++++++ .../api/table/[tableId]/import-csv/route.ts | 267 +++++++++ apps/sim/app/api/table/import-csv/route.ts | 163 +----- .../[tableId]/components/table/table.tsx | 34 +- .../import-csv-dialog/import-csv-dialog.tsx | 513 ++++++++++++++++++ .../components/import-csv-dialog/index.ts | 1 + .../[workspaceId]/tables/components/index.ts | 1 + .../table-context-menu/table-context-menu.tsx | 15 +- .../workspace/[workspaceId]/tables/tables.tsx | 20 +- apps/sim/hooks/queries/tables.ts | 71 +++ .../lib/copilot/generated/tool-catalog-v1.ts | 11 + .../lib/copilot/generated/tool-schemas-v1.ts | 185 ++++--- .../lib/copilot/request/tools/files.test.ts | 99 ++++ apps/sim/lib/copilot/request/tools/files.ts | 45 +- .../tools/server/table/user-table.test.ts | 220 ++++++++ .../copilot/tools/server/table/user-table.ts | 271 ++++----- apps/sim/lib/table/csv-import.test.ts | 277 ++++++++++ apps/sim/lib/table/csv-import.ts | 391 +++++++++++++ apps/sim/lib/table/index.ts | 1 + apps/sim/lib/table/service.ts | 116 ++++ apps/sim/lib/table/types.ts | 12 + 21 files changed, 2600 insertions(+), 408 deletions(-) create mode 100644 apps/sim/app/api/table/[tableId]/import-csv/route.test.ts create mode 100644 apps/sim/app/api/table/[tableId]/import-csv/route.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/tables/components/import-csv-dialog/import-csv-dialog.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/tables/components/import-csv-dialog/index.ts create mode 100644 apps/sim/lib/copilot/request/tools/files.test.ts create mode 100644 apps/sim/lib/copilot/tools/server/table/user-table.test.ts create mode 100644 apps/sim/lib/table/csv-import.test.ts create mode 100644 apps/sim/lib/table/csv-import.ts diff --git a/apps/sim/app/api/table/[tableId]/import-csv/route.test.ts b/apps/sim/app/api/table/[tableId]/import-csv/route.test.ts new file mode 100644 index 00000000000..9e2f31edc9a --- /dev/null +++ b/apps/sim/app/api/table/[tableId]/import-csv/route.test.ts @@ -0,0 +1,295 @@ +/** + * @vitest-environment node + */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { TableDefinition } from '@/lib/table' + +const { + mockCheckSessionOrInternalAuth, + mockCheckAccess, + mockBatchInsertRows, + mockReplaceTableRows, +} = vi.hoisted(() => ({ + mockCheckSessionOrInternalAuth: vi.fn(), + mockCheckAccess: vi.fn(), + mockBatchInsertRows: vi.fn(), + mockReplaceTableRows: vi.fn(), +})) + +vi.mock('@/lib/auth/hybrid', () => ({ + AuthType: { SESSION: 'session', API_KEY: 'api_key', INTERNAL_JWT: 'internal_jwt' }, + checkSessionOrInternalAuth: mockCheckSessionOrInternalAuth, +})) + +vi.mock('@/lib/core/utils/request', () => ({ + generateRequestId: vi.fn().mockReturnValue('req-test-123'), +})) + +vi.mock('@/lib/core/utils/uuid', () => ({ + generateId: vi.fn().mockReturnValue('deadbeefcafef00d'), + generateShortId: vi.fn().mockReturnValue('short-id'), +})) + +vi.mock('@/app/api/table/utils', async () => { + const { NextResponse } = await import('next/server') + return { + checkAccess: mockCheckAccess, + accessError: (result: { status: number }) => { + const message = result.status === 404 ? 'Table not found' : 'Access denied' + return NextResponse.json({ error: message }, { status: result.status }) + }, + } +}) + +/** + * The route imports `batchInsertRows` and `replaceTableRows` from the barrel, + * which forwards them from `./service`. Mocking the service module replaces + * both without having to touch the other real helpers (`parseCsvBuffer`, + * `coerceRowsForTable`, etc.) exported through the barrel. + */ +vi.mock('@/lib/table/service', () => ({ + batchInsertRows: mockBatchInsertRows, + replaceTableRows: mockReplaceTableRows, +})) + +import { POST } from '@/app/api/table/[tableId]/import-csv/route' + +function createCsvFile(contents: string, name = 'data.csv', type = 'text/csv'): File { + return new File([contents], name, { type }) +} + +function createFormData( + file: File, + options?: { + workspaceId?: string | null + mode?: string | null + mapping?: unknown + } +): FormData { + const form = new FormData() + form.append('file', file) + if (options?.workspaceId !== null) { + form.append('workspaceId', options?.workspaceId ?? 'workspace-1') + } + if (options?.mode !== null) { + form.append('mode', options?.mode ?? 'append') + } + if (options?.mapping !== undefined) { + form.append( + 'mapping', + typeof options.mapping === 'string' ? options.mapping : JSON.stringify(options.mapping) + ) + } + return form +} + +function buildTable(overrides: Partial = {}): TableDefinition { + return { + id: 'tbl_1', + name: 'People', + description: null, + schema: { + columns: [ + { name: 'name', type: 'string', required: true }, + { name: 'age', type: 'number' }, + ], + }, + metadata: null, + rowCount: 0, + maxRows: 100, + workspaceId: 'workspace-1', + createdBy: 'user-1', + archivedAt: null, + createdAt: new Date('2024-01-01'), + updatedAt: new Date('2024-01-01'), + ...overrides, + } +} + +async function callPost(form: FormData, { tableId }: { tableId: string } = { tableId: 'tbl_1' }) { + const req = new NextRequest(`http://localhost:3000/api/table/${tableId}/import-csv`, { + method: 'POST', + body: form, + }) + return POST(req, { params: Promise.resolve({ tableId }) }) +} + +describe('POST /api/table/[tableId]/import-csv', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckSessionOrInternalAuth.mockResolvedValue({ + success: true, + userId: 'user-1', + authType: 'session', + }) + mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() }) + mockBatchInsertRows.mockImplementation(async (data: { rows: unknown[] }) => + data.rows.map((_, i) => ({ id: `row_${i}` })) + ) + mockReplaceTableRows.mockResolvedValue({ deletedCount: 0, insertedCount: 0 }) + }) + + it('returns 401 when the user is not authenticated', async () => { + mockCheckSessionOrInternalAuth.mockResolvedValueOnce({ + success: false, + error: 'Authentication required', + }) + const response = await callPost(createFormData(createCsvFile('name,age\nAlice,30'))) + expect(response.status).toBe(401) + }) + + it('returns 400 when the mode is invalid', async () => { + const response = await callPost( + createFormData(createCsvFile('name,age\nAlice,30'), { mode: 'bogus' }) + ) + expect(response.status).toBe(400) + const data = await response.json() + expect(data.error).toMatch(/Invalid mode/) + }) + + it('returns 403 when the user lacks workspace write access', async () => { + mockCheckAccess.mockResolvedValueOnce({ ok: false, status: 403 }) + const response = await callPost(createFormData(createCsvFile('name,age\nAlice,30'))) + expect(response.status).toBe(403) + }) + + it('returns 400 when the target table is archived', async () => { + mockCheckAccess.mockResolvedValueOnce({ + ok: true, + table: buildTable({ archivedAt: new Date('2024-01-02') }), + }) + const response = await callPost(createFormData(createCsvFile('name,age\nAlice,30'))) + expect(response.status).toBe(400) + const data = await response.json() + expect(data.error).toMatch(/archived/i) + }) + + it('returns 400 when the CSV is missing a required column', async () => { + const response = await callPost(createFormData(createCsvFile('age\n30'))) + expect(response.status).toBe(400) + const data = await response.json() + expect(data.error).toMatch(/missing required columns/i) + expect(data.details?.missingRequired).toEqual(['name']) + expect(mockBatchInsertRows).not.toHaveBeenCalled() + }) + + it('appends rows via batchInsertRows', async () => { + const response = await callPost( + createFormData(createCsvFile('name,age\nAlice,30\nBob,40'), { mode: 'append' }) + ) + expect(response.status).toBe(200) + const data = await response.json() + expect(data.data.mode).toBe('append') + expect(data.data.insertedCount).toBe(2) + expect(mockBatchInsertRows).toHaveBeenCalledTimes(1) + const callArgs = mockBatchInsertRows.mock.calls[0][0] as { rows: unknown[] } + expect(callArgs.rows).toEqual([ + { name: 'Alice', age: 30 }, + { name: 'Bob', age: 40 }, + ]) + expect(mockReplaceTableRows).not.toHaveBeenCalled() + }) + + it('rejects append when it would exceed maxRows', async () => { + mockCheckAccess.mockResolvedValueOnce({ + ok: true, + table: buildTable({ rowCount: 99, maxRows: 100 }), + }) + const response = await callPost( + createFormData(createCsvFile('name,age\nAlice,30\nBob,40'), { mode: 'append' }) + ) + expect(response.status).toBe(400) + const data = await response.json() + expect(data.error).toMatch(/exceed table row limit/) + expect(mockBatchInsertRows).not.toHaveBeenCalled() + }) + + it('replaces rows via replaceTableRows', async () => { + mockReplaceTableRows.mockResolvedValueOnce({ deletedCount: 5, insertedCount: 2 }) + const response = await callPost( + createFormData(createCsvFile('name,age\nAlice,30\nBob,40'), { mode: 'replace' }) + ) + expect(response.status).toBe(200) + const data = await response.json() + expect(data.data.mode).toBe('replace') + expect(data.data.deletedCount).toBe(5) + expect(data.data.insertedCount).toBe(2) + expect(mockReplaceTableRows).toHaveBeenCalledTimes(1) + expect(mockBatchInsertRows).not.toHaveBeenCalled() + }) + + it('uses an explicit mapping when provided', async () => { + const response = await callPost( + createFormData(createCsvFile('First Name,Years\nAlice,30\nBob,40', 'people.csv'), { + mode: 'append', + mapping: { 'First Name': 'name', Years: 'age' }, + }) + ) + expect(response.status).toBe(200) + const data = await response.json() + expect(data.data.mappedColumns).toEqual(['First Name', 'Years']) + const callArgs = mockBatchInsertRows.mock.calls[0][0] as { rows: unknown[] } + expect(callArgs.rows).toEqual([ + { name: 'Alice', age: 30 }, + { name: 'Bob', age: 40 }, + ]) + }) + + it('returns 400 when the mapping targets a non-existent column', async () => { + const response = await callPost( + createFormData(createCsvFile('a\nAlice'), { + mode: 'append', + mapping: { a: 'nonexistent' }, + }) + ) + expect(response.status).toBe(400) + const data = await response.json() + expect(data.error).toMatch(/do not exist on the table/) + }) + + it('returns 400 when a mapping value is not a string or null', async () => { + const response = await callPost( + createFormData(createCsvFile('name,age\nAlice,30'), { + mode: 'append', + mapping: { name: 42 }, + }) + ) + expect(response.status).toBe(400) + const data = await response.json() + expect(data.error).toMatch(/Mapping values must be/) + }) + + it('surfaces unique violations from batchInsertRows as 400', async () => { + mockBatchInsertRows.mockRejectedValueOnce( + new Error('Row 1: Column "name" must be unique. Value "Alice" already exists in row row_xxx') + ) + const response = await callPost( + createFormData(createCsvFile('name,age\nAlice,30'), { mode: 'append' }) + ) + expect(response.status).toBe(400) + const data = await response.json() + expect(data.error).toMatch(/must be unique/) + expect(data.data?.insertedCount).toBe(0) + }) + + it('accepts TSV files', async () => { + const response = await callPost( + createFormData( + createCsvFile('name\tage\nAlice\t30', 'data.tsv', 'text/tab-separated-values'), + { mode: 'append' } + ) + ) + expect(response.status).toBe(200) + expect(mockBatchInsertRows).toHaveBeenCalledTimes(1) + }) + + it('returns 400 for unsupported file extensions', async () => { + const response = await callPost( + createFormData(createCsvFile('name,age', 'data.json', 'application/json')) + ) + expect(response.status).toBe(400) + const data = await response.json() + expect(data.error).toMatch(/CSV and TSV/) + }) +}) diff --git a/apps/sim/app/api/table/[tableId]/import-csv/route.ts b/apps/sim/app/api/table/[tableId]/import-csv/route.ts new file mode 100644 index 00000000000..7d6e32e5e38 --- /dev/null +++ b/apps/sim/app/api/table/[tableId]/import-csv/route.ts @@ -0,0 +1,267 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' +import { generateRequestId } from '@/lib/core/utils/request' +import { generateId } from '@/lib/core/utils/uuid' +import { + batchInsertRows, + buildAutoMapping, + CSV_MAX_BATCH_SIZE, + CSV_MAX_FILE_SIZE_BYTES, + type CsvHeaderMapping, + CsvImportValidationError, + coerceRowsForTable, + parseCsvBuffer, + replaceTableRows, + validateMapping, +} from '@/lib/table' +import { accessError, checkAccess } from '@/app/api/table/utils' + +const logger = createLogger('TableImportCSVExisting') + +const IMPORT_MODES = new Set(['append', 'replace']) + +interface RouteParams { + params: Promise<{ tableId: string }> +} + +export async function POST(request: NextRequest, { params }: RouteParams) { + const requestId = generateRequestId() + const { tableId } = await params + + try { + const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) + if (!authResult.success || !authResult.userId) { + return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) + } + + const formData = await request.formData() + const file = formData.get('file') + const workspaceId = formData.get('workspaceId') as string | null + const rawMode = (formData.get('mode') as string | null) ?? 'append' + const rawMapping = formData.get('mapping') as string | null + + if (!file || !(file instanceof File)) { + return NextResponse.json({ error: 'CSV file is required' }, { status: 400 }) + } + + if (file.size > CSV_MAX_FILE_SIZE_BYTES) { + return NextResponse.json( + { + error: `File exceeds maximum allowed size of ${CSV_MAX_FILE_SIZE_BYTES / (1024 * 1024)} MB`, + }, + { status: 400 } + ) + } + + if (!workspaceId) { + return NextResponse.json({ error: 'Workspace ID is required' }, { status: 400 }) + } + + if (!IMPORT_MODES.has(rawMode)) { + return NextResponse.json( + { error: `Invalid mode "${rawMode}". Must be "append" or "replace".` }, + { status: 400 } + ) + } + const mode = rawMode as 'append' | 'replace' + + const ext = file.name.split('.').pop()?.toLowerCase() + if (ext !== 'csv' && ext !== 'tsv') { + return NextResponse.json({ error: 'Only CSV and TSV files are supported' }, { status: 400 }) + } + + const accessResult = await checkAccess(tableId, authResult.userId, 'write') + if (!accessResult.ok) return accessError(accessResult, requestId, tableId) + + const { table } = accessResult + + if (table.workspaceId !== workspaceId) { + logger.warn( + `[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${workspaceId}, Actual: ${table.workspaceId}` + ) + return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) + } + + if (table.archivedAt) { + return NextResponse.json({ error: 'Cannot import into an archived table' }, { status: 400 }) + } + + let mapping: CsvHeaderMapping | undefined + if (rawMapping) { + try { + const parsed = JSON.parse(rawMapping) + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { + return NextResponse.json( + { error: 'mapping must be a JSON object mapping CSV headers to column names' }, + { status: 400 } + ) + } + mapping = parsed as CsvHeaderMapping + } catch { + return NextResponse.json({ error: 'mapping must be valid JSON' }, { status: 400 }) + } + } + + const buffer = Buffer.from(await file.arrayBuffer()) + const delimiter = ext === 'tsv' ? '\t' : ',' + const { headers, rows } = await parseCsvBuffer(buffer, delimiter) + + const effectiveMapping = mapping ?? buildAutoMapping(headers, table.schema) + + let validation: ReturnType + try { + validation = validateMapping({ + csvHeaders: headers, + mapping: effectiveMapping, + tableSchema: table.schema, + }) + } catch (err) { + if (err instanceof CsvImportValidationError) { + return NextResponse.json({ error: err.message, details: err.details }, { status: 400 }) + } + throw err + } + + if (validation.mappedHeaders.length === 0) { + return NextResponse.json( + { + error: `No CSV headers map to columns on the table. CSV headers: ${headers.join(', ')}. Table columns: ${table.schema.columns.map((c) => c.name).join(', ')}`, + }, + { status: 400 } + ) + } + + const coerced = coerceRowsForTable(rows, table.schema, validation.effectiveMap) + + if (mode === 'append') { + if (table.rowCount + coerced.length > table.maxRows) { + const deficit = table.rowCount + coerced.length - table.maxRows + return NextResponse.json( + { + error: `Append would exceed table row limit (${table.maxRows}). Currently ${table.rowCount} rows, ${coerced.length} new rows, ${deficit} over.`, + }, + { status: 400 } + ) + } + + let inserted = 0 + try { + for (let i = 0; i < coerced.length; i += CSV_MAX_BATCH_SIZE) { + const batch = coerced.slice(i, i + CSV_MAX_BATCH_SIZE) + const batchRequestId = generateId().slice(0, 8) + const result = await batchInsertRows( + { + tableId: table.id, + rows: batch, + workspaceId, + userId: authResult.userId, + }, + table, + batchRequestId + ) + inserted += result.length + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + logger.warn(`[${requestId}] Append failed mid-import for table ${tableId}`, { + inserted, + total: coerced.length, + error: message, + }) + const isClientError = + message.includes('row limit') || + message.includes('Insufficient capacity') || + message.includes('Schema validation') || + message.includes('must be unique') || + message.includes('Row size exceeds') || + /^Row \d+:/.test(message) + return NextResponse.json( + { + error: isClientError ? message : 'Failed to import CSV', + data: { insertedCount: inserted }, + }, + { status: isClientError ? 400 : 500 } + ) + } + + logger.info(`[${requestId}] Append CSV imported`, { + tableId: table.id, + fileName: file.name, + mode, + inserted, + mappedColumns: validation.mappedHeaders.length, + skippedHeaders: validation.skippedHeaders.length, + }) + + return NextResponse.json({ + success: true, + data: { + tableId: table.id, + mode, + insertedCount: inserted, + mappedColumns: validation.mappedHeaders, + skippedHeaders: validation.skippedHeaders, + unmappedColumns: validation.unmappedColumns, + sourceFile: file.name, + }, + }) + } + + try { + const result = await replaceTableRows( + { tableId: table.id, rows: coerced, workspaceId, userId: authResult.userId }, + table, + requestId + ) + + logger.info(`[${requestId}] Replace CSV imported`, { + tableId: table.id, + fileName: file.name, + mode, + deleted: result.deletedCount, + inserted: result.insertedCount, + mappedColumns: validation.mappedHeaders.length, + }) + + return NextResponse.json({ + success: true, + data: { + tableId: table.id, + mode, + deletedCount: result.deletedCount, + insertedCount: result.insertedCount, + mappedColumns: validation.mappedHeaders, + skippedHeaders: validation.skippedHeaders, + unmappedColumns: validation.unmappedColumns, + sourceFile: file.name, + }, + }) + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + const isClientError = + message.includes('row limit') || + message.includes('Schema validation') || + message.includes('must be unique') || + message.includes('Row size exceeds') || + /^Row \d+:/.test(message) + if (isClientError) { + return NextResponse.json({ error: message }, { status: 400 }) + } + throw err + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + logger.error(`[${requestId}] CSV import into existing table failed:`, error) + + const isClientError = + message.includes('CSV file has no') || + message.includes('already exists') || + message.includes('Invalid column name') + + return NextResponse.json( + { error: isClientError ? message : 'Failed to import CSV' }, + { status: isClientError ? 400 : 500 } + ) + } +} diff --git a/apps/sim/app/api/table/import-csv/route.ts b/apps/sim/app/api/table/import-csv/route.ts index f10074180c2..7c8bc1389d1 100644 --- a/apps/sim/app/api/table/import-csv/route.ts +++ b/apps/sim/app/api/table/import-csv/route.ts @@ -5,157 +5,22 @@ import { generateRequestId } from '@/lib/core/utils/request' import { generateId } from '@/lib/core/utils/uuid' import { batchInsertRows, + CSV_MAX_BATCH_SIZE, + CSV_MAX_FILE_SIZE_BYTES, + coerceRowsForTable, createTable, deleteTable, getWorkspaceTableLimits, + inferSchemaFromCsv, + parseCsvBuffer, + sanitizeName, type TableSchema, } from '@/lib/table' -import type { ColumnDefinition, RowData } from '@/lib/table/types' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' import { normalizeColumn } from '@/app/api/table/utils' const logger = createLogger('TableImportCSV') -const MAX_CSV_FILE_SIZE = 50 * 1024 * 1024 -const MAX_BATCH_SIZE = 1000 -const SCHEMA_SAMPLE_SIZE = 100 - -type ColumnType = 'string' | 'number' | 'boolean' | 'date' - -async function parseCsvBuffer( - buffer: Buffer, - delimiter = ',' -): Promise<{ headers: string[]; rows: Record[] }> { - const { parse } = await import('csv-parse/sync') - const parsed = parse(buffer.toString('utf-8'), { - columns: true, - skip_empty_lines: true, - trim: true, - relax_column_count: true, - relax_quotes: true, - skip_records_with_error: true, - cast: false, - delimiter, - }) as Record[] - - if (parsed.length === 0) { - throw new Error('CSV file has no data rows') - } - - const headers = Object.keys(parsed[0]) - if (headers.length === 0) { - throw new Error('CSV file has no headers') - } - - return { headers, rows: parsed } -} - -function inferColumnType(values: unknown[]): ColumnType { - const nonEmpty = values.filter((v) => v !== null && v !== undefined && v !== '') - if (nonEmpty.length === 0) return 'string' - - const allNumber = nonEmpty.every((v) => { - const n = Number(v) - return !Number.isNaN(n) && String(v).trim() !== '' - }) - if (allNumber) return 'number' - - const allBoolean = nonEmpty.every((v) => { - const s = String(v).toLowerCase() - return s === 'true' || s === 'false' - }) - if (allBoolean) return 'boolean' - - const isoDatePattern = /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2})?)?/ - const allDate = nonEmpty.every((v) => { - const s = String(v) - return isoDatePattern.test(s) && !Number.isNaN(Date.parse(s)) - }) - if (allDate) return 'date' - - return 'string' -} - -function inferSchema(headers: string[], rows: Record[]): ColumnDefinition[] { - const sample = rows.slice(0, SCHEMA_SAMPLE_SIZE) - const seen = new Set() - - return headers.map((name) => { - let colName = sanitizeName(name) - let suffix = 2 - while (seen.has(colName.toLowerCase())) { - colName = `${sanitizeName(name)}_${suffix}` - suffix++ - } - seen.add(colName.toLowerCase()) - - return { - name: colName, - type: inferColumnType(sample.map((r) => r[name])), - } - }) -} - -/** - * Strips non-alphanumeric characters (except underscore), collapses runs of - * underscores, and ensures the name starts with a letter or underscore. - * Used for both table names and column names to satisfy NAME_PATTERN. - */ -function sanitizeName(raw: string, fallbackPrefix = 'col'): string { - let name = raw - .trim() - .replace(/[^a-zA-Z0-9_]/g, '_') - .replace(/_+/g, '_') - .replace(/^_+|_+$/g, '') - - if (!name || /^\d/.test(name)) { - name = `${fallbackPrefix}_${name}` - } - - return name -} - -function coerceValue(value: unknown, colType: ColumnType): string | number | boolean | null { - if (value === null || value === undefined || value === '') return null - switch (colType) { - case 'number': { - const n = Number(value) - return Number.isNaN(n) ? null : n - } - case 'boolean': { - const s = String(value).toLowerCase() - if (s === 'true') return true - if (s === 'false') return false - return null - } - case 'date': { - const d = new Date(String(value)) - return Number.isNaN(d.getTime()) ? String(value) : d.toISOString() - } - default: - return String(value) - } -} - -function coerceRows( - rows: Record[], - columns: ColumnDefinition[], - headerToColumn: Map -): RowData[] { - const colTypeMap = new Map(columns.map((c) => [c.name, c.type as ColumnType])) - - return rows.map((row) => { - const coerced: RowData = {} - for (const [header, value] of Object.entries(row)) { - const colName = headerToColumn.get(header) - if (colName) { - coerced[colName] = coerceValue(value, colTypeMap.get(colName) ?? 'string') - } - } - return coerced - }) -} - export async function POST(request: NextRequest) { const requestId = generateRequestId() @@ -173,9 +38,11 @@ export async function POST(request: NextRequest) { return NextResponse.json({ error: 'CSV file is required' }, { status: 400 }) } - if (file.size > MAX_CSV_FILE_SIZE) { + if (file.size > CSV_MAX_FILE_SIZE_BYTES) { return NextResponse.json( - { error: `File exceeds maximum allowed size of ${MAX_CSV_FILE_SIZE / (1024 * 1024)} MB` }, + { + error: `File exceeds maximum allowed size of ${CSV_MAX_FILE_SIZE_BYTES / (1024 * 1024)} MB`, + }, { status: 400 } ) } @@ -198,9 +65,7 @@ export async function POST(request: NextRequest) { const delimiter = ext === 'tsv' ? '\t' : ',' const { headers, rows } = await parseCsvBuffer(buffer, delimiter) - const columns = inferSchema(headers, rows) - const headerToColumn = new Map(headers.map((h, i) => [h, columns[i].name])) - + const { columns, headerToColumn } = inferSchemaFromCsv(headers, rows) const tableName = sanitizeName(file.name.replace(/\.[^.]+$/, ''), 'imported_table') const planLimits = await getWorkspaceTableLimits(workspaceId) @@ -222,10 +87,10 @@ export async function POST(request: NextRequest) { ) try { - const coerced = coerceRows(rows, columns, headerToColumn) + const coerced = coerceRowsForTable(rows, normalizedSchema, headerToColumn) let inserted = 0 - for (let i = 0; i < coerced.length; i += MAX_BATCH_SIZE) { - const batch = coerced.slice(i, i + MAX_BATCH_SIZE) + for (let i = 0; i < coerced.length; i += CSV_MAX_BATCH_SIZE) { + const batch = coerced.slice(i, i + CSV_MAX_BATCH_SIZE) const batchRequestId = generateId().slice(0, 8) const result = await batchInsertRows( { tableId: table.id, rows: batch, workspaceId, userId: authResult.userId }, diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table/table.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table/table.tsx index 558394b9ad1..c24ad9e6b87 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table/table.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table/table.tsx @@ -22,6 +22,7 @@ import { ModalFooter, ModalHeader, Skeleton, + Upload, } from '@/components/emcn' import { ArrowLeft, @@ -45,6 +46,7 @@ import type { ColumnDefinition, Filter, SortDirection, TableRow as TableRowType import type { ColumnOption, SortConfig } from '@/app/workspace/[workspaceId]/components' import { ResourceHeader, ResourceOptionsBar } from '@/app/workspace/[workspaceId]/components' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' +import { ImportCsvDialog } from '@/app/workspace/[workspaceId]/tables/components/import-csv-dialog' import { useAddTableColumn, useBatchCreateTableRows, @@ -200,6 +202,7 @@ export function Table({ const lastCheckboxRowRef = useRef(null) const [showDeleteTableConfirm, setShowDeleteTableConfirm] = useState(false) const [deletingColumn, setDeletingColumn] = useState(null) + const [isImportCsvOpen, setIsImportCsvOpen] = useState(false) const [columnWidths, setColumnWidths] = useState>({}) const columnWidthsRef = useRef(columnWidths) @@ -1555,6 +1558,21 @@ export function Table({ [handleAddColumn, addColumnMutation.isPending] ) + const headerActions = useMemo( + () => + tableData + ? [ + { + label: 'Import CSV', + icon: Upload, + onClick: () => setIsImportCsvOpen(true), + disabled: userPermissions.canEdit !== true, + }, + ] + : undefined, + [tableData, userPermissions.canEdit] + ) + const activeSortState = useMemo(() => { if (!queryOptions.sort) return null const entries = Object.entries(queryOptions.sort) @@ -1619,7 +1637,12 @@ export function Table({
{!embedded && ( <> - + )} + {tableData && ( + + )} + { diff --git a/apps/sim/app/workspace/[workspaceId]/tables/components/import-csv-dialog/import-csv-dialog.tsx b/apps/sim/app/workspace/[workspaceId]/tables/components/import-csv-dialog/import-csv-dialog.tsx new file mode 100644 index 00000000000..6760a73fd7a --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/tables/components/import-csv-dialog/import-csv-dialog.tsx @@ -0,0 +1,513 @@ +'use client' + +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { createLogger } from '@sim/logger' +import { + Button, + ButtonGroup, + ButtonGroupItem, + Combobox, + type ComboboxOption, + Label, + Modal, + ModalBody, + ModalContent, + ModalFooter, + ModalHeader, + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, + toast, +} from '@/components/emcn' +import { cn } from '@/lib/core/utils/cn' +import { buildAutoMapping, parseCsvBuffer } from '@/lib/table/csv-import' +import type { TableDefinition } from '@/lib/table/types' +import { type CsvImportMode, useImportCsvIntoTable } from '@/hooks/queries/tables' + +const logger = createLogger('ImportCsvDialog') + +const MAX_SAMPLE_ROWS = 5 +const MAX_EXAMPLES_IN_ERROR = 3 +/** + * Sentinel value for the "Do not import" option in the mapping combobox. The + * whitespace is intentional: valid column names must match `NAME_PATTERN` + * (`/^[a-z_][a-z0-9_]*$/i`), so no real column can share this value. + */ +const SKIP_VALUE = '__ skip __' + +/** + * Converts the verbose backend error messages into a short, human-friendly + * summary suitable for the modal footer. Specifically collapses repeated + * `Row N: Column "X" must be unique. Value "Y" already exists in row row_abc` + * segments into a single concise summary without internal row IDs. + */ +function summarizeImportError(message: string): string { + const uniqueMatches = [ + ...message.matchAll(/Column\s+"([^"]+)"\s+must be unique\.\s+Value\s+"([^"]+)"/g), + ] + if (uniqueMatches.length > 0) { + const column = uniqueMatches[0][1] + const values = Array.from(new Set(uniqueMatches.map((m) => m[2]))) + const preview = values + .slice(0, MAX_EXAMPLES_IN_ERROR) + .map((v) => `"${v}"`) + .join(', ') + const extra = values.length - MAX_EXAMPLES_IN_ERROR + const suffix = extra > 0 ? `, +${extra} more` : '' + return `${values.length} row${values.length === 1 ? '' : 's'} conflict on unique column "${column}" (${preview}${suffix})` + } + + const requiredMatch = message.match(/missing required columns?:\s*(.+)/i) + if (requiredMatch) { + return `Missing required column(s): ${requiredMatch[1].replace(/[.;]+$/, '')}` + } + + const rowLimitMatch = message.match(/row limit[^.;]*/i) + if (rowLimitMatch) { + return rowLimitMatch[0].trim() + } + + const stripped = message.replace(/\s+in row\s+row_[a-f0-9]+/gi, '').trim() + if (stripped.length > 180) return `${stripped.slice(0, 177)}...` + return stripped +} + +interface ImportCsvDialogProps { + open: boolean + onOpenChange: (open: boolean) => void + workspaceId: string + table: TableDefinition + onImported?: (result: { insertedCount?: number; deletedCount?: number }) => void +} + +interface ParsedCsv { + file: File + headers: string[] + sampleRows: Record[] + totalRows: number +} + +export function ImportCsvDialog({ + open, + onOpenChange, + workspaceId, + table, + onImported, +}: ImportCsvDialogProps) { + const [parsed, setParsed] = useState(null) + const [parseError, setParseError] = useState(null) + const [submitError, setSubmitError] = useState(null) + const [parsing, setParsing] = useState(false) + const [mapping, setMapping] = useState>({}) + const [mode, setMode] = useState('append') + const [isDragging, setIsDragging] = useState(false) + const fileInputRef = useRef(null) + const importMutation = useImportCsvIntoTable() + + const resetState = useCallback(() => { + setParsed(null) + setParseError(null) + setSubmitError(null) + setMapping({}) + setMode('append') + setIsDragging(false) + setParsing(false) + if (fileInputRef.current) fileInputRef.current.value = '' + }, []) + + useEffect(() => { + if (!open) resetState() + }, [open, resetState]) + + useEffect(() => { + resetState() + }, [table.id, resetState]) + + const columnOptions: ComboboxOption[] = useMemo(() => { + const options: ComboboxOption[] = [{ label: 'Do not import', value: SKIP_VALUE }] + for (const col of table.schema.columns) { + options.push({ + label: col.required ? `${col.name} (required)` : col.name, + value: col.name, + }) + } + return options + }, [table.schema.columns]) + + const handleFileSelected = useCallback( + async (file: File) => { + const ext = file.name.split('.').pop()?.toLowerCase() + if (ext !== 'csv' && ext !== 'tsv') { + setParseError('Only CSV and TSV files are supported') + return + } + setParsing(true) + setParseError(null) + try { + const arrayBuffer = await file.arrayBuffer() + const delimiter = ext === 'tsv' ? '\t' : ',' + const { headers, rows } = await parseCsvBuffer(new Uint8Array(arrayBuffer), delimiter) + const autoMapping = buildAutoMapping(headers, table.schema) + setParsed({ + file, + headers, + sampleRows: rows.slice(0, MAX_SAMPLE_ROWS), + totalRows: rows.length, + }) + setMapping(autoMapping) + } catch (err) { + const message = err instanceof Error ? err.message : 'Failed to parse CSV' + logger.error('CSV parse failed', err) + setParseError(message) + } finally { + setParsing(false) + } + }, + [table.schema] + ) + + const handleFileInputChange = useCallback( + (e: React.ChangeEvent) => { + const file = e.target.files?.[0] + if (file) void handleFileSelected(file) + }, + [handleFileSelected] + ) + + const handleDragEnter = useCallback((e: React.DragEvent) => { + e.preventDefault() + setIsDragging(true) + }, []) + + const handleDragOver = useCallback((e: React.DragEvent) => { + e.preventDefault() + }, []) + + const handleDragLeave = useCallback((e: React.DragEvent) => { + e.preventDefault() + setIsDragging(false) + }, []) + + const handleDrop = useCallback( + (e: React.DragEvent) => { + e.preventDefault() + setIsDragging(false) + const file = e.dataTransfer.files?.[0] + if (file) void handleFileSelected(file) + }, + [handleFileSelected] + ) + + const handleMappingChange = useCallback((header: string, value: string) => { + setSubmitError(null) + setMapping((prev) => ({ + ...prev, + [header]: value === SKIP_VALUE ? null : value, + })) + }, []) + + const handleModeChange = useCallback((value: string) => { + setSubmitError(null) + setMode(value as CsvImportMode) + }, []) + + const { missingRequired, duplicateTargets, mappedCount, skipCount } = useMemo(() => { + const mappedTargets = new Map() + let mapped = 0 + let skipped = 0 + for (const header of parsed?.headers ?? []) { + const target = mapping[header] + if (!target) { + skipped++ + continue + } + mapped++ + const existing = mappedTargets.get(target) ?? [] + existing.push(header) + mappedTargets.set(target, existing) + } + const dupes = [...mappedTargets.entries()] + .filter(([, headers]) => headers.length > 1) + .map(([col]) => col) + const mappedSet = new Set(mappedTargets.keys()) + const missing = table.schema.columns + .filter((c) => c.required && !mappedSet.has(c.name)) + .map((c) => c.name) + return { + missingRequired: missing, + duplicateTargets: dupes, + mappedCount: mapped, + skipCount: skipped, + } + }, [mapping, parsed?.headers, table.schema.columns]) + + const appendCapacityDeficit = useMemo(() => { + if (!parsed || mode !== 'append') return 0 + const projected = table.rowCount + parsed.totalRows + return projected > table.maxRows ? projected - table.maxRows : 0 + }, [mode, parsed, table.maxRows, table.rowCount]) + + const replaceCapacityDeficit = useMemo(() => { + if (!parsed || mode !== 'replace') return 0 + return parsed.totalRows > table.maxRows ? parsed.totalRows - table.maxRows : 0 + }, [mode, parsed, table.maxRows]) + + const canSubmit = + parsed !== null && + !importMutation.isPending && + missingRequired.length === 0 && + duplicateTargets.length === 0 && + mappedCount > 0 && + appendCapacityDeficit === 0 && + replaceCapacityDeficit === 0 + + const importCsv = importMutation.mutateAsync + const handleSubmit = useCallback(async () => { + if (!parsed || !canSubmit) return + setSubmitError(null) + try { + const result = await importCsv({ + workspaceId, + tableId: table.id, + file: parsed.file, + mode, + mapping, + }) + const data = result.data + if (mode === 'append') { + toast.success(`Imported ${data?.insertedCount ?? 0} rows into "${table.name}"`) + } else { + toast.success( + `Replaced rows in "${table.name}": deleted ${data?.deletedCount ?? 0}, inserted ${data?.insertedCount ?? 0}` + ) + } + onImported?.({ + insertedCount: data?.insertedCount, + deletedCount: data?.deletedCount, + }) + onOpenChange(false) + } catch (err) { + const message = err instanceof Error ? err.message : 'Failed to import CSV' + setSubmitError(summarizeImportError(message)) + logger.error('CSV import into existing table failed', err) + } + }, [ + canSubmit, + importCsv, + mapping, + mode, + onImported, + onOpenChange, + parsed, + table.id, + table.name, + workspaceId, + ]) + + const hasWarning = + missingRequired.length > 0 || + duplicateTargets.length > 0 || + appendCapacityDeficit > 0 || + replaceCapacityDeficit > 0 + + return ( + + + Import CSV into {table.name} + + {!parsed ? ( +
+ + + {parseError && ( +

{parseError}

+ )} +
+ ) : ( +
+
+
+ + {parsed.file.name} + + + {parsed.totalRows.toLocaleString()} rows · {parsed.headers.length} columns + +
+ +
+ +
+ + + Append + Replace all rows + +
+ +
+ +
+
+ + + + CSV column + Target column + + + + {parsed.headers.map((header) => { + const sample = parsed.sampleRows + .map((r) => + r[header] === '' || r[header] == null ? '' : String(r[header]) + ) + .filter(Boolean) + .slice(0, 2) + .join(', ') + return ( + + +
+ + {header} + + {sample && ( + + {sample} + + )} +
+ + + handleMappingChange(header, value)} + size='sm' + className='w-full' + /> + + + ) + })} + +
+
+
+ + {mappedCount} mapped · {skipCount} skipped + +
+ + {hasWarning && ( +
+ {missingRequired.length > 0 && ( +

+ Missing required column(s): {missingRequired.join(', ')} +

+ )} + {duplicateTargets.length > 0 && ( +

+ Multiple CSV columns target: {duplicateTargets.join(', ')} (pick one) +

+ )} + {appendCapacityDeficit > 0 && ( +

+ Append would exceed the row limit ({table.maxRows.toLocaleString()}) by{' '} + {appendCapacityDeficit.toLocaleString()} row(s). Remove rows or switch to + Replace. +

+ )} + {replaceCapacityDeficit > 0 && ( +

+ CSV has {parsed.totalRows.toLocaleString()} rows, which exceeds the table + limit of {table.maxRows.toLocaleString()} by{' '} + {replaceCapacityDeficit.toLocaleString()}. +

+ )} +
+ )} + + {mode === 'replace' && !hasWarning && ( +

+ Replace will permanently delete the {table.rowCount.toLocaleString()} existing + row(s) before inserting the new rows. +

+ )} + + {submitError && ( +

+ {submitError} +

+ )} +
+ )} + + + + + + + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/tables/components/import-csv-dialog/index.ts b/apps/sim/app/workspace/[workspaceId]/tables/components/import-csv-dialog/index.ts new file mode 100644 index 00000000000..d9ee61eb41f --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/tables/components/import-csv-dialog/index.ts @@ -0,0 +1 @@ +export * from './import-csv-dialog' diff --git a/apps/sim/app/workspace/[workspaceId]/tables/components/index.ts b/apps/sim/app/workspace/[workspaceId]/tables/components/index.ts index b1c93333c52..4a74ec95484 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/components/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/components/index.ts @@ -1,2 +1,3 @@ +export * from './import-csv-dialog' export * from './table-context-menu' export * from './tables-list-context-menu' diff --git a/apps/sim/app/workspace/[workspaceId]/tables/components/table-context-menu/table-context-menu.tsx b/apps/sim/app/workspace/[workspaceId]/tables/components/table-context-menu/table-context-menu.tsx index 051557b10cd..ef493231c5c 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/components/table-context-menu/table-context-menu.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/components/table-context-menu/table-context-menu.tsx @@ -6,6 +6,7 @@ import { DropdownMenuItem, DropdownMenuSeparator, DropdownMenuTrigger, + Upload, } from '@/components/emcn' import { Copy, Database, Pencil, Trash } from '@/components/emcn/icons' @@ -17,8 +18,10 @@ interface TableContextMenuProps { onDelete?: () => void onViewSchema?: () => void onRename?: () => void + onImportCsv?: () => void disableDelete?: boolean disableRename?: boolean + disableImport?: boolean menuRef?: React.RefObject } @@ -30,8 +33,10 @@ export function TableContextMenu({ onDelete, onViewSchema, onRename, + onImportCsv, disableDelete = false, disableRename = false, + disableImport = false, }: TableContextMenuProps) { return ( !open && onClose()} modal={false}> @@ -67,7 +72,15 @@ export function TableContextMenu({ Rename )} - {(onViewSchema || onRename) && (onCopyId || onDelete) && } + {onImportCsv && ( + + + Import CSV... + + )} + {(onViewSchema || onRename || onImportCsv) && (onCopyId || onDelete) && ( + + )} {onCopyId && ( diff --git a/apps/sim/app/workspace/[workspaceId]/tables/tables.tsx b/apps/sim/app/workspace/[workspaceId]/tables/tables.tsx index 0761513cdf4..c017e766ba0 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/tables.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/tables.tsx @@ -27,7 +27,10 @@ import type { } from '@/app/workspace/[workspaceId]/components' import { ownerCell, Resource, timeCell } from '@/app/workspace/[workspaceId]/components' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' -import { TablesListContextMenu } from '@/app/workspace/[workspaceId]/tables/components' +import { + ImportCsvDialog, + TablesListContextMenu, +} from '@/app/workspace/[workspaceId]/tables/components' import { TableContextMenu } from '@/app/workspace/[workspaceId]/tables/components/table-context-menu' import { useContextMenu } from '@/app/workspace/[workspaceId]/w/components/sidebar/hooks' import { @@ -76,6 +79,7 @@ export function Tables() { const uploadCsv = useUploadCsvToTable() const [isDeleteDialogOpen, setIsDeleteDialogOpen] = useState(false) + const [isImportDialogOpen, setIsImportDialogOpen] = useState(false) const [activeTable, setActiveTable] = useState(null) const [searchTerm, setSearchTerm] = useState('') const debouncedSearchTerm = useDebounce(searchTerm, 300) @@ -525,10 +529,24 @@ export function Tables() { if (activeTable) navigator.clipboard.writeText(activeTable.id) }} onDelete={() => setIsDeleteDialogOpen(true)} + onImportCsv={() => setIsImportDialogOpen(true)} disableDelete={userPermissions.canEdit !== true} disableRename={userPermissions.canEdit !== true} + disableImport={userPermissions.canEdit !== true} /> + {activeTable && ( + { + setIsImportDialogOpen(open) + if (!open) setActiveTable(null) + }} + workspaceId={workspaceId} + table={activeTable} + /> + )} + Delete Table diff --git a/apps/sim/hooks/queries/tables.ts b/apps/sim/hooks/queries/tables.ts index cef8e4447fc..9f67d50e9a3 100644 --- a/apps/sim/hooks/queries/tables.ts +++ b/apps/sim/hooks/queries/tables.ts @@ -780,6 +780,77 @@ export function useUploadCsvToTable() { }) } +export type CsvHeaderMapping = Record +export type CsvImportMode = 'append' | 'replace' + +interface ImportCsvIntoTableParams { + workspaceId: string + tableId: string + file: File + mode: CsvImportMode + mapping?: CsvHeaderMapping +} + +interface ImportCsvIntoTableResponse { + success: boolean + data?: { + tableId: string + mode: CsvImportMode + insertedCount?: number + deletedCount?: number + mappedColumns?: string[] + skippedHeaders?: string[] + unmappedColumns?: string[] + sourceFile?: string + } +} + +/** + * Upload a CSV file to an existing table in append or replace mode. Supports + * an optional explicit header-to-column mapping; when omitted the server + * auto-maps headers by sanitized name. + */ +export function useImportCsvIntoTable() { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: async ({ + workspaceId, + tableId, + file, + mode, + mapping, + }: ImportCsvIntoTableParams): Promise => { + const formData = new FormData() + formData.append('file', file) + formData.append('workspaceId', workspaceId) + formData.append('mode', mode) + if (mapping) { + formData.append('mapping', JSON.stringify(mapping)) + } + + const response = await fetch(`/api/table/${tableId}/import-csv`, { + method: 'POST', + body: formData, + }) + + if (!response.ok) { + const data = await response.json().catch(() => ({})) + throw new Error(data.error || 'CSV import failed') + } + + return response.json() + }, + onSettled: (_data, _error, variables) => { + if (!variables) return + invalidateRowCount(queryClient, variables.workspaceId, variables.tableId) + }, + onError: (error) => { + logger.error('Failed to import CSV into table:', error) + }, + }) +} + export function useDeleteColumn({ workspaceId, tableId }: RowMutationContext) { const queryClient = useQueryClient() diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index 2af7b1d660e..2d3930bc54d 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -2830,6 +2830,17 @@ export const UserTable: ToolCatalogEntry = { type: 'number', description: 'Maximum rows to return or affect (optional, default 100)', }, + mapping: { + type: 'object', + description: + 'Optional explicit CSV-header → table-column mapping for import_file, as { "csvHeader": "columnName" | null }. When omitted, headers are auto-matched by sanitized name (case-insensitive fallback). Use null to skip a CSV column.', + }, + mode: { + type: 'string', + description: + "Import mode for import_file. 'append' (default) adds rows; 'replace' truncates existing rows in a transaction before inserting the new rows.", + enum: ['append', 'replace'], + }, name: { type: 'string', description: "Table name (required for 'create')" }, newName: { type: 'string', description: 'New column name (required for rename_column)' }, newType: { diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index 78e624c8473..fdf139dd72c 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -10,7 +10,7 @@ export interface ToolRuntimeSchemaEntry { } export const TOOL_RUNTIME_SCHEMAS: Record = { - agent: { + ['agent']: { parameters: { properties: { request: { @@ -23,7 +23,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - auth: { + ['auth']: { parameters: { properties: { request: { @@ -36,7 +36,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - check_deployment_status: { + ['check_deployment_status']: { parameters: { type: 'object', properties: { @@ -48,7 +48,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - complete_job: { + ['complete_job']: { parameters: { type: 'object', properties: { @@ -61,7 +61,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - context_write: { + ['context_write']: { parameters: { type: 'object', properties: { @@ -78,7 +78,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - crawl_website: { + ['crawl_website']: { parameters: { type: 'object', properties: { @@ -113,7 +113,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - create_file: { + ['create_file']: { parameters: { type: 'object', properties: { @@ -149,7 +149,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - create_folder: { + ['create_folder']: { parameters: { type: 'object', properties: { @@ -170,7 +170,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - create_job: { + ['create_job']: { parameters: { type: 'object', properties: { @@ -220,7 +220,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - create_workflow: { + ['create_workflow']: { parameters: { type: 'object', properties: { @@ -245,7 +245,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - create_workspace_mcp_server: { + ['create_workspace_mcp_server']: { parameters: { type: 'object', properties: { @@ -266,7 +266,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - debug: { + ['debug']: { parameters: { properties: { context: { @@ -285,7 +285,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - delete_file: { + ['delete_file']: { parameters: { type: 'object', properties: { @@ -314,7 +314,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - delete_folder: { + ['delete_folder']: { parameters: { type: 'object', properties: { @@ -330,7 +330,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - delete_workflow: { + ['delete_workflow']: { parameters: { type: 'object', properties: { @@ -346,7 +346,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - delete_workspace_mcp_server: { + ['delete_workspace_mcp_server']: { parameters: { type: 'object', properties: { @@ -359,7 +359,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - deploy: { + ['deploy']: { parameters: { properties: { request: { @@ -373,7 +373,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - deploy_api: { + ['deploy_api']: { parameters: { type: 'object', properties: { @@ -447,7 +447,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { ], }, }, - deploy_chat: { + ['deploy_chat']: { parameters: { type: 'object', properties: { @@ -595,7 +595,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { ], }, }, - deploy_mcp: { + ['deploy_mcp']: { parameters: { type: 'object', properties: { @@ -711,7 +711,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['deploymentType', 'deploymentStatus'], }, }, - download_to_workspace_file: { + ['download_to_workspace_file']: { parameters: { type: 'object', properties: { @@ -730,7 +730,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - edit_content: { + ['edit_content']: { parameters: { type: 'object', properties: { @@ -762,7 +762,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - edit_workflow: { + ['edit_workflow']: { parameters: { type: 'object', properties: { @@ -801,13 +801,13 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - file: { + ['file']: { parameters: { type: 'object', }, resultSchema: undefined, }, - function_execute: { + ['function_execute']: { parameters: { type: 'object', properties: { @@ -868,7 +868,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - generate_api_key: { + ['generate_api_key']: { parameters: { type: 'object', properties: { @@ -886,7 +886,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - generate_image: { + ['generate_image']: { parameters: { type: 'object', properties: { @@ -923,7 +923,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - generate_visualization: { + ['generate_visualization']: { parameters: { type: 'object', properties: { @@ -963,7 +963,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_block_outputs: { + ['get_block_outputs']: { parameters: { type: 'object', properties: { @@ -984,7 +984,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_block_upstream_references: { + ['get_block_upstream_references']: { parameters: { type: 'object', properties: { @@ -1006,7 +1006,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_deployed_workflow_state: { + ['get_deployed_workflow_state']: { parameters: { type: 'object', properties: { @@ -1019,7 +1019,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_deployment_version: { + ['get_deployment_version']: { parameters: { type: 'object', properties: { @@ -1036,7 +1036,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_execution_summary: { + ['get_execution_summary']: { parameters: { type: 'object', properties: { @@ -1063,7 +1063,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_job_logs: { + ['get_job_logs']: { parameters: { type: 'object', properties: { @@ -1088,7 +1088,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_page_contents: { + ['get_page_contents']: { parameters: { type: 'object', properties: { @@ -1116,14 +1116,14 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_platform_actions: { + ['get_platform_actions']: { parameters: { type: 'object', properties: {}, }, resultSchema: undefined, }, - get_workflow_data: { + ['get_workflow_data']: { parameters: { type: 'object', properties: { @@ -1142,7 +1142,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - get_workflow_logs: { + ['get_workflow_logs']: { parameters: { type: 'object', properties: { @@ -1168,7 +1168,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - glob: { + ['glob']: { parameters: { type: 'object', properties: { @@ -1187,7 +1187,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - grep: { + ['grep']: { parameters: { type: 'object', properties: { @@ -1234,7 +1234,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - job: { + ['job']: { parameters: { properties: { request: { @@ -1247,7 +1247,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - knowledge: { + ['knowledge']: { parameters: { properties: { request: { @@ -1260,7 +1260,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - knowledge_base: { + ['knowledge_base']: { parameters: { type: 'object', properties: { @@ -1452,7 +1452,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - list_folders: { + ['list_folders']: { parameters: { type: 'object', properties: { @@ -1464,14 +1464,14 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - list_user_workspaces: { + ['list_user_workspaces']: { parameters: { type: 'object', properties: {}, }, resultSchema: undefined, }, - list_workspace_mcp_servers: { + ['list_workspace_mcp_servers']: { parameters: { type: 'object', properties: { @@ -1483,7 +1483,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - manage_credential: { + ['manage_credential']: { parameters: { type: 'object', properties: { @@ -1512,7 +1512,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - manage_custom_tool: { + ['manage_custom_tool']: { parameters: { type: 'object', properties: { @@ -1591,7 +1591,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - manage_job: { + ['manage_job']: { parameters: { type: 'object', properties: { @@ -1661,7 +1661,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - manage_mcp_tool: { + ['manage_mcp_tool']: { parameters: { type: 'object', properties: { @@ -1712,7 +1712,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - manage_skill: { + ['manage_skill']: { parameters: { type: 'object', properties: { @@ -1744,7 +1744,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - materialize_file: { + ['materialize_file']: { parameters: { type: 'object', properties: { @@ -1778,7 +1778,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - move_folder: { + ['move_folder']: { parameters: { type: 'object', properties: { @@ -1796,7 +1796,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - move_workflow: { + ['move_workflow']: { parameters: { type: 'object', properties: { @@ -1816,7 +1816,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - oauth_get_auth_link: { + ['oauth_get_auth_link']: { parameters: { type: 'object', properties: { @@ -1830,7 +1830,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - oauth_request_access: { + ['oauth_request_access']: { parameters: { type: 'object', properties: { @@ -1844,7 +1844,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - open_resource: { + ['open_resource']: { parameters: { type: 'object', properties: { @@ -1872,7 +1872,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - read: { + ['read']: { parameters: { type: 'object', properties: { @@ -1899,7 +1899,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - redeploy: { + ['redeploy']: { parameters: { type: 'object', properties: { @@ -1967,7 +1967,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { ], }, }, - rename_file: { + ['rename_file']: { parameters: { type: 'object', properties: { @@ -2002,7 +2002,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - rename_workflow: { + ['rename_workflow']: { parameters: { type: 'object', properties: { @@ -2019,7 +2019,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - research: { + ['research']: { parameters: { properties: { topic: { @@ -2032,7 +2032,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - respond: { + ['respond']: { parameters: { additionalProperties: true, properties: { @@ -2055,7 +2055,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - restore_resource: { + ['restore_resource']: { parameters: { type: 'object', properties: { @@ -2073,7 +2073,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - revert_to_version: { + ['revert_to_version']: { parameters: { type: 'object', properties: { @@ -2090,7 +2090,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - run: { + ['run']: { parameters: { properties: { context: { @@ -2107,7 +2107,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - run_block: { + ['run_block']: { parameters: { type: 'object', properties: { @@ -2139,7 +2139,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - run_from_block: { + ['run_from_block']: { parameters: { type: 'object', properties: { @@ -2171,7 +2171,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - run_workflow: { + ['run_workflow']: { parameters: { type: 'object', properties: { @@ -2199,7 +2199,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - run_workflow_until_block: { + ['run_workflow_until_block']: { parameters: { type: 'object', properties: { @@ -2231,7 +2231,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - scrape_page: { + ['scrape_page']: { parameters: { type: 'object', properties: { @@ -2252,7 +2252,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - search_documentation: { + ['search_documentation']: { parameters: { type: 'object', properties: { @@ -2269,7 +2269,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - search_library_docs: { + ['search_library_docs']: { parameters: { type: 'object', properties: { @@ -2290,7 +2290,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - search_online: { + ['search_online']: { parameters: { type: 'object', properties: { @@ -2331,7 +2331,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - search_patterns: { + ['search_patterns']: { parameters: { type: 'object', properties: { @@ -2353,7 +2353,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - set_block_enabled: { + ['set_block_enabled']: { parameters: { type: 'object', properties: { @@ -2375,7 +2375,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - set_environment_variables: { + ['set_environment_variables']: { parameters: { type: 'object', properties: { @@ -2409,7 +2409,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - set_global_workflow_variables: { + ['set_global_workflow_variables']: { parameters: { type: 'object', properties: { @@ -2447,7 +2447,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - superagent: { + ['superagent']: { parameters: { properties: { task: { @@ -2461,7 +2461,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - table: { + ['table']: { parameters: { properties: { request: { @@ -2474,7 +2474,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - tool_search_tool_regex: { + ['tool_search_tool_regex']: { parameters: { properties: { case_insensitive: { @@ -2495,7 +2495,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - update_job_history: { + ['update_job_history']: { parameters: { type: 'object', properties: { @@ -2513,7 +2513,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - update_workspace_mcp_server: { + ['update_workspace_mcp_server']: { parameters: { type: 'object', properties: { @@ -2538,7 +2538,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - user_memory: { + ['user_memory']: { parameters: { type: 'object', properties: { @@ -2586,7 +2586,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, - user_table: { + ['user_table']: { parameters: { type: 'object', properties: { @@ -2635,6 +2635,17 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { type: 'number', description: 'Maximum rows to return or affect (optional, default 100)', }, + mapping: { + type: 'object', + description: + 'Optional explicit CSV-header → table-column mapping for import_file, as { "csvHeader": "columnName" | null }. When omitted, headers are auto-matched by sanitized name (case-insensitive fallback). Use null to skip a CSV column.', + }, + mode: { + type: 'string', + description: + "Import mode for import_file. 'append' (default) adds rows; 'replace' truncates existing rows in a transaction before inserting the new rows.", + enum: ['append', 'replace'], + }, name: { type: 'string', description: "Table name (required for 'create')", @@ -2761,13 +2772,13 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['success', 'message'], }, }, - workflow: { + ['workflow']: { parameters: { type: 'object', }, resultSchema: undefined, }, - workspace_file: { + ['workspace_file']: { parameters: { type: 'object', properties: { diff --git a/apps/sim/lib/copilot/request/tools/files.test.ts b/apps/sim/lib/copilot/request/tools/files.test.ts new file mode 100644 index 00000000000..10bf4e57356 --- /dev/null +++ b/apps/sim/lib/copilot/request/tools/files.test.ts @@ -0,0 +1,99 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + extractTabularData, + serializeOutputForFile, + unwrapFunctionExecuteOutput, +} from '@/lib/copilot/request/tools/files' + +describe('unwrapFunctionExecuteOutput', () => { + it('unwraps the function_execute envelope { result, stdout }', () => { + expect(unwrapFunctionExecuteOutput({ result: 'name,age\nAlice,30', stdout: '' })).toBe( + 'name,age\nAlice,30' + ) + }) + + it('passes through objects that do not have both result + stdout', () => { + const output = { data: { rows: [], totalCount: 0 } } + expect(unwrapFunctionExecuteOutput(output)).toBe(output) + }) + + it('passes through strings and arrays untouched', () => { + expect(unwrapFunctionExecuteOutput('hello')).toBe('hello') + const arr: unknown[] = [{ a: 1 }] + expect(unwrapFunctionExecuteOutput(arr)).toBe(arr) + }) +}) + +describe('serializeOutputForFile (csv)', () => { + it('returns raw CSV text when function_execute result is already a CSV string', () => { + const output = { + result: 'name,age\nAlice,30\nBob,40', + stdout: '(2 rows)', + } + expect(serializeOutputForFile(output, 'csv')).toBe('name,age\nAlice,30\nBob,40') + }) + + it('converts a result array of objects into CSV', () => { + const output = { + result: [ + { name: 'Alice', age: 30 }, + { name: 'Bob', age: 40 }, + ], + stdout: '', + } + expect(serializeOutputForFile(output, 'csv')).toBe('name,age\nAlice,30\nBob,40') + }) + + it('returns the raw string when the non-envelope output is already a CSV string', () => { + expect(serializeOutputForFile('a,b\n1,2', 'csv')).toBe('a,b\n1,2') + }) + + it('falls back to JSON.stringify when the payload is not tabular and not a string', () => { + const output = { result: { foo: 'bar' }, stdout: '' } + expect(serializeOutputForFile(output, 'csv')).toBe('{\n "foo": "bar"\n}') + }) +}) + +describe('serializeOutputForFile (json / txt / md)', () => { + it('unwraps the envelope for json format so the file contains only result', () => { + const output = { result: { hello: 'world' }, stdout: 'log' } + expect(serializeOutputForFile(output, 'json')).toBe('{\n "hello": "world"\n}') + }) + + it('returns the string payload as-is for txt/md/html formats', () => { + const output = { result: '# Report\n\nHello', stdout: '' } + expect(serializeOutputForFile(output, 'md')).toBe('# Report\n\nHello') + expect(serializeOutputForFile(output, 'txt')).toBe('# Report\n\nHello') + expect(serializeOutputForFile(output, 'html')).toBe('# Report\n\nHello') + }) +}) + +describe('extractTabularData', () => { + it('extracts rows from function_execute { result: [...] } via unwrap', () => { + const rows = extractTabularData({ + result: [{ a: 1 }, { a: 2 }], + stdout: '', + }) + expect(rows).toEqual([{ a: 1 }, { a: 2 }]) + }) + + it('returns null when function_execute result is a non-tabular string', () => { + expect(extractTabularData({ result: 'name,age\nAlice,30', stdout: '' })).toBeNull() + }) + + it('still extracts rows from the user_table query_rows shape', () => { + const rows = extractTabularData({ + data: { + rows: [ + { id: 'row_1', data: { name: 'Alice' } }, + { id: 'row_2', data: { name: 'Bob' } }, + ], + totalCount: 2, + }, + }) + expect(rows).toEqual([{ name: 'Alice' }, { name: 'Bob' }]) + }) +}) diff --git a/apps/sim/lib/copilot/request/tools/files.ts b/apps/sim/lib/copilot/request/tools/files.ts index 64bb4bf6a1d..1e26303c777 100644 --- a/apps/sim/lib/copilot/request/tools/files.ts +++ b/apps/sim/lib/copilot/request/tools/files.ts @@ -25,29 +25,40 @@ export const FORMAT_TO_CONTENT_TYPE: Record = { html: 'text/html', } +/** + * Unwraps the `function_execute` response envelope `{ result, stdout }` so the + * rest of the serialization code works on the user's actual payload (a string, + * array, object, etc.) instead of JSON-stringifying the envelope itself. + * + * Only unwraps when both keys are present — that's the unique shape of + * `function_execute` (see `apps/sim/tools/function/types.ts` `CodeExecutionOutput`). + * `user_table` returns `{ data, message, success }` which is left alone. + */ +export function unwrapFunctionExecuteOutput(output: unknown): unknown { + if (!output || typeof output !== 'object' || Array.isArray(output)) return output + const obj = output as Record + if ('result' in obj && 'stdout' in obj) { + return obj.result + } + return output +} + /** * Try to pull a flat array of row-objects out of the various shapes that * `function_execute` and `user_table` can return. */ export function extractTabularData(output: unknown): Record[] | null { - if (!output || typeof output !== 'object') return null + const unwrapped = unwrapFunctionExecuteOutput(output) + if (!unwrapped || typeof unwrapped !== 'object') return null - if (Array.isArray(output)) { - if (output.length > 0 && typeof output[0] === 'object' && output[0] !== null) { - return output as Record[] + if (Array.isArray(unwrapped)) { + if (unwrapped.length > 0 && typeof unwrapped[0] === 'object' && unwrapped[0] !== null) { + return unwrapped as Record[] } return null } - const obj = output as Record - - // function_execute shape: { result: [...], stdout: "..." } - if (Array.isArray(obj.result)) { - const rows = obj.result - if (rows.length > 0 && typeof rows[0] === 'object' && rows[0] !== null) { - return rows as Record[] - } - } + const obj = unwrapped as Record // user_table query_rows shape: { data: { rows: [{ data: {...} }], totalCount } } if (obj.data && typeof obj.data === 'object' && !Array.isArray(obj.data)) { @@ -112,16 +123,18 @@ export function resolveOutputFormat(fileName: string, explicit?: string): Output } export function serializeOutputForFile(output: unknown, format: OutputFormat): string { - if (typeof output === 'string') return output + const unwrapped = unwrapFunctionExecuteOutput(output) + + if (typeof unwrapped === 'string') return unwrapped if (format === 'csv') { - const rows = extractTabularData(output) + const rows = extractTabularData(unwrapped) if (rows && rows.length > 0) { return convertRowsToCsv(rows) } } - return JSON.stringify(output, null, 2) + return JSON.stringify(unwrapped, null, 2) } export async function maybeWriteOutputToFile( diff --git a/apps/sim/lib/copilot/tools/server/table/user-table.test.ts b/apps/sim/lib/copilot/tools/server/table/user-table.test.ts new file mode 100644 index 00000000000..2110a5e5e27 --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/table/user-table.test.ts @@ -0,0 +1,220 @@ +/** + * @vitest-environment node + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { TableDefinition } from '@/lib/table' + +const { + mockResolveWorkspaceFileReference, + mockDownloadWorkspaceFile, + mockGetTableById, + mockBatchInsertRows, + mockReplaceTableRows, +} = vi.hoisted(() => ({ + mockResolveWorkspaceFileReference: vi.fn(), + mockDownloadWorkspaceFile: vi.fn(), + mockGetTableById: vi.fn(), + mockBatchInsertRows: vi.fn(), + mockReplaceTableRows: vi.fn(), +})) + +vi.mock('@sim/logger', () => ({ + createLogger: () => ({ + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }), +})) + +vi.mock('@/lib/core/utils/uuid', () => ({ + generateId: vi.fn().mockReturnValue('deadbeefcafef00d'), + generateShortId: vi.fn().mockReturnValue('short-id'), +})) + +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ + resolveWorkspaceFileReference: mockResolveWorkspaceFileReference, + downloadWorkspaceFile: mockDownloadWorkspaceFile, +})) + +vi.mock('@/lib/table/service', () => ({ + addTableColumn: vi.fn(), + batchInsertRows: mockBatchInsertRows, + batchUpdateRows: vi.fn(), + createTable: vi.fn(), + deleteColumn: vi.fn(), + deleteColumns: vi.fn(), + deleteRow: vi.fn(), + deleteRowsByFilter: vi.fn(), + deleteRowsByIds: vi.fn(), + deleteTable: vi.fn(), + getRowById: vi.fn(), + getTableById: mockGetTableById, + insertRow: vi.fn(), + queryRows: vi.fn(), + renameColumn: vi.fn(), + renameTable: vi.fn(), + replaceTableRows: mockReplaceTableRows, + updateColumnConstraints: vi.fn(), + updateColumnType: vi.fn(), + updateRow: vi.fn(), + updateRowsByFilter: vi.fn(), +})) + +import { userTableServerTool } from '@/lib/copilot/tools/server/table/user-table' + +function buildTable(overrides: Partial = {}): TableDefinition { + return { + id: 'tbl_1', + name: 'People', + description: null, + schema: { + columns: [ + { name: 'name', type: 'string', required: true }, + { name: 'age', type: 'number' }, + ], + }, + metadata: null, + rowCount: 0, + maxRows: 100, + workspaceId: 'workspace-1', + createdBy: 'user-1', + archivedAt: null, + createdAt: new Date('2024-01-01'), + updatedAt: new Date('2024-01-01'), + ...overrides, + } +} + +describe('userTableServerTool.import_file', () => { + beforeEach(() => { + vi.clearAllMocks() + mockResolveWorkspaceFileReference.mockResolvedValue({ + name: 'people.csv', + type: 'text/csv', + }) + mockDownloadWorkspaceFile.mockResolvedValue(Buffer.from('name,age\nAlice,30\nBob,40')) + mockGetTableById.mockResolvedValue(buildTable()) + mockBatchInsertRows.mockImplementation(async (data: { rows: unknown[] }) => + data.rows.map((_, i) => ({ id: `row_${i}` })) + ) + mockReplaceTableRows.mockResolvedValue({ deletedCount: 0, insertedCount: 0 }) + }) + + it('appends rows using auto-mapping by default', async () => { + const result = await userTableServerTool.execute( + { + operation: 'import_file', + args: { tableId: 'tbl_1', fileId: 'file-1' }, + }, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + + expect(result.success).toBe(true) + expect(result.data?.mode).toBe('append') + expect(result.data?.rowCount).toBe(2) + expect(mockBatchInsertRows).toHaveBeenCalledTimes(1) + expect(mockReplaceTableRows).not.toHaveBeenCalled() + const call = mockBatchInsertRows.mock.calls[0][0] as { rows: unknown[] } + expect(call.rows).toEqual([ + { name: 'Alice', age: 30 }, + { name: 'Bob', age: 40 }, + ]) + }) + + it('replaces rows in replace mode', async () => { + mockReplaceTableRows.mockResolvedValueOnce({ deletedCount: 3, insertedCount: 2 }) + const result = await userTableServerTool.execute( + { + operation: 'import_file', + args: { tableId: 'tbl_1', fileId: 'file-1', mode: 'replace' }, + }, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + + expect(result.success).toBe(true) + expect(result.data?.mode).toBe('replace') + expect(result.data?.deletedCount).toBe(3) + expect(result.data?.insertedCount).toBe(2) + expect(mockReplaceTableRows).toHaveBeenCalledTimes(1) + expect(mockBatchInsertRows).not.toHaveBeenCalled() + }) + + it('uses the caller-provided mapping', async () => { + mockDownloadWorkspaceFile.mockResolvedValueOnce( + Buffer.from('Full Name,Years\nAlice,30\nBob,40') + ) + const result = await userTableServerTool.execute( + { + operation: 'import_file', + args: { + tableId: 'tbl_1', + fileId: 'file-1', + mapping: { 'Full Name': 'name', Years: 'age' }, + }, + }, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + + expect(result.success).toBe(true) + const call = mockBatchInsertRows.mock.calls[0][0] as { rows: unknown[] } + expect(call.rows).toEqual([ + { name: 'Alice', age: 30 }, + { name: 'Bob', age: 40 }, + ]) + }) + + it('rejects unknown modes', async () => { + const result = await userTableServerTool.execute( + { + operation: 'import_file', + args: { tableId: 'tbl_1', fileId: 'file-1', mode: 'merge' }, + }, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + expect(result.success).toBe(false) + expect(result.message).toMatch(/Invalid mode/) + expect(mockBatchInsertRows).not.toHaveBeenCalled() + }) + + it('refuses to import into an archived table', async () => { + mockGetTableById.mockResolvedValueOnce(buildTable({ archivedAt: new Date('2024-02-01') })) + const result = await userTableServerTool.execute( + { + operation: 'import_file', + args: { tableId: 'tbl_1', fileId: 'file-1' }, + }, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + expect(result.success).toBe(false) + expect(result.message).toMatch(/archived/i) + }) + + it('refuses to import when the table belongs to a different workspace', async () => { + mockGetTableById.mockResolvedValueOnce(buildTable({ workspaceId: 'workspace-other' })) + const result = await userTableServerTool.execute( + { + operation: 'import_file', + args: { tableId: 'tbl_1', fileId: 'file-1' }, + }, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + expect(result.success).toBe(false) + expect(result.message).toMatch(/not found/i) + expect(mockBatchInsertRows).not.toHaveBeenCalled() + }) + + it('reports missing required columns instead of inserting', async () => { + mockDownloadWorkspaceFile.mockResolvedValueOnce(Buffer.from('age\n30')) + const result = await userTableServerTool.execute( + { + operation: 'import_file', + args: { tableId: 'tbl_1', fileId: 'file-1' }, + }, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + expect(result.success).toBe(false) + expect(result.message).toMatch(/missing required columns/i) + expect(mockBatchInsertRows).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/copilot/tools/server/table/user-table.ts b/apps/sim/lib/copilot/tools/server/table/user-table.ts index 626de7a938d..af545b4baff 100644 --- a/apps/sim/lib/copilot/tools/server/table/user-table.ts +++ b/apps/sim/lib/copilot/tools/server/table/user-table.ts @@ -7,6 +7,17 @@ import { } from '@/lib/copilot/tools/server/base-tool' import { generateId } from '@/lib/core/utils/uuid' import { COLUMN_TYPES } from '@/lib/table/constants' +import { + buildAutoMapping, + CSV_MAX_BATCH_SIZE, + type CsvHeaderMapping, + CsvImportValidationError, + coerceRowsForTable, + inferSchemaFromCsv, + parseCsvBuffer, + sanitizeName, + validateMapping, +} from '@/lib/table/csv-import' import { addTableColumn, batchInsertRows, @@ -24,12 +35,13 @@ import { queryRows, renameColumn, renameTable, + replaceTableRows, updateColumnConstraints, updateColumnType, updateRow, updateRowsByFilter, } from '@/lib/table/service' -import type { ColumnDefinition, RowData, TableDefinition } from '@/lib/table/types' +import type { RowData, TableDefinition } from '@/lib/table/types' import { downloadWorkspaceFile, resolveWorkspaceFileReference, @@ -48,22 +60,27 @@ type UserTableResult = { data?: any } -const MAX_BATCH_SIZE = 1000 -const SCHEMA_SAMPLE_SIZE = 100 - -type ColumnType = 'string' | 'number' | 'boolean' | 'date' | 'json' +const MAX_BATCH_SIZE = CSV_MAX_BATCH_SIZE -function sanitizeColumnName(raw: string): string { - let name = raw - .trim() - .replace(/[^a-zA-Z0-9_]/g, '_') - .replace(/_+/g, '_') - .replace(/^_|_$/g, '') - if (!name || /^\d/.test(name)) name = `col_${name}` - return name +async function resolveWorkspaceFile( + fileReference: string, + workspaceId: string +): Promise<{ buffer: Buffer; name: string; type: string }> { + const record = await resolveWorkspaceFileReference(workspaceId, fileReference) + if (!record) { + throw new Error( + `File not found: "${fileReference}". Use glob("files/by-id/*/meta.json") to list canonical file IDs.` + ) + } + const buffer = await downloadWorkspaceFile(record) + return { buffer, name: record.name, type: record.type } } -function sanitizeHeaders( +/** + * Sanitizes raw JSON headers/rows so they conform to the same rules as CSV + * imports (so `inferSchemaFromCsv` and friends can be reused). + */ +function sanitizeJsonHeaders( headers: string[], rows: Record[] ): { headers: string[]; rows: Record[] } { @@ -71,7 +88,7 @@ function sanitizeHeaders( const seen = new Set() for (const raw of headers) { - let safe = sanitizeColumnName(raw) + let safe = sanitizeName(raw) while (seen.has(safe)) safe = `${safe}_` seen.add(safe) renamed.set(raw, safe) @@ -92,35 +109,6 @@ function sanitizeHeaders( } } -async function resolveWorkspaceFile( - fileReference: string, - workspaceId: string -): Promise<{ buffer: Buffer; name: string; type: string }> { - const record = await resolveWorkspaceFileReference(workspaceId, fileReference) - if (!record) { - throw new Error( - `File not found: "${fileReference}". Use glob("files/by-id/*/meta.json") to list canonical file IDs.` - ) - } - const buffer = await downloadWorkspaceFile(record) - return { buffer, name: record.name, type: record.type } -} - -function parseFileRows( - buffer: Buffer, - fileName: string, - contentType: string -): Promise<{ headers: string[]; rows: Record[] }> { - const ext = fileName.split('.').pop()?.toLowerCase() - if (ext === 'json' || contentType === 'application/json') { - return parseJsonRows(buffer) - } - if (ext === 'csv' || ext === 'tsv' || contentType === 'text/csv') { - return parseCsvRows(buffer) - } - throw new Error(`Unsupported file format: "${ext}". Supported: csv, tsv, json`) -} - async function parseJsonRows( buffer: Buffer ): Promise<{ headers: string[]; rows: Record[] }> { @@ -138,98 +126,23 @@ async function parseJsonRows( } for (const key of Object.keys(row)) headerSet.add(key) } - return sanitizeHeaders([...headerSet], parsed) + return sanitizeJsonHeaders([...headerSet], parsed) } -async function parseCsvRows( - buffer: Buffer +async function parseFileRows( + buffer: Buffer, + fileName: string, + contentType: string ): Promise<{ headers: string[]; rows: Record[] }> { - const { parse } = await import('csv-parse/sync') - const parsed = parse(buffer.toString('utf-8'), { - columns: true, - skip_empty_lines: true, - trim: true, - relax_column_count: true, - relax_quotes: true, - skip_records_with_error: true, - cast: false, - }) as Record[] - if (parsed.length === 0) { - throw new Error('CSV file has no data rows') - } - const headers = Object.keys(parsed[0]) - if (headers.length === 0) { - throw new Error('CSV file has no headers') + const ext = fileName.split('.').pop()?.toLowerCase() + if (ext === 'json' || contentType === 'application/json') { + return parseJsonRows(buffer) } - return sanitizeHeaders(headers, parsed) -} - -function inferColumnType(values: unknown[]): ColumnType { - const nonEmpty = values.filter((v) => v !== null && v !== undefined && v !== '') - if (nonEmpty.length === 0) return 'string' - - const allNumber = nonEmpty.every((v) => { - const n = Number(v) - return !Number.isNaN(n) && String(v).trim() !== '' - }) - if (allNumber) return 'number' - - const allBoolean = nonEmpty.every((v) => { - const s = String(v).toLowerCase() - return s === 'true' || s === 'false' - }) - if (allBoolean) return 'boolean' - - const isoDatePattern = /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2})?)?/ - const allDate = nonEmpty.every((v) => { - const s = String(v) - return isoDatePattern.test(s) && !Number.isNaN(Date.parse(s)) - }) - if (allDate) return 'date' - - return 'string' -} - -function inferSchema(headers: string[], rows: Record[]): ColumnDefinition[] { - const sample = rows.slice(0, SCHEMA_SAMPLE_SIZE) - return headers.map((name) => ({ - name, - type: inferColumnType(sample.map((r) => r[name])), - })) -} - -function coerceValue(value: unknown, colType: ColumnType): string | number | boolean | null { - if (value === null || value === undefined || value === '') return null - switch (colType) { - case 'number': { - const n = Number(value) - return Number.isNaN(n) ? null : n - } - case 'boolean': { - const s = String(value).toLowerCase() - return s === 'true' - } - case 'date': - return new Date(String(value)).toISOString() - default: - return String(value) + if (ext === 'csv' || ext === 'tsv' || contentType === 'text/csv') { + const delimiter = ext === 'tsv' ? '\t' : ',' + return parseCsvBuffer(buffer, delimiter) } -} - -function coerceRows( - rows: Record[], - columns: ColumnDefinition[], - columnMap: Map -): RowData[] { - return rows.map((row) => { - const coerced: RowData = {} - for (const col of columns) { - if (row[col.name] !== undefined) { - coerced[col.name] = coerceValue(row[col.name], col.type as ColumnType) - } - } - return coerced - }) + throw new Error(`Unsupported file format: "${ext}". Supported: csv, tsv, json`) } async function batchInsertAll( @@ -724,7 +637,7 @@ export const userTableServerTool: BaseServerTool return { success: false, message: 'File contains no data rows' } } - const columns = inferSchema(headers, rows) + const { columns, headerToColumn } = inferSchemaFromCsv(headers, rows) const tableName = args.name || file.name.replace(/\.[^.]+$/, '') const requestId = generateId().slice(0, 8) assertNotAborted() @@ -739,8 +652,7 @@ export const userTableServerTool: BaseServerTool requestId ) - const columnMap = new Map(columns.map((c) => [c.name, c])) - const coerced = coerceRows(rows, columns, columnMap) + const coerced = coerceRowsForTable(rows, { columns }, headerToColumn) const inserted = await batchInsertAll(table.id, coerced, table, workspaceId, context) logger.info('Table created from file', { @@ -769,6 +681,10 @@ export const userTableServerTool: BaseServerTool const filePath = (args as Record).filePath as string | undefined const tableId = (args as Record).tableId as string | undefined const fileReference = fileId || filePath + const rawMode = (args as Record).mode as string | undefined + const rawMapping = (args as Record).mapping as + | CsvHeaderMapping + | undefined if (!fileReference) { return { success: false, @@ -782,11 +698,21 @@ export const userTableServerTool: BaseServerTool if (!workspaceId) { return { success: false, message: 'Workspace ID is required' } } + if (rawMode && rawMode !== 'append' && rawMode !== 'replace') { + return { + success: false, + message: `Invalid mode "${rawMode}". Must be "append" or "replace".`, + } + } + const mode: 'append' | 'replace' = rawMode === 'replace' ? 'replace' : 'append' const table = await getTableById(tableId) - if (!table) { + if (!table || table.workspaceId !== workspaceId) { return { success: false, message: `Table not found: ${tableId}` } } + if (table.archivedAt) { + return { success: false, message: `Table is archived: ${tableId}` } + } const file = await resolveWorkspaceFile(fileReference, workspaceId) const { headers, rows } = await parseFileRows(file.buffer, file.name, file.type) @@ -794,47 +720,86 @@ export const userTableServerTool: BaseServerTool return { success: false, message: 'File contains no data rows' } } - const tableColumns = table.schema.columns as ColumnDefinition[] - const tableColNames = new Set(tableColumns.map((c) => c.name)) - const mappedHeaders = headers.filter((h) => tableColNames.has(h)) - if (mappedHeaders.length === 0) { + const mapping: CsvHeaderMapping = rawMapping ?? buildAutoMapping(headers, table.schema) + + let validation: ReturnType + try { + validation = validateMapping({ + csvHeaders: headers, + mapping, + tableSchema: table.schema, + }) + } catch (err) { + if (err instanceof CsvImportValidationError) { + return { success: false, message: err.message } + } + throw err + } + + if (validation.mappedHeaders.length === 0) { return { success: false, - message: `No matching columns between file (${headers.join(', ')}) and table (${tableColumns.map((c) => c.name).join(', ')})`, + message: `No matching columns between file (${headers.join(', ')}) and table (${table.schema.columns.map((c) => c.name).join(', ')})`, } } - const requiredMissing = tableColumns - .filter((c) => c.required && !headers.includes(c.name)) - .map((c) => c.name) - if (requiredMissing.length > 0) { + const coerced = coerceRowsForTable(rows, table.schema, validation.effectiveMap) + + if (mode === 'replace') { + assertNotAborted() + const requestId = generateId().slice(0, 8) + const result = await replaceTableRows( + { tableId: table.id, rows: coerced, workspaceId }, + table, + requestId + ) + + logger.info('Rows replaced from file', { + tableId: table.id, + fileName: file.name, + mode, + matchedColumns: validation.mappedHeaders.length, + deleted: result.deletedCount, + inserted: result.insertedCount, + userId: context.userId, + }) + return { - success: false, - message: `File is missing required columns: ${requiredMissing.join(', ')}`, + success: true, + message: `Replaced rows in "${table.name}" from "${file.name}": deleted ${result.deletedCount}, inserted ${result.insertedCount}`, + data: { + tableId: table.id, + tableName: table.name, + mode, + matchedColumns: validation.mappedHeaders, + skippedColumns: validation.skippedHeaders, + deletedCount: result.deletedCount, + insertedCount: result.insertedCount, + sourceFile: file.name, + }, } } - const columnMap = new Map(tableColumns.map((c) => [c.name, c])) - const matchedColumns = tableColumns.filter((c) => headers.includes(c.name)) - const coerced = coerceRows(rows, matchedColumns, columnMap) const inserted = await batchInsertAll(table.id, coerced, table, workspaceId, context) logger.info('Rows imported from file', { tableId: table.id, fileName: file.name, - matchedColumns: mappedHeaders.length, + mode, + matchedColumns: validation.mappedHeaders.length, rows: inserted, userId: context.userId, }) return { success: true, - message: `Imported ${inserted} rows into "${table.name}" from "${file.name}" (${mappedHeaders.length} columns matched)`, + message: `Imported ${inserted} rows into "${table.name}" from "${file.name}" (${validation.mappedHeaders.length} columns matched)`, data: { tableId: table.id, tableName: table.name, - matchedColumns: mappedHeaders, - skippedColumns: headers.filter((h) => !tableColNames.has(h)), + mode, + matchedColumns: validation.mappedHeaders, + skippedColumns: validation.skippedHeaders, rowCount: inserted, sourceFile: file.name, }, diff --git a/apps/sim/lib/table/csv-import.test.ts b/apps/sim/lib/table/csv-import.test.ts new file mode 100644 index 00000000000..b342fc7c988 --- /dev/null +++ b/apps/sim/lib/table/csv-import.test.ts @@ -0,0 +1,277 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + buildAutoMapping, + CsvImportValidationError, + coerceRowsForTable, + coerceValue, + inferColumnType, + inferSchemaFromCsv, + parseCsvBuffer, + sanitizeName, + validateMapping, +} from '@/lib/table/csv-import' +import type { TableSchema } from '@/lib/table/types' + +describe('csv-import', () => { + describe('parseCsvBuffer', () => { + it('parses a CSV string and extracts headers', async () => { + const { headers, rows } = await parseCsvBuffer('a,b\n1,2\n3,4') + expect(headers).toEqual(['a', 'b']) + expect(rows).toEqual([ + { a: '1', b: '2' }, + { a: '3', b: '4' }, + ]) + }) + + it('strips a UTF-8 BOM from the first header', async () => { + const text = `\uFEFFname,age\nAlice,30` + const { headers } = await parseCsvBuffer(text) + expect(headers).toEqual(['name', 'age']) + }) + + it('parses a Uint8Array input in browser-like environments', async () => { + const bytes = new TextEncoder().encode('a,b\n1,2') + const { headers, rows } = await parseCsvBuffer(bytes) + expect(headers).toEqual(['a', 'b']) + expect(rows).toHaveLength(1) + }) + + it('parses TSV when delimiter is tab', async () => { + const { headers, rows } = await parseCsvBuffer('a\tb\n1\t2', '\t') + expect(headers).toEqual(['a', 'b']) + expect(rows).toEqual([{ a: '1', b: '2' }]) + }) + + it('throws when the file has no data rows', async () => { + await expect(parseCsvBuffer('a,b')).rejects.toThrow(/no data rows/i) + }) + }) + + describe('inferColumnType', () => { + it('returns "string" for empty samples', () => { + expect(inferColumnType([])).toBe('string') + expect(inferColumnType([null, undefined, ''])).toBe('string') + }) + + it('detects numeric columns', () => { + expect(inferColumnType(['1', '2', '3.14'])).toBe('number') + }) + + it('detects boolean columns (case-insensitive)', () => { + expect(inferColumnType(['true', 'FALSE', 'True'])).toBe('boolean') + }) + + it('detects ISO date columns', () => { + expect(inferColumnType(['2024-01-01', '2024-02-01T12:00:00'])).toBe('date') + }) + + it('falls back to "string"', () => { + expect(inferColumnType(['abc', 'def'])).toBe('string') + expect(inferColumnType(['1', 'abc'])).toBe('string') + }) + }) + + describe('sanitizeName', () => { + it('strips unsupported chars and collapses underscores', () => { + expect(sanitizeName('Hello World!')).toBe('Hello_World') + expect(sanitizeName(' foo-bar ')).toBe('foo_bar') + }) + + it('prefixes names that start with a digit', () => { + expect(sanitizeName('123abc')).toBe('col_123abc') + }) + + it('fills in an empty name with the prefix', () => { + expect(sanitizeName('$$$')).toBe('col_') + }) + }) + + describe('inferSchemaFromCsv', () => { + it('produces sanitized column names and inferred types', () => { + const { columns, headerToColumn } = inferSchemaFromCsv( + ['First Name', 'Age', 'Active'], + [ + { 'First Name': 'Alice', Age: '30', Active: 'true' }, + { 'First Name': 'Bob', Age: '40', Active: 'false' }, + ] + ) + expect(columns).toEqual([ + { name: 'First_Name', type: 'string' }, + { name: 'Age', type: 'number' }, + { name: 'Active', type: 'boolean' }, + ]) + expect(headerToColumn.get('First Name')).toBe('First_Name') + expect(headerToColumn.get('Age')).toBe('Age') + }) + + it('disambiguates duplicate sanitized headers', () => { + const { columns } = inferSchemaFromCsv( + ['a b', 'a-b', 'a.b'], + [{ 'a b': '1', 'a-b': '2', 'a.b': '3' }] + ) + expect(columns.map((c) => c.name)).toEqual(['a_b', 'a_b_2', 'a_b_3']) + }) + }) + + describe('coerceValue', () => { + it('returns null for empty values', () => { + expect(coerceValue(null, 'string')).toBeNull() + expect(coerceValue(undefined, 'number')).toBeNull() + expect(coerceValue('', 'boolean')).toBeNull() + }) + + it('coerces numbers', () => { + expect(coerceValue('42', 'number')).toBe(42) + expect(coerceValue('not a number', 'number')).toBeNull() + }) + + it('coerces booleans strictly', () => { + expect(coerceValue('true', 'boolean')).toBe(true) + expect(coerceValue('FALSE', 'boolean')).toBe(false) + expect(coerceValue('yes', 'boolean')).toBeNull() + }) + + it('coerces dates to ISO strings and falls back to the original string', () => { + expect(coerceValue('2024-01-01', 'date')).toBe(new Date('2024-01-01').toISOString()) + expect(coerceValue('not-a-date', 'date')).toBe('not-a-date') + }) + }) + + describe('buildAutoMapping', () => { + const schema: TableSchema = { + columns: [ + { name: 'First_Name', type: 'string' }, + { name: 'age', type: 'number' }, + ], + } + + it('maps by exact sanitized name', () => { + const mapping = buildAutoMapping(['First_Name', 'age'], schema) + expect(mapping).toEqual({ First_Name: 'First_Name', age: 'age' }) + }) + + it('falls back to a case/punctuation-insensitive match', () => { + const mapping = buildAutoMapping(['first name', 'AGE'], schema) + expect(mapping).toEqual({ 'first name': 'First_Name', AGE: 'age' }) + }) + + it('returns null for headers without a match', () => { + const mapping = buildAutoMapping(['unmatched'], schema) + expect(mapping).toEqual({ unmatched: null }) + }) + }) + + describe('validateMapping', () => { + const schema: TableSchema = { + columns: [ + { name: 'name', type: 'string', required: true }, + { name: 'age', type: 'number' }, + ], + } + + it('accepts a valid mapping and lists skipped/unmapped', () => { + const result = validateMapping({ + csvHeaders: ['name', 'age', 'extra'], + mapping: { name: 'name', age: 'age', extra: null }, + tableSchema: schema, + }) + expect(result.mappedHeaders).toEqual(['name', 'age']) + expect(result.skippedHeaders).toEqual(['extra']) + expect(result.unmappedColumns).toEqual([]) + expect(result.effectiveMap.get('name')).toBe('name') + expect(result.effectiveMap.has('extra')).toBe(false) + }) + + it('throws when a required column is missing', () => { + expect(() => + validateMapping({ + csvHeaders: ['age'], + mapping: { age: 'age' }, + tableSchema: schema, + }) + ).toThrow(CsvImportValidationError) + }) + + it('throws when a mapping targets a non-existent column', () => { + expect(() => + validateMapping({ + csvHeaders: ['name'], + mapping: { name: 'nonexistent' }, + tableSchema: schema, + }) + ).toThrow(/do not exist on the table/) + }) + + it('throws when multiple headers map to the same column', () => { + expect(() => + validateMapping({ + csvHeaders: ['a', 'b'], + mapping: { a: 'name', b: 'name' }, + tableSchema: schema, + }) + ).toThrow(/same column/) + }) + + it('throws when mapping references an unknown CSV header', () => { + expect(() => + validateMapping({ + csvHeaders: ['name'], + mapping: { name: 'name', bogus: 'age' }, + tableSchema: schema, + }) + ).toThrow(/unknown CSV headers/) + }) + + it('throws when a mapping value is neither a string nor null', () => { + expect(() => + validateMapping({ + csvHeaders: ['name'], + mapping: { name: 42 as unknown as string }, + tableSchema: schema, + }) + ).toThrow(/Mapping values must be/) + }) + }) + + describe('coerceRowsForTable', () => { + const schema: TableSchema = { + columns: [ + { name: 'name', type: 'string' }, + { name: 'age', type: 'number' }, + { name: 'active', type: 'boolean' }, + ], + } + + it('applies the table column type using the effective mapping', () => { + const rows = coerceRowsForTable( + [ + { Name: 'Alice', Age: '30', Active: 'true' }, + { Name: 'Bob', Age: 'oops', Active: 'false' }, + ], + schema, + new Map([ + ['Name', 'name'], + ['Age', 'age'], + ['Active', 'active'], + ]) + ) + + expect(rows).toEqual([ + { name: 'Alice', age: 30, active: true }, + { name: 'Bob', age: null, active: false }, + ]) + }) + + it('drops CSV headers absent from the mapping', () => { + const rows = coerceRowsForTable( + [{ name: 'Alice', extra: 'keep me out' }], + schema, + new Map([['name', 'name']]) + ) + expect(rows).toEqual([{ name: 'Alice' }]) + }) + }) +}) diff --git a/apps/sim/lib/table/csv-import.ts b/apps/sim/lib/table/csv-import.ts new file mode 100644 index 00000000000..029da125fe1 --- /dev/null +++ b/apps/sim/lib/table/csv-import.ts @@ -0,0 +1,391 @@ +/** + * Shared CSV import helpers for user-defined tables. + * + * Used by: + * - `POST /api/table/import-csv` (create new table from CSV) + * - `POST /api/table/[tableId]/import-csv` (append/replace into existing table) + * - Copilot `user-table` tool (`create_from_file`, `import_file`) + * + * Keeping a single implementation avoids drift between HTTP and agent code paths. + */ + +import type { ColumnDefinition, RowData, TableSchema } from '@/lib/table/types' + +/** Narrower type than `COLUMN_TYPES` used internally for coercion. */ +export type CsvColumnType = 'string' | 'number' | 'boolean' | 'date' | 'json' + +/** Number of CSV rows sampled when inferring column types for a new table. */ +export const CSV_SCHEMA_SAMPLE_SIZE = 100 + +/** Maximum rows inserted per `batchInsertRows` call during import. */ +export const CSV_MAX_BATCH_SIZE = 1000 + +/** Maximum CSV/TSV file size accepted by import routes (50 MB). */ +export const CSV_MAX_FILE_SIZE_BYTES = 50 * 1024 * 1024 + +/** + * Error thrown when the user-supplied mapping or CSV does not line up with the + * target table. Callers should translate this into a 400 response. + */ +export class CsvImportValidationError extends Error { + readonly code = 'CSV_IMPORT_VALIDATION' as const + readonly details: { + missingRequired?: string[] + duplicateTargets?: string[] + unknownColumns?: string[] + unknownHeaders?: string[] + } + + constructor( + message: string, + details: { + missingRequired?: string[] + duplicateTargets?: string[] + unknownColumns?: string[] + unknownHeaders?: string[] + } = {} + ) { + super(message) + this.name = 'CsvImportValidationError' + this.details = details + } +} + +/** + * Parses a CSV/TSV payload using `csv-parse/sync`. Accepts a Node `Buffer`, + * browser-friendly `Uint8Array`, or already-decoded string. Strips a leading + * UTF-8 BOM so headers are not silently prefixed with `\uFEFF`. + */ +export async function parseCsvBuffer( + input: Buffer | Uint8Array | string, + delimiter = ',' +): Promise<{ headers: string[]; rows: Record[] }> { + const { parse } = await import('csv-parse/sync') + + let text: string + if (typeof input === 'string') { + text = input + } else if (typeof Buffer !== 'undefined' && Buffer.isBuffer(input)) { + text = input.toString('utf-8') + } else { + text = new TextDecoder('utf-8').decode(input as Uint8Array) + } + text = text.replace(/^\uFEFF/, '') + + const parsed = parse(text, { + columns: true, + skip_empty_lines: true, + trim: true, + relax_column_count: true, + relax_quotes: true, + skip_records_with_error: true, + cast: false, + delimiter, + }) as Record[] + + if (parsed.length === 0) { + throw new Error('CSV file has no data rows') + } + + const headers = Object.keys(parsed[0]) + if (headers.length === 0) { + throw new Error('CSV file has no headers') + } + + return { headers, rows: parsed } +} + +/** + * Infers a column type from a sample of non-empty values. Order matters: we + * prefer narrower types (number > boolean > ISO date) and fall back to string. + * JSON is never inferred automatically. + */ +export function inferColumnType(values: unknown[]): Exclude { + const nonEmpty = values.filter((v) => v !== null && v !== undefined && v !== '') + if (nonEmpty.length === 0) return 'string' + + const allNumber = nonEmpty.every((v) => { + const n = Number(v) + return !Number.isNaN(n) && String(v).trim() !== '' + }) + if (allNumber) return 'number' + + const allBoolean = nonEmpty.every((v) => { + const s = String(v).toLowerCase() + return s === 'true' || s === 'false' + }) + if (allBoolean) return 'boolean' + + const isoDatePattern = /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2})?)?/ + const allDate = nonEmpty.every((v) => { + const s = String(v) + return isoDatePattern.test(s) && !Number.isNaN(Date.parse(s)) + }) + if (allDate) return 'date' + + return 'string' +} + +/** + * Sanitizes a raw header into a valid column/table name. Strips disallowed + * characters, collapses runs of underscores, and ensures the first character + * is a letter or underscore (prefixing with `fallbackPrefix` otherwise). + */ +export function sanitizeName(raw: string, fallbackPrefix = 'col'): string { + let name = raw + .trim() + .replace(/[^a-zA-Z0-9_]/g, '_') + .replace(/_+/g, '_') + .replace(/^_+|_+$/g, '') + + if (!name || /^\d/.test(name)) { + name = `${fallbackPrefix}_${name}` + } + + return name +} + +/** + * Returns column definitions inferred from CSV headers + sample rows. Duplicate + * sanitized names are suffixed with `_2`, `_3`, etc. Also returns the header -> + * column-name mapping used when coercing row values. + */ +export function inferSchemaFromCsv( + headers: string[], + rows: Record[] +): { columns: ColumnDefinition[]; headerToColumn: Map } { + const sample = rows.slice(0, CSV_SCHEMA_SAMPLE_SIZE) + const seen = new Set() + const headerToColumn = new Map() + + const columns = headers.map((header) => { + const base = sanitizeName(header) + let colName = base + let suffix = 2 + while (seen.has(colName.toLowerCase())) { + colName = `${base}_${suffix}` + suffix++ + } + seen.add(colName.toLowerCase()) + headerToColumn.set(header, colName) + + return { + name: colName, + type: inferColumnType(sample.map((r) => r[header])), + } satisfies ColumnDefinition + }) + + return { columns, headerToColumn } +} + +/** + * Coerces a single value to the requested column type. Returns `null` for + * empty inputs or values that cannot be parsed (numbers/booleans). Dates fall + * back to the original string when unparseable so that schema validation can + * reject it with context rather than silently inserting `null`. + */ +export function coerceValue( + value: unknown, + colType: CsvColumnType +): string | number | boolean | null | Record | unknown[] { + if (value === null || value === undefined || value === '') return null + switch (colType) { + case 'number': { + const n = Number(value) + return Number.isNaN(n) ? null : n + } + case 'boolean': { + const s = String(value).toLowerCase() + if (s === 'true') return true + if (s === 'false') return false + return null + } + case 'date': { + const d = new Date(String(value)) + return Number.isNaN(d.getTime()) ? String(value) : d.toISOString() + } + case 'json': { + if (typeof value === 'object') return value as Record | unknown[] + try { + return JSON.parse(String(value)) + } catch { + return String(value) + } + } + default: + return String(value) + } +} + +/** + * Mapping from raw CSV header to target column name, with `null` indicating + * "do not import this column". + */ +export type CsvHeaderMapping = Record + +export interface CsvMappingValidationResult { + /** Columns present in the CSV that landed on a real table column. */ + mappedHeaders: string[] + /** Columns in the CSV that the user/client chose to skip. */ + skippedHeaders: string[] + /** Target column names that ended up unmapped (resolved from the mapping). */ + unmappedColumns: string[] + /** Effective header -> column map (after dropping unknown / null targets). */ + effectiveMap: Map +} + +/** + * Validates a user-supplied mapping against the target table schema. Rejects + * unknown target columns, duplicate targets, and required table columns that + * are not covered by the CSV. Returns the normalized header -> column map. + */ +export function validateMapping(params: { + csvHeaders: string[] + mapping: CsvHeaderMapping + tableSchema: TableSchema +}): CsvMappingValidationResult { + const { csvHeaders, mapping, tableSchema } = params + const columnByName = new Map(tableSchema.columns.map((c) => [c.name, c])) + + const unknownHeaders = Object.keys(mapping).filter((h) => !csvHeaders.includes(h)) + if (unknownHeaders.length > 0) { + throw new CsvImportValidationError( + `Mapping references unknown CSV headers: ${unknownHeaders.join(', ')}`, + { unknownHeaders } + ) + } + + const invalidTargets = Object.entries(mapping).filter( + ([, target]) => target !== null && typeof target !== 'string' + ) + if (invalidTargets.length > 0) { + throw new CsvImportValidationError( + `Mapping values must be a column name (string) or null, got: ${invalidTargets + .map(([header]) => header) + .join(', ')}` + ) + } + + const targetsSeen = new Map() + const unknownColumns: string[] = [] + const effectiveMap = new Map() + const skippedHeaders: string[] = [] + + for (const header of csvHeaders) { + const target = header in mapping ? mapping[header] : undefined + if (target === null || target === undefined) { + skippedHeaders.push(header) + continue + } + if (!columnByName.has(target)) { + unknownColumns.push(target) + continue + } + const existing = targetsSeen.get(target) ?? [] + existing.push(header) + targetsSeen.set(target, existing) + effectiveMap.set(header, target) + } + + if (unknownColumns.length > 0) { + throw new CsvImportValidationError( + `Mapping references columns that do not exist on the table: ${unknownColumns.join(', ')}`, + { unknownColumns } + ) + } + + const duplicateTargets = [...targetsSeen.entries()] + .filter(([, headers]) => headers.length > 1) + .map(([col]) => col) + if (duplicateTargets.length > 0) { + throw new CsvImportValidationError( + `Multiple CSV headers map to the same column(s): ${duplicateTargets.join(', ')}`, + { duplicateTargets } + ) + } + + const mappedTargets = new Set(effectiveMap.values()) + const unmappedColumns = tableSchema.columns + .filter((c) => !mappedTargets.has(c.name)) + .map((c) => c.name) + + const missingRequired = tableSchema.columns + .filter((c) => c.required && !mappedTargets.has(c.name)) + .map((c) => c.name) + if (missingRequired.length > 0) { + throw new CsvImportValidationError( + `CSV is missing required columns: ${missingRequired.join(', ')}`, + { missingRequired } + ) + } + + return { + mappedHeaders: [...effectiveMap.keys()], + skippedHeaders, + unmappedColumns, + effectiveMap, + } +} + +/** + * Builds an auto-mapping from CSV headers to table columns: prefers exact + * sanitized-name matches and falls back to a case- and punctuation-insensitive + * comparison. Unmapped headers are set to `null`. + */ +export function buildAutoMapping(csvHeaders: string[], tableSchema: TableSchema): CsvHeaderMapping { + const mapping: CsvHeaderMapping = {} + const columns = tableSchema.columns + + const exactByName = new Map(columns.map((c) => [c.name, c.name])) + const loose = new Map() + for (const col of columns) { + loose.set(col.name.toLowerCase().replace(/[^a-z0-9]/g, ''), col.name) + } + + const usedTargets = new Set() + + for (const header of csvHeaders) { + const sanitized = sanitizeName(header) + const exact = exactByName.get(sanitized) + if (exact && !usedTargets.has(exact)) { + mapping[header] = exact + usedTargets.add(exact) + continue + } + const key = header.toLowerCase().replace(/[^a-z0-9]/g, '') + const fuzzy = loose.get(key) + if (fuzzy && !usedTargets.has(fuzzy)) { + mapping[header] = fuzzy + usedTargets.add(fuzzy) + continue + } + mapping[header] = null + } + + return mapping +} + +/** + * Coerces parsed CSV rows into `RowData` objects keyed by target column name, + * applying the column types declared in `tableSchema`. Headers not present in + * `headerToColumn` are dropped. Missing table columns remain unset (schema + * validation decides whether that's acceptable). + */ +export function coerceRowsForTable( + rows: Record[], + tableSchema: TableSchema, + headerToColumn: Map +): RowData[] { + const typeByName = new Map(tableSchema.columns.map((c) => [c.name, c.type as CsvColumnType])) + + return rows.map((row) => { + const coerced: RowData = {} + for (const [header, value] of Object.entries(row)) { + const colName = headerToColumn.get(header) + if (!colName) continue + const colType = typeByName.get(colName) ?? 'string' + coerced[colName] = coerceValue(value, colType) as RowData[string] + } + return coerced + }) +} diff --git a/apps/sim/lib/table/index.ts b/apps/sim/lib/table/index.ts index 5e960d09fd3..a384f30e6a6 100644 --- a/apps/sim/lib/table/index.ts +++ b/apps/sim/lib/table/index.ts @@ -7,6 +7,7 @@ export * from './billing' export * from './constants' +export * from './csv-import' export * from './llm' export * from './query-builder' export * from './service' diff --git a/apps/sim/lib/table/service.ts b/apps/sim/lib/table/service.ts index 5a874be671b..cfa1f5dbecb 100644 --- a/apps/sim/lib/table/service.ts +++ b/apps/sim/lib/table/service.ts @@ -30,6 +30,8 @@ import type { QueryOptions, QueryResult, RenameColumnData, + ReplaceRowsData, + ReplaceRowsResult, RowData, TableDefinition, TableMetadata, @@ -776,6 +778,120 @@ export async function batchInsertRows( })) } +/** + * Replaces all rows in a table with a new set of rows. Deletes existing rows + * and inserts the provided rows inside a single transaction so the table is + * never observed in an empty intermediate state by other readers. + * + * Validates each row against the schema, enforces unique constraints within the + * new rows (existing rows are deleted, so DB-side checks are unnecessary), and + * enforces `maxRows` before the replace executes. + * + * @param data - Replace data (rows to install) + * @param table - Table definition + * @param requestId - Request ID for logging + * @returns Count of rows deleted and inserted + * @throws Error if validation fails or capacity exceeded + */ +export async function replaceTableRows( + data: ReplaceRowsData, + table: TableDefinition, + requestId: string +): Promise { + if (data.tableId !== table.id) { + throw new Error(`Table ID mismatch: ${data.tableId} vs ${table.id}`) + } + if (data.workspaceId !== table.workspaceId) { + throw new Error(`Workspace ID mismatch: ${data.workspaceId} does not own table ${data.tableId}`) + } + if (data.rows.length > table.maxRows) { + throw new Error( + `Cannot replace: ${data.rows.length} rows exceeds table row limit (${table.maxRows})` + ) + } + + for (let i = 0; i < data.rows.length; i++) { + const row = data.rows[i] + + const sizeValidation = validateRowSize(row) + if (!sizeValidation.valid) { + throw new Error(`Row ${i + 1}: ${sizeValidation.errors.join(', ')}`) + } + + const schemaValidation = validateRowAgainstSchema(row, table.schema) + if (!schemaValidation.valid) { + throw new Error(`Row ${i + 1}: ${schemaValidation.errors.join(', ')}`) + } + } + + const uniqueColumns = getUniqueColumns(table.schema) + if (uniqueColumns.length > 0 && data.rows.length > 0) { + const seen = new Map>() + for (const col of uniqueColumns) { + seen.set(col.name, new Map()) + } + for (let i = 0; i < data.rows.length; i++) { + const row = data.rows[i] + for (const col of uniqueColumns) { + const value = row[col.name] + if (value === null || value === undefined) continue + const normalized = typeof value === 'string' ? value.toLowerCase() : JSON.stringify(value) + const map = seen.get(col.name)! + if (map.has(normalized)) { + throw new Error( + `Row ${i + 1}: Column "${col.name}" must be unique. Value "${String(value)}" duplicates row ${map.get(normalized)! + 1} in batch` + ) + } + map.set(normalized, i) + } + } + } + + const now = new Date() + + const result = await db.transaction(async (trx) => { + await trx.execute( + sql`SELECT 1 FROM user_table_definitions WHERE id = ${data.tableId} FOR UPDATE` + ) + + const deletedRows = await trx + .delete(userTableRows) + .where(eq(userTableRows.tableId, data.tableId)) + .returning({ id: userTableRows.id }) + + let insertedCount = 0 + if (data.rows.length > 0) { + const rowsToInsert = data.rows.map((rowData, i) => ({ + id: `row_${generateId().replace(/-/g, '')}`, + tableId: data.tableId, + workspaceId: data.workspaceId, + data: rowData, + position: i, + createdAt: now, + updatedAt: now, + ...(data.userId ? { createdBy: data.userId } : {}), + })) + + const batchSize = TABLE_LIMITS.MAX_BATCH_INSERT_SIZE + for (let i = 0; i < rowsToInsert.length; i += batchSize) { + const chunk = rowsToInsert.slice(i, i + batchSize) + const inserted = await trx.insert(userTableRows).values(chunk).returning({ + id: userTableRows.id, + }) + insertedCount += inserted.length + } + } + + return { deletedCount: deletedRows.length, insertedCount } + }) + + logger.info( + `[${requestId}] Replaced rows in table ${data.tableId}: deleted ${result.deletedCount}, inserted ${result.insertedCount}` + ) + + return result +} + /** * Upserts a row: updates an existing row if a match is found on the conflict target * column, otherwise inserts a new row. diff --git a/apps/sim/lib/table/types.ts b/apps/sim/lib/table/types.ts index ea97b292152..6b361d24310 100644 --- a/apps/sim/lib/table/types.ts +++ b/apps/sim/lib/table/types.ts @@ -241,6 +241,18 @@ export interface BulkDeleteByIdsResult { missingRowIds: string[] } +export interface ReplaceRowsData { + tableId: string + rows: RowData[] + workspaceId: string + userId?: string +} + +export interface ReplaceRowsResult { + deletedCount: number + insertedCount: number +} + export interface RenameColumnData { tableId: string oldName: string From 630f4a18018ee205db8163cbd36e862ccd34345a Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Thu, 16 Apr 2026 12:23:03 -0700 Subject: [PATCH 2/4] update types --- apps/sim/lib/copilot/generated/tool-catalog-v1.ts | 5 +++++ apps/sim/lib/copilot/generated/tool-schemas-v1.ts | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index 2d3930bc54d..df406483840 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -2834,6 +2834,11 @@ export const UserTable: ToolCatalogEntry = { type: 'object', description: 'Optional explicit CSV-header → table-column mapping for import_file, as { "csvHeader": "columnName" | null }. When omitted, headers are auto-matched by sanitized name (case-insensitive fallback). Use null to skip a CSV column.', + additionalProperties: { + type: 'string', + description: + 'Target column name on the table. Use null to skip this CSV header instead of a column name.', + }, }, mode: { type: 'string', diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index fdf139dd72c..95538651008 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -2639,6 +2639,11 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { type: 'object', description: 'Optional explicit CSV-header → table-column mapping for import_file, as { "csvHeader": "columnName" | null }. When omitted, headers are auto-matched by sanitized name (case-insensitive fallback). Use null to skip a CSV column.', + additionalProperties: { + type: 'string', + description: + 'Target column name on the table. Use null to skip this CSV header instead of a column name.', + }, }, mode: { type: 'string', From 6bf67c4445572f5bfb22d5a6802ecd8a521b773d Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Thu, 16 Apr 2026 12:28:28 -0700 Subject: [PATCH 3/4] address comments --- .../lib/copilot/tools/server/table/user-table.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/apps/sim/lib/copilot/tools/server/table/user-table.ts b/apps/sim/lib/copilot/tools/server/table/user-table.ts index af545b4baff..366df11ae7e 100644 --- a/apps/sim/lib/copilot/tools/server/table/user-table.ts +++ b/apps/sim/lib/copilot/tools/server/table/user-table.ts @@ -153,11 +153,16 @@ async function batchInsertAll( context?: ServerToolContext ): Promise { let inserted = 0 + const userId = context?.userId for (let i = 0; i < rows.length; i += MAX_BATCH_SIZE) { assertServerToolNotAborted(context, 'Request aborted before table mutation could be applied.') const batch = rows.slice(i, i + MAX_BATCH_SIZE) const requestId = generateId().slice(0, 8) - const result = await batchInsertRows({ tableId, rows: batch, workspaceId }, table, requestId) + const result = await batchInsertRows( + { tableId, rows: batch, workspaceId, userId }, + table, + requestId + ) inserted += result.length } return inserted @@ -297,7 +302,7 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) assertNotAborted() const row = await insertRow( - { tableId: args.tableId, data: args.data, workspaceId }, + { tableId: args.tableId, data: args.data, workspaceId, userId: context.userId }, table, requestId ) @@ -328,7 +333,7 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) assertNotAborted() const rows = await batchInsertRows( - { tableId: args.tableId, rows: args.rows, workspaceId }, + { tableId: args.tableId, rows: args.rows, workspaceId, userId: context.userId }, table, requestId ) @@ -749,7 +754,7 @@ export const userTableServerTool: BaseServerTool assertNotAborted() const requestId = generateId().slice(0, 8) const result = await replaceTableRows( - { tableId: table.id, rows: coerced, workspaceId }, + { tableId: table.id, rows: coerced, workspaceId, userId: context.userId }, table, requestId ) From 1a45b8ee0ecb50375db015b984c6e9184aec9d25 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Thu, 16 Apr 2026 13:28:38 -0700 Subject: [PATCH 4/4] address comment --- .../lib/copilot/request/tools/files.test.ts | 23 +++++++++++-------- apps/sim/lib/copilot/request/tools/files.ts | 18 ++++++++------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/apps/sim/lib/copilot/request/tools/files.test.ts b/apps/sim/lib/copilot/request/tools/files.test.ts index 10bf4e57356..e72d6455f89 100644 --- a/apps/sim/lib/copilot/request/tools/files.test.ts +++ b/apps/sim/lib/copilot/request/tools/files.test.ts @@ -72,19 +72,18 @@ describe('serializeOutputForFile (json / txt / md)', () => { }) describe('extractTabularData', () => { - it('extracts rows from function_execute { result: [...] } via unwrap', () => { - const rows = extractTabularData({ - result: [{ a: 1 }, { a: 2 }], - stdout: '', - }) - expect(rows).toEqual([{ a: 1 }, { a: 2 }]) + it('extracts rows directly from an array input', () => { + expect(extractTabularData([{ a: 1 }, { a: 2 }])).toEqual([{ a: 1 }, { a: 2 }]) }) - it('returns null when function_execute result is a non-tabular string', () => { - expect(extractTabularData({ result: 'name,age\nAlice,30', stdout: '' })).toBeNull() + it('does NOT unwrap function_execute envelopes on its own (callers must pre-unwrap)', () => { + // Caller is responsible for unwrapping { result, stdout } envelopes first. + // Keeping that concern out of this function prevents a double unwrap when + // the user's payload itself happens to have matching keys. + expect(extractTabularData({ result: [{ a: 1 }], stdout: '' })).toBeNull() }) - it('still extracts rows from the user_table query_rows shape', () => { + it('extracts rows from the user_table query_rows shape', () => { const rows = extractTabularData({ data: { rows: [ @@ -96,4 +95,10 @@ describe('extractTabularData', () => { }) expect(rows).toEqual([{ name: 'Alice' }, { name: 'Bob' }]) }) + + it('returns null for non-tabular inputs', () => { + expect(extractTabularData('plain string')).toBeNull() + expect(extractTabularData(null)).toBeNull() + expect(extractTabularData({ foo: 'bar' })).toBeNull() + }) }) diff --git a/apps/sim/lib/copilot/request/tools/files.ts b/apps/sim/lib/copilot/request/tools/files.ts index 1e26303c777..51c4e307c8c 100644 --- a/apps/sim/lib/copilot/request/tools/files.ts +++ b/apps/sim/lib/copilot/request/tools/files.ts @@ -44,21 +44,23 @@ export function unwrapFunctionExecuteOutput(output: unknown): unknown { } /** - * Try to pull a flat array of row-objects out of the various shapes that - * `function_execute` and `user_table` can return. + * Try to pull a flat array of row-objects out of an already-unwrapped tool + * payload. Callers are responsible for stripping any `function_execute` + * envelope first (via {@link unwrapFunctionExecuteOutput}) — this function + * does not re-unwrap, so a user payload that coincidentally has `result` and + * `stdout` keys is not mistaken for another envelope. */ export function extractTabularData(output: unknown): Record[] | null { - const unwrapped = unwrapFunctionExecuteOutput(output) - if (!unwrapped || typeof unwrapped !== 'object') return null + if (!output || typeof output !== 'object') return null - if (Array.isArray(unwrapped)) { - if (unwrapped.length > 0 && typeof unwrapped[0] === 'object' && unwrapped[0] !== null) { - return unwrapped as Record[] + if (Array.isArray(output)) { + if (output.length > 0 && typeof output[0] === 'object' && output[0] !== null) { + return output as Record[] } return null } - const obj = unwrapped as Record + const obj = output as Record // user_table query_rows shape: { data: { rows: [{ data: {...} }], totalCount } } if (obj.data && typeof obj.data === 'object' && !Array.isArray(obj.data)) {