diff --git a/.server-changes/clickhouse-array-of-dynamic-inference.md b/.server-changes/clickhouse-array-of-dynamic-inference.md new file mode 100644 index 0000000000..fad5f3ad81 --- /dev/null +++ b/.server-changes/clickhouse-array-of-dynamic-inference.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Infer mixed-type JSON arrays as Array(Dynamic) instead of nested tuples when writing run, event, metric, and session data to avoid ClickHouse type-complexity merge failures diff --git a/internal-packages/clickhouse/src/metrics.ts b/internal-packages/clickhouse/src/metrics.ts index 0367280f8d..7e03086179 100644 --- a/internal-packages/clickhouse/src/metrics.ts +++ b/internal-packages/clickhouse/src/metrics.ts @@ -22,6 +22,7 @@ export function insertMetrics(ch: ClickhouseWriter) { settings: { enable_json_type: 1, type_json_skip_duplicated_paths: 1, + input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1, input_format_json_throw_on_bad_escape_sequence: 0, input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects: 1, }, diff --git a/internal-packages/clickhouse/src/sessions.ts b/internal-packages/clickhouse/src/sessions.ts index bebb8b1549..81cf8badce 100644 --- a/internal-packages/clickhouse/src/sessions.ts +++ b/internal-packages/clickhouse/src/sessions.ts @@ -118,6 +118,7 @@ export function insertSessionsCompactArrays(ch: ClickhouseWriter, settings?: Cli settings: { enable_json_type: 1, type_json_skip_duplicated_paths: 1, + input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1, ...settings, }, }); @@ -131,6 +132,7 @@ export function insertSessions(ch: ClickhouseWriter, settings?: ClickHouseSettin settings: { enable_json_type: 1, type_json_skip_duplicated_paths: 1, + input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1, ...settings, }, }); diff --git a/internal-packages/clickhouse/src/taskEvents.ts b/internal-packages/clickhouse/src/taskEvents.ts index 4c17d5068d..9c53db31be 100644 --- a/internal-packages/clickhouse/src/taskEvents.ts +++ b/internal-packages/clickhouse/src/taskEvents.ts @@ -31,6 +31,7 @@ export function insertTaskEvents(ch: ClickhouseWriter, settings?: ClickHouseSett settings: { enable_json_type: 1, type_json_skip_duplicated_paths: 1, + input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1, input_format_json_throw_on_bad_escape_sequence: 0, input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects: 1, ...settings, @@ -206,6 +207,7 @@ export function insertTaskEventsV2(ch: ClickhouseWriter, settings?: ClickHouseSe settings: { enable_json_type: 1, type_json_skip_duplicated_paths: 1, + input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1, input_format_json_throw_on_bad_escape_sequence: 0, input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects: 1, ...settings, diff --git a/internal-packages/clickhouse/src/taskRuns.test.ts b/internal-packages/clickhouse/src/taskRuns.test.ts index 0d4ec995c2..00dcd4bfba 100644 --- a/internal-packages/clickhouse/src/taskRuns.test.ts +++ b/internal-packages/clickhouse/src/taskRuns.test.ts @@ -156,6 +156,111 @@ describe("Task Runs V2", () => { ); }); + clickhouseTest( + "should insert and read back JSON arrays with mixed element types", + async ({ clickhouseContainer }) => { + // Regression test for input_format_json_infer_array_of_dynamic_from_array_of_different_types. + // Arrays with mixed element types (e.g. [1, "hello", {...}, [...]]) must be inferred as + // Array(Dynamic) rather than deeply nested Tuple types, which otherwise blow up the binary + // type-complexity limit during background merges (ClickHouse Code 117). + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const insert = insertTaskRunsCompactArrays(client, { + async_insert: 0, // turn off async insert for this test + }); + + const mixedArray = [1, "hello", { nested: "object" }, [1, 2, 3]]; + + const now = Date.now(); + const taskRunData: TaskRunInsertArray = [ + "env_mixed", // environment_id + "org_mixed", // organization_id + "project_mixed", // project_id + "run_mixed", // run_id + now, // updated_at + now, // created_at + "COMPLETED_SUCCESSFULLY", // status + "DEVELOPMENT", // environment_type + "friendly_mixed", // friendly_id + 1, // attempt + "V2", // engine + "my-task", // task_identifier + "my-queue", // queue + "", // schedule_id + "", // batch_id + null, // completed_at + null, // started_at + null, // executed_at + null, // delay_until + null, // queued_at + null, // expired_at + 0, // usage_duration_ms + 0, // cost_in_cents + 0, // base_cost_in_cents + { data: { items: mixedArray } }, // output + { data: null }, // error + "", // error_fingerprint + [], // tags + "", // task_version + "", // sdk_version + "", // cli_version + "", // machine_preset + "", // root_run_id + "", // parent_run_id + 0, // depth + "span_mixed", // span_id + "trace_mixed", // trace_id + "", // idempotency_key + "", // idempotency_key_user + "", // idempotency_key_scope + "", // expiration_ttl + true, // is_test + "1", // _version + 0, // _is_deleted + "", // concurrency_key + [], // bulk_action_group_ids + "", // worker_queue + "", // region + "", // plan_type + null, // max_duration_in_seconds + "", // trigger_source + "", // root_trigger_source + "", // task_kind + null, // is_warm_start + ]; + + const [insertError, insertResult] = await insert([taskRunData]); + + expect(insertError).toBeNull(); + expect(insertResult).toEqual(expect.objectContaining({ executed: true })); + expect(insertResult?.summary?.written_rows).toEqual("1"); + + // output_text is a materialized String column that extracts the `data` field, so it + // round-trips the mixed-type array back out as JSON regardless of the internal storage type. + const query = client.query({ + name: "query-task-runs-mixed", + query: + "SELECT run_id, output_text FROM trigger_dev.task_runs_v2 WHERE run_id = {run_id: String}", + schema: z.object({ + run_id: z.string(), + output_text: z.string(), + }), + params: z.object({ + run_id: z.string(), + }), + }); + + const [queryError, result] = await query({ run_id: "run_mixed" }); + + expect(queryError).toBeNull(); + expect(result).toHaveLength(1); + expect(JSON.parse(result![0].output_text)).toEqual({ items: mixedArray }); + } + ); + clickhouseTest("should deduplicate on the _version column", async ({ clickhouseContainer }) => { const client = new ClickhouseClient({ name: "test", diff --git a/internal-packages/clickhouse/src/taskRuns.ts b/internal-packages/clickhouse/src/taskRuns.ts index 67dd0371f1..cbcd6260b4 100644 --- a/internal-packages/clickhouse/src/taskRuns.ts +++ b/internal-packages/clickhouse/src/taskRuns.ts @@ -211,6 +211,7 @@ export function insertTaskRunsCompactArrays(ch: ClickhouseWriter, settings?: Cli settings: { enable_json_type: 1, type_json_skip_duplicated_paths: 1, + input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1, ...settings, }, }); @@ -225,6 +226,7 @@ export function insertTaskRuns(ch: ClickhouseWriter, settings?: ClickHouseSettin settings: { enable_json_type: 1, type_json_skip_duplicated_paths: 1, + input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1, ...settings, }, }); @@ -349,6 +351,7 @@ export function insertRawTaskRunPayloadsCompactArrays( async_insert_busy_timeout_ms: 1000, enable_json_type: 1, type_json_skip_duplicated_paths: 1, + input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1, ...settings, }, }); @@ -367,6 +370,7 @@ export function insertRawTaskRunPayloads(ch: ClickhouseWriter, settings?: ClickH async_insert_busy_timeout_ms: 1000, enable_json_type: 1, type_json_skip_duplicated_paths: 1, + input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1, ...settings, }, });