feat(clickhouse): infer mixed-type JSON arrays as Array(Dynamic) on insert

.server-changes/clickhouse-array-of-dynamic-inference.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: fix
+---
+
+Infer mixed-type JSON arrays as Array(Dynamic) instead of nested tuples when writing run, event, metric, and session data to avoid ClickHouse type-complexity merge failures

internal-packages/clickhouse/src/metrics.ts

@@ -22,6 +22,7 @@ export function insertMetrics(ch: ClickhouseWriter) {
     settings: {
       enable_json_type: 1,
       type_json_skip_duplicated_paths: 1,
+      input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
       input_format_json_throw_on_bad_escape_sequence: 0,
       input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects: 1,
     },

internal-packages/clickhouse/src/sessions.ts

@@ -118,6 +118,7 @@ export function insertSessionsCompactArrays(ch: ClickhouseWriter, settings?: Cli
     settings: {
       enable_json_type: 1,
       type_json_skip_duplicated_paths: 1,
+      input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
       ...settings,
     },
   });
@@ -131,6 +132,7 @@ export function insertSessions(ch: ClickhouseWriter, settings?: ClickHouseSettin
     settings: {
       enable_json_type: 1,
       type_json_skip_duplicated_paths: 1,
+      input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
       ...settings,
     },
   });

internal-packages/clickhouse/src/taskEvents.ts

@@ -31,6 +31,7 @@ export function insertTaskEvents(ch: ClickhouseWriter, settings?: ClickHouseSett
     settings: {
       enable_json_type: 1,
       type_json_skip_duplicated_paths: 1,
+      input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
       input_format_json_throw_on_bad_escape_sequence: 0,
       input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects: 1,
       ...settings,
@@ -206,6 +207,7 @@ export function insertTaskEventsV2(ch: ClickhouseWriter, settings?: ClickHouseSe
     settings: {
       enable_json_type: 1,
       type_json_skip_duplicated_paths: 1,
+      input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
       input_format_json_throw_on_bad_escape_sequence: 0,
       input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects: 1,
       ...settings,

internal-packages/clickhouse/src/taskRuns.test.ts

@@ -156,6 +156,111 @@ describe("Task Runs V2", () => {
     );
   });
 
+  clickhouseTest(
+    "should insert and read back JSON arrays with mixed element types",
+    async ({ clickhouseContainer }) => {
+      // Regression test for input_format_json_infer_array_of_dynamic_from_array_of_different_types.
+      // Arrays with mixed element types (e.g. [1, "hello", {...}, [...]]) must be inferred as
+      // Array(Dynamic) rather than deeply nested Tuple types, which otherwise blow up the binary
+      // type-complexity limit during background merges (ClickHouse Code 117).
+      const client = new ClickhouseClient({
+        name: "test",
+        url: clickhouseContainer.getConnectionUrl(),
+      });
+
+      const insert = insertTaskRunsCompactArrays(client, {
+        async_insert: 0, // turn off async insert for this test
+      });
+
+      const mixedArray = [1, "hello", { nested: "object" }, [1, 2, 3]];
+
+      const now = Date.now();
+      const taskRunData: TaskRunInsertArray = [
+        "env_mixed", // environment_id
+        "org_mixed", // organization_id
+        "project_mixed", // project_id
+        "run_mixed", // run_id
+        now, // updated_at
+        now, // created_at
+        "COMPLETED_SUCCESSFULLY", // status
+        "DEVELOPMENT", // environment_type
+        "friendly_mixed", // friendly_id
+        1, // attempt
+        "V2", // engine
+        "my-task", // task_identifier
+        "my-queue", // queue
+        "", // schedule_id
+        "", // batch_id
+        null, // completed_at
+        null, // started_at
+        null, // executed_at
+        null, // delay_until
+        null, // queued_at
+        null, // expired_at
+        0, // usage_duration_ms
+        0, // cost_in_cents
+        0, // base_cost_in_cents
+        { data: { items: mixedArray } }, // output
+        { data: null }, // error
+        "", // error_fingerprint
+        [], // tags
+        "", // task_version
+        "", // sdk_version
+        "", // cli_version
+        "", // machine_preset
+        "", // root_run_id
+        "", // parent_run_id
+        0, // depth
+        "span_mixed", // span_id
+        "trace_mixed", // trace_id
+        "", // idempotency_key
+        "", // idempotency_key_user
+        "", // idempotency_key_scope
+        "", // expiration_ttl
+        true, // is_test
+        "1", // _version
+        0, // _is_deleted
+        "", // concurrency_key
+        [], // bulk_action_group_ids
+        "", // worker_queue
+        "", // region
+        "", // plan_type
+        null, // max_duration_in_seconds
+        "", // trigger_source
+        "", // root_trigger_source
+        "", // task_kind
+        null, // is_warm_start
+      ];
+
+      const [insertError, insertResult] = await insert([taskRunData]);
+
+      expect(insertError).toBeNull();
+      expect(insertResult).toEqual(expect.objectContaining({ executed: true }));
+      expect(insertResult?.summary?.written_rows).toEqual("1");
+
+      // output_text is a materialized String column that extracts the `data` field, so it
+      // round-trips the mixed-type array back out as JSON regardless of the internal storage type.
+      const query = client.query({
+        name: "query-task-runs-mixed",
+        query:
+          "SELECT run_id, output_text FROM trigger_dev.task_runs_v2 WHERE run_id = {run_id: String}",
+        schema: z.object({
+          run_id: z.string(),
+          output_text: z.string(),
+        }),
+        params: z.object({
+          run_id: z.string(),
+        }),
+      });
+
+      const [queryError, result] = await query({ run_id: "run_mixed" });
+
+      expect(queryError).toBeNull();
+      expect(result).toHaveLength(1);
+      expect(JSON.parse(result![0].output_text)).toEqual({ items: mixedArray });
+    }
+  );
+
   clickhouseTest("should deduplicate on the _version column", async ({ clickhouseContainer }) => {
     const client = new ClickhouseClient({
       name: "test",

internal-packages/clickhouse/src/taskRuns.ts

@@ -211,6 +211,7 @@ export function insertTaskRunsCompactArrays(ch: ClickhouseWriter, settings?: Cli
     settings: {
       enable_json_type: 1,
       type_json_skip_duplicated_paths: 1,
+      input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
       ...settings,
     },
   });
@@ -225,6 +226,7 @@ export function insertTaskRuns(ch: ClickhouseWriter, settings?: ClickHouseSettin
     settings: {
       enable_json_type: 1,
       type_json_skip_duplicated_paths: 1,
+      input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
       ...settings,
     },
   });
@@ -349,6 +351,7 @@ export function insertRawTaskRunPayloadsCompactArrays(
       async_insert_busy_timeout_ms: 1000,
       enable_json_type: 1,
       type_json_skip_duplicated_paths: 1,
+      input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
       ...settings,
     },
   });
@@ -367,6 +370,7 @@ export function insertRawTaskRunPayloads(ch: ClickhouseWriter, settings?: ClickH
       async_insert_busy_timeout_ms: 1000,
       enable_json_type: 1,
       type_json_skip_duplicated_paths: 1,
+      input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
       ...settings,
     },
   });