From 0328db3e520c06a664b968b56208fad5f74ef989 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Sun, 19 Jul 2026 13:25:57 +0000 Subject: [PATCH 1/4] feat(client-v2, jdbc-v2): add QBit data type support QBit(element_type, dimension) is transmitted over RowBinary exactly like Array(element_type) - a var-int length followed by that many element values - so it is read and written as a Java array of its element type (float[] for BFloat16/Float32, double[] for Float64) reusing the existing array codec, while keeping dataType == QBit for metadata fidelity. In jdbc-v2 QBit maps to java.sql.Types.ARRAY and is exposed as a java.sql.Array. Element types: BFloat16, Float32, Float64. The bit-transposed on-disk layout is a server storage detail and is not exposed to the client. Implements: https://github.com/ClickHouse/clickhouse-java/issues/2610 --- CHANGELOG.md | 8 ++ .../com/clickhouse/data/ClickHouseColumn.java | 32 +++++++ .../clickhouse/data/ClickHouseColumnTest.java | 29 ++++++ .../internal/BinaryStreamReader.java | 2 + .../internal/SerializerUtils.java | 2 + .../api/internal/DataTypeConverter.java | 2 + .../client/datatypes/DataTypeTests.java | 93 ++++++++++++++++++- docs/features.md | 3 + .../clickhouse/jdbc/internal/JdbcUtils.java | 2 +- .../java/com/clickhouse/jdbc/types/Array.java | 4 +- .../clickhouse/jdbc/JdbcDataTypeTests.java | 55 +++++++++++ 11 files changed, 229 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4f9aabad..c34909c67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,14 @@ `getFloat`/`setFloat` and `getObject` accessors, and reported as such by `ResultSetMetaData` and `DatabaseMetaData`. Previously reading or writing a `BFloat16` column failed with an unsupported-data-type error. (https://github.com/ClickHouse/clickhouse-java/issues/2279) +- **[client-v2, jdbc-v2]** Added support for the experimental `QBit(element_type, dimension)` vector data type + (ClickHouse `25.10+`; the `allow_experimental_qbit_type` server setting is required to create a column), with + `BFloat16`, `Float32`, or `Float64` element types. A `QBit` value is transmitted over `RowBinary` exactly like + `Array(element_type)`, so it is read and written as a Java array of the element type (`float[]` for + `BFloat16`/`Float32`, `double[]` for `Float64`) through generic records, binary readers, and POJO binding. In the + JDBC driver (`jdbc-v2`) `QBit` maps to `java.sql.Types.ARRAY` and is returned as a `java.sql.Array` from + `getObject`/`getArray`. Previously `QBit` was an unimplemented type constant and reading or writing such a column + failed. (https://github.com/ClickHouse/clickhouse-java/issues/2610) - **[client-v2, jdbc-v2]** Added TLS cipher suite selection. `Client.Builder.setSSLCipherSuites(String...)` (client-v2) and the comma-separated `ssl_cipher_suites` connection property (client-v2 and jdbc-v2) restrict the cipher suites enabled on secure connections; when unset, the transport defaults are used. Cipher-suite selection is independent of the diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java index 64ed857cc..b4118d7c3 100644 --- a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java @@ -73,6 +73,7 @@ public final class ClickHouseColumn implements Serializable { private static final String KEYWORD_NESTED = ClickHouseDataType.Nested.name(); private static final String KEYWORD_VARIANT = ClickHouseDataType.Variant.name(); private static final String KEYWORD_JSON = ClickHouseDataType.JSON.name(); + private static final String KEYWORD_QBIT = ClickHouseDataType.QBit.name(); private int columnCount; private int columnIndex; @@ -146,6 +147,17 @@ private static ClickHouseColumn update(ClickHouseColumn column) { } } break; + case QBit: + // QBit(element_type, dimension) is a one-level array of its element type on the + // wire; the dimension parameter is kept as the column precision. + if (!column.nested.isEmpty()) { + column.arrayLevel = 1; + column.arrayBaseColumn = column.nested.get(0); + } + if (size > 1) { + column.precision = Integer.parseInt(column.parameters.get(1).trim()); + } + break; case Bool: column.template = ClickHouseBoolValue.ofNull(); break; @@ -567,6 +579,26 @@ protected static int readColumn(String args, int startIndex, int len, String nam fixedLength = false; estimatedLength++; } + } else if (args.startsWith(KEYWORD_QBIT, i)) { + int index = args.indexOf('(', i + KEYWORD_QBIT.length()); + if (index < i) { + throw new IllegalArgumentException(ERROR_MISSING_NESTED_TYPE); + } + List params = new LinkedList<>(); + i = ClickHouseUtils.readParameters(args, index, len, params); + if (params.size() < 2) { + throw new IllegalArgumentException( + "QBit requires an element type and a dimension, e.g. QBit(Float32, 8)"); + } + // QBit(element_type, dimension) is transmitted over RowBinary exactly like + // Array(element_type): a var-int length followed by that many element values. The + // first parameter is the element type and drives the nested (item) column. + List nestedColumns = new LinkedList<>(); + nestedColumns.add(ClickHouseColumn.of("", params.get(0))); + column = new ClickHouseColumn(ClickHouseDataType.QBit, name, args.substring(startIndex, i), + nullable, lowCardinality, params, nestedColumns); + fixedLength = false; + estimatedLength++; } if (column == null) { diff --git a/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java index 6a3508b11..fb6cd89fe 100644 --- a/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java @@ -508,4 +508,33 @@ public Object[][] testJSONBinaryFormat_dp() { {"JSON(max_dynamic_types=3,max_dynamic_paths=3, SKIP REGEXP '^-.*',SKIP ff, flags Array(Array(Array(Int8))), SKIP alt_count)", 2, Arrays.asList("flags")}, }; } + + @DataProvider(name = "qbitTypesProvider") + private static Object[][] qbitTypesProvider() { + return new Object[][] { + { "QBit(BFloat16, 4)", ClickHouseDataType.BFloat16, 4 }, + { "QBit(Float32, 8)", ClickHouseDataType.Float32, 8 }, + { "QBit(Float64, 1536)", ClickHouseDataType.Float64, 1536 }, + }; + } + + @Test(groups = { "unit" }, dataProvider = "qbitTypesProvider") + public void testParseQBit(String typeName, ClickHouseDataType elementType, int dimension) { + ClickHouseColumn column = ClickHouseColumn.of("vec", typeName); + Assert.assertEquals(column.getDataType(), ClickHouseDataType.QBit); + Assert.assertEquals(column.getOriginalTypeName(), typeName); + // QBit(element_type, dimension) is a one-level array of its element type on the wire. + Assert.assertEquals(column.getArrayNestedLevel(), 1); + Assert.assertNotNull(column.getArrayBaseColumn()); + Assert.assertEquals(column.getArrayBaseColumn().getDataType(), elementType); + Assert.assertEquals(column.getNestedColumns().size(), 1); + Assert.assertEquals(column.getNestedColumns().get(0).getDataType(), elementType); + // The fixed vector dimension is retained as the column precision. + Assert.assertEquals(column.getPrecision(), dimension); + } + + @Test(groups = { "unit" }, expectedExceptions = IllegalArgumentException.class) + public void testParseQBitRequiresDimension() { + ClickHouseColumn.of("vec", "QBit(Float32)"); + } } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java index 782b6d0d1..8f6e2d448 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java @@ -234,6 +234,8 @@ public T readValue(ClickHouseColumn column, Class typeHint) throws IOExce return (T) readJsonData(input, actualColumn); } // case Object: // deprecated https://clickhouse.com/docs/en/sql-reference/data-types/object-data-type + case QBit: + // QBit(element_type, dimension) is transmitted like Array(element_type). case Array: if (typeHint == null) { typeHint = arrayDefaultTypeHint;} return convertArray(readArray(actualColumn), typeHint); diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java index e00438f12..1cb4a7760 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java @@ -63,6 +63,8 @@ public class SerializerUtils { public static void serializeData(OutputStream stream, Object value, ClickHouseColumn column) throws IOException { //Serialize the value to the stream based on the data type switch (column.getDataType()) { + case QBit: + // QBit(element_type, dimension) is serialized like Array(element_type). case Array: serializeArrayData(stream, value, column); break; diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/DataTypeConverter.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/DataTypeConverter.java index d7f4a0f34..06eaa5bea 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/DataTypeConverter.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/DataTypeConverter.java @@ -67,6 +67,8 @@ public String convertToString(Object value, ClickHouseColumn column) { case IPv4: case IPv6: return ipvToString(value, column); + case QBit: + // QBit is rendered as an array literal of its element type, like Array(element_type). case Array: return arrayToString(value, column); case Point: diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java index b24883ca9..c2d5a8175 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java @@ -92,8 +92,17 @@ public void tearDown() { private void writeReadVerify(String table, String tableDef, Class dtoClass, List data, BiConsumer, T> rowVerifier) throws Exception { + writeReadVerify(table, tableDef, dtoClass, data, rowVerifier, null); + } + + private void writeReadVerify(String table, String tableDef, Class dtoClass, List data, + BiConsumer, T> rowVerifier, CommandSettings ddlSettings) throws Exception { client.execute("DROP TABLE IF EXISTS " + table).get(); - client.execute(tableDef); + if (ddlSettings == null) { + client.execute(tableDef); + } else { + client.execute(tableDef, ddlSettings).get(); + } final TableSchema tableSchema = client.getTableSchema(table); client.register(dtoClass, tableSchema); @@ -107,6 +116,88 @@ private void writeReadVerify(String table, String tableDef, Class dtoClas Assert.assertEquals(rowCount.get(), data.size()); } + // QBit was introduced in ClickHouse 25.10. + private static final String QBIT_UNSUPPORTED_VERSIONS = "(,25.9]"; + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class QBitFloat32DTO { + private long rowId; + private float[] vec; + private int tail; + + public static String tblCreateSQL(String table) { + return tableDefinition(table, "rowId Int64", "vec QBit(Float32, 8)", "tail Int32"); + } + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class QBitFloat64DTO { + private long rowId; + private double[] vec; + private int tail; + + public static String tblCreateSQL(String table) { + return tableDefinition(table, "rowId Int64", "vec QBit(Float64, 8)", "tail Int32"); + } + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class QBitBFloat16DTO { + private long rowId; + private float[] vec; + private int tail; + + public static String tblCreateSQL(String table) { + return tableDefinition(table, "rowId Int64", "vec QBit(BFloat16, 8)", "tail Int32"); + } + } + + @Test(groups = {"integration"}) + public void testQBit() throws Exception { + if (isVersionMatch(QBIT_UNSUPPORTED_VERSIONS)) { + throw new SkipException("QBit requires ClickHouse 25.10+"); + } + + // A QBit(element_type, dimension) value is transmitted over RowBinary exactly like an + // Array(element_type): a var-int length followed by that many element values. The full + // client write -> server -> client read round-trip is verified for every supported + // element type. The trailing Int32 column would shift (and the assertion fail) if the + // QBit codec consumed the wrong number of bytes. + CommandSettings ddl = (CommandSettings) new CommandSettings() + .serverSetting("allow_experimental_qbit_type", "1"); + + final float[] f32 = {1f, -2f, 3.5f, 4f, 5f, 6f, 7f, 8f}; + writeReadVerify("test_qbit_f32", QBitFloat32DTO.tblCreateSQL("test_qbit_f32"), + QBitFloat32DTO.class, Arrays.asList(new QBitFloat32DTO(0, f32, 42)), + (all, dto) -> { + Assert.assertEquals(dto.getVec(), f32); + Assert.assertEquals(dto.getTail(), 42); + }, ddl); + + final double[] f64 = {1d, -2d, 3.5d, 4d, 5d, 6d, 7d, 8d}; + writeReadVerify("test_qbit_f64", QBitFloat64DTO.tblCreateSQL("test_qbit_f64"), + QBitFloat64DTO.class, Arrays.asList(new QBitFloat64DTO(0, f64, 42)), + (all, dto) -> { + Assert.assertEquals(dto.getVec(), f64); + Assert.assertEquals(dto.getTail(), 42); + }, ddl); + + // Integers up to 256 are exactly representable in BFloat16, so these round-trip bit-for-bit. + final float[] bf16 = {1f, 2f, 4f, 8f, 16f, 32f, 64f, 128f}; + writeReadVerify("test_qbit_bf16", QBitBFloat16DTO.tblCreateSQL("test_qbit_bf16"), + QBitBFloat16DTO.class, Arrays.asList(new QBitBFloat16DTO(0, bf16, 42)), + (all, dto) -> { + Assert.assertEquals(dto.getVec(), bf16); + Assert.assertEquals(dto.getTail(), 42); + }, ddl); + } + @Test(groups = {"integration"}) public void testNestedDataTypes() throws Exception { final String table = "test_nested_types"; diff --git a/docs/features.md b/docs/features.md index 73b01df59..7126628d2 100644 --- a/docs/features.md +++ b/docs/features.md @@ -21,6 +21,7 @@ This document lists stable, user-visible behavior in `client-v2` and `jdbc-v2` t - JSONEachRow text reader: Can stream `JSONEachRow` responses through a caller-supplied `JsonParser`, with Jackson and Gson parser factory implementations available as optional classpath dependencies, and infers a best-effort schema from the first row. - Data type conversion: Maps ClickHouse types to Java values for binary reads, POJO binding, and SQL parameter formatting, including date/time handling. - BFloat16 type support: For ClickHouse `24.11+`, reads and writes the `BFloat16` type through generic records, binary readers, POJO binding, and `Nullable`/`Dynamic`/`Variant` wrappers. `BFloat16` maps to the Java `float` type: a read widens the stored 16-bit value losslessly, while a write keeps the high 16 bits of the `float`. In `jdbc-v2`, `BFloat16` maps to `java.sql.Types.FLOAT` / `java.lang.Float` and is read and written through the standard `getFloat`/`setFloat` and `getObject` accessors. +- QBit type support: For ClickHouse `25.10+` (requires the `allow_experimental_qbit_type` server setting to create a column), reads and writes the experimental `QBit(element_type, dimension)` vector type, where `element_type` is `BFloat16`, `Float32`, or `Float64`. On the wire a `QBit` value is encoded exactly like `Array(element_type)` (a length-prefixed list of elements), so the client reads and writes it as a Java array of the element type — `float[]` for `BFloat16`/`Float32`, `double[]` for `Float64` — through generic records, binary readers, and POJO binding, using the same code path as `Array(element_type)`. The bit-transposed on-disk layout is a server storage detail and is not exposed to the client. - Geometry type support: For ClickHouse `25.11+`, where `Geometry` changed from a string alias to `Variant(Point, Ring, LineString, MultiLineString, Polygon, MultiPolygon)`, the client reads and writes `Geometry` values through generic records, binary readers, POJO binding, and SQL parameter formatting, using Java array dimensionality to represent the geometry shape. - Insert APIs: Supports inserting registered POJOs, raw streams, and callback-driven writers, with optional column lists and format selection. - Insert controls: Supports insert-specific settings such as deduplication token, query id, compression behavior, and request headers. @@ -43,6 +44,7 @@ Compatibility-sensitive traits: - Identifier quoting behavior is stable API for helper callers: identifiers are double-quoted, embedded double quotes are doubled, and optional quoting keeps simple identifiers unchanged. - Instant formatting is type-sensitive and should not drift: `Date` formatting depends on an explicit timezone, `DateTime` is serialized as epoch seconds, and higher-precision timestamps preserve up to 9 fractional digits. - `BFloat16` conversion is precision-sensitive and should not drift: a write keeps only the high 16 bits of the `float` (the low mantissa bits are truncated toward zero, matching the server's `Float32` → `BFloat16` conversion), so values that are not exactly representable in `BFloat16` change when written; a read widens the 16-bit value back to `float` losslessly. +- `QBit` is wire-compatible with `Array(element_type)` and should not drift: the client transmits the logical vector as a length-prefixed array of its element type (`float[]` for `BFloat16`/`Float32`, `double[]` for `Float64`) rather than the server's bit-transposed on-disk layout, so a `QBit(E, N)` value read from or written to the server round-trips as an array of `E`. - Timezone conversion helpers preserve nanoseconds and can intentionally shift local date or time when interpreted in a different timezone; this behavior is covered by tests and should not be normalized away. - `Geometry` handling is shape-sensitive: supported values are 1D through 4D Java arrays representing the nested geometry variants, and unsupported shapes or non-array values are rejected during serialization. - `Geometry` write inference is dimension-based rather than fully type-specific: point, ring/line string, polygon/multi-line string, and multi-polygon are selected from array depth, so writing `Geometry` cannot currently distinguish `Ring` from `LineString` or `Polygon` from `MultiLineString`. @@ -76,6 +78,7 @@ Compatibility-sensitive traits: - Database metadata: Implements JDBC `DatabaseMetaData` for ClickHouse catalogs, schemas, tables, columns, and related capability reporting. - Parameter metadata: Reports prepared-statement parameter counts. - Type mapping and conversions: Maps ClickHouse types to JDBC types and Java classes, including date/time handling and `java.time` support. +- QBit type mapping: For ClickHouse `25.10+`, JDBC exposes the experimental `QBit(element_type, dimension)` type as `ARRAY`, returning the vector as a `java.sql.Array` of the element type from `getObject()`/`getArray()`. Supported element types are `BFloat16` and `Float32` (both `java.lang.Float`) and `Float64` (`java.lang.Double`). The `allow_experimental_qbit_type` server setting is required to create a `QBit` column. - Custom result-set type map: `ResultSet#getObject(int|String, Map>)` accepts both ClickHouse type names and JDBC `SQLType` names as map keys. Only unwrapped type names are used — `Nullable(...)` and `LowCardinality(...)` wrappers are stripped before lookup, so a key like `"Int32"` matches both `Int32` and `Nullable(Int32)` columns, and keys like `"Nullable(Int32)"` are not recognized. Lookup order is `ClickHouseColumn#getDataType().name()` (e.g. `"Int32"`, `"String"`, `"DateTime"`) then `SQLType.getName()` (e.g. `"INTEGER"`, `"VARCHAR"`, `"TIMESTAMP"`); a missing entry leaves the value uncoerced (read as-is). The feature is supported for primitive ClickHouse types only — `Array`, `Tuple`, `Map`, `Nested`, and geometry types (`Point`, `Ring`, `LineString`, `Polygon`, `MultiPolygon`, `MultiLineString`, `Geometry`) bypass the type map and are returned in their native form. - Arrays and tuples: Supports JDBC arrays plus ClickHouse tuple values through custom `Array` and `Struct` implementations. - Geometry type mapping: For ClickHouse `25.11+`, where `Geometry` changed from a string alias to `Variant(Point, Ring, LineString, MultiLineString, Polygon, MultiPolygon)`, JDBC exposes `Geometry` as `ARRAY`, returns nested Java arrays from `getObject()`/`getArray()`, and accepts `Struct` or nested `Array` inputs for prepared-statement inserts depending on the geometry shape. diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcUtils.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcUtils.java index bf7292428..6c13034c3 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcUtils.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcUtils.java @@ -117,7 +117,7 @@ private static Map generateTypeMap() { map.put(ClickHouseDataType.AggregateFunction, JDBCType.OTHER); map.put(ClickHouseDataType.Variant, JDBCType.OTHER); map.put(ClickHouseDataType.Dynamic, JDBCType.OTHER); - map.put(ClickHouseDataType.QBit, JDBCType.OTHER); + map.put(ClickHouseDataType.QBit, JDBCType.ARRAY); return ImmutableMap.copyOf(map); diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/types/Array.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/types/Array.java index ac82a3fd2..b7b23cc7f 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/types/Array.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/types/Array.java @@ -27,7 +27,9 @@ public class Array implements java.sql.Array { public Array(ClickHouseColumn column, Object[] elements) throws SQLException { this.column = column; this.array = elements; - ClickHouseColumn baseColumn = (this.column.isArray() ? this.column.getArrayBaseColumn() : this.column); + // QBit is not an Array (isArray() == false) but is still a one-level array of its element + // type, so use the array nesting level (set for both Array and QBit) to find the element column. + ClickHouseColumn baseColumn = (this.column.getArrayNestedLevel() > 0 ? this.column.getArrayBaseColumn() : this.column); this.baseDataType = baseColumn.getDataType(); this.elementTypeName = baseColumn.getOriginalTypeName(); this.type = JdbcUtils.CLICKHOUSE_TO_SQL_TYPE_MAP.getOrDefault(baseDataType, JDBCType.OTHER).getVendorTypeNumber(); diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/JdbcDataTypeTests.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/JdbcDataTypeTests.java index a3b46496b..c35a50171 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/JdbcDataTypeTests.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/JdbcDataTypeTests.java @@ -570,6 +570,61 @@ public void testBFloat16() throws SQLException { } } + // QBit was introduced in ClickHouse 25.10. + private static final String QBIT_UNSUPPORTED_VERSIONS = "(,25.9]"; + + @Test(groups = { "integration" }) + public void testQBit() throws SQLException { + if (ClickHouseVersion.of(getServerVersion()).check(QBIT_UNSUPPORTED_VERSIONS)) { + throw new SkipException("QBit was introduced in ClickHouse 25.10"); + } + + // QBit(element_type, dimension) is exposed through JDBC as an ARRAY of its element type + // (the same way Array/Geometry are), written from and read as a Java array. The + // experimental type flag is only required to create the column, so the insert/read + // connection does not need it. + final String table = "test_qbit_jdbc"; + Properties properties = new Properties(); + properties.setProperty(ClientConfigProperties.serverSetting("allow_experimental_qbit_type"), "1"); + runQuery("DROP TABLE IF EXISTS " + table); + runQuery("CREATE TABLE " + table + + " (rowId Int32, vec QBit(Float32, 8)) ENGINE = MergeTree ORDER BY rowId", properties); + + final float[] expected = { 1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f }; + + // Write path: a Java float[] bound to a QBit parameter is rendered as an array literal. + try (Connection conn = getJdbcConnection(); + PreparedStatement ps = conn.prepareStatement("INSERT INTO " + table + " VALUES (?, ?)")) { + ps.setInt(1, 1); + ps.setObject(2, expected); + ps.executeUpdate(); + } + + // Read path: metadata reports ARRAY with the element type, and getArray/getObject return + // a java.sql.Array of the element values. + try (Connection conn = getJdbcConnection(); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT rowId, vec FROM " + table + " ORDER BY rowId")) { + ResultSetMetaData meta = rs.getMetaData(); + assertEquals(meta.getColumnType(2), Types.ARRAY, "QBit should map to Types.ARRAY"); + + assertTrue(rs.next()); + java.sql.Array array = rs.getArray("vec"); + assertTrue(array != null, "getArray must return the QBit vector"); + assertEquals(array.getBaseType(), Types.FLOAT, "QBit(Float32) element type should be Types.FLOAT"); + assertEquals(array.getBaseTypeName(), "Float32", "QBit(Float32) element type name should be Float32"); + Object elements = array.getArray(); + assertEquals(java.lang.reflect.Array.getLength(elements), expected.length); + for (int i = 0; i < expected.length; i++) { + assertEquals(((Number) java.lang.reflect.Array.get(elements, i)).floatValue(), expected[i]); + } + + assertTrue(rs.getObject("vec") instanceof java.sql.Array, + "getObject on a QBit column should return java.sql.Array"); + assertFalse(rs.next()); + } + } + @Test(groups = { "integration" }) public void testBFloat16WriteAsFloat() throws SQLException { if (ClickHouseVersion.of(getServerVersion()).check(BFLOAT16_UNSUPPORTED_VERSIONS)) { From 0e1a6bb7c660993ec97c6e2c943a2b54de4303d0 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Sun, 19 Jul 2026 13:58:25 +0000 Subject: [PATCH 2/4] feat(client-v2): validate QBit vector dimension on write QBit(element_type, dimension) is a fixed-size vector: the server requires exactly `dimension` elements and rejects any other length over RowBinary with a late SERIALIZATION_ERROR (verified against server 26.5). The write path reused serializeArrayData, which wrote the actual array/list length without checking it, so a wrong-sized (including empty) vector was serialized like a normal Array and only failed on the server after a round-trip. Split the QBit case out of the Array fall-through into serializeQBitData, which validates the element count against column.getPrecision() (the dimension) and throws a clear IllegalArgumentException naming the column, expected dimension, and actual length before delegating to serializeArrayData. This mirrors the client-side length enforcement already applied to the other fixed-size type, FixedString(N). A correctly-sized QBit is still byte-for-byte identical to Array(element_type). Addresses Cursor Bugbot review on PR #2939. --- .../internal/SerializerUtils.java | 29 ++++++++++++- .../internal/SerializerUtilsTest.java | 41 +++++++++++++++++++ docs/features.md | 2 +- 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java index 1cb4a7760..f92f1562f 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java @@ -64,7 +64,8 @@ public static void serializeData(OutputStream stream, Object value, ClickHouseCo //Serialize the value to the stream based on the data type switch (column.getDataType()) { case QBit: - // QBit(element_type, dimension) is serialized like Array(element_type). + serializeQBitData(stream, value, column); + break; case Array: serializeArrayData(stream, value, column); break; @@ -472,6 +473,32 @@ public static void serializeArrayData(OutputStream stream, Object value, ClickHo } } + /** + * Serializes a {@code QBit(element_type, dimension)} value. On the wire a {@code QBit} is + * transmitted exactly like {@code Array(element_type)} — a var-int length followed by that many + * element values — but the element count is fixed and must equal the declared dimension. The + * count is validated up-front so a wrong-sized (including empty) vector fails fast on the client + * with a clear message instead of a late server {@code SERIALIZATION_ERROR}, mirroring the + * client-side length enforcement already applied to the other fixed-size type, + * {@code FixedString(N)}. + */ + private static void serializeQBitData(OutputStream stream, Object value, ClickHouseColumn column) throws IOException { + if (value != null) { + int length = -1; + if (value.getClass().isArray()) { + length = Array.getLength(value); + } else if (value instanceof List) { + length = ((List) value).size(); + } + int dimension = column.getPrecision(); + if (length >= 0 && length != dimension) { + throw new IllegalArgumentException("QBit column '" + column.getColumnName() + + "' expects exactly " + dimension + " elements but got " + length); + } + } + serializeArrayData(stream, value, column); + } + /** DO NOT USE - part of internal API that will be changed */ diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java index 9574bed72..87cb9a136 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java @@ -143,6 +143,47 @@ public void testGeometrySerializationRejectsMalformedList() { ClickHouseColumn.of("v", "Geometry"))); } + @Test(dataProvider = "qbitWrongDimension") + public void testQBitSerializationRejectsWrongDimension(String typeName, Object value, int actualLength) { + ClickHouseColumn column = ClickHouseColumn.of("vec", typeName); + + IllegalArgumentException ex = Assert.expectThrows(IllegalArgumentException.class, + () -> SerializerUtils.serializeData(new ByteArrayOutputStream(), value, column)); + String message = ex.getMessage(); + Assert.assertTrue(message.contains("vec"), "Message should name the column: " + message); + Assert.assertTrue(message.contains("8"), "Message should state the expected dimension: " + message); + Assert.assertTrue(message.contains("got " + actualLength), + "Message should state the actual length: " + message); + } + + @DataProvider(name = "qbitWrongDimension") + private Object[][] qbitWrongDimension() { + // A QBit(E, 8) column requires exactly 8 elements: empty, too-short, and too-long vectors are + // all invalid, for both the Java-array and List representations and every element type. + return new Object[][] { + {"QBit(Float32, 8)", new float[0], 0}, + {"QBit(Float32, 8)", new float[] {1f, 2f, 3f, 4f, 5f}, 5}, + {"QBit(Float32, 8)", new float[] {1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f, 9f, 10f}, 10}, + {"QBit(Float64, 8)", new double[] {1d, 2d, 3d, 4d, 5d}, 5}, + {"QBit(BFloat16, 8)", new float[] {1f, 2f, 3f}, 3}, + {"QBit(Float32, 8)", Arrays.asList(1f, 2f, 3f), 3}, + }; + } + + @Test + public void testQBitSerializationAcceptsExactDimensionAndMatchesArray() throws Exception { + float[] vec = {1f, -2f, 3.5f, 4f, 5f, 6f, 7f, 8f}; + + ByteArrayOutputStream qbitOut = new ByteArrayOutputStream(); + SerializerUtils.serializeData(qbitOut, vec, ClickHouseColumn.of("vec", "QBit(Float32, 8)")); + + // A correctly-sized QBit passes validation and is serialized byte-for-byte identically to + // Array(element_type), which is the wire contract the reader relies on. + ByteArrayOutputStream arrayOut = new ByteArrayOutputStream(); + SerializerUtils.serializeData(arrayOut, vec, ClickHouseColumn.of("vec", "Array(Float32)")); + Assert.assertEquals(qbitOut.toByteArray(), arrayOut.toByteArray()); + } + @Test(dataProvider = "nonNullableEnumTypes") public void testNullIntoNonNullableEnumThrowsIllegalArgument(String typeName) { ClickHouseColumn column = ClickHouseColumn.of("bs_flag", typeName); diff --git a/docs/features.md b/docs/features.md index 7126628d2..3f3a013f4 100644 --- a/docs/features.md +++ b/docs/features.md @@ -44,7 +44,7 @@ Compatibility-sensitive traits: - Identifier quoting behavior is stable API for helper callers: identifiers are double-quoted, embedded double quotes are doubled, and optional quoting keeps simple identifiers unchanged. - Instant formatting is type-sensitive and should not drift: `Date` formatting depends on an explicit timezone, `DateTime` is serialized as epoch seconds, and higher-precision timestamps preserve up to 9 fractional digits. - `BFloat16` conversion is precision-sensitive and should not drift: a write keeps only the high 16 bits of the `float` (the low mantissa bits are truncated toward zero, matching the server's `Float32` → `BFloat16` conversion), so values that are not exactly representable in `BFloat16` change when written; a read widens the 16-bit value back to `float` losslessly. -- `QBit` is wire-compatible with `Array(element_type)` and should not drift: the client transmits the logical vector as a length-prefixed array of its element type (`float[]` for `BFloat16`/`Float32`, `double[]` for `Float64`) rather than the server's bit-transposed on-disk layout, so a `QBit(E, N)` value read from or written to the server round-trips as an array of `E`. +- `QBit` is wire-compatible with `Array(element_type)` and should not drift: the client transmits the logical vector as a length-prefixed array of its element type (`float[]` for `BFloat16`/`Float32`, `double[]` for `Float64`) rather than the server's bit-transposed on-disk layout, so a `QBit(E, N)` value read from or written to the server round-trips as an array of `E`. A written `QBit(E, N)` value must contain exactly `N` elements: a wrong-sized (including empty) vector is rejected during binary serialization with an `IllegalArgumentException` rather than deferred to a server error, matching the fixed dimension the server enforces. - Timezone conversion helpers preserve nanoseconds and can intentionally shift local date or time when interpreted in a different timezone; this behavior is covered by tests and should not be normalized away. - `Geometry` handling is shape-sensitive: supported values are 1D through 4D Java arrays representing the nested geometry variants, and unsupported shapes or non-array values are rejected during serialization. - `Geometry` write inference is dimension-based rather than fully type-specific: point, ring/line string, polygon/multi-line string, and multi-polygon are selected from array depth, so writing `Geometry` cannot currently distinguish `Ring` from `LineString` or `Polygon` from `MultiLineString`. From e2b23e05d4aa7826b9d6367fd70e874010789bde Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Sun, 19 Jul 2026 14:25:56 +0000 Subject: [PATCH 3/4] fix(client-v2): reject non-array/non-List QBit write values A non-null QBit value that is neither a Java array nor a List fell through serializeQBitData to serializeArrayData, which writes no bytes for such a value, desynchronizing the RowBinary stream so the following columns are misread or rejected by the server. Reject it up-front with a clear IllegalArgumentException, consistent with the existing fixed-dimension validation and the Geometry reject-unsupported path. Addresses Cursor Bugbot review on #2939. --- .../internal/SerializerUtils.java | 13 +++++++-- .../internal/SerializerUtilsTest.java | 29 +++++++++++++++++++ docs/features.md | 2 +- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java index f92f1562f..0016beec7 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java @@ -480,18 +480,25 @@ public static void serializeArrayData(OutputStream stream, Object value, ClickHo * count is validated up-front so a wrong-sized (including empty) vector fails fast on the client * with a clear message instead of a late server {@code SERIALIZATION_ERROR}, mirroring the * client-side length enforcement already applied to the other fixed-size type, - * {@code FixedString(N)}. + * {@code FixedString(N)}. A non-null value that is neither a Java array nor a {@code List} cannot + * carry a vector and is rejected as well — otherwise it would fall through to + * {@link #serializeArrayData} and write no bytes for the column, desynchronizing the + * {@code RowBinary} stream and corrupting the columns that follow. */ private static void serializeQBitData(OutputStream stream, Object value, ClickHouseColumn column) throws IOException { if (value != null) { - int length = -1; + int length; if (value.getClass().isArray()) { length = Array.getLength(value); } else if (value instanceof List) { length = ((List) value).size(); + } else { + throw new IllegalArgumentException("QBit column '" + column.getColumnName() + + "' expects a Java array or List of its element type but got " + + value.getClass().getName()); } int dimension = column.getPrecision(); - if (length >= 0 && length != dimension) { + if (length != dimension) { throw new IllegalArgumentException("QBit column '" + column.getColumnName() + "' expects exactly " + dimension + " elements but got " + length); } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java index 87cb9a136..5b2a8d587 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java @@ -170,6 +170,35 @@ private Object[][] qbitWrongDimension() { }; } + @Test(dataProvider = "qbitWrongType") + public void testQBitSerializationRejectsNonArrayValue(Object value) { + // A non-null QBit value that is neither a Java array nor a List cannot carry a vector. It must + // be rejected up-front: otherwise it falls through to the Array serializer, which writes no + // bytes for the column, desynchronizing the RowBinary stream and corrupting the following + // columns. Writing into a byte sink so any (wrongly) emitted payload would be observable. + ClickHouseColumn column = ClickHouseColumn.of("vec", "QBit(Float32, 8)"); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + + IllegalArgumentException ex = Assert.expectThrows(IllegalArgumentException.class, + () -> SerializerUtils.serializeData(out, value, column)); + Assert.assertTrue(ex.getMessage().contains("vec"), + "Message should name the column: " + ex.getMessage()); + Assert.assertEquals(out.size(), 0, + "Nothing should be written to the stream when the value is rejected"); + } + + @DataProvider(name = "qbitWrongType") + private Object[][] qbitWrongType() { + // Values that are neither a Java array nor a List: a String, boxed scalars of the element + // type, and a Map. None of these can represent a QBit(E, N) vector. + return new Object[][] { + {"not-a-vector"}, + {3.14f}, + {42d}, + {newMap("k", "v")}, + }; + } + @Test public void testQBitSerializationAcceptsExactDimensionAndMatchesArray() throws Exception { float[] vec = {1f, -2f, 3.5f, 4f, 5f, 6f, 7f, 8f}; diff --git a/docs/features.md b/docs/features.md index 3f3a013f4..87bd19a42 100644 --- a/docs/features.md +++ b/docs/features.md @@ -44,7 +44,7 @@ Compatibility-sensitive traits: - Identifier quoting behavior is stable API for helper callers: identifiers are double-quoted, embedded double quotes are doubled, and optional quoting keeps simple identifiers unchanged. - Instant formatting is type-sensitive and should not drift: `Date` formatting depends on an explicit timezone, `DateTime` is serialized as epoch seconds, and higher-precision timestamps preserve up to 9 fractional digits. - `BFloat16` conversion is precision-sensitive and should not drift: a write keeps only the high 16 bits of the `float` (the low mantissa bits are truncated toward zero, matching the server's `Float32` → `BFloat16` conversion), so values that are not exactly representable in `BFloat16` change when written; a read widens the 16-bit value back to `float` losslessly. -- `QBit` is wire-compatible with `Array(element_type)` and should not drift: the client transmits the logical vector as a length-prefixed array of its element type (`float[]` for `BFloat16`/`Float32`, `double[]` for `Float64`) rather than the server's bit-transposed on-disk layout, so a `QBit(E, N)` value read from or written to the server round-trips as an array of `E`. A written `QBit(E, N)` value must contain exactly `N` elements: a wrong-sized (including empty) vector is rejected during binary serialization with an `IllegalArgumentException` rather than deferred to a server error, matching the fixed dimension the server enforces. +- `QBit` is wire-compatible with `Array(element_type)` and should not drift: the client transmits the logical vector as a length-prefixed array of its element type (`float[]` for `BFloat16`/`Float32`, `double[]` for `Float64`) rather than the server's bit-transposed on-disk layout, so a `QBit(E, N)` value read from or written to the server round-trips as an array of `E`. A written `QBit(E, N)` value must be a Java array or `List` holding exactly `N` elements: a wrong-sized (including empty) vector, or a non-null value that is neither an array nor a `List`, is rejected during binary serialization with an `IllegalArgumentException` rather than deferred to a server error or silently writing a misaligned stream, matching the fixed dimension the server enforces. - Timezone conversion helpers preserve nanoseconds and can intentionally shift local date or time when interpreted in a different timezone; this behavior is covered by tests and should not be normalized away. - `Geometry` handling is shape-sensitive: supported values are 1D through 4D Java arrays representing the nested geometry variants, and unsupported shapes or non-array values are rejected during serialization. - `Geometry` write inference is dimension-based rather than fully type-specific: point, ring/line string, polygon/multi-line string, and multi-polygon are selected from array depth, so writing `Geometry` cannot currently distinguish `Ring` from `LineString` or `Polygon` from `MultiLineString`. From 557768b4d537f87fd4d629dde975e390b420b9da Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:01:58 +0000 Subject: [PATCH 4/4] =?UTF-8?q?fix(client-v2):=20address=20QBit=20review?= =?UTF-8?q?=20=E2=80=94=20reject=20QBit=20in=20Native=20format,=20reject?= =?UTF-8?q?=20null=20QBit,=20await=20DDL=20in=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review response on PR #2939: - NativeFormatReader: reject reading any column that is or contains a QBit (recursive check). The server transmits QBit in the Native format using its internal bit-transposed Tuple(FixedString(...)) layout, not the Array-like RowBinary representation this reader decodes, so the previous fall-through misread the bytes and desynchronized the block. Fail loud with a clear ClientException pointing to a RowBinary format instead of decoding garbage. - SerializerUtils.serializeQBitData: reject a null value. A fixed-dimension QBit cannot be null; a QBit nested in a container (Tuple/Map/Array) reaches this path via serializeNestedData without the top-level preamble, so without the guard the null was written as a zero-length vector, desynchronizing the stream. - DataTypeTests.writeReadVerify: await client.execute(tableDef) in the no-ddl branch (it was unawaited, racing the subsequent getTableSchema). - Tests: Native rejection (top-level + nested Map(String,QBit)); null QBit rejection (direct + nested-in-Tuple). docs/features.md + CHANGELOG updated. Co-Authored-By: Claude Opus 4.7 --- CHANGELOG.md | 4 +- .../api/data_formats/NativeFormatReader.java | 35 +++++++++++++++ .../internal/SerializerUtils.java | 45 ++++++++++++------- .../internal/SerializerUtilsTest.java | 41 +++++++++++++++++ .../client/datatypes/DataTypeTests.java | 39 +++++++++++++++- docs/features.md | 2 +- 6 files changed, 147 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c34909c67..9cb4935dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,9 @@ `BFloat16`/`Float32`, `double[]` for `Float64`) through generic records, binary readers, and POJO binding. In the JDBC driver (`jdbc-v2`) `QBit` maps to `java.sql.Types.ARRAY` and is returned as a `java.sql.Array` from `getObject`/`getArray`. Previously `QBit` was an unimplemented type constant and reading or writing such a column - failed. (https://github.com/ClickHouse/clickhouse-java/issues/2610) + failed. Reading `QBit` through the `Native` output format is not supported — the server transmits it there using a + different internal layout — and fails fast with a clear error; use a `RowBinary` format instead. + (https://github.com/ClickHouse/clickhouse-java/issues/2610) - **[client-v2, jdbc-v2]** Added TLS cipher suite selection. `Client.Builder.setSSLCipherSuites(String...)` (client-v2) and the comma-separated `ssl_cipher_suites` connection property (client-v2 and jdbc-v2) restrict the cipher suites enabled on secure connections; when unset, the transport defaults are used. Cipher-suite selection is independent of the diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/NativeFormatReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/NativeFormatReader.java index ddcf0049b..6bd910f99 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/NativeFormatReader.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/NativeFormatReader.java @@ -91,6 +91,21 @@ private boolean readBlock() throws IOException { names.add(column.getColumnName()); types.add(column.getDataType().name()); + if (containsQBit(column)) { + // QBit is transmitted in the Native format using its internal bit-transposed + // Tuple(FixedString(...)) layout, which is NOT the Array(element_type)-like + // representation used in RowBinary (the only representation this reader decodes for + // QBit). Reading it through the columnar/per-row paths below would misread those bytes + // and desynchronize the block, corrupting the columns that follow. Fail loudly instead + // of silently decoding garbage. This also covers a QBit nested inside another type + // (e.g. Map(String, QBit(...))). QBit can be read through a RowBinary format. + throw new ClientException("Reading column '" + column.getColumnName() + "' (" + + column.getOriginalTypeName() + ") from the Native format is not supported " + + "because it contains a QBit type: QBit is serialized in the Native format " + + "using an internal layout this reader does not decode. Use a RowBinary format " + + "(e.g. RowBinaryWithNamesAndTypes) to read QBit values"); + } + List values = new ArrayList<>(nRows); if (column.isArray()) { int[] sizes = new int[nRows]; @@ -116,6 +131,26 @@ private boolean readBlock() throws IOException { return true; } + /** + * Returns {@code true} if {@code column} is a {@code QBit} or contains a {@code QBit} anywhere in + * its nested type tree (e.g. {@code Array(QBit(...))}, {@code Tuple(..., QBit(...))}, + * {@code Map(String, QBit(...))}). {@code QBit} uses a different, internal wire layout in the + * Native format than in RowBinary, so this reader cannot decode it and rejects such columns + * up-front rather than misreading the block. {@code Nullable}/{@code LowCardinality} wrappers are + * flags on the column, so a wrapped {@code QBit} still reports {@code dataType == QBit} here. + */ + private static boolean containsQBit(ClickHouseColumn column) { + if (column.getDataType() == ClickHouseDataType.QBit) { + return true; + } + for (ClickHouseColumn nested : column.getNestedColumns()) { + if (nested != column && containsQBit(nested)) { + return true; + } + } + return false; + } + private static class Block { final List names; final List types; diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java index 0016beec7..f389239c8 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java @@ -484,24 +484,37 @@ public static void serializeArrayData(OutputStream stream, Object value, ClickHo * carry a vector and is rejected as well — otherwise it would fall through to * {@link #serializeArrayData} and write no bytes for the column, desynchronizing the * {@code RowBinary} stream and corrupting the columns that follow. + *

+ * A {@code null} value is rejected for the same reason: a fixed-dimension {@code QBit} cannot be + * represented by {@code null}. A top-level {@code null} non-nullable {@code QBit} is already + * rejected by + * {@link com.clickhouse.client.api.data_formats.RowBinaryFormatSerializer#writeValuePreamble}, + * but a {@code QBit} nested inside a container ({@code Tuple}/{@code Map}/{@code Array}) is written + * through {@link #serializeNestedData}, which does not route a non-nullable element through that + * preamble; without this guard the {@code null} would delegate to {@link #serializeArrayData} and + * be written as a zero-length vector (var-int {@code 0}), again desynchronizing the stream. (A + * {@code Nullable(QBit)} {@code null} never reaches here: its null-marker is written earlier by the + * preamble or by {@link #serializeNestedData}, which then return.) */ private static void serializeQBitData(OutputStream stream, Object value, ClickHouseColumn column) throws IOException { - if (value != null) { - int length; - if (value.getClass().isArray()) { - length = Array.getLength(value); - } else if (value instanceof List) { - length = ((List) value).size(); - } else { - throw new IllegalArgumentException("QBit column '" + column.getColumnName() - + "' expects a Java array or List of its element type but got " - + value.getClass().getName()); - } - int dimension = column.getPrecision(); - if (length != dimension) { - throw new IllegalArgumentException("QBit column '" + column.getColumnName() - + "' expects exactly " + dimension + " elements but got " + length); - } + if (value == null) { + throw new IllegalArgumentException("QBit column '" + column.getColumnName() + + "' cannot be null; expected exactly " + column.getPrecision() + " elements"); + } + int length; + if (value.getClass().isArray()) { + length = Array.getLength(value); + } else if (value instanceof List) { + length = ((List) value).size(); + } else { + throw new IllegalArgumentException("QBit column '" + column.getColumnName() + + "' expects a Java array or List of its element type but got " + + value.getClass().getName()); + } + int dimension = column.getPrecision(); + if (length != dimension) { + throw new IllegalArgumentException("QBit column '" + column.getColumnName() + + "' expects exactly " + dimension + " elements but got " + length); } serializeArrayData(stream, value, column); } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java index 5b2a8d587..9f5ba46c4 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java @@ -213,6 +213,47 @@ public void testQBitSerializationAcceptsExactDimensionAndMatchesArray() throws E Assert.assertEquals(qbitOut.toByteArray(), arrayOut.toByteArray()); } + @Test + public void testQBitSerializationRejectsNull() { + // A QBit has a fixed dimension, so a null value cannot satisfy it. A top-level null + // non-nullable QBit is already rejected by RowBinaryFormatSerializer.writeValuePreamble, but a + // QBit nested inside a Tuple/Map/Array is serialized through serializeNestedData, which does + // NOT route a non-nullable element through that preamble. Without an explicit guard the null + // would delegate to the Array serializer and be written as a zero-length vector (var-int 0), + // desynchronizing the RowBinary stream and corrupting the following columns. Writing into a + // byte sink so any (wrongly) emitted payload would be observable. + ClickHouseColumn column = ClickHouseColumn.of("vec", "QBit(Float32, 8)"); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + + IllegalArgumentException ex = Assert.expectThrows(IllegalArgumentException.class, + () -> SerializerUtils.serializeData(out, null, column)); + Assert.assertTrue(ex.getMessage().contains("vec"), + "Message should name the column: " + ex.getMessage()); + Assert.assertTrue(ex.getMessage().contains("null"), + "Message should state the value cannot be null: " + ex.getMessage()); + Assert.assertEquals(out.size(), 0, + "Nothing should be written to the stream when a null QBit is rejected"); + } + + @Test + public void testQBitNestedInTupleRejectsNullElement() throws Exception { + // Exercises the production-reachable path for the null guard: a non-nullable QBit nested in a + // Tuple is written through serializeNestedData, which does NOT apply the top-level + // writeValuePreamble null-into-non-nullable check to a non-nullable element. Without the guard + // in serializeQBitData the null element would be written as a zero-length vector (var-int 0), + // desynchronizing the stream and corrupting the rest of the row. + ClickHouseColumn tuple = ClickHouseColumn.of("t", "Tuple(QBit(Float32, 8))"); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + List tupleValue = Arrays.asList((Object) null); + + IllegalArgumentException ex = Assert.expectThrows(IllegalArgumentException.class, + () -> SerializerUtils.serializeData(out, tupleValue, tuple)); + Assert.assertTrue(ex.getMessage().contains("cannot be null"), + "Message should explain the null QBit is rejected: " + ex.getMessage()); + Assert.assertEquals(out.size(), 0, + "Nothing should be written when the nested null QBit element is rejected"); + } + @Test(dataProvider = "nonNullableEnumTypes") public void testNullIntoNonNullableEnumThrowsIllegalArgument(String typeName) { ClickHouseColumn column = ClickHouseColumn.of("bs_flag", typeName); diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java index c2d5a8175..a946b3024 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java @@ -17,6 +17,7 @@ import com.clickhouse.client.api.query.QuerySettings; import com.clickhouse.client.api.sql.SQLUtils; import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseFormat; import com.clickhouse.data.ClickHouseVersion; import lombok.AllArgsConstructor; import lombok.Data; @@ -99,7 +100,7 @@ private void writeReadVerify(String table, String tableDef, Class dtoClas BiConsumer, T> rowVerifier, CommandSettings ddlSettings) throws Exception { client.execute("DROP TABLE IF EXISTS " + table).get(); if (ddlSettings == null) { - client.execute(tableDef); + client.execute(tableDef).get(); } else { client.execute(tableDef, ddlSettings).get(); } @@ -198,6 +199,42 @@ public void testQBit() throws Exception { }, ddl); } + @Test(groups = {"integration"}) + public void testQBitNativeFormatRejected() throws Exception { + if (isVersionMatch(QBIT_UNSUPPORTED_VERSIONS)) { + throw new SkipException("QBit requires ClickHouse 25.10+"); + } + + // In the Native format the server transmits a QBit column using its internal bit-transposed + // layout, which is NOT the Array(element_type)-like representation the client decodes for QBit + // over RowBinary. Reading QBit via Native must therefore fail loudly with a clear error rather + // than silently decoding garbage and misaligning the trailing column that follows it. + QuerySettings settings = new QuerySettings() + .setFormat(ClickHouseFormat.Native) + .serverSetting("allow_experimental_qbit_type", "1"); + try (QueryResponse response = client.query( + "SELECT CAST([1, 2, 3, 4, 5, 6, 7, 8] AS QBit(Float32, 8)) AS q, 42 AS tail", settings).get()) { + ClientException ex = Assert.expectThrows(ClientException.class, + () -> client.newBinaryFormatReader(response)); + Assert.assertTrue(ex.getMessage().contains("QBit"), + "Expected a clear QBit message, got: " + ex.getMessage()); + Assert.assertTrue(ex.getMessage().contains("Native"), + "Expected the message to mention the Native format, got: " + ex.getMessage()); + } + + // The same rejection applies to a QBit nested inside another type (here Map(String, QBit)), + // which the server does support and would otherwise be misread column-by-column. + try (QueryResponse response = client.query( + "SELECT CAST(map('a', [1, 2, 3]) AS Map(String, QBit(Float32, 3))) AS m", settings).get()) { + ClientException ex = Assert.expectThrows(ClientException.class, + () -> client.newBinaryFormatReader(response)); + Assert.assertTrue(ex.getMessage().contains("QBit"), + "Expected a clear QBit message for the nested case, got: " + ex.getMessage()); + Assert.assertTrue(ex.getMessage().contains("Native"), + "Expected the message to mention the Native format, got: " + ex.getMessage()); + } + } + @Test(groups = {"integration"}) public void testNestedDataTypes() throws Exception { final String table = "test_nested_types"; diff --git a/docs/features.md b/docs/features.md index 87bd19a42..bded88a4f 100644 --- a/docs/features.md +++ b/docs/features.md @@ -21,7 +21,7 @@ This document lists stable, user-visible behavior in `client-v2` and `jdbc-v2` t - JSONEachRow text reader: Can stream `JSONEachRow` responses through a caller-supplied `JsonParser`, with Jackson and Gson parser factory implementations available as optional classpath dependencies, and infers a best-effort schema from the first row. - Data type conversion: Maps ClickHouse types to Java values for binary reads, POJO binding, and SQL parameter formatting, including date/time handling. - BFloat16 type support: For ClickHouse `24.11+`, reads and writes the `BFloat16` type through generic records, binary readers, POJO binding, and `Nullable`/`Dynamic`/`Variant` wrappers. `BFloat16` maps to the Java `float` type: a read widens the stored 16-bit value losslessly, while a write keeps the high 16 bits of the `float`. In `jdbc-v2`, `BFloat16` maps to `java.sql.Types.FLOAT` / `java.lang.Float` and is read and written through the standard `getFloat`/`setFloat` and `getObject` accessors. -- QBit type support: For ClickHouse `25.10+` (requires the `allow_experimental_qbit_type` server setting to create a column), reads and writes the experimental `QBit(element_type, dimension)` vector type, where `element_type` is `BFloat16`, `Float32`, or `Float64`. On the wire a `QBit` value is encoded exactly like `Array(element_type)` (a length-prefixed list of elements), so the client reads and writes it as a Java array of the element type — `float[]` for `BFloat16`/`Float32`, `double[]` for `Float64` — through generic records, binary readers, and POJO binding, using the same code path as `Array(element_type)`. The bit-transposed on-disk layout is a server storage detail and is not exposed to the client. +- QBit type support: For ClickHouse `25.10+` (requires the `allow_experimental_qbit_type` server setting to create a column), reads and writes the experimental `QBit(element_type, dimension)` vector type, where `element_type` is `BFloat16`, `Float32`, or `Float64`. On the wire a `QBit` value is encoded exactly like `Array(element_type)` (a length-prefixed list of elements), so the client reads and writes it as a Java array of the element type — `float[]` for `BFloat16`/`Float32`, `double[]` for `Float64` — through generic records, binary readers, and POJO binding, using the same code path as `Array(element_type)`. This `Array`-like encoding is what the server uses over `RowBinary` formats; the `Native` format instead transmits `QBit` using its internal bit-transposed `Tuple(FixedString(...))` layout, which the client does not decode, so reading any column that is or contains a `QBit` (including a nested `QBit`, e.g. `Map(String, QBit(...))`) through the `Native` format is rejected with a clear `ClientException` (use a `RowBinary` format such as `RowBinaryWithNamesAndTypes` instead). - Geometry type support: For ClickHouse `25.11+`, where `Geometry` changed from a string alias to `Variant(Point, Ring, LineString, MultiLineString, Polygon, MultiPolygon)`, the client reads and writes `Geometry` values through generic records, binary readers, POJO binding, and SQL parameter formatting, using Java array dimensionality to represent the geometry shape. - Insert APIs: Supports inserting registered POJOs, raw streams, and callback-driven writers, with optional column lists and format selection. - Insert controls: Supports insert-specific settings such as deduplication token, query id, compression behavior, and request headers.