From ea4b65635efe72a1a10ee94380282f54e2a27939 Mon Sep 17 00:00:00 2001 From: alekjarmov Date: Fri, 17 Jul 2026 12:17:33 +0200 Subject: [PATCH 1/4] [MINOR][SQL] Classify Databricks syntax errors by message Recognize remote Databricks parse errors when the JDBC driver reports the SYNTAX_ERROR condition only in its message. --- .../org/apache/spark/sql/jdbc/DatabricksDialect.scala | 5 +++-- .../datasources/v2/jdbc/JDBCTableCatalogSuite.scala | 9 ++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/DatabricksDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/DatabricksDialect.scala index a56aa90d6d72b..9a309ed326ae1 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/DatabricksDialect.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/DatabricksDialect.scala @@ -55,9 +55,10 @@ private case class DatabricksDialect() extends JdbcDialect with NoLegacyJDBCErro case _ => None } - // See https://docs.databricks.com/aws/en/error-messages/sqlstates + // The driver may report SYNTAX_ERROR only in the message. override def isSyntaxErrorBestEffort(exception: SQLException): Boolean = { - Option(exception.getSQLState).exists(_.startsWith("42")) + Option(exception.getSQLState).exists(_.startsWith("42")) || + Option(exception.getMessage).exists(_.contains("SYNTAX_ERROR")) } override def quoteIdentifier(colName: String): String = { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala index b3b0eb811ead9..ae53535f26308 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.datasources.v2.jdbc -import java.sql.{Connection, DriverManager} +import java.sql.{Connection, DriverManager, SQLException} import java.util.Properties import scala.jdk.CollectionConverters._ @@ -32,6 +32,7 @@ import org.apache.spark.sql.connector.catalog.{Identifier, TableSummary} import org.apache.spark.sql.errors.DataTypeErrors.{toSQLConf, toSQLStmt} import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.jdbc.JdbcDialects import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ import org.apache.spark.sql.util.CaseInsensitiveStringMap @@ -228,6 +229,12 @@ class JDBCTableCatalogSuite extends SharedSparkSession { } } + test("DatabricksDialect recognizes syntax errors in messages") { + val exception = new SQLException("[PARSE_SYNTAX_ERROR] Syntax error at or near 'SQL'", "07000") + val dialect = JdbcDialects.get("jdbc:databricks://account.cloud.databricks.com") + assert(dialect.isSyntaxErrorBestEffort(exception)) + } + test("create a table") { withTable("h2.test.new_table") { sql("CREATE TABLE h2.test.new_table(i INT, j STRING)") From 52471af0e615177364470b87cd46b2ddcbb4a02c Mon Sep 17 00:00:00 2001 From: alekjarmov Date: Fri, 17 Jul 2026 12:22:08 +0200 Subject: [PATCH 2/4] [MINOR][SQL] Match Databricks syntax errors case-insensitively Normalize JDBC error messages before detecting the remote SYNTAX_ERROR condition while retaining the SQLState documentation. --- .../scala/org/apache/spark/sql/jdbc/DatabricksDialect.scala | 6 ++++-- .../datasources/v2/jdbc/JDBCTableCatalogSuite.scala | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/DatabricksDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/DatabricksDialect.scala index 9a309ed326ae1..0c606ce71f914 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/DatabricksDialect.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/DatabricksDialect.scala @@ -55,10 +55,12 @@ private case class DatabricksDialect() extends JdbcDialect with NoLegacyJDBCErro case _ => None } - // The driver may report SYNTAX_ERROR only in the message. + // See https://docs.databricks.com/aws/en/error-messages/sqlstates. + // The driver may report syntax errors with a non-class-42 SQLState. override def isSyntaxErrorBestEffort(exception: SQLException): Boolean = { Option(exception.getSQLState).exists(_.startsWith("42")) || - Option(exception.getMessage).exists(_.contains("SYNTAX_ERROR")) + Option(exception.getMessage) + .exists(_.toUpperCase(Locale.ROOT).contains("SYNTAX_ERROR")) } override def quoteIdentifier(colName: String): String = { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala index ae53535f26308..c07e52ecf2aa7 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala @@ -230,7 +230,7 @@ class JDBCTableCatalogSuite extends SharedSparkSession { } test("DatabricksDialect recognizes syntax errors in messages") { - val exception = new SQLException("[PARSE_SYNTAX_ERROR] Syntax error at or near 'SQL'", "07000") + val exception = new SQLException("[parse_syntax_error] Syntax error at or near 'SQL'", "07000") val dialect = JdbcDialects.get("jdbc:databricks://account.cloud.databricks.com") assert(dialect.isSyntaxErrorBestEffort(exception)) } From f10335e73dc6dbd9dbf76d306c5e09db0b3864ab Mon Sep 17 00:00:00 2001 From: alekjarmov Date: Fri, 17 Jul 2026 16:58:48 +0200 Subject: [PATCH 3/4] [MINOR][SQL] Organize Databricks syntax error tests Move dialect coverage to JDBCSuite and verify that unrelated errors are not classified as syntax errors. --- .../datasources/v2/jdbc/JDBCTableCatalogSuite.scala | 9 +-------- .../test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala | 9 ++++++++- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala index c07e52ecf2aa7..b3b0eb811ead9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.datasources.v2.jdbc -import java.sql.{Connection, DriverManager, SQLException} +import java.sql.{Connection, DriverManager} import java.util.Properties import scala.jdk.CollectionConverters._ @@ -32,7 +32,6 @@ import org.apache.spark.sql.connector.catalog.{Identifier, TableSummary} import org.apache.spark.sql.errors.DataTypeErrors.{toSQLConf, toSQLStmt} import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.jdbc.JdbcDialects import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ import org.apache.spark.sql.util.CaseInsensitiveStringMap @@ -229,12 +228,6 @@ class JDBCTableCatalogSuite extends SharedSparkSession { } } - test("DatabricksDialect recognizes syntax errors in messages") { - val exception = new SQLException("[parse_syntax_error] Syntax error at or near 'SQL'", "07000") - val dialect = JdbcDialects.get("jdbc:databricks://account.cloud.databricks.com") - assert(dialect.isSyntaxErrorBestEffort(exception)) - } - test("create a table") { withTable("h2.test.new_table") { sql("CREATE TABLE h2.test.new_table(i INT, j STRING)") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala index 58797e68b8ab4..14b4ec3e62f3b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql.jdbc import java.math.BigDecimal -import java.sql.{Connection, Date, DriverManager, ResultSet, Statement, Timestamp} +import java.sql.{Connection, Date, DriverManager, ResultSet, SQLException, Statement, Timestamp} import java.time.{Instant, LocalDate, LocalDateTime} import java.time.format.DateTimeFormatter import java.util.{Calendar, GregorianCalendar, Properties, TimeZone} @@ -2608,6 +2608,13 @@ class JDBCSuite extends SharedSparkSession { .getJDBCType(BinaryType).map(_.databaseTypeDefinition).get == "BINARY") } + test("DatabricksDialect syntax error detection") { + val dialect = DatabricksDialect() + assert(dialect.isSyntaxErrorBestEffort( + new SQLException("[parse_syntax_error] Syntax error at or near 'SQL'", "07000"))) + assert(!dialect.isSyntaxErrorBestEffort(new SQLException("Connection reset", "08001"))) + } + test("SPARK-45425: Mapped TINYINT to ShortType for MsSqlServerDialect") { val msSqlServerDialect = JdbcDialects.get("jdbc:sqlserver") val metadata = new MetadataBuilder().putLong("scale", 1) From f89739a54fbfac932f73cfec5048dde6e3506d52 Mon Sep 17 00:00:00 2001 From: alekjarmov Date: Fri, 17 Jul 2026 17:00:55 +0200 Subject: [PATCH 4/4] [MINOR][SQL] Name Databricks syntax test after SPARK-58193 --- .../src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala index 14b4ec3e62f3b..de863b244be8a 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala @@ -2608,7 +2608,7 @@ class JDBCSuite extends SharedSparkSession { .getJDBCType(BinaryType).map(_.databaseTypeDefinition).get == "BINARY") } - test("DatabricksDialect syntax error detection") { + test("SPARK-58193: DatabricksDialect syntax error detection") { val dialect = DatabricksDialect() assert(dialect.isSyntaxErrorBestEffort( new SQLException("[parse_syntax_error] Syntax error at or near 'SQL'", "07000")))