From 86d81c5f6f231b191e0be3abca68226b59ee6092 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sat, 19 Sep 2026 03:13:05 +0700 Subject: [PATCH 1/5] refactor(editor): move the SQL lexer and statement scanner into a shared package --- Packages/TableProCore/Package.swift | 8 +- .../PLSQLUnitTracker.swift | 25 ++-- .../SQLLexicalGrammar.swift | 64 +++++++++ .../SQLRoutineBodyTracker.swift | 25 ++-- .../SQLStatementBoundaryTracking.swift | 24 +--- .../SQLStatementScanner.swift | 129 +++++++++--------- .../SqlBlockStructure.swift | 44 +++--- .../TableProSQLGrammar}/SqlDollarQuote.swift | 21 ++- .../TableProSQLGrammar}/SqlLexer.swift | 101 ++++++-------- .../TableProSQLGrammar}/StatementBlank.swift | 23 ++-- .../Autocomplete/SQLContextAnalyzer.swift | 1 + .../Core/Compare/CompareSyncExecutor.swift | 1 + ...yExecutionCoordinator+MultiStatement.swift | 1 + ...QueryExecutionCoordinator+Parameters.swift | 1 + .../QueryExecutionCoordinator.swift | 1 + .../Access/DatabaseAccessBridge.swift | 1 + .../Core/ObjectCopy/ObjectCopyPlanner.swift | 1 + .../Execution/BatchTransactionPolicy.swift | 1 + .../Services/Query/LeadingRowsStatement.swift | 1 + .../JavaScriptStatementScanner.swift | 1 + .../JavaScript/QueryStatementModel.swift | 1 + .../SQL/CatalogChangeClassifier.swift | 1 + .../SQL/Folding/SQLFoldScanner.swift | 1 + .../Utilities/SQL/QueryClassifier+PLSQL.swift | 1 + .../Core/Utilities/SQL/QueryClassifier.swift | 1 + .../Core/Utilities/SQL/SQLFileParser.swift | 1 + .../Core/Utilities/SQL/SQLLimitDetector.swift | 1 + .../Core/Utilities/SQL/SQLNonCodeSpan.swift | 1 + .../Utilities/SQL/SQLParameterExtractor.swift | 1 + .../Utilities/SQL/SQLQueryFingerprint.swift | 1 + .../Core/Utilities/SQL/SQLTokenCursor.swift | 1 + .../SQL/SelectSourceTableParser.swift | 1 + .../SQL/SqlDialect+LexicalGrammar.swift | 120 ++++++++++++++++ TablePro/Models/Query/StatementAnchor.swift | 1 + .../Views/Editor/StatementRunController.swift | 1 + .../MainContentCoordinator+ExecuteAll.swift | 1 + .../MainContentCoordinator+Explain.swift | 1 + .../Views/Main/MainContentCoordinator.swift | 1 + .../BatchTransactionPolicyTests.swift | 1 + .../Utilities/QueryStatementModelTests.swift | 1 + .../Utilities/SQL/QueryClassifierTests.swift | 1 + .../SQL/SQLExecutableStatementTests.swift | 1 + .../Utilities/SQL/SQLFoldScannerTests.swift | 1 + .../SQL/SQLStatementBlockSplittingTests.swift | 1 + .../SQL/SQLStatementNavigationTests.swift | 1 + .../SQL/SQLStatementPLSQLSplittingTests.swift | 1 + .../SQL/SQLStatementRangeTests.swift | 1 + .../Core/Utilities/SQL/SqlLexerTests.swift | 1 + .../Utilities/SQL/StatementBlankTests.swift | 1 + .../SQLStatementScannerLocatedTests.swift | 1 + .../Utilities/SQLStatementScannerTests.swift | 1 + .../Models/Query/StatementAnchorTests.swift | 1 + .../StatementNavigationCommandTests.swift | 1 + .../StatementRunPerformanceGuardTests.swift | 1 + .../Views/Main/TabQueryIsolationTests.swift | 1 + project.yml | 4 +- 56 files changed, 406 insertions(+), 226 deletions(-) rename {TablePro/Core/Utilities/SQL => Packages/TableProCore/Sources/TableProSQLGrammar}/PLSQLUnitTracker.swift (96%) create mode 100644 Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift rename {TablePro/Core/Utilities/SQL => Packages/TableProCore/Sources/TableProSQLGrammar}/SQLRoutineBodyTracker.swift (85%) rename {TablePro/Core/Utilities/SQL => Packages/TableProCore/Sources/TableProSQLGrammar}/SQLStatementBoundaryTracking.swift (78%) rename {TablePro/Core/Utilities/SQL => Packages/TableProCore/Sources/TableProSQLGrammar}/SQLStatementScanner.swift (81%) rename {TablePro/Core/Utilities/SQL => Packages/TableProCore/Sources/TableProSQLGrammar}/SqlBlockStructure.swift (86%) rename {TablePro/Core/Utilities/SQL => Packages/TableProCore/Sources/TableProSQLGrammar}/SqlDollarQuote.swift (81%) rename {TablePro/Core/Utilities/SQL => Packages/TableProCore/Sources/TableProSQLGrammar}/SqlLexer.swift (73%) rename {TablePro/Core/Utilities/SQL => Packages/TableProCore/Sources/TableProSQLGrammar}/StatementBlank.swift (76%) create mode 100644 TablePro/Core/Utilities/SQL/SqlDialect+LexicalGrammar.swift diff --git a/Packages/TableProCore/Package.swift b/Packages/TableProCore/Package.swift index f021837f1b..143afd80a6 100644 --- a/Packages/TableProCore/Package.swift +++ b/Packages/TableProCore/Package.swift @@ -28,7 +28,8 @@ let package = Package( .library(name: "TableProNumberFormatting", targets: ["TableProNumberFormatting"]), .library(name: "TableProDocumentPath", targets: ["TableProDocumentPath"]), .library(name: "TableProR2SQLCore", targets: ["TableProR2SQLCore"]), - .library(name: "TableProConnectionLibrary", targets: ["TableProConnectionLibrary"]) + .library(name: "TableProConnectionLibrary", targets: ["TableProConnectionLibrary"]), + .library(name: "TableProSQLGrammar", targets: ["TableProSQLGrammar"]) ], targets: [ .target( @@ -131,6 +132,11 @@ let package = Package( dependencies: [], path: "Sources/TableProConnectionLibrary" ), + .target( + name: "TableProSQLGrammar", + dependencies: [], + path: "Sources/TableProSQLGrammar" + ), .testTarget( name: "TableProConnectionLibraryTests", dependencies: ["TableProConnectionLibrary"], diff --git a/TablePro/Core/Utilities/SQL/PLSQLUnitTracker.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/PLSQLUnitTracker.swift similarity index 96% rename from TablePro/Core/Utilities/SQL/PLSQLUnitTracker.swift rename to Packages/TableProCore/Sources/TableProSQLGrammar/PLSQLUnitTracker.swift index b97d0d91f5..276b8bca05 100644 --- a/TablePro/Core/Utilities/SQL/PLSQLUnitTracker.swift +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/PLSQLUnitTracker.swift @@ -1,8 +1,3 @@ -// -// PLSQLUnitTracker.swift -// TablePro -// - import Foundation /// Statement boundaries in Oracle's language, where a `;` inside a PL/SQL unit separates the unit's own statements. @@ -27,7 +22,7 @@ import Foundation /// The final `;` of a unit belongs to it, except after a trigger whose body is a `CALL`: measured on Oracle 23ai, /// `CREATE TRIGGER ... CALL p(:NEW.a);` is stored INVALID and the same text without the `;` compiles. /// `scripts/check-oracle-plsql-terminators.sh` re-measures both rules against a live server. -struct PLSQLUnitTracker: SQLStatementBoundaryTracking { +public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { private enum Lead { case start case afterCreate @@ -76,11 +71,13 @@ struct PLSQLUnitTracker: SQLStatementBoundaryTracking { private var expectsDeclaration = false private var inlineQueryStarted = false - var needsWords: Bool { + public init() {} + + public var needsWords: Bool { lead != .decided || tracksConstructs } - var terminator: SQLStatementTerminator { + public var terminator: SQLStatementTerminator { switch kind { case .anonymousBlock, .opaque: return .partOfStatement @@ -91,7 +88,7 @@ struct PLSQLUnitTracker: SQLStatementBoundaryTracking { } } - var acceptsBindParameters: Bool { + public var acceptsBindParameters: Bool { switch kind { case .storedUnit, .embeddedSourceHeader, .opaque: return false @@ -100,7 +97,7 @@ struct PLSQLUnitTracker: SQLStatementBoundaryTracking { } } - mutating func observeWord(_ word: String) { + public mutating func observeWord(_ word: String) { previousSymbol = nil guard !inLabel else { return } let isMember = followsPeriod @@ -155,7 +152,7 @@ struct PLSQLUnitTracker: SQLStatementBoundaryTracking { observeBodyWord(word) } - mutating func observeSymbol(_ symbol: UInt16) { + public mutating func observeSymbol(_ symbol: UInt16) { if symbol == Self.lessThan, previousSymbol == Self.lessThan { inLabel = true previousSymbol = nil @@ -180,14 +177,14 @@ struct PLSQLUnitTracker: SQLStatementBoundaryTracking { followsPeriod = symbol == Self.period } - mutating func observeOpaqueToken() { + public mutating func observeOpaqueToken() { previousSymbol = nil followsPeriod = false guard tracksConstructs else { return } settlePendingBeforeNonWord() } - mutating func observeSemicolon() -> Bool { + public mutating func observeSemicolon() -> Bool { previousSymbol = nil followsPeriod = false guard lead == .decided else { return true } @@ -212,7 +209,7 @@ struct PLSQLUnitTracker: SQLStatementBoundaryTracking { } } - mutating func reset() { + public mutating func reset() { self = PLSQLUnitTracker() } diff --git a/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift new file mode 100644 index 0000000000..8a25983309 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift @@ -0,0 +1,64 @@ +import Foundation + +/// Where one SQL engine ends a string, a quoted identifier and a comment, as plain data. +/// +/// Every reader that walks SQL text takes one of these rather than an engine name, so the statement scanner, the +/// folding scanner, the import parser and the classifiers cannot disagree about where a literal ends. The value +/// carries no behaviour of its own: ``SqlLexer`` and ``SQLStatementScanner`` read it. +public struct SQLLexicalGrammar: OptionSet, Hashable, Sendable { + public let rawValue: UInt32 + + public init(rawValue: UInt32) { + self.rawValue = rawValue + } + + /// `'a\'b'` is one literal: a backslash keeps a single-quoted string open. + public static let backslashEscapesInSingleQuotes = SQLLexicalGrammar(rawValue: 1 << 0) + + /// `"a\"b"` is one token: a backslash keeps a double-quoted string or identifier open. + public static let backslashEscapesInDoubleQuotes = SQLLexicalGrammar(rawValue: 1 << 1) + + /// A backslash keeps a backtick-quoted identifier open. + public static let backslashEscapesInBackticks = SQLLexicalGrammar(rawValue: 1 << 2) + + /// `` `name` `` is a quoted identifier. + public static let backtickQuotes = SQLLexicalGrammar(rawValue: 1 << 3) + + /// `$$...$$` and `$tag$...$tag$` are literals whose body nothing inside can end. + public static let taggedDollarQuotes = SQLLexicalGrammar(rawValue: 1 << 9) + + /// `#` starts a comment that runs to the end of the line. + public static let hashLineComments = SQLLexicalGrammar(rawValue: 1 << 11) + + /// Oracle's `q'[...]'` literal, whose body runs to the matching delimiter followed by a quote. + public static let alternativeQuoting = SQLLexicalGrammar(rawValue: 1 << 7) + + /// A line holding only `/` ends the statement, as SQL*Plus reads it. + public static let slashLineTerminators = SQLLexicalGrammar(rawValue: 1 << 14) + + /// `$` and `#` continue an identifier, so `V$SESSION` is one word, and `$` may start a conditional compilation + /// directive such as `$IF`. + public static let dollarAndHashInIdentifiers = SQLLexicalGrammar(rawValue: 1 << 15) + + /// A `;` inside a PL/SQL unit belongs to the unit, which ``PLSQLUnitTracker`` decides. + public static let plsqlBlocks = SQLLexicalGrammar(rawValue: 1 << 16) + + public func backslashEscapes(inQuote quote: UInt16) -> Bool { + switch quote { + case SqlLexer.singleQuote: + return contains(.backslashEscapesInSingleQuotes) + case SqlLexer.doubleQuote: + return contains(.backslashEscapesInDoubleQuotes) + case SqlLexer.backtick: + return contains(.backslashEscapesInBackticks) + default: + return false + } + } + + public func isQuote(_ character: UInt16) -> Bool { + character == SqlLexer.singleQuote + || character == SqlLexer.doubleQuote + || (character == SqlLexer.backtick && contains(.backtickQuotes)) + } +} diff --git a/TablePro/Core/Utilities/SQL/SQLRoutineBodyTracker.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift similarity index 85% rename from TablePro/Core/Utilities/SQL/SQLRoutineBodyTracker.swift rename to Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift index 3e4bd64684..3170444f88 100644 --- a/TablePro/Core/Utilities/SQL/SQLRoutineBodyTracker.swift +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift @@ -1,8 +1,3 @@ -// -// SQLRoutineBodyTracker.swift -// TablePro -// - import Foundation /// Statement boundaries for the dialects whose only `;`-holding construct is a routine body written `BEGIN ... END`. @@ -10,26 +5,28 @@ import Foundation /// A `BEGIN` opens a body only inside a statement that defines a routine, for the safety reason recorded on /// ``SqlBlockStructure/opensRoutineDefinition(_:)``. The `;` never belongs to the statement: every one of these /// engines accepts a statement without it. -struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { +public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { private var sawStatementKeyword = false private var definesRoutine = false private var depth = 0 private var pendingBegin = false private var pendingEnd = false - var needsWords: Bool { + public init() {} + + public var needsWords: Bool { !sawStatementKeyword || definesRoutine } - var terminator: SQLStatementTerminator { + public var terminator: SQLStatementTerminator { .separator } - var acceptsBindParameters: Bool { + public var acceptsBindParameters: Bool { true } - mutating func observeWord(_ word: String) { + public mutating func observeWord(_ word: String) { if settlePending(before: word) { return } if !sawStatementKeyword { sawStatementKeyword = true @@ -48,15 +45,15 @@ struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { } } - mutating func observeSymbol(_ symbol: UInt16) { + public mutating func observeSymbol(_ symbol: UInt16) { settlePendingBeforeNonWord() } - mutating func observeOpaqueToken() { + public mutating func observeOpaqueToken() { settlePendingBeforeNonWord() } - mutating func observeSemicolon() -> Bool { + public mutating func observeSemicolon() -> Bool { pendingBegin = false if pendingEnd { pendingEnd = false @@ -65,7 +62,7 @@ struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { return depth == 0 } - mutating func reset() { + public mutating func reset() { self = SQLRoutineBodyTracker() } diff --git a/TablePro/Core/Utilities/SQL/SQLStatementBoundaryTracking.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementBoundaryTracking.swift similarity index 78% rename from TablePro/Core/Utilities/SQL/SQLStatementBoundaryTracking.swift rename to Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementBoundaryTracking.swift index 2a1d90e49a..4ece222eea 100644 --- a/TablePro/Core/Utilities/SQL/SQLStatementBoundaryTracking.swift +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementBoundaryTracking.swift @@ -1,13 +1,7 @@ -// -// SQLStatementBoundaryTracking.swift -// TablePro -// - import Foundation -import TableProPluginKit /// What the `;` that ends a statement is to that statement. -enum SQLStatementTerminator: Equatable, Sendable { +public enum SQLStatementTerminator: Equatable, Sendable { /// It separates the statement from the next one, and the driver never sees it. case separator @@ -26,7 +20,7 @@ enum SQLStatementTerminator: Equatable, Sendable { /// /// Words arrive uppercased. Comments and whitespace never arrive; a string literal, a quoted identifier or a /// dollar-quoted body arrives as ``observeOpaqueToken()``. -protocol SQLStatementBoundaryTracking { +public protocol SQLStatementBoundaryTracking { /// Whether words still bear on where this statement ends. A reader that has to assemble words itself can stop /// doing so once this is false, which keeps a plain `INSERT` dump free of the cost. var needsWords: Bool { get } @@ -47,14 +41,10 @@ protocol SQLStatementBoundaryTracking { mutating func reset() } -enum SQLStatementBoundaries { - /// The one place a dialect is matched to its grammar, so no reader can pick a different one. - static func makeTracker(for dialect: SqlDialect) -> any SQLStatementBoundaryTracking { - switch dialect { - case .oracle: - return PLSQLUnitTracker() - default: - return SQLRoutineBodyTracker() - } +public enum SQLStatementBoundaries { + /// The one place a grammar is matched to its statement boundaries, so no reader can pick a different one. + public static func makeTracker(for grammar: SQLLexicalGrammar) -> any SQLStatementBoundaryTracking { + guard grammar.contains(.plsqlBlocks) else { return SQLRoutineBodyTracker() } + return PLSQLUnitTracker() } } diff --git a/TablePro/Core/Utilities/SQL/SQLStatementScanner.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift similarity index 81% rename from TablePro/Core/Utilities/SQL/SQLStatementScanner.swift rename to Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift index e9514286d5..de06467029 100644 --- a/TablePro/Core/Utilities/SQL/SQLStatementScanner.swift +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift @@ -1,20 +1,14 @@ -// -// SQLStatementScanner.swift -// TablePro -// - import Foundation -import TableProPluginKit -enum SQLStatementScanner { - struct LocatedStatement { - let sql: String - let offset: Int - let hasContent: Bool - let terminator: SQLStatementTerminator - let acceptsBindParameters: Bool +public enum SQLStatementScanner { + public struct LocatedStatement: Sendable { + public let sql: String + public let offset: Int + public let hasContent: Bool + public let terminator: SQLStatementTerminator + public let acceptsBindParameters: Bool - init( + public init( sql: String, offset: Int, hasContent: Bool = true, @@ -29,7 +23,7 @@ enum SQLStatementScanner { } /// The statement's whole span in the document, in UTF-16 units. - var range: NSRange { + public var range: NSRange { NSRange(location: offset, length: (sql as NSString).length) } @@ -38,7 +32,7 @@ enum SQLStatementScanner { /// `offset` is the index just past the previous semicolon, so in a script written one statement per line it /// lands on the newline that ended the previous line. A decoration or a gutter anchor placed from ``range`` /// therefore starts a line early, and uses this instead. - var contentRange: NSRange { + public var contentRange: NSRange { let content = StatementBlank.contentRange(of: sql) return NSRange(location: offset + content.location, length: content.length) } @@ -53,21 +47,21 @@ enum SQLStatementScanner { /// The range is relative to the text the scan was given. A run started from a selection or from a single /// statement scans a fragment, so those callers shift the range onto the tab's whole query with ``offset(by:)`` /// before it travels any further. Everything downstream may then assume tab coordinates. - struct ExecutableStatement { - let sql: String - let range: NSRange + public struct ExecutableStatement: Sendable { + public let sql: String + public let range: NSRange /// False for a definition, whose `:name` is never a bind parameter; see /// ``SQLStatementBoundaryTracking/acceptsBindParameters``. - let acceptsBindParameters: Bool + public let acceptsBindParameters: Bool - init(sql: String, range: NSRange, acceptsBindParameters: Bool = true) { + public init(sql: String, range: NSRange, acceptsBindParameters: Bool = true) { self.sql = sql self.range = range self.acceptsBindParameters = acceptsBindParameters } - func offset(by delta: Int) -> ExecutableStatement { + public func offset(by delta: Int) -> ExecutableStatement { guard delta != 0 else { return self } return ExecutableStatement( sql: sql, @@ -79,12 +73,12 @@ enum SQLStatementScanner { /// Every statement in the document, with its span, in document order. /// - /// Unlike ``allStatements(in:dialect:)`` this keeps the empty and comment-only segments, flagged by + /// Unlike ``allStatements(in:grammar:)`` this keeps the empty and comment-only segments, flagged by /// ``LocatedStatement/hasContent``, because a caller drawing per-statement decorations has to be able to tell a /// segment that carries nothing from one that was never scanned. - static func locatedStatements(in sql: String, dialect: SqlDialect = .generic) -> [LocatedStatement] { + public static func locatedStatements(in sql: String, grammar: SQLLexicalGrammar) -> [LocatedStatement] { var results: [LocatedStatement] = [] - scan(sql: sql, cursorPosition: nil, dialect: dialect) { statement in + scan(sql: sql, cursorPosition: nil, grammar: grammar) { statement in results.append(statement) return true } @@ -97,8 +91,8 @@ enum SQLStatementScanner { /// caret-statement band and the navigation commands. A segment that carries nothing, meaning a comment or trailing /// whitespace, is not somewhere a caret should be sent and not something worth offering to run, so it is dropped /// here rather than at each call site where the three could drift apart. - static func navigableStatements(in sql: String, dialect: SqlDialect = .generic) -> [LocatedStatement] { - locatedStatements(in: sql, dialect: dialect) + public static func navigableStatements(in sql: String, grammar: SQLLexicalGrammar) -> [LocatedStatement] { + locatedStatements(in: sql, grammar: grammar) .filter { $0.hasContent && $0.contentRange.length > 0 } } @@ -109,13 +103,13 @@ enum SQLStatementScanner { /// /// A caret sitting in the trivia between two statements belongs to neither, so this answers with the next /// statement that starts after it. - static func statementStart( + public static func statementStart( after offset: Int, in sql: String, - dialect: SqlDialect = .generic + grammar: SQLLexicalGrammar ) -> Int? { var found: Int? - scan(sql: sql, cursorPosition: nil, dialect: dialect) { statement in + scan(sql: sql, cursorPosition: nil, grammar: grammar) { statement in guard statement.hasContent, statement.contentRange.length > 0 else { return true } guard statement.contentRange.location > offset else { return true } found = statement.contentRange.location @@ -128,15 +122,15 @@ enum SQLStatementScanner { /// /// Selection wants the far edge of the text, not the start of the next statement, or the last statement's own body /// could never be selected: past its start there is no next statement to reach for. - static func statementSelectionEnd( + public static func statementSelectionEnd( after offset: Int, in sql: String, - dialect: SqlDialect = .generic + grammar: SQLLexicalGrammar ) -> Int? { - if let next = statementStart(after: offset, in: sql, dialect: dialect) { + if let next = statementStart(after: offset, in: sql, grammar: grammar) { return next } - let end = navigableStatements(in: sql, dialect: dialect).last?.contentRange.upperBound + let end = navigableStatements(in: sql, grammar: grammar).last?.contentRange.upperBound return end.flatMap { $0 > offset ? $0 : nil } } @@ -144,13 +138,13 @@ enum SQLStatementScanner { /// /// A caret already past the start of its own statement goes to that statement's start first, which is how a /// reader steps back through a script without overshooting the statement they were reading. - static func statementStart( + public static func statementStart( before offset: Int, in sql: String, - dialect: SqlDialect = .generic + grammar: SQLLexicalGrammar ) -> Int? { var found: Int? - scan(sql: sql, cursorPosition: nil, dialect: dialect) { statement in + scan(sql: sql, cursorPosition: nil, grammar: grammar) { statement in guard statement.hasContent, statement.contentRange.length > 0 else { return true } guard statement.contentRange.location < offset else { return false } found = statement.contentRange.location @@ -160,18 +154,18 @@ enum SQLStatementScanner { } /// Returns statements as the driver receives them, for driver execution. - static func allStatements(in sql: String, dialect: SqlDialect = .generic) -> [String] { - executableStatements(in: sql, dialect: dialect).map(\.sql) + public static func allStatements(in sql: String, grammar: SQLLexicalGrammar) -> [String] { + executableStatements(in: sql, grammar: grammar).map(\.sql) } - /// The same statements ``allStatements(in:dialect:)`` returns, each with its span in the document. + /// The same statements ``allStatements(in:grammar:)`` returns, each with its span in the document. /// /// One enumeration produces both, because the alternative is two filters that have to agree and that nothing - /// checks. ``navigableStatements(in:dialect:)`` is deliberately not that second filter: it keeps the terminating + /// checks. ``navigableStatements(in:grammar:)`` is deliberately not that second filter: it keeps the terminating /// semicolon, while execution strips one, so pointing execution at it would change which text reaches the driver. - static func executableStatements(in sql: String, dialect: SqlDialect = .generic) -> [ExecutableStatement] { + public static func executableStatements(in sql: String, grammar: SQLLexicalGrammar) -> [ExecutableStatement] { var results: [ExecutableStatement] = [] - scan(sql: sql, cursorPosition: nil, dialect: dialect) { located in + scan(sql: sql, cursorPosition: nil, grammar: grammar) { located in guard located.hasContent, let statement = executableStatement(from: located) else { return true } results.append(statement) return true @@ -182,9 +176,9 @@ enum SQLStatementScanner { /// `text` as a driver receives it when it is sent whole: trimmed, and ending where its last statement's executable /// form ends, so a trailing separator comes off and a terminator that belongs to the statement stays. Empty when /// nothing but separators, comments or blanks is left. - static func executableText(of text: String, dialect: SqlDialect) -> String { + public static func executableText(of text: String, grammar: SQLLexicalGrammar) -> String { let trimmed = StatementBlank.trimming(text) - guard let last = executableStatements(in: trimmed, dialect: dialect).last else { return "" } + guard let last = executableStatements(in: trimmed, grammar: grammar).last else { return "" } let end = last.range.location + last.range.length return StatementBlank.trimming((trimmed as NSString).substring(to: end)) } @@ -193,7 +187,7 @@ enum SQLStatementScanner { /// /// A `;` that belongs to the statement stays, but only when there is something before it: a unit reduced to its /// terminator is as empty as any other. - static func executableStatement(from located: LocatedStatement) -> ExecutableStatement? { + public static func executableStatement(from located: LocatedStatement) -> ExecutableStatement? { let rawSQL = located.sql var content = StatementBlank.trimming(rawSQL[...]) if content.last == ";" { @@ -214,9 +208,9 @@ enum SQLStatementScanner { } /// Returns statements preserving trailing semicolons, for display/history/favorites. - static func allStatementsPreservingSemicolons(in sql: String) -> [String] { + public static func allStatementsPreservingSemicolons(in sql: String, grammar: SQLLexicalGrammar) -> [String] { var results: [String] = [] - scan(sql: sql, cursorPosition: nil) { statement in + scan(sql: sql, cursorPosition: nil, grammar: grammar) { statement in guard statement.hasContent else { return true } let trimmed = StatementBlank.trimming(statement.sql) let withoutSemicolon = trimmed.hasSuffix(";") @@ -230,14 +224,14 @@ enum SQLStatementScanner { return results } - static func statementAtCursor(in sql: String, cursorPosition: Int, dialect: SqlDialect = .generic) -> String { - let located = locatedStatementAtCursor(in: sql, cursorPosition: cursorPosition, dialect: dialect) + public static func statementAtCursor(in sql: String, cursorPosition: Int, grammar: SQLLexicalGrammar) -> String { + let located = locatedStatementAtCursor(in: sql, cursorPosition: cursorPosition, grammar: grammar) return executableStatement(from: located)?.sql ?? "" } - static func locatedStatementAtCursor(in sql: String, cursorPosition: Int, dialect: SqlDialect = .generic) -> LocatedStatement { + public static func locatedStatementAtCursor(in sql: String, cursorPosition: Int, grammar: SQLLexicalGrammar) -> LocatedStatement { var result = LocatedStatement(sql: "", offset: 0, hasContent: false) - scan(sql: sql, cursorPosition: cursorPosition, dialect: dialect) { statement in + scan(sql: sql, cursorPosition: cursorPosition, grammar: grammar) { statement in result = statement return false } @@ -255,7 +249,7 @@ enum SQLStatementScanner { private static func scan( sql: String, cursorPosition: Int?, - dialect: SqlDialect = .generic, + grammar: SQLLexicalGrammar, onStatement: (LocatedStatement) -> Bool ) { let nsQuery = sql as NSString @@ -264,12 +258,12 @@ enum SQLStatementScanner { let safePosition = cursorPosition.map { min(max(0, $0), length) } - var tracker = SQLStatementBoundaries.makeTracker(for: dialect) - var nonCode = NonCodeSpan(backslashEscapes: dialect != .oracle) + var tracker = SQLStatementBoundaries.makeTracker(for: grammar) + var nonCode = NonCodeSpan(grammar: grammar) var currentStart = 0 var hasStatementContent = false - let dollarQuotesEnabled = dialect.supportsDollarQuotes - let hashCommentsEnabled = dialect.supportsHashLineComments + let dollarQuotesEnabled = grammar.contains(.taggedDollarQuotes) + let hashCommentsEnabled = grammar.contains(.hashLineComments) var i = 0 var lastStatementWithContent: LocatedStatement? @@ -330,7 +324,7 @@ enum SQLStatementScanner { continue } - if SqlLexer.isQuote(ch) { + if grammar.isQuote(ch) { nonCode.state = .string(quote: ch) hasStatementContent = true tracker.observeOpaqueToken() @@ -347,21 +341,21 @@ enum SQLStatementScanner { continue } - if SqlBlockStructure.startsWord(nsQuery, at: i, length: length, dialect: dialect) { + if SqlBlockStructure.startsWord(nsQuery, at: i, length: length, grammar: grammar) { hasStatementContent = true - if dialect.supportsAlternativeQuoting, + if grammar.contains(.alternativeQuoting), let literal = SqlLexer.skipAlternativeQuotedString(nsQuery, at: i, length: length) { tracker.observeOpaqueToken() i = literal.next continue } if tracker.needsWords { - let word = SqlBlockStructure.readKeyword(nsQuery, at: i, length: length, dialect: dialect) + let word = SqlBlockStructure.readKeyword(nsQuery, at: i, length: length, grammar: grammar) tracker.observeWord(word.text) i = word.end } else { i += 1 - while i < length, SqlBlockStructure.continuesWord(nsQuery.character(at: i), dialect: dialect) { + while i < length, SqlBlockStructure.continuesWord(nsQuery.character(at: i), grammar: grammar) { i += 1 } } @@ -381,7 +375,7 @@ enum SQLStatementScanner { continue } - if dialect.endsStatementsAtSlashLines, ch == SqlLexer.slash, isSlashLine(nsQuery, at: i, length: length) { + if grammar.contains(.slashLineTerminators), ch == SqlLexer.slash, isSlashLine(nsQuery, at: i, length: length) { if hasStatementContent { guard finishSegment(at: i, hasContent: true) else { return } } @@ -426,9 +420,8 @@ enum SQLStatementScanner { var state = State.code - /// Whether a backslash keeps a string open. Every dialect but Oracle is scanned as if it did, which only ever - /// merges two statements; Oracle never escapes with one. - let backslashEscapes: Bool + /// Which quotes a backslash keeps open. + let grammar: SQLLexicalGrammar var isOpen: Bool { state != .code @@ -457,7 +450,7 @@ enum SQLStatementScanner { state = .code return i + (tag as NSString).length + 2 case let .string(quote): - if backslashEscapes, ch == SqlLexer.backslash, i + 1 < length { + if grammar.backslashEscapes(inQuote: quote), ch == SqlLexer.backslash, i + 1 < length { return i + 2 } guard ch == quote else { return i + 1 } @@ -472,7 +465,7 @@ enum SQLStatementScanner { /// Whether the `/` at `offset` stands alone on its line, which is what makes it SQL*Plus's terminator rather than /// a division. - static func isSlashLine(_ text: NSString, at offset: Int, length: Int) -> Bool { + public static func isSlashLine(_ text: NSString, at offset: Int, length: Int) -> Bool { var before = offset - 1 while before >= 0, isLineBlank(text.character(at: before)) { before -= 1 diff --git a/TablePro/Core/Utilities/SQL/SqlBlockStructure.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SqlBlockStructure.swift similarity index 86% rename from TablePro/Core/Utilities/SQL/SqlBlockStructure.swift rename to Packages/TableProCore/Sources/TableProSQLGrammar/SqlBlockStructure.swift index 6ab1c0f5d3..1401a4f959 100644 --- a/TablePro/Core/Utilities/SQL/SqlBlockStructure.swift +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SqlBlockStructure.swift @@ -1,10 +1,4 @@ -// -// SqlBlockStructure.swift -// TablePro -// - import Foundation -import TableProPluginKit /// The word level rules for the blocks a semicolon does not end. /// @@ -17,9 +11,9 @@ import TableProPluginKit /// What they do not share is how much they will let a block swallow: see `allowsBlock` on ``effect(of:endingAt:in:length:allowsBlock:)``. /// /// This sits beside ``SqlLexer`` rather than inside it because these rules are about words, not characters. -enum SqlBlockStructure { +public enum SqlBlockStructure { /// What a keyword does to the block nesting around it. - enum Effect: Equatable { + public enum Effect: Equatable, Sendable { case opensBlock /// Closes a block and swallows the keyword that follows up to `resumeAt`, which is how `END CASE` reads: the /// `CASE` names what is closing rather than opening another. @@ -28,7 +22,7 @@ enum SqlBlockStructure { } /// What an `END` means, given the word after it. - enum EndFollower: Equatable { + public enum EndFollower: Equatable, Sendable { /// `END IF`, `END LOOP` and their kin close a construct that never opened a block, and the word is theirs. case closesControlFlow /// `END CASE` closes the `CASE` that opened a block, and the word is theirs. @@ -65,16 +59,16 @@ enum SqlBlockStructure { private static let routineDefinitionOpeners: Set = ["CREATE", "ALTER", "REPLACE", "DECLARE"] /// Whether a statement opening with `keyword` can carry a `BEGIN ... END` body. - static func opensRoutineDefinition(_ keyword: String) -> Bool { + public static func opensRoutineDefinition(_ keyword: String) -> Bool { routineDefinitionOpeners.contains(keyword) } - static func beginStartsTransaction(followedBy keyword: String?) -> Bool { + public static func beginStartsTransaction(followedBy keyword: String?) -> Bool { guard let keyword else { return true } return transactionFollowers.contains(keyword) } - static func endingFollowedBy(_ keyword: String) -> EndFollower { + public static func endingFollowedBy(_ keyword: String) -> EndFollower { if keyword == "CASE" { return .closesCaseStatement } return controlFlowFollowers.contains(keyword) ? .closesControlFlow : .closesBlock } @@ -83,41 +77,43 @@ enum SqlBlockStructure { /// /// Returns an empty string when `offset` does not start an identifier, along with the next offset, so a caller can /// advance one character and carry on without a second bounds check. - static func readKeyword(_ text: NSString, at offset: Int, length: Int) -> (text: String, end: Int) { - readKeyword(text, at: offset, length: length, dialect: .generic) + public static func readKeyword(_ text: NSString, at offset: Int, length: Int) -> (text: String, end: Int) { + readKeyword(text, at: offset, length: length, grammar: []) } - /// The keyword at `offset` as `dialect` spells identifiers. + /// The keyword at `offset` as `grammar` spells identifiers. /// /// Oracle continues an identifier with `$` and `#`, so `V$SESSION` is one word, and starts a conditional /// compilation directive with `$`, so `$END` is one word that ``PLSQLUnitTracker`` can tell apart from `END`. - static func readKeyword( + public static func readKeyword( _ text: NSString, at offset: Int, length: Int, - dialect: SqlDialect + grammar: SQLLexicalGrammar ) -> (text: String, end: Int) { - guard offset < length, startsWord(text, at: offset, length: length, dialect: dialect) else { + guard offset < length, startsWord(text, at: offset, length: length, grammar: grammar) else { return ("", offset + 1) } var cursor = offset + 1 - while cursor < length, continuesWord(text.character(at: cursor), dialect: dialect) { + while cursor < length, continuesWord(text.character(at: cursor), grammar: grammar) { cursor += 1 } let word = text.substring(with: NSRange(location: offset, length: cursor - offset)) return (word.uppercased(), cursor) } - static func startsWord(_ text: NSString, at offset: Int, length: Int, dialect: SqlDialect) -> Bool { + public static func startsWord(_ text: NSString, at offset: Int, length: Int, grammar: SQLLexicalGrammar) -> Bool { let character = text.character(at: offset) if SqlDollarQuote.isIdentifierStart(character) { return true } - guard dialect == .oracle, character == SqlDollarQuote.dollar, offset + 1 < length else { return false } + guard grammar.contains(.dollarAndHashInIdentifiers), character == SqlDollarQuote.dollar, offset + 1 < length + else { return false } return SqlDollarQuote.isIdentifierStart(text.character(at: offset + 1)) } - static func continuesWord(_ character: UInt16, dialect: SqlDialect) -> Bool { + public static func continuesWord(_ character: UInt16, grammar: SQLLexicalGrammar) -> Bool { if SqlDollarQuote.isIdentifierPart(character) { return true } - return dialect == .oracle && (character == SqlDollarQuote.dollar || character == SqlLexer.hash) + return grammar.contains(.dollarAndHashInIdentifiers) + && (character == SqlDollarQuote.dollar || character == SqlLexer.hash) } /// What `keyword`, which ends at `wordEnd`, does to the block nesting. @@ -125,7 +121,7 @@ enum SqlBlockStructure { /// - Parameter allowsBlock: whether a block may open here at all. Folding passes `true`, because an anonymous /// `BEGIN ... END` is foldable and folding executes nothing. Splitting passes ``opensRoutineDefinition(_:)`` for /// the statement's first keyword, for the reason given on `routineDefinitionOpeners`. - static func effect( + public static func effect( of keyword: String, endingAt wordEnd: Int, in text: NSString, diff --git a/TablePro/Core/Utilities/SQL/SqlDollarQuote.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SqlDollarQuote.swift similarity index 81% rename from TablePro/Core/Utilities/SQL/SqlDollarQuote.swift rename to Packages/TableProCore/Sources/TableProSQLGrammar/SqlDollarQuote.swift index ccb40c288b..ec4bb57960 100644 --- a/TablePro/Core/Utilities/SQL/SqlDollarQuote.swift +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SqlDollarQuote.swift @@ -1,24 +1,19 @@ -// -// SqlDollarQuote.swift -// TablePro -// - import Foundation -enum SqlDollarQuote { - enum Opener { +public enum SqlDollarQuote { + public enum Opener: Sendable { case opener(length: Int, tag: String) case notOpener case needsMoreData } - static let dollar: unichar = 0x24 + public static let dollar: unichar = 0x24 - static func isIdentifierStart(_ ch: unichar) -> Bool { + public static func isIdentifierStart(_ ch: unichar) -> Bool { (ch >= 0x41 && ch <= 0x5A) || (ch >= 0x61 && ch <= 0x7A) || ch == 0x5F } - static func isIdentifierPart(_ ch: unichar) -> Bool { + public static func isIdentifierPart(_ ch: unichar) -> Bool { isIdentifierStart(ch) || (ch >= 0x30 && ch <= 0x39) } @@ -26,7 +21,7 @@ enum SqlDollarQuote { /// per PostgreSQL's rule that a dollar quote must be separated from a /// preceding identifier by whitespace (so `a$$b` is one identifier, not an /// opener). - static func isIdentifierContinuation(_ ch: unichar) -> Bool { + public static func isIdentifierContinuation(_ ch: unichar) -> Bool { isIdentifierPart(ch) || ch == dollar } @@ -34,7 +29,7 @@ enum SqlDollarQuote { /// like `$1`, or a non-tag dollar. A `$` glued to a preceding identifier is /// not an opener. Returns `needsMoreData` when the buffer ends mid-tag; a /// whole-string caller treats that as `notOpener`. - static func scanOpener(at pos: Int, in buffer: NSString, bufLen: Int) -> Opener { + public static func scanOpener(at pos: Int, in buffer: NSString, bufLen: Int) -> Opener { if pos > 0, isIdentifierContinuation(buffer.character(at: pos - 1)) { return .notOpener } @@ -62,7 +57,7 @@ enum SqlDollarQuote { /// Whether the closing delimiter for `tag` starts at `pos`. The tag match is /// exact and case-sensitive, per PostgreSQL. - static func matchesClose(at pos: Int, tag: String, in buffer: NSString, bufLen: Int) -> Bool { + public static func matchesClose(at pos: Int, tag: String, in buffer: NSString, bufLen: Int) -> Bool { let closeLen = (tag as NSString).length + 2 guard pos + closeLen <= bufLen else { return false } if buffer.character(at: pos) != dollar { return false } diff --git a/TablePro/Core/Utilities/SQL/SqlLexer.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SqlLexer.swift similarity index 73% rename from TablePro/Core/Utilities/SQL/SqlLexer.swift rename to Packages/TableProCore/Sources/TableProSQLGrammar/SqlLexer.swift index d8d612b8c9..756fa3048b 100644 --- a/TablePro/Core/Utilities/SQL/SqlLexer.swift +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SqlLexer.swift @@ -1,10 +1,4 @@ -// -// SqlLexer.swift -// TablePro -// - import Foundation -import TableProPluginKit /// The character level rules every SQL scanner in the app agrees on: which UTF-16 units matter, and how far a comment, /// a quoted string or a dollar quoted body runs. @@ -12,31 +6,31 @@ import TableProPluginKit /// Scanners differ in what they do with the structure they find, so they are not merged. Offsets are UTF-16 units, so /// an `NSString` can be walked in constant time per character. /// -/// ``skipQuotedString`` gates backslash escapes on the dialect, which is what PostgreSQL requires. +/// ``skipQuotedString`` gates backslash escapes on the grammar, which is what PostgreSQL requires. /// `SQLStatementScanner` deliberately keeps its own ungated handling, because splitting a script for execution is /// safer when a backslash never ends a string early; `SQLStatementScannerTests` pins that behaviour. Oracle is the /// exception there: a backslash is never an escape in Oracle, and scripts written for it routinely quote Windows paths. -enum SqlLexer { - static let space = UInt16(UnicodeScalar(" ").value) - static let tab = UInt16(UnicodeScalar("\t").value) - static let newline = UInt16(UnicodeScalar("\n").value) - static let carriageReturn = UInt16(UnicodeScalar("\r").value) - static let singleQuote = UInt16(UnicodeScalar("'").value) - static let doubleQuote = UInt16(UnicodeScalar("\"").value) - static let backtick = UInt16(UnicodeScalar("`").value) - static let backslash = UInt16(UnicodeScalar("\\").value) - static let dash = UInt16(UnicodeScalar("-").value) - static let slash = UInt16(UnicodeScalar("/").value) - static let star = UInt16(UnicodeScalar("*").value) - static let hash = UInt16(UnicodeScalar("#").value) - static let semicolon = UInt16(UnicodeScalar(";").value) - static let openParen = UInt16(UnicodeScalar("(").value) - static let closeParen = UInt16(UnicodeScalar(")").value) - static let exclamationMark = UInt16(UnicodeScalar("!").value) - static let smallQ = UInt16(UnicodeScalar("q").value) - static let capitalQ = UInt16(UnicodeScalar("Q").value) - static let smallN = UInt16(UnicodeScalar("n").value) - static let capitalN = UInt16(UnicodeScalar("N").value) +public enum SqlLexer { + public static let space = UInt16(UnicodeScalar(" ").value) + public static let tab = UInt16(UnicodeScalar("\t").value) + public static let newline = UInt16(UnicodeScalar("\n").value) + public static let carriageReturn = UInt16(UnicodeScalar("\r").value) + public static let singleQuote = UInt16(UnicodeScalar("'").value) + public static let doubleQuote = UInt16(UnicodeScalar("\"").value) + public static let backtick = UInt16(UnicodeScalar("`").value) + public static let backslash = UInt16(UnicodeScalar("\\").value) + public static let dash = UInt16(UnicodeScalar("-").value) + public static let slash = UInt16(UnicodeScalar("/").value) + public static let star = UInt16(UnicodeScalar("*").value) + public static let hash = UInt16(UnicodeScalar("#").value) + public static let semicolon = UInt16(UnicodeScalar(";").value) + public static let openParen = UInt16(UnicodeScalar("(").value) + public static let closeParen = UInt16(UnicodeScalar(")").value) + public static let exclamationMark = UInt16(UnicodeScalar("!").value) + public static let smallQ = UInt16(UnicodeScalar("q").value) + public static let capitalQ = UInt16(UnicodeScalar("Q").value) + public static let smallN = UInt16(UnicodeScalar("n").value) + public static let capitalN = UInt16(UnicodeScalar("N").value) private static let openBracket = UInt16(UnicodeScalar("[").value) private static let closeBracket = UInt16(UnicodeScalar("]").value) private static let openBrace = UInt16(UnicodeScalar("{").value) @@ -45,36 +39,41 @@ enum SqlLexer { private static let greaterThan = UInt16(UnicodeScalar(">").value) /// How far a scan ran, and how many lines it crossed. A caller that does not track lines ignores `newlines`. - struct Span { - let next: Int - let newlines: Int + public struct Span: Sendable { + public let next: Int + public let newlines: Int + + public init(next: Int, newlines: Int) { + self.next = next + self.newlines = newlines + } } - static func isWhitespace(_ character: UInt16) -> Bool { + public static func isWhitespace(_ character: UInt16) -> Bool { character == space || character == tab || character == newline || character == carriageReturn } - static func isQuote(_ character: UInt16) -> Bool { + public static func isQuote(_ character: UInt16) -> Bool { character == singleQuote || character == doubleQuote || character == backtick } - static func startsLineComment(_ text: NSString, at offset: Int, length: Int) -> Bool { + public static func startsLineComment(_ text: NSString, at offset: Int, length: Int) -> Bool { text.character(at: offset) == dash && offset + 1 < length && text.character(at: offset + 1) == dash } - static func startsBlockComment(_ text: NSString, at offset: Int, length: Int) -> Bool { + public static func startsBlockComment(_ text: NSString, at offset: Int, length: Int) -> Bool { text.character(at: offset) == slash && offset + 1 < length && text.character(at: offset + 1) == star } /// A MySQL conditional comment, whose body is executed rather than ignored. - static func startsConditionalComment(_ text: NSString, at offset: Int, length: Int) -> Bool { + public static func startsConditionalComment(_ text: NSString, at offset: Int, length: Int) -> Bool { startsBlockComment(text, at: offset, length: length) && offset + 2 < length && text.character(at: offset + 2) == exclamationMark } /// The offset of the newline that ends the line, or the end of the document. - static func endOfLine(_ text: NSString, from offset: Int, length: Int) -> Int { + public static func endOfLine(_ text: NSString, from offset: Int, length: Int) -> Int { var cursor = min(offset, length) while cursor < length, text.character(at: cursor) != newline { cursor += 1 @@ -83,7 +82,7 @@ enum SqlLexer { } /// Runs past `*/`, or to the end of the document when the comment is never closed. - static func skipBlockComment(_ text: NSString, from offset: Int, length: Int) -> Span { + public static func skipBlockComment(_ text: NSString, from offset: Int, length: Int) -> Span { var cursor = offset + 2 var newlines = 0 while cursor < length { @@ -99,7 +98,7 @@ enum SqlLexer { return Span(next: length, newlines: newlines) } - static func skipNestedBlockComment(_ text: NSString, from offset: Int, length: Int) -> Span { + public static func skipNestedBlockComment(_ text: NSString, from offset: Int, length: Int) -> Span { var cursor = offset + 2 var depth = 1 var newlines = 0 @@ -126,25 +125,9 @@ enum SqlLexer { /// Runs past the closing quote, or to the end of the document when the string is never closed. /// - /// A doubled quote always escapes. A backslash only escapes where the dialect says it does, so `'a\'` ends the + /// A doubled quote always escapes. A backslash only escapes where the grammar says it does, so `'a\'` ends the /// string on PostgreSQL and continues it on MySQL. - static func skipQuotedString( - _ text: NSString, - from offset: Int, - quote: UInt16, - length: Int, - dialect: SqlDialect - ) -> Span { - skipQuotedString( - text, - from: offset, - quote: quote, - length: length, - backslashEscapes: dialect.requiresBackslashEscapesInSingleQuotes - ) - } - - static func skipQuotedString( + public static func skipQuotedString( _ text: NSString, from offset: Int, quote: UInt16, @@ -180,7 +163,7 @@ enum SqlLexer { /// The body ends at the closing delimiter followed by a quote, so `q'[it's]'` is one literal although a plain scan /// would end it at `it'`. Bracket-like delimiters close with their partner. Returns nil when `offset` does not /// start one; a caller must only ask at the start of a word, because `xq'` is an identifier followed by a string. - static func skipAlternativeQuotedString(_ text: NSString, at offset: Int, length: Int) -> Span? { + public static func skipAlternativeQuotedString(_ text: NSString, at offset: Int, length: Int) -> Span? { var cursor = offset let first = text.character(at: cursor) if first == smallN || first == capitalN { @@ -218,7 +201,7 @@ enum SqlLexer { } /// Runs to the closing `$tag$`. `bodyEnd` is where the body stops, `next` is past the closing tag. - static func skipDollarQuotedBody( + public static func skipDollarQuotedBody( _ text: NSString, from bodyStart: Int, tag: String, diff --git a/TablePro/Core/Utilities/SQL/StatementBlank.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/StatementBlank.swift similarity index 76% rename from TablePro/Core/Utilities/SQL/StatementBlank.swift rename to Packages/TableProCore/Sources/TableProSQLGrammar/StatementBlank.swift index c41f996045..eaa63ae30c 100644 --- a/TablePro/Core/Utilities/SQL/StatementBlank.swift +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/StatementBlank.swift @@ -1,16 +1,11 @@ -// -// StatementBlank.swift -// TablePro -// - import Foundation -internal enum StatementBlank { +public enum StatementBlank { private static let interlinearAnnotations: ClosedRange = 0xFFF9...0xFFFB private static let asciiDelete: UInt32 = 0x7F private static let asciiSpace: UInt32 = 0x20 - static func isBlank(_ scalar: Unicode.Scalar) -> Bool { + public static func isBlank(_ scalar: Unicode.Scalar) -> Bool { guard !scalar.isASCII else { return scalar.value <= asciiSpace || scalar.value == asciiDelete } let properties = scalar.properties guard !properties.isAlphabetic else { return false } @@ -20,34 +15,34 @@ internal enum StatementBlank { || interlinearAnnotations.contains(scalar.value) } - static func isBlank(_ character: Character) -> Bool { + public static func isBlank(_ character: Character) -> Bool { character.unicodeScalars.allSatisfy { isBlank($0) } } - static func hasContent(_ text: String) -> Bool { + public static func hasContent(_ text: String) -> Bool { text.contains { !isBlank($0) } } - static func blankLength(in text: NSString, at offset: Int) -> Int { + public static func blankLength(in text: NSString, at offset: Int) -> Int { guard let scalar = scalar(in: text, at: offset), isBlank(scalar) else { return 0 } return scalar.utf16.count } - static func trimming(_ text: String) -> String { + public static func trimming(_ text: String) -> String { String(trimming(text[...])) } - static func trimming(_ text: Substring) -> Substring { + public static func trimming(_ text: Substring) -> Substring { let leading = trimmingLeading(text) guard let last = leading.lastIndex(where: { !isBlank($0) }) else { return leading[leading.endIndex...] } return leading[...last] } - static func trimmingLeading(_ text: Substring) -> Substring { + public static func trimmingLeading(_ text: Substring) -> Substring { text.drop { isBlank($0) } } - static func contentRange(of text: String) -> NSRange { + public static func contentRange(of text: String) -> NSRange { let content = trimming(text[...]) return NSRange(content.startIndex.. [LocatedStatement] { + locatedStatements(in: sql, grammar: dialect.lexicalGrammar) + } + + static func navigableStatements(in sql: String, dialect: SqlDialect = .generic) -> [LocatedStatement] { + navigableStatements(in: sql, grammar: dialect.lexicalGrammar) + } + + static func statementStart(after offset: Int, in sql: String, dialect: SqlDialect = .generic) -> Int? { + statementStart(after: offset, in: sql, grammar: dialect.lexicalGrammar) + } + + static func statementSelectionEnd(after offset: Int, in sql: String, dialect: SqlDialect = .generic) -> Int? { + statementSelectionEnd(after: offset, in: sql, grammar: dialect.lexicalGrammar) + } + + static func statementStart(before offset: Int, in sql: String, dialect: SqlDialect = .generic) -> Int? { + statementStart(before: offset, in: sql, grammar: dialect.lexicalGrammar) + } + + static func allStatements(in sql: String, dialect: SqlDialect = .generic) -> [String] { + allStatements(in: sql, grammar: dialect.lexicalGrammar) + } + + static func executableStatements(in sql: String, dialect: SqlDialect = .generic) -> [ExecutableStatement] { + executableStatements(in: sql, grammar: dialect.lexicalGrammar) + } + + static func executableText(of text: String, dialect: SqlDialect) -> String { + executableText(of: text, grammar: dialect.lexicalGrammar) + } + + static func allStatementsPreservingSemicolons(in sql: String) -> [String] { + allStatementsPreservingSemicolons(in: sql, grammar: SqlDialect.generic.lexicalGrammar) + } + + static func statementAtCursor(in sql: String, cursorPosition: Int, dialect: SqlDialect = .generic) -> String { + statementAtCursor(in: sql, cursorPosition: cursorPosition, grammar: dialect.lexicalGrammar) + } + + static func locatedStatementAtCursor( + in sql: String, + cursorPosition: Int, + dialect: SqlDialect = .generic + ) -> LocatedStatement { + locatedStatementAtCursor(in: sql, cursorPosition: cursorPosition, grammar: dialect.lexicalGrammar) + } +} + +extension SqlLexer { + static func skipQuotedString( + _ text: NSString, + from offset: Int, + quote: UInt16, + length: Int, + dialect: SqlDialect + ) -> Span { + skipQuotedString( + text, + from: offset, + quote: quote, + length: length, + backslashEscapes: dialect.requiresBackslashEscapesInSingleQuotes + ) + } +} + +extension SqlBlockStructure { + static func readKeyword( + _ text: NSString, + at offset: Int, + length: Int, + dialect: SqlDialect + ) -> (text: String, end: Int) { + readKeyword(text, at: offset, length: length, grammar: dialect.lexicalGrammar) + } + + static func startsWord(_ text: NSString, at offset: Int, length: Int, dialect: SqlDialect) -> Bool { + startsWord(text, at: offset, length: length, grammar: dialect.lexicalGrammar) + } + + static func continuesWord(_ character: UInt16, dialect: SqlDialect) -> Bool { + continuesWord(character, grammar: dialect.lexicalGrammar) + } +} + +extension SQLStatementBoundaries { + static func makeTracker(for dialect: SqlDialect) -> any SQLStatementBoundaryTracking { + makeTracker(for: dialect.lexicalGrammar) + } +} diff --git a/TablePro/Models/Query/StatementAnchor.swift b/TablePro/Models/Query/StatementAnchor.swift index 4e4086d433..63f2ccff86 100644 --- a/TablePro/Models/Query/StatementAnchor.swift +++ b/TablePro/Models/Query/StatementAnchor.swift @@ -7,6 +7,7 @@ import Foundation import TableProPluginKit +import TableProSQLGrammar /// The statement a result set was produced by, as something that can be found again. /// diff --git a/TablePro/Views/Editor/StatementRunController.swift b/TablePro/Views/Editor/StatementRunController.swift index 29722a12d3..a1b455e8ef 100644 --- a/TablePro/Views/Editor/StatementRunController.swift +++ b/TablePro/Views/Editor/StatementRunController.swift @@ -6,6 +6,7 @@ import AppKit import TableProEditorKit import TableProPluginKit +import TableProSQLGrammar import TableProTextEngine /// Which way a statement navigation command moves the caret. diff --git a/TablePro/Views/Main/Extensions/MainContentCoordinator+ExecuteAll.swift b/TablePro/Views/Main/Extensions/MainContentCoordinator+ExecuteAll.swift index 8e81946173..972ed0b2cc 100644 --- a/TablePro/Views/Main/Extensions/MainContentCoordinator+ExecuteAll.swift +++ b/TablePro/Views/Main/Extensions/MainContentCoordinator+ExecuteAll.swift @@ -4,6 +4,7 @@ // import Foundation +import TableProSQLGrammar extension MainContentCoordinator { func runAllStatements(extraCapabilities: CallerCapabilities = []) { diff --git a/TablePro/Views/Main/Extensions/MainContentCoordinator+Explain.swift b/TablePro/Views/Main/Extensions/MainContentCoordinator+Explain.swift index 74181db4f0..9ab35db601 100644 --- a/TablePro/Views/Main/Extensions/MainContentCoordinator+Explain.swift +++ b/TablePro/Views/Main/Extensions/MainContentCoordinator+Explain.swift @@ -10,6 +10,7 @@ import Foundation import TableProEditorKit import TableProPluginKit +import TableProSQLGrammar extension MainContentCoordinator { func runExplain(variant: ExplainVariant? = nil) { diff --git a/TablePro/Views/Main/MainContentCoordinator.swift b/TablePro/Views/Main/MainContentCoordinator.swift index 7913e4e27c..80fa97b69c 100644 --- a/TablePro/Views/Main/MainContentCoordinator.swift +++ b/TablePro/Views/Main/MainContentCoordinator.swift @@ -12,6 +12,7 @@ import os import SwiftUI import TableProEditorKit import TableProPluginKit +import TableProSQLGrammar /// Discard action types for unified alert handling enum DiscardAction { diff --git a/TableProTests/Core/Services/Execution/BatchTransactionPolicyTests.swift b/TableProTests/Core/Services/Execution/BatchTransactionPolicyTests.swift index d380f928e3..d4f72af965 100644 --- a/TableProTests/Core/Services/Execution/BatchTransactionPolicyTests.swift +++ b/TableProTests/Core/Services/Execution/BatchTransactionPolicyTests.swift @@ -5,6 +5,7 @@ @testable import TablePro import TableProPluginKit +import TableProSQLGrammar import Testing @Suite("Batch transaction policy") diff --git a/TableProTests/Core/Utilities/QueryStatementModelTests.swift b/TableProTests/Core/Utilities/QueryStatementModelTests.swift index 5ad53319a1..dac3c229f6 100644 --- a/TableProTests/Core/Utilities/QueryStatementModelTests.swift +++ b/TableProTests/Core/Utilities/QueryStatementModelTests.swift @@ -4,6 +4,7 @@ // import Foundation +import TableProSQLGrammar import Testing @testable import TablePro diff --git a/TableProTests/Core/Utilities/SQL/QueryClassifierTests.swift b/TableProTests/Core/Utilities/SQL/QueryClassifierTests.swift index 115b768fb9..8739574c01 100644 --- a/TableProTests/Core/Utilities/SQL/QueryClassifierTests.swift +++ b/TableProTests/Core/Utilities/SQL/QueryClassifierTests.swift @@ -5,6 +5,7 @@ import Foundation @testable import TablePro +import TableProSQLGrammar import Testing @Suite("QueryClassifier isExplainStatement") diff --git a/TableProTests/Core/Utilities/SQL/SQLExecutableStatementTests.swift b/TableProTests/Core/Utilities/SQL/SQLExecutableStatementTests.swift index b1937fb29b..0b83cacb93 100644 --- a/TableProTests/Core/Utilities/SQL/SQLExecutableStatementTests.swift +++ b/TableProTests/Core/Utilities/SQL/SQLExecutableStatementTests.swift @@ -8,6 +8,7 @@ import Foundation import TableProPluginKit +import TableProSQLGrammar import Testing @testable import TablePro diff --git a/TableProTests/Core/Utilities/SQL/SQLFoldScannerTests.swift b/TableProTests/Core/Utilities/SQL/SQLFoldScannerTests.swift index 694e15fab5..fd3990b599 100644 --- a/TableProTests/Core/Utilities/SQL/SQLFoldScannerTests.swift +++ b/TableProTests/Core/Utilities/SQL/SQLFoldScannerTests.swift @@ -5,6 +5,7 @@ import Foundation import TableProPluginKit +import TableProSQLGrammar import Testing @testable import TablePro diff --git a/TableProTests/Core/Utilities/SQL/SQLStatementBlockSplittingTests.swift b/TableProTests/Core/Utilities/SQL/SQLStatementBlockSplittingTests.swift index 3cc7edcbd5..15a3297740 100644 --- a/TableProTests/Core/Utilities/SQL/SQLStatementBlockSplittingTests.swift +++ b/TableProTests/Core/Utilities/SQL/SQLStatementBlockSplittingTests.swift @@ -9,6 +9,7 @@ import Foundation import TableProPluginKit +import TableProSQLGrammar import Testing @testable import TablePro diff --git a/TableProTests/Core/Utilities/SQL/SQLStatementNavigationTests.swift b/TableProTests/Core/Utilities/SQL/SQLStatementNavigationTests.swift index e0f67784e2..c4a18a7b16 100644 --- a/TableProTests/Core/Utilities/SQL/SQLStatementNavigationTests.swift +++ b/TableProTests/Core/Utilities/SQL/SQLStatementNavigationTests.swift @@ -8,6 +8,7 @@ import Foundation import TableProPluginKit +import TableProSQLGrammar import Testing @testable import TablePro diff --git a/TableProTests/Core/Utilities/SQL/SQLStatementPLSQLSplittingTests.swift b/TableProTests/Core/Utilities/SQL/SQLStatementPLSQLSplittingTests.swift index ce0eea609b..cb26f069b5 100644 --- a/TableProTests/Core/Utilities/SQL/SQLStatementPLSQLSplittingTests.swift +++ b/TableProTests/Core/Utilities/SQL/SQLStatementPLSQLSplittingTests.swift @@ -10,6 +10,7 @@ import Foundation @testable import TablePro import TableProPluginKit +import TableProSQLGrammar import Testing @Suite("SQL statement scanner - Oracle PL/SQL units") diff --git a/TableProTests/Core/Utilities/SQL/SQLStatementRangeTests.swift b/TableProTests/Core/Utilities/SQL/SQLStatementRangeTests.swift index af2efb555b..4bc4ac5242 100644 --- a/TableProTests/Core/Utilities/SQL/SQLStatementRangeTests.swift +++ b/TableProTests/Core/Utilities/SQL/SQLStatementRangeTests.swift @@ -8,6 +8,7 @@ import Foundation import TableProPluginKit +import TableProSQLGrammar import Testing @testable import TablePro diff --git a/TableProTests/Core/Utilities/SQL/SqlLexerTests.swift b/TableProTests/Core/Utilities/SQL/SqlLexerTests.swift index 5ea59edd9d..7890cea36b 100644 --- a/TableProTests/Core/Utilities/SQL/SqlLexerTests.swift +++ b/TableProTests/Core/Utilities/SQL/SqlLexerTests.swift @@ -5,6 +5,7 @@ import Foundation import TableProPluginKit +import TableProSQLGrammar import Testing @testable import TablePro diff --git a/TableProTests/Core/Utilities/SQL/StatementBlankTests.swift b/TableProTests/Core/Utilities/SQL/StatementBlankTests.swift index a1d40f90b1..b0aea34e89 100644 --- a/TableProTests/Core/Utilities/SQL/StatementBlankTests.swift +++ b/TableProTests/Core/Utilities/SQL/StatementBlankTests.swift @@ -5,6 +5,7 @@ import Foundation @testable import TablePro +import TableProSQLGrammar import TableProTextEngine import Testing diff --git a/TableProTests/Core/Utilities/SQLStatementScannerLocatedTests.swift b/TableProTests/Core/Utilities/SQLStatementScannerLocatedTests.swift index 016738ec19..c82872ac07 100644 --- a/TableProTests/Core/Utilities/SQLStatementScannerLocatedTests.swift +++ b/TableProTests/Core/Utilities/SQLStatementScannerLocatedTests.swift @@ -8,6 +8,7 @@ import Foundation import TableProPluginKit +import TableProSQLGrammar import Testing @testable import TablePro diff --git a/TableProTests/Core/Utilities/SQLStatementScannerTests.swift b/TableProTests/Core/Utilities/SQLStatementScannerTests.swift index 30bcc6d5ed..e31e6609bf 100644 --- a/TableProTests/Core/Utilities/SQLStatementScannerTests.swift +++ b/TableProTests/Core/Utilities/SQLStatementScannerTests.swift @@ -5,6 +5,7 @@ @testable import TablePro import TableProPluginKit +import TableProSQLGrammar import XCTest final class SQLStatementScannerTests: XCTestCase { diff --git a/TableProTests/Models/Query/StatementAnchorTests.swift b/TableProTests/Models/Query/StatementAnchorTests.swift index e27097cd42..f2314084ec 100644 --- a/TableProTests/Models/Query/StatementAnchorTests.swift +++ b/TableProTests/Models/Query/StatementAnchorTests.swift @@ -8,6 +8,7 @@ import Foundation import TableProPluginKit +import TableProSQLGrammar import Testing @testable import TablePro diff --git a/TableProTests/Views/Editor/StatementNavigationCommandTests.swift b/TableProTests/Views/Editor/StatementNavigationCommandTests.swift index b89d0b6d80..bff00b7514 100644 --- a/TableProTests/Views/Editor/StatementNavigationCommandTests.swift +++ b/TableProTests/Views/Editor/StatementNavigationCommandTests.swift @@ -12,6 +12,7 @@ import Foundation import TableProEditorKit import TableProGrammars import TableProPluginKit +import TableProSQLGrammar import TableProTextEngine import Testing diff --git a/TableProTests/Views/Editor/StatementRunPerformanceGuardTests.swift b/TableProTests/Views/Editor/StatementRunPerformanceGuardTests.swift index 624697f464..b0e569ba2f 100644 --- a/TableProTests/Views/Editor/StatementRunPerformanceGuardTests.swift +++ b/TableProTests/Views/Editor/StatementRunPerformanceGuardTests.swift @@ -13,6 +13,7 @@ import Foundation import TableProEditorKit import TableProGrammars import TableProPluginKit +import TableProSQLGrammar import TableProTextEngine import Testing diff --git a/TableProTests/Views/Main/TabQueryIsolationTests.swift b/TableProTests/Views/Main/TabQueryIsolationTests.swift index ed12f8ecfd..2fd0f087d0 100644 --- a/TableProTests/Views/Main/TabQueryIsolationTests.swift +++ b/TableProTests/Views/Main/TabQueryIsolationTests.swift @@ -15,6 +15,7 @@ import Foundation @testable import TablePro import TableProPluginKit +import TableProSQLGrammar import Testing @Suite("Tab query isolation", .serialized) diff --git a/project.yml b/project.yml index 3d748151bf..9eb7ed479f 100644 --- a/project.yml +++ b/project.yml @@ -234,7 +234,7 @@ targets: - package: Sparkle - package: Yams - package: TableProCore - products: [TableProAnalytics, TableProConnectionLibrary, TableProGeometry, TableProGoogleCloud, TableProImport, TableProNumberFormatting, TableProSyncTransport] + products: [TableProAnalytics, TableProConnectionLibrary, TableProGeometry, TableProGoogleCloud, TableProImport, TableProNumberFormatting, TableProSQLGrammar, TableProSyncTransport] settings: base: ASSETCATALOG_COMPILER_APPICON_NAME: AppIcon @@ -693,7 +693,7 @@ targets: dependencies: - target: TablePro - package: TableProCore - products: [TableProConnectionLibrary, TableProDocumentPath, TableProGeometry, TableProMSSQLCore, TableProNumberFormatting, TableProWeaviateCore] + products: [TableProConnectionLibrary, TableProDocumentPath, TableProGeometry, TableProMSSQLCore, TableProNumberFormatting, TableProSQLGrammar, TableProWeaviateCore] # The Kafka integration suite drives the real driver, so the test target links what # the plugin target links: zstd for decompression and NIO for the transport. - package: zstd From 47ad36d04bdef7ba0be64d685255a7b2b7823bc2 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 20 Sep 2026 10:32:10 +0700 Subject: [PATCH 2/5] fix(editor): lex SQL the way each engine does, so Safe Mode sees every statement --- CHANGELOG.md | 7 + Packages/TableProCore/Package.swift | 7 +- .../TableProQuery/SQLWriteClassifier.swift | 132 +----- .../SQLCodeProjection.swift | 44 ++ .../SQLLexicalGrammar.swift | 54 ++- .../SQLLexicalProfile.swift | 264 +++++++++++ .../SQLLexicalReadings.swift | 152 ++++++ .../TableProSQLGrammar/SQLNonCodeSpan.swift | 273 +++++++++++ .../SQLSeparatingCharacter.swift | 39 ++ .../SQLStatementScanner.swift | 111 +---- .../SqlBlockStructure.swift | 40 +- .../TableProSQLGrammar/SqlDollarQuote.swift | 58 ++- .../Sources/TableProSQLGrammar/SqlLexer.swift | 228 ++++++++- .../SQLWriteClassifierTests.swift | 46 ++ .../SQLLexicalCorpusTests.swift | 132 ++++++ .../SQLLexicalReadingsTests.swift | 90 ++++ .../SQLNonCodeSpanTests.swift | 141 ++++++ Plugins/DamengDriverPlugin/DamengPlugin.swift | 16 + .../DuckDBDriverPlugin/DuckDBConnection.swift | 2 +- .../DuckDBLexicalFeatures.swift | 18 + .../MariaDBPluginConnection.swift | 13 + .../MySQLLexicalFeatures.swift | 47 ++ .../MySQLDriverPlugin/MySQLPluginDriver.swift | 22 +- .../MySQLSessionFootprint.swift | 16 +- .../LibPQDriverCore.swift | 15 + .../SpannerPluginDriver.swift | 13 + .../PluginDatabaseDriver.swift | 8 + .../SQLDialectDescriptor.swift | 51 +- .../TableProPluginKit/SQLFeatureLexer.swift | 363 ++++++++++++++ .../SQLLexicalFeatures.swift | 98 ++++ .../SQLStatementSplitting.swift | 59 ++- .../SQLTransactionTracking.swift | 12 +- .../Autocomplete/SQLCompletionProvider.swift | 5 +- .../Autocomplete/SQLContextAnalyzer.swift | 4 +- .../Core/Compare/CompareSyncExecutor.swift | 2 +- .../QueryExecutionCoordinator+Helpers.swift | 3 +- ...QueryExecutionCoordinator+Parameters.swift | 13 +- .../QueryExecutionCoordinator.swift | 2 +- .../Access/DatabaseAccessBridge.swift | 16 +- TablePro/Core/Database/DatabaseDriver.swift | 6 + .../Diagnostics/ConfusableSQLCharacter.swift | 22 +- .../Core/Diagnostics/QueryDiagnostic.swift | 5 +- ...nfusableCharacterDiagnosticsProducer.swift | 5 +- .../SQLConfusableCharacterScanner.swift | 14 +- .../Core/Diagnostics/SQLLexicalRules.swift | 42 -- TablePro/Core/MCP/MCPConnectionBridge.swift | 2 +- .../Core/ObjectCopy/ObjectCopyPlanner.swift | 4 +- .../Core/Plugins/PluginDriverAdapter.swift | 1 + .../Core/Plugins/SqlFileImportSource.swift | 9 +- .../AutocommitOnlyStatement+MySQL.swift | 5 +- .../AutocommitOnlyStatement+PostgreSQL.swift | 5 +- .../AutocommitOnlyStatement+SQLServer.swift | 5 +- .../AutocommitOnlyStatement+SQLite.swift | 9 +- .../Execution/AutocommitOnlyStatement.swift | 17 +- .../Execution/BatchCommitStatement.swift | 9 +- .../Execution/BatchTransactionPolicy.swift | 24 +- .../Core/Services/Export/ImportService.swift | 3 +- .../Services/Query/LeadingRowsStatement.swift | 6 +- .../Core/Services/Query/QuerySqlParser.swift | 8 +- .../JavaScript/QueryStatementModel.swift | 36 +- .../SQL/CatalogChangeClassifier.swift | 13 +- .../SQL/Folding/SQLFoldScanner.swift | 102 ++-- .../SQL/InvisibleCharacterRemover.swift | 5 +- .../Utilities/SQL/QueryClassifier+PLSQL.swift | 50 +- .../Core/Utilities/SQL/QueryClassifier.swift | 444 +++++++++++++----- .../Core/Utilities/SQL/SQLFileParser.swift | 301 +++++++++--- .../Utilities/SQL/SQLLexicalResolver.swift | 101 ++++ .../Core/Utilities/SQL/SQLLimitDetector.swift | 104 +--- .../Core/Utilities/SQL/SQLNonCodeSpan.swift | 122 ----- .../Core/Utilities/SQL/SQLTokenCursor.swift | 83 +--- .../SQL/SelectSourceTableParser.swift | 197 +++----- .../SQL/SqlDialect+LexicalGrammar.swift | 120 ----- TablePro/Models/Query/StatementAnchor.swift | 4 +- .../Editor/Folding/FoldProviderResolver.swift | 7 +- .../Editor/Folding/SQLLineFoldProvider.swift | 10 +- ...ditorCoordinator+InvisibleCharacters.swift | 5 +- .../Views/Editor/SQLEditorCoordinator.swift | 5 +- .../Views/Editor/StatementRunController.swift | 20 +- TablePro/Views/Import/ImportDialog.swift | 4 +- .../MainContentCoordinator+Explain.swift | 4 +- .../Views/Main/MainContentCoordinator.swift | 14 +- .../DatabaseAccessBridgeStatementTests.swift | 7 +- .../SQLConfusableCharacterScannerTests.swift | 123 ++--- .../Diagnostics/SQLLexicalRulesTests.swift | 47 -- .../ExternalStatementGateLexicalTests.swift | 62 +++ .../Tools/MCPStatementGateTests.swift | 12 + .../SqlFileImportSourceCleanupTests.swift | 7 +- .../AutocommitOnlyStatementTests.swift | 22 +- .../Execution/BatchCommitStatementTests.swift | 2 +- .../BatchTransactionPolicyTests.swift | 2 +- .../Services/LeadingRowsStatementTests.swift | 5 +- .../Services/Query/QueryExecutorTests.swift | 25 +- .../Utilities/QueryStatementModelTests.swift | 16 +- .../SQL/InvisibleCharacterRemoverTests.swift | 25 +- .../SQL/QueryClassifierLexicalTests.swift | 177 +++++++ .../SQL/QueryClassifierPLSQLTests.swift | 8 +- .../SQL/SQLFileParserPLSQLTests.swift | 33 +- .../Utilities/SQL/SQLFileParserTests.swift | 43 +- .../Utilities/SQL/SQLFoldScannerTests.swift | 28 +- .../Utilities/SQL/SQLLimitDetectorTests.swift | 35 +- .../Utilities/SQL/SQLNonCodeSpanTests.swift | 37 +- .../SQL/SQLSetAssignmentsTests.swift | 5 +- .../SQL/SQLStatementBlockSplittingTests.swift | 40 +- .../SQL/SQLStatementNavigationTests.swift | 2 +- .../SQL/SQLStatementPLSQLSplittingTests.swift | 32 +- .../Utilities/SQL/SQLTokenCursorTests.swift | 59 ++- .../SQL/SelectSourceTableParserTests.swift | 51 +- .../Core/Utilities/SQL/SqlLexerTests.swift | 12 +- .../SQLStatementScannerLocatedTests.swift | 4 +- .../Utilities/SQLStatementScannerTests.swift | 29 +- TableProTests/Helpers/TestGrammar.swift | 93 ++++ .../Plugins/MySQLSessionFootprintTests.swift | 8 +- .../MySQLStatementClassificationTests.swift | 3 +- .../SQLLexicalFeatureMappingTests.swift | 86 ++++ .../Plugins/SQLStatementSplittingTests.swift | 34 ++ .../Plugins/SQLTransactionTrackingTests.swift | 16 + .../Editor/FoldCommandBehaviourTests.swift | 3 +- .../Editor/FoldPreviewHitTestTests.swift | 3 +- .../Editor/SQLFoldPerformanceGuardTests.swift | 9 +- .../StatementNavigationCommandTests.swift | 13 +- .../Editor/StatementRunControllerTests.swift | 10 +- .../StatementRunPerformanceGuardTests.swift | 12 +- docs/features/safe-mode.mdx | 4 + project.yml | 2 + scripts/check-sql-lexical-grammar.sh | 240 ++++++++++ 125 files changed, 4376 insertions(+), 1678 deletions(-) create mode 100644 Packages/TableProCore/Sources/TableProSQLGrammar/SQLCodeProjection.swift create mode 100644 Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift create mode 100644 Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift create mode 100644 Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift create mode 100644 Packages/TableProCore/Sources/TableProSQLGrammar/SQLSeparatingCharacter.swift create mode 100644 Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalCorpusTests.swift create mode 100644 Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalReadingsTests.swift create mode 100644 Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift create mode 100644 Plugins/DuckDBDriverPlugin/DuckDBLexicalFeatures.swift create mode 100644 Plugins/MySQLDriverPlugin/MySQLLexicalFeatures.swift create mode 100644 Plugins/TableProPluginKit/SQLFeatureLexer.swift create mode 100644 Plugins/TableProPluginKit/SQLLexicalFeatures.swift delete mode 100644 TablePro/Core/Diagnostics/SQLLexicalRules.swift create mode 100644 TablePro/Core/Utilities/SQL/SQLLexicalResolver.swift delete mode 100644 TablePro/Core/Utilities/SQL/SQLNonCodeSpan.swift delete mode 100644 TablePro/Core/Utilities/SQL/SqlDialect+LexicalGrammar.swift delete mode 100644 TableProTests/Core/Diagnostics/SQLLexicalRulesTests.swift create mode 100644 TableProTests/Core/Execution/ExternalStatementGateLexicalTests.swift create mode 100644 TableProTests/Core/Utilities/SQL/QueryClassifierLexicalTests.swift create mode 100644 TableProTests/Helpers/TestGrammar.swift create mode 100644 TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift create mode 100755 scripts/check-sql-lexical-grammar.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 674a0591f8..51f5f33fe4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -136,6 +136,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Require Face ID** turned off on iPhone and iPad without authenticating. - SSH private keys pasted or picked on iPhone and iPad saved in plain text in the connections file. - Test Connection on iPhone and iPad saving its credentials to the Keychain, synced with Sync Passwords on. +- Statements hidden behind a backslash in a string skipped Safe Mode on PostgreSQL, DuckDB, SQL Server, SQLite and Dameng. +- Statements hidden inside a nested block comment skipped Safe Mode on PostgreSQL, DuckDB and SQL Server. +- Statements hidden behind a bracketed identifier skipped Safe Mode on SQL Server and SQLite. +- Statements hidden in a dollar-quoted string skipped Safe Mode on DuckDB, Snowflake and Cassandra, or PostgreSQL with a non-ASCII tag. +- Statements hidden behind an engine's own literal or comment forms, such as `E'\''`, `'''` or `--1`, skipped Safe Mode. +- Statements hidden the same ways passed the one-statement check on MCP and AI chat queries. +- Writes hidden in a dollar-quoted string, a nested comment or a bracketed identifier skipped Safe Mode on iPhone and iPad. ## [0.75.0] - 2026-09-18 diff --git a/Packages/TableProCore/Package.swift b/Packages/TableProCore/Package.swift index 143afd80a6..0713da1b9a 100644 --- a/Packages/TableProCore/Package.swift +++ b/Packages/TableProCore/Package.swift @@ -74,7 +74,7 @@ let package = Package( ), .target( name: "TableProQuery", - dependencies: ["TableProModels", "TableProPluginKit", "TableProCoreTypes"], + dependencies: ["TableProModels", "TableProPluginKit", "TableProCoreTypes", "TableProSQLGrammar"], path: "Sources/TableProQuery" ), .target( @@ -177,6 +177,11 @@ let package = Package( dependencies: ["TableProQuery", "TableProModels", "TableProPluginKit"], path: "Tests/TableProQueryTests" ), + .testTarget( + name: "TableProSQLGrammarTests", + dependencies: ["TableProSQLGrammar"], + path: "Tests/TableProSQLGrammarTests" + ), .testTarget( name: "TableProAnalyticsTests", dependencies: ["TableProAnalytics"], diff --git a/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift b/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift index 864b6942ae..4d0d6c4aa0 100644 --- a/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift +++ b/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift @@ -1,5 +1,6 @@ import Foundation import TableProModels +import TableProSQLGrammar /// Decides whether a statement batch writes, so Safe Mode can block or confirm it. /// @@ -7,6 +8,10 @@ import TableProModels /// short, closed set of read verbs. Everything else writes, including a keyword this classifier has /// never heard of. A write-keyword allowlist cannot be safe, because anything it has not been /// taught, or anything hidden behind a leading comment, runs unguarded. +/// +/// The batch is split into statements the way its engine lexes it, by the grammar the Mac reads +/// too, and under every reading the engine could be using: iOS drivers send the whole text in one +/// call, so a `DELETE` hidden behind a quote only one reading closes still runs. public enum SQLWriteClassifier { /// `EXPLAIN` and `PRAGMA` are deliberately absent: `EXPLAIN ANALYZE DELETE …` runs the delete on /// PostgreSQL, and `PRAGMA journal_mode = WAL` writes on SQLite and DuckDB. @@ -25,18 +30,23 @@ public enum SQLWriteClassifier { public static func isWriteQuery(_ sql: String, databaseType: DatabaseType) -> Bool { if databaseType == .redis { return redisWrites(sql) } - let statements = splitStatements(sql) - guard !statements.isEmpty else { return false } - return statements.contains(where: statementWrites) + let readings = SQLLexicalReadings.resolve(databaseTypeId: databaseType.rawValue, declared: nil, session: nil) + return readings.distinct(for: sql).contains { grammar in + SQLStatementScanner.executableStatements(in: sql, grammar: grammar).contains { statement in + statementWrites(statement.sql, grammar: grammar) + } + } } - private static func statementWrites(_ statement: String) -> Bool { - let body = strippingLeadingTrivia(statement) - // Content this cannot name is content it cannot vouch for. + /// Reads the statement's code with every literal and comment blanked by its engine's own rules. A MySQL + /// `/*! ... */` is kept as the code it is, so a keyword the server runs from one is never mistaken for a read. + private static func statementWrites(_ statement: String, grammar: SQLLexicalGrammar) -> Bool { + let code = SQLCodeProjection.code(of: statement, grammar: grammar, revealingExecutableComments: true) + let body = String(code.drop { $0.isWhitespace }) guard let keyword = leadingKeyword(of: body) else { return true } - if keyword == "WITH" { return commonTableExpressionWrites(body) } + if keyword == "WITH" { return commonTableExpressionWrites(code) } if readUnlessIntoKeywords.contains(keyword) { - return containsWord("INTO", in: maskingLiteralsAndComments(body).uppercased()) + return containsWord("INTO", in: code.uppercased()) } return !readKeywords.contains(keyword) } @@ -76,10 +86,10 @@ public enum SQLWriteClassifier { "TS.RANGE", "TS.REVRANGE", "TS.GET", "TS.MGET", "TS.INFO", "FT.SEARCH", "FT.INFO" ] - /// A CTE's leading keyword says nothing about what the statement finally does, so the body is - /// searched for a write verb with its literals and comments blanked out first. - private static func commonTableExpressionWrites(_ statement: String) -> Bool { - let masked = maskingLiteralsAndComments(statement).uppercased() + /// A CTE's leading keyword says nothing about what the statement finally does, so its code is + /// searched for a write verb. + private static func commonTableExpressionWrites(_ code: String) -> Bool { + let masked = code.uppercased() return writeKeywordsInsideCTE.contains { keyword in containsWord(keyword, in: masked) } @@ -131,102 +141,4 @@ public enum SQLWriteClassifier { } return String(rest) } - - /// Splits on semicolons that are not inside a string, an identifier quote, or a comment. - private static func splitStatements(_ sql: String) -> [String] { - let characters = Array(sql) - let quoted = quotedOrCommentMask(characters) - var statements: [String] = [] - var current = "" - - for (index, character) in characters.enumerated() { - if character == ";", !quoted[index] { - appendIfMeaningful(current, to: &statements) - current = "" - continue - } - current.append(character) - } - appendIfMeaningful(current, to: &statements) - return statements - } - - private static func appendIfMeaningful(_ statement: String, to statements: inout [String]) { - let body = strippingLeadingTrivia(statement).trimmingCharacters(in: .whitespacesAndNewlines) - guard !body.isEmpty else { return } - statements.append(statement) - } - - private static func maskingLiteralsAndComments(_ sql: String) -> String { - let characters = Array(sql) - let quoted = quotedOrCommentMask(characters) - return String(characters.enumerated().map { quoted[$0.offset] ? " " : $0.element }) - } - - /// One pass marking every position that sits inside a string literal, a quoted identifier, or a - /// comment. Doubled and backslash-escaped quotes do not end a literal. Splitting and blanking - /// both read this rather than re-deriving the state, so neither can drift from the other. - private static func quotedOrCommentMask(_ characters: [Character]) -> [Bool] { - var mask = [Bool](repeating: false, count: characters.count) - var index = 0 - var quote: Character? - - while index < characters.count { - let character = characters[index] - let following = index + 1 < characters.count ? characters[index + 1] : nil - - if let open = quote { - mask[index] = true - // A backslash is not an escape under PostgreSQL's standard_conforming_strings, which - // PostgreSQLDriver sets on. Treating it as one would swallow the terminating quote - // and hide the rest of the batch, so it is left alone: ending a literal early splits - // more statements, and more statements can only classify toward write. - if character == open { - if following == open { - mask[index + 1] = true - index += 2 - continue - } - quote = nil - } - index += 1 - continue - } - - if character == "-", following == "-" { - while index < characters.count, !characters[index].isNewline { - mask[index] = true - index += 1 - } - continue - } - - if character == "/", following == "*" { - mask[index] = true - mask[index + 1] = true - index += 2 - while index < characters.count { - if characters[index] == "*", index + 1 < characters.count, characters[index + 1] == "/" { - mask[index] = true - mask[index + 1] = true - index += 2 - break - } - mask[index] = true - index += 1 - } - continue - } - - if character == "'" || character == "\"" || character == "`" { - quote = character - mask[index] = true - index += 1 - continue - } - - index += 1 - } - return mask - } } diff --git a/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCodeProjection.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCodeProjection.swift new file mode 100644 index 0000000000..6c8339d59a --- /dev/null +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCodeProjection.swift @@ -0,0 +1,44 @@ +import Foundation + +/// SQL text with every comment, literal and quoted identifier blanked, read by one grammar. +/// +/// A classifier looks for keywords in what this returns, so a `DROP` inside a string never counts and a `DROP` after +/// a string that ended where the engine ends it always does. Each blanked UTF-16 unit becomes a space and each line +/// feed stays, so an offset into the projection is an offset into the original text. +public enum SQLCodeProjection { + /// - Parameter revealingExecutableComments: whether a MySQL `/*! ... */` body is kept as code. A gate reveals it + /// whatever the grammar says, because the only cost of reading an ignored comment as code is a stricter tier. + public static func code( + of text: String, + grammar: SQLLexicalGrammar, + revealingExecutableComments: Bool = false + ) -> String { + let source = text as NSString + let length = source.length + guard length > 0 else { return "" } + var units = [UInt16](repeating: 0, count: length) + source.getCharacters(&units, range: NSRange(location: 0, length: length)) + + var index = 0 + while index < length { + if revealingExecutableComments, + let opener = SqlLexer.executableCommentOpenerLength(source, at: index, length: length) { + index += opener + continue + } + guard let span = SQLNonCodeSpan.span(at: index, in: source, grammar: grammar) else { + index += 1 + continue + } + blank(&units, from: span.start, to: span.end) + index = max(span.end, index + 1) + } + return String(utf16CodeUnits: units, count: length) + } + + private static func blank(_ units: inout [UInt16], from start: Int, to end: Int) { + for offset in start.. Bool { switch quote { case SqlLexer.singleQuote: diff --git a/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift new file mode 100644 index 0000000000..e9670bc16b --- /dev/null +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift @@ -0,0 +1,264 @@ +import Foundation + +/// What TablePro knows about one engine's lexing: the grammar it splits with, and the facts it cannot settle alone. +/// +/// A fact lands in ``undetermined`` for one of two reasons. Either the server decides it per session, as MySQL's +/// `NO_BACKSLASH_ESCAPES`, PostgreSQL's `standard_conforming_strings` and Dameng's `BACKSLASH_ESCAPE` do, or nobody has +/// measured it against a live server. Either way a gate reads both values, which is always the safe direction: an +/// extra reading can only raise the statement count and the tier. `scripts/check-sql-lexical-grammar.sh` re-measures +/// the facts marked measured below. +public struct SQLLexicalProfile: Sendable, Hashable { + public let grammar: SQLLexicalGrammar + + /// Whole grammars the engine may be running instead of ``grammar``, as Spanner does per database. + public let alternatives: [SQLLexicalGrammar] + + public let undetermined: SQLLexicalGrammar + + /// Every combination of the undetermined facts over ``grammar`` and each alternative, ``grammar`` first. + public let readings: [SQLLexicalGrammar] + + public init( + grammar: SQLLexicalGrammar, + alternatives: [SQLLexicalGrammar] = [], + undetermined: SQLLexicalGrammar = [] + ) { + self.grammar = grammar + self.alternatives = alternatives + self.undetermined = undetermined + self.readings = Self.readings(of: grammar, alternatives: alternatives, undetermined: undetermined) + } + + private static func readings( + of grammar: SQLLexicalGrammar, + alternatives: [SQLLexicalGrammar], + undetermined: SQLLexicalGrammar + ) -> [SQLLexicalGrammar] { + let bits = (0..<32).map { SQLLexicalGrammar(rawValue: 1 << $0) }.filter { undetermined.contains($0) } + var subsets: [SQLLexicalGrammar] = [[]] + for bit in bits { + subsets += subsets.map { $0.union(bit) } + } + var seen: Set = [grammar] + var result = [grammar] + for base in [grammar] + alternatives { + for subset in subsets { + let reading = base.subtracting(undetermined).union(subset) + if seen.insert(reading).inserted { + result.append(reading) + } + } + } + return result + } + + public static func curated(forDatabaseTypeId typeId: String) -> SQLLexicalProfile? { + profiles[typeId] + } + + /// The database type ids the curated table covers. + public static var curatedDatabaseTypeIds: Set { + Set(profiles.keys) + } + + /// Every reading of every curated engine, the readings a gate has to take for an engine it knows nothing about. + /// A PL/SQL unit grammar is left out: it chooses statement boundaries rather than token rules, and an unknown + /// engine keeps the routine-body boundaries every non-Oracle engine has. + public static let everyKnownReading: [SQLLexicalGrammar] = { + var seen: Set = [] + var result: [SQLLexicalGrammar] = [] + for typeId in profiles.keys.sorted() { + guard let profile = profiles[typeId] else { continue } + for reading in profile.readings { + let unitless = reading.subtracting([.plsqlBlocks, .delimiterDirective]) + if seen.insert(unitless).inserted { + result.append(unitless) + } + } + } + return result + }() + + // MARK: - Grammars + + /// PostgreSQL 17.11, measured: a backslash is literal under `standard_conforming_strings`, block comments nest, + /// `$tag$` takes non-ASCII and case-sensitive tags, `E'\''` escapes, a carriage return ends `--`, and `#` and `//` + /// are operators. + static let postgreSQL: SQLLexicalGrammar = [ + .taggedDollarQuotes, .nestedBlockComments, .escapeStringPrefix, .carriageReturnEndsLineComments, + ] + + /// MySQL 8.4.11 and MariaDB 11.8.9, measured: a backslash escapes in `'` and `"` strings and is literal in a + /// backtick identifier, block comments do not nest, `#` is a comment, `/*! */` runs, `--` needs whitespace after + /// it, and only a line feed ends a line comment. + static let mySQL: SQLLexicalGrammar = [ + .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backtickQuotes, .hashLineComments, + .executableComments, .dashCommentsNeedWhitespace, .delimiterDirective, + ] + + /// SQLite 3.54, measured: brackets and backticks quote identifiers, `]]` does not escape, a backslash is literal, + /// a Tcl-style `$name(...)` parameter swallows quotes and semicolons, and only a line feed ends `--`. + static let sqlite: SQLLexicalGrammar = [ + .backtickQuotes, .bracketQuotedIdentifiers, .parenthesizedParameterNames, + ] + + /// DuckDB 1.5.2 and 1.5.4, measured: lexes strings, comments and dollar quotes as PostgreSQL does. + static let duckDB: SQLLexicalGrammar = [ + .taggedDollarQuotes, .nestedBlockComments, .escapeStringPrefix, .carriageReturnEndsLineComments, + ] + + /// Oracle 23.26, measured: a backslash is literal, block comments do not nest, `q'[...]'` is a literal, `$` and + /// `#` continue an identifier, and only a line feed ends `--`. + static let oracle: SQLLexicalGrammar = [ + .alternativeQuoting, .slashLineTerminators, .dollarAndHashInIdentifiers, .plsqlBlocks, + ] + + /// DM8 V8, measured in compatibility modes 0, 2, 4 and 7: `q'[...]'` is a literal, `//` is a comment, block + /// comments do not nest, `$` and `#` continue an identifier, a carriage return ends a line comment, and a + /// backslash is literal unless the server runs with `BACKSLASH_ESCAPE = 1`, which the plugin detects at connect. + static let dameng: SQLLexicalGrammar = [ + .alternativeQuoting, .doubleSlashLineComments, .dollarAndHashInIdentifiers, .carriageReturnEndsLineComments, + ] + + /// Azure SQL Edge 15.0 (the SQL Server 2019 engine), measured: brackets quote identifiers and `]]` escapes, block + /// comments nest, a backslash is literal, a carriage return ends `--`, and T-SQL has no `$$`, `#` or `//` comment. + static let sqlServer: SQLLexicalGrammar = [ + .bracketQuotedIdentifiers, .doubledClosingBracketEscapes, .nestedBlockComments, + .carriageReturnEndsLineComments, + ] + + /// Spanner's GoogleSQL dialect on the emulator, measured, and BigQuery by the ZetaSQL grammar the two share: a + /// backslash escapes in every quote including backticks, `'''` and `"""` literals, `#` comments, flat block + /// comments. + static let googleSQL: SQLLexicalGrammar = [ + .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backslashEscapesInBackticks, + .backtickQuotes, .tripleQuotedStrings, .hashLineComments, + ] + + /// Spanner's PostgreSQL dialect on the emulator, measured: lexes as PostgreSQL does. + static let spannerPostgreSQL: SQLLexicalGrammar = [ + .taggedDollarQuotes, .nestedBlockComments, .escapeStringPrefix, + ] + + /// ClickHouse, from its syntax reference: a backslash escapes in strings and in both quoted identifier forms, `#` + /// and `--` are comments, and `$heredoc$` bodies are literals. Not measured. + static let clickHouse: SQLLexicalGrammar = [ + .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backslashEscapesInBackticks, + .backtickQuotes, .hashLineComments, .taggedDollarQuotes, + ] + + /// Snowflake, from its reference: a backslash escapes in single-quoted strings, `$$` bodies are literals and `//` + /// is a comment. Not measured. + static let snowflake: SQLLexicalGrammar = [ + .backslashEscapesInSingleQuotes, .untaggedDollarQuotes, .doubleSlashLineComments, + ] + + /// Databend, from its tokenizer: a backslash escapes in `'` and `"`, backticks quote identifiers and `$$` bodies + /// are literals. Not measured. + static let databend: SQLLexicalGrammar = [ + .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backtickQuotes, .untaggedDollarQuotes, + ] + + /// CQL for Cassandra and ScyllaDB, from the reference: `$$` bodies are literals and `//` is a comment. Not + /// measured. + static let cql: SQLLexicalGrammar = [.untaggedDollarQuotes, .doubleSlashLineComments] + + /// SurrealQL, from its reference: a backslash escapes in both quotes, and `#`, `//` and `--` are comments. Not + /// measured. + static let surrealQL: SQLLexicalGrammar = [ + .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backtickQuotes, .hashLineComments, + .doubleSlashLineComments, + ] + + /// Engines whose statements are commands or JSON documents rather than SQL. Their splitting is what it has always + /// been: a backslash escapes inside any quote, as it does in JSON and in `redis-cli`. + static let commandLine: SQLLexicalGrammar = [ + .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backslashEscapesInBackticks, + .backtickQuotes, + ] + + private static let profiles: [String: SQLLexicalProfile] = { + let postgreSQLFamily = SQLLexicalProfile(grammar: postgreSQL, undetermined: [.backslashEscapesInSingleQuotes]) + let mySQLFamily = SQLLexicalProfile( + grammar: mySQL, + undetermined: [.backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes] + ) + let mySQLCompatible = SQLLexicalProfile( + grammar: mySQL, + undetermined: [ + .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .dashCommentsNeedWhitespace, + .executableComments, .carriageReturnEndsLineComments, + ] + ) + let sqliteFamily = SQLLexicalProfile(grammar: sqlite) + let cqlFamily = SQLLexicalProfile(grammar: cql, undetermined: [.carriageReturnEndsLineComments]) + let commandLineFamily = SQLLexicalProfile(grammar: commandLine) + return [ + "PostgreSQL": postgreSQLFamily, + "Greenplum": postgreSQLFamily, + "AlloyDB": postgreSQLFamily, + "Citus": postgreSQLFamily, + "PGlite": postgreSQLFamily, + "Redshift": SQLLexicalProfile( + grammar: postgreSQL, + undetermined: [.backslashEscapesInSingleQuotes, .nestedBlockComments, .escapeStringPrefix] + ), + "CockroachDB": SQLLexicalProfile( + grammar: postgreSQL, + undetermined: [.nestedBlockComments, .taggedDollarQuotes, .carriageReturnEndsLineComments] + ), + "MySQL": SQLLexicalProfile( + grammar: mySQL, + undetermined: [.backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .taggedDollarQuotes] + ), + "MariaDB": mySQLFamily, + "TiDB": mySQLCompatible, + "OceanBase": mySQLCompatible, + "Databend": SQLLexicalProfile( + grammar: databend, + undetermined: [.hashLineComments, .nestedBlockComments, .carriageReturnEndsLineComments] + ), + "SQLite": sqliteFamily, + "libSQL": sqliteFamily, + "Turso": sqliteFamily, + "Cloudflare D1": sqliteFamily, + "DuckDB": SQLLexicalProfile(grammar: duckDB), + "Oracle": SQLLexicalProfile(grammar: oracle), + "Dameng": SQLLexicalProfile( + grammar: dameng, + undetermined: [.backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes] + ), + "SQL Server": SQLLexicalProfile(grammar: sqlServer), + "ClickHouse": SQLLexicalProfile( + grammar: clickHouse, + undetermined: [.nestedBlockComments, .taggedDollarQuotes, .carriageReturnEndsLineComments] + ), + "Snowflake": SQLLexicalProfile( + grammar: snowflake, + undetermined: [.nestedBlockComments, .carriageReturnEndsLineComments] + ), + "BigQuery": SQLLexicalProfile(grammar: googleSQL, undetermined: [.carriageReturnEndsLineComments]), + "Spanner": SQLLexicalProfile( + grammar: googleSQL, + alternatives: [spannerPostgreSQL], + undetermined: [.carriageReturnEndsLineComments] + ), + "Trino": SQLLexicalProfile(grammar: .ansi, undetermined: [.carriageReturnEndsLineComments]), + "Teradata": SQLLexicalProfile( + grammar: .ansi, + undetermined: [.nestedBlockComments, .carriageReturnEndsLineComments] + ), + "Cassandra": cqlFamily, + "ScyllaDB": cqlFamily, + "DynamoDB": SQLLexicalProfile(grammar: .ansi, undetermined: [.carriageReturnEndsLineComments]), + "SurrealDB": SQLLexicalProfile(grammar: surrealQL, undetermined: [.carriageReturnEndsLineComments]), + "Redis": commandLineFamily, + "MongoDB": commandLineFamily, + "etcd": commandLineFamily, + "Elasticsearch": commandLineFamily, + "Typesense": commandLineFamily, + "Weaviate": commandLineFamily, + "Kafka": commandLineFamily, + ] + }() +} diff --git a/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift new file mode 100644 index 0000000000..e7a29f3b20 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift @@ -0,0 +1,152 @@ +import Foundation + +/// The lexical facts a driver learned from its own session, such as MySQL's `NO_BACKSLASH_ESCAPES` from the status +/// flags of the last reply. +public struct SQLSessionLexicalFacts: Sendable, Hashable { + /// The facts the session settled. + public let determined: SQLLexicalGrammar + + /// The settled facts that hold. Anything outside ``determined`` is ignored. + public let enabled: SQLLexicalGrammar + + public init(determined: SQLLexicalGrammar, enabled: SQLLexicalGrammar) { + self.determined = determined + self.enabled = enabled.intersection(determined) + } + + public func applied(to grammar: SQLLexicalGrammar) -> SQLLexicalGrammar { + grammar.subtracting(determined).union(enabled) + } +} + +/// Every way an engine could lex a text, and the one it is split with for execution. +/// +/// A gate never trusts a single reading where the engine could be using another: it classifies under every reading in +/// ``all`` and takes the worst, the highest statement count and the most severe tier. That covers a fact a server +/// decides per session and a driver did not report, a fact nobody measured, and an engine TablePro does not know at +/// all. Execution splits with ``execution`` alone, because a statement can only be sent one way. +/// +/// A fact the session reported never narrows what a gate reads. It widens it when it disagrees with the curated +/// reading, and it chooses ``execution``. Narrowing would be wrong twice over: MySQL applies a `SET sql_mode` to the +/// rest of the same call, and its status flag after connect does not yet show a mode `init_connect` set, both +/// measured on 8.4 and 11.8. +public struct SQLLexicalReadings: Sendable, Hashable { + public let execution: SQLLexicalGrammar + + /// Every plausible reading, ``execution`` first. + public let all: [SQLLexicalGrammar] + + public init(execution: SQLLexicalGrammar, all: [SQLLexicalGrammar]) { + self.execution = execution + self.all = Self.deduplicated([execution] + all) + } + + public static func single(_ grammar: SQLLexicalGrammar) -> SQLLexicalReadings { + SQLLexicalReadings(execution: grammar, all: [grammar]) + } + + public var isAmbiguous: Bool { + all.count > 1 + } + + /// Resolves the readings for a connection: the curated profile when TablePro knows the engine, which wins over + /// anything a plugin declares because a registry plugin may be built against an older kit; otherwise what the + /// plugin declared; otherwise every grammar TablePro knows, split for execution as standard SQL. + public static func resolve( + databaseTypeId: String, + declared: SQLLexicalGrammar?, + session: SQLSessionLexicalFacts? + ) -> SQLLexicalReadings { + let base = baseReadings(databaseTypeId: databaseTypeId, declared: declared) + guard let session else { return base } + let execution = session.applied(to: base.execution) + return SQLLexicalReadings(execution: execution, all: base.all) + } + + /// The readings that can lex `text` differently from each other, execution first. + /// + /// A fact no character in `text` can trigger is dropped from every reading before they are compared, so a query + /// with no backslash is lexed once however many backslash rules are in doubt. + public func distinct(for text: String) -> [SQLLexicalGrammar] { + guard isAmbiguous else { return all } + let relevant = SQLLexicalRelevance.facts(triggeredBy: text as NSString) + return Self.deduplicated(all.map { $0.intersection(relevant) }) + } + + private static func baseReadings(databaseTypeId: String, declared: SQLLexicalGrammar?) -> SQLLexicalReadings { + if let profile = SQLLexicalProfile.curated(forDatabaseTypeId: databaseTypeId) { + return SQLLexicalReadings(execution: profile.grammar, all: profile.readings) + } + if let declared { + return .single(declared) + } + return SQLLexicalReadings(execution: .ansi, all: SQLLexicalProfile.everyKnownReading) + } + + private static func deduplicated(_ grammars: [SQLLexicalGrammar]) -> [SQLLexicalGrammar] { + var seen: Set = [] + return grammars.filter { seen.insert($0).inserted } + } +} + +/// Which lexical facts a text could possibly exercise, found in one pass over its UTF-16 units. +enum SQLLexicalRelevance { + private static let openBracket = UInt16(UnicodeScalar("[").value) + private static let at = UInt16(UnicodeScalar("@").value) + private static let colon = UInt16(UnicodeScalar(":").value) + + static func facts(triggeredBy text: NSString) -> SQLLexicalGrammar { + let length = text.length + var facts: SQLLexicalGrammar = [.plsqlBlocks, .delimiterDirective] + var blockCommentOpeners = 0 + var index = 0 + while index < length { + let unit = text.character(at: index) + let next: UInt16 = index + 1 < length ? text.character(at: index + 1) : 0 + switch unit { + case SqlLexer.backslash: + facts.formUnion([ + .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backslashEscapesInBackticks, + .escapeStringPrefix, + ]) + case SqlLexer.backtick: + facts.insert(.backtickQuotes) + case openBracket: + facts.formUnion([.bracketQuotedIdentifiers, .doubledClosingBracketEscapes]) + case SqlLexer.singleQuote, SqlLexer.doubleQuote: + if next == unit, index + 2 < length, text.character(at: index + 2) == unit { + facts.insert(.tripleQuotedStrings) + } + case SqlDollarQuote.dollar: + facts.formUnion([ + .untaggedDollarQuotes, .taggedDollarQuotes, .dollarAndHashInIdentifiers, + .parenthesizedParameterNames, + ]) + case SqlLexer.hash: + facts.formUnion([.hashLineComments, .dollarAndHashInIdentifiers, .parenthesizedParameterNames]) + case at, colon: + facts.insert(.parenthesizedParameterNames) + case SqlLexer.slash: + facts.insert(.slashLineTerminators) + if next == SqlLexer.slash { facts.insert(.doubleSlashLineComments) } + if next == SqlLexer.star { + blockCommentOpeners += 1 + facts.insert(.executableComments) + } + case SqlLexer.dash where next == SqlLexer.dash: + facts.insert(.dashCommentsNeedWhitespace) + case SqlLexer.carriageReturn: + facts.insert(.carriageReturnEndsLineComments) + case SqlLexer.smallQ, SqlLexer.capitalQ: + if next == SqlLexer.singleQuote { facts.insert(.alternativeQuoting) } + default: + break + } + index += 1 + } + if blockCommentOpeners > 1 { + facts.insert(.nestedBlockComments) + } + return facts + } +} diff --git a/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift new file mode 100644 index 0000000000..642f6d969c --- /dev/null +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift @@ -0,0 +1,273 @@ +import Foundation + +/// The one lexer: where a comment, a literal or a quoted identifier that starts at an offset ends, read by an engine's +/// ``SQLLexicalGrammar``. +/// +/// Everything that has to know where code stops asks here: the statement scanner, the folding scanner, the token +/// cursor, the row-limit detector, the code-only projection the classifiers read, and the diagnostics. Two readers +/// that disagree about where a string ends can disagree about where a statement ends, and the gate that tiers one +/// statement would then let the engine run two. +public enum SQLNonCodeSpan { + public enum Kind: Sendable, Equatable { + case lineComment + case blockComment + + /// MySQL's `/*! ... */`, whose body the server runs. It is code to a reader of statements, and one opaque + /// token to a reader of boundaries. + case executableComment + + /// A string, a dollar-quoted body, or a quoted identifier: one token nothing inside can end early. + case quoted + + /// SQLite's `$name(...)` parameter, which swallows quotes and semicolons up to its `)`. + case parameter + + public var isComment: Bool { + self == .lineComment || self == .blockComment + } + } + + public struct Span: Sendable, Equatable { + public let kind: Kind + public let start: Int + + /// Past the closing delimiter, or the end of the text when the span never closes. + public let end: Int + + /// Where the body stops, before the closing delimiter. + public let contentEnd: Int + + /// Line feeds inside the span. A line comment ends at its line feed and does not count it. + public let newlines: Int + + /// False when the text ended before the span's closing delimiter, so everything after its start is inside it. + public let isTerminated: Bool + + public init(kind: Kind, start: Int, end: Int, contentEnd: Int, newlines: Int, isTerminated: Bool) { + self.kind = kind + self.start = start + self.end = end + self.contentEnd = contentEnd + self.newlines = newlines + self.isTerminated = isTerminated + } + } + + private static let openBracket = UInt16(UnicodeScalar("[").value) + private static let closeBracket = UInt16(UnicodeScalar("]").value) + private static let capitalE = UInt16(UnicodeScalar("E").value) + private static let smallE = UInt16(UnicodeScalar("e").value) + + /// The comment, literal or quoted identifier starting at `index`, or nil when `index` starts code. + public static func span(at index: Int, in text: NSString, grammar: SQLLexicalGrammar) -> Span? { + let length = text.length + guard index >= 0, index < length else { return nil } + let character = text.character(at: index) + + if let comment = commentSpan(at: index, character: character, in: text, length: length, grammar: grammar) { + return comment + } + if grammar.isQuote(character) { + return quotedSpan(at: index, quote: character, in: text, length: length, grammar: grammar) + } + if grammar.contains(.bracketQuotedIdentifiers), character == openBracket { + let span = SqlLexer.skipBracketedIdentifier( + text, + from: index, + length: length, + doubledCloseEscapes: grammar.contains(.doubledClosingBracketEscapes) + ) + return closedSpan(.quoted, at: index, span, closerLength: 1) + } + if let prefixed = prefixedLiteralSpan(at: index, character: character, in: text, length: length, grammar: grammar) { + return prefixed + } + if character == SqlDollarQuote.dollar, let dollar = dollarQuotedSpan(at: index, in: text, grammar: grammar) { + return dollar + } + guard grammar.contains(.parenthesizedParameterNames), + let parameter = SqlLexer.skipParenthesizedParameterName(text, at: index, length: length) + else { + return nil + } + return Span( + kind: .parameter, + start: index, + end: parameter.next, + contentEnd: parameter.next, + newlines: 0, + isTerminated: parameter.isClosed + ) + } + + /// Where the span starting at `index` ends, or nil when `index` starts code. An executable comment is code, so it + /// answers nil: the reader walks into it and reads its body. + public static func end(at index: Int, in text: NSString, grammar: SQLLexicalGrammar) -> Int? { + guard let span = span(at: index, in: text, grammar: grammar), span.kind != .executableComment else { + return nil + } + return span.end + } + + /// Whether `unit` continues a word, so a prefix like `E` or `q` glued to it is part of an identifier rather than + /// the start of a literal. + public static func isWordUnit(_ unit: UInt16) -> Bool { + if unit < 0x80 { + return SqlDollarQuote.isIdentifierPart(unit) + } + return !SQLSeparatingCharacter.isSeparating(unit) + } + + // MARK: - Comments + + private static func commentSpan( + at index: Int, + character: UInt16, + in text: NSString, + length: Int, + grammar: SQLLexicalGrammar + ) -> Span? { + if startsLineComment(at: index, character: character, in: text, length: length, grammar: grammar) { + let end = SqlLexer.endOfLineComment( + text, + from: index, + length: length, + carriageReturnEnds: grammar.contains(.carriageReturnEndsLineComments) + ) + return Span(kind: .lineComment, start: index, end: end, contentEnd: end, newlines: 0, isTerminated: true) + } + guard SqlLexer.startsBlockComment(text, at: index, length: length) else { return nil } + if grammar.contains(.executableComments), + let opener = SqlLexer.executableCommentOpenerLength(text, at: index, length: length) { + let span = SqlLexer.skipExecutableComment( + text, + from: index, + openerLength: opener, + length: length, + backslashEscapes: grammar.backslashEscapes(inQuote:) + ) + return closedSpan(.executableComment, at: index, span, closerLength: 2) + } + let span = grammar.contains(.nestedBlockComments) + ? SqlLexer.skipNestedBlockComment(text, from: index, length: length) + : SqlLexer.skipBlockComment(text, from: index, length: length) + return closedSpan(.blockComment, at: index, span, closerLength: 2) + } + + private static func startsLineComment( + at index: Int, + character: UInt16, + in text: NSString, + length: Int, + grammar: SQLLexicalGrammar + ) -> Bool { + if character == SqlLexer.dash { + return SqlLexer.startsDashComment( + text, + at: index, + length: length, + needsWhitespace: grammar.contains(.dashCommentsNeedWhitespace) + ) + } + if character == SqlLexer.hash { + return grammar.contains(.hashLineComments) + } + guard character == SqlLexer.slash, grammar.contains(.doubleSlashLineComments) else { return false } + return SqlLexer.startsDoubleSlash(text, at: index, length: length) + } + + // MARK: - Literals + + private static func quotedSpan( + at index: Int, + quote: UInt16, + in text: NSString, + length: Int, + grammar: SQLLexicalGrammar + ) -> Span { + let backslashEscapes = grammar.backslashEscapes(inQuote: quote) + if grammar.contains(.tripleQuotedStrings), SqlLexer.startsTripleQuote(text, at: index, length: length) { + let span = SqlLexer.skipTripleQuotedString( + text, + from: index, + length: length, + backslashEscapes: backslashEscapes + ) + return closedSpan(.quoted, at: index, span, closerLength: 3) + } + let span = SqlLexer.skipQuotedString( + text, + from: index, + quote: quote, + length: length, + backslashEscapes: backslashEscapes + ) + return closedSpan(.quoted, at: index, span, closerLength: 1) + } + + /// `E'...'` and `q'[...]'`, which only start where a word could: `xE'` is an identifier followed by a string. + private static func prefixedLiteralSpan( + at index: Int, + character: UInt16, + in text: NSString, + length: Int, + grammar: SQLLexicalGrammar + ) -> Span? { + guard index == 0 || !isWordUnit(text.character(at: index - 1)) else { return nil } + if grammar.contains(.escapeStringPrefix), + character == capitalE || character == smallE, + index + 1 < length, + text.character(at: index + 1) == SqlLexer.singleQuote { + let span = SqlLexer.skipQuotedString( + text, + from: index + 1, + quote: SqlLexer.singleQuote, + length: length, + backslashEscapes: true + ) + return closedSpan(.quoted, at: index, span, closerLength: 1) + } + guard grammar.contains(.alternativeQuoting), + let span = SqlLexer.skipAlternativeQuotedString(text, at: index, length: length) + else { + return nil + } + return closedSpan(.quoted, at: index, span, closerLength: 2) + } + + private static func dollarQuotedSpan(at index: Int, in text: NSString, grammar: SQLLexicalGrammar) -> Span? { + let length = text.length + guard let style = grammar.dollarQuoteStyle, + case .opener(let openerLength, let tag) = SqlDollarQuote.scanOpener( + at: index, + in: text, + bufLen: length, + style: style + ) + else { + return nil + } + let body = SqlLexer.skipDollarQuotedBody(text, from: index + openerLength, tag: tag, length: length) + return Span( + kind: .quoted, + start: index, + end: body.span.next, + contentEnd: body.bodyEnd, + newlines: body.span.newlines, + isTerminated: body.span.isClosed + ) + } + + /// A span whose closer is `closerLength` units long. One that ran to the end of the text without a closer runs its + /// content to the end of the text. + private static func closedSpan(_ kind: Kind, at start: Int, _ span: SqlLexer.Span, closerLength: Int) -> Span { + Span( + kind: kind, + start: start, + end: span.next, + contentEnd: span.isClosed ? max(start, span.next - closerLength) : span.next, + newlines: span.newlines, + isTerminated: span.isClosed + ) + } +} diff --git a/Packages/TableProCore/Sources/TableProSQLGrammar/SQLSeparatingCharacter.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLSeparatingCharacter.swift new file mode 100644 index 0000000000..50646f8130 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLSeparatingCharacter.swift @@ -0,0 +1,39 @@ +import Foundation + +/// The non-ASCII characters that end a word although no engine reads them as SQL: full-width punctuation, curly +/// quotes and non-ASCII spaces, which arrive by pasting from a word processor or typing with a CJK input method. +/// +/// Every other non-ASCII character continues a word, which is how unquoted non-Latin identifiers stay whole. +public enum SQLSeparatingCharacter: Sendable, Equatable { + case fullWidthPunctuation + case curlyQuote + case nonASCIISpace + + public static let fullWidthForms: ClosedRange = 0xFF01...0xFF5E + + public static func isFullWidthWordUnit(_ unit: UInt16) -> Bool { + switch unit { + case 0xFF10...0xFF19, 0xFF21...0xFF3A, 0xFF3F, 0xFF41...0xFF5A: + return true + default: + return false + } + } + + public static func kind(of unit: UInt16) -> SQLSeparatingCharacter? { + switch unit { + case fullWidthForms where !isFullWidthWordUnit(unit): + return .fullWidthPunctuation + case 0x2018, 0x2019, 0x201C, 0x201D: + return .curlyQuote + case 0x00A0, 0x2000...0x200A, 0x202F, 0x205F, 0x3000: + return .nonASCIISpace + default: + return nil + } + } + + public static func isSeparating(_ unit: UInt16) -> Bool { + kind(of: unit) != nil + } +} diff --git a/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift index de06467029..efc3d6f575 100644 --- a/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift @@ -259,11 +259,8 @@ public enum SQLStatementScanner { let safePosition = cursorPosition.map { min(max(0, $0), length) } var tracker = SQLStatementBoundaries.makeTracker(for: grammar) - var nonCode = NonCodeSpan(grammar: grammar) var currentStart = 0 var hasStatementContent = false - let dollarQuotesEnabled = grammar.contains(.taggedDollarQuotes) - let hashCommentsEnabled = grammar.contains(.hashLineComments) var i = 0 var lastStatementWithContent: LocatedStatement? @@ -298,57 +295,22 @@ public enum SQLStatementScanner { while i < length { let ch = nsQuery.character(at: i) - if nonCode.isOpen { - i = nonCode.advance(from: i, in: nsQuery, length: length) - continue - } - - if SqlLexer.startsLineComment(nsQuery, at: i, length: length) { - nonCode.state = .lineComment - i += 2 - continue - } - - if hashCommentsEnabled && ch == SqlLexer.hash { - nonCode.state = .lineComment - i += 1 - continue - } - - if SqlLexer.startsBlockComment(nsQuery, at: i, length: length) { - if SqlLexer.startsConditionalComment(nsQuery, at: i, length: length) { + if let span = SQLNonCodeSpan.span(at: i, in: nsQuery, grammar: grammar) { + switch span.kind { + case .lineComment, .blockComment: + break + case .executableComment: hasStatementContent = true + case .quoted, .parameter: + hasStatementContent = true + tracker.observeOpaqueToken() } - nonCode.state = .blockComment - i += 2 - continue - } - - if grammar.isQuote(ch) { - nonCode.state = .string(quote: ch) - hasStatementContent = true - tracker.observeOpaqueToken() - i += 1 - continue - } - - if dollarQuotesEnabled, ch == SqlDollarQuote.dollar, - case .opener(let openerLength, let tag) = SqlDollarQuote.scanOpener(at: i, in: nsQuery, bufLen: length) { - nonCode.state = .dollarQuote(tag: tag) - hasStatementContent = true - tracker.observeOpaqueToken() - i += openerLength + i = max(span.end, i + 1) continue } if SqlBlockStructure.startsWord(nsQuery, at: i, length: length, grammar: grammar) { hasStatementContent = true - if grammar.contains(.alternativeQuoting), - let literal = SqlLexer.skipAlternativeQuotedString(nsQuery, at: i, length: length) { - tracker.observeOpaqueToken() - i = literal.next - continue - } if tracker.needsWords { let word = SqlBlockStructure.readKeyword(nsQuery, at: i, length: length, grammar: grammar) tracker.observeWord(word.text) @@ -408,61 +370,6 @@ public enum SQLStatementScanner { } } - /// The literal or comment the scan is inside, where nothing is a token and no `;` ends anything. - private struct NonCodeSpan { - enum State: Equatable { - case code - case lineComment - case blockComment - case string(quote: UInt16) - case dollarQuote(tag: String) - } - - var state = State.code - - /// Which quotes a backslash keeps open. - let grammar: SQLLexicalGrammar - - var isOpen: Bool { - state != .code - } - - /// Steps past one unit of the open span, closing it where it ends, and returns the next offset. - mutating func advance(from i: Int, in text: NSString, length: Int) -> Int { - let ch = text.character(at: i) - switch state { - case .code: - return i + 1 - case .lineComment: - if ch == SqlLexer.newline { state = .code } - return i + 1 - case .blockComment: - guard ch == SqlLexer.star, i + 1 < length, text.character(at: i + 1) == SqlLexer.slash else { - return i + 1 - } - state = .code - return i + 2 - case let .dollarQuote(tag): - guard ch == SqlDollarQuote.dollar, - SqlDollarQuote.matchesClose(at: i, tag: tag, in: text, bufLen: length) else { - return i + 1 - } - state = .code - return i + (tag as NSString).length + 2 - case let .string(quote): - if grammar.backslashEscapes(inQuote: quote), ch == SqlLexer.backslash, i + 1 < length { - return i + 2 - } - guard ch == quote else { return i + 1 } - if i + 1 < length, text.character(at: i + 1) == quote { - return i + 2 - } - state = .code - return i + 1 - } - } - } - /// Whether the `/` at `offset` stands alone on its line, which is what makes it SQL*Plus's terminator rather than /// a division. public static func isSlashLine(_ text: NSString, at offset: Int, length: Int) -> Bool { diff --git a/Packages/TableProCore/Sources/TableProSQLGrammar/SqlBlockStructure.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SqlBlockStructure.swift index 1401a4f959..212bdca685 100644 --- a/Packages/TableProCore/Sources/TableProSQLGrammar/SqlBlockStructure.swift +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SqlBlockStructure.swift @@ -8,7 +8,7 @@ import Foundation /// to run a fragment. ``SQLStatementScanner`` and ``SQLFoldScanner`` both read the vocabulary from here for that /// reason, including the `END IF` disambiguation that is easy to get subtly different twice. /// -/// What they do not share is how much they will let a block swallow: see `allowsBlock` on ``effect(of:endingAt:in:length:allowsBlock:)``. +/// What they do not share is how much they will let a block swallow: see `allowsBlock` on ``effect(of:endingAt:in:length:allowsBlock:grammar:)``. /// /// This sits beside ``SqlLexer`` rather than inside it because these rules are about words, not characters. public enum SqlBlockStructure { @@ -126,17 +126,18 @@ public enum SqlBlockStructure { endingAt wordEnd: Int, in text: NSString, length: Int, - allowsBlock: Bool + allowsBlock: Bool, + grammar: SQLLexicalGrammar ) -> Effect { guard allowsBlock else { return .none } switch keyword { case "BEGIN": - return startsTransaction(after: wordEnd, in: text, length: length) ? .none : .opensBlock + return startsTransaction(after: wordEnd, in: text, length: length, grammar: grammar) ? .none : .opensBlock case "CASE": return .opensBlock case "END": - return endEffect(after: wordEnd, in: text, length: length) + return endEffect(after: wordEnd, in: text, length: length, grammar: grammar) default: return .none } @@ -144,15 +145,25 @@ public enum SqlBlockStructure { // MARK: - Private - private static func startsTransaction(after offset: Int, in text: NSString, length: Int) -> Bool { - let cursor = skipTrivia(from: offset, in: text, length: length) + private static func startsTransaction( + after offset: Int, + in text: NSString, + length: Int, + grammar: SQLLexicalGrammar + ) -> Bool { + let cursor = skipTrivia(from: offset, in: text, length: length, grammar: grammar) guard cursor < length else { return true } guard text.character(at: cursor) != SqlLexer.semicolon else { return true } return beginStartsTransaction(followedBy: readKeyword(text, at: cursor, length: length).text) } - private static func endEffect(after offset: Int, in text: NSString, length: Int) -> Effect { - let cursor = skipTrivia(from: offset, in: text, length: length) + private static func endEffect( + after offset: Int, + in text: NSString, + length: Int, + grammar: SQLLexicalGrammar + ) -> Effect { + let cursor = skipTrivia(from: offset, in: text, length: length, grammar: grammar) guard cursor < length else { return .closesBlock(resumeAt: offset) } let follower = readKeyword(text, at: cursor, length: length) switch endingFollowedBy(follower.text) { @@ -165,22 +176,17 @@ public enum SqlBlockStructure { } } - private static func skipTrivia(from offset: Int, in text: NSString, length: Int) -> Int { + private static func skipTrivia(from offset: Int, in text: NSString, length: Int, grammar: SQLLexicalGrammar) -> Int { var cursor = offset while cursor < length { if SqlLexer.isWhitespace(text.character(at: cursor)) { cursor += 1 continue } - if SqlLexer.startsLineComment(text, at: cursor, length: length) { - cursor = SqlLexer.endOfLine(text, from: cursor, length: length) - continue - } - if SqlLexer.startsBlockComment(text, at: cursor, length: length) { - cursor = SqlLexer.skipBlockComment(text, from: cursor, length: length).next - continue + guard let span = SQLNonCodeSpan.span(at: cursor, in: text, grammar: grammar), span.kind.isComment else { + break } - break + cursor = max(span.end, cursor + 1) } return cursor } diff --git a/Packages/TableProCore/Sources/TableProSQLGrammar/SqlDollarQuote.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SqlDollarQuote.swift index ec4bb57960..e0a33dc376 100644 --- a/Packages/TableProCore/Sources/TableProSQLGrammar/SqlDollarQuote.swift +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SqlDollarQuote.swift @@ -7,6 +7,15 @@ public enum SqlDollarQuote { case needsMoreData } + /// Which dollar quotes an engine reads. + public enum Style: Sendable, Equatable { + /// `$$` alone. Snowflake, CQL and Databend read `$name` as a variable, so a tag would swallow one. + case untagged + + /// `$$` and `$tag$`, PostgreSQL's rule, which DuckDB and Spanner's PostgreSQL dialect share. + case tagged + } + public static let dollar: unichar = 0x24 public static func isIdentifierStart(_ ch: unichar) -> Bool { @@ -17,37 +26,44 @@ public enum SqlDollarQuote { isIdentifierStart(ch) || (ch >= 0x30 && ch <= 0x39) } - /// Whether a `$` following this character is part of the preceding identifier, - /// per PostgreSQL's rule that a dollar quote must be separated from a - /// preceding identifier by whitespace (so `a$$b` is one identifier, not an - /// opener). + /// Whether a `$` following this character is part of the preceding identifier, per PostgreSQL's rule that a + /// dollar quote must be separated from a preceding identifier (so `a$$b` is one identifier, not an opener). + /// + /// PostgreSQL's lexer reads every byte from 0x80 up as an identifier character, so a non-ASCII letter glues a + /// `$` to itself exactly as an ASCII one does. public static func isIdentifierContinuation(_ ch: unichar) -> Bool { - isIdentifierPart(ch) || ch == dollar + isIdentifierPart(ch) || ch == dollar || ch >= 0x80 } - /// Resolves a `$` at `pos` to a dollar-quote opener, a positional parameter - /// like `$1`, or a non-tag dollar. A `$` glued to a preceding identifier is - /// not an opener. Returns `needsMoreData` when the buffer ends mid-tag; a - /// whole-string caller treats that as `notOpener`. + /// Resolves a `$` at `pos` with PostgreSQL's tagged rule. public static func scanOpener(at pos: Int, in buffer: NSString, bufLen: Int) -> Opener { + scanOpener(at: pos, in: buffer, bufLen: bufLen, style: .tagged) + } + + /// Resolves a `$` at `pos` to a dollar-quote opener, a positional parameter like `$1`, or a non-tag dollar. A `$` + /// glued to a preceding identifier is not an opener. Returns `needsMoreData` when the buffer ends mid-tag; a + /// whole-string caller treats that as `notOpener`. + /// + /// A tag starts with a letter or an underscore and continues with letters, digits and underscores, where a letter + /// is any non-ASCII character as well, which is how PostgreSQL accepts `$ü$`. + public static func scanOpener(at pos: Int, in buffer: NSString, bufLen: Int, style: Style) -> Opener { if pos > 0, isIdentifierContinuation(buffer.character(at: pos - 1)) { return .notOpener } - var p = pos + 1 + guard pos + 1 < bufLen else { return .needsMoreData } + if buffer.character(at: pos + 1) == dollar { + return .opener(length: 2, tag: "") + } + guard style == .tagged, isTagStart(buffer.character(at: pos + 1)) else { return .notOpener } + var p = pos + 2 while p < bufLen { let ch = buffer.character(at: p) if ch == dollar { let tagLen = p - pos - 1 - if tagLen == 0 { - return .opener(length: 2, tag: "") - } - if !isIdentifierStart(buffer.character(at: pos + 1)) { - return .notOpener - } let tag = buffer.substring(with: NSRange(location: pos + 1, length: tagLen)) return .opener(length: tagLen + 2, tag: tag) } - if !isIdentifierPart(ch) { + if !isTagPart(ch) { return .notOpener } p += 1 @@ -66,4 +82,12 @@ public enum SqlDollarQuote { let tagRange = NSRange(location: pos + 1, length: (tag as NSString).length) return buffer.substring(with: tagRange) == tag } + + private static func isTagStart(_ ch: unichar) -> Bool { + isIdentifierStart(ch) || ch >= 0x80 + } + + private static func isTagPart(_ ch: unichar) -> Bool { + isIdentifierPart(ch) || ch >= 0x80 + } } diff --git a/Packages/TableProCore/Sources/TableProSQLGrammar/SqlLexer.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SqlLexer.swift index 756fa3048b..9d3ea485ca 100644 --- a/Packages/TableProCore/Sources/TableProSQLGrammar/SqlLexer.swift +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SqlLexer.swift @@ -1,15 +1,11 @@ import Foundation -/// The character level rules every SQL scanner in the app agrees on: which UTF-16 units matter, and how far a comment, -/// a quoted string or a dollar quoted body runs. +/// The character level pieces every SQL scanner is built from: which UTF-16 units matter, and how far one comment, +/// quoted string or dollar quoted body runs once its kind is known. /// -/// Scanners differ in what they do with the structure they find, so they are not merged. Offsets are UTF-16 units, so -/// an `NSString` can be walked in constant time per character. -/// -/// ``skipQuotedString`` gates backslash escapes on the grammar, which is what PostgreSQL requires. -/// `SQLStatementScanner` deliberately keeps its own ungated handling, because splitting a script for execution is -/// safer when a backslash never ends a string early; `SQLStatementScannerTests` pins that behaviour. Oracle is the -/// exception there: a backslash is never an escape in Oracle, and scripts written for it routinely quote Windows paths. +/// Which kind starts at an offset is ``SQLNonCodeSpan``'s decision, made from an ``SQLLexicalGrammar``; these +/// functions only run a span to its end. Offsets are UTF-16 units, so an `NSString` can be walked in constant time per +/// character. public enum SqlLexer { public static let space = UInt16(UnicodeScalar(" ").value) public static let tab = UInt16(UnicodeScalar("\t").value) @@ -39,13 +35,16 @@ public enum SqlLexer { private static let greaterThan = UInt16(UnicodeScalar(">").value) /// How far a scan ran, and how many lines it crossed. A caller that does not track lines ignores `newlines`. + /// `isClosed` is false when the text ended before the closing delimiter. public struct Span: Sendable { public let next: Int public let newlines: Int + public let isClosed: Bool - public init(next: Int, newlines: Int) { + public init(next: Int, newlines: Int, isClosed: Bool = true) { self.next = next self.newlines = newlines + self.isClosed = isClosed } } @@ -95,7 +94,7 @@ public enum SqlLexer { } cursor += 1 } - return Span(next: length, newlines: newlines) + return Span(next: length, newlines: newlines, isClosed: false) } public static func skipNestedBlockComment(_ text: NSString, from offset: Int, length: Int) -> Span { @@ -120,7 +119,7 @@ public enum SqlLexer { } cursor += 1 } - return Span(next: length, newlines: newlines) + return Span(next: length, newlines: newlines, isClosed: false) } /// Runs past the closing quote, or to the end of the document when the string is never closed. @@ -154,7 +153,7 @@ public enum SqlLexer { } cursor += 1 } - return Span(next: length, newlines: newlines) + return Span(next: length, newlines: newlines, isClosed: false) } /// Runs past an Oracle `q'...'` literal, or its national form `nq'...'`, when one starts at @@ -187,7 +186,7 @@ public enum SqlLexer { } cursor += 1 } - return Span(next: length, newlines: newlines) + return Span(next: length, newlines: newlines, isClosed: false) } private static func alternativeQuoteCloser(for opener: UInt16) -> UInt16 { @@ -221,6 +220,205 @@ public enum SqlLexer { } cursor += 1 } - return (length, Span(next: length, newlines: newlines)) + return (length, Span(next: length, newlines: newlines, isClosed: false)) + } + + /// Whether `--` at `offset` starts a comment. MySQL and MariaDB read `--` as a comment only when a space or a + /// control character follows it, measured on 8.4 and 11.8 as `SELECT 1--1` returning 2. + public static func startsDashComment(_ text: NSString, at offset: Int, length: Int, needsWhitespace: Bool) -> Bool { + guard startsLineComment(text, at: offset, length: length) else { return false } + guard needsWhitespace, offset + 2 < length else { return true } + return text.character(at: offset + 2) <= space + } + + /// Whether `//` starts at `offset`. + public static func startsDoubleSlash(_ text: NSString, at offset: Int, length: Int) -> Bool { + text.character(at: offset) == slash && offset + 1 < length && text.character(at: offset + 1) == slash + } + + /// The offset of the line break that ends a line comment, or the end of the document. + public static func endOfLineComment( + _ text: NSString, + from offset: Int, + length: Int, + carriageReturnEnds: Bool + ) -> Int { + var cursor = min(offset, length) + while cursor < length { + let character = text.character(at: cursor) + if character == newline || (carriageReturnEnds && character == carriageReturn) { + return cursor + } + cursor += 1 + } + return cursor + } + + /// The length of a MySQL `/*!NNNNN` or MariaDB `/*M!NNNNN` opener at `offset`, or nil when none starts there. + public static func executableCommentOpenerLength(_ text: NSString, at offset: Int, length: Int) -> Int? { + guard startsBlockComment(text, at: offset, length: length) else { return nil } + var cursor = offset + 2 + if cursor < length, text.character(at: cursor) == capitalM || text.character(at: cursor) == smallM { + cursor += 1 + } + guard cursor < length, text.character(at: cursor) == exclamationMark else { return nil } + cursor += 1 + while cursor < length, isDigit(text.character(at: cursor)) { + cursor += 1 + } + return cursor - offset + } + + /// Runs past a `[...]` identifier. With `doubledCloseEscapes`, `]]` stands for one `]`, as T-SQL reads it; SQLite + /// ends the identifier at the first `]`. + public static func skipBracketedIdentifier( + _ text: NSString, + from offset: Int, + length: Int, + doubledCloseEscapes: Bool + ) -> Span { + var cursor = offset + 1 + var newlines = 0 + while cursor < length { + let character = text.character(at: cursor) + if character == newline { + newlines += 1 + } + guard character == closeBracket else { + cursor += 1 + continue + } + guard doubledCloseEscapes, cursor + 1 < length, text.character(at: cursor + 1) == closeBracket else { + return Span(next: cursor + 1, newlines: newlines) + } + cursor += 2 + } + return Span(next: length, newlines: newlines, isClosed: false) + } + + /// Whether three of `quote` start at `offset`, which opens a GoogleSQL triple-quoted literal. + public static func startsTripleQuote(_ text: NSString, at offset: Int, length: Int) -> Bool { + let quote = text.character(at: offset) + guard quote == singleQuote || quote == doubleQuote, offset + 2 < length else { return false } + return text.character(at: offset + 1) == quote && text.character(at: offset + 2) == quote + } + + /// Runs past a triple-quoted literal starting at `offset`, which only three of its own quote end. + public static func skipTripleQuotedString( + _ text: NSString, + from offset: Int, + length: Int, + backslashEscapes: Bool + ) -> Span { + let quote = text.character(at: offset) + var cursor = offset + 3 + var newlines = 0 + while cursor < length { + let character = text.character(at: cursor) + if character == newline { + newlines += 1 + } + if backslashEscapes, character == backslash, cursor + 1 < length { + cursor += 2 + continue + } + if character == quote, cursor + 2 < length, + text.character(at: cursor + 1) == quote, text.character(at: cursor + 2) == quote { + return Span(next: cursor + 3, newlines: newlines) + } + cursor += 1 + } + return Span(next: length, newlines: newlines, isClosed: false) + } + + /// Runs past SQLite's Tcl-style parameter `$name(...)`, `@name(...)`, `:name(...)` or `#name(...)` starting at + /// `offset`, or returns nil when none starts there. + /// + /// SQLite's tokenizer takes everything from the `(` to the first `)` or whitespace as part of the name, quotes + /// and semicolons included, so `$a('); DROP TABLE t; --'` is the parameter `$a(')` followed by a `DROP` the server + /// runs, measured on 3.54 for all four prefixes. A name that meets whitespace first is an illegal token, which ends + /// where the whitespace starts. `::` continues a name, as in `$a::b(...)`. + public static func skipParenthesizedParameterName(_ text: NSString, at offset: Int, length: Int) -> Span? { + guard parameterPrefixes.contains(text.character(at: offset)) else { return nil } + var cursor = offset + 1 + while cursor < length { + let character = text.character(at: cursor) + if isSQLiteIdentifierCharacter(character) { + cursor += 1 + continue + } + guard character == colonUnit, cursor + 1 < length, text.character(at: cursor + 1) == colonUnit else { break } + cursor += 2 + } + guard cursor > offset + 1, cursor < length, text.character(at: cursor) == openParen else { return nil } + cursor += 1 + while cursor < length { + let character = text.character(at: cursor) + if character == closeParen { + return Span(next: cursor + 1, newlines: 0) + } + if character <= space { + return Span(next: cursor, newlines: 0, isClosed: false) + } + cursor += 1 + } + return Span(next: length, newlines: 0, isClosed: false) + } + + /// Runs past a MySQL executable comment whose opener is `openerLength` long. The server lexes the body as SQL, so + /// a `*/` inside a quoted string does not close it, measured on 8.4 and 11.8 with `/*! , '*/' */`. + public static func skipExecutableComment( + _ text: NSString, + from offset: Int, + openerLength: Int, + length: Int, + backslashEscapes: (UInt16) -> Bool + ) -> Span { + var cursor = offset + openerLength + var newlines = 0 + while cursor < length { + let character = text.character(at: cursor) + if character == newline { + newlines += 1 + } + if character == star, cursor + 1 < length, text.character(at: cursor + 1) == slash { + return Span(next: cursor + 2, newlines: newlines) + } + if character == singleQuote || character == doubleQuote || character == backtick { + let quoted = skipQuotedString( + text, + from: cursor, + quote: character, + length: length, + backslashEscapes: backslashEscapes(character) + ) + newlines += quoted.newlines + cursor = quoted.next + continue + } + cursor += 1 + } + return Span(next: length, newlines: newlines, isClosed: false) + } + + private static let smallM = UInt16(UnicodeScalar("m").value) + private static let capitalM = UInt16(UnicodeScalar("M").value) + private static let digitZero = UInt16(UnicodeScalar("0").value) + private static let digitNine = UInt16(UnicodeScalar("9").value) + private static let colonUnit = UInt16(UnicodeScalar(":").value) + private static let parameterPrefixes: Set = [ + SqlDollarQuote.dollar, + UInt16(UnicodeScalar("@").value), + UInt16(UnicodeScalar(":").value), + UInt16(UnicodeScalar("#").value), + ] + + private static func isDigit(_ character: UInt16) -> Bool { + character >= digitZero && character <= digitNine + } + + /// SQLite's `IdChar`: ASCII letters, digits, `_`, `$`, and every unit from 0x80 up. + private static func isSQLiteIdentifierCharacter(_ character: UInt16) -> Bool { + SqlDollarQuote.isIdentifierPart(character) || character == SqlDollarQuote.dollar || character >= 0x80 } } diff --git a/Packages/TableProCore/Tests/TableProQueryTests/SQLWriteClassifierTests.swift b/Packages/TableProCore/Tests/TableProQueryTests/SQLWriteClassifierTests.swift index 02d4db4c0b..2e3ee87058 100644 --- a/Packages/TableProCore/Tests/TableProQueryTests/SQLWriteClassifierTests.swift +++ b/Packages/TableProCore/Tests/TableProQueryTests/SQLWriteClassifierTests.swift @@ -158,4 +158,50 @@ struct SQLWriteClassifierTests { #expect(!isWrite("SELECT 1 -- ; DELETE FROM users")) #expect(!isWrite("SELECT 1 /* ; DELETE FROM users */")) } + + // MARK: - Statements a lexical trick hid from the old splitter + + @Test("PostgreSQL deleted every row behind SELECT $$'$$, measured on 17 through the iOS libpq sequence") + func dollarQuoteHidesAWrite() { + #expect(isWrite("SELECT $$'$$; DELETE FROM t")) + #expect(isWrite("SELECT $ü$'$ü$; DELETE FROM t")) + } + + @Test("an E'' literal ends where PostgreSQL ends it") + func escapeStringHidesAWrite() { + #expect(isWrite("SELECT E'\\''; DELETE FROM t; --'")) + } + + @Test("a nested comment ends where PostgreSQL ends it") + func nestedCommentHidesAWrite() { + #expect(isWrite("SELECT 1 /* /* */ ' */; DELETE FROM t; --'")) + } + + @Test("a backslash is literal on PostgreSQL, so the quote after it closes the string") + func standardStringHidesAWrite() { + #expect(isWrite("SELECT 'C:\\' AS p; DELETE FROM t")) + } + + @Test("MySQL reads the batch both ways a backslash can go") + func mySQLBackslashHidesAWrite() { + #expect(isWrite("SELECT 'a\\'; DELETE FROM t; -- '", .mysql)) + #expect(isWrite("SELECT 'a\\''; DELETE FROM t; -- '", .mysql)) + } + + @Test("a bracketed identifier holds a quote on SQL Server") + func bracketHidesAWrite() { + #expect(isWrite("SELECT 1 AS [a'b]; DELETE FROM t", .mssql)) + #expect(!isWrite("SELECT 1 AS [a;b]", .mssql)) + } + + @Test("an Oracle q'[...]' literal holds a quote") + func alternativeQuoteHidesAWrite() { + #expect(isWrite("SELECT q'[it's]' FROM dual; DELETE FROM t", .oracle)) + #expect(!isWrite("SELECT q'[it's; fine]' FROM dual", .oracle)) + } + + @Test("a MySQL executable comment runs what it holds") + func executableCommentIsAWrite() { + #expect(isWrite("/*!40101 DELETE FROM t */", .mysql)) + } } diff --git a/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalCorpusTests.swift b/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalCorpusTests.swift new file mode 100644 index 0000000000..f267d0759b --- /dev/null +++ b/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalCorpusTests.swift @@ -0,0 +1,132 @@ +import Foundation +import TableProSQLGrammar +import Testing + +/// The statement count each engine's server produced for texts that hide a statement behind a lexical trick, +/// measured on 2026-09-19 against PostgreSQL 17.11, MySQL 8.4.11, MariaDB 11.8.9, Azure SQL Edge 15.0, DM8 V8, +/// Oracle 23.26, DuckDB 1.5.2 and SQLite 3.54 (`scripts/check-sql-lexical-grammar.sh` re-runs them). `executed` is +/// the count the execution grammar splits into, `plausible` the highest count any reading of the engine finds, which +/// is what a gate counts. +@Suite("SQL lexical corpus") +struct SQLLexicalCorpusTests { + struct Case: CustomTestStringConvertible, Sendable { + let engine: String + let sql: String + let executed: Int + let plausible: Int + let measured: String + + var testDescription: String { "\(engine): \(measured)" } + } + + static let backslash = "SELECT 'C:\\' AS p; DROP TABLE lexer_canary" + static let nestedComment = "SELECT 1 /* /* */ ' */; DROP TABLE lexer_canary; --'" + static let bracket = "SELECT [it's] FROM lexer_t; DROP TABLE lexer_canary; SELECT 'x'" + static let dollar = "SELECT $$it's$$; DROP TABLE lexer_canary; SELECT 'x'" + static let nonASCIITag = "SELECT $ü$it's$ü$; DROP TABLE lexer_canary; SELECT 'x'" + static let escapeString = "SELECT E'\\''; DROP TABLE lexer_canary; --'" + static let hashComment = "SELECT 1 # '\n; DROP TABLE lexer_canary; -- '" + static let mySQLEscape = "SELECT 'a\\'; DROP TABLE lexer_canary; -- '" + static let doubledBracket = "SELECT 1 AS [a]]'b]; DROP TABLE lexer_canary; SELECT 'x'" + static let alternativeQuote = "SELECT q'[it's]' FROM dual; DROP TABLE lexer_canary; --'" + static let doubleSlash = "SELECT 1 FROM DUAL // '\n; DELETE FROM lexer_canary; -- '" + static let carriageReturn = "SELECT 1 -- x\r; DROP TABLE lexer_canary" + static let tightDashes = "SELECT 1 --x; DROP TABLE lexer_canary" + static let tclParameter = "SELECT $a('); DROP TABLE lexer_canary; --'" + static let gluedDollars = "SELECT 1 AS x$$; DROP TABLE lexer_canary; --$$" + static let gluedNonASCIIDollars = "SELECT 1 AS é$$; DROP TABLE lexer_canary; --$$" + static let tripleQuote = "SELECT '''it's'''; DROP TABLE lexer_canary; SELECT 'x'" + + static let cases: [Case] = [ + Case(engine: "PostgreSQL", sql: backslash, executed: 2, plausible: 2, measured: "PQexec ran the DROP"), + Case(engine: "DuckDB", sql: backslash, executed: 2, plausible: 2, measured: "duckdb_query ran the DROP"), + Case(engine: "SQL Server", sql: backslash, executed: 2, plausible: 2, measured: "the batch ran the DROP"), + Case(engine: "SQLite", sql: backslash, executed: 2, plausible: 2, measured: "sqlite3 ran the DROP"), + Case(engine: "Dameng", sql: backslash, executed: 2, plausible: 2, measured: "DM8 ran the DELETE"), + Case(engine: "Oracle", sql: backslash, executed: 2, plausible: 2, measured: "ORA-03405 at the ;"), + Case(engine: "MySQL", sql: backslash, executed: 1, plausible: 2, measured: "one string, 2 with NO_BACKSLASH"), + + Case(engine: "PostgreSQL", sql: nestedComment, executed: 2, plausible: 2, measured: "comments nest"), + Case(engine: "DuckDB", sql: nestedComment, executed: 2, plausible: 2, measured: "comments nest"), + Case(engine: "SQL Server", sql: nestedComment, executed: 2, plausible: 2, measured: "comments nest"), + Case(engine: "MySQL", sql: nestedComment, executed: 1, plausible: 1, measured: "flat, the DROP stayed"), + Case(engine: "SQLite", sql: nestedComment, executed: 1, plausible: 1, measured: "flat, the DROP stayed"), + Case(engine: "Oracle", sql: nestedComment, executed: 1, plausible: 1, measured: "flat"), + Case(engine: "Dameng", sql: nestedComment, executed: 1, plausible: 1, measured: "flat"), + + Case(engine: "SQL Server", sql: bracket, executed: 3, plausible: 3, measured: "[it's] is an identifier"), + Case(engine: "SQLite", sql: bracket, executed: 3, plausible: 3, measured: "[it's] is an identifier"), + Case(engine: "PostgreSQL", sql: bracket, executed: 1, plausible: 1, measured: "a syntax error"), + Case(engine: "ClickHouse", sql: bracket, executed: 1, plausible: 1, measured: "[ is an array"), + + Case(engine: "PostgreSQL", sql: dollar, executed: 3, plausible: 3, measured: "PQexec ran the DROP"), + Case(engine: "DuckDB", sql: dollar, executed: 3, plausible: 3, measured: "duckdb_query ran the DROP"), + Case(engine: "Snowflake", sql: dollar, executed: 3, plausible: 3, measured: "$$ body, from the reference"), + Case(engine: "Cassandra", sql: dollar, executed: 3, plausible: 3, measured: "$$ body, from the reference"), + Case(engine: "SQLite", sql: dollar, executed: 1, plausible: 1, measured: "$$ is not a quote"), + Case(engine: "MySQL", sql: dollar, executed: 1, plausible: 3, measured: "8.4 refuses $$, 9 reads it"), + + Case(engine: "PostgreSQL", sql: nonASCIITag, executed: 3, plausible: 3, measured: "$ü$ is a tag"), + Case(engine: "DuckDB", sql: nonASCIITag, executed: 3, plausible: 3, measured: "$ü$ is a tag"), + Case(engine: "Snowflake", sql: nonASCIITag, executed: 1, plausible: 1, measured: "only $$ quotes"), + + Case(engine: "PostgreSQL", sql: escapeString, executed: 2, plausible: 2, measured: "E'\\'' ran the DROP"), + Case(engine: "DuckDB", sql: escapeString, executed: 2, plausible: 2, measured: "E'\\'' ran the DROP"), + Case(engine: "SQLite", sql: escapeString, executed: 1, plausible: 1, measured: "no E'' prefix"), + + Case(engine: "MySQL", sql: hashComment, executed: 2, plausible: 2, measured: "# is a comment"), + Case(engine: "MariaDB", sql: hashComment, executed: 2, plausible: 2, measured: "# is a comment"), + Case(engine: "PostgreSQL", sql: hashComment, executed: 1, plausible: 1, measured: "# is an operator"), + + Case(engine: "MySQL", sql: mySQLEscape, executed: 1, plausible: 2, measured: "one string by default"), + Case(engine: "PostgreSQL", sql: mySQLEscape, executed: 2, plausible: 2, measured: "PQexec ran the DROP"), + Case(engine: "SQL Server", sql: mySQLEscape, executed: 2, plausible: 2, measured: "the batch ran the DROP"), + + Case(engine: "SQL Server", sql: doubledBracket, executed: 3, plausible: 3, measured: "]] escapes"), + Case(engine: "SQLite", sql: doubledBracket, executed: 1, plausible: 1, measured: "]] closes, the DROP stayed"), + + Case(engine: "Oracle", sql: alternativeQuote, executed: 2, plausible: 2, measured: "q'[it's]' is one literal"), + Case(engine: "Dameng", sql: alternativeQuote, executed: 2, plausible: 2, measured: "DM8 ran the DELETE"), + + Case(engine: "Dameng", sql: doubleSlash, executed: 2, plausible: 2, measured: "// is a comment on DM8"), + Case(engine: "Oracle", sql: doubleSlash, executed: 1, plausible: 1, measured: "// is not a comment"), + + Case(engine: "PostgreSQL", sql: carriageReturn, executed: 2, plausible: 2, measured: "CR ends --"), + Case(engine: "SQL Server", sql: carriageReturn, executed: 2, plausible: 2, measured: "CR ends --"), + Case(engine: "MySQL", sql: carriageReturn, executed: 1, plausible: 1, measured: "only LF ends --"), + Case(engine: "SQLite", sql: carriageReturn, executed: 1, plausible: 1, measured: "only LF ends --"), + + Case(engine: "MySQL", sql: tightDashes, executed: 2, plausible: 2, measured: "--x is arithmetic"), + Case(engine: "PostgreSQL", sql: tightDashes, executed: 1, plausible: 1, measured: "--x is a comment"), + + Case(engine: "SQLite", sql: tclParameter, executed: 2, plausible: 2, measured: "$a(') ran the DROP"), + Case(engine: "PostgreSQL", sql: gluedDollars, executed: 2, plausible: 2, measured: "x$$ is an identifier"), + Case(engine: "PostgreSQL", sql: gluedNonASCIIDollars, executed: 2, plausible: 2, measured: "é$$ ran the DROP"), + Case(engine: "DuckDB", sql: gluedNonASCIIDollars, executed: 2, plausible: 2, measured: "é$$ ran the DROP"), + + Case(engine: "Spanner", sql: tripleQuote, executed: 3, plausible: 3, measured: "''' literal on GoogleSQL"), + Case(engine: "BigQuery", sql: tripleQuote, executed: 3, plausible: 3, measured: "''' literal, from ZetaSQL"), + ] + + @Test(arguments: cases) + func splitsWhereTheEngineSplits(_ corpus: Case) throws { + let readings = SQLLexicalReadings.resolve(databaseTypeId: corpus.engine, declared: nil, session: nil) + let executed = SQLStatementScanner.executableStatements(in: corpus.sql, grammar: readings.execution).count + let plausible = readings.distinct(for: corpus.sql).map { grammar in + SQLStatementScanner.executableStatements(in: corpus.sql, grammar: grammar).count + }.max() + #expect(executed == corpus.executed) + #expect(plausible == corpus.plausible) + } + + @Test("An engine TablePro does not know is counted under every grammar it does know") + func unknownEngineTakesTheHighestCount() { + let readings = SQLLexicalReadings.resolve(databaseTypeId: "Nonesuch", declared: nil, session: nil) + for sql in [Self.backslash, Self.nestedComment, Self.bracket, Self.dollar, Self.hashComment] { + let counts = readings.distinct(for: sql).map { grammar in + SQLStatementScanner.executableStatements(in: sql, grammar: grammar).count + } + #expect((counts.max() ?? 0) > 1, "\(sql)") + } + } +} diff --git a/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalReadingsTests.swift b/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalReadingsTests.swift new file mode 100644 index 0000000000..b83d9f76ea --- /dev/null +++ b/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalReadingsTests.swift @@ -0,0 +1,90 @@ +import Foundation +import TableProSQLGrammar +import Testing + +@Suite("SQL lexical readings") +struct SQLLexicalReadingsTests { + @Test("Every SQL engine TablePro ships has a curated grammar") + func curatedTableCoversShippedEngines() { + let shipped = [ + "MySQL", "MariaDB", "TiDB", "Databend", "OceanBase", "PostgreSQL", "Redshift", "CockroachDB", "PGlite", + "SQLite", "libSQL", "Turso", "Cloudflare D1", "DuckDB", "Oracle", "Dameng", "SQL Server", "ClickHouse", + "Snowflake", "BigQuery", "Spanner", "Trino", "Teradata", "Cassandra", "ScyllaDB", "DynamoDB", + "SurrealDB", "Redis", "MongoDB", "etcd", "Elasticsearch", "Typesense", "Weaviate", "Kafka", + ] + let missing = shipped.filter { SQLLexicalProfile.curated(forDatabaseTypeId: $0) == nil } + #expect(missing.isEmpty) + } + + @Test("A curated engine ignores what a plugin declares, because a registry plugin may be older than the app") + func curatedGrammarWinsOverDeclaration() { + let readings = SQLLexicalReadings.resolve( + databaseTypeId: "PostgreSQL", + declared: [.backslashEscapesInSingleQuotes, .hashLineComments], + session: nil + ) + #expect(!readings.execution.contains(.hashLineComments)) + #expect(readings.execution.contains(.nestedBlockComments)) + } + + @Test("An engine TablePro does not know is split as its plugin declares and read as nothing else") + func declaredGrammarServesUnknownEngines() { + let declared: SQLLexicalGrammar = [.bracketQuotedIdentifiers, .nestedBlockComments] + let readings = SQLLexicalReadings.resolve(databaseTypeId: "Nonesuch", declared: declared, session: nil) + #expect(readings.execution == declared) + #expect(readings.all == [declared]) + } + + @Test("An engine nobody declared is split as standard SQL and read under every grammar TablePro knows") + func undeclaredUnknownEngineReadsEverything() { + let readings = SQLLexicalReadings.resolve(databaseTypeId: "Nonesuch", declared: nil, session: nil) + #expect(readings.execution == .ansi) + #expect(readings.all.count > 10) + #expect(readings.all.allSatisfy { !$0.contains(.plsqlBlocks) }) + } + + @Test("A session fact chooses the execution grammar without narrowing what a gate reads") + func sessionFactsChooseExecutionOnly() { + let noBackslash = SQLSessionLexicalFacts( + determined: [.backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes], + enabled: [] + ) + let readings = SQLLexicalReadings.resolve(databaseTypeId: "MySQL", declared: nil, session: noBackslash) + #expect(!readings.execution.contains(.backslashEscapesInSingleQuotes)) + #expect(readings.all.contains { $0.contains(.backslashEscapesInSingleQuotes) }) + #expect(readings.all.contains { !$0.contains(.backslashEscapesInSingleQuotes) }) + } + + @Test("A fact the session reports outside the curated doubt still reaches the gate") + func sessionFactOutsideTheProfileWidensTheReadings() { + let escapes = SQLSessionLexicalFacts(determined: .hashLineComments, enabled: .hashLineComments) + let readings = SQLLexicalReadings.resolve(databaseTypeId: "Oracle", declared: nil, session: escapes) + #expect(readings.execution.contains(.hashLineComments)) + #expect(readings.all.count == 2) + } + + @Test("A fact no character in the text can trigger collapses the readings to one") + func irrelevantFactsCollapse() { + let readings = SQLLexicalReadings.resolve(databaseTypeId: "MySQL", declared: nil, session: nil) + #expect(readings.all.count > 1) + #expect(readings.distinct(for: "SELECT a FROM t WHERE b = 'c'").count == 1) + #expect(readings.distinct(for: "SELECT 'a\\'").count == 4) + } + + @Test("Spanner's grammar is either dialect until the database says which") + func spannerReadsBothDialects() { + let profile = SQLLexicalProfile.curated(forDatabaseTypeId: "Spanner") + let readings = profile?.readings ?? [] + #expect(readings.contains { $0.contains(.tripleQuotedStrings) }) + #expect(readings.contains { $0.contains(.taggedDollarQuotes) }) + } + + @Test("Every combination of the undetermined facts is a reading") + func undeterminedFactsExpand() { + let profile = SQLLexicalProfile(grammar: .ansi, undetermined: [.hashLineComments, .nestedBlockComments]) + #expect(Set(profile.readings) == [ + [], [.hashLineComments], [.nestedBlockComments], [.hashLineComments, .nestedBlockComments], + ]) + #expect(profile.readings.first == .ansi) + } +} diff --git a/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift b/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift new file mode 100644 index 0000000000..db703df0da --- /dev/null +++ b/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift @@ -0,0 +1,141 @@ +import Foundation +import TableProSQLGrammar +import Testing + +@Suite("SQL non-code spans") +struct SQLNonCodeSpanTests { + private func end(_ text: String, at index: Int = 0, _ grammar: SQLLexicalGrammar) -> Int? { + SQLNonCodeSpan.span(at: index, in: text as NSString, grammar: grammar)?.end + } + + @Test("A backslash keeps a quote open only where the grammar says so") + func backslashFollowsTheGrammar() { + #expect(end("'a\\' b'", .ansi) == 4) + #expect(end("'a\\' b'", .backslashEscapesInSingleQuotes) == 7) + #expect(end("\"a\\\" b\"", .backslashEscapesInSingleQuotes) == 4) + #expect(end("`a\\` b`", [.backtickQuotes, .backslashEscapesInSingleQuotes]) == 4) + #expect(end("`a\\` b`", [.backtickQuotes, .backslashEscapesInBackticks]) == 7) + } + + @Test("A backtick quotes only where the grammar reads it") + func backtickFollowsTheGrammar() { + #expect(end("`a;b`", .ansi) == nil) + #expect(end("`a;b`", .backtickQuotes) == 5) + } + + @Test("Block comments nest only where the grammar says so") + func nestingFollowsTheGrammar() { + let text = "/* a /* b */ c */ d" + #expect(end(text, .ansi) == 12) + #expect(end(text, .nestedBlockComments) == 17) + } + + @Test("Brackets quote an identifier only where the grammar says so, and ]] escapes only on T-SQL") + func bracketsFollowTheGrammar() { + #expect(end("[a]]b] c", .ansi) == nil) + #expect(end("[a]]b] c", .bracketQuotedIdentifiers) == 3) + #expect(end("[a]]b] c", [.bracketQuotedIdentifiers, .doubledClosingBracketEscapes]) == 6) + } + + @Test("Line comments start with #, // and -- as the grammar reads them") + func lineCommentsFollowTheGrammar() { + #expect(end("# a\nb", .ansi) == nil) + #expect(end("# a\nb", .hashLineComments) == 3) + #expect(end("// a\nb", .ansi) == nil) + #expect(end("// a\nb", .doubleSlashLineComments) == 4) + #expect(end("--a\nb", .ansi) == 3) + #expect(end("--a\nb", .dashCommentsNeedWhitespace) == nil) + #expect(end("-- a\nb", .dashCommentsNeedWhitespace) == 4) + #expect(end("--", .dashCommentsNeedWhitespace) == 2) + } + + @Test("A lone carriage return ends a line comment only where the engine ends it there") + func carriageReturnFollowsTheGrammar() { + #expect(end("-- a\rb\nc", .ansi) == 6) + #expect(end("-- a\rb\nc", .carriageReturnEndsLineComments) == 4) + } + + @Test("E'...' escapes with a backslash, and only where a word could start") + func escapeStringPrefix() { + #expect(end("E'\\'' x", .escapeStringPrefix) == 5) + #expect(end("xE'\\'' x", at: 1, .escapeStringPrefix) == nil) + #expect(end("E'\\'' x", .ansi) == nil) + } + + @Test("q'[...]' runs to its closing delimiter, and only where a word could start") + func alternativeQuoting() { + #expect(end("q'[it's]' x", .alternativeQuoting) == 9) + #expect(end("nq'{a'b}' x", .alternativeQuoting) == 9) + #expect(end("xq'[it's]'", at: 1, .alternativeQuoting) == nil) + } + + @Test("Tagged dollar quotes take non-ASCII tags, untagged ones take only $$") + func dollarQuoteStyles() { + #expect(end("$ü$a;b$ü$ c", .taggedDollarQuotes) == 9) + #expect(end("$tag$a;b$tag$ c", .taggedDollarQuotes) == 13) + #expect(end("$tag$a;b$tag$ c", .untaggedDollarQuotes) == nil) + #expect(end("$$a;b$$ c", .untaggedDollarQuotes) == 7) + #expect(end("$1", .taggedDollarQuotes) == nil) + } + + @Test("A dollar glued to an identifier, ASCII or not, never opens a body") + func gluedDollarIsIdentifier() { + #expect(end("x$$;$$", at: 1, .taggedDollarQuotes) == nil) + #expect(end("é$$;$$", at: 1, .taggedDollarQuotes) == nil) + #expect(end(" $$;$$", at: 1, .taggedDollarQuotes) == 6) + } + + @Test("A triple-quoted literal holds a lone quote") + func tripleQuotes() { + #expect(end("'''it's''' x", .tripleQuotedStrings) == 10) + #expect(end("'''it's''' x", .ansi) == 6) + } + + @Test("SQLite's $name(...) parameter runs to its ) or to whitespace") + func parenthesizedParameters() { + #expect(end("$a('); x", .parenthesizedParameterNames) == 5) + #expect(end("@a::b(;) x", .parenthesizedParameterNames) == 8) + #expect(end("$ab(x y)", .parenthesizedParameterNames) == 5) + #expect(end("$(x)", .parenthesizedParameterNames) == nil) + #expect(end("$a('); x", .ansi) == nil) + } + + @Test("A MySQL executable comment reads its body as SQL, so a quoted */ does not close it") + func executableCommentIsQuoteAware() { + let text = "/*!40101 , '*/' */ x" + let span = SQLNonCodeSpan.span(at: 0, in: text as NSString, grammar: [.executableComments]) + #expect(span?.kind == .executableComment) + #expect(span?.end == 18) + #expect(SQLNonCodeSpan.end(at: 0, in: text as NSString, grammar: [.executableComments]) == nil) + #expect(end(text, .ansi) == 14) + } + + @Test("A span the text ends inside is reported unterminated") + func unterminatedSpans() { + let grammar: SQLLexicalGrammar = [.taggedDollarQuotes, .bracketQuotedIdentifiers] + for text in ["'abc", "/* abc", "$$abc", "[abc", "'abc\\'"] { + let span = SQLNonCodeSpan.span(at: 0, in: text as NSString, grammar: grammar.union(.backslashEscapesInSingleQuotes)) + #expect(span?.isTerminated == false, "\(text)") + } + #expect(SQLNonCodeSpan.span(at: 0, in: "'abc'" as NSString, grammar: .ansi)?.isTerminated == true) + } + + @Test("The code projection keeps every offset and blanks every literal and comment") + func codeProjectionKeepsOffsets() { + let text = "SELECT 'a;b', \"c\" /* d */ FROM t -- e\nWHERE x = $$f$$" + let code = SQLCodeProjection.code(of: text, grammar: .taggedDollarQuotes) + #expect((code as NSString).length == (text as NSString).length) + #expect(!code.contains("a;b")) + #expect(!code.contains(" d ")) + #expect(code.contains("FROM t")) + #expect(code.contains("WHERE x =")) + #expect(!code.contains("f")) + } + + @Test("Revealing an executable comment keeps its body as code under any grammar") + func codeProjectionRevealsExecutableComments() { + let text = "SELECT 1 /*!40101 DROP TABLE t */" + #expect(!SQLCodeProjection.code(of: text, grammar: .ansi).contains("DROP")) + #expect(SQLCodeProjection.code(of: text, grammar: .ansi, revealingExecutableComments: true).contains("DROP")) + } +} diff --git a/Plugins/DamengDriverPlugin/DamengPlugin.swift b/Plugins/DamengDriverPlugin/DamengPlugin.swift index 6c9cd42996..3f6eb538b1 100644 --- a/Plugins/DamengDriverPlugin/DamengPlugin.swift +++ b/Plugins/DamengDriverPlugin/DamengPlugin.swift @@ -352,6 +352,22 @@ final class DamengPluginDriver: PluginDatabaseDriver, @unchecked Sendable { var requiresBackslashEscapingInLiterals: Bool { textEscaping == .backslashEscape } + /// The server's `BACKSLASH_ESCAPE` mode as the connect probe measured it. It is a static `dm.ini` parameter, so + /// it holds for the whole session; an unknown mode leaves the app reading both. + var sessionLexicalState: PluginSessionLexicalState? { + switch textEscaping { + case .backslashEscape: + return PluginSessionLexicalState( + determined: .backslashEscapesInSingleQuotes, + enabled: .backslashEscapesInSingleQuotes + ) + case .backslashLiteral: + return PluginSessionLexicalState(determined: .backslashEscapesInSingleQuotes, enabled: []) + case .unknown: + return nil + } + } + func escapeStringLiteral(_ value: String) -> String { let stripped = String(String.UnicodeScalarView(value.unicodeScalars.filter { $0 != "\0" })) let escaping: DamengTextEscaping = textEscaping == .backslashLiteral ? .backslashLiteral : .backslashEscape diff --git a/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift b/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift index 76404692fd..8eb0e3d4aa 100644 --- a/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift +++ b/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift @@ -278,7 +278,7 @@ actor DuckDBConnectionActor { /// genuinely open, and the next release would close the handle and roll it back. private func noteActivity(_ query: String) { lastActivity = ContinuousClock.now - switch SQLTransactionTracking.effect(of: query) { + switch SQLTransactionTracking.effect(of: query, lexicalFeatures: DuckDBLexicalFeatures.features) { case .opens: hasOpenTransaction = true case .closes: hasOpenTransaction = false case .unchanged: break diff --git a/Plugins/DuckDBDriverPlugin/DuckDBLexicalFeatures.swift b/Plugins/DuckDBDriverPlugin/DuckDBLexicalFeatures.swift new file mode 100644 index 0000000000..3009fd7d5b --- /dev/null +++ b/Plugins/DuckDBDriverPlugin/DuckDBLexicalFeatures.swift @@ -0,0 +1,18 @@ +// +// DuckDBLexicalFeatures.swift +// DuckDBDriverPlugin +// + +import Foundation +import TableProPluginKit + +/// How DuckDB lexes a statement, for the batches the plugin reads itself to track its transaction. +/// +/// Measured on 1.5.2 (the linked library) and 1.5.4 (the CLI): `$$` and `$tag$` bodies with non-ASCII tags, nested +/// block comments, `E'...'` escapes, a carriage return ending `--`, and a backslash that never escapes a plain string. +/// The app's curated table holds the same facts and `SQLLexicalFeatureMappingTests` keeps the two equal. +enum DuckDBLexicalFeatures { + static let features: SQLLexicalFeatures = [ + .taggedDollarQuotes, .nestedBlockComments, .escapeStringPrefix, .carriageReturnEndsLineComments, + ] +} diff --git a/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift b/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift index 3f6af39040..1f0fd7ba5d 100644 --- a/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift +++ b/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift @@ -206,12 +206,24 @@ final class MariaDBPluginConnection: @unchecked Sendable { private var _isInTransaction = false + /// Whether the session runs with `NO_BACKSLASH_ESCAPES`, from the status flags of the last reply, or nil before + /// any reply this connection read. Measured on MySQL 8.4.11 and MariaDB 11.8.9: the flag follows every session + /// `sql_mode` change on the next OK packet, and a mode `init_connect` sets shows only from the first reply after + /// connect, which is why it is read after the session setup statement rather than from the handshake. + var noBackslashEscapes: Bool? { + stateLock.withLock { _noBackslashEscapes } + } + + private var _noBackslashEscapes: Bool? + private func recordTransactionState(on mysql: UnsafeMutablePointer) { var serverStatus: UInt32 = 0 guard mariadb_get_info(mysql, MARIADB_CONNECTION_SERVER_STATUS, &serverStatus) == 0 else { return } let isOpen = (serverStatus & UInt32(SERVER_STATUS_IN_TRANS)) != 0 + let escapesOff = (serverStatus & UInt32(SERVER_STATUS_NO_BACKSLASH_ESCAPES)) != 0 stateLock.lock() _isInTransaction = isOpen + _noBackslashEscapes = escapesOff stateLock.unlock() } @@ -300,6 +312,7 @@ final class MariaDBPluginConnection: @unchecked Sendable { self.mysql = handle self._isConnected = true self.stateLock.unlock() + self.recordTransactionState(on: handle) } } diff --git a/Plugins/MySQLDriverPlugin/MySQLLexicalFeatures.swift b/Plugins/MySQLDriverPlugin/MySQLLexicalFeatures.swift new file mode 100644 index 0000000000..9fc43e6dae --- /dev/null +++ b/Plugins/MySQLDriverPlugin/MySQLLexicalFeatures.swift @@ -0,0 +1,47 @@ +// +// MySQLLexicalFeatures.swift +// MySQLDriverPlugin +// + +import Foundation +import TableProPluginKit + +/// How the engines this plugin serves lex a statement, for the statements the plugin reads itself. +/// +/// The app's curated table is the authority for its gates; these are the same facts, held here because a plugin +/// cannot link the app's package, and `SQLLexicalFeatureMappingTests` holds the two equal. +enum MySQLLexicalFeatures { + /// MySQL 8.4.11 and MariaDB 11.8.9, measured. + static let mySQL: SQLLexicalFeatures = [ + .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backtickQuotes, .hashLineComments, + .executableComments, .dashCommentsNeedWhitespace, .delimiterDirective, + ] + + /// Databend, from its tokenizer. + static let databend: SQLLexicalFeatures = [ + .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backtickQuotes, .untaggedDollarQuotes, + ] + + static func features(for flavor: MySQLServerFlavor, noBackslashEscapes: Bool?) -> SQLLexicalFeatures { + guard !flavor.isDatabend else { return databend } + guard let state = sessionState(noBackslashEscapes: noBackslashEscapes) else { return mySQL } + return mySQL.subtracting(state.determined).union(state.enabled) + } + + /// What the status flags of the last reply settle. `NO_BACKSLASH_ESCAPES` turns the backslash off in both string + /// quotes. With it off a double-quoted token is still a string or an identifier depending on `ANSI_QUOTES`, which + /// MySQL reports nowhere, so only the single quote is settled then. + static func sessionState(noBackslashEscapes: Bool?) -> PluginSessionLexicalState? { + guard let noBackslashEscapes else { return nil } + guard noBackslashEscapes else { + return PluginSessionLexicalState( + determined: .backslashEscapesInSingleQuotes, + enabled: .backslashEscapesInSingleQuotes + ) + } + return PluginSessionLexicalState( + determined: [.backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes], + enabled: [] + ) + } +} diff --git a/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift b/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift index 960f9e7ef0..5a75170e26 100644 --- a/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift +++ b/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift @@ -90,6 +90,22 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { var supportsTransactions: Bool { true } var requiresBackslashEscapingInLiterals: Bool { true } + /// `NO_BACKSLASH_ESCAPES` from the status flags of the last reply the connection read. Databend reports no such + /// flag, so it says nothing. + var sessionLexicalState: PluginSessionLexicalState? { + guard !flavor.isDatabend else { return nil } + return MySQLLexicalFeatures.sessionState(noBackslashEscapes: noBackslashEscapes) + } + + /// The features this session's statements are split with when the plugin reads them itself. + private var lexicalFeatures: SQLLexicalFeatures { + MySQLLexicalFeatures.features(for: flavor, noBackslashEscapes: noBackslashEscapes) + } + + private var noBackslashEscapes: Bool? { + sessionLock.withLock { mariadbConnection }?.noBackslashEscapes + } + var capabilities: PluginCapabilities { guard !flavor.isDatabend else { return Self.databendCapabilities } return [ @@ -366,8 +382,9 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Idle connection release private func noteActivity(_ sql: String) { + let features = lexicalFeatures sessionLock.withLock { - footprint.observe(sql) + footprint.observe(sql, lexicalFeatures: features) lastActivity = ContinuousClock.now } } @@ -377,7 +394,8 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { /// the table lock: a `LOCK TABLES` that errors holds nothing, and releases what the session held /// before it. private func noteFailure(_ sql: String) { - sessionLock.withLock { footprint.observeFailure(of: sql) } + let features = lexicalFeatures + sessionLock.withLock { footprint.observeFailure(of: sql, lexicalFeatures: features) } } private func mayReplay(_ query: String) -> Bool { diff --git a/Plugins/MySQLDriverPlugin/MySQLSessionFootprint.swift b/Plugins/MySQLDriverPlugin/MySQLSessionFootprint.swift index 66773be43e..ce66a46a54 100644 --- a/Plugins/MySQLDriverPlugin/MySQLSessionFootprint.swift +++ b/Plugins/MySQLDriverPlugin/MySQLSessionFootprint.swift @@ -89,10 +89,12 @@ struct MySQLSessionFootprint: Equatable { return nil } - mutating func observe(_ sql: String) { - for statement in SQLStatementSplitting.statements(in: sql) { + /// `lexicalFeatures` are the connection's own, so a `;` inside a Databend `$$` body or a string the session's + /// `NO_BACKSLASH_ESCAPES` closes early is split where the server splits it. + mutating func observe(_ sql: String, lexicalFeatures: SQLLexicalFeatures) { + for statement in SQLStatementSplitting.statements(in: sql, lexicalFeatures: lexicalFeatures) { let body = Self.executableBody(of: statement) - observeTransaction(body) + observeTransaction(body, lexicalFeatures: lexicalFeatures) observeStatement(body) } } @@ -121,8 +123,8 @@ struct MySQLSessionFootprint: Equatable { /// holds the lock and fails on the `INSERT`: clearing the flag for any `LOCK TABLES` anywhere in /// the text released a lock the session was still holding, and the idle release then handed the /// connection back. - mutating func observeFailure(of sql: String) { - let statements = SQLStatementSplitting.statements(in: sql) + mutating func observeFailure(of sql: String, lexicalFeatures: SQLLexicalFeatures) { + let statements = SQLStatementSplitting.statements(in: sql, lexicalFeatures: lexicalFeatures) guard statements.count == 1, let failed = statements.first else { return } let head = Self.collapsedHead(of: Self.executableBody(of: failed).uppercased()) guard head.hasPrefix("LOCK TABLE") else { return } @@ -139,8 +141,8 @@ struct MySQLSessionFootprint: Equatable { return hasLockedTables ? .holdsSessionLocks : .idle } - private mutating func observeTransaction(_ statement: String) { - switch SQLTransactionTracking.effect(of: statement) { + private mutating func observeTransaction(_ statement: String, lexicalFeatures: SQLLexicalFeatures) { + switch SQLTransactionTracking.effect(of: statement, lexicalFeatures: lexicalFeatures) { case .opens: hasOpenTransaction = true case .closes: hasOpenTransaction = false case .unchanged: break diff --git a/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift b/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift index 950717a1ff..4c1efb92d6 100644 --- a/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift +++ b/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift @@ -46,6 +46,17 @@ final class LibPQDriverCore: @unchecked Sendable { var serverVersionNumber: Int32 { libpqConnection?.serverVersionNumber() ?? 0 } var standardConformingStrings: Bool { libpqConnection?.standardConformingStrings ?? true } + /// Whether a backslash escapes in a plain string literal on this session, which is exactly what + /// `standard_conforming_strings` decides. The connection re-reads it from every `ParameterStatus` the server + /// sends, so a `SET standard_conforming_strings` the user runs is reflected once it has run. + var sessionLexicalState: PluginSessionLexicalState? { + guard let connection = libpqConnection else { return nil } + return PluginSessionLexicalState( + determined: .backslashEscapesInSingleQuotes, + enabled: connection.standardConformingStrings ? [] : .backslashEscapesInSingleQuotes + ) + } + init( config: DriverConnectionConfig, schemaFallbackQueries: [String] = PostgreSQLSchemaQueries.schemaFallbackQueries, @@ -247,6 +258,10 @@ protocol LibPQBackedDriver: PluginDatabaseDriver { } extension LibPQBackedDriver { + var sessionLexicalState: PluginSessionLexicalState? { + core.sessionLexicalState + } + /// The new name must be bare. Every libpq engine here rejects a qualified one, because this /// statement renames in place and never moves the object; `SET SCHEMA` is the separate verb. /// diff --git a/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift b/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift index eb19d15a5a..c86c002d5e 100644 --- a/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift +++ b/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift @@ -23,6 +23,19 @@ internal final class SpannerPluginDriver: PluginDatabaseDriver, @unchecked Senda lock.withLock { connectedDialect } } + /// Spanner fixes a database's dialect when it is created, so once connected the whole grammar is known: GoogleSQL + /// or PostgreSQL, each measured on the emulator. + var sessionLexicalState: PluginSessionLexicalState? { + guard lock.withLock({ connectedExecutor != nil }) else { return nil } + let googleSQL: SQLLexicalFeatures = [ + .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backslashEscapesInBackticks, + .backtickQuotes, .tripleQuotedStrings, .hashLineComments, + ] + let postgreSQL: SQLLexicalFeatures = [.taggedDollarQuotes, .nestedBlockComments, .escapeStringPrefix] + let enabled = dialect == .postgreSQL ? postgreSQL : googleSQL + return PluginSessionLexicalState(determined: googleSQL.union(postgreSQL), enabled: enabled) + } + var capabilities: PluginCapabilities { [.multiSchema, .cancelQuery, .transactions, .truncateTable] } diff --git a/Plugins/TableProPluginKit/PluginDatabaseDriver.swift b/Plugins/TableProPluginKit/PluginDatabaseDriver.swift index d673cdffc7..6bd2e0ef00 100644 --- a/Plugins/TableProPluginKit/PluginDatabaseDriver.swift +++ b/Plugins/TableProPluginKit/PluginDatabaseDriver.swift @@ -215,6 +215,12 @@ public protocol PluginDatabaseDriver: AnyObject, Sendable { var requiresBackslashEscapingInLiterals: Bool { get } + /// The lexical facts the server decides for this session and the driver has read, such as MySQL's + /// `NO_BACKSLASH_ESCAPES`. `nil` means the driver cannot tell, which leaves the app reading every value the + /// engine allows. The app splits statements for execution by it; the Safe Mode and external gates keep reading + /// every plausible value whatever it says. + var sessionLexicalState: PluginSessionLexicalState? { get } + func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] var providesBulkColumnFetch: Bool { get } @@ -672,6 +678,8 @@ public extension PluginDatabaseDriver { var requiresBackslashEscapingInLiterals: Bool { false } + var sessionLexicalState: PluginSessionLexicalState? { nil } + func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? { nil } /// Answers whether `fetchAllColumns` is a single query rather than the N+1 default below, and diff --git a/Plugins/TableProPluginKit/SQLDialectDescriptor.swift b/Plugins/TableProPluginKit/SQLDialectDescriptor.swift index 37bab70e73..97f6dec73c 100644 --- a/Plugins/TableProPluginKit/SQLDialectDescriptor.swift +++ b/Plugins/TableProPluginKit/SQLDialectDescriptor.swift @@ -103,6 +103,10 @@ public struct SQLDialectDescriptor: Sendable { /// tell those apart, which is why the dialect has to say. public let functionNamesAreCaseInsensitive: Bool + /// Where the engine ends a string, a quoted identifier and a comment. `nil` means the plugin does not say, and the + /// gates then read every grammar TablePro knows. Ignored for an engine the app curates itself. + public let lexicalFeatures: SQLLexicalFeatures? + public enum CaseSensitivityStyle: String, Sendable { case ilikeOperator // PostgreSQL, CockroachDB, PGlite, DuckDB, Snowflake case caseFoldFunction // Oracle, BigQuery, ClickHouse, Redshift @@ -285,6 +289,7 @@ public struct SQLDialectDescriptor: Sendable { ) } + @_disfavoredOverload public init( identifierQuote: String, keywords: Set, @@ -304,6 +309,49 @@ public struct SQLDialectDescriptor: Sendable { textCastTypeName: String?, functionNamesAreCaseInsensitive: Bool ) { + self.init( + identifierQuote: identifierQuote, + keywords: keywords, + functions: functions, + dataTypes: dataTypes, + tableOptions: tableOptions, + regexSyntax: regexSyntax, + booleanLiteralStyle: booleanLiteralStyle, + likeEscapeStyle: likeEscapeStyle, + paginationStyle: paginationStyle, + offsetFetchOrderBy: offsetFetchOrderBy, + requiresBackslashEscaping: requiresBackslashEscaping, + autoLimitStyle: autoLimitStyle, + caseSensitivityStyle: caseSensitivityStyle, + caseFoldFunction: caseFoldFunction, + operators: operators, + textCastTypeName: textCastTypeName, + functionNamesAreCaseInsensitive: functionNamesAreCaseInsensitive, + lexicalFeatures: nil + ) + } + + public init( + identifierQuote: String, + keywords: Set, + functions: Set, + dataTypes: Set, + tableOptions: [String] = [], + regexSyntax: RegexSyntax = .unsupported, + booleanLiteralStyle: BooleanLiteralStyle = .numeric, + likeEscapeStyle: LikeEscapeStyle = .explicit, + paginationStyle: PaginationStyle = .limit, + offsetFetchOrderBy: String = "ORDER BY (SELECT NULL)", + requiresBackslashEscaping: Bool = false, + autoLimitStyle: AutoLimitStyle = .limit, + caseSensitivityStyle: CaseSensitivityStyle = .unsupported, + caseFoldFunction: String = SQLDialectDescriptor.defaultCaseFoldFunction, + operators: [SQLOperatorDescriptor] = [], + textCastTypeName: String?, + functionNamesAreCaseInsensitive: Bool, + lexicalFeatures: SQLLexicalFeatures? = nil + ) { + self.lexicalFeatures = lexicalFeatures self.identifierQuote = identifierQuote self.keywords = keywords self.functions = functions @@ -346,7 +394,8 @@ public struct SQLDialectDescriptor: Sendable { caseFoldFunction: caseFoldFunction, operators: operators, textCastTypeName: textCastTypeName, - functionNamesAreCaseInsensitive: functionNamesAreCaseInsensitive + functionNamesAreCaseInsensitive: functionNamesAreCaseInsensitive, + lexicalFeatures: lexicalFeatures ) } } diff --git a/Plugins/TableProPluginKit/SQLFeatureLexer.swift b/Plugins/TableProPluginKit/SQLFeatureLexer.swift new file mode 100644 index 0000000000..85ae597597 --- /dev/null +++ b/Plugins/TableProPluginKit/SQLFeatureLexer.swift @@ -0,0 +1,363 @@ +// +// SQLFeatureLexer.swift +// TableProPluginKit +// + +import Foundation + +/// Where a comment, a literal or a quoted identifier ends, read by ``SQLLexicalFeatures``. +/// +/// This is the plugin side of the app's lexer: a plugin cannot link the app's package, so the kit carries the same +/// rules for the text a driver reads itself. The app's tests run one corpus through both. +struct SQLFeatureLexer { + enum Kind: Equatable { + case comment + case executableComment + case quoted + } + + struct Span: Equatable { + let kind: Kind + let end: Int + } + + private static let space: UInt16 = 0x20 + private static let newline: UInt16 = 0x0A + private static let carriageReturn: UInt16 = 0x0D + private static let singleQuote: UInt16 = 0x27 + private static let doubleQuote: UInt16 = 0x22 + private static let backtick: UInt16 = 0x60 + private static let backslash: UInt16 = 0x5C + private static let dash: UInt16 = 0x2D + private static let slash: UInt16 = 0x2F + private static let star: UInt16 = 0x2A + private static let hash: UInt16 = 0x23 + private static let dollar: UInt16 = 0x24 + private static let openBracket: UInt16 = 0x5B + private static let closeBracket: UInt16 = 0x5D + private static let openParen: UInt16 = 0x28 + private static let closeParen: UInt16 = 0x29 + private static let openBrace: UInt16 = 0x7B + private static let closeBrace: UInt16 = 0x7D + private static let lessThan: UInt16 = 0x3C + private static let greaterThan: UInt16 = 0x3E + private static let exclamationMark: UInt16 = 0x21 + private static let at: UInt16 = 0x40 + private static let colon: UInt16 = 0x3A + + let units: [UInt16] + let features: SQLLexicalFeatures + + init(_ text: String, features: SQLLexicalFeatures) { + self.units = Array(text.utf16) + self.features = features + } + + var count: Int { + units.count + } + + func span(at index: Int) -> Span? { + guard index < units.count else { return nil } + let unit = units[index] + if let comment = commentSpan(at: index, unit: unit) { + return comment + } + if unit == Self.singleQuote || unit == Self.doubleQuote + || (unit == Self.backtick && features.contains(.backtickQuotes)) { + return Span(kind: .quoted, end: quotedEnd(at: index, quote: unit)) + } + if unit == Self.openBracket, features.contains(.bracketQuotedIdentifiers) { + return Span(kind: .quoted, end: bracketEnd(at: index)) + } + if let prefixed = prefixedLiteralEnd(at: index, unit: unit) { + return Span(kind: .quoted, end: prefixed) + } + if unit == Self.dollar, let dollarEnd = dollarQuotedEnd(at: index) { + return Span(kind: .quoted, end: dollarEnd) + } + guard features.contains(.parenthesizedParameterNames), let parameterEnd = parameterEnd(at: index) else { + return nil + } + return Span(kind: .quoted, end: parameterEnd) + } + + // MARK: - Comments + + private func commentSpan(at index: Int, unit: UInt16) -> Span? { + if startsLineComment(at: index, unit: unit) { + return Span(kind: .comment, end: lineCommentEnd(from: index)) + } + guard unit == Self.slash, unitAt(index + 1) == Self.star else { return nil } + if features.contains(.executableComments), isExecutableCommentOpener(at: index) { + return Span(kind: .executableComment, end: executableCommentEnd(from: index)) + } + return Span(kind: .comment, end: blockCommentEnd(from: index, nests: features.contains(.nestedBlockComments))) + } + + private func startsLineComment(at index: Int, unit: UInt16) -> Bool { + switch unit { + case Self.dash: + guard unitAt(index + 1) == Self.dash else { return false } + guard features.contains(.dashCommentsNeedWhitespace), let next = unitAt(index + 2) else { return true } + return next <= Self.space + case Self.hash: + return features.contains(.hashLineComments) + case Self.slash: + return features.contains(.doubleSlashLineComments) && unitAt(index + 1) == Self.slash + default: + return false + } + } + + private func lineCommentEnd(from index: Int) -> Int { + let endsAtCarriageReturn = features.contains(.carriageReturnEndsLineComments) + var cursor = index + while cursor < units.count { + let unit = units[cursor] + if unit == Self.newline || (endsAtCarriageReturn && unit == Self.carriageReturn) { + return cursor + } + cursor += 1 + } + return cursor + } + + private func blockCommentEnd(from index: Int, nests: Bool) -> Int { + var cursor = index + 2 + var depth = 1 + while cursor < units.count { + if nests, units[cursor] == Self.slash, unitAt(cursor + 1) == Self.star { + depth += 1 + cursor += 2 + continue + } + if units[cursor] == Self.star, unitAt(cursor + 1) == Self.slash { + depth -= 1 + cursor += 2 + if depth == 0 { return cursor } + continue + } + cursor += 1 + } + return cursor + } + + /// The server lexes an executable comment's body as SQL, so a `*/` inside a quoted string does not close it. + private func executableCommentEnd(from index: Int) -> Int { + var cursor = index + 2 + while cursor < units.count { + let unit = units[cursor] + if unit == Self.star, unitAt(cursor + 1) == Self.slash { + return cursor + 2 + } + if unit == Self.singleQuote || unit == Self.doubleQuote || unit == Self.backtick { + cursor = quotedEnd(from: cursor + 1, quote: unit, backslashEscapes: backslashEscapes(inQuote: unit)) + continue + } + cursor += 1 + } + return cursor + } + + private func isExecutableCommentOpener(at index: Int) -> Bool { + var cursor = index + 2 + if unitAt(cursor) == 0x4D || unitAt(cursor) == 0x6D { + cursor += 1 + } + return unitAt(cursor) == Self.exclamationMark + } + + // MARK: - Literals + + private func backslashEscapes(inQuote quote: UInt16) -> Bool { + switch quote { + case Self.singleQuote: return features.contains(.backslashEscapesInSingleQuotes) + case Self.doubleQuote: return features.contains(.backslashEscapesInDoubleQuotes) + default: return features.contains(.backslashEscapesInBackticks) + } + } + + private func quotedEnd(at index: Int, quote: UInt16) -> Int { + let backslashEscapes = backslashEscapes(inQuote: quote) + if quote != Self.backtick, features.contains(.tripleQuotedStrings), + unitAt(index + 1) == quote, unitAt(index + 2) == quote { + return tripleQuotedEnd(from: index + 3, quote: quote, backslashEscapes: backslashEscapes) + } + return quotedEnd(from: index + 1, quote: quote, backslashEscapes: backslashEscapes) + } + + private func quotedEnd(from start: Int, quote: UInt16, backslashEscapes: Bool) -> Int { + var cursor = start + while cursor < units.count { + let unit = units[cursor] + if backslashEscapes, unit == Self.backslash, cursor + 1 < units.count { + cursor += 2 + continue + } + if unit == quote { + if unitAt(cursor + 1) == quote { + cursor += 2 + continue + } + return cursor + 1 + } + cursor += 1 + } + return cursor + } + + private func tripleQuotedEnd(from start: Int, quote: UInt16, backslashEscapes: Bool) -> Int { + var cursor = start + while cursor < units.count { + let unit = units[cursor] + if backslashEscapes, unit == Self.backslash, cursor + 1 < units.count { + cursor += 2 + continue + } + if unit == quote, unitAt(cursor + 1) == quote, unitAt(cursor + 2) == quote { + return cursor + 3 + } + cursor += 1 + } + return cursor + } + + private func bracketEnd(at index: Int) -> Int { + let doubledEscapes = features.contains(.doubledClosingBracketEscapes) + var cursor = index + 1 + while cursor < units.count { + guard units[cursor] == Self.closeBracket else { + cursor += 1 + continue + } + guard doubledEscapes, unitAt(cursor + 1) == Self.closeBracket else { return cursor + 1 } + cursor += 2 + } + return cursor + } + + private func prefixedLiteralEnd(at index: Int, unit: UInt16) -> Int? { + guard index == 0 || !isWordUnit(units[index - 1]) else { return nil } + if features.contains(.escapeStringPrefix), unit == 0x45 || unit == 0x65, unitAt(index + 1) == Self.singleQuote { + return quotedEnd(from: index + 2, quote: Self.singleQuote, backslashEscapes: true) + } + guard features.contains(.alternativeQuoting) else { return nil } + var cursor = index + if unit == 0x4E || unit == 0x6E { + cursor += 1 + } + guard let prefix = unitAt(cursor), prefix == 0x51 || prefix == 0x71, + unitAt(cursor + 1) == Self.singleQuote, + let opener = unitAt(cursor + 2), opener > Self.space + else { + return nil + } + let closer = alternativeQuoteCloser(for: opener) + cursor += 3 + while cursor < units.count { + if units[cursor] == closer, unitAt(cursor + 1) == Self.singleQuote { + return cursor + 2 + } + cursor += 1 + } + return cursor + } + + private func alternativeQuoteCloser(for opener: UInt16) -> UInt16 { + switch opener { + case Self.openBracket: return Self.closeBracket + case Self.openParen: return Self.closeParen + case Self.openBrace: return Self.closeBrace + case Self.lessThan: return Self.greaterThan + default: return opener + } + } + + private func dollarQuotedEnd(at index: Int) -> Int? { + let tagged = features.contains(.taggedDollarQuotes) + guard tagged || features.contains(.untaggedDollarQuotes) else { return nil } + if index > 0, continuesIdentifierBeforeDollar(units[index - 1]) { return nil } + guard let second = unitAt(index + 1) else { return nil } + var tagEnd = index + 1 + if second != Self.dollar { + guard tagged, isTagStart(second) else { return nil } + tagEnd = index + 2 + while tagEnd < units.count, units[tagEnd] != Self.dollar { + guard isTagPart(units[tagEnd]) else { return nil } + tagEnd += 1 + } + guard tagEnd < units.count else { return nil } + } + let delimiter = Array(units[index...tagEnd]) + var cursor = tagEnd + 1 + while cursor + delimiter.count <= units.count { + if units[cursor] == Self.dollar, Array(units[cursor..<(cursor + delimiter.count)]) == delimiter { + return cursor + delimiter.count + } + cursor += 1 + } + return units.count + } + + private func parameterEnd(at index: Int) -> Int? { + let prefix = units[index] + guard prefix == Self.dollar || prefix == Self.at || prefix == Self.colon || prefix == Self.hash else { + return nil + } + var cursor = index + 1 + while cursor < units.count, isSQLiteIdentifierUnit(units[cursor]) { + cursor += 1 + } + guard cursor > index + 1, unitAt(cursor) == Self.openParen else { return nil } + cursor += 1 + while cursor < units.count { + if units[cursor] == Self.closeParen { return cursor + 1 } + if units[cursor] <= Self.space { return cursor } + cursor += 1 + } + return cursor + } + + // MARK: - Characters + + private func unitAt(_ index: Int) -> UInt16? { + index < units.count ? units[index] : nil + } + + private func isASCIIIdentifierPart(_ unit: UInt16) -> Bool { + (unit >= 0x41 && unit <= 0x5A) || (unit >= 0x61 && unit <= 0x7A) || (unit >= 0x30 && unit <= 0x39) + || unit == 0x5F + } + + private func isWordUnit(_ unit: UInt16) -> Bool { + unit < 0x80 ? isASCIIIdentifierPart(unit) : !isSeparating(unit) + } + + private func isSeparating(_ unit: UInt16) -> Bool { + switch unit { + case 0xFF10...0xFF19, 0xFF21...0xFF3A, 0xFF3F, 0xFF41...0xFF5A: + return false + case 0xFF01...0xFF5E, 0x2018, 0x2019, 0x201C, 0x201D, 0x00A0, 0x2000...0x200A, 0x202F, 0x205F, 0x3000: + return true + default: + return false + } + } + + private func continuesIdentifierBeforeDollar(_ unit: UInt16) -> Bool { + isASCIIIdentifierPart(unit) || unit == Self.dollar || unit >= 0x80 + } + + private func isTagStart(_ unit: UInt16) -> Bool { + (unit >= 0x41 && unit <= 0x5A) || (unit >= 0x61 && unit <= 0x7A) || unit == 0x5F || unit >= 0x80 + } + + private func isTagPart(_ unit: UInt16) -> Bool { + isASCIIIdentifierPart(unit) || unit >= 0x80 + } + + private func isSQLiteIdentifierUnit(_ unit: UInt16) -> Bool { + isASCIIIdentifierPart(unit) || unit == Self.dollar || unit >= 0x80 + } +} diff --git a/Plugins/TableProPluginKit/SQLLexicalFeatures.swift b/Plugins/TableProPluginKit/SQLLexicalFeatures.swift new file mode 100644 index 0000000000..a20ee720c3 --- /dev/null +++ b/Plugins/TableProPluginKit/SQLLexicalFeatures.swift @@ -0,0 +1,98 @@ +// +// SQLLexicalFeatures.swift +// TableProPluginKit +// + +import Foundation + +/// How an engine ends a string, a quoted identifier and a comment, as a plugin declares it. +/// +/// A plugin for an engine TablePro does not know declares this through ``SQLDialectDescriptor/lexicalFeatures`` so +/// the editor splits its scripts where the engine does, and so the Safe Mode and external gates read the statements +/// the engine will actually run. For an engine TablePro ships, the app's own curated table wins, because a registry +/// plugin may be built against an older kit. The bit values are ABI: a new feature takes a new bit. +public struct SQLLexicalFeatures: OptionSet, Sendable, Hashable { + public let rawValue: UInt32 + + public init(rawValue: UInt32) { + self.rawValue = rawValue + } + + /// `'a\'b'` is one literal. + public static let backslashEscapesInSingleQuotes = SQLLexicalFeatures(rawValue: 1 << 0) + + /// `"a\"b"` is one token. + public static let backslashEscapesInDoubleQuotes = SQLLexicalFeatures(rawValue: 1 << 1) + + /// A backslash keeps a backtick-quoted identifier open. + public static let backslashEscapesInBackticks = SQLLexicalFeatures(rawValue: 1 << 2) + + /// `` `name` `` is a quoted identifier. + public static let backtickQuotes = SQLLexicalFeatures(rawValue: 1 << 3) + + /// `[name]` is a quoted identifier rather than an array or a subscript. + public static let bracketQuotedIdentifiers = SQLLexicalFeatures(rawValue: 1 << 4) + + /// `'''...'''` and `"""..."""` are literals a lone quote inside cannot end. + public static let tripleQuotedStrings = SQLLexicalFeatures(rawValue: 1 << 5) + + /// `E'...'` is a literal a backslash escapes in. + public static let escapeStringPrefix = SQLLexicalFeatures(rawValue: 1 << 6) + + /// Oracle's `q'[...]'` literal. + public static let alternativeQuoting = SQLLexicalFeatures(rawValue: 1 << 7) + + /// `$$...$$` is a literal; `$name` stays a variable. + public static let untaggedDollarQuotes = SQLLexicalFeatures(rawValue: 1 << 8) + + /// `$$...$$` and `$tag$...$tag$` are literals. + public static let taggedDollarQuotes = SQLLexicalFeatures(rawValue: 1 << 9) + + /// `/* /* */ */` is one comment. + public static let nestedBlockComments = SQLLexicalFeatures(rawValue: 1 << 10) + + /// `#` starts a line comment. + public static let hashLineComments = SQLLexicalFeatures(rawValue: 1 << 11) + + /// `//` starts a line comment. + public static let doubleSlashLineComments = SQLLexicalFeatures(rawValue: 1 << 12) + + /// MySQL's `/*! ... */`, whose body the server runs. + public static let executableComments = SQLLexicalFeatures(rawValue: 1 << 13) + + /// A line holding only `/` ends a statement, as SQL*Plus reads it. + public static let slashLineTerminators = SQLLexicalFeatures(rawValue: 1 << 14) + + /// `$` and `#` continue an identifier. + public static let dollarAndHashInIdentifiers = SQLLexicalFeatures(rawValue: 1 << 15) + + /// A `;` inside a PL/SQL unit belongs to the unit. + public static let plsqlBlocks = SQLLexicalFeatures(rawValue: 1 << 16) + + /// A script line `DELIMITER //` changes the statement terminator. + public static let delimiterDirective = SQLLexicalFeatures(rawValue: 1 << 17) + + /// `--` starts a comment only when a space or a control character follows it. + public static let dashCommentsNeedWhitespace = SQLLexicalFeatures(rawValue: 1 << 18) + + /// SQLite's `$name(...)` parameter, which runs to its `)`. + public static let parenthesizedParameterNames = SQLLexicalFeatures(rawValue: 1 << 19) + + /// `]]` inside `[...]` stands for one `]`. + public static let doubledClosingBracketEscapes = SQLLexicalFeatures(rawValue: 1 << 20) + + /// A carriage return on its own ends a line comment. + public static let carriageReturnEndsLineComments = SQLLexicalFeatures(rawValue: 1 << 21) +} + +/// The lexical facts a driver's session settled, such as MySQL's `NO_BACKSLASH_ESCAPES` from the status flags of the +/// last reply. A fact outside ``determined`` is left to the declared grammar. +public struct PluginSessionLexicalState: Sendable, Hashable { + public let determined: SQLLexicalFeatures + public let enabled: SQLLexicalFeatures + + public init(determined: SQLLexicalFeatures, enabled: SQLLexicalFeatures) { + self.determined = determined + self.enabled = enabled.intersection(determined) + } +} diff --git a/Plugins/TableProPluginKit/SQLStatementSplitting.swift b/Plugins/TableProPluginKit/SQLStatementSplitting.swift index 5b63ef7ce1..3a4a2b34c2 100644 --- a/Plugins/TableProPluginKit/SQLStatementSplitting.swift +++ b/Plugins/TableProPluginKit/SQLStatementSplitting.swift @@ -14,12 +14,61 @@ import Foundation /// other direction: `-- staging` on the line above `CREATE TEMPORARY TABLE` pushes the keyword off /// the front, so a session-state check that reads the first word sees a comment and finds nothing. /// -/// So the scan tracks single quotes, double quotes, backticks, dollar-quoted bodies, `--` and `#` -/// line comments and `/* */` block comments, and a `;` inside any of them is not a separator. -/// Doubled quotes (`''`) and backslash escapes both keep the string open, because MySQL honours -/// the backslash form and DuckDB does not, and treating a literal as longer than it is only ever -/// merges two statements, which is the safe direction here. +/// ``statements(in:lexicalFeatures:)`` splits the way the engine lexes: pass the features of the +/// connection's own engine, so a `;` inside a dollar-quoted body, a nested comment or a bracketed +/// identifier stays where the engine keeps it. +/// +/// ``statements(in:)`` is the older, engine-blind form. It tracks single quotes, double quotes, +/// backticks, `--` and `#` line comments and flat `/* */` block comments, and nothing else: a +/// dollar-quoted body is split at its first `;`. A backslash keeps any quote open, which is MySQL's +/// rule and not DuckDB's, so it can only ever merge two statements. public enum SQLStatementSplitting { + public static func statements(in sql: String, lexicalFeatures: SQLLexicalFeatures) -> [String] { + let lexer = SQLFeatureLexer(sql, features: lexicalFeatures) + var pieces: [String] = [] + var start = 0 + var index = 0 + while index < lexer.count { + if let span = lexer.span(at: index) { + index = max(span.end, index + 1) + continue + } + if lexer.units[index] == semicolon { + pieces.append(String(decoding: lexer.units[start.. String { + let lexer = SQLFeatureLexer(statement, features: lexicalFeatures) + var index = 0 + while index < lexer.count { + if isWhitespace(lexer.units[index]) { + index += 1 + continue + } + guard let span = lexer.span(at: index), span.kind == .comment else { break } + index = max(span.end, index + 1) + } + return String(decoding: lexer.units[min(index, lexer.count)...], as: UTF16.self) + } + + private static let semicolon: UInt16 = 0x3B + + private static func isWhitespace(_ unit: UInt16) -> Bool { + unit == 0x20 || (unit >= 0x09 && unit <= 0x0D) + } + public static func statements(in sql: String) -> [String] { var statements: [String] = [] var current = "" diff --git a/Plugins/TableProPluginKit/SQLTransactionTracking.swift b/Plugins/TableProPluginKit/SQLTransactionTracking.swift index 9f094964de..c8b9daa07f 100644 --- a/Plugins/TableProPluginKit/SQLTransactionTracking.swift +++ b/Plugins/TableProPluginKit/SQLTransactionTracking.swift @@ -49,9 +49,19 @@ public enum SQLTransactionTracking { "ABORT", "ABORT TRANSACTION", ] + /// The effect of `sql` split the way the engine lexes it, so a `COMMIT` inside a dollar-quoted + /// body is not read as one. + public static func effect(of sql: String, lexicalFeatures: SQLLexicalFeatures) -> Effect { + effect(ofStatements: SQLStatementSplitting.statements(in: sql, lexicalFeatures: lexicalFeatures)) + } + public static func effect(of sql: String) -> Effect { + effect(ofStatements: SQLStatementSplitting.statements(in: sql)) + } + + private static func effect(ofStatements statements: [String]) -> Effect { var effect = Effect.unchanged - for statement in SQLStatementSplitting.statements(in: sql) { + for statement in statements { if closes(statement) { effect = .closes } else if opens(statement) { diff --git a/TablePro/Core/Autocomplete/SQLCompletionProvider.swift b/TablePro/Core/Autocomplete/SQLCompletionProvider.swift index 556aa1a4dd..41522a756c 100644 --- a/TablePro/Core/Autocomplete/SQLCompletionProvider.swift +++ b/TablePro/Core/Autocomplete/SQLCompletionProvider.swift @@ -7,6 +7,7 @@ import Foundation import TableProPluginKit +import TableProSQLGrammar /// Main provider for SQL autocomplete suggestions final class SQLCompletionProvider { @@ -87,7 +88,9 @@ final class SQLCompletionProvider { cursorPosition: Int, forcedTableReferences: [TableReference]? = nil ) async -> (items: [SQLCompletionItem], candidates: [SQLCompletionItem], context: SQLContext) { - var context = contextAnalyzer.analyze(query: text, cursorPosition: cursorPosition) + var context = contextAnalyzer.analyze( + query: text, cursorPosition: cursorPosition, grammar: databaseType?.lexicalGrammar ?? .ansi + ) if let forcedTableReferences { context = context.replacingTableReferences(forcedTableReferences) } diff --git a/TablePro/Core/Autocomplete/SQLContextAnalyzer.swift b/TablePro/Core/Autocomplete/SQLContextAnalyzer.swift index 6014b6c18d..ce4ae72cf2 100644 --- a/TablePro/Core/Autocomplete/SQLContextAnalyzer.swift +++ b/TablePro/Core/Autocomplete/SQLContextAnalyzer.swift @@ -284,13 +284,13 @@ final class SQLContextAnalyzer { // MARK: - Main Analysis /// Analyze the query at the given cursor position - func analyze(query: String, cursorPosition: Int) -> SQLContext { + func analyze(query: String, cursorPosition: Int, grammar: SQLLexicalGrammar) -> SQLContext { let nsQuery = query as NSString let safePosition = min(cursorPosition, nsQuery.length) // Extract the current statement for multi-statement queries let located = SQLStatementScanner.locatedStatementAtCursor( - in: nsQuery as String, cursorPosition: safePosition + in: nsQuery as String, cursorPosition: safePosition, grammar: grammar ) let currentStatement = located.sql let statementOffset = located.offset diff --git a/TablePro/Core/Compare/CompareSyncExecutor.swift b/TablePro/Core/Compare/CompareSyncExecutor.swift index 97d5bb8551..4c981b49db 100644 --- a/TablePro/Core/Compare/CompareSyncExecutor.swift +++ b/TablePro/Core/Compare/CompareSyncExecutor.swift @@ -198,7 +198,7 @@ internal actor CompareSyncExecutor { /// the way the editor sends them. Every other engine takes the script text as written. private static func driverText(of statement: SyncStatement, dialect: SqlDialect) -> String { guard dialect == .oracle else { return statement.sql } - return SQLStatementScanner.executableText(of: statement.sql, dialect: dialect) + return SQLStatementScanner.executableText(of: statement.sql, grammar: DatabaseType.oracle.lexicalGrammar) } private func run( diff --git a/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift b/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift index 931b1601d2..6fa7d3a6ba 100644 --- a/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift +++ b/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift @@ -7,6 +7,7 @@ import AppKit import Foundation import os import TableProPluginKit +import TableProSQLGrammar private let helpersLogger = Logger(subsystem: "com.TablePro", category: "QueryExecutionCoordinator") @@ -22,7 +23,7 @@ extension QueryExecutionCoordinator { guard !SQLLimitDetector.hasExplicitRowLimit( sql, autoLimitStyle: PluginManager.shared.autoLimitStyle(for: parent.connection.type), - lexicalDialect: parent.sqlDialect + grammar: parent.lexicalGrammar ) else { return nil } diff --git a/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift b/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift index 9cb8219afa..fbd03574fb 100644 --- a/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift +++ b/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift @@ -306,11 +306,8 @@ extension QueryExecutionCoordinator { let statementTexts = statements.map(\.sql) let transactionKind = OperationKind.worst(of: statementTexts, databaseType: conn.type) - let rules = SQLLexicalRules( - databaseType: conn.type, - descriptor: PluginManager.shared.sqlDialect(for: conn.type) - ) - let plan = BatchTransactionPolicy.plan(for: statementTexts, databaseType: conn.type, rules: rules) + let grammar = parent.lexicalGrammar + let plan = BatchTransactionPolicy.plan(for: statementTexts, databaseType: conn.type, grammar: grammar) let prepared = statements.map { statement in prepareStatement( statement: statement, @@ -318,7 +315,7 @@ extension QueryExecutionCoordinator { style: style, tabType: tabType, bypassRowLimit: bypassRowLimit, - rules: rules + grammar: grammar ) } @@ -403,7 +400,7 @@ extension QueryExecutionCoordinator { style: ParameterStyle, tabType: TabType, bypassRowLimit: Bool, - rules: SQLLexicalRules + grammar: SQLLexicalGrammar ) -> PreparedStatement { let sql = statement.sql let parameterNames = parameters.isEmpty || !statement.acceptsBindParameters @@ -421,7 +418,7 @@ extension QueryExecutionCoordinator { parameterValues: conversion?.values, rowCap: bounded.rowCap, anchor: StatementAnchor(statement), - isCommitPoint: BatchCommitStatement.matches(sql, rules: rules) + isCommitPoint: BatchCommitStatement.matches(sql, grammar: grammar) ) } diff --git a/TablePro/Core/Coordinators/QueryExecutionCoordinator.swift b/TablePro/Core/Coordinators/QueryExecutionCoordinator.swift index cfc277096b..a3cbd3ad8e 100644 --- a/TablePro/Core/Coordinators/QueryExecutionCoordinator.swift +++ b/TablePro/Core/Coordinators/QueryExecutionCoordinator.swift @@ -23,7 +23,7 @@ final class QueryExecutionCoordinator: ObservableObject { tab.tabType == .query else { return } let statements = QueryStatementScanner.executableStatements( - in: tab.content.query, model: parent.statementModel, dialect: parent.sqlDialect + in: tab.content.query, model: parent.statementModel, grammar: parent.lexicalGrammar ) guard !statements.isEmpty else { return } diff --git a/TablePro/Core/Database/Access/DatabaseAccessBridge.swift b/TablePro/Core/Database/Access/DatabaseAccessBridge.swift index de33a94202..431d6d78ea 100644 --- a/TablePro/Core/Database/Access/DatabaseAccessBridge.swift +++ b/TablePro/Core/Database/Access/DatabaseAccessBridge.swift @@ -176,14 +176,11 @@ internal actor DatabaseAccessBridge { timeoutSeconds: Int, cancellation: (any StatementCancellationSignal)? ) async throws -> StatementOutcome { - guard !Self.statementText(query, dialect: .generic).isEmpty else { + guard !Self.statementText(query, grammar: .ansi).isEmpty else { throw DatabaseAccessError.invalidArgument(String(localized: "The query is empty.")) } let databaseType = try await ensureConnected(scope.connectionId) - let normalizedQuery = Self.statementText( - query, - dialect: SqlDialect.from(databaseTypeId: databaseType.rawValue) - ) + let normalizedQuery = Self.statementText(query, grammar: databaseType.lexicalGrammar) guard !normalizedQuery.isEmpty else { throw DatabaseAccessError.invalidArgument(String(localized: "The query is empty.")) } @@ -306,11 +303,10 @@ internal actor DatabaseAccessBridge { /// The text an external client's statement reaches the driver as. /// /// Trailing separators come off and a terminator that belongs to the statement stays, as the editor decides it: - /// a PL/SQL unit sent without the `;` after its `END` fails, or is stored INVALID, on Oracle. The emptiness of a - /// statement does not depend on the dialect for anything but an Oracle `/` line, so a caller may check it with - /// `.generic` before it knows the engine and again once it does. - internal static func statementText(_ query: String, dialect: SqlDialect) -> String { - SQLStatementScanner.executableText(of: query, dialect: dialect) + /// a PL/SQL unit sent without the `;` after its `END` fails, or is stored INVALID, on Oracle. A caller may check + /// emptiness with `.ansi` before it knows the engine and again once it does. + internal static func statementText(_ query: String, grammar: SQLLexicalGrammar) -> String { + SQLStatementScanner.executableText(of: query, grammar: grammar) } } diff --git a/TablePro/Core/Database/DatabaseDriver.swift b/TablePro/Core/Database/DatabaseDriver.swift index 2f1509337f..970ea7d0ae 100644 --- a/TablePro/Core/Database/DatabaseDriver.swift +++ b/TablePro/Core/Database/DatabaseDriver.swift @@ -25,6 +25,10 @@ protocol DatabaseDriver: AnyObject, Sendable { /// Optional - not all drivers may implement this var serverVersion: String? { get } + /// The lexical facts the server decided for this session and the driver has read. See + /// `PluginDatabaseDriver.sessionLexicalState`. + var sessionLexicalState: PluginSessionLexicalState? { get } + // MARK: - Connection Management /// Connect to the database @@ -391,6 +395,8 @@ extension DatabaseDriver { /// Override in drivers that support version querying var serverVersion: String? { nil } + var sessionLexicalState: PluginSessionLexicalState? { nil } + func connectReporting(stage report: @escaping ConnectionStageReporter) async throws { try await connect() } diff --git a/TablePro/Core/Diagnostics/ConfusableSQLCharacter.swift b/TablePro/Core/Diagnostics/ConfusableSQLCharacter.swift index c035b8dbd1..2bf78f10b3 100644 --- a/TablePro/Core/Diagnostics/ConfusableSQLCharacter.swift +++ b/TablePro/Core/Diagnostics/ConfusableSQLCharacter.swift @@ -4,6 +4,7 @@ // import Foundation +import TableProSQLGrammar enum ConfusableSQLCharacter: Equatable, Sendable { case fullWidthPunctuation(Unicode.Scalar) @@ -11,29 +12,22 @@ enum ConfusableSQLCharacter: Equatable, Sendable { case curlyQuote(Unicode.Scalar) case nonASCIISpace(Unicode.Scalar) - private static let fullWidthForms: ClosedRange = 0xFF01...0xFF5E + private static let fullWidthForms = SQLSeparatingCharacter.fullWidthForms private static let fullWidthOffset: UInt16 = 0xFEE0 static func isFullWidthWordUnit(_ unit: UInt16) -> Bool { - switch unit { - case 0xFF10...0xFF19, 0xFF21...0xFF3A, 0xFF3F, 0xFF41...0xFF5A: - return true - default: - return false - } + SQLSeparatingCharacter.isFullWidthWordUnit(unit) } static func separating(_ unit: UInt16) -> ConfusableSQLCharacter? { - guard let scalar = Unicode.Scalar(unit) else { return nil } - switch unit { - case fullWidthForms where !isFullWidthWordUnit(unit): + guard let scalar = Unicode.Scalar(unit), let kind = SQLSeparatingCharacter.kind(of: unit) else { return nil } + switch kind { + case .fullWidthPunctuation: return .fullWidthPunctuation(scalar) - case 0x2018, 0x2019, 0x201C, 0x201D: + case .curlyQuote: return .curlyQuote(scalar) - case 0x00A0, 0x2000...0x200A, 0x202F, 0x205F, 0x3000: + case .nonASCIISpace: return .nonASCIISpace(scalar) - default: - return nil } } diff --git a/TablePro/Core/Diagnostics/QueryDiagnostic.swift b/TablePro/Core/Diagnostics/QueryDiagnostic.swift index 036af2cb45..dea1176908 100644 --- a/TablePro/Core/Diagnostics/QueryDiagnostic.swift +++ b/TablePro/Core/Diagnostics/QueryDiagnostic.swift @@ -51,10 +51,7 @@ enum QueryDiagnosticsFactory { case .sql: return CombinedQueryDiagnosticsProducer(producers: [ SQLDiagnosticsProducer(), - SQLConfusableCharacterDiagnosticsProducer(rules: SQLLexicalRules( - databaseType: resolvedType, - descriptor: PluginManager.shared.sqlDialect(for: resolvedType) - )) + SQLConfusableCharacterDiagnosticsProducer(grammar: resolvedType.lexicalGrammar) ]) default: return SQLDiagnosticsProducer() diff --git a/TablePro/Core/Diagnostics/SQLConfusableCharacterDiagnosticsProducer.swift b/TablePro/Core/Diagnostics/SQLConfusableCharacterDiagnosticsProducer.swift index 8f13e522b3..4dadc4ab92 100644 --- a/TablePro/Core/Diagnostics/SQLConfusableCharacterDiagnosticsProducer.swift +++ b/TablePro/Core/Diagnostics/SQLConfusableCharacterDiagnosticsProducer.swift @@ -4,15 +4,16 @@ // import Foundation +import TableProSQLGrammar struct SQLConfusableCharacterDiagnosticsProducer: QueryDiagnosticsProducing { - let rules: SQLLexicalRules + let grammar: SQLLexicalGrammar func diagnostics(for text: String) -> [QueryDiagnostic] { let source = text as NSString guard source.length > 0, source.length <= QueryDiagnosticsLimits.maximumDocumentLength else { return [] } - return SQLConfusableCharacterScanner.scan(source, rules: rules).map { match in + return SQLConfusableCharacterScanner.scan(source, grammar: grammar).map { match in QueryDiagnostic(range: match.range, message: match.character.message, severity: .warning) } } diff --git a/TablePro/Core/Diagnostics/SQLConfusableCharacterScanner.swift b/TablePro/Core/Diagnostics/SQLConfusableCharacterScanner.swift index eeaaa80a82..c5f680b914 100644 --- a/TablePro/Core/Diagnostics/SQLConfusableCharacterScanner.swift +++ b/TablePro/Core/Diagnostics/SQLConfusableCharacterScanner.swift @@ -4,11 +4,11 @@ // import Foundation -import TableProPluginKit +import TableProSQLGrammar enum SQLConfusableCharacterScanner { - static func scan(_ text: NSString, rules: SQLLexicalRules) -> [ConfusableSQLCharacterMatch] { - var scan = ConfusableCharacterScan(text: text, rules: rules) + static func scan(_ text: NSString, grammar: SQLLexicalGrammar) -> [ConfusableSQLCharacterMatch] { + var scan = ConfusableCharacterScan(text: text, grammar: grammar) scan.run() return scan.matches } @@ -17,15 +17,15 @@ enum SQLConfusableCharacterScanner { private struct ConfusableCharacterScan { private let text: NSString private let length: Int - private let rules: SQLLexicalRules + private let grammar: SQLLexicalGrammar private(set) var matches: [ConfusableSQLCharacterMatch] = [] private var index = 0 - init(text: NSString, rules: SQLLexicalRules) { + init(text: NSString, grammar: SQLLexicalGrammar) { self.text = text self.length = text.length - self.rules = rules + self.grammar = grammar } mutating func run() { @@ -37,7 +37,7 @@ private struct ConfusableCharacterScan { private mutating func step() { let character = text.character(at: index) - if let end = SQLNonCodeSpan.end(at: index, in: text, rules: rules) { + if let end = SQLNonCodeSpan.end(at: index, in: text, grammar: grammar) { index = end return } diff --git a/TablePro/Core/Diagnostics/SQLLexicalRules.swift b/TablePro/Core/Diagnostics/SQLLexicalRules.swift deleted file mode 100644 index bfb43143d7..0000000000 --- a/TablePro/Core/Diagnostics/SQLLexicalRules.swift +++ /dev/null @@ -1,42 +0,0 @@ -// -// SQLLexicalRules.swift -// TablePro -// - -import Foundation -import TableProPluginKit - -struct SQLLexicalRules: Equatable, Sendable { - private static let enginesAcceptingBracketedIdentifiers: Set = [ - .sqlite, .libsql, .turso, .cloudflareD1 - ] - - let dialect: SqlDialect - let backslashEscapes: Bool - let bracketsDelimitIdentifiers: Bool - - init(dialect: SqlDialect, backslashEscapes: Bool, bracketsDelimitIdentifiers: Bool) { - self.dialect = dialect - self.backslashEscapes = backslashEscapes - self.bracketsDelimitIdentifiers = bracketsDelimitIdentifiers - } - - init(dialect: SqlDialect) { - self.init( - dialect: dialect, - backslashEscapes: dialect.requiresBackslashEscapesInSingleQuotes, - bracketsDelimitIdentifiers: false - ) - } - - init(databaseType: DatabaseType, descriptor: SQLDialectDescriptor?) { - let dialect = SqlDialect.from(databaseTypeId: databaseType.rawValue) - self.init( - dialect: dialect, - backslashEscapes: dialect.requiresBackslashEscapesInSingleQuotes - || descriptor?.requiresBackslashEscaping == true, - bracketsDelimitIdentifiers: descriptor?.identifierQuote == "[" - || Self.enginesAcceptingBracketedIdentifiers.contains(databaseType) - ) - } -} diff --git a/TablePro/Core/MCP/MCPConnectionBridge.swift b/TablePro/Core/MCP/MCPConnectionBridge.swift index 383bcb8f30..47bdbc449f 100644 --- a/TablePro/Core/MCP/MCPConnectionBridge.swift +++ b/TablePro/Core/MCP/MCPConnectionBridge.swift @@ -219,7 +219,7 @@ public actor MCPConnectionBridge { } static func statementText(_ query: String, databaseType: DatabaseType) -> String { - DatabaseAccessBridge.statementText(query, dialect: SqlDialect.from(databaseTypeId: databaseType.rawValue)) + DatabaseAccessBridge.statementText(query, grammar: databaseType.lexicalGrammar) } } diff --git a/TablePro/Core/ObjectCopy/ObjectCopyPlanner.swift b/TablePro/Core/ObjectCopy/ObjectCopyPlanner.swift index 25b6854971..a4633beedc 100644 --- a/TablePro/Core/ObjectCopy/ObjectCopyPlanner.swift +++ b/TablePro/Core/ObjectCopy/ObjectCopyPlanner.swift @@ -495,7 +495,9 @@ internal struct ObjectCopyPlanner { )) ?? [] for sequence in found where claimedSequences.insert(sequence.name.lowercased()).inserted { - sequences += SQLStatementScanner.allStatements(in: sequence.ddl) + sequences += SQLStatementScanner.allStatements( + in: sequence.ddl, grammar: driver.connection.type.lexicalGrammar + ) } } parts[input.id] = SourceParts( diff --git a/TablePro/Core/Plugins/PluginDriverAdapter.swift b/TablePro/Core/Plugins/PluginDriverAdapter.swift index c2074b317a..a41f059787 100644 --- a/TablePro/Core/Plugins/PluginDriverAdapter.swift +++ b/TablePro/Core/Plugins/PluginDriverAdapter.swift @@ -29,6 +29,7 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor } var serverVersion: String? { pluginDriver.serverVersion } + var sessionLexicalState: PluginSessionLexicalState? { pluginDriver.sessionLexicalState } var parameterStyle: ParameterStyle { pluginDriver.parameterStyle } func pluginGenerateStatements( diff --git a/TablePro/Core/Plugins/SqlFileImportSource.swift b/TablePro/Core/Plugins/SqlFileImportSource.swift index bdd6b743f7..5b30677364 100644 --- a/TablePro/Core/Plugins/SqlFileImportSource.swift +++ b/TablePro/Core/Plugins/SqlFileImportSource.swift @@ -6,13 +6,14 @@ import Foundation import os import TableProPluginKit +import TableProSQLGrammar final class SqlFileImportSource: PluginImportSource, @unchecked Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "SqlFileImportSource") private let url: URL private let encoding: String.Encoding - private let dialect: SqlDialect + private let grammar: SQLLexicalGrammar private let parser = SQLFileParser() private let externalDecompressedURL: URL? @@ -22,13 +23,13 @@ final class SqlFileImportSource: PluginImportSource, @unchecked Sendable { init( url: URL, encoding: String.Encoding, - dialect: SqlDialect = .generic, + grammar: SQLLexicalGrammar = .ansi, decompressedURL: URL? = nil, ownsDecompressedFile: Bool? = nil ) { self.url = url self.encoding = encoding - self.dialect = dialect + self.grammar = grammar self.externalDecompressedURL = decompressedURL self.ownsDecompressedFile = ownsDecompressedFile ?? (decompressedURL == nil) } @@ -50,7 +51,7 @@ final class SqlFileImportSource: PluginImportSource, @unchecked Sendable { func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { let fileURL = try await resolveURL() - return parser.parseFile(url: fileURL, encoding: encoding, dialect: dialect) + return parser.parseFile(url: fileURL, encoding: encoding, grammar: grammar) } /// `ownsDecompressedFile` answers only whether the caller handed over a file to delete. A file diff --git a/TablePro/Core/Services/Execution/AutocommitOnlyStatement+MySQL.swift b/TablePro/Core/Services/Execution/AutocommitOnlyStatement+MySQL.swift index 89822237b7..410fdbc091 100644 --- a/TablePro/Core/Services/Execution/AutocommitOnlyStatement+MySQL.swift +++ b/TablePro/Core/Services/Execution/AutocommitOnlyStatement+MySQL.swift @@ -4,14 +4,15 @@ // import Foundation +import TableProSQLGrammar internal extension AutocommitOnlyStatement { /// Measured on MySQL 8.4.11 with `gtid_mode = ON` and MariaDB 11.4.13 with the binary log on, /// each statement run after `START TRANSACTION` and an `INSERT`. The errors are 1694, 1679, /// 1685, 1766, 1953, 1929, 1179, 1192 and 1568 depending on the variable; what they share is /// that the statement works on its own and fails inside the wrap. - static func matchesMySQLFamily(_ statement: NSString, rules: SQLLexicalRules) -> Bool { - var cursor = SQLTokenCursor(statement, rules: rules) + static func matchesMySQLFamily(_ statement: NSString, grammar: SQLLexicalGrammar) -> Bool { + var cursor = SQLTokenCursor(statement, grammar: grammar) guard let keyword = cursor.next()?.word else { return false } switch keyword { case "SET": diff --git a/TablePro/Core/Services/Execution/AutocommitOnlyStatement+PostgreSQL.swift b/TablePro/Core/Services/Execution/AutocommitOnlyStatement+PostgreSQL.swift index c278383791..754ef61a17 100644 --- a/TablePro/Core/Services/Execution/AutocommitOnlyStatement+PostgreSQL.swift +++ b/TablePro/Core/Services/Execution/AutocommitOnlyStatement+PostgreSQL.swift @@ -4,6 +4,7 @@ // import Foundation +import TableProSQLGrammar internal extension AutocommitOnlyStatement { /// PostgreSQL 17.11, each statement run after `BEGIN`. Redshift and CockroachDB take the @@ -14,9 +15,9 @@ internal extension AutocommitOnlyStatement { static func matchesPostgresFamily( _ statement: NSString, family: TransactionEngineFamily, - rules: SQLLexicalRules + grammar: SQLLexicalGrammar ) -> Bool { - var cursor = SQLTokenCursor(statement, rules: rules) + var cursor = SQLTokenCursor(statement, grammar: grammar) guard let keyword = cursor.next()?.word else { return false } switch keyword { case "VACUUM": diff --git a/TablePro/Core/Services/Execution/AutocommitOnlyStatement+SQLServer.swift b/TablePro/Core/Services/Execution/AutocommitOnlyStatement+SQLServer.swift index 4afc1b42c4..b281ccf547 100644 --- a/TablePro/Core/Services/Execution/AutocommitOnlyStatement+SQLServer.swift +++ b/TablePro/Core/Services/Execution/AutocommitOnlyStatement+SQLServer.swift @@ -4,6 +4,7 @@ // import Foundation +import TableProSQLGrammar internal extension AutocommitOnlyStatement { /// From the T-SQL transaction locking and row versioning guide, which lists the statements an @@ -11,8 +12,8 @@ internal extension AutocommitOnlyStatement { /// catalog and index statements, `BACKUP`, `RESTORE` and `RECONFIGURE`. Unmeasured: there is no /// SQL Server here. The statements that begin `CREATE DATABASE` without being one, such as /// `CREATE DATABASE SCOPED CREDENTIAL`, keep the wrap. - static func matchesSQLServer(_ statement: NSString, rules: SQLLexicalRules) -> Bool { - var cursor = SQLTokenCursor(statement, rules: rules) + static func matchesSQLServer(_ statement: NSString, grammar: SQLLexicalGrammar) -> Bool { + var cursor = SQLTokenCursor(statement, grammar: grammar) guard let keyword = cursor.next()?.word else { return false } switch keyword { case "RECONFIGURE": diff --git a/TablePro/Core/Services/Execution/AutocommitOnlyStatement+SQLite.swift b/TablePro/Core/Services/Execution/AutocommitOnlyStatement+SQLite.swift index fea4d48f59..d4b1ec7960 100644 --- a/TablePro/Core/Services/Execution/AutocommitOnlyStatement+SQLite.swift +++ b/TablePro/Core/Services/Execution/AutocommitOnlyStatement+SQLite.swift @@ -4,14 +4,15 @@ // import Foundation +import TableProSQLGrammar internal extension AutocommitOnlyStatement { /// SQLite 3.54.0, each statement run after `BEGIN`. `VACUUM` and a journal-mode or safety-level /// change are refused outright; `PRAGMA foreign_keys` is the quiet one, applying no change and /// reading back `0` after the `COMMIT` with no error at any point. `wal_checkpoint` is refused /// once the transaction has run anything, in every form. - static func matchesSQLite(_ statement: NSString, rules: SQLLexicalRules) -> Bool { - var cursor = SQLTokenCursor(statement, rules: rules) + static func matchesSQLite(_ statement: NSString, grammar: SQLLexicalGrammar) -> Bool { + var cursor = SQLTokenCursor(statement, grammar: grammar) guard let keyword = cursor.next()?.word else { return false } switch keyword { case "VACUUM", "DETACH": @@ -26,8 +27,8 @@ internal extension AutocommitOnlyStatement { /// DuckDB v1.5.4. It takes `VACUUM`, `ATTACH`, `SET` and every `PRAGMA` inside a transaction, /// so it shares none of SQLite's rules despite sharing its lexing dialect. What it refuses is a /// checkpoint and a detach of a database the transaction has touched. - static func matchesDuckDB(_ statement: NSString, rules: SQLLexicalRules) -> Bool { - var cursor = SQLTokenCursor(statement, rules: rules) + static func matchesDuckDB(_ statement: NSString, grammar: SQLLexicalGrammar) -> Bool { + var cursor = SQLTokenCursor(statement, grammar: grammar) guard let keyword = cursor.next()?.word else { return false } switch keyword { case "DETACH", "CHECKPOINT": diff --git a/TablePro/Core/Services/Execution/AutocommitOnlyStatement.swift b/TablePro/Core/Services/Execution/AutocommitOnlyStatement.swift index e715f8df40..d274324ac9 100644 --- a/TablePro/Core/Services/Execution/AutocommitOnlyStatement.swift +++ b/TablePro/Core/Services/Execution/AutocommitOnlyStatement.swift @@ -4,6 +4,7 @@ // import Foundation +import TableProSQLGrammar /// Whether the engine refuses this statement, or silently ignores it, inside a transaction block. /// @@ -23,27 +24,27 @@ internal enum AutocommitOnlyStatement { internal static func matches( _ statement: String, family: TransactionEngineFamily, - rules: SQLLexicalRules + grammar: SQLLexicalGrammar ) -> Bool { - matches(statement as NSString, family: family, rules: rules) + matches(statement as NSString, family: family, grammar: grammar) } internal static func matches( _ statement: NSString, family: TransactionEngineFamily, - rules: SQLLexicalRules + grammar: SQLLexicalGrammar ) -> Bool { switch family { case .postgres, .redshift, .cockroach: - return matchesPostgresFamily(statement, family: family, rules: rules) + return matchesPostgresFamily(statement, family: family, grammar: grammar) case .mysql: - return matchesMySQLFamily(statement, rules: rules) + return matchesMySQLFamily(statement, grammar: grammar) case .sqlite: - return matchesSQLite(statement, rules: rules) + return matchesSQLite(statement, grammar: grammar) case .duckdb: - return matchesDuckDB(statement, rules: rules) + return matchesDuckDB(statement, grammar: grammar) case .sqlServer: - return matchesSQLServer(statement, rules: rules) + return matchesSQLServer(statement, grammar: grammar) case .redis, .other: return false } diff --git a/TablePro/Core/Services/Execution/BatchCommitStatement.swift b/TablePro/Core/Services/Execution/BatchCommitStatement.swift index 31f2cfe45c..6045428d26 100644 --- a/TablePro/Core/Services/Execution/BatchCommitStatement.swift +++ b/TablePro/Core/Services/Execution/BatchCommitStatement.swift @@ -4,6 +4,7 @@ // import Foundation +import TableProSQLGrammar /// Whether one statement of a batch is the script's own point of no return. /// @@ -24,8 +25,8 @@ import Foundation internal enum BatchCommitStatement { private static let transactionNouns: Set = ["TRANSACTION", "WORK"] - internal static func matches(_ statement: NSString, rules: SQLLexicalRules) -> Bool { - var cursor = SQLTokenCursor(statement, rules: rules) + internal static func matches(_ statement: NSString, grammar: SQLLexicalGrammar) -> Bool { + var cursor = SQLTokenCursor(statement, grammar: grammar) guard let keyword = cursor.next()?.word else { return false } switch keyword { case "COMMIT": @@ -38,7 +39,7 @@ internal enum BatchCommitStatement { } } - internal static func matches(_ statement: String, rules: SQLLexicalRules) -> Bool { - matches(statement as NSString, rules: rules) + internal static func matches(_ statement: String, grammar: SQLLexicalGrammar) -> Bool { + matches(statement as NSString, grammar: grammar) } } diff --git a/TablePro/Core/Services/Execution/BatchTransactionPolicy.swift b/TablePro/Core/Services/Execution/BatchTransactionPolicy.swift index d6adeb9ae5..fc67aead80 100644 --- a/TablePro/Core/Services/Execution/BatchTransactionPolicy.swift +++ b/TablePro/Core/Services/Execution/BatchTransactionPolicy.swift @@ -29,19 +29,19 @@ internal enum BatchTransactionPolicy { internal static func plan( for statements: [String], databaseType: DatabaseType, - rules: SQLLexicalRules + grammar: SQLLexicalGrammar ) -> BatchTransactionPlan { let family = TransactionEngineFamily.of(databaseType) - guard family.wrapsBatchInTransaction else { return queuedBlockPlan(for: statements, rules: rules) } + guard family.wrapsBatchInTransaction else { return queuedBlockPlan(for: statements, grammar: grammar) } var runsInAutocommit = false var holdsASavepoint = false for statement in statements { let text = statement as NSString - if takesTransactionControl(text, family: family, rules: rules) { return .scriptTransaction } - if !runsInAutocommit, AutocommitOnlyStatement.matches(text, family: family, rules: rules) { + if takesTransactionControl(text, family: family, grammar: grammar) { return .scriptTransaction } + if !runsInAutocommit, AutocommitOnlyStatement.matches(text, family: family, grammar: grammar) { runsInAutocommit = true } - if family.savepointOpensTransaction, !holdsASavepoint, startsWithSavepoint(text, rules: rules) { + if family.savepointOpensTransaction, !holdsASavepoint, startsWithSavepoint(text, grammar: grammar) { holdsASavepoint = true } } @@ -56,10 +56,10 @@ internal enum BatchTransactionPolicy { /// anything else. private static func queuedBlockPlan( for statements: [String], - rules: SQLLexicalRules + grammar: SQLLexicalGrammar ) -> BatchTransactionPlan { let opensABlock = statements.contains { statement in - var cursor = SQLTokenCursor(statement as NSString, rules: rules) + var cursor = SQLTokenCursor(statement as NSString, grammar: grammar) return cursor.next()?.word == "MULTI" } return opensABlock ? .scriptTransaction : .autocommit @@ -68,13 +68,13 @@ internal enum BatchTransactionPolicy { private static func takesTransactionControl( _ statement: NSString, family: TransactionEngineFamily, - rules: SQLLexicalRules + grammar: SQLLexicalGrammar ) -> Bool { - var cursor = SQLTokenCursor(statement, rules: rules) + var cursor = SQLTokenCursor(statement, grammar: grammar) guard let keyword = cursor.next()?.word else { return false } switch keyword { case "BEGIN": - guard rules.dialect != .oracle else { return false } + guard !grammar.contains(.plsqlBlocks) else { return false } return SqlBlockStructure.beginStartsTransaction(followedBy: cursor.next()?.word) case "START": return cursor.next()?.word == "TRANSACTION" @@ -110,8 +110,8 @@ internal enum BatchTransactionPolicy { return false } - private static func startsWithSavepoint(_ statement: NSString, rules: SQLLexicalRules) -> Bool { - var cursor = SQLTokenCursor(statement, rules: rules) + private static func startsWithSavepoint(_ statement: NSString, grammar: SQLLexicalGrammar) -> Bool { + var cursor = SQLTokenCursor(statement, grammar: grammar) return cursor.next()?.word == "SAVEPOINT" } } diff --git a/TablePro/Core/Services/Export/ImportService.swift b/TablePro/Core/Services/Export/ImportService.swift index 9aadeae9c4..4328a99d69 100644 --- a/TablePro/Core/Services/Export/ImportService.swift +++ b/TablePro/Core/Services/Export/ImportService.swift @@ -80,11 +80,10 @@ final class ImportService: ObservableObject { if type(of: plugin).requiresTargetTable { source = PlainFileImportSource(url: decompressedURL ?? url) } else { - let dialect = SqlDialect.from(databaseTypeId: connection.type.rawValue) source = SqlFileImportSource( url: url, encoding: encoding, - dialect: dialect, + grammar: connection.type.lexicalGrammar, decompressedURL: decompressedURL, ownsDecompressedFile: ownsDecompressedFile ) diff --git a/TablePro/Core/Services/Query/LeadingRowsStatement.swift b/TablePro/Core/Services/Query/LeadingRowsStatement.swift index b237e6e750..8aed09b4d2 100644 --- a/TablePro/Core/Services/Query/LeadingRowsStatement.swift +++ b/TablePro/Core/Services/Query/LeadingRowsStatement.swift @@ -22,10 +22,10 @@ struct LeadingRowsStatement: Equatable { rowCap: Int?, maximumRows: Int, autoLimitStyle: AutoLimitStyle, - lexicalDialect: SqlDialect + grammar: SQLLexicalGrammar ) -> LeadingRowsStatement { let unchanged = LeadingRowsStatement(sql: sql, rowCap: rowCap) - guard !SQLLimitDetector.hasExplicitRowLimit(sql, autoLimitStyle: autoLimitStyle, lexicalDialect: lexicalDialect) + guard !SQLLimitDetector.hasExplicitRowLimit(sql, autoLimitStyle: autoLimitStyle, grammar: grammar) else { return unchanged } let cap = rowCap.map { min($0, maximumRows) } @@ -65,7 +65,7 @@ extension LeadingRowsStatement { rowCap: rowCap, maximumRows: maximumRows, autoLimitStyle: PluginManager.shared.autoLimitStyle(for: databaseType), - lexicalDialect: SqlDialect.from(databaseTypeId: databaseType.rawValue) + grammar: databaseType.lexicalGrammar ) } } diff --git a/TablePro/Core/Services/Query/QuerySqlParser.swift b/TablePro/Core/Services/Query/QuerySqlParser.swift index c7dad42323..13b77e1805 100644 --- a/TablePro/Core/Services/Query/QuerySqlParser.swift +++ b/TablePro/Core/Services/Query/QuerySqlParser.swift @@ -1,5 +1,6 @@ import Foundation import TableProPluginKit +import TableProSQLGrammar enum QuerySqlParser { private static let mongoCollectionRegex = try? NSRegularExpression( @@ -28,9 +29,10 @@ enum QuerySqlParser { static func extractTableName( from sql: String, dialect: SqlDialect = .generic, + readings: SQLLexicalReadings = .single(.ansi), browseSchema: String? = nil ) -> String? { - if let source = SelectSourceTableParser.singleSourceTable(in: sql, dialect: dialect) { + if let source = SelectSourceTableParser.singleSourceTable(in: sql, dialect: dialect, readings: readings) { guard let schema = source.schema else { return source.name } guard let browseSchema, matchesSessionSchema(schema, browseSchema) else { return nil } return source.name @@ -71,10 +73,10 @@ enum QuerySqlParser { /// Appending it to the end instead produced `... LIMIT 100 ORDER BY "total" ASC`, a syntax /// error on every engine, and stripping the old ORDER BY took the user's LIMIT with it, which /// silently replaced their limit with the app's row cap. - static func applyingOrderBy(_ orderClause: String, to sql: String, lexicalDialect: SqlDialect) -> String { + static func applyingOrderBy(_ orderClause: String, to sql: String, grammar: SQLLexicalGrammar) -> String { let trimmed = sql.trimmingCharacters(in: .whitespacesAndNewlines) let buffer = trimmed as NSString - let splitOffset = SQLLimitDetector.firstRowLimitClauseOffset(trimmed, lexicalDialect: lexicalDialect) + let splitOffset = SQLLimitDetector.firstRowLimitClauseOffset(trimmed, grammar: grammar) let head = splitOffset.map { buffer.substring(to: $0) } ?? trimmed let tail = splitOffset.map { buffer.substring(from: $0) } ?? "" diff --git a/TablePro/Core/Utilities/JavaScript/QueryStatementModel.swift b/TablePro/Core/Utilities/JavaScript/QueryStatementModel.swift index f0a07340d2..6f60617b79 100644 --- a/TablePro/Core/Utilities/JavaScript/QueryStatementModel.swift +++ b/TablePro/Core/Utilities/JavaScript/QueryStatementModel.swift @@ -40,11 +40,11 @@ enum QueryStatementScanner { static func locatedStatements( in text: String, model: QueryStatementModel, - dialect: SqlDialect = .generic + grammar: SQLLexicalGrammar ) -> [SQLStatementScanner.LocatedStatement] { switch model { case .sql: - return SQLStatementScanner.locatedStatements(in: text, dialect: dialect) + return SQLStatementScanner.locatedStatements(in: text, grammar: grammar) case .javascript: return JavaScriptStatementScanner.locatedStatements(in: text).map(located) } @@ -53,11 +53,11 @@ enum QueryStatementScanner { static func navigableStatements( in text: String, model: QueryStatementModel, - dialect: SqlDialect = .generic + grammar: SQLLexicalGrammar ) -> [SQLStatementScanner.LocatedStatement] { switch model { case .sql: - return SQLStatementScanner.navigableStatements(in: text, dialect: dialect) + return SQLStatementScanner.navigableStatements(in: text, grammar: grammar) case .javascript: return JavaScriptStatementScanner.executableStatements(in: text) .map(located) @@ -68,11 +68,11 @@ enum QueryStatementScanner { static func executableStatements( in text: String, model: QueryStatementModel, - dialect: SqlDialect = .generic + grammar: SQLLexicalGrammar ) -> [SQLStatementScanner.ExecutableStatement] { switch model { case .sql: - return SQLStatementScanner.executableStatements(in: text, dialect: dialect) + return SQLStatementScanner.executableStatements(in: text, grammar: grammar) case .javascript: return JavaScriptStatementScanner.executableStatements(in: text).compactMap { statement in let located = located(statement) @@ -89,12 +89,12 @@ enum QueryStatementScanner { in text: String, cursorPosition: Int, model: QueryStatementModel, - dialect: SqlDialect = .generic + grammar: SQLLexicalGrammar ) -> SQLStatementScanner.LocatedStatement { switch model { case .sql: return SQLStatementScanner.locatedStatementAtCursor( - in: text, cursorPosition: cursorPosition, dialect: dialect + in: text, cursorPosition: cursorPosition, grammar: grammar ) case .javascript: guard let statement = JavaScriptStatementScanner.statementAtCursor( @@ -110,13 +110,13 @@ enum QueryStatementScanner { after offset: Int, in text: String, model: QueryStatementModel, - dialect: SqlDialect = .generic + grammar: SQLLexicalGrammar ) -> Int? { switch model { case .sql: - return SQLStatementScanner.statementStart(after: offset, in: text, dialect: dialect) + return SQLStatementScanner.statementStart(after: offset, in: text, grammar: grammar) case .javascript: - return navigableStatements(in: text, model: model) + return navigableStatements(in: text, model: model, grammar: grammar) .first { $0.contentRange.location > offset }? .contentRange.location } @@ -126,13 +126,13 @@ enum QueryStatementScanner { before offset: Int, in text: String, model: QueryStatementModel, - dialect: SqlDialect = .generic + grammar: SQLLexicalGrammar ) -> Int? { switch model { case .sql: - return SQLStatementScanner.statementStart(before: offset, in: text, dialect: dialect) + return SQLStatementScanner.statementStart(before: offset, in: text, grammar: grammar) case .javascript: - return navigableStatements(in: text, model: model) + return navigableStatements(in: text, model: model, grammar: grammar) .last { $0.contentRange.location < offset }? .contentRange.location } @@ -142,14 +142,14 @@ enum QueryStatementScanner { after offset: Int, in text: String, model: QueryStatementModel, - dialect: SqlDialect = .generic + grammar: SQLLexicalGrammar ) -> Int? { switch model { case .sql: - return SQLStatementScanner.statementSelectionEnd(after: offset, in: text, dialect: dialect) + return SQLStatementScanner.statementSelectionEnd(after: offset, in: text, grammar: grammar) case .javascript: - if let next = statementStart(after: offset, in: text, model: model) { return next } - let end = navigableStatements(in: text, model: model).last?.contentRange.upperBound + if let next = statementStart(after: offset, in: text, model: model, grammar: grammar) { return next } + let end = navigableStatements(in: text, model: model, grammar: grammar).last?.contentRange.upperBound return end.flatMap { $0 > offset ? $0 : nil } } } diff --git a/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift b/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift index a6a91a306a..d0082b7978 100644 --- a/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift +++ b/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift @@ -46,14 +46,15 @@ enum CatalogChangeClassifier { let tier = QueryClassifier.classifyTier(trimmed, databaseType: databaseType) return tier == .safe ? .none : opaque } - if QueryClassifier.runsPLSQL(trimmed, databaseType: databaseType) { + let grammar = databaseType.lexicalGrammar + if QueryClassifier.runsPLSQL(trimmed, grammar: grammar) { return opaque } - return sqlEffect(trimmed) + return sqlEffect(trimmed, grammar: grammar) } - private static func sqlEffect(_ trimmed: String) -> CatalogStatementEffect { - let tokens = leadingTokens(of: trimmed) + private static func sqlEffect(_ trimmed: String, grammar: SQLLexicalGrammar) -> CatalogStatementEffect { + let tokens = leadingTokens(of: trimmed, grammar: grammar) let leading = leadingKeywordEffect(tokens: tokens, trimmed: trimmed) guard leading.kinds.isEmpty else { return leading } return leading.union(embeddedDefinitionEffect(tokens: tokens)) @@ -119,9 +120,9 @@ enum CatalogChangeClassifier { /// Identifier tokens of the statement's opening, with MySQL executable comments revealed and /// their version numbers dropped, so `/*!50001 CREATE ... VIEW */` reads as `CREATE VIEW`. - private static func leadingTokens(of trimmed: String) -> [String] { + private static func leadingTokens(of trimmed: String, grammar: SQLLexicalGrammar) -> [String] { let prefix = String(trimmed.prefix(classifiedPrefixLength)) - let body = QueryClassifier.strippingStringLiterals(prefix, revealingConditionalComments: true).uppercased() + let body = SQLCodeProjection.code(of: prefix, grammar: grammar, revealingExecutableComments: true).uppercased() var tokens: [String] = [] var current = "" for character in body { diff --git a/TablePro/Core/Utilities/SQL/Folding/SQLFoldScanner.swift b/TablePro/Core/Utilities/SQL/Folding/SQLFoldScanner.swift index 71764b7f48..15a9150db5 100644 --- a/TablePro/Core/Utilities/SQL/Folding/SQLFoldScanner.swift +++ b/TablePro/Core/Utilities/SQL/Folding/SQLFoldScanner.swift @@ -13,9 +13,9 @@ import TableProSQLGrammar /// stack, so a nested region always reports a deeper level than the region containing it. Regions that open and close /// on the same line are discarded, because there is nothing to hide. enum SQLFoldScanner { - static func scan(_ text: NSString, dialect: SqlDialect) -> SQLFoldStructure { + static func scan(_ text: NSString, grammar: SQLLexicalGrammar) -> SQLFoldStructure { guard text.length > 0 else { return .empty } - var scan = Scan(text: text, dialect: dialect) + var scan = Scan(text: text, grammar: grammar) scan.run() return scan.structure } @@ -35,7 +35,7 @@ private struct Scan { private let text: NSString private let length: Int - private let dialect: SqlDialect + private let grammar: SQLLexicalGrammar private var frames: [Frame] = [] private var completed: [SQLFoldRegion] = [] @@ -46,11 +46,11 @@ private struct Scan { /// in the same gutter agree about where a statement stops. private var boundaries: any SQLStatementBoundaryTracking - init(text: NSString, dialect: SqlDialect) { + init(text: NSString, grammar: SQLLexicalGrammar) { self.text = text self.length = text.length - self.dialect = dialect - self.boundaries = SQLStatementBoundaries.makeTracker(for: dialect) + self.grammar = grammar + self.boundaries = SQLStatementBoundaries.makeTracker(for: grammar) } // MARK: - Pass @@ -66,10 +66,7 @@ private struct Scan { let character = text.character(at: index) if consumeTrivia(character) { return } - if consumeComment(character) { return } - if consumeQuotedString(character) { return } - if consumeDollarQuotedBody(character) { return } - if consumeAlternativeQuotedString() { return } + if consumeNonCode() { return } if consumeSlashLine(character) { return } openStatementIfNeeded() @@ -172,7 +169,7 @@ private struct Scan { let endsStatement = boundaries.observeSemicolon() if endsStatement { boundaries.reset() - if dialect == .oracle { + if grammar.contains(.plsqlBlocks) { closeFramesThroughStatement(end: index) } } @@ -191,7 +188,7 @@ private struct Scan { } private mutating func consumeKeyword(_ character: UInt16) { - let word = SqlBlockStructure.readKeyword(text, at: index, length: length, dialect: dialect) + let word = SqlBlockStructure.readKeyword(text, at: index, length: length, grammar: grammar) guard !word.text.isEmpty else { if !SqlLexer.isWhitespace(character) { boundaries.observeSymbol(character) @@ -206,7 +203,8 @@ private struct Scan { endingAt: word.end, in: text, length: length, - allowsBlock: true + allowsBlock: true, + grammar: grammar ) { case .opensBlock: pushFrame(.keywordBlock, openingToken: word.end, startLine: line) @@ -232,50 +230,35 @@ private struct Scan { return true } - private mutating func consumeComment(_ character: UInt16) -> Bool { - if SqlLexer.startsLineComment(text, at: index, length: length) - || (dialect.supportsHashLineComments && character == SqlLexer.hash) { - index = SqlLexer.endOfLine(text, from: index, length: length) - return true - } - - guard SqlLexer.startsBlockComment(text, at: index, length: length) else { return false } + /// A comment, a literal or a quoted identifier, ended where the grammar ends it. A block comment and a dollar + /// quoted body that span lines fold; nothing inside either is read as structure. + private mutating func consumeNonCode() -> Bool { + guard let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) else { return false } let start = SqlLexer.endOfLine(text, from: index, length: length) let startLine = line - let span = SqlLexer.skipBlockComment(text, from: index, length: length) - line += span.newlines - appendSpanningRegion(.blockComment, start: start, startLine: startLine, end: span.next - 2) - index = span.next - return true - } - - private mutating func consumeQuotedString(_ character: UInt16) -> Bool { - guard SqlLexer.isQuote(character) else { return false } - boundaries.observeOpaqueToken() - let span = SqlLexer.skipQuotedString(text, from: index, quote: character, length: length, dialect: dialect) - line += span.newlines - index = span.next - return true - } - - /// An Oracle `q'[...]'` literal, which only starts where a word could, so `xq'` stays an identifier and a string. - private mutating func consumeAlternativeQuotedString() -> Bool { - guard dialect.supportsAlternativeQuoting, - index == 0 || !SqlBlockStructure.continuesWord(text.character(at: index - 1), dialect: dialect), - let span = SqlLexer.skipAlternativeQuotedString(text, at: index, length: length) - else { - return false + switch span.kind { + case .lineComment: + break + case .blockComment, .executableComment: + line += span.newlines + appendSpanningRegion(.blockComment, start: start, startLine: startLine, end: span.contentEnd) + case .quoted where text.character(at: index) == SqlDollarQuote.dollar: + openStatementIfNeeded() + boundaries.observeOpaqueToken() + line += span.newlines + appendSpanningRegion(.quotedBody, start: start, startLine: startLine, end: span.contentEnd) + case .quoted, .parameter: + boundaries.observeOpaqueToken() + line += span.newlines } - boundaries.observeOpaqueToken() - line += span.newlines - index = span.next + index = max(span.end, index + 1) return true } /// A `/` alone on its line ends whatever statement is open, as SQL*Plus reads it. The statement ends where its own /// text does, on a line above the slash, so a one-line statement stays unfoldable. private mutating func consumeSlashLine(_ character: UInt16) -> Bool { - guard dialect.endsStatementsAtSlashLines, character == SqlLexer.slash, + guard grammar.contains(.slashLineTerminators), character == SqlLexer.slash, SQLStatementScanner.isSlashLine(text, at: index, length: length) else { return false @@ -297,29 +280,6 @@ private struct Scan { return true } - /// A dollar quoted body is one opaque region, so nothing inside it is read as structure. The statement around it - /// opens first, because the body is part of that statement. - private mutating func consumeDollarQuotedBody(_ character: UInt16) -> Bool { - guard dialect.supportsDollarQuotes, character == SqlDollarQuote.dollar, - case .opener(let openerLength, let tag) = SqlDollarQuote.scanOpener( - at: index, - in: text, - bufLen: length - ) else { - return false - } - - openStatementIfNeeded() - boundaries.observeOpaqueToken() - let start = SqlLexer.endOfLine(text, from: index, length: length) - let startLine = line - let result = SqlLexer.skipDollarQuotedBody(text, from: index + openerLength, tag: tag, length: length) - line += result.span.newlines - appendSpanningRegion(.quotedBody, start: start, startLine: startLine, end: result.bodyEnd) - index = result.span.next - return true - } - /// Records a region that was scanned in one go rather than opened and closed on the frame stack, when it turned /// out to span more than one line. private mutating func appendSpanningRegion( diff --git a/TablePro/Core/Utilities/SQL/InvisibleCharacterRemover.swift b/TablePro/Core/Utilities/SQL/InvisibleCharacterRemover.swift index 77c4f7e8ca..949a2f53c7 100644 --- a/TablePro/Core/Utilities/SQL/InvisibleCharacterRemover.swift +++ b/TablePro/Core/Utilities/SQL/InvisibleCharacterRemover.swift @@ -4,6 +4,7 @@ // import Foundation +import TableProSQLGrammar import TableProTextEngine internal enum InvisibleCharacterRemover { @@ -14,14 +15,14 @@ internal enum InvisibleCharacterRemover { in text: NSString, scope: NSRange, skippingLiteralsAndComments: Bool, - rules: SQLLexicalRules, + grammar: SQLLexicalGrammar, lineEnding: String ) -> [TextReplacement] { let end = min(NSMaxRange(scope), text.length) var replacements: [TextReplacement] = [] var index = max(scope.location, 0) while index < end { - if skippingLiteralsAndComments, let skipTo = SQLNonCodeSpan.end(at: index, in: text, rules: rules) { + if skippingLiteralsAndComments, let skipTo = SQLNonCodeSpan.end(at: index, in: text, grammar: grammar) { index = skipTo continue } diff --git a/TablePro/Core/Utilities/SQL/QueryClassifier+PLSQL.swift b/TablePro/Core/Utilities/SQL/QueryClassifier+PLSQL.swift index 60dc6fd1c2..f3a6cd2c6b 100644 --- a/TablePro/Core/Utilities/SQL/QueryClassifier+PLSQL.swift +++ b/TablePro/Core/Utilities/SQL/QueryClassifier+PLSQL.swift @@ -17,8 +17,14 @@ import TableProSQLGrammar /// of a stored procedure is: a write whose effect the text does not show. extension QueryClassifier { static func runsPLSQL(_ sql: String, databaseType: DatabaseType) -> Bool { - guard SqlDialect.from(databaseTypeId: databaseType.rawValue) == .oracle else { return false } - return startsPLSQL(sql) + runsPLSQL(sql, grammar: databaseType.lexicalGrammar) + } + + /// Whether `sql` is a PL/SQL unit to a grammar that has them. Its leading comments are read by that grammar, so a + /// block after a comment is found wherever the comment ends. + static func runsPLSQL(_ sql: String, grammar: SQLLexicalGrammar) -> Bool { + guard grammar.contains(.plsqlBlocks) else { return false } + return startsPLSQL(SQLCodeProjection.code(of: sql, grammar: grammar)) } /// An anonymous block opens with `DECLARE` or `BEGIN`, after any number of `<