feat(sidebar): find tables in every schema from Open Quickly and the sidebar filter

CHANGELOG.md

@@ -38,6 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **File > Session**, with the agent session commands and the assistant's conversation commands.
 - Eight more rebindable commands in **Settings > Keyboard**, among them the sidebar's lists and the session commands.
 - **Global** on a saved query folder's menu, for a folder every connection shows.
+- Tables from every schema in Open Quickly and the sidebar filter, and `schema.table` searches in both. (#3048)
 - Recent-tab switching on Control-Tab, with a list of the window's tabs while Control is held. (#2524)
 - **Extensions** for SQLite and local libSQL connections, loading sqlite-vec, SpatiaLite and other libraries on connect. (#2502)
 
@@ -74,6 +75,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- Tables in an expanded Oracle or Snowflake schema missing from Open Quickly until the next refresh.
+- Schemas missing from Open Quickly on every reopen after one failed to load.
+- Unexpanded schemas hidden by the sidebar filter in the Tree layout.
+- Empty object sections opened as "No items" under every match while filtering the sidebar tree.
+- **Drop View** offered in Recent for a sequence or materialized view opened from Open Quickly.
 - Unresponsive app and a dropped keystroke when typing in the row inspector's JSON field. (#3051)
 - Raw Oracle driver error in the schema switch failure dialog. (#3053)
 - Oracle health check closing a connection a statement was still running on. (#3053)

Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift

@@ -166,10 +166,17 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable {
     // MARK: - Schema
 
     func fetchTables(schema: String?) async throws -> [PluginTableInfo] {
-        let schemaName = schema ?? core.currentSchema
+        try await listTables(in: .schema(schema ?? core.currentSchema))
+    }
+
+    func fetchTablesInAllSchemas() async throws -> [PluginTableInfo]? {
+        try await listTables(in: .allSchemas)
+    }
+
+    private func listTables(in listing: PostgreSQLTableListingScope) async throws -> [PluginTableInfo] {
         func query(_ attempt: PostgreSQLTableListingAttempt) -> String {
             PostgreSQLSchemaQueries.fetchTables(
-                schema: schemaName,
+                in: listing,
                 includeMaterializedViews: attempt.includeOptionalCatalogs && includesMaterializedViews(),
                 includeForeignTables: attempt.includeOptionalCatalogs && includesForeignTables(),
                 includeComments: attempt.includeComments,
@@ -203,7 +210,13 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable {
             }
             let comment = row[safe: 2]?.asText?.nilIfEmpty
             let partitionCount = row[safe: 3]?.asText.flatMap(Int.init)
-            return PluginTableInfo(name: name, type: type, comment: comment, partitionCount: partitionCount)
+            return PluginTableInfo(
+                name: name,
+                type: type,
+                schema: row[safe: 4]?.asText,
+                comment: comment,
+                partitionCount: partitionCount
+            )
         }
     }
 

Plugins/PostgreSQLDriverPlugin/PostgreSQLSchemaQueries.swift

@@ -15,6 +15,11 @@ enum PostgreSQLSchemaProbe: Equatable {
     case failed
 }
 
+enum PostgreSQLTableListingScope: Sendable, Equatable {
+    case schema(String)
+    case allSchemas
+}
+
 enum PostgreSQLSchemaQueries {
     /// Returns the first schema on the effective search path, or SQL NULL
     /// when the path is empty (neither `$user` nor `public` exists).
@@ -110,7 +115,38 @@ enum PostgreSQLSchemaQueries {
         includeComments: Bool = true,
         includePartitionAwareness: Bool = true
     ) -> String {
-        let schemaLiteral = PostgreSQLObjectQueries.quoteLiteral(schema)
+        fetchTables(
+            in: .schema(schema),
+            includeMaterializedViews: includeMaterializedViews,
+            includeForeignTables: includeForeignTables,
+            includeComments: includeComments,
+            includePartitionAwareness: includePartitionAwareness
+        )
+    }
+
+    /// The same listing over one schema or over every schema `listSchemas` returns. The second
+    /// filters by that query itself rather than restating its predicate, so a table is listed here
+    /// exactly when its schema is listed there, and projects each row's schema, which the
+    /// one-schema listing leaves to the caller.
+    static func fetchTables(
+        in listing: PostgreSQLTableListingScope,
+        includeMaterializedViews: Bool,
+        includeForeignTables: Bool,
+        includeComments: Bool = true,
+        includePartitionAwareness: Bool = true
+    ) -> String {
+        func schemaFilter(_ column: String) -> String {
+            switch listing {
+            case .schema(let schema):
+                return "\(column) = \(PostgreSQLObjectQueries.quoteLiteral(schema))"
+            case .allSchemas:
+                return "\(column) IN (\n\(listSchemas)\n)"
+            }
+        }
+        func schemaColumn(_ column: String) -> String {
+            listing == .allSchemas ? ",\n       \(column) AS schema_name" : ""
+        }
+        let orderBy = listing == .allSchemas ? "ORDER BY schema_name, table_name" : "ORDER BY table_name"
         func commentColumn(_ oidExpression: String) -> String {
             includeComments ? "obj_description(\(oidExpression), 'pg_class')" : "NULL::text"
         }
@@ -140,9 +176,9 @@ enum PostgreSQLSchemaQueries {
             """
             SELECT t.table_name, \(tableTypeColumn) AS table_type,
                    \(commentColumn("pc.oid")) AS table_comment,
-                   \(partitionCountColumn) AS partition_count
+                   \(partitionCountColumn) AS partition_count\(schemaColumn("t.table_schema"))
             FROM information_schema.tables t\(classJoin)
-            WHERE t.table_schema = \(schemaLiteral)
+            WHERE \(schemaFilter("t.table_schema"))
               AND t.table_type IN ('BASE TABLE', 'VIEW')\(partitionFilter)
             """
         ]
@@ -157,9 +193,9 @@ enum PostgreSQLSchemaQueries {
                 """
                 SELECT m.matviewname AS table_name, 'MATERIALIZED VIEW' AS table_type,
                        \(commentColumn("mc.oid")) AS table_comment,
-                       NULL::bigint AS partition_count
+                       NULL::bigint AS partition_count\(schemaColumn("m.schemaname"))
                 FROM pg_matviews m\(matviewJoin)
-                WHERE m.schemaname = \(schemaLiteral)
+                WHERE \(schemaFilter("m.schemaname"))
                 """
             )
         }
@@ -172,16 +208,16 @@ enum PostgreSQLSchemaQueries {
                 """
                 SELECT c.relname AS table_name, 'FOREIGN TABLE' AS table_type,
                        \(commentColumn("c.oid")) AS table_comment,
-                       NULL::bigint AS partition_count
+                       NULL::bigint AS partition_count\(schemaColumn("n.nspname"))
                 FROM pg_foreign_table ft
                 JOIN pg_class c ON c.oid = ft.ftrelid
                 JOIN pg_namespace n ON n.oid = c.relnamespace
-                WHERE n.nspname = \(schemaLiteral)\(foreignPartitionFilter)
+                WHERE \(schemaFilter("n.nspname"))\(foreignPartitionFilter)
                 """
             )
         }
 
-        return unions.joined(separator: "\nUNION ALL\n") + "\nORDER BY table_name"
+        return unions.joined(separator: "\nUNION ALL\n") + "\n" + orderBy
     }
 
     /// The predicate that keeps a partition out of a flat listing. A foreign

Plugins/TableProPluginKit/PluginDatabaseDriver.swift

@@ -93,6 +93,11 @@ public protocol PluginDatabaseDriver: AnyObject, Sendable {
     func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult?
 
     func fetchTables(schema: String?) async throws -> [PluginTableInfo]
+
+    /// What `fetchTables(schema:)` lists for every schema `fetchSchemas()` lists, in one call, each
+    /// row carrying its own schema. Nil means the engine has no single call for it, and the host
+    /// asks each schema in turn instead.
+    func fetchTablesInAllSchemas() async throws -> [PluginTableInfo]?
     func fetchPartitions(table: String, schema: String?) async throws -> [PluginTableInfo]
 
     /// The same partitions as `fetchPartitions`, with the bound, the ordinal position and the row
@@ -641,6 +646,8 @@ public extension PluginDatabaseDriver {
 
     func fetchSchemas() async throws -> [String] { [] }
 
+    func fetchTablesInAllSchemas() async throws -> [PluginTableInfo]? { nil }
+
     /// Schemas whose objects live in a catalog outside the database itself, such
     /// as Redshift external schemas backed by Glue, Hive, or a federated source.
     /// Engines without that concept keep the empty default.

TablePro/Core/Concurrency/CatalogFreshness.swift

@@ -0,0 +1,43 @@
+//
+//  CatalogFreshness.swift
+//  TablePro
+//
+
+import Foundation
+
+/// Which cached lists a catalog change has overtaken, for a cache that refetches when it is next
+/// read rather than the moment the catalog changes.
+///
+/// A change moves its key's revision, and a fetch carries the revision it started under. A fetch
+/// that was already running when the change landed still delivers its rows, but it cannot make
+/// the key current again, so the next read fetches once more. A key whose fetch failed was never
+/// committed, which leaves it stale and retried on the next read too.
+struct CatalogFreshness<Key: Hashable> {
+    private var revisions: [Key: Int] = [:]
+    private var committed: [Key: Int] = [:]
+
+    func revision(for key: Key) -> Int {
+        revisions[key, default: 0]
+    }
+
+    func isCurrent(_ key: Key) -> Bool {
+        committed[key] == revision(for: key)
+    }
+
+    mutating func markChanged(_ key: Key) {
+        revisions[key, default: 0] &+= 1
+    }
+
+    /// False when a fetch that started later has already committed, so an older fetch finishing
+    /// last cannot put its rows back over newer ones.
+    mutating func commit(_ revision: Int, for key: Key) -> Bool {
+        if let current = committed[key], current > revision { return false }
+        committed[key] = revision
+        return true
+    }
+
+    mutating func removeAll(where shouldRemove: (Key) -> Bool) {
+        revisions = revisions.filter { !shouldRemove($0.key) }
+        committed = committed.filter { !shouldRemove($0.key) }
+    }
+}

TablePro/Core/Database/BackupScopeLoader.swift

@@ -92,32 +92,26 @@ enum BackupScopeLoader {
         }
     }
 
+    /// A schema that could not be listed fails the whole list, as it did when each schema was read
+    /// in turn: a picker that silently lacks a schema would back up less than the user chose.
     @MainActor
     private static func schemaQualifiedObjects(scope: DatabaseScope) async throws -> [NativeDumpObject] {
-        let schemas = try await DatabaseManager.shared.withMetadataDriver(scope: scope) { driver in
-            try await driver.fetchSchemas()
-        }
-        var objects: [NativeDumpObject] = []
-        for schema in schemas {
-            let qualified = DatabaseScope(
-                connectionId: scope.connectionId, database: scope.database, schema: schema
-            )
-            let tables = try await DatabaseManager.shared.withMetadataDriver(
-                scope: qualified, workload: .bulk
-            ) { driver in
-                try await driver.fetchTables(schema: schema)
+        let listing = try await CatalogTableListing.tables(in: scope, excludingSchemas: [])
+        guard listing.unlistedSchemas.isEmpty else { throw BackupScopeLoadError.schemasNotListed }
+        return listing.tables
+            .filter(\.type.isBackupSelectable)
+            .compactMap { table in
+                guard let schema = table.schema else { return nil }
+                return NativeDumpObject(
+                    name: table.name,
+                    schema: schema,
+                    isPartitionedParent: table.type == .partitionedTable
+                )
             }
-            objects += tables
-                .filter(\.type.isBackupSelectable)
-                .map {
-                    NativeDumpObject(
-                        name: $0.name,
-                        schema: schema,
-                        isPartitionedParent: $0.type == .partitionedTable
-                    )
-                }
-        }
-        return objects
+    }
+
+    private enum BackupScopeLoadError: Error {
+        case schemasNotListed
     }
 
     /// Everything the dump tool has to be told about to reproduce the chosen objects.

TablePro/Core/Database/CatalogTableListing.swift

@@ -0,0 +1,111 @@
+//
+//  CatalogTableListing.swift
+//  TablePro
+//
+
+import Foundation
+import os
+
+/// Every table one database holds, across all of its schemas.
+///
+/// An engine that answers `fetchTablesInAllSchemas()` is asked once. Any other is asked schema by
+/// schema, and each of those reads queues on the metadata lane by itself, so a sidebar expansion
+/// that arrives in the middle waits behind one schema rather than behind all of them. Every read
+/// goes through the one scope the caller names: a scope per schema would open a pooled connection
+/// per schema.
+@MainActor
+internal enum CatalogTableListing {
+    /// A schema whose own read failed is named rather than dropped. Read as empty, it would tell a
+    /// search that nothing in it matches, and hide exactly the table the search was looking for.
+    internal struct Result: Sendable, Equatable {
+        internal let tables: [TableInfo]
+        internal let unlistedSchemas: Set<String>
+
+        /// This listing with another read of some of its unlisted schemas folded in. A schema the
+        /// read listed replaces what was known of it; one it still could not list keeps its rows
+        /// and stays unlisted.
+        internal func merging(_ retry: Result, retried schemas: Set<String>) -> Result {
+            let listedNow = schemas.subtracting(retry.unlistedSchemas)
+            let kept = tables.filter { table in
+                guard let schema = table.schema else { return true }
+                return !listedNow.contains(schema)
+            }
+            return Result(
+                tables: kept + retry.tables,
+                unlistedSchemas: unlistedSchemas.subtracting(schemas).union(retry.unlistedSchemas)
+            )
+        }
+
+        /// A refresh that could not read a schema says nothing new about it, so the rows an earlier
+        /// listing had for that schema are carried over rather than dropped.
+        internal func keepingRows(from previous: Result?) -> Result {
+            guard let previous, !unlistedSchemas.isEmpty else { return self }
+            let carried = previous.tables.filter { table in
+                guard let schema = table.schema else { return false }
+                return unlistedSchemas.contains(schema)
+            }
+            return Result(tables: tables + carried, unlistedSchemas: unlistedSchemas)
+        }
+    }
+
+    private static let logger = Logger(subsystem: "com.TablePro", category: "CatalogTableListing")
+
+    internal static func tables(
+        in scope: DatabaseScope,
+        excludingSchemas excluded: Set<String>,
+        metadata: ScopedMetadataProviding = DatabaseManager.shared
+    ) async throws -> Result {
+        let listed = try await metadata.withMetadataDriver(scope: scope, workload: .bulk) { driver in
+            try await driver.fetchTablesInAllSchemas()
+        }
+        if let listed {
+            let tables = listed.filter { table in
+                guard let schema = table.schema else { return true }
+                return !excluded.contains(schema)
+            }
+            return Result(tables: tables, unlistedSchemas: [])
+        }
+        let schemas = try await metadata.withMetadataDriver(scope: scope, workload: .bulk) { driver in
+            try await driver.fetchSchemas()
+        }
+        return try await tables(inSchemas: schemas.filter { !excluded.contains($0) }, scope: scope, metadata: metadata)
+    }
+
+    /// The named schemas one by one, which is also how a listing asks again for the schemas it
+    /// could not read the first time.
+    ///
+    /// Only a failure that belongs to one schema is recorded against it. A lost connection fails
+    /// every schema the same way, and recording that as a listing of nothing would read as a
+    /// database with no tables, so it fails the whole read instead, as does every schema failing.
+    internal static func tables(
+        inSchemas schemas: [String],
+        scope: DatabaseScope,
+        metadata: ScopedMetadataProviding = DatabaseManager.shared
+    ) async throws -> Result {
+        var tables: [TableInfo] = []
+        var unlisted: Set<String> = []
+        var lastError: Error?
+        for schema in schemas {
+            try Task.checkCancellation()
+            do {
+                tables += try await metadata.withMetadataDriver(scope: scope, workload: .bulk) { driver in
+                    try await driver.fetchTables(schema: schema)
+                }
+            } catch is CancellationError {
+                throw CancellationError()
+            } catch let error as DatabaseError {
+                throw error
+            } catch {
+                logger.warning(
+                    "[catalog] schema not listed schema=\(schema, privacy: .private(mask: .hash)) error=\(error.publicLogShape, privacy: .public)"
+                )
+                unlisted.insert(schema)
+                lastError = error
+            }
+        }
+        if let lastError, !schemas.isEmpty, unlisted.count == schemas.count {
+            throw lastError
+        }
+        return Result(tables: tables, unlistedSchemas: unlisted)
+    }
+}

TablePro/Core/Database/DatabaseDriver.swift

@@ -99,6 +99,10 @@ protocol DatabaseDriver: AnyObject, Sendable {
 
     func fetchTables(schema: String?) async throws -> [TableInfo]
 
+    /// Every schema's tables in one call, or nil when the engine has no such call and the caller
+    /// has to ask each schema itself. `CatalogTableListing` is the caller that does.
+    func fetchTablesInAllSchemas() async throws -> [TableInfo]?
+
     /// Fetch the direct partitions of one partitioned table, with each one's bound, position and
     /// row estimate. A partition is not a table on every engine, so this cannot answer `TableInfo`:
     /// a MySQL or Oracle partition name is unique only within its own table.
@@ -704,6 +708,8 @@ extension DatabaseDriver {
         try await fetchTables()
     }
 
+    func fetchTablesInAllSchemas() async throws -> [TableInfo]? { nil }
+
     func fetchRoutines(schema: String?) async throws -> [RoutineInfo] { [] }
 
     func fetchRoutineDDL(_ routine: RoutineInfo) async throws -> String {